pax_global_header00006660000000000000000000000064147731466620014532gustar00rootroot0000000000000052 comment=02ccc30a9c8b0904259f56e106d5e5c52642ad47 elasticsearch-curator-8.0.21/000077500000000000000000000000001477314666200161115ustar00rootroot00000000000000elasticsearch-curator-8.0.21/.dockerignore000066400000000000000000000000131477314666200205570ustar00rootroot00000000000000Dockerfile elasticsearch-curator-8.0.21/.github/000077500000000000000000000000001477314666200174515ustar00rootroot00000000000000elasticsearch-curator-8.0.21/.github/PULL_REQUEST_TEMPLATE.md000066400000000000000000000000521477314666200232470ustar00rootroot00000000000000Fixes # ## Proposed Changes - - - elasticsearch-curator-8.0.21/.github/issue_template.md000066400000000000000000000020321477314666200230130ustar00rootroot00000000000000## For usage questions and help Please create a topic at https://discuss.elastic.co/c/elasticsearch Perhaps a topic there already has an answer for you! ## To submit a bug or report an issue ### Expected Behavior ### Actual Behavior ### Steps to Reproduce the Problem 1. 1. 1. ### Specifications - Version: - Platform: - Subsystem: ## Context (Environment) ## Detailed Description elasticsearch-curator-8.0.21/.github/workflows/000077500000000000000000000000001477314666200215065ustar00rootroot00000000000000elasticsearch-curator-8.0.21/.github/workflows/docs-build.yml000066400000000000000000000005601477314666200242570ustar00rootroot00000000000000name: docs-build on: workflow_dispatch: ~ push: branches: - master pull_request_target: ~ merge_group: ~ jobs: docs-preview: uses: elastic/docs-builder/.github/workflows/preview-build.yml@main with: path-pattern: docs/** permissions: deployments: write id-token: write contents: read pull-requests: read elasticsearch-curator-8.0.21/.github/workflows/docs-cleanup.yml000066400000000000000000000003771477314666200246150ustar00rootroot00000000000000name: docs-cleanup on: pull_request_target: types: - closed jobs: docs-preview: uses: elastic/docs-builder/.github/workflows/preview-cleanup.yml@main permissions: contents: none id-token: write deployments: write elasticsearch-curator-8.0.21/.gitignore000066400000000000000000000063131477314666200201040ustar00rootroot00000000000000.DS_Store localhost.es oneliners.py cacert.pem docs/asciidoc.bak docker_test/.env docker_test/repo/ docker_test/curatortestenv html_docs/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so .black .flake8 pylintrc pylintrc.toml # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .mypy.ini .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ .vscode elasticsearch-curator-8.0.21/.readthedocs.yaml000066400000000000000000000006411477314666200213410ustar00rootroot00000000000000# .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # Set the version of Python and other tools you might need build: os: ubuntu-22.04 tools: python: "3.11" # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py python: install: - method: pip path: . elasticsearch-curator-8.0.21/CONTRIBUTING.md000066400000000000000000000065761477314666200203600ustar00rootroot00000000000000# Contributing to Curator All contributions are welcome: ideas, patches, documentation, bug reports, complaints, etc! Programming is not a required skill, and there are many ways to help out! It is more important to us that you are able to contribute. That said, some basic guidelines, which you are free to ignore :) ## Want to learn? Want to write your own code to do something Curator doesn't do out of the box? * [Curator API Documentation](http://curator.readthedocs.io/) Since version 2.0, Curator ships with both an API and wrapper scripts (which are actually defined as entry points). This allows you to write your own scripts to accomplish similar goals, or even new and different things with the [Curator API](http://curator.readthedocs.io/), [es_client](https://es-client.readthedocs.io), and the [Elasticsearch Python Client Library](http://elasticsearch-py.readthedocs.io/). Want to know how to use the command-line interface (CLI)? * [Curator CLI Documentation](http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html) The Curator CLI Documentation is now a part of the document repository at http://elastic.co/guide at http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html ## Have a Question? Or an Idea or Feature Request? * File a ticket on [github](https://github.com/elastic/curator/issues) ## Something Not Working? Found a Bug? If you think you found a bug, it probably is a bug. * File it on [github](https://github.com/elastic/curator/issues) # Contributing Documentation and Code Changes If you have a bugfix or new feature that you would like to contribute to Curator, and you think it will take more than a few minutes to produce the fix (ie; write code), it is worth discussing the change with the Curator users and developers first! You can reach us via [github](https://github.com/elastic/curator/issues). Documentation is in two parts: API and CLI documentation. API documentation is generated from comments inside the classes and methods within the code. This documentation is rendered and hosted at http://curator.readthedocs.io CLI documentation is in Asciidoc format in the GitHub repository at https://github.com/elastic/curator/tree/master/docs/asciidoc. This documentation can be changed via a pull request as with any other code change. ## Contribution Steps 1. Test your changes! Run the test suite ('pytest --cov=curator'). Please note that this requires an Elasticsearch instance. The tests will try to connect to a local elasticsearch instance and run integration tests against it. **This will delete all the data stored there!** You can use the env variable `TEST_ES_SERVER` to point to a different instance (for example 'otherhost:9203'). 2. Please make sure you have signed our [Contributor License Agreement](http://www.elastic.co/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. 3. Send a pull request! Push your changes to your fork of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). In the pull request, describe what your changes do and mention any bugs/issues related to the pull request. elasticsearch-curator-8.0.21/CONTRIBUTORS000066400000000000000000000044421477314666200177750ustar00rootroot00000000000000The following is a list of people who have contributed ideas, code, bug reports, or in general have helped curator along its way. Contributors: * Jordan Sissel (jordansissel) (For Logstash, first and foremost) * Shay Banon (kimchy) (For Elasticsearch, of course!) * Aaron Mildenstein (untergeek) * Njal Karevoll * François Deppierraz * Honza Kral (HonzaKral) * Benjamin Smith (benjaminws) * Colin Moller (LeftyBC) * Elliot (edgeofnite) * Ram Viswanadha (ramv) * Chris Meisinger (cmeisinger) * Stuart Warren (stuart-warren) * (gitshaw) * (sfritz) * (sjoelsam) * Jose Diaz-Gonzalez (josegonzalez) * Arie Bro (arieb) * David Harrigan (dharrigan) * Mathieu Geli (gelim) * Nick Ethier (nickethier) * Mohab Usama (mohabusama) * (gitshaw) * Stuart Warren (stuart-warren) * Xavier Calland (xavier-calland) * Chad Schellenger (cschellenger) * Kamil Essekkat (ekamil) * (gbutt) * Ben Buchacher (bbuchacher) * Ehtesh Choudhury (shurane) * Markus Fischer (mfn) * Fabien Wernli (faxm0dem) * Michael Weiser (michaelweiser) * (digital-wonderland) * cassiano (cassianoleal) * Matt Dainty (bodgit) * Alex Philipp (alex-sf) * (krzaczek) * Justin Lintz (jlintz) * Jeremy Falling (jjfalling) * Ian Babrou (bobrik) * Ferenc Erki (ferki) * George Heppner (gheppner) * Matt Hughes (matthughes) * Brian Lalor (blalor) * Paweł Krzaczkowski (krzaczek) * Ben Tse (bt5e) * Tom Hendrikx (whyscream) * Christian Vozar (christianvozar) * Magnus Baeck (magnusbaeck) * Robin Kearney (rk295) * (cfeio) * (malagoli) * Dan Sheridan (djs52) * Michael-Keith Bernard (SegFaultAX) * Simon Lundström (simmel) * (pkr1234) * Mark Feltner (feltnerm) * William Jimenez (wjimenez5271) * Jeremy Canady (jrmycanady) * Steven Ottenhoff (steffo) * Ole Rößner (Basster) * Jack (univerio) * Tomáš Mózes (hydrapolic) * Gary Gao (garyelephant) * Panagiotis Moustafellos (pmoust) * (pbamba) * Pavel Strashkin (xaka) * Wadim Kruse (wkruse) * Richard Megginson (richm) * Thibaut Ackermann (thib-ack) * (zzugg) * Julien Mancuso (petitout) * Spencer Herzberg (sherzberg) * Luke Waite (lukewaite) * (dtrv) * Christopher "Chief" Najewicz (chiefy) * Filipe Gonçalves (basex) * Sönke Liebau (soenkeliebau) * Timothy Schroder (tschroeder-zendesk) * Jared Carey (jpcarey) * Juraj Seffer (jurajseffer) * Roger Steneteg (rsteneteg) * Muhammad Junaid Muzammil (junmuz) * Loet Avramson (psypuff) elasticsearch-curator-8.0.21/Dockerfile000066400000000000000000000027011477314666200201030ustar00rootroot00000000000000# syntax=docker/dockerfile:1 ARG PYVER=3.12.9 ARG ALPTAG=3.21 FROM python:${PYVER}-alpine${ALPTAG} AS builder # Add the community repo for access to patchelf binary package ARG ALPTAG RUN echo "https://dl-cdn.alpinelinux.org/alpine/v${ALPTAG}/community/" >> /etc/apk/repositories RUN apk --no-cache upgrade && apk --no-cache add build-base tar musl-utils openssl-dev patchelf # patchelf-wrapper is necessary now for cx_Freeze, but not for Curator itself. RUN pip3 install setuptools cx_Freeze patchelf-wrapper COPY . . # alpine4docker.sh does some link magic necessary for cx_Freeze execution # These files are platform dependent because the architecture is in the file name. # This script handles it, effectively: # ARCH=$(uname -m) # ln -s /lib/libc.musl-${ARCH}.so.1 ldd # ln -s /lib /lib64 RUN /bin/sh alpine4docker.sh # Install Curator locally RUN pip3 install . # Build (or rather Freeze) Curator RUN cxfreeze build # Rename 'build/exe.{system().lower()}-{machine()}-{MAJOR}.{MINOR}' to curator_build RUN python3 post4docker.py ### End `builder` segment ### Copy frozen binary to the container that will actually be published ARG ALPTAG FROM alpine:${ALPTAG} RUN apk --no-cache upgrade && apk --no-cache add openssl-dev expat # The path `curator_build` is from `builder` and `post4docker.py` COPY --from=builder curator_build /curator/ RUN mkdir /.curator USER nobody:nobody ENV LD_LIBRARY_PATH=/curator/lib:\$LD_LIBRARY_PATH ENTRYPOINT ["/curator/curator"] elasticsearch-curator-8.0.21/LICENSE000066400000000000000000000011261477314666200171160ustar00rootroot00000000000000Copyright 2011–2024 Elasticsearch and contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. elasticsearch-curator-8.0.21/NOTICE000066400000000000000000000103641477314666200170210ustar00rootroot00000000000000In accordance with section 4d of the Apache 2.0 license (http://www.apache.org/licenses/LICENSE-2.0), this NOTICE file is included. All users mentioned in the CONTRIBUTORS file at https://github.com/elastic/curator/blob/master/CONTRIBUTORS must be included in any derivative work. All conditions of section 4 of the Apache 2.0 license will be enforced: 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: a. You must give any other recipients of the Work or Derivative Works a copy of this License; and b. You must cause any modified files to carry prominent notices stating that You changed the files; and c. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and d. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. Contributors: * Jordan Sissel (jordansissel) (For Logstash, first and foremost) * Shay Banon (kimchy) (For Elasticsearch, of course!) * Aaron Mildenstein (untergeek) * Njal Karevoll * François Deppierraz * Honza Kral (HonzaKral) * Benjamin Smith (benjaminws) * Colin Moller (LeftyBC) * Elliot (edgeofnite) * Ram Viswanadha (ramv) * Chris Meisinger (cmeisinger) * Stuart Warren (stuart-warren) * (gitshaw) * (sfritz) * (sjoelsam) * Jose Diaz-Gonzalez (josegonzalez) * Arie Bro (arieb) * David Harrigan (dharrigan) * Mathieu Geli (gelim) * Nick Ethier (nickethier) * Mohab Usama (mohabusama) * (gitshaw) * Stuart Warren (stuart-warren) * Xavier Calland (xavier-calland) * Chad Schellenger (cschellenger) * Kamil Essekkat (ekamil) * (gbutt) * Ben Buchacher (bbuchacher) * Ehtesh Choudhury (shurane) * Markus Fischer (mfn) * Fabien Wernli (faxm0dem) * Michael Weiser (michaelweiser) * (digital-wonderland) * cassiano (cassianoleal) * Matt Dainty (bodgit) * Alex Philipp (alex-sf) * (krzaczek) * Justin Lintz (jlintz) * Jeremy Falling (jjfalling) * Ian Babrou (bobrik) * Ferenc Erki (ferki) * George Heppner (gheppner) * Matt Hughes (matthughes) * Brian Lalor (blalor) * Paweł Krzaczkowski (krzaczek) * Ben Tse (bt5e) * Tom Hendrikx (whyscream) * Christian Vozar (christianvozar) * Magnus Baeck (magnusbaeck) * Robin Kearney (rk295) * (cfeio) * (malagoli) * Dan Sheridan (djs52) * Michael-Keith Bernard (SegFaultAX) * Simon Lundström (simmel) * (pkr1234) * Mark Feltner (feltnerm) * William Jimenez (wjimenez5271) * Jeremy Canady (jrmycanady) * Steven Ottenhoff (steffo) * Ole Rößner (Basster) * Jack (univerio) * Tomáš Mózes (hydrapolic) * Gary Gao (garyelephant) * Panagiotis Moustafellos (pmoust) * (pbamba) * Pavel Strashkin (xaka) * Wadim Kruse (wkruse) * Richard Megginson (richm) * Thibaut Ackermann (thib-ack) * (zzugg) * Julien Mancuso (petitout) elasticsearch-curator-8.0.21/README.rst000066400000000000000000000036401477314666200176030ustar00rootroot00000000000000.. _readme: Curator ======= Have indices in Elasticsearch? This is the tool for you! Like a museum curator manages the exhibits and collections on display, Elasticsearch Curator helps you curate, or manage your indices. ANNOUNCEMENT ------------ Curator is breaking into version dependent releases. Curator 6.x will work with Elasticsearch 6.x, Curator 7.x will work with Elasticsearch 7.x, and when it is released, Curator 8.x will work with Elasticsearch 8.x. Watch this space for updates when that is coming. Additional support for Elasticsearch 7.14.0 - 7.17.x **************************************************** Starting with Curator 8.0.18, Curator 8 can execute against Elasticsearch versions 7.14.0 - 7.17.x in addition to all versions of Elasticsearch 8.x. New Client Configuration ------------------------ Curator now connects using the ``es_client`` Python module. This separation makes it much easier to update the client connection portion separate from Curator. It is largely derived from the original Curator client configuration, but with important updates. The updated configuration file structure requires ``elasticsearch`` at the root level:: --- elasticsearch: client: hosts: https://10.11.12.13:9200 cloud_id: bearer_auth: opaque_id: request_timeout: 60 http_compress: verify_certs: ca_certs: client_cert: client_key: ssl_assert_hostname: ssl_assert_fingerprint: ssl_version: other_settings: master_only: skip_version_test: username: password: api_key: id: api_key: logging: loglevel: INFO logfile: /path/to/file.log logformat: default blacklist: [] Action File Configuration ------------------------- Action file structure is unchanged, for now. A few actions may have had the options modified a bit. elasticsearch-curator-8.0.21/alpine4docker.sh000077500000000000000000000001121477314666200211660ustar00rootroot00000000000000ARCH=$(uname -m) ln -s /lib/libc.musl-${ARCH}.so.1 ldd ln -s /lib /lib64 elasticsearch-curator-8.0.21/curator/000077500000000000000000000000001477314666200175705ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/__init__.py000066400000000000000000000006161477314666200217040ustar00rootroot00000000000000"""Tending your Elasticsearch indices and snapshots""" from curator._version import __version__ from curator.helpers import * from curator.exceptions import * from curator.defaults import * from curator.validators import * from curator.indexlist import IndexList from curator.snapshotlist import SnapshotList from curator.actions import * from curator.cli import * from curator.repomgrcli import * elasticsearch-curator-8.0.21/curator/_version.py000066400000000000000000000000561477314666200217670ustar00rootroot00000000000000"""Curator Version""" __version__ = '8.0.21' elasticsearch-curator-8.0.21/curator/actions/000077500000000000000000000000001477314666200212305ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/actions/__init__.py000066400000000000000000000024771477314666200233530ustar00rootroot00000000000000"""Use __init__ to make these not need to be nested under lowercase.Capital""" from curator.actions.alias import Alias from curator.actions.allocation import Allocation from curator.actions.close import Close from curator.actions.cluster_routing import ClusterRouting from curator.actions.cold2frozen import Cold2Frozen from curator.actions.create_index import CreateIndex from curator.actions.delete_indices import DeleteIndices from curator.actions.forcemerge import ForceMerge from curator.actions.index_settings import IndexSettings from curator.actions.open import Open from curator.actions.reindex import Reindex from curator.actions.replicas import Replicas from curator.actions.rollover import Rollover from curator.actions.shrink import Shrink from curator.actions.snapshot import Snapshot, DeleteSnapshots, Restore CLASS_MAP = { 'alias' : Alias, 'allocation' : Allocation, 'close' : Close, 'cluster_routing' : ClusterRouting, 'cold2frozen': Cold2Frozen, 'create_index' : CreateIndex, 'delete_indices' : DeleteIndices, 'delete_snapshots' : DeleteSnapshots, 'forcemerge' : ForceMerge, 'index_settings' : IndexSettings, 'open' : Open, 'reindex' : Reindex, 'replicas' : Replicas, 'restore' : Restore, 'rollover' : Rollover, 'snapshot' : Snapshot, 'shrink' : Shrink, } elasticsearch-curator-8.0.21/curator/actions/alias.py000066400000000000000000000160431477314666200226770ustar00rootroot00000000000000"""Alias action""" import logging # pylint: disable=import-error from curator.exceptions import ActionError, MissingArgument, NoIndices from curator.helpers.date_ops import parse_date_pattern, parse_datemath from curator.helpers.testers import verify_index_list from curator.helpers.utils import report_failure class Alias: """Alias Action Class""" # pylint: disable=unused-argument def __init__(self, name=None, extra_settings=None, **kwargs): """ :param name: The alias name :param extra_settings: Extra settings, including filters and routing. For more information see `here `_. :type name: str :type extra_settings: dict """ if extra_settings is None: extra_settings = {} if not name: raise MissingArgument('No value for "name" provided.') #: The :py:func:`~.curator.helpers.date_ops.parse_date_pattern` rendered #: version of what was passed by param ``name``. self.name = parse_date_pattern(name) #: The list of actions to perform. Populated by #: :py:meth:`~.curator.actions.Alias.add` and #: :py:meth:`~.curator.actions.Alias.remove` self.actions = [] #: The :py:class:`~.elasticsearch.Elasticsearch` client object which will #: later be set by :py:meth:`~.curator.actions.Alias.add` or #: :py:meth:`~.curator.actions.Alias.remove` self.client = None #: Any extra things to add to the alias, like filters, or routing. Gets #: the value from param ``extra_settings``. self.extra_settings = extra_settings self.loggit = logging.getLogger('curator.actions.alias') #: Preset default value to ``False``. self.warn_if_no_indices = False def add(self, ilo, warn_if_no_indices=False): """ Create ``add`` statements for each index in ``ilo`` for :py:attr:`name`, then append them to :py:attr:`actions`. Add any :py:attr:`extra_settings` that may be there. :param ilo: An IndexList Object :type ilo: :py:class:`~.curator.indexlist.IndexList` """ verify_index_list(ilo) self.loggit.debug('ADD -> ILO = %s', ilo.indices) if not self.client: self.client = ilo.client self.name = parse_datemath(self.client, self.name) try: ilo.empty_list_check() except NoIndices as exc: # Add a warning if there are no indices to add, if so set in options if warn_if_no_indices: self.warn_if_no_indices = True self.loggit.warning( 'No indices found after processing filters. Nothing to add to %s', self.name, ) return # Re-raise the exceptions.NoIndices so it will behave as before raise NoIndices('No indices to add to alias') from exc for index in ilo.working_list(): self.loggit.debug( 'Adding index %s to alias %s with extra settings %s', index, self.name, self.extra_settings, ) add_dict = {'add': {'index': index, 'alias': self.name}} add_dict['add'].update(self.extra_settings) self.actions.append(add_dict) def remove(self, ilo, warn_if_no_indices=False): """ Create ``remove`` statements for each index in ``ilo`` for :py:attr:`name`, then append them to :py:attr:`actions`. :param ilo: An IndexList Object :type ilo: :py:class:`~.curator.indexlist.IndexList` """ verify_index_list(ilo) self.loggit.debug('REMOVE -> ILO = %s', ilo.indices) if not self.client: self.client = ilo.client self.name = parse_datemath(self.client, self.name) try: ilo.empty_list_check() except NoIndices as exc: # Add a warning if there are no indices to add, if so set in options if warn_if_no_indices: self.warn_if_no_indices = True self.loggit.warning( 'No indices found after processing filters. ' 'Nothing to remove from %s', self.name, ) return # Re-raise the exceptions.NoIndices so it will behave as before raise NoIndices('No indices to remove from alias') from exc aliases = self.client.indices.get_alias(expand_wildcards=['open', 'closed']) for index in ilo.working_list(): if index in aliases: self.loggit.debug('Index %s in get_aliases output', index) # Only remove if the index is associated with the alias if self.name in aliases[index]['aliases']: self.loggit.debug( 'Removing index %s from alias %s', index, self.name ) self.actions.append( {'remove': {'index': index, 'alias': self.name}} ) else: self.loggit.debug( 'Can not remove: Index %s is not associated with alias %s', index, self.name, ) def check_actions(self): """ :returns: :py:attr:`actions` for use with the :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` API call if actions exist, otherwise an exception is raised. """ if not self.actions: if not self.warn_if_no_indices: raise ActionError('No "add" or "remove" operations') raise NoIndices('No "adds" or "removes" found. Taking no action') self.loggit.debug('Alias actions: %s', self.actions) return self.actions def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') for item in self.check_actions(): job = list(item.keys())[0] index = item[job]['index'] alias = item[job]['alias'] # We want our log to look clever, so if job is "remove", strip the # 'e' so "remove" can become "removing". "adding" works already. msg = ( f"DRY-RUN: alias: {job.rstrip('e')}ing index \"{index}\" " f"{'to' if job == 'add' else 'from'} alias \"{alias}\"" ) self.loggit.info(msg) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` for :py:attr:`name` with :py:attr:`actions` """ self.loggit.info('Updating aliases...') self.loggit.info('Alias actions: %s', self.actions) try: self.client.indices.update_aliases(actions=self.actions) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/allocation.py000066400000000000000000000110141477314666200237240ustar00rootroot00000000000000"""Allocation action class""" import logging # pylint: disable=import-error from curator.exceptions import MissingArgument from curator.helpers.testers import verify_index_list from curator.helpers.waiters import wait_for_it from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv class Allocation: """Allocation Action Class""" def __init__( self, ilo, key=None, value=None, allocation_type='require', wait_for_completion=False, wait_interval=3, max_wait=-1, ): """ :param ilo: An IndexList Object :param key: An arbitrary metadata attribute key. Must match the key assigned to at least one node. :param value: An arbitrary metadata attribute value. Must correspond to values associated with at least one node. :param allocation_type: Type of allocation to apply. Default is ``require`` :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :type ilo: :py:class:`~.curator.indexlist.IndexList` :type key: str :type value: str :type allocation_type: str :type wait_for_completion: bool :type wait_interval: int :type max_wait: int .. note:: See more about `shard allocation filtering `_. """ verify_index_list(ilo) if not key: raise MissingArgument('No value for "key" provided') if allocation_type not in ['require', 'include', 'exclude']: raise ValueError( f'{allocation_type} is an invalid allocation_type. Must be one of ' f'"require", "include", "exclude".' ) #: The :py:class:`~.curator.indexlist.IndexList` object passed as ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client self.loggit = logging.getLogger('curator.actions.allocation') bkey = f'index.routing.allocation.{allocation_type}.{key}' #: Populated at instance creation time. Value is built from the passed params #: ``allocation_type``, ``key``, and ``value``, e.g. #: ``index.routing.allocation.allocation_type.key.value`` self.settings = {bkey: value} #: Object attribute that gets the value of param ``wait_for_completion`` self.wfc = wait_for_completion #: Object attribute that gets the value of param ``wait_interval`` self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait`` self.max_wait = max_wait def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run(self.index_list, 'allocation', settings=self.settings) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.put_settings` to indices in :py:attr:`index_list` with :py:attr:`settings`. """ self.loggit.debug( 'Cannot get change shard routing allocation of closed indices. ' 'Omitting any closed indices.' ) self.index_list.filter_closed() self.index_list.empty_list_check() self.loggit.info( 'Updating %s selected indices: %s', len(self.index_list.indices), self.index_list.indices, ) self.loggit.info('Updating index setting %s', self.settings) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: self.client.indices.put_settings( index=to_csv(lst), settings=self.settings ) if self.wfc: self.loggit.debug( 'Waiting for shards to complete relocation for indices: %s', to_csv(lst), ) wait_for_it( self.client, 'allocation', wait_interval=self.wait_interval, max_wait=self.max_wait, ) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/close.py000066400000000000000000000070301477314666200227070ustar00rootroot00000000000000"""Close index action class""" import logging import warnings from elasticsearch8.exceptions import ElasticsearchWarning from curator.helpers.testers import verify_index_list from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv class Close: """Close Action Class""" def __init__(self, ilo, delete_aliases=False, skip_flush=False): """ :param ilo: An IndexList Object :param delete_aliases: Delete any associated aliases before closing indices. :param skip_flush: Do not flush indices before closing. :type ilo: :py:class:`~.curator.indexlist.IndexList` :type delete_aliases: bool :type skip_flush: bool """ verify_index_list(ilo) #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The value passed as ``delete_aliases`` self.delete_aliases = delete_aliases #: The value passed as ``skip_flush`` self.skip_flush = skip_flush #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client self.loggit = logging.getLogger('curator.actions.close') def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run( self.index_list, 'close', **{'delete_aliases': self.delete_aliases} ) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.close` open indices in :py:attr:`index_list` """ self.index_list.filter_closed() self.index_list.empty_list_check() self.loggit.info( 'Closing %s selected indices: %s', len(self.index_list.indices), self.index_list.indices, ) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: lst_as_csv = to_csv(lst) self.loggit.debug('CSV list of indices to close: %s', lst_as_csv) if self.delete_aliases: self.loggit.info('Deleting aliases from indices before closing.') self.loggit.debug('Deleting aliases from: %s', lst) try: self.client.indices.delete_alias(index=lst_as_csv, name='*') self.loggit.debug('Deleted aliases from: %s', lst) # pylint: disable=broad-except except Exception as err: self.loggit.warning( 'Some indices may not have had aliases. Exception: %s', err ) if not self.skip_flush: self.client.indices.flush( index=lst_as_csv, ignore_unavailable=True, force=True ) # ElasticsearchWarning: the default value for the # wait_for_active_shards parameter will change from '0' to # 'index-setting' in version 8; # specify 'wait_for_active_shards=index-setting' to adopt the # future default behaviour, or # 'wait_for_active_shards=0' to preserve today's behaviour warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.close(index=lst_as_csv, ignore_unavailable=True) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/cluster_routing.py000066400000000000000000000106731477314666200250410ustar00rootroot00000000000000"""Cluster Routing action class""" import logging # pylint: disable=import-error from curator.helpers.testers import verify_client_object from curator.helpers.utils import report_failure from curator.helpers.waiters import wait_for_it class ClusterRouting: """ClusterRouting Action Class""" def __init__( self, client, routing_type=None, setting=None, value=None, wait_for_completion=False, wait_interval=9, max_wait=-1, ): """ For now, the cluster routing settings are hardcoded to be ``transient`` :param client: A client connection object :param routing_type: Type of routing to apply. Either ``allocation`` or ``rebalance`` :param setting: Currently, the only acceptable value for ``setting`` is ``enable``. This is here in case that changes. :param value: Used only if ``setting`` is ``enable``. Semi-dependent on ``routing_type``. Acceptable values for ``allocation`` and ``rebalance`` are ``all``, ``primaries``, and ``none`` (string, not :py:class:`None`). If ``routing_type`` is ``allocation``, this can also be ``new_primaries``, and if ``rebalance``, it can be ``replicas``. :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :type client: :py:class:`~.elasticsearch.Elasticsearch` :type routing_type: str :type setting: str :type value: str :type wait_for_completion: bool :type wait_interval: int :type max_wait: int """ verify_client_object(client) #: An :py:class:`~.elasticsearch.Elasticsearch` client object self.client = client self.loggit = logging.getLogger('curator.actions.cluster_routing') #: Object attribute that gets the value of param ``wait_for_completion`` self.wfc = wait_for_completion #: Object attribute that gets the value of param ``wait_interval`` self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait``. How long #: in seconds to :py:attr:`wfc` before returning with an exception. A value #: of ``-1`` means wait forever. self.max_wait = max_wait if setting != 'enable': raise ValueError(f'Invalid value for "setting": {setting}.') if routing_type == 'allocation': if value not in ['all', 'primaries', 'new_primaries', 'none']: raise ValueError( f'Invalid "value": {value} with "routing_type":{routing_type}.' ) elif routing_type == 'rebalance': if value not in ['all', 'primaries', 'replicas', 'none']: raise ValueError( f'Invalid "value": {value} with "routing_type": {routing_type}.' ) else: raise ValueError(f'Invalid value for "routing_type":{routing_type}.') bkey = f'cluster.routing.{routing_type}.{setting}' #: Populated at instance creation time. Value is built from the passed #: values from params ``routing_type`` and ``setting``, e.g. #: ``cluster.routing.routing_type.setting`` self.settings = {bkey: value} def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') msg = f'DRY-RUN: Update cluster routing transient settings: {self.settings}' self.loggit.info(msg) def do_action(self): """ :py:meth:`~.elasticsearch.client.ClusterClient.put_settings` to the cluster with :py:attr:`settings`. """ self.loggit.info('Updating cluster settings: %s', self.settings) try: self.client.cluster.put_settings(transient=self.settings) if self.wfc: self.loggit.debug( 'Waiting for shards to complete routing and/or rebalancing' ) wait_for_it( self.client, 'cluster_routing', wait_interval=self.wait_interval, max_wait=self.max_wait, ) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/cold2frozen.py000066400000000000000000000232051477314666200240330ustar00rootroot00000000000000"""Snapshot and Restore action classes""" import logging from curator.helpers.getters import get_alias_actions, get_tier_preference, meta_getter from curator.helpers.testers import ( has_lifecycle_name, is_idx_partial, verify_index_list, ) from curator.helpers.utils import report_failure from curator.exceptions import ( CuratorException, FailedExecution, SearchableSnapshotException, ) class Cold2Frozen: """Cold to Frozen Tier Searchable Snapshot Action Class For manually migrating snapshots not associated with ILM from the cold tier to the frozen tier. """ DEFAULTS = { 'index_settings': None, 'ignore_index_settings': ['index.refresh_interval'], 'wait_for_completion': True, } def __init__(self, ilo, **kwargs): """ :param ilo: An IndexList Object :param index_settings: (Optional) Settings that should be added to the index when it is mounted. If not set, set the ``_tier_preference`` to the tiers available, coldest first. :param ignore_index_settings: (Optional, array of strings) Names of settings that should be removed from the index when it is mounted. :param wait_for_completion: Wait for completion before returning. :type ilo: :py:class:`~.curator.indexlist.IndexList` :type index_settings: dict :type ignore_index_settings: list :type wait_for_completion: bool """ self.loggit = logging.getLogger('curator.actions.cold2frozen') verify_index_list(ilo) # Check here and don't bother with the rest of this if there are no # indices in the index list. ilo.empty_list_check() #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: Object attribute that contains the :py:func:`~.curator.helpers.utils.to_csv` #: output of the indices in :py:attr:`index_list`. self.indices = ilo #: Object attribute that gets the value of ``index_settings``. self.index_settings = None #: Object attribute that gets the value of ``ignore_index_settings``. self.ignore_index_settings = None #: Object attribute that gets the value of param ``wait_for_completion``. self.wait_for_completion = None # Parse the kwargs into attributes self.assign_kwargs(**kwargs) def assign_kwargs(self, **kwargs): """ Assign the kwargs to the attribute of the same name with the passed value or the default from DEFAULTS """ # Handy little loop here only adds kwargs that exist in DEFAULTS, or the # default value. It ignores any non-relevant kwargs for key, value in self.DEFAULTS.items(): if key in kwargs: setattr(self, key, kwargs[key]) else: setattr(self, key, value) def action_generator(self): """Yield a dict for use in :py:meth:`do_action` and :py:meth:`do_dry_run` :returns: A generator object containing the settings necessary to migrate indices from cold to frozen :rtype: dict """ for idx in self.index_list.indices: idx_settings = meta_getter(self.client, idx, get='settings') self.loggit.debug('Index %s has settings: %s', idx, idx_settings) if has_lifecycle_name(idx_settings): self.loggit.critical( 'Index %s is associated with an ILM policy and this action ' 'will never work on an index associated with an ILM policy', idx, ) raise CuratorException(f'Index {idx} is associated with an ILM policy') if is_idx_partial(idx_settings): self.loggit.critical('Index %s is already in the frozen tier', idx) raise SearchableSnapshotException('Index is already in frozen tier') snap = idx_settings['store']['snapshot']['snapshot_name'] snap_idx = idx_settings['store']['snapshot']['index_name'] repo = idx_settings['store']['snapshot']['repository_name'] msg = ( f'Index {idx} Snapshot name: {snap}, Snapshot index: {snap_idx}, ' f'repo: {repo}' ) self.loggit.debug(msg) aliases = meta_getter(self.client, idx, get='alias') renamed = f'partial-{idx}' if not self.index_settings: self.index_settings = { "routing": { "allocation": { "include": { "_tier_preference": get_tier_preference(self.client) } } } } yield { 'repository': repo, 'snapshot': snap, 'index': snap_idx, 'renamed_index': renamed, 'index_settings': self.index_settings, 'ignore_index_settings': self.ignore_index_settings, 'storage': 'shared_cache', 'wait_for_completion': self.wait_for_completion, 'aliases': aliases, 'current_idx': idx, } def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') for kwargs in self.action_generator(): aliases = kwargs.pop('aliases') current_idx = kwargs.pop('current_idx') msg = ( f'DRY-RUN: cold2frozen: from snapshot {kwargs["snapshot"]} in ' f'repository {kwargs["repository"]}, mount index {kwargs["index"]} ' f'renamed as {kwargs["renamed_index"]} with index settings: ' f'{kwargs["index_settings"]} and ignoring settings: ' f'{kwargs["ignore_index_settings"]}. wait_for_completion: ' f'{kwargs["wait_for_completion"]}. Restore aliases: {aliases}. ' f'Current index name: {current_idx}' ) self.loggit.info(msg) def mount_index(self, newidx, kwargs): """ Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices in :py:attr:`ilo` in the Frozen tier. """ try: self.loggit.debug('Mounting new index %s in frozen tier...', newidx) self.client.searchable_snapshots.mount(**kwargs) # pylint: disable=broad-except except Exception as err: report_failure(err) def verify_mount(self, newidx): """ Verify that newidx is a mounted index """ self.loggit.debug('Verifying new index %s is mounted properly...', newidx) idx_settings = self.client.indices.get(index=newidx)[newidx] if is_idx_partial(idx_settings['settings']['index']): self.loggit.info('Index %s is mounted for frozen tier', newidx) else: report_failure( SearchableSnapshotException( f'Index {newidx} not a mounted searchable snapshot' ) ) def update_aliases(self, current_idx, newidx, aliases): """ Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new frozen index with the aliases from the old cold-tier index. Verify aliases look good. """ alias_names = aliases.keys() if not alias_names: self.loggit.warning('No aliases associated with index %s', current_idx) else: self.loggit.debug('Transferring aliases to new index %s', newidx) self.client.indices.update_aliases( actions=get_alias_actions(current_idx, newidx, aliases) ) verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys() if alias_names != verify: self.loggit.error( 'Alias names do not match! %s does not match: %s', alias_names, verify, ) report_failure( FailedExecution('Aliases failed to transfer to new index') ) def cleanup(self, current_idx, newidx): """ Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index. """ self.loggit.debug('Deleting old index: %s', current_idx) try: self.client.indices.delete(index=current_idx) # pylint: disable=broad-except except Exception as err: report_failure(err) self.loggit.info( 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx ) def do_action(self): """ Do the actions outlined: Extract values from generated kwargs Mount Verify Update Aliases Cleanup """ for kwargs in self.action_generator(): aliases = kwargs.pop('aliases') current_idx = kwargs.pop('current_idx') newidx = kwargs['renamed_index'] # Mount the index self.mount_index(newidx, kwargs) # Verify it's mounted as a partial now: self.verify_mount(newidx) # Update Aliases self.update_aliases(current_idx, newidx, aliases) # Clean up old index self.cleanup(current_idx, newidx) elasticsearch-curator-8.0.21/curator/actions/create_index.py000066400000000000000000000105401477314666200242340ustar00rootroot00000000000000"""Create index action class""" import logging # pylint: disable=import-error, broad-except from elasticsearch8.exceptions import RequestError from curator.exceptions import ConfigurationError, FailedExecution from curator.helpers.date_ops import parse_date_pattern from curator.helpers.utils import report_failure class CreateIndex: """Create Index Action Class""" def __init__(self, client, name=None, extra_settings=None, ignore_existing=False): """ :param client: A client connection object :param name: A name, which can contain :py:func:`time.strftime` strings :param extra_settings: The `settings` and `mappings` for the index. For more information see `the create indices documentation `_. :param ignore_existing: If an index already exists, and this setting is ``True``, ignore the 400 error that results in a ``resource_already_exists_exception`` and return that it was successful. :type client: :py:class:`~.elasticsearch.Elasticsearch` :type name: str :type extra_settings: dict :type ignore_existing: bool """ if extra_settings is None: extra_settings = {} if not name: raise ConfigurationError('Value for "name" not provided.') #: The :py:func:`~.curator.helpers.date_ops.parse_date_pattern` rendered #: version of what was passed as ``name``. self.name = parse_date_pattern(name) #: Extracted from the action definition, it should be a boolean informing #: whether to ignore the error if the index already exists. self.ignore_existing = ignore_existing #: An :py:class:`~.elasticsearch.Elasticsearch` client object self.client = client #: Any extra settings for the index, like aliases, mappings, or settings. #: Gets the value from param ``extra_settings``. self.extra_settings = extra_settings #: Gets any ``aliases`` from :py:attr:`extra_settings` or is :py:class:`None` self.aliases = None #: Gets any ``mappings`` from :py:attr:`extra_settings` or is :py:class:`None` self.mappings = None #: Gets any ``settings`` from :py:attr:`extra_settings` or is :py:class:`None` self.settings = None if 'aliases' in extra_settings: self.aliases = extra_settings.pop('aliases') if 'mappings' in extra_settings: self.mappings = extra_settings.pop('mappings') if 'settings' in extra_settings: self.settings = extra_settings.pop('settings') self.loggit = logging.getLogger('curator.actions.create_index') def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') msg = ( f'DRY-RUN: create_index "{self.name}" with arguments: {self.extra_settings}' ) self.loggit.info(msg) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.create` index identified by :py:attr:`name` with values from :py:attr:`aliases`, :py:attr:`mappings`, and :py:attr:`settings` """ msg = f'Creating index "{self.name}" with settings: {self.extra_settings}' self.loggit.info(msg) try: self.client.indices.create( index=self.name, aliases=self.aliases, mappings=self.mappings, settings=self.settings, ) # Most likely error is a 400, `resource_already_exists_exception` except RequestError as err: match_list = [ "index_already_exists_exception", "resource_already_exists_exception", ] if err.error in match_list: if self.ignore_existing: self.loggit.warning('Index %s already exists.', self.name) else: raise FailedExecution(f'Index {self.name} already exists.') from err else: msg = f'Unable to create index "{self.name}". Error: {err.error}' raise FailedExecution(msg) from err # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/delete_indices.py000066400000000000000000000075041477314666200245500ustar00rootroot00000000000000"""Delete index action class""" import logging # pylint: disable=import-error from curator.helpers.getters import get_indices from curator.helpers.testers import verify_index_list from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv class DeleteIndices: """Delete Indices Action Class""" def __init__(self, ilo, master_timeout=30): """ :param ilo: An IndexList Object :param master_timeout: Number of seconds to wait for master node response :type ilo: :py:class:`~.curator.indexlist.IndexList` :type master_timeout: int """ verify_index_list(ilo) if not isinstance(master_timeout, int): raise TypeError( f'Incorrect type for "master_timeout": {type(master_timeout)}. ' f'Should be integer value.' ) #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: String value of param ``master_timeout`` + ``s``, for seconds. self.master_timeout = str(master_timeout) + 's' self.loggit = logging.getLogger('curator.actions.delete_indices') self.loggit.debug('master_timeout value: %s', self.master_timeout) def _verify_result(self, result, count): """ Breakout method to aid readability :param result: A list of indices from :py:meth:`__chunk_loop` :param count: The number of tries that have occurred :type result: list :type count: int :returns: ``True`` if result is verified successful, else ``False`` :rtype: bool """ if isinstance(result, list) and result: self.loggit.error( 'The following indices failed to delete on try #%s:', count ) for idx in result: self.loggit.error("---%s", idx) retval = False else: self.loggit.debug('Successfully deleted all indices on try #%s', count) retval = True return retval def __chunk_loop(self, chunk_list): """ Loop through deletes 3 times to ensure they complete :param chunk_list: A list of indices pre-chunked so it won't overload the URL size limit. :type chunk_list: list """ working_list = chunk_list for count in range(1, 4): # Try 3 times for i in working_list: self.loggit.info("---deleting index %s", i) self.client.indices.delete( index=to_csv(working_list), master_timeout=self.master_timeout ) result = [i for i in working_list if i in get_indices(self.client)] if self._verify_result(result, count): return working_list = result self.loggit.error( 'Unable to delete the following indices after 3 attempts: %s', result ) def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run(self.index_list, 'delete_indices') def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.delete` indices in :py:attr:`index_list` """ self.index_list.empty_list_check() msg = ( f'Deleting {len(self.index_list.indices)} selected indices: ' f'{self.index_list.indices}' ) self.loggit.info(msg) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: self.__chunk_loop(lst) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/forcemerge.py000066400000000000000000000056011477314666200237220ustar00rootroot00000000000000"""Forcemerge action class""" import logging from time import sleep # pylint: disable=import-error from curator.exceptions import MissingArgument from curator.helpers.testers import verify_index_list from curator.helpers.utils import report_failure, show_dry_run class ForceMerge: """ForceMerge Action Class""" def __init__(self, ilo, max_num_segments=None, delay=0): """ :param ilo: An IndexList Object :param max_num_segments: Number of segments per shard to forceMerge :param delay: Number of seconds to delay between forceMerge operations :type ilo: :py:class:`~.curator.indexlist.IndexList` :type max_num_segments: int :type delay: int """ verify_index_list(ilo) if not max_num_segments: raise MissingArgument('Missing value for "max_num_segments"') #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: Object attribute that gets the value of param ``max_num_segments``. self.max_num_segments = max_num_segments #: Object attribute that gets the value of param ``delay``. self.delay = delay self.loggit = logging.getLogger('curator.actions.forcemerge') def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run( self.index_list, 'forcemerge', max_num_segments=self.max_num_segments, delay=self.delay, ) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.forcemerge` indices in :py:attr:`index_list` """ self.index_list.filter_closed() self.index_list.filter_forceMerged(max_num_segments=self.max_num_segments) self.index_list.empty_list_check() msg = ( f'forceMerging {len(self.index_list.indices)} ' f'selected indices: {self.index_list.indices}' ) self.loggit.info(msg) try: for index_name in self.index_list.indices: msg = ( f'forceMerging index {index_name} to {self.max_num_segments} ' f'segments per shard. Please wait...' ) self.loggit.info(msg) self.client.indices.forcemerge( index=index_name, max_num_segments=self.max_num_segments ) if self.delay > 0: self.loggit.info( 'Pausing for %s seconds before continuing...', self.delay ) sleep(self.delay) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/index_settings.py000066400000000000000000000136301477314666200246340ustar00rootroot00000000000000"""Index settings action class""" import logging # pylint: disable=import-error from curator.exceptions import ActionError, ConfigurationError, MissingArgument from curator.helpers.testers import verify_index_list from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv class IndexSettings: """Index Settings Action Class""" def __init__( self, ilo, index_settings=None, ignore_unavailable=False, preserve_existing=False, ): """ :param ilo: An IndexList Object :param index_settings: A settings structure with one or more index settings to change. :param ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed) :param preserve_existing: Whether to update existing settings. If set to ``True``, existing settings on an index remain unchanged. The default is ``False`` :type ilo: :py:class:`~.curator.indexlist.IndexList` :type index_settings: dict :type ignore_unavailable: bool :type preserve_existing: bool """ if index_settings is None: index_settings = {} verify_index_list(ilo) if not index_settings: raise MissingArgument('Missing value for "index_settings"') #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: Object attribute that gets the value of param ``index_settings``. self.body = index_settings #: Object attribute that gets the value of param ``ignore_unavailable``. self.ignore_unavailable = ignore_unavailable #: Object attribute that gets the value of param ``preserve_existing``. self.preserve_existing = preserve_existing self.loggit = logging.getLogger('curator.actions.index_settings') self._body_check() def _body_check(self): # The body only passes the skimpiest of requirements by having 'index' # as the only root-level key, and having a 'dict' as its value if len(self.body) == 1: if 'index' in self.body: if isinstance(self.body['index'], dict): return True raise ConfigurationError(f'Bad value for "index_settings": {self.body}') def _static_settings(self): return [ 'number_of_shards', 'shard', 'codec', 'routing_partition_size', ] def _dynamic_settings(self): return [ 'number_of_replicas', 'auto_expand_replicas', 'refresh_interval', 'max_result_window', 'max_rescore_window', 'blocks', 'max_refresh_listeners', 'mapping', 'merge', 'translog', ] def _settings_check(self): # Detect if even one index is open. Save all found to open_index_list. open_index_list = [] open_indices = False # This action requires index settings and state to be present # Calling these here should not cause undue problems, even if it's a repeat call self.index_list.get_index_state() self.index_list.get_index_settings() for idx in self.index_list.indices: if self.index_list.index_info[idx]['state'] == 'open': open_index_list.append(idx) open_indices = True for k in self.body['index']: if k in self._static_settings(): if not self.ignore_unavailable: if open_indices: msg = ( f'Static Setting "{k}" detected with open indices: ' f'{open_index_list}. Static settings can only be used ' f'with closed indices. Recommend filtering out open ' f'indices, or setting ignore_unavailable to True' ) raise ActionError(msg) elif k in self._dynamic_settings(): # Dynamic settings should be appliable to open or closed indices # Act here if the case is different for some settings. pass else: msg = ( f'"{k}" is not a setting Curator recognizes and may or may ' f' not work.' ) self.loggit.warning(msg) def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run(self.index_list, 'indexsettings', **self.body) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.put_settings` in :py:attr:`body` to indices in :py:attr:`index_list` """ self._settings_check() # Ensure that the open indices filter applied in _settings_check() # didn't result in an empty list (or otherwise empty) self.index_list.empty_list_check() msg = ( f'Applying index settings to {len(self.index_list.indices)} indices: ' f'{self.index_list.indices}' ) self.loggit.info(msg) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: response = self.client.indices.put_settings( index=to_csv(lst), body=self.body, ignore_unavailable=self.ignore_unavailable, preserve_existing=self.preserve_existing, ) self.loggit.debug('PUT SETTINGS RESPONSE: %s', response) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/open.py000066400000000000000000000030121477314666200225370ustar00rootroot00000000000000"""Open index action class""" import logging from curator.helpers.testers import verify_index_list from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv class Open: """Open Action Class""" def __init__(self, ilo): """ :param ilo: An IndexList Object :type ilo: :py:class:`~.curator.indexlist.IndexList` """ verify_index_list(ilo) #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client self.loggit = logging.getLogger('curator.actions.open') def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run(self.index_list, 'open') def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.open` indices in :py:attr:`index_list` """ self.index_list.empty_list_check() msg = ( f'Opening {len(self.index_list.indices)} selected indices: ' f'{self.index_list.indices}' ) self.loggit.info(msg) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: self.client.indices.open(index=to_csv(lst)) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/reindex.py000066400000000000000000000437551477314666200232560ustar00rootroot00000000000000"""Reindex action class""" import logging from copy import deepcopy from dotmap import DotMap # type: ignore # pylint: disable=broad-except, R0902,R0912,R0913,R0914,R0915 from es_client.builder import Builder from es_client.helpers.utils import ensure_list, verify_url_schema from es_client.exceptions import ConfigurationError from curator.exceptions import CuratorException, FailedExecution, NoIndices # Separate from es_client from curator.defaults.settings import VERSION_MAX from curator.exceptions import ConfigurationError as CuratorConfigError from curator.helpers.testers import verify_index_list from curator.helpers.utils import report_failure from curator.helpers.waiters import wait_for_it from curator import IndexList class Reindex: """Reindex Action Class""" def __init__( self, ilo, request_body, refresh=True, requests_per_second=-1, slices=1, timeout=60, wait_for_active_shards=1, wait_for_completion=True, max_wait=-1, wait_interval=9, remote_certificate=None, remote_client_cert=None, remote_client_key=None, remote_filters=None, migration_prefix='', migration_suffix='', ): """ :param ilo: An IndexList Object :param request_body: The body to send to :py:meth:`~.elasticsearch.Elasticsearch.reindex`, which must be complete and usable, as Curator will do no vetting of the request_body. If it fails to function, Curator will return an exception. :param refresh: Whether to refresh the entire target index after the operation is complete. :param requests_per_second: The throttle to set on this request in sub-requests per second. ``-1`` means set no throttle as does ``unlimited`` which is the only non-float this accepts. :param slices: The number of slices this task should be divided into. ``1`` means the task will not be sliced into subtasks. (Default: ``1``) :param timeout: The length in seconds each individual bulk request should wait for shards that are unavailable. (default: ``60``) :param wait_for_active_shards: Sets the number of shard copies that must be active before proceeding with the reindex operation. (Default: ``1``) means the primary shard only. Set to ``all`` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :param remote_certificate: Path to SSL/TLS certificate :param remote_client_cert: Path to SSL/TLS client certificate (public key) :param remote_client_key: Path to SSL/TLS private key :param migration_prefix: When migrating, prepend this value to the index name. :param migration_suffix: When migrating, append this value to the index name. :type ilo: :py:class:`~.curator.indexlist.IndexList` :type request_body: dict :type refresh: bool :type requests_per_second: int :type slices: int :type timeout: int :type wait_for_active_shards: int :type wait_for_completion: bool :type wait_interval: int :type max_wait: int :type remote_certificate: str :type remote_cclient_cert: str :type remote_cclient_key: str :type migration_prefix: str :type migration_suffix: str """ if remote_filters is None: remote_filters = {} self.loggit = logging.getLogger('curator.actions.reindex') verify_index_list(ilo) if not isinstance(request_body, dict): raise CuratorConfigError('"request_body" is not of type dictionary') #: Object attribute that gets the value of param ``request_body``. self.body = request_body self.loggit.debug('REQUEST_BODY = %s', request_body) #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: Object attribute that gets the value of param ``refresh``. self.refresh = refresh #: Object attribute that gets the value of param ``requests_per_second``. self.requests_per_second = requests_per_second #: Object attribute that gets the value of param ``slices``. self.slices = slices #: Object attribute that gets the value of param ``timeout``, convert to #: :py:class:`str` and add ``s`` for seconds. self.timeout = f'{timeout}s' #: Object attribute that gets the value of param ``wait_for_active_shards``. self.wait_for_active_shards = wait_for_active_shards #: Object attribute that gets the value of param ``wait_for_completion``. self.wfc = wait_for_completion #: Object attribute that gets the value of param ``wait_interval``. self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait``. self.max_wait = max_wait #: Object attribute that gets the value of param ``migration_prefix``. self.mpfx = migration_prefix #: Object attribute that gets the value of param ``migration_suffix``. self.msfx = migration_suffix #: Object attribute that is set ``False`` unless :py:attr:`body` has #: ``{'source': {'remote': {}}}``, then it is set ``True`` self.remote = False if 'remote' in self.body['source']: self.remote = True #: Object attribute that is set ``False`` unless :py:attr:`body` has #: ``{'dest': {'index': 'MIGRATION'}}``, then it is set ``True`` self.migration = False if self.body['dest']['index'] == 'MIGRATION': self.migration = True if self.migration: if not self.remote and not self.mpfx and not self.msfx: raise CuratorConfigError( 'MIGRATION can only be used locally with one or both of ' 'migration_prefix or migration_suffix.' ) # REINDEX_SELECTION is the designated token. If you use this for the # source "index," it will be replaced with the list of indices from the # provided 'ilo' (index list object). if self.body['source']['index'] == 'REINDEX_SELECTION' and not self.remote: self.body['source']['index'] = self.index_list.indices # Remote section elif self.remote: rclient_args = DotMap() rother_args = DotMap() self.loggit.debug('Remote reindex request detected') if 'host' not in self.body['source']['remote']: raise CuratorConfigError('Missing remote "host"') try: rclient_args.hosts = verify_url_schema( self.body['source']['remote']['host'] ) except ConfigurationError as exc: raise CuratorConfigError(exc) from exc # Now that the URL schema is verified, these will pass. self.remote_host = rclient_args.hosts.split(':')[-2] self.remote_host = self.remote_host.split('/')[2] self.remote_port = rclient_args.hosts.split(':')[-1] if 'username' in self.body['source']['remote']: rother_args.username = self.body['source']['remote']['username'] if 'password' in self.body['source']['remote']: rother_args.password = self.body['source']['remote']['password'] if remote_certificate: rclient_args.ca_certs = remote_certificate if remote_client_cert: rclient_args.client_cert = remote_client_cert if remote_client_key: rclient_args.client_key = remote_client_key # Let's set a decent remote timeout for initially reading # the indices on the other side, and collecting their metadata rclient_args.request_timeout = 180 # The rest only applies if using filters for remote indices if self.body['source']['index'] == 'REINDEX_SELECTION': self.loggit.debug('Filtering indices from remote') msg = ( f'Remote client args: ' f'hosts={rclient_args.hosts} ' f'username=REDACTED ' f'password=REDACTED ' f'certificate={remote_certificate} ' f'client_cert={remote_client_cert} ' f'client_key={remote_client_key} ' f'request_timeout={rclient_args.request_timeout} ' f'skip_version_test=True' ) self.loggit.debug(msg) remote_config = { 'elasticsearch': { 'client': rclient_args.toDict(), 'other_settings': rother_args.toDict(), } } try: # let's try to build a remote connection with these! builder = Builder( configdict=remote_config, version_max=VERSION_MAX, version_min=(1, 4, 2), ) builder.version_min = (1, 0, 0) builder.connect() rclient = builder.client except Exception as err: self.loggit.error( 'Unable to establish connection to remote Elasticsearch' ' with provided credentials/certificates/settings.' ) report_failure(err) try: rio = IndexList(rclient) rio.iterate_filters({'filters': remote_filters}) try: rio.empty_list_check() except NoIndices as exc: raise FailedExecution( 'No actionable remote indices selected after applying ' 'filters.' ) from exc self.body['source']['index'] = rio.indices except Exception as err: self.loggit.error('Unable to get/filter list of remote indices.') report_failure(err) self.loggit.debug('Reindexing indices: %s', self.body['source']['index']) def _get_request_body(self, source, dest): body = deepcopy(self.body) body['source']['index'] = source body['dest']['index'] = dest return body def _get_reindex_args(self, source, dest): # Always set wait_for_completion to False. Let 'wait_for_it' do its # thing if wait_for_completion is set to True. Report the task_id # either way. reindex_args = { 'refresh': self.refresh, 'requests_per_second': self.requests_per_second, 'slices': self.slices, 'timeout': self.timeout, 'wait_for_active_shards': self.wait_for_active_shards, 'wait_for_completion': False, } for keyname in [ 'dest', 'source', 'conflicts', 'max_docs', 'size', '_source', 'script', ]: if keyname in self.body: reindex_args[keyname] = self.body[keyname] # Mimic the _get_request_body(source, dest) behavior by casting these values # here instead reindex_args['dest']['index'] = dest reindex_args['source']['index'] = source return reindex_args def get_processed_items(self, task_id): """ This function calls :py:func:`~.elasticsearch.client.TasksClient.get` with the provided ``task_id``. It will get the value from ``'response.total'`` as the total number of elements processed during reindexing. If the value is not found, it will return ``-1`` :param task_id: A task_id which ostensibly matches a task searchable in the tasks API. """ try: task_data = self.client.tasks.get(task_id=task_id) except Exception as exc: raise CuratorException( f'Unable to obtain task information for task_id "{task_id}". ' f'Exception {exc}' ) from exc total_processed_items = -1 task = task_data['task'] if task['action'] == 'indices:data/write/reindex': self.loggit.debug("It's a REINDEX TASK'") self.loggit.debug('TASK_DATA: %s', task_data) self.loggit.debug('TASK_DATA keys: %s', list(task_data.keys())) if 'response' in task_data: response = task_data['response'] total_processed_items = response['total'] self.loggit.debug('total_processed_items = %s', total_processed_items) return total_processed_items def _post_run_quick_check(self, index_name, task_id): # Check whether any documents were processed # if no documents processed, the target index "dest" won't exist processed_items = self.get_processed_items(task_id) if processed_items == 0: msg = ( f'No items were processed. Will not check if target index ' f'"{index_name}" exists' ) self.loggit.info(msg) else: # Verify the destination index is there after the fact index_exists = self.client.indices.exists(index=index_name) alias_instead = self.client.indices.exists_alias(name=index_name) if not index_exists and not alias_instead: # pylint: disable=logging-fstring-interpolation self.loggit.error( f'The index described as "{index_name}" was not found after the ' f'reindex operation. Check Elasticsearch logs for more ' f'information.' ) if self.remote: # pylint: disable=logging-fstring-interpolation self.loggit.error( f'Did you forget to add "reindex.remote.whitelist: ' f'{self.remote_host}:{self.remote_port}" to the ' f'elasticsearch.yml file on the "dest" node?' ) raise FailedExecution( f'Reindex failed. The index or alias identified by "{index_name}" ' f'was not found.' ) def sources(self): """Generator for Reindexing ``sources`` & ``dests``""" dest = self.body['dest']['index'] source_list = ensure_list(self.body['source']['index']) self.loggit.debug('source_list: %s', source_list) if not source_list or source_list == ['REINDEX_SELECTED']: # Empty list raise NoIndices if not self.migration: yield self.body['source']['index'], dest # Loop over all sources (default will only be one) else: for source in source_list: if self.migration: dest = self.mpfx + source + self.msfx yield source, dest def show_run_args(self, source, dest): """Show what will run""" return ( f'request body: {self._get_request_body(source, dest)} with arguments: ' f'refresh={self.refresh} ' f'requests_per_second={self.requests_per_second} ' f'slices={self.slices} ' f'timeout={self.timeout} ' f'wait_for_active_shards={self.wait_for_active_shards} ' f'wait_for_completion={self.wfc}' ) def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') for source, dest in self.sources(): self.loggit.info('DRY-RUN: REINDEX: %s', self.show_run_args(source, dest)) def do_action(self): """ Execute :py:meth:`~.elasticsearch.Elasticsearch.reindex` operation with the ``request_body`` from :py:meth:`_get_request_body` and arguments :py:attr:`refresh`, :py:attr:`requests_per_second`, :py:attr:`slices`, :py:attr:`timeout`, :py:attr:`wait_for_active_shards`, and :py:attr:`wfc`. """ try: # Loop over all sources (default will only be one) for source, dest in self.sources(): self.loggit.info('Commencing reindex operation') self.loggit.debug('REINDEX: %s', self.show_run_args(source, dest)) response = self.client.reindex(**self._get_reindex_args(source, dest)) self.loggit.debug('TASK ID = %s', response['task']) if self.wfc: wait_for_it( self.client, 'reindex', task_id=response['task'], wait_interval=self.wait_interval, max_wait=self.max_wait, ) self._post_run_quick_check(dest, response['task']) else: msg = ( f'"wait_for_completion" set to {self.wfc}. Remember to check ' f"task_id \"{response['task']}\" for successful completion " f"manually." ) self.loggit.warning(msg) except NoIndices as exc: raise NoIndices( 'Source index must be list of actual indices. It must not be an empty ' 'list.' ) from exc except Exception as exc: report_failure(exc) elasticsearch-curator-8.0.21/curator/actions/replicas.py000066400000000000000000000071141477314666200234070ustar00rootroot00000000000000"""Index replica count action class""" import logging from curator.exceptions import MissingArgument from curator.helpers.testers import verify_index_list from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv from curator.helpers.waiters import wait_for_it class Replicas: """Replica Action Class""" def __init__( self, ilo, count=None, wait_for_completion=False, wait_interval=9, max_wait=-1 ): """ :param ilo: An IndexList Object :param count: The count of replicas per shard :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :type ilo: :py:class:`~.curator.indexlist.IndexList` :type count: int :type wait_for_completion: bool :type wait_interval: int :type max_wait: int """ verify_index_list(ilo) # It's okay for count to be zero if count == 0: pass elif not count: raise MissingArgument('Missing value for "count"') #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: Object attribute that gets the value of param ``count``. self.count = count #: Object attribute that gets the value of param ``wait_for_completion``. self.wfc = wait_for_completion #: Object attribute that gets the value of param ``wait_interval``. self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait``. self.max_wait = max_wait self.loggit = logging.getLogger('curator.actions.replicas') def do_dry_run(self): """Log what the output would be, but take no action.""" show_dry_run(self.index_list, 'replicas', count=self.count) def do_action(self): """ Update ``number_of_replicas`` with :py:attr:`count` and :py:meth:`~.elasticsearch.client.IndicesClient.put_settings` to indices in :py:attr:`index_list` """ self.loggit.debug( 'Cannot get update replica count of closed indices. Omitting any ' 'closed indices.' ) self.index_list.filter_closed() self.index_list.empty_list_check() msg = ( f'Setting the replica count to {self.count} for ' f'{len(self.index_list.indices)} indices: {self.index_list.indices}' ) self.loggit.info(msg) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: self.client.indices.put_settings( index=to_csv(lst), settings={'number_of_replicas': self.count} ) if self.wfc and self.count > 0: msg = ( f'Waiting for shards to complete replication for indices: ' f'{to_csv(lst)}' ) self.loggit.debug(msg) wait_for_it( self.client, 'replicas', wait_interval=self.wait_interval, max_wait=self.max_wait, ) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/rollover.py000066400000000000000000000122771477314666200234570ustar00rootroot00000000000000"""Open index action class""" import logging from curator.exceptions import ConfigurationError from curator.helpers.date_ops import parse_date_pattern from curator.helpers.testers import rollable_alias, verify_client_object from curator.helpers.utils import report_failure class Rollover: """Rollover Action Class""" def __init__( self, client, name=None, conditions=None, new_index=None, extra_settings=None, wait_for_active_shards=1, ): """ :param client: A client connection object :param name: The name of the single-index-mapped alias to test for rollover conditions. :param new_index: A new index name :param conditions: Conditions to test :param extra_settings: Must be either ``None``, or a dictionary of settings to apply to the new index on rollover. This is used in place of ``settings`` in the Rollover API, mostly because it's already existent in other places here in Curator :param wait_for_active_shards: The number of shards expected to be active before returning. :type client: :py:class:`~.elasticsearch.Elasticsearch` :type name: str :type new_index: str :type conditions: dict :type extra_settings: dict or None :type wait_for_active_shards: int """ self.loggit = logging.getLogger('curator.actions.rollover') if not isinstance(conditions, dict): raise ConfigurationError('"conditions" must be a dictionary') else: self.loggit.debug('"conditions" is %s', conditions) if not isinstance(extra_settings, dict) and extra_settings is not None: raise ConfigurationError('"extra_settings" must be a dictionary or None') verify_client_object(client) #: Object attribute that gets the value of param ``client``. self.client = client #: Object attribute that gets the value of param ``conditions``. self.conditions = conditions #: Object attribute that gets the value of param ``extra_settings``. self.settings = extra_settings #: The :py:func:`~.curator.helpers.date_ops.parse_date_pattern` rendered #: version of what was passed as ``new_index``, or else ``None`` self.new_index = parse_date_pattern(new_index) if new_index else new_index #: Object attribute that gets the value of param ``wait_for_active_shards``. self.wait_for_active_shards = wait_for_active_shards #: Object attribute that gets the value of param ``name``. self.name = None # Verify that `conditions` and `settings` are good? # Verify that `name` is an alias, and is only mapped to one index. if rollable_alias(client, name): self.name = name else: raise ValueError( f'Unable to perform index rollover with alias ' f'"{name}". See previous logs for more details.' ) def log_result(self, result): """Log the results based on whether the index rolled over or not""" dryrun_string = '' if result['dry_run']: dryrun_string = 'DRY-RUN: ' self.loggit.debug('%sResult: %s', dryrun_string, result) rollover_string = ( f"{dryrun_string}Old index {result['old_index']} " f"rolled over to new index {result['new_index']}" ) # Success is determined by at one condition being True success = False for k in list(result['conditions'].keys()): if result['conditions'][k]: success = True if result['dry_run'] and success: # log "successful" dry-run self.loggit.info(rollover_string) elif result['rolled_over']: self.loggit.info(rollover_string) else: msg = ( f"{dryrun_string}Rollover conditions not met. " f"Index {result['old_index']} not rolled over." ) self.loggit.info(msg) def doit(self, dry_run=False): """ This exists solely to prevent having to have duplicate code in both :py:meth:`do_dry_run` and :py:meth:`do_action` because :py:meth:`~.elasticsearch.client.IndicesClient.rollover` has its own ``dry_run`` flag. """ return self.client.indices.rollover( alias=self.name, new_index=self.new_index, conditions=self.conditions, settings=self.settings, dry_run=dry_run, wait_for_active_shards=self.wait_for_active_shards, ) def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') self.log_result(self.doit(dry_run=True)) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.rollover` the index referenced by alias :py:attr:`name` """ self.loggit.info('Performing index rollover') try: self.log_result(self.doit()) # pylint: disable=broad-except except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/actions/shrink.py000066400000000000000000000633311477314666200231060ustar00rootroot00000000000000"""Reindex action class""" import logging # pylint: disable=broad-except from curator.defaults.settings import DATA_NODE_ROLES from curator.exceptions import ActionError, ConfigurationError from curator.helpers.getters import ( index_size, name_to_node_id, node_id_to_name, node_roles, ) from curator.helpers.testers import verify_index_list from curator.helpers.utils import chunk_index_list, report_failure from curator.helpers.waiters import health_check, wait_for_it class Shrink: """Shrink Action Class""" def __init__( self, ilo, shrink_node='DETERMINISTIC', node_filters=None, number_of_shards=1, number_of_replicas=1, shrink_prefix='', shrink_suffix='-shrink', copy_aliases=False, delete_after=True, post_allocation=None, wait_for_active_shards=1, wait_for_rebalance=True, extra_settings=None, wait_for_completion=True, wait_interval=9, max_wait=-1, ): """ :param ilo: An IndexList Object :param shrink_node: The node name to use as the shrink target, or ``DETERMINISTIC``, which will use the values in ``node_filters`` to determine which node will be the shrink node. :param node_filters: If the value of ``shrink_node`` is ``DETERMINISTIC``, the values in ``node_filters`` will be used while determining which node to allocate the shards on before performing the shrink. :param number_of_shards: The number of shards the shrunk index should have :param number_of_replicas: The number of replicas for the shrunk index :param shrink_prefix: Prepend the shrunk index with this value :param shrink_suffix: Append the value to the shrunk index (Default: ``-shrink``) :param copy_aliases: Whether to copy each source index aliases to target index after shrinking. The aliases will be added to target index and deleted from source index at the same time. (Default: ``False``) :param delete_after: Whether to delete each index after shrinking. (Default: ``True``) :param post_allocation: If populated, the ``allocation_type``, ``key``, and ``value`` will be applied to the shrunk index to re-route it. :param extra_settings: Permitted root keys are ``settings`` and ``aliases``. :param wait_for_active_shards: Wait for this many active shards before returning. :param wait_for_rebalance: Wait for rebalance. (Default: ``True``) :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :type ilo: :py:class:`~.curator.indexlist.IndexList` :type shrink_node: str :type node_filters: dict :type number_of_shards: int :type number_of_replicas: int :type shrink_prefix: str :type shrink_suffix: str :type copy_aliases: bool :type delete_after: bool :type post_allocation: dict :type extra_settings: dict :type wait_for_active_shards: int :type wait_for_rebalance: bool :type wait_for_completion: bool :type wait_interval: int :type max_wait: int """ if node_filters is None: node_filters = {} if post_allocation is None: post_allocation = {} if extra_settings is None: extra_settings = {} self.loggit = logging.getLogger('curator.actions.shrink') verify_index_list(ilo) if 'permit_masters' not in node_filters: node_filters['permit_masters'] = False #: The :py:class:`~.curator.indexlist.IndexList` object passed from #: param ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: Object attribute that gets the value of param ``shrink_node``. self.shrink_node = shrink_node #: Object attribute that gets the value of param ``node_filters``. self.node_filters = node_filters #: Object attribute that gets the value of param ``shrink_prefix``. self.shrink_prefix = shrink_prefix #: Object attribute that gets the value of param ``shrink_suffix``. self.shrink_suffix = shrink_suffix #: Object attribute that gets the value of param ``copy_aliases``. self.copy_aliases = copy_aliases #: Object attribute that gets the value of param ``delete_after``. self.delete_after = delete_after #: Object attribute that gets the value of param ``post_allocation``. self.post_allocation = post_allocation #: Object attribute that gets the value of param ``wait_for_rebalance``. self.wait_for_rebalance = wait_for_rebalance #: Object attribute that gets the value of param ``wait_for_completion``. self.wfc = wait_for_completion #: Object attribute that gets the value of param ``wait_interval``. self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait``. self.max_wait = max_wait #: Object attribute that gets the value of param ``number_of_shards``. self.number_of_shards = number_of_shards #: Object attribute that gets the value of param ``wait_for_active_shards``. self.wait_for_active_shards = wait_for_active_shards #: Object attribute that represents the target node for shrinking. self.shrink_node_name = None #: Object attribute that represents whether :py:attr:`shrink_node_name` #: is available self.shrink_node_avail = None #: Object attribute that represents the node_id of :py:attr:`shrink_node_name` self.shrink_node_id = None #: Object attribute that gets values from params ``number_of_shards`` and #: ``number_of_replicas``. self.settings = { 'index.number_of_shards': number_of_shards, 'index.number_of_replicas': number_of_replicas, } if extra_settings: self._merge_extra_settings(extra_settings) self._merge_extra_settings( { 'settings': { 'index.routing.allocation.require._name': None, 'index.blocks.write': None, } } ) def _merge_extra_settings(self, extra_settings): self.loggit.debug('Adding extra_settings to shrink body: %s', extra_settings) # Pop these here, otherwise we could overwrite our default number of # shards and replicas if 'settings' in extra_settings: settings = extra_settings.pop('settings') try: self.settings.update(settings) except Exception as exc: raise ConfigurationError( f"Unable to apply extra settings \"{{'settings':settings}}\" " f"to shrink body. Exception: {exc}" ) from exc if extra_settings: try: # Apply any remaining keys, should there be any. self.settings.update(extra_settings) except Exception as exc: raise ConfigurationError( f'Unable to apply extra settings "{extra_settings}" ' f'to shrink body. Exception: {exc}' ) from exc def _data_node(self, node_id): roles = node_roles(self.client, node_id) name = node_id_to_name(self.client, node_id) is_data_node = False for role in roles: if role in DATA_NODE_ROLES: is_data_node = True break # At least one data node role type qualifies if not is_data_node: self.loggit.info('Skipping node "%s": non-data node', name) return False if 'master' in roles and not self.node_filters['permit_masters']: self.loggit.info('Skipping node "%s": master node', name) return False if 'master' in roles and self.node_filters['permit_masters']: msg = ( f'Not skipping node "{name}" which is a master node (not recommended)' f', but permit_masters is True' ) self.loggit.warning(msg) return True # Implied else: It does have a qualifying data role and is not a master node return True def _exclude_node(self, name): if 'exclude_nodes' in self.node_filters: if name in self.node_filters['exclude_nodes']: self.loggit.info('Excluding node "%s" due to node_filters', name) return True return False def _shrink_target(self, name): return f'{self.shrink_prefix}{name}{self.shrink_suffix}' def qualify_single_node(self): """Qualify a single node as a shrink target""" node_id = name_to_node_id(self.client, self.shrink_node) if node_id: self.shrink_node_id = node_id self.shrink_node_name = self.shrink_node else: raise ConfigurationError(f'Unable to find node named: "{self.shrink_node}"') if self._exclude_node(self.shrink_node): raise ConfigurationError(f'Node "{self.shrink_node}" listed for exclusion') if not self._data_node(node_id): raise ActionError( f'Node "{self.shrink_node}" is not usable as a shrink node' ) self.shrink_node_avail = self.client.nodes.stats()['nodes'][node_id]['fs'][ 'total' ]['available_in_bytes'] def most_available_node(self): """ Determine which data node name has the most available free space, and meets the other node filters settings. """ mvn_avail = 0 # mvn_total = 0 mvn_name = None mvn_id = None nodes = self.client.nodes.stats()['nodes'] for node_id in nodes: name = nodes[node_id]['name'] if self._exclude_node(name): self.loggit.debug('Node "%s" excluded by node filters', name) continue if not self._data_node(node_id): self.loggit.debug('Node "%s" is not a data node', name) continue value = nodes[node_id]['fs']['total']['available_in_bytes'] if value > mvn_avail: mvn_name = name mvn_id = node_id mvn_avail = value self.shrink_node_name = mvn_name self.shrink_node_id = mvn_id self.shrink_node_avail = mvn_avail def route_index(self, idx, allocation_type, key, value): """Apply the indicated shard routing allocation""" bkey = f'index.routing.allocation.{allocation_type}.{key}' routing = {bkey: value} try: self.client.indices.put_settings(index=idx, settings=routing) if self.wait_for_rebalance: wait_for_it( self.client, 'allocation', wait_interval=self.wait_interval, max_wait=self.max_wait, ) else: wait_for_it( self.client, 'relocate', index=idx, wait_interval=self.wait_interval, max_wait=self.max_wait, ) except Exception as err: report_failure(err) def __log_action(self, error_msg, dry_run=False): if not dry_run: raise ActionError(error_msg) else: self.loggit.warning('DRY-RUN: %s', error_msg) def _block_writes(self, idx): block = {'index.blocks.write': True} self.client.indices.put_settings(index=idx, settings=block) def _unblock_writes(self, idx): unblock = {'index.blocks.write': False} self.client.indices.put_settings(index=idx, settings=unblock) def _check_space(self, idx, dry_run=False): # Disk watermark calculation is already baked into `available_in_bytes` size = index_size(self.client, idx, value='primaries') padded = (size * 2) + (32 * 1024) if padded < self.shrink_node_avail: msg = ( f'Sufficient space available for 2x the size of index "{idx}". ' f'Required: {padded}, available: {self.shrink_node_avail}' ) self.loggit.debug(msg) else: error_msg = ( f'Insufficient space available for 2x the size of index "{idx}", ' f'shrinking will exceed space available. Required: {padded}, ' f'available: {self.shrink_node_avail}' ) self.__log_action(error_msg, dry_run) def _check_node(self): if self.shrink_node != 'DETERMINISTIC': if not self.shrink_node_name: self.qualify_single_node() else: self.most_available_node() # At this point, we should have the three shrink-node identifying # instance variables: # - self.shrink_node_name # - self.shrink_node_id # - self.shrink_node_avail # # - self.shrink_node_total - only if needed in the future def _check_target_exists(self, idx, dry_run=False): target = self._shrink_target(idx) if self.client.indices.exists(index=target): error_msg = f'Target index "{target}" already exists' self.__log_action(error_msg, dry_run) def _check_doc_count(self, idx, dry_run=False): max_docs = 2147483519 doc_count = self.client.indices.stats(index=idx)['indices'][idx]['primaries'][ 'docs' ]['count'] if doc_count > (max_docs * self.number_of_shards): error_msg = ( f'Too many documents ({doc_count}) to fit in {self.number_of_shards} ' f'shard(s). Maximum number of docs per shard is {max_docs}' ) self.__log_action(error_msg, dry_run) def _check_shard_count(self, idx, src_shards, dry_run=False): if self.number_of_shards >= src_shards: error_msg = ( f'Target number of shards ({self.number_of_shards}) must be less than ' f'current number of shards ({src_shards}) in index "{idx}"' ) self.__log_action(error_msg, dry_run) def _check_shard_factor(self, idx, src_shards, dry_run=False): # Find the list of factors of src_shards factors = [x for x in range(1, src_shards + 1) if src_shards % x == 0] # Pop the last one, because it will be the value of src_shards factors.pop() if self.number_of_shards not in factors: error_msg = ( f'"{self.number_of_shards}" is not a valid factor of {src_shards} ' f'shards of index {idx}. Valid values are {factors}' ) self.__log_action(error_msg, dry_run) def _check_all_shards(self, idx): shards = self.client.cluster.state(index=idx)['routing_table']['indices'][idx][ 'shards' ] found = [] for shardnum in shards: for shard_idx in range(0, len(shards[shardnum])): if shards[shardnum][shard_idx]['node'] == self.shrink_node_id: found.append( { 'shard': shardnum, 'primary': shards[shardnum][shard_idx]['primary'], } ) if len(shards) != len(found): self.loggit.debug( 'Found these shards on node "%s": %s', self.shrink_node_name, found ) raise ActionError( f'Unable to shrink index "{idx}" as not all shards were found on the ' f'designated shrink node ({self.shrink_node_name}): {found}' ) def pre_shrink_check(self, idx, dry_run=False): """Do a shrink preflight check""" self.loggit.debug('BEGIN PRE_SHRINK_CHECK') self.loggit.debug('Check that target exists') self._check_target_exists(idx, dry_run) self.loggit.debug('Check doc count constraints') self._check_doc_count(idx, dry_run) self.loggit.debug('Check shard count') src_shards = int( self.client.indices.get(index=idx)[idx]['settings']['index'][ 'number_of_shards' ] ) self._check_shard_count(idx, src_shards, dry_run) self.loggit.debug('Check shard factor') self._check_shard_factor(idx, src_shards, dry_run) self.loggit.debug('Check node availability') self._check_node() self.loggit.debug('Check available disk space') self._check_space(idx, dry_run) self.loggit.debug('FINISH PRE_SHRINK_CHECK') def do_copy_aliases(self, source_idx, target_idx): """Copy the aliases to the shrunk index""" alias_actions = [] aliases = self.client.indices.get_alias(index=source_idx) for alias in aliases[source_idx]['aliases']: self.loggit.debug('alias: %s', alias) alias_actions.append({'remove': {'index': source_idx, 'alias': alias}}) alias_actions.append({'add': {'index': target_idx, 'alias': alias}}) if alias_actions: self.loggit.info('Copy alias actions: %s', alias_actions) self.client.indices.update_aliases(actions=alias_actions) def do_dry_run(self): """Show what a regular run would do, but don't actually do it.""" self.index_list.filter_closed() self.index_list.filter_by_shards(number_of_shards=self.number_of_shards) self.index_list.empty_list_check() try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: for idx in lst: # Shrink can only be done one at a time... target = self._shrink_target(idx) self.pre_shrink_check(idx, dry_run=True) self.loggit.info( 'DRY-RUN: Moving shards to shrink node: "%s"', self.shrink_node_name, ) msg = ( f'DRY-RUN: Shrinking index "{idx}" to "{target}" with ' f'settings: {self.settings}, wait_for_active_shards=' f'{self.wait_for_active_shards}' ) self.loggit.info(msg) if self.post_allocation: submsg = ( f"index.routing.allocation." f"{self.post_allocation['allocation_type']}." f"{self.post_allocation['key']}:" f"{self.post_allocation['value']}" ) msg = ( f'DRY-RUN: Applying post-shrink allocation rule "{submsg}" ' f'to index "{target}"' ) self.loggit.info(msg) if self.copy_aliases: msg = ( f'DRY-RUN: Copy source index aliases ' f'"{self.client.indices.get_alias(index=idx)}"' ) self.loggit.info(msg) if self.delete_after: self.loggit.info('DRY-RUN: Deleting source index "%s"', idx) except Exception as err: report_failure(err) def do_action(self): """ :py:meth:`~.elasticsearch.client.IndicesClient.shrink` the indices in :py:attr:`index_list` """ self.index_list.filter_closed() self.index_list.filter_by_shards(number_of_shards=self.number_of_shards) self.index_list.empty_list_check() msg = ( f'Shrinking {len(self.index_list.indices)} selected indices: ' f'{self.index_list.indices}' ) self.loggit.info(msg) try: index_lists = chunk_index_list(self.index_list.indices) for lst in index_lists: for idx in lst: # Shrink can only be done one at a time... target = self._shrink_target(idx) self.loggit.info( 'Source index: %s -- Target index: %s', idx, target ) # Pre-check ensures disk space available for each pass of the loop self.pre_shrink_check(idx) # Route the index to the shrink node self.loggit.info( 'Moving shards to shrink node: "%s"', self.shrink_node_name ) self.route_index(idx, 'require', '_name', self.shrink_node_name) # Ensure a copy of each shard is present self._check_all_shards(idx) # Block writes on index self._block_writes(idx) # Do final health check if not health_check(self.client, status='green'): msg = ( 'Unable to proceed with shrink action. ' 'Cluster health is not "green"' ) raise ActionError(msg) # Do the shrink msg = ( f'Shrinking index "{idx}" to "{target}" with settings: ' f'{self.settings}, wait_for_active_shards=' f'{self.wait_for_active_shards}' ) self.loggit.info(msg) try: self.client.indices.shrink( index=idx, target=target, settings=self.settings, wait_for_active_shards=self.wait_for_active_shards, ) # Wait for it to complete if self.wfc: self.loggit.debug( 'Wait for shards to complete allocation for index: %s', target, ) if self.wait_for_rebalance: wait_for_it( self.client, 'shrink', wait_interval=self.wait_interval, max_wait=self.max_wait, ) else: wait_for_it( self.client, 'relocate', index=target, wait_interval=self.wait_interval, max_wait=self.max_wait, ) except Exception as exc: if self.client.indices.exists(index=target): msg = ( f'Deleting target index "{target}" due to failure ' f'to complete shrink' ) self.loggit.error(msg) self.client.indices.delete(index=target) raise ActionError( f'Unable to shrink index "{idx}" -- Error: {exc}' ) from exc self.loggit.info( 'Index "%s" successfully shrunk to "%s"', idx, target ) # Do post-shrink steps # Unblock writes on index (just in case) self._unblock_writes(idx) # Post-allocation, if enabled if self.post_allocation: submsg = ( f"index.routing.allocation." f"{self.post_allocation['allocation_type']}." f"{self.post_allocation['key']}:" f"{self.post_allocation['value']}" ) msg = ( f'Applying post-shrink allocation rule "{submsg}" ' f'to index "{target}"' ) self.loggit.info(msg) self.route_index( target, self.post_allocation['allocation_type'], self.post_allocation['key'], self.post_allocation['value'], ) # Copy aliases, if flagged if self.copy_aliases: self.loggit.info('Copy source index aliases "%s"', idx) self.do_copy_aliases(idx, target) # Delete, if flagged if self.delete_after: self.loggit.info('Deleting source index "%s"', idx) self.client.indices.delete(index=idx) else: # Let's unset the routing we applied here. self.loggit.info( 'Unassigning routing for source index: "%s"', idx ) self.route_index(idx, 'require', '_name', '') except Exception as err: # Just in case it fails after attempting to meet this condition self._unblock_writes(idx) report_failure(err) elasticsearch-curator-8.0.21/curator/actions/snapshot.py000066400000000000000000000544671477314666200234610ustar00rootroot00000000000000"""Snapshot and Restore action classes""" import logging import re from es_client.helpers.utils import ensure_list from curator.helpers.date_ops import parse_datemath, parse_date_pattern from curator.helpers.getters import get_indices from curator.helpers.testers import ( repository_exists, snapshot_running, verify_index_list, verify_repository, verify_snapshot_list, ) from curator.helpers.utils import report_failure, to_csv, multitarget_match from curator.helpers.waiters import wait_for_it # pylint: disable=broad-except from curator.exceptions import ( ActionError, CuratorException, FailedRestore, FailedSnapshot, MissingArgument, SnapshotInProgress, ) class Snapshot(object): """Snapshot Action Class Read more about identically named settings at: :py:meth:`elasticsearch.client.SnapshotClient.create` """ def __init__( self, ilo, repository=None, name=None, ignore_unavailable=False, include_global_state=True, partial=False, wait_for_completion=True, wait_interval=9, max_wait=-1, skip_repo_fs_check=True, ): """ :param ilo: An IndexList Object :param repository: Repository name. :param name: Snapshot name. :param ignore_unavailable: Ignore unavailable shards/indices. :param include_global_state: Store cluster global state with snapshot. :param partial: Do not fail if primary shard is unavailable. :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :param skip_repo_fs_check: Do not validate write access to repository on all cluster nodes before proceeding. Useful for shared filesystems where intermittent timeouts can affect validation, but won't likely affect snapshot success. (Default: ``True``) :type ilo: :py:class:`~.curator.indexlist.IndexList` :type repository: str :type name: str :type ignore_unavailable: bool :type include_global_state: bool :type partial: bool :type wait_for_completion: bool :type wait_interval: int :type max_wait: int :type skip_repo_fs_check: bool """ verify_index_list(ilo) # Check here and don't bother with the rest of this if there are no # indices in the index list. ilo.empty_list_check() if not repository_exists(ilo.client, repository=repository): raise ActionError( f'Cannot snapshot indices to missing repository: {repository}' ) if not name: raise MissingArgument('No value for "name" provided.') #: The :py:class:`~.curator.indexlist.IndexList` object passed from param #: ``ilo`` self.index_list = ilo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`index_list` self.client = ilo.client #: The :py:func:`~.curator.helpers.date_ops.parse_date_pattern` rendered #: version of what was passed by param ``name``. self.name = parse_datemath(self.client, parse_date_pattern(name)) #: Object attribute that gets the value of param ``repository``. self.repository = repository #: Object attribute that gets the value of param ``wait_for_completion``. self.wait_for_completion = wait_for_completion #: Object attribute that gets the value of param ``wait_interval``. self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait``. self.max_wait = max_wait #: Object attribute that gets the value of param ``skip_repo_fs_check``. self.skip_repo_fs_check = skip_repo_fs_check #: Object attribute that tracks the snapshot state. self.state = None #: Object attribute that contains the :py:func:`~.curator.helpers.utils.to_csv` #: output of the indices in :py:attr:`index_list`. self.indices = to_csv(ilo.indices) #: Object attribute that gets the value of param ``ignore_unavailable``. self.ignore_unavailable = ignore_unavailable #: Object attribute that gets the value of param ``include_global_state``. self.include_global_state = include_global_state #: Object attribute that gets the value of param ``partial``. self.partial = partial #: Object attribute dictionary compiled from :py:attr:`indices`, #: :py:attr:`ignore_unavailable`, :py:attr:`include_global_state`, and #: :py:attr:`partial` self.settings = { 'indices': ilo.indices, 'ignore_unavailable': self.ignore_unavailable, 'include_global_state': self.include_global_state, 'partial': self.partial, } self.loggit = logging.getLogger('curator.actions.snapshot') def get_state(self): """Get the state of the snapshot and set :py:attr:`state`""" try: self.state = self.client.snapshot.get( repository=self.repository, snapshot=self.name )['snapshots'][0]['state'] return self.state except IndexError as exc: raise CuratorException( f'Snapshot "{self.name}" not found in repository "{self.repository}"' ) from exc def report_state(self): """ Log the :py:attr:`state` of the snapshot and raise :py:exc:`FailedSnapshot` if :py:attr:`state` is not ``SUCCESS`` """ self.get_state() if self.state == 'SUCCESS': self.loggit.info('Snapshot %s successfully completed.', self.name) else: msg = f'Snapshot {self.name} completed with state: {self.state}' self.loggit.error(msg) raise FailedSnapshot(msg) def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') msg = ( f'DRY-RUN: snapshot: {self.name} in repository {self.repository} ' f'with arguments: {self.settings}' ) self.loggit.info(msg) def do_action(self): """ :py:meth:`elasticsearch.client.SnapshotClient.create` a snapshot of :py:attr:`indices`, with passed parameters. """ if not self.skip_repo_fs_check: verify_repository(self.client, self.repository) if snapshot_running(self.client): raise SnapshotInProgress('Snapshot already in progress.') try: self.loggit.info( 'Creating snapshot "%s" from indices: %s', self.name, self.index_list.indices, ) # Always set wait_for_completion to False. Let 'wait_for_it' do its # thing if wait_for_completion is set to True. Report the task_id # either way. self.client.snapshot.create( repository=self.repository, snapshot=self.name, ignore_unavailable=self.ignore_unavailable, include_global_state=self.include_global_state, indices=self.indices, partial=self.partial, wait_for_completion=False, ) if self.wait_for_completion: wait_for_it( self.client, 'snapshot', snapshot=self.name, repository=self.repository, wait_interval=self.wait_interval, max_wait=self.max_wait, ) self.report_state() else: msg = ( f'"wait_for_completion" set to {self.wait_for_completion}. ' f'Remember to check for successful completion manually.' ) self.loggit.warning(msg) except Exception as err: report_failure(err) class DeleteSnapshots: """Delete Snapshots Action Class""" def __init__(self, slo, retry_interval=120, retry_count=3): """ :param slo: A SnapshotList object :type slo: :py:class:`~.curator.snapshotlist.SnapshotList` :param retry_interval: Seconds to delay betwen retries. (Default: ``120``) :type retry_interval: int :param retry_count: Number of attempts to make. (Default: ``3``) :type retry_count: int """ verify_snapshot_list(slo) #: The :py:class:`~.curator.snapshotlist.SnapshotList` object passed from param #: ``slo`` self.snapshot_list = slo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`snapshot_list` self.client = slo.client #: Object attribute that gets the value of param ``retry_interval``. self.retry_interval = retry_interval #: Object attribute that gets the value of param ``retry_count``. self.retry_count = retry_count #: Object attribute that gets its value from :py:attr:`snapshot_list`. self.repository = slo.repository self.loggit = logging.getLogger('curator.actions.delete_snapshots') def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') mykwargs = { 'repository': self.repository, 'retry_interval': self.retry_interval, 'retry_count': self.retry_count, } for snap in self.snapshot_list.snapshots: self.loggit.info( 'DRY-RUN: delete_snapshot: %s with arguments: %s', snap, mykwargs ) def do_action(self): """ :py:meth:`~.elasticsearch.client.SnapshotClient.delete` snapshots in :py:attr:`snapshot_list`. Retry up to :py:attr:`retry_count` times, pausing :py:attr:`retry_interval` seconds between retries. """ self.snapshot_list.empty_list_check() msg = ( f'Deleting {len(self.snapshot_list.snapshots)} ' f'selected snapshots: {self.snapshot_list.snapshots}' ) self.loggit.info(msg) try: for snap in self.snapshot_list.snapshots: self.loggit.info('Deleting snapshot %s...', snap) self.client.snapshot.delete(repository=self.repository, snapshot=snap) # pylint: disable=broad-except except Exception as err: report_failure(err) class Restore(object): """Restore Action Class Read more about identically named settings at: :py:meth:`elasticsearch.client.SnapshotClient.restore` """ def __init__( self, slo, name=None, indices=None, include_aliases=False, ignore_unavailable=False, include_global_state=False, partial=False, rename_pattern=None, rename_replacement=None, extra_settings=None, wait_for_completion=True, wait_interval=9, max_wait=-1, skip_repo_fs_check=True, ): """ :param slo: A SnapshotList object :param name: Name of the snapshot to restore. If ``None``, use the most recent snapshot. :param indices: Indices to restore. If ``None``, all in the snapshot will be restored. :param include_aliases: Restore aliases with the indices. :param ignore_unavailable: Ignore unavailable shards/indices. :param include_global_state: Restore cluster global state with snapshot. :param partial: Do not fail if primary shard is unavailable. :param rename_pattern: A regular expression pattern with one or more captures, e.g. ``index_(.+)`` :param rename_replacement: A target index name pattern with `$#` numbered references to the captures in ``rename_pattern``, e.g. ``restored_index_$1`` :param extra_settings: Index settings to apply to restored indices. :param wait_for_completion: Wait for completion before returning. :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :param skip_repo_fs_check: Do not validate write access to repository on all cluster nodes before proceeding. Useful for shared filesystems where intermittent timeouts can affect validation, but won't likely affect snapshot success. (Default: ``True``) :type slo: :py:class:`~.curator.snapshotlist.SnapshotList` :type name: str :type indices: list :type include_aliases: bool :type ignore_unavailable: bool :type include_global_state: bool :type partial: bool :type rename_pattern: str :type rename_replacement: str :type extra_settings: dict :type wait_for_completion: bool :type wait_interval: int :type max_wait: int :type skip_repo_fs_check: bool """ if extra_settings is None: extra_settings = {} self.loggit = logging.getLogger('curator.actions.snapshot') verify_snapshot_list(slo) # Get the most recent snapshot. most_recent = slo.most_recent() self.loggit.debug('"most_recent" snapshot: %s', most_recent) #: Object attribute that gets the value of param ``name`` if not ``None``, #: or the output from :py:meth:`~.curator.SnapshotList.most_recent`. self.name = name if name else most_recent # Stop here now, if it's not a successful snapshot. if slo.snapshot_info[self.name]['state'] == 'PARTIAL' and partial: self.loggit.warning('Performing restore of snapshot in state PARTIAL.') elif slo.snapshot_info[self.name]['state'] != 'SUCCESS': raise CuratorException( 'Restore operation can only be performed on snapshots with ' 'state "SUCCESS", or "PARTIAL" if partial=True.' ) #: Internal reference to `slo` self.snapshot_list = slo #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from #: :py:attr:`snapshot_list` self.client = slo.client #: Object attribute that gets the value of ``repository`` from #: :py:attr:`snapshot_list`. self.repository = slo.repository self.loggit.debug('indices: %s', indices) if indices: self.indices = ensure_list(indices) else: self.indices = slo.snapshot_info[self.name]['indices'] self.loggit.debug('self.indices: %s', self.indices) #: Object attribute that gets the value of param ``wait_for_completion``. self.wfc = wait_for_completion #: Object attribute that gets the value of param ``wait_interval``. self.wait_interval = wait_interval #: Object attribute that gets the value of param ``max_wait``. self.max_wait = max_wait #: Object attribute that gets the value of param ``rename_pattern``. #: Empty :py:class:`str` if ``None`` self.rename_pattern = rename_pattern if rename_replacement is not None else '' #: Object attribute that gets the value of param ``rename_replacement``. Empty #: :py:class:`str` if ``None`` self.rename_replacement = ( rename_replacement if rename_replacement is not None else '' ) #: Object attribute derived from :py:attr:`rename_replacement`. but with Java #: regex group designations of ``$#`` converted to Python's ``\\#`` style. self.py_rename_replacement = self.rename_replacement.replace('$', '\\') #: Object attribute that gets the value of param ``max_wait``. self.skip_repo_fs_check = skip_repo_fs_check #: Object attribute that gets populated from other params/attributes. #: Deprecated, but not removed. Lazy way to keep from updating #: :py:meth:`do_dry_run`. Will fix later. self.body = { 'indices': self.indices, 'include_aliases': include_aliases, 'ignore_unavailable': ignore_unavailable, 'include_global_state': include_global_state, 'partial': partial, 'rename_pattern': self.rename_pattern, 'rename_replacement': self.rename_replacement, } #: Object attribute that gets the value of param ``include_aliases``. self.include_aliases = include_aliases #: Object attribute that gets the value of param ``ignore_unavailable``. self.ignore_unavailable = ignore_unavailable #: Object attribute that gets the value of param ``include_global_state``. self.include_global_state = include_global_state #: Object attribute that gets the value of param ``include_aliases``. self.include_aliases = include_aliases #: Object attribute that gets the value of param ``partial``. self.partial = partial #: Object attribute that gets the value of param ``extra_settings``. self.index_settings = None if extra_settings: self.loggit.debug( 'Adding extra_settings to restore body: %s', extra_settings ) self.index_settings = extra_settings try: self.body.update(extra_settings) except Exception: self.loggit.error('Unable to apply extra settings to restore body') self.loggit.debug('REPOSITORY: %s', self.repository) self.loggit.debug('WAIT_FOR_COMPLETION: %s', self.wfc) self.loggit.debug('SKIP_REPO_FS_CHECK: %s', self.skip_repo_fs_check) self.loggit.debug('BODY: %s', self.body) self._get_expected_output() def _get_expected_output(self): if self.indices == self.snapshot_list.snapshot_info[self.name]['indices']: indices = self.indices else: indices = multitarget_match( to_csv(self.indices), self.snapshot_list.snapshot_info[self.name]['indices'], ) if not self.rename_pattern and not self.rename_replacement: self.expected_output = indices self.loggit.debug('Expected output: %s', indices) return # Don't stick around if we're not replacing anything self.expected_output = [] for index in indices: self.expected_output.append( re.sub(self.rename_pattern, self.py_rename_replacement, index) ) msg = f'index: {index} replacement: {self.expected_output[-1]}' self.loggit.debug(msg) def report_state(self): """ Log the state of the restore. This should only be done if ``wait_for_completion`` is ``True``, and only after completing the restore. """ all_indices = get_indices(self.client) self.loggit.debug('All indices: %s', all_indices) self.loggit.debug('Expected output: %s', self.expected_output) found_count = 0 missing = [] for index in self.expected_output: if index in all_indices: found_count += 1 self.loggit.info('Found restored index %s', index) else: missing.append(index) if found_count == len(self.expected_output): self.loggit.info('All indices appear to have been restored.') else: msg = ( f'Some of the indices do not appear to have been restored. ' f'Missing: {missing}' ) self.loggit.error(msg) raise FailedRestore(msg) def do_dry_run(self): """Log what the output would be, but take no action.""" self.loggit.info('DRY-RUN MODE. No changes will be made.') args = {'wait_for_completion': self.wfc, 'body': self.body} msg = ( f'DRY-RUN: restore: Repository: {self.repository} ' f'Snapshot name: {self.name} Arguments: {args}' ) self.loggit.info(msg) for index in self.indices: if self.rename_pattern and self.rename_replacement: _ = re.sub(self.rename_pattern, self.py_rename_replacement, index) rmsg = f'as {_}' else: rmsg = '' self.loggit.info('DRY-RUN: restore: Index %s %s', index, rmsg) def do_action(self): """ :py:meth:`~.elasticsearch.client.SnapshotClient.restore` :py:attr:`indices` from :py:attr:`name` with passed params. """ if not self.skip_repo_fs_check: verify_repository(self.client, self.repository) if snapshot_running(self.client): raise SnapshotInProgress('Cannot restore while a snapshot is in progress.') try: self.loggit.info( 'Restoring indices "%s" from snapshot: %s', self.indices, self.name ) # Always set wait_for_completion to False. Let 'wait_for_it' do its # thing if wait_for_completion is set to True. Report the task_id # either way. self.client.snapshot.restore( repository=self.repository, snapshot=self.name, ignore_index_settings=None, ignore_unavailable=self.ignore_unavailable, include_aliases=self.include_aliases, include_global_state=self.include_global_state, index_settings=self.index_settings, indices=self.indices, partial=self.partial, rename_pattern=self.rename_pattern, rename_replacement=self.rename_replacement, wait_for_completion=False, ) if self.wfc: wait_for_it( self.client, 'restore', index_list=self.expected_output, wait_interval=self.wait_interval, max_wait=self.max_wait, ) self.report_state() else: msg = ( f'"wait_for_completion" set to {self.wfc}. ' f'Remember to check for successful completion manually.' ) self.loggit.warning(msg) except Exception as err: report_failure(err) elasticsearch-curator-8.0.21/curator/classdef.py000066400000000000000000000230041477314666200217250ustar00rootroot00000000000000"""Other Classes""" import logging from es_client.exceptions import FailedValidation from es_client.helpers.schemacheck import password_filter from es_client.helpers.utils import get_yaml from curator import IndexList, SnapshotList from curator.actions import CLASS_MAP from curator.exceptions import ConfigurationError from curator.helpers.testers import validate_actions # Let me tell you the story of the nearly wasted afternoon and the research that went # into this seemingly simple work-around. Actually, no. It's even more wasted time # writing that story here. Suffice to say that I couldn't use the CLASS_MAP with class # objects to directly map them to class instances. The Wrapper class and the # ActionDef.instantiate method do all of the work for me, allowing me to easily and # cleanly pass *args and **kwargs to the individual action classes of CLASS_MAP. class Wrapper: """Wrapper Class""" def __init__(self, cls): """Instantiate with passed Class (not instance or object) :param cls: A class (not an instance of the class) """ #: The class itself (not an instance of it), passed from ``cls`` self.class_object = cls #: An instance of :py:attr:`class_object` self.class_instance = None def set_instance(self, *args, **kwargs): """Set up :py:attr:`class_instance` from :py:attr:`class_object`""" self.class_instance = self.class_object(*args, **kwargs) def get_instance(self, *args, **kwargs): """Return the instance with ``*args`` and ``**kwargs``""" self.set_instance(*args, **kwargs) return self.class_instance class ActionsFile: """Class to parse and verify entire actions file Individual actions are :py:class:`~.curator.classdef.ActionDef` objects """ def __init__(self, action_file): self.logger = logging.getLogger(__name__) #: The full, validated configuration from ``action_file``. self.fullconfig = self.get_validated(action_file) self.logger.debug('Action Configuration: %s', password_filter(self.fullconfig)) #: A dict of all actions in the provided configuration. Each original key name #: is preserved and the value is now an #: :py:class:`~.curator.classdef.ActionDef`, rather than a dict. self.actions = None self.set_actions(self.fullconfig['actions']) def get_validated(self, action_file): """ :param action_file: The path to a valid YAML action configuration file :type action_file: str :returns: The result from passing ``action_file`` to :py:func:`~.curator.helpers.testers.validate_actions` """ try: return validate_actions(get_yaml(action_file)) except (FailedValidation, UnboundLocalError) as err: self.logger.critical('Configuration Error: %s', err) raise ConfigurationError from err def parse_actions(self, all_actions): """Parse the individual actions found in ``all_actions['actions']`` :param all_actions: All actions, each its own dictionary behind a numeric key. Making the keys numeric guarantees that if they are sorted, they will always be executed in order. :type all_actions: dict :returns: :rtype: list of :py:class:`~.curator.classdef.ActionDef` """ acts = {} for idx in all_actions.keys(): acts[idx] = ActionDef(all_actions[idx]) return acts def set_actions(self, all_actions): """Set the actions via :py:meth:`~.curator.classdef.ActionsFile.parse_actions` :param all_actions: All actions, each its own dictionary behind a numeric key. Making the keys numeric guarantees that if they are sorted, they will always be executed in order. :type all_actions: dict :rtype: None """ self.actions = self.parse_actions(all_actions) # In this case, I just don't care that pylint thinks I'm overdoing it with attributes # pylint: disable=too-many-instance-attributes class ActionDef: """Individual Action Definition Class Instances of this class represent an individual action from an action file. """ def __init__(self, action_dict): #: The whole action dictionary self.action_dict = action_dict #: The action name self.action = None #: The action's class (Alias, Allocation, etc.) self.action_cls = None #: Only when action is alias will this be a :py:class:`~.curator.IndexList` self.alias_adds = None #: Only when action is alias will this be a :py:class:`~.curator.IndexList` self.alias_removes = None #: The list class, either :py:class:`~.curator.IndexList` or #: :py:class:`~.curator.SnapshotList`. Default is #: :py:class:`~.curator.IndexList` self.list_obj = Wrapper(IndexList) #: The action ``description`` self.description = None #: The action ``options`` :py:class:`dict` self.options = {} #: The action ``filters`` :py:class:`list` self.filters = None #: The action option ``disable_action`` self.disabled = None #: The action option ``continue_if_exception`` self.cif = None #: The action option ``timeout_override`` self.timeout_override = None #: The action option ``ignore_empty_list`` self.iel = None #: The action option ``allow_ilm_indices`` self.allow_ilm = None self.set_root_attrs() self.set_option_attrs() self.log_the_options() self.get_action_class() def instantiate(self, attribute, *args, **kwargs): """ Convert ``attribute`` from being a :py:class:`~.curator.classdef.Wrapper` of a Class to an instantiated object of that Class. This is madness or genius. You decide. This entire method plus the :py:class:`~.curator.classdef.Wrapper` class came about because I couldn't cleanly instantiate a class variable into a class object. It works, and that's good enough for me. :param attribute: The `name` of an attribute that references a Wrapper class instance :type attribute: str """ try: wrapper = getattr(self, attribute) except AttributeError as exc: raise AttributeError( f'Bad Attribute: {attribute}. Exception: {exc}' ) from exc setattr(self, attribute, self.get_obj_instance(wrapper, *args, **kwargs)) def get_obj_instance(self, wrapper, *args, **kwargs): """Get the class instance wrapper identified by ``wrapper`` Pass all other args and kwargs to the :py:meth:`~.curator.classdef.Wrapper.get_instance` method. :returns: An instance of the class that :py:class:`~.curator.classdef.Wrapper` is wrapping """ if not isinstance(wrapper, Wrapper): raise ConfigurationError( f'{__name__} was passed wrapper which was of type {type(wrapper)}' ) return wrapper.get_instance(*args, **kwargs) def set_alias_extras(self): """Populate the :py:attr:`alias_adds` and :py:attr:`alias_removes` attributes""" self.alias_adds = Wrapper(IndexList) self.alias_removes = Wrapper(IndexList) def get_action_class(self): """Get the action class from :py:const:`~.curator.actions.CLASS_MAP` Do extra setup when action is ``alias`` Set :py:attr:`list_obj` to :py:class:`~.curator.SnapshotList` when :py:attr:`~.curator.classdef.ActionDef.action` is ``delete_snapshots`` or ``restore`` """ self.action_cls = Wrapper(CLASS_MAP[self.action]) if self.action == 'alias': self.set_alias_extras() if self.action in ['delete_snapshots', 'restore']: self.list_obj = Wrapper(SnapshotList) def set_option_attrs(self): """ Iteratively get the keys and values from :py:attr:`~.curator.classdef.ActionDef.options` and set the attributes """ attmap = { 'disable_action': 'disabled', 'continue_if_exception': 'cif', 'ignore_empty_list': 'iel', 'allow_ilm_indices': 'allow_ilm', 'timeout_override': 'timeout_override', } for key in self.action_dict['options']: if key in attmap: setattr(self, attmap[key], self.action_dict['options'][key]) else: self.options[key] = self.action_dict['options'][key] def set_root_attrs(self): """ Iteratively get the keys and values from :py:attr:`~.curator.classdef.ActionDef.action_dict` and set the attributes """ for key, value in self.action_dict.items(): # Gonna grab options in get_option_attrs() if key == 'options': continue if value is not None: setattr(self, key, value) def log_the_options(self): """Log options at initialization time""" logger = logging.getLogger('curator.cli.ActionDef') msg = ( f'For action {self.action}: disable_action={self.disabled}' f'continue_if_exception={self.cif}, ' f'timeout_override={self.timeout_override}, ' f'ignore_empty_list={self.iel}, allow_ilm_indices={self.allow_ilm}' ) logger.debug(msg) if self.allow_ilm: logger.warning('Permitting operation on indices with an ILM policy') elasticsearch-curator-8.0.21/curator/cli.py000066400000000000000000000244361477314666200207220ustar00rootroot00000000000000"""Main CLI for Curator""" import sys import logging import click from es_client.defaults import OPTION_DEFAULTS from es_client.helpers.config import ( cli_opts, context_settings, generate_configdict, get_client, get_config, options_from_dict, ) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper, prune_nones from curator.exceptions import ClientException from curator.classdef import ActionsFile from curator.defaults.settings import ( CLICK_DRYRUN, VERSION_MAX, VERSION_MIN, default_config_file, footer, snapshot_actions, ) from curator.exceptions import NoIndices, NoSnapshots from curator.helpers.testers import ilm_policy_check from curator._version import __version__ ONOFF = {'on': '', 'off': 'no-'} click_opt_wrap = option_wrapper() # pylint: disable=R0913, R0914, W0613, W0622, W0718 def ilm_action_skip(client, action_def): """ Skip rollover action if ``allow_ilm_indices`` is ``false``. For all other non-snapshot actions, add the ``ilm`` filtertype to the :py:attr:`~.curator.ActionDef.filters` list. :param action_def: An action object :type action_def: :py:class:`~.curator.classdef.ActionDef` :returns: ``True`` if ``action_def.action`` is ``rollover`` and the alias identified by ``action_def.options['name']`` is associated with an ILM policy. This hacky work-around is because the Rollover action does not use :py:class:`~.curator.IndexList` :rtype: bool """ logger = logging.getLogger(__name__) if not action_def.allow_ilm and action_def.action not in snapshot_actions(): if action_def.action == 'rollover': if ilm_policy_check(client, action_def.options['name']): logger.info( 'Alias %s is associated with ILM policy.', action_def.options['name'], ) return True elif action_def.filters: action_def.filters.append({'filtertype': 'ilm'}) else: action_def.filters = [{'filtertype': 'ilm'}] return False def exception_handler(action_def, err): """Do the grunt work with the exception :param action_def: An action object :param err: The exception :type action_def: :py:class:`~.curator.classdef.ActionDef` :type err: :py:exc:`Exception` """ logger = logging.getLogger(__name__) if isinstance(err, (NoIndices, NoSnapshots)): if action_def.iel: logger.info( 'Skipping action "%s" due to empty list: %s', action_def.action, type(err), ) else: logger.error( 'Unable to complete action "%s". No actionable items in list: %s', action_def.action, type(err), ) sys.exit(1) else: logger.error( 'Failed to complete action: %s. %s: %s', action_def.action, type(err), err ) if action_def.cif: logger.info( 'Continuing execution with next action because "continue_if_exception" ' 'is set to True for action %s', action_def.action, ) else: sys.exit(1) def process_action(client, action_def, dry_run=False): """ Do the ``action`` in ``action_def.action``, using the associated options and any ``kwargs``. :param client: A client connection object :param action_def: The ``action`` object :type client: :py:class:`~.elasticsearch.Elasticsearch` :type action_def: :py:class:`~.curator.classdef.ActionDef` :rtype: None """ logger = logging.getLogger(__name__) logger.debug('Configuration dictionary: %s', action_def.action_dict) mykwargs = {} logger.debug('INITIAL Action kwargs: %s', mykwargs) # Add some settings to mykwargs... if action_def.action == 'delete_indices': mykwargs['master_timeout'] = 30 # Update the defaults with whatever came with opts, minus any Nones mykwargs.update(prune_nones(action_def.options)) # Pop out the search_pattern option, if present. ptrn = mykwargs.pop('search_pattern', '*') hidn = mykwargs.pop('include_hidden', False) logger.debug('Action kwargs: %s', mykwargs) logger.debug('Post search_pattern & include_hidden Action kwargs: %s', mykwargs) # Set up the action logger.debug('Running "%s"', action_def.action.upper()) if action_def.action == 'alias': # Special behavior for this action, as it has 2 index lists action_def.instantiate('action_cls', **mykwargs) action_def.instantiate( 'alias_adds', client, search_pattern=ptrn, include_hidden=hidn ) action_def.instantiate( 'alias_removes', client, search_pattern=ptrn, include_hidden=hidn ) if 'remove' in action_def.action_dict: logger.debug('Removing indices from alias "%s"', action_def.options['name']) action_def.alias_removes.iterate_filters(action_def.action_dict['remove']) action_def.action_cls.remove( action_def.alias_removes, warn_if_no_indices=action_def.options['warn_if_no_indices'], ) if 'add' in action_def.action_dict: logger.debug('Adding indices to alias "%s"', action_def.options['name']) action_def.alias_adds.iterate_filters(action_def.action_dict['add']) action_def.action_cls.add( action_def.alias_adds, warn_if_no_indices=action_def.options['warn_if_no_indices'], ) elif action_def.action in ['cluster_routing', 'create_index', 'rollover']: action_def.instantiate('action_cls', client, **mykwargs) else: if action_def.action in ['delete_snapshots', 'restore']: mykwargs.pop('repository') # We don't need to send this value to the action action_def.instantiate( 'list_obj', client, repository=action_def.options['repository'] ) else: action_def.instantiate( 'list_obj', client, search_pattern=ptrn, include_hidden=hidn ) action_def.list_obj.iterate_filters({'filters': action_def.filters}) logger.debug(f'Pre Instantiation Action kwargs: {mykwargs}') action_def.instantiate('action_cls', action_def.list_obj, **mykwargs) # Do the action if dry_run: action_def.action_cls.do_dry_run() else: logger.debug('Doing the action here.') action_def.action_cls.do_action() def run(ctx: click.Context) -> None: """ :param ctx: The Click command context :type ctx: :py:class:`Context ` Called by :py:func:`cli` to execute what was collected at the command-line """ logger = logging.getLogger(__name__) logger.debug('action_file: %s', ctx.params['action_file']) all_actions = ActionsFile(ctx.params['action_file']) for idx in sorted(list(all_actions.actions.keys())): action_def = all_actions.actions[idx] # Skip to next action if 'disabled' if action_def.disabled: logger.info( 'Action ID: %s: "%s" not performed because "disable_action" ' 'is set to True', idx, action_def.action, ) continue logger.info('Preparing Action ID: %s, "%s"', idx, action_def.action) # Override the timeout, if specified, otherwise use the default. if action_def.timeout_override: ctx.obj['configdict']['elasticsearch']['client'][ 'request_timeout' ] = action_def.timeout_override # Create a client object for each action... logger.info('Creating client object and testing connection') try: client = get_client( configdict=ctx.obj['configdict'], version_max=VERSION_MAX, version_min=VERSION_MIN, ) except ClientException as exc: # No matter where logging is set to go, make sure we dump these messages to # the CLI click.echo('Unable to establish client connection to Elasticsearch!') click.echo(f'Exception: {exc}') sys.exit(1) except Exception as other: logger.debug('Fatal exception encountered: %s', other) # Filter ILM indices unless expressly permitted if ilm_action_skip(client, action_def): continue # # Process the action # msg = ( f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}' ) try: logger.info(msg) process_action(client, action_def, dry_run=ctx.params['dry_run']) except Exception as err: exception_handler(action_def, err) logger.info('Action ID: %s, "%s" completed.', idx, action_def.action) logger.info('All actions completed.') @click.command( context_settings=context_settings(), epilog=footer(__version__, tail='command-line.html'), ) @options_from_dict(OPTION_DEFAULTS) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) @click.argument('action_file', type=click.Path(exists=True), nargs=1) @click.version_option(__version__, '-v', '--version', prog_name="curator") @click.pass_context def cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, loglevel, logfile, logformat, blacklist, dry_run, action_file, ): """ Curator for Elasticsearch indices The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. Some less-frequently used client configuration options are now hidden. To see the full list, run: curator_cli -h """ ctx.obj = {} ctx.obj['default_config'] = default_config_file() get_config(ctx) configure_logging(ctx) generate_configdict(ctx) run(ctx) elasticsearch-curator-8.0.21/curator/cli_singletons/000077500000000000000000000000001477314666200226045ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/cli_singletons/__init__.py000066400000000000000000000012521477314666200247150ustar00rootroot00000000000000"""Use __init__ to make these not need to be nested under lowercase.Capital""" from curator.cli_singletons.alias import alias from curator.cli_singletons.allocation import allocation from curator.cli_singletons.close import close from curator.cli_singletons.delete import delete_indices, delete_snapshots from curator.cli_singletons.forcemerge import forcemerge from curator.cli_singletons.open_indices import open_indices from curator.cli_singletons.replicas import replicas from curator.cli_singletons.restore import restore from curator.cli_singletons.rollover import rollover from curator.cli_singletons.shrink import shrink from curator.cli_singletons.snapshot import snapshot elasticsearch-curator-8.0.21/curator/cli_singletons/alias.py000066400000000000000000000043311477314666200242500ustar00rootroot00000000000000"""Alias Singleton""" # pylint: disable=R0913,R0917 import logging import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json @click.command() @click.option('--name', type=str, help='Alias name', required=True) @click.option( '--add', callback=validate_filter_json, help='JSON array of filters selecting indices to ADD to alias', default=None, ) @click.option( '--remove', callback=validate_filter_json, help='JSON array of filters selecting indices to REMOVE from alias', default=None, ) @click.option( '--warn_if_no_indices', is_flag=True, help='Do not raise exception if there are no actionable indices in add/remove', ) @click.option( '--extra_settings', help='JSON version of extra_settings (see documentation)', callback=json_to_dict, ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.pass_context def alias( ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_indices, include_hidden, ): """ Add/Remove Indices to/from Alias """ logger = logging.getLogger('cli.singleton.alias') manual_options = { 'name': name, 'extra_settings': extra_settings, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } logger.debug('manual_options %s', manual_options) # ctx.info_name is the name of the function or name specified in # @click.command decorator ignore_empty_list = warn_if_no_indices action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, [], # filter_list is empty in our case ignore_empty_list, add=add, remove=remove, warn_if_no_indices=warn_if_no_indices, # alias specific kwargs ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/allocation.py000066400000000000000000000051161477314666200253060ustar00rootroot00000000000000"""Allocation Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option('--key', type=str, required=True, help='Node identification tag') @click.option('--value', type=str, default=None, help='Value associated with --key') @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude'])) @click.option( '--wait_for_completion/--no-wait_for_completion', default=False, help='Wait for the allocation to complete', show_default=True, ) @click.option( '--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion', show_default=True, ) @click.option( '--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.', show_default=True, ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def allocation( ctx, search_pattern, key, value, allocation_type, wait_for_completion, max_wait, wait_interval, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Shard Routing Allocation """ manual_options = { 'search_pattern': search_pattern, 'key': key, 'value': value, 'allocation_type': allocation_type, 'wait_for_completion': wait_for_completion, 'max_wait': max_wait, 'wait_interval': wait_interval, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/close.py000066400000000000000000000036051477314666200242670ustar00rootroot00000000000000"""Close Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option( '--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed', ) @click.option( '--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed' ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def close( ctx, search_pattern, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Close Indices """ manual_options = { 'search_pattern': search_pattern, 'skip_flush': skip_flush, 'delete_aliases': delete_aliases, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/delete.py000066400000000000000000000060331477314666200244220ustar00rootroot00000000000000"""Delete Index and Delete Snapshot Singletons""" # pylint: disable=R0913,R0917 import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json # Indices @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern', ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def delete_indices( ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list ): """ Delete Indices """ # ctx.info_name is the name of the function or name specified in @click.command # decorator action = CLIAction( 'delete_indices', ctx.obj['configdict'], {'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices}, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) # Snapshots @click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option('--retry_count', type=int, help='Number of times to retry (max 3)') @click.option('--retry_interval', type=int, help='Time in seconds between retries') @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable snapshots', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', required=True, ) @click.pass_context def delete_snapshots( ctx, repository, retry_count, retry_interval, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Delete Snapshots """ manual_options = { 'retry_count': retry_count, 'retry_interval': retry_interval, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in @click.command # decorator action = CLIAction( 'delete_snapshots', ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, repository=repository, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/forcemerge.py000066400000000000000000000037651477314666200253070ustar00rootroot00000000000000"""ForceMerge Singleton""" # pylint: disable=R0913,R0917 import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option( '--max_num_segments', type=int, required=True, help='Maximum number of segments per shard (minimum of 1)', ) @click.option( '--delay', type=float, help='Time in seconds to delay between operations. Default 0. Maximum 3600', ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def forcemerge( ctx, search_pattern, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ forceMerge Indices (reduce segment count) """ manual_options = { 'search_pattern': search_pattern, 'max_num_segments': max_num_segments, 'delay': delay, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/object_class.py000066400000000000000000000250211477314666200256110ustar00rootroot00000000000000"""Object builder""" # pylint: disable=W0718,R0902,R0912,R0913,R0914,R0917 import logging import sys from voluptuous import Schema from es_client.builder import Builder from es_client.exceptions import FailedValidation from es_client.helpers.schemacheck import SchemaCheck from es_client.helpers.utils import prune_nones from curator import IndexList, SnapshotList from curator.actions import ( Alias, Allocation, Close, ClusterRouting, CreateIndex, DeleteIndices, ForceMerge, IndexSettings, Open, Reindex, Replicas, Rollover, Shrink, Snapshot, DeleteSnapshots, Restore, ) from curator.defaults.settings import VERSION_MAX, VERSION_MIN, snapshot_actions from curator.exceptions import ConfigurationError, NoIndices, NoSnapshots from curator.helpers.testers import validate_filters from curator.validators import options from curator.validators.filter_functions import validfilters CLASS_MAP = { 'alias': Alias, 'allocation': Allocation, 'close': Close, 'cluster_routing': ClusterRouting, 'create_index': CreateIndex, 'delete_indices': DeleteIndices, 'delete_snapshots': DeleteSnapshots, 'forcemerge': ForceMerge, 'index_settings': IndexSettings, 'open': Open, 'reindex': Reindex, 'replicas': Replicas, 'restore': Restore, 'rollover': Rollover, 'shrink': Shrink, 'snapshot': Snapshot, } EXCLUDED_OPTIONS = [ 'ignore_empty_list', 'timeout_override', 'continue_if_exception', 'disable_action', ] class CLIAction: """ Unified class for all CLI singleton actions """ def __init__( self, action, configdict, option_dict, filter_list, ignore_empty_list, **kwargs ): """Class setup :param action: The action name. :param configdict: ``dict`` containing everything needed for :py:class:`~.es_client.builder.Builder` to build an :py:class:`~.elasticsearch.Elasticsearch` client object. :param option_dict: Options for ``action``. :param filter_list: Filters to select indices for ``action``. :param ignore_empty_list: Exit ``0`` even if filters result in no indices for ``action``. :param kwargs: Other keyword args to pass to ``action``. :type action: str :type configdict: dict :type option_dict: dict :type filter_list: list :type ignore_empty_list: bool :type kwargs: dict """ self.logger = logging.getLogger('curator.cli_singletons.cli_action.' + action) self.filters = [] self.action = action self.list_object = None self.repository = kwargs['repository'] if 'repository' in kwargs else None if action[:5] != 'show_': # Ignore CLASS_MAP for show_indices/show_snapshots try: self.action_class = CLASS_MAP[action] except KeyError: self.logger.critical('Action must be one of %s', list(CLASS_MAP.keys())) self.check_options(option_dict) else: self.options = option_dict self.search_pattern = self.options.pop('search_pattern', '*') self.include_hidden = self.options.pop('include_hidden', False) # Extract allow_ilm_indices so it can be handled separately. if 'allow_ilm_indices' in self.options: self.allow_ilm = self.options.pop('allow_ilm_indices') else: self.allow_ilm = False if action == 'alias': self.logger.debug('ACTION = ALIAS') self.alias = { 'name': option_dict['name'], 'extra_settings': option_dict['extra_settings'], 'wini': ( kwargs['warn_if_no_indices'] if 'warn_if_no_indices' in kwargs else False ), } for k in ['add', 'remove']: if k in kwargs: self.alias[k] = {} self.check_filters(kwargs[k], loc='alias singleton', key=k) self.alias[k]['filters'] = self.filters if self.allow_ilm: self.alias[k]['filters'].append({'filtertype': 'ilm'}) # No filters for these actions elif action in ['cluster_routing', 'create_index', 'rollover']: self.action_kwargs = {} if action == 'rollover': self.logger.debug('rollover option_dict = %s', option_dict) else: self.check_filters(filter_list) try: builder = Builder( configdict=configdict, version_max=VERSION_MAX, version_min=VERSION_MIN ) builder.connect() # pylint: disable=broad-except except Exception as exc: raise ConfigurationError( f'Unable to connect to Elasticsearch as configured: {exc}' ) from exc # If we're here, we'll see the output from GET http(s)://hostname.tld:PORT self.logger.debug('Connection result: %s', builder.client.info()) self.client = builder.client self.ignore = ignore_empty_list def prune_excluded(self, option_dict): """Prune excluded options""" for k in list(option_dict.keys()): if k in EXCLUDED_OPTIONS: del option_dict[k] return option_dict def check_options(self, option_dict): """Validate provided options""" try: self.logger.debug('Validating provided options: %s', option_dict) # Kludgy work-around to needing 'repository' in options for these actions # but only to pass the schema check. It's removed again below. if self.action in ['delete_snapshots', 'restore']: option_dict['repository'] = self.repository _ = SchemaCheck( prune_nones(option_dict), options.get_schema(self.action), 'options', f'{self.action} singleton action "options"', ).result() self.options = self.prune_excluded(_) # Remove this after the schema check, as the action class won't need # it as an arg if self.action in ['delete_snapshots', 'restore']: del self.options['repository'] except FailedValidation as exc: self.logger.critical('Unable to parse options: %s', exc) sys.exit(1) def check_filters(self, filter_dict, loc='singleton', key='filters'): """Validate provided filters""" try: self.logger.debug('Validating provided filters: %s', filter_dict) _ = SchemaCheck( filter_dict, Schema(validfilters(self.action, location=loc)), key, f'{self.action} singleton action "{key}"', ).result() self.filters = validate_filters(self.action, _) except FailedValidation as exc: self.logger.critical('Unable to parse filters: %s', exc) sys.exit(1) def do_filters(self): """Actually run the filters""" self.logger.debug('Running filters and testing for empty list object') if self.allow_ilm: self.filters.append({'filtertype': 'ilm', 'exclude': True}) try: self.list_object.iterate_filters({'filters': self.filters}) self.list_object.empty_list_check() except (NoIndices, NoSnapshots) as exc: otype = 'index' if isinstance(exc, NoIndices) else 'snapshot' if self.ignore: self.logger.info('Singleton action not performed: empty %s list', otype) sys.exit(0) else: self.logger.error('Singleton action failed due to empty %s list', otype) sys.exit(1) def get_list_object(self): """Get either a SnapshotList or IndexList object""" if self.action in snapshot_actions() or self.action == 'show_snapshots': self.list_object = SnapshotList(self.client, repository=self.repository) else: self.list_object = IndexList( self.client, search_pattern=self.search_pattern, include_hidden=self.include_hidden, ) def get_alias_obj(self): """Get the Alias object""" action_obj = Alias( name=self.alias['name'], extra_settings=self.alias['extra_settings'] ) for k in ['remove', 'add']: if k in self.alias: msg = ( f"{'Add' if k == 'add' else 'Remov'}ing matching indices " f"{'to' if k == 'add' else 'from'} alias \"{self.alias['name']}\"" ) self.logger.debug(msg) self.alias[k]['ilo'] = IndexList( self.client, search_pattern=self.search_pattern, include_hidden=self.include_hidden, ) self.alias[k]['ilo'].iterate_filters( {'filters': self.alias[k]['filters']} ) fltr = getattr(action_obj, k) fltr(self.alias[k]['ilo'], warn_if_no_indices=self.alias['wini']) return action_obj def do_singleton_action(self, dry_run=False): """Execute the (ostensibly) completely ready to run action""" self.logger.debug('Doing the singleton "%s" action here.', self.action) try: if self.action == 'alias': action_obj = self.get_alias_obj() elif self.action in ['cluster_routing', 'create_index', 'rollover']: action_obj = self.action_class(self.client, **self.options) else: self.get_list_object() self.do_filters() self.logger.debug('OPTIONS = %s', self.options) action_obj = self.action_class(self.list_object, **self.options) if dry_run: action_obj.do_dry_run() else: action_obj.do_action() except NoIndices: # Speficically to address #1704 if not self.ignore: self.logger.critical('No indices in list after filtering. Exiting.') sys.exit(1) self.logger.info('No indices in list after filtering. Skipping action.') except Exception as exc: self.logger.critical( 'Failed to complete action: %s. Exception: %s', self.action, exc ) sys.exit(1) self.logger.info('"%s" action completed.', self.action) sys.exit(0) elasticsearch-curator-8.0.21/curator/cli_singletons/open_indices.py000066400000000000000000000031101477314666200256100ustar00rootroot00000000000000"""Open (closed) Index Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json @click.command(name='open') @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def open_indices( ctx, search_pattern, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Open Indices """ # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], { 'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, }, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/replicas.py000066400000000000000000000036721477314666200247700ustar00rootroot00000000000000"""Change Replica Count Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option('--count', type=int, required=True, help='Number of replicas (max 10)') @click.option( '--wait_for_completion/--no-wait_for_completion', default=False, help='Wait for replication to complete', show_default=True, ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def replicas( ctx, search_pattern, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Change Replica Count """ manual_options = { 'search_pattern': search_pattern, 'count': count, 'wait_for_completion': wait_for_completion, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/restore.py000066400000000000000000000100451477314666200246410ustar00rootroot00000000000000"""Snapshot Restore Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json # pylint: disable=line-too-long @click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option('--name', type=str, help='Snapshot name', required=False, default=None) @click.option( '--index', multiple=True, help='Index name to restore. (Can invoke repeatedly for multiple indices)', ) @click.option( '--rename_pattern', type=str, help='Rename pattern', required=False, default=None ) @click.option( '--rename_replacement', type=str, help='Rename replacement', required=False, default=None, ) @click.option( '--extra_settings', type=str, help='JSON version of extra_settings (see documentation)', callback=json_to_dict, ) @click.option( '--include_aliases', is_flag=True, show_default=True, help='Include aliases with restored indices.', ) @click.option( '--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.', ) @click.option( '--include_global_state', is_flag=True, show_default=True, help='Restore cluster global state with snapshot.', ) @click.option( '--partial', is_flag=True, show_default=True, help='Restore partial data (from snapshot taken with --partial).', ) @click.option( '--wait_for_completion/--no-wait_for_completion', default=True, show_default=True, help='Wait for the snapshot to complete', ) @click.option( '--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.', ) @click.option( '--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion', ) @click.option( '--skip_repo_fs_check', is_flag=True, show_default=True, help='Skip repository filesystem access validation.', ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', required=True, ) @click.pass_context def restore( ctx, repository, name, index, rename_pattern, rename_replacement, extra_settings, include_aliases, ignore_unavailable, include_global_state, partial, wait_for_completion, wait_interval, max_wait, skip_repo_fs_check, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Restore Indices """ indices = list(index) manual_options = { 'name': name, 'extra_settings': extra_settings, 'indices': indices, 'rename_pattern': rename_pattern, 'rename_replacement': rename_replacement, 'ignore_unavailable': ignore_unavailable, 'include_aliases': include_aliases, 'include_global_state': include_global_state, 'partial': partial, 'skip_repo_fs_check': skip_repo_fs_check, 'wait_for_completion': wait_for_completion, 'max_wait': max_wait, 'wait_interval': wait_interval, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, repository=repository, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/rollover.py000066400000000000000000000045361477314666200250320ustar00rootroot00000000000000"""Index Rollover Singleton""" import click from es_client.helpers.utils import prune_nones from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict # pylint: disable=line-too-long @click.command() @click.option('--name', type=str, help='Alias name', required=True) @click.option('--max_age', type=str, help='max_age condition value (see documentation)') @click.option( '--max_docs', type=str, help='max_docs condition value (see documentation)' ) @click.option( '--max_size', type=str, help='max_size condition value (see documentation)' ) @click.option( '--extra_settings', type=str, help='JSON version of extra_settings (see documentation)', callback=json_to_dict, ) @click.option( '--new_index', type=str, help='Optional new index name (see documentation)' ) @click.option( '--wait_for_active_shards', type=int, default=1, show_default=True, help='Wait for number of shards to be active before returning', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.pass_context def rollover( ctx, name, max_age, max_docs, max_size, extra_settings, new_index, wait_for_active_shards, allow_ilm_indices, include_hidden, ): """ Rollover Index associated with Alias """ conditions = prune_nones( { 'max_age': max_age, 'max_docs': max_docs, 'max_size': max_size, } ) manual_options = { 'name': name, 'conditions': conditions, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, [], True, extra_settings=extra_settings, new_index=new_index, wait_for_active_shards=wait_for_active_shards, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/show.py000066400000000000000000000123411477314666200241370ustar00rootroot00000000000000"""Show Index/Snapshot Singletons""" from datetime import datetime import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json from curator.helpers.getters import byte_size from curator.defaults.settings import footer from curator._version import __version__ # ### Indices ### # pylint: disable=line-too-long @click.command( epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots') ) @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True) @click.option( '--header', help='Print header if --verbose', is_flag=True, show_default=True ) @click.option( '--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.', ) @click.pass_context def show_indices( ctx, search_pattern, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Show Indices """ # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( 'show_indices', ctx.obj['configdict'], { 'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, }, filter_list, ignore_empty_list, ) action.get_list_object() action.do_filters() indices = sorted(action.list_object.indices) # Do some calculations to figure out the proper column sizes allbytes = [] alldocs = [] for idx in indices: allbytes.append(byte_size(action.list_object.index_info[idx]['size_in_bytes'])) alldocs.append(str(action.list_object.index_info[idx]['docs'])) if epoch: timeformat = '{6:>13}' column = 'creation_date' else: timeformat = '{6:>20}' column = 'Creation Timestamp' formatting = ( '{0:' + str(len(max(indices, key=len))) + '} ' '{1:>5} ' '{2:>' + str(len(max(allbytes, key=len)) + 1) + '} ' '{3:>' + str(len(max(alldocs, key=len)) + 1) + '} ' '{4:>3} {5:>3} ' + timeformat ) # Print the header, if both verbose and header are enabled if header and verbose: click.secho( formatting.format('Index', 'State', 'Size', 'Docs', 'Pri', 'Rep', column), bold=True, underline=True, ) # Loop through indices and print info, if verbose for idx in indices: data = action.list_object.index_info[idx] if verbose: if epoch: datefield = ( data['age']['creation_date'] if 'creation_date' in data['age'] else 0 ) else: datefield = ( datetime.utcfromtimestamp(data['age']['creation_date']).isoformat() if 'creation_date' in data['age'] else 'unknown/closed' ) click.echo( formatting.format( idx, data['state'], byte_size(data['size_in_bytes']), data['docs'], data['number_of_shards'], data['number_of_replicas'], f'{datefield}Z', ) ) else: click.secho(f'{idx}') # ### Snapshots ### # pylint: disable=line-too-long @click.command( epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots') ) @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable snapshots', ) @click.option( '--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.', ) @click.pass_context def show_snapshots(ctx, repository, ignore_empty_list, filter_list): """ Show Snapshots """ # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( 'show_snapshots', ctx.obj['configdict'], {}, filter_list, ignore_empty_list, repository=repository, ) action.get_list_object() action.do_filters() for snapshot in sorted(action.list_object.snapshots): click.secho(f'{snapshot}') elasticsearch-curator-8.0.21/curator/cli_singletons/shrink.py000066400000000000000000000107321477314666200244570ustar00rootroot00000000000000"""Shrink Index Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json # pylint: disable=line-too-long @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option( '--shrink_node', default='DETERMINISTIC', type=str, help='Named node, or DETERMINISTIC', show_default=True, ) @click.option( '--node_filters', help='JSON version of node_filters (see documentation)', callback=json_to_dict, ) @click.option( '--number_of_shards', default=1, type=int, help='Shrink to this many shards per index', ) @click.option( '--number_of_replicas', default=1, type=int, help='Number of replicas for the target index', show_default=True, ) @click.option('--shrink_prefix', type=str, help='Prefix for the target index name') @click.option( '--shrink_suffix', default='-shrink', type=str, help='Suffix for the target index name', show_default=True, ) @click.option( '--copy_aliases', is_flag=True, help='Copy each source index aliases to target index', ) @click.option( '--delete_after/--no-delete_after', default=True, help='Delete source index after shrink', show_default=True, ) @click.option( '--post_allocation', help='JSON version of post_allocation (see documentation)', callback=json_to_dict, ) @click.option( '--extra_settings', help='JSON version of extra_settings (see documentation)', callback=json_to_dict, ) @click.option( '--wait_for_active_shards', default=1, type=int, help='Wait for number of active shards before continuing', ) @click.option( '--wait_for_rebalance/--no-wait_for_rebalance', default=True, help='Wait for rebalance to complete', ) @click.option( '--wait_for_completion/--no-wait_for_completion', default=True, help='Wait for the shrink to complete', ) @click.option( '--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.', ) @click.option( '--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion', ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def shrink( ctx, search_pattern, shrink_node, node_filters, number_of_shards, number_of_replicas, shrink_prefix, shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings, wait_for_active_shards, wait_for_rebalance, wait_for_completion, wait_interval, max_wait, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Shrink Indices to --number_of_shards """ manual_options = { 'search_pattern': search_pattern, 'shrink_node': shrink_node, 'node_filters': node_filters, 'number_of_shards': number_of_shards, 'number_of_replicas': number_of_replicas, 'shrink_prefix': shrink_prefix, 'shrink_suffix': shrink_suffix, 'copy_aliases': copy_aliases, 'delete_after': delete_after, 'post_allocation': post_allocation, 'extra_settings': extra_settings, 'wait_for_active_shards': wait_for_active_shards, 'wait_for_rebalance': wait_for_rebalance, 'wait_for_completion': wait_for_completion, 'wait_interval': wait_interval, 'max_wait': max_wait, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/snapshot.py000066400000000000000000000064151477314666200250230ustar00rootroot00000000000000"""Snapshot Singleton""" import click from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json # pylint: disable=line-too-long @click.command() @click.option( '--search_pattern', type=str, default='*', help='Elasticsearch Index Search Pattern' ) @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option( '--name', type=str, help='Snapshot name', show_default=True, default='curator-%Y%m%d%H%M%S', ) @click.option( '--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.', ) @click.option( '--include_global_state', is_flag=True, show_default=True, help='Store cluster global state with snapshot.', ) @click.option( '--partial', is_flag=True, show_default=True, help='Do not fail if primary shard is unavailable.', ) @click.option( '--wait_for_completion/--no-wait_for_completion', default=True, show_default=True, help='Wait for the snapshot to complete', ) @click.option( '--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.', ) @click.option( '--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion', ) @click.option( '--skip_repo_fs_check', is_flag=True, show_default=True, help='Skip repository filesystem access validation.', ) @click.option( '--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices', ) @click.option( '--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True, ) @click.option( '--include_hidden/--no-include_hidden', help='Allow Curator to operate on hidden indices (and data_streams).', default=False, show_default=True, ) @click.option( '--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True, ) @click.pass_context def snapshot( ctx, search_pattern, repository, name, ignore_unavailable, include_global_state, partial, skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list, allow_ilm_indices, include_hidden, filter_list, ): """ Snapshot Indices """ manual_options = { 'search_pattern': search_pattern, 'name': name, 'repository': repository, 'ignore_unavailable': ignore_unavailable, 'include_global_state': include_global_state, 'partial': partial, 'skip_repo_fs_check': skip_repo_fs_check, 'wait_for_completion': wait_for_completion, 'max_wait': max_wait, 'wait_interval': wait_interval, 'allow_ilm_indices': allow_ilm_indices, 'include_hidden': include_hidden, } # ctx.info_name is the name of the function or name specified in # @click.command decorator action = CLIAction( ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, ) action.do_singleton_action(dry_run=ctx.obj['dry_run']) elasticsearch-curator-8.0.21/curator/cli_singletons/utils.py000066400000000000000000000020301477314666200243110ustar00rootroot00000000000000"""Singleton Utils Module""" import json from click import BadParameter from es_client.helpers.utils import ensure_list # Click functions require ctx and param to be passed positionally even if not used # pylint: disable=unused-argument def json_to_dict(ctx, param, value): """Convert JSON to dictionary""" if value is None: return {} try: return json.loads(value) except ValueError as exc: raise BadParameter(f'Invalid JSON for {param}: {value}') from exc # Click functions require ctx and param to be passed positionally even if not used # pylint: disable=unused-argument def validate_filter_json(ctx, param, value): """Validate the JSON provided from the command-line""" # The "None" edge case should only be for optional filters, like alias add/remove if value is None: return value try: filter_list = ensure_list(json.loads(value)) return filter_list except ValueError as exc: raise BadParameter(f'Filter list is invalid JSON: {value}') from exc elasticsearch-curator-8.0.21/curator/defaults/000077500000000000000000000000001477314666200213775ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/defaults/__init__.py000066400000000000000000000000001477314666200234760ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/defaults/filter_elements.py000066400000000000000000000257071477314666200251450ustar00rootroot00000000000000"""Filter element schema definitions All member functions return a :class:`voluptuous.schema_builder.Schema` object """ from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required from curator.defaults import settings # pylint: disable=unused-argument, line-too-long def aliases(**kwargs): """ :returns: {Required('aliases'): Any(list, str)} """ return {Required('aliases'): Any(list, str)} def allocation_type(**kwargs): """ :returns: {Optional('allocation_type', default='require'): All(Any(str), Any('require', 'include', 'exclude'))} """ return { Optional('allocation_type', default='require'): All( Any(str), Any('require', 'include', 'exclude') ) } def count(**kwargs): """ This setting is only used with the count filtertype and is required :returns: {Required('count'): All(Coerce(int), Range(min=1))} """ return {Required('count'): All(Coerce(int), Range(min=1))} def date_from(**kwargs): """ This setting is only used with the period filtertype. :returns: {Optional('date_from'): Any(str)} """ return {Optional('date_from'): Any(str)} def date_from_format(**kwargs): """ This setting is only used with the period filtertype. :returns: {Optional('date_from_format'): Any(str)} """ return {Optional('date_from_format'): Any(str)} def date_to(**kwargs): """ This setting is only used with the period filtertype. :returns: {Optional('date_to'): Any(str)} """ return {Optional('date_to'): Any(str)} def date_to_format(**kwargs): """ This setting is only used with the period filtertype. :returns: {Optional('date_to_format'): Any(str)} """ return {Optional('date_to_format'): Any(str)} def direction(**kwargs): """ This setting is only used with the ``age`` filtertype. :returns: {Required('direction'): Any('older', 'younger')} """ return {Required('direction'): Any('older', 'younger')} def disk_space(**kwargs): """ This setting is only used with the ``space`` filtertype and is required :returns: {Required('disk_space'): Any(Coerce(float))} """ return {Required('disk_space'): Any(Coerce(float))} def epoch(**kwargs): """ This setting is only used with the ``age`` filtertype. :returns: {Optional('epoch', default=None): Any(Coerce(int), None)} """ return {Optional('epoch', default=None): Any(Coerce(int), None)} def exclude(**kwargs): """ This setting is available in all filter types. The default ``val`` is ``True`` if ``exclude`` in ``kwargs``, otherwise ``False`` :returns: {Optional('exclude', default=val): Any(bool, All(Any(str), Boolean()))} """ val = bool('exclude' in kwargs and kwargs['exclude']) # pylint: disable=no-value-for-parameter return {Optional('exclude', default=val): Any(bool, All(Any(str), Boolean()))} def field(**kwargs): """ This setting is only used with the ``age`` filtertype. :returns: {Required('field'): Any(str)} if ``kwargs['required']`` is ``True`` otherwise {Optional('field'): Any(str)} """ if 'required' in kwargs and kwargs['required']: return {Required('field'): Any(str)} return {Optional('field'): Any(str)} def intersect(**kwargs): """ This setting is only used with the period filtertype when using field_stats, i.e. indices only. :returns: {Optional('intersect', default=False): Any(bool, All(Any(str), Boolean()))} """ # pylint: disable=no-value-for-parameter return {Optional('intersect', default=False): Any(bool, All(Any(str), Boolean()))} def key(**kwargs): """ This setting is only used with the allocated filtertype. :returns: {Required('key'): Any(str)} """ return {Required('key'): Any(str)} def kind(**kwargs): """ This setting is only used with the pattern filtertype and is required :returns: {Required('kind'): Any('prefix', 'suffix', 'timestring', 'regex')} """ return {Required('kind'): Any('prefix', 'suffix', 'timestring', 'regex')} def max_num_segments(**kwargs): """ :returns: {Required('max_num_segments'): All(Coerce(int), Range(min=1))} """ return {Required('max_num_segments'): All(Coerce(int), Range(min=1))} def number_of_shards(**kwargs): """ :returns: {Required('number_of_shards'): All(Coerce(int), Range(min=1))} """ return {Required('number_of_shards'): All(Coerce(int), Range(min=1))} def pattern(**kwargs): """ :returns: {Optional('pattern'): Any(str)} """ return {Optional('pattern'): Any(str)} def period_type(**kwargs): """ This setting is only used with the period filtertype. :returns: {Optional('period_type', default='relative'): Any('relative', 'absolute')} """ return {Optional('period_type', default='relative'): Any('relative', 'absolute')} def range_from(**kwargs): """ :returns: {Optional('range_from'): Coerce(int)} """ return {Optional('range_from'): Coerce(int)} def range_to(**kwargs): """ :returns: {Optional('range_to'): Coerce(int)} """ return {Optional('range_to'): Coerce(int)} def reverse(**kwargs): """ Only used with ``space`` filtertype. Should be ignored if ```use_age``` is True :returns: {Optional('reverse', default=True): Any(bool, All(Any(str), Boolean()))} """ # pylint: disable=no-value-for-parameter return {Optional('reverse', default=True): Any(bool, All(Any(str), Boolean()))} def shard_filter_behavior(**kwargs): """ This setting is only used with the shards filtertype and defaults to 'greater_than'. :returns: {Optional('shard_filter_behavior', default='greater_than'): Any('greater_than', 'less_than', 'greater_than_or_equal', 'less_than_or_equal', 'equal')} """ return { Optional('shard_filter_behavior', default='greater_than'): Any( 'greater_than', 'less_than', 'greater_than_or_equal', 'less_than_or_equal', 'equal', ) } def size_threshold(**kwargs): """ This setting is only used with the size filtertype and is required :returns: {Required('size_threshold'): Any(Coerce(float))} """ return {Required('size_threshold'): Any(Coerce(float))} def size_behavior(**kwargs): """ This setting is only used with the size filtertype and defaults to 'primary'. :returns: {Optional('size_behavior', default='primary'): Any('primary', 'total')} """ return {Optional('size_behavior', default='primary'): Any('primary', 'total')} def source(**kwargs): """ This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype when ``use_age`` is set to True. :ivar valuelist: If ``kwargs['action']`` is in :py:func:`curator.defaults.settings.snapshot_actions`, then it is ``Any('name', 'creation_date')``, otherwise ``Any('name', 'creation_date', 'field_stats')`` :returns: {Required('source'): valuelist} if ``kwargs['required']``, else {Optional('source'): valuelist} """ if 'action' in kwargs and kwargs['action'] in settings.snapshot_actions(): valuelist = Any('name', 'creation_date') valuelist = Any('name', 'creation_date', 'field_stats') if 'required' in kwargs and kwargs['required']: return {Required('source'): valuelist} return {Optional('source'): valuelist} def state(**kwargs): """ This setting is only used with the state filtertype. :returns: {Optional('state', default='SUCCESS'): Any('SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS')} """ return { Optional('state', default='SUCCESS'): Any( 'SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS' ) } def stats_result(**kwargs): """ This setting is only used with the ``age`` filtertype. :returns: {Optional('stats_result', default='min_value'): Any('min_value', 'max_value')} """ return { Optional('stats_result', default='min_value'): Any('min_value', 'max_value') } def timestring(**kwargs): """ This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype if ``use_age`` is set to ``True``. :returns: {Required('timestring'): Any(str)} if ``kwargs['required']`` else {Optional('timestring', default=None): Any(None, str)} """ if 'required' in kwargs and kwargs['required']: return {Required('timestring'): Any(str)} return {Optional('timestring', default=None): Any(None, str)} def threshold_behavior(**kwargs): """ This setting is only used with the space and size filtertype and defaults to 'greater_than'. :returns: {Optional('threshold_behavior', default='greater_than'): Any('greater_than', 'less_than')} """ return { Optional('threshold_behavior', default='greater_than'): Any( 'greater_than', 'less_than' ) } def unit(**kwargs): """ This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype if ``use_age`` is set to ``True``. :returns: {Required('unit'): Any('seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years')} """ return { Required('unit'): Any( 'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years' ) } def unit_count(**kwargs): """ This setting is only used with the ``age`` filtertype, or with the ``space`` filtertype if ``use_age`` is set to ``True``. :returns: {Required('unit_count'): Coerce(int)} """ return {Required('unit_count'): Coerce(int)} def unit_count_pattern(**kwargs): """ This setting is used with the ``age`` filtertype to define whether the ``unit_count`` value is taken from the configuration or read from the index name via a regular expression :returns: {Optional('unit_count_pattern'): Any(str)} """ return {Optional('unit_count_pattern'): Any(str)} def use_age(**kwargs): """ Use of this setting requires the additional setting, ``source``. :returns: {Optional('use_age', default=False): Any(bool, All(Any(str), Boolean()))} """ # pylint: disable=no-value-for-parameter return {Optional('use_age', default=False): Any(bool, All(Any(str), Boolean()))} def value(**kwargs): """ This setting is only used with the ``pattern`` filtertype and is a required setting. There is a separate value option associated with the ``Allocation`` action, and the ``allocated`` filtertype. :returns: {Required('value'): Any(str)} """ return {Required('value'): Any(str)} def week_starts_on(**kwargs): """ :returns: {Optional('week_starts_on', default='sunday'): Any('Sunday', 'sunday', 'SUNDAY', 'Monday', 'monday', 'MONDAY', None)} """ return { Optional('week_starts_on', default='sunday'): Any( 'Sunday', 'sunday', 'SUNDAY', 'Monday', 'monday', 'MONDAY', None ) } elasticsearch-curator-8.0.21/curator/defaults/filtertypes.py000066400000000000000000000200301477314666200243160ustar00rootroot00000000000000"""Filtertype schema definitions""" import logging from curator.defaults import filter_elements, settings # pylint: disable=missing-docstring, unused-argument, line-too-long def _age_elements(action, config): """ Sort which filter types that have ``use_age`` are suitable for :py:class:`~.curator.IndexList` and which are acceptable in :py:class:`~.curator.SnapshotList`, which are required, and which are not. :param action: The name of an action :type action: str :param config: The configuration block for one filter of ``action`` :type config: dict :returns: A :py:class:`list` containing one or more :py:class:`~.voluptuous.schema_builder.Optional` or :py:class:`~.voluptuous.schema_builder.Required` options from :py:mod:`~.curator.defaults.filter_elements`, defining acceptable values for each for the given ``action`` :rtype: list """ retval = [] is_req = True if config['filtertype'] in ['count', 'space']: # is_req = True if 'use_age' in config and config['use_age'] else False is_req = bool('use_age' in config and config['use_age']) retval.append(filter_elements.source(action=action, required=is_req)) if action in settings.index_actions(): retval.append(filter_elements.stats_result()) # This is a silly thing here, because the absence of 'source' will # show up in the actual schema check, but it keeps code from breaking here ts_req = False if 'source' in config: if config['source'] == 'name': ts_req = True elif action in settings.index_actions(): # field_stats must _only_ exist for Index actions (not Snapshot) if config['source'] == 'field_stats': retval.append(filter_elements.field(required=True)) else: retval.append(filter_elements.field(required=False)) retval.append(filter_elements.timestring(required=ts_req)) else: # If source isn't in the config, then the other elements are not # required, but should be Optional to prevent false positives retval.append(filter_elements.field(required=False)) retval.append(filter_elements.timestring(required=ts_req)) return retval # ### Schema information ### def alias(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_by_alias` """ return [ filter_elements.aliases(), filter_elements.exclude(), ] def age(action, config): """ :returns: Filter elements acceptable for :py:class:`~.curator.IndexList` :py:meth:`~.curator.IndexList.filter_by_age` or :py:class:`~.curator.SnapshotList` :py:meth:`~.curator.SnapshotList.filter_by_age` """ # Required & Optional logger = logging.getLogger('curator.defaults.filtertypes.age') retval = [ filter_elements.direction(), filter_elements.unit(), filter_elements.unit_count(), filter_elements.unit_count_pattern(), filter_elements.epoch(), filter_elements.exclude(), ] retval += _age_elements(action, config) logger.debug('AGE FILTER = %s', retval) return retval def allocated(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_allocated` """ return [ filter_elements.key(), filter_elements.value(), filter_elements.allocation_type(), filter_elements.exclude(exclude=True), ] def closed(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_closed` """ return [filter_elements.exclude(exclude=True)] def count(action, config): """ :returns: Filter elements acceptable for :py:class:`~.curator.IndexList` :py:meth:`~.curator.IndexList.filter_by_count` or :py:class:`~.curator.SnapshotList` :py:meth:`~.curator.SnapshotList.filter_by_count` """ retval = [ filter_elements.count(), filter_elements.use_age(), filter_elements.pattern(), filter_elements.reverse(), filter_elements.exclude(exclude=True), ] retval += _age_elements(action, config) return retval def empty(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_empty` """ return [filter_elements.exclude()] def forcemerged(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_forceMerged` """ return [ filter_elements.max_num_segments(), filter_elements.exclude(exclude=True), ] def ilm(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_ilm` """ return [filter_elements.exclude(exclude=True)] def kibana(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_kibana` """ return [filter_elements.exclude(exclude=True)] def none(action, config): """ :returns: Filter elements acceptable for :py:class:`~.curator.IndexList` :py:meth:`~.curator.IndexList.filter_none` or :py:class:`~.curator.SnapshotList` :py:meth:`~.curator.SnapshotList.filter_none` """ return [] def opened(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_opened` """ return [filter_elements.exclude(exclude=True)] def pattern(action, config): """ :returns: Filter elements acceptable for :py:class:`~.curator.IndexList` :py:meth:`~.curator.IndexList.filter_by_regex` or :py:class:`~.curator.SnapshotList` :py:meth:`~.curator.SnapshotList.filter_by_regex` """ return [ filter_elements.kind(), filter_elements.value(), filter_elements.exclude(), ] def period(action, config): """ :returns: Filter elements acceptable for :py:class:`~.curator.IndexList` :py:meth:`~.curator.IndexList.filter_period` or :py:class:`~.curator.SnapshotList` :py:meth:`~.curator.SnapshotList.filter_period` """ retval = [ filter_elements.unit(period=True), filter_elements.range_from(), filter_elements.range_to(), filter_elements.week_starts_on(), filter_elements.epoch(), filter_elements.exclude(), filter_elements.period_type(), filter_elements.date_from(), filter_elements.date_from_format(), filter_elements.date_to(), filter_elements.date_to_format(), ] # Only add intersect() to index actions. if action in settings.index_actions(): retval.append(filter_elements.intersect()) retval += _age_elements(action, config) return retval def shards(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_by_shards` """ return [ filter_elements.number_of_shards(), filter_elements.shard_filter_behavior(), filter_elements.exclude(), ] def size(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_by_size` """ return [ filter_elements.size_threshold(), filter_elements.threshold_behavior(), filter_elements.size_behavior(), filter_elements.exclude(), ] def space(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.IndexList.filter_by_space` """ retval = [ filter_elements.disk_space(), filter_elements.reverse(), filter_elements.use_age(), filter_elements.exclude(), filter_elements.threshold_behavior(), ] retval += _age_elements(action, config) return retval def state(action, config): """ :returns: Filter elements acceptable for :py:meth:`~.curator.SnapshotList.filter_by_state` """ return [ filter_elements.state(), filter_elements.exclude(), ] elasticsearch-curator-8.0.21/curator/defaults/option_defaults.py000066400000000000000000000454431477314666200251620ustar00rootroot00000000000000"""Action Option Schema definitions""" from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required # pylint: disable=E1120 def allocation_type(): """ :returns: {Optional('allocation_type', default='require'): All(Any(str), Any('require', 'include', 'exclude'))} """ return { Optional('allocation_type', default='require'): All( Any(str), Any('require', 'include', 'exclude') ) } def allow_ilm_indices(): """ :returns: {Optional('allow_ilm_indices', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('allow_ilm_indices', default=False): Any( bool, All(Any(str), Boolean()) ) } def conditions(): """ :returns: {Optional('conditions'): {Optional('max_age'): Any(str), Optional('max_docs'): Coerce(int), Optional('max_size'): Any(str)}} """ return { Optional('conditions'): { Optional('max_age'): Any(str), Optional('max_docs'): Coerce(int), Optional('max_size'): Any(str), } } def continue_if_exception(): """ :returns: {Optional('continue_if_exception', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('continue_if_exception', default=False): Any( bool, All(Any(str), Boolean()) ) } def count(): """ :returns: {Required('count'): All(Coerce(int), Range(min=0, max=10))} """ return {Required('count'): All(Coerce(int), Range(min=0, max=10))} def delay(): """ :returns: {Optional('delay', default=0): All(Coerce(float), Range(min=0.0, max=3600.0))} """ return { Optional('delay', default=0): All(Coerce(float), Range(min=0.0, max=3600.0)) } def c2f_index_settings(): """ Only for the :py:class:`~.curator.actions.Cold2Frozen` action :returns: {Optional('index_settings'): Any(None, dict)} """ return {Optional('index_settings', default=None): Any(None, dict)} def c2f_ignore_index_settings(): """ Only for the :py:class:`~.curator.actions.Cold2Frozen` action :returns: {Optional('ignore_index_settings'): Any(None, list)} """ return {Optional('ignore_index_settings', default=None): Any(None, list)} def copy_aliases(): """ :returns: {Optional('copy_aliases', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('copy_aliases', default=False): Any(bool, All(Any(str), Boolean())) } def delete_after(): """ :returns: {Optional('delete_after', default=True): Any(bool, All(Any(str), Boolean()))} """ return {Optional('delete_after', default=True): Any(bool, All(Any(str), Boolean()))} def delete_aliases(): """ :returns: {Optional('delete_aliases', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('delete_aliases', default=False): Any(bool, All(Any(str), Boolean())) } def skip_flush(): """ :returns: {Optional('skip_flush', default=False): Any(bool, All(Any(str), Boolean()))} """ return {Optional('skip_flush', default=False): Any(bool, All(Any(str), Boolean()))} def disable_action(): """ :returns: {Optional('disable_action', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('disable_action', default=False): Any(bool, All(Any(str), Boolean())) } def extra_settings(): """ :returns: {Optional('extra_settings', default={}): dict} """ return {Optional('extra_settings', default={}): dict} def ignore_empty_list(): """ :returns: {Optional('ignore_empty_list', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('ignore_empty_list', default=False): Any( bool, All(Any(str), Boolean()) ) } def ignore_existing(): """ :returns: {Optional('ignore_existing', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('ignore_existing', default=False): Any(bool, All(Any(str), Boolean())) } def ignore_unavailable(): """ :returns: {Optional('ignore_unavailable', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('ignore_unavailable', default=False): Any( bool, All(Any(str), Boolean()) ) } def include_aliases(): """ :returns: {Optional('include_aliases', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('include_aliases', default=False): Any(bool, All(Any(str), Boolean())) } def include_global_state(action): """ :returns: {Optional('include_global_state', default=default): Any(bool, All(Any(str), Boolean()))} """ default = False if action == 'snapshot': default = True return { Optional('include_global_state', default=default): Any( bool, All(Any(str), Boolean()) ) } def include_hidden(): """ :returns: {Optional('include_hidden', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('include_hidden', default=False): Any(bool, All(Any(str), Boolean())) } def index_settings(): """ :returns: {Required('index_settings'): {'index': dict}} """ return {Required('index_settings'): {'index': dict}} def indices(): """ :returns: {Optional('indices', default=None): Any(None, list)} """ return {Optional('indices', default=None): Any(None, list)} def key(): """ :returns: {Required('key'): Any(str)} """ return {Required('key'): Any(str)} def max_num_segments(): """ :returns: {Required('max_num_segments'): All(Coerce(int), Range(min=1, max=32768))} """ return {Required('max_num_segments'): All(Coerce(int), Range(min=1, max=32768))} # pylint: disable=unused-argument def max_wait(action): """ :returns: {Optional('max_wait', default=defval): Any(-1, Coerce(int), None)} """ # The separation is here in case I want to change defaults later... defval = -1 # if action in ['allocation', 'cluster_routing', 'replicas']: # defval = -1 # elif action in ['restore', 'snapshot', 'reindex', 'shrink']: # defval = -1 return {Optional('max_wait', default=defval): Any(-1, Coerce(int), None)} def migration_prefix(): """ :returns: {Optional('migration_prefix', default=''): Any(None, str)} """ return {Optional('migration_prefix', default=''): Any(None, str)} def migration_suffix(): """ :returns: {Optional('migration_suffix', default=''): Any(None, str)} """ return {Optional('migration_suffix', default=''): Any(None, str)} def name(action): """ :returns: The proper name based on what action it is: ``alias``, ``create_index``, ``rollover``: {Required('name'): Any(str)} ``snapshot``: {Optional('name', default='curator-%Y%m%d%H%M%S'): Any(str)} ``restore``: {Optional('name'): Any(str)} """ if action in ['alias', 'create_index', 'rollover']: return {Required('name'): Any(str)} if action == 'snapshot': return {Optional('name', default='curator-%Y%m%d%H%M%S'): Any(str)} if action == 'restore': return {Optional('name'): Any(str)} def new_index(): """ :returns: {Optional('new_index', default=None): Any(None, str)} """ return {Optional('new_index', default=None): Any(None, str)} def node_filters(): """ :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. """ return { Optional('node_filters', default={}): { Optional('permit_masters', default=False): Any( bool, All(Any(str), Boolean()) ), Optional('exclude_nodes', default=[]): Any(list, None), } } def number_of_replicas(): """ :returns: {Optional('number_of_replicas', default=1): All(Coerce(int), Range(min=0, max=10))} """ return { Optional('number_of_replicas', default=1): All( Coerce(int), Range(min=0, max=10) ) } def number_of_shards(): """ :returns: {Optional('number_of_shards', default=1): All(Coerce(int), Range(min=1, max=99))} """ return { Optional('number_of_shards', default=1): All(Coerce(int), Range(min=1, max=99)) } def partial(): """ :returns: {Optional('partial', default=False): Any(bool, All(Any(str), Boolean()))} """ return {Optional('partial', default=False): Any(bool, All(Any(str), Boolean()))} def post_allocation(): """ :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. """ return { Optional('post_allocation', default={}): Any( {}, All( { Required('allocation_type', default='require'): All( Any(str), Any('require', 'include', 'exclude') ), Required('key'): Any(str), Required('value', default=None): Any(None, str), } ), ) } def preserve_existing(): """ :returns: {Optional('preserve_existing', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('preserve_existing', default=False): Any( bool, All(Any(str), Boolean()) ) } def refresh(): """ :returns: {Optional('refresh', default=True): Any(bool, All(Any(str), Boolean()))} """ return {Optional('refresh', default=True): Any(bool, All(Any(str), Boolean()))} def remote_certificate(): """ :returns: {Optional('remote_certificate', default=None): Any(None, str)} """ return {Optional('remote_certificate', default=None): Any(None, str)} def remote_client_cert(): """ :returns: {Optional('remote_client_cert', default=None): Any(None, str)} """ return {Optional('remote_client_cert', default=None): Any(None, str)} def remote_client_key(): """ :returns: {Optional('remote_client_key', default=None): Any(None, str)} """ return {Optional('remote_client_key', default=None): Any(None, str)} def remote_filters(): """ :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. """ # This is really just a basic check here. The real check is in the # validate_actions() method in utils.py return { Optional( 'remote_filters', default=[ { 'filtertype': 'pattern', 'kind': 'regex', 'value': '.*', 'exclude': True, } ], ): Any(list, None) } def rename_pattern(): """ :returns: {Optional('rename_pattern'): Any(str)} """ return {Optional('rename_pattern'): Any(str)} def rename_replacement(): """ :returns: {Optional('rename_replacement'): Any(str)} """ return {Optional('rename_replacement'): Any(str)} def repository(): """ :returns: {Required('repository'): Any(str)} """ return {Required('repository'): Any(str)} def request_body(): """ :returns: A :py:class:`voluptuous.schema_builder.Schema` object. See code for more details. """ return { Required('request_body'): { Optional('conflicts'): Any('proceed', 'abort'), Optional('max_docs'): Coerce(int), Required('source'): { Required('index'): Any(Any(str), list), Optional('query'): dict, Optional('remote'): { Optional('host'): Any(str), Optional('username'): Any(str), Optional('password'): Any(str), Optional('socket_timeout'): Any(str), Optional('connect_timeout'): Any(str), Optional('headers'): Any(str), }, Optional('size'): Coerce(int), Optional('_source'): Any(bool, Boolean()), }, Required('dest'): { Required('index'): Any(str), Optional('version_type'): Any( 'internal', 'external', 'external_gt', 'external_gte' ), Optional('op_type'): Any(str), Optional('pipeline'): Any(str), }, Optional('script'): { Optional('source'): Any(str), Optional('lang'): Any('painless', 'expression', 'mustache', 'java'), }, } } def requests_per_second(): """ :returns: {Optional('requests_per_second', default=-1): Any(-1, Coerce(int), None)} """ return {Optional('requests_per_second', default=-1): Any(-1, Coerce(int), None)} def retry_count(): """ :returns: {Optional('retry_count', default=3): All(Coerce(int), Range(min=0, max=100))} """ return {Optional('retry_count', default=3): All(Coerce(int), Range(min=0, max=100))} def retry_interval(): """ :returns: {Optional('retry_interval', default=120): All(Coerce(int), Range(min=1, max=600))} """ return { Optional('retry_interval', default=120): All(Coerce(int), Range(min=1, max=600)) } def routing_type(): """ :returns: {Required('routing_type'): Any('allocation', 'rebalance')} """ return {Required('routing_type'): Any('allocation', 'rebalance')} def cluster_routing_setting(): """ :returns: {Required('setting'): Any('enable')} """ return {Required('setting'): Any('enable')} def cluster_routing_value(): """ :returns: {Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')} """ return { Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas') } def search_pattern(): """ :returns: {Optional('search_pattern', default='*'): Any(str)} """ return {Optional('search_pattern', default='*'): Any(str)} def shrink_node(): """ :returns: {Required('shrink_node'): Any(str)} """ return {Required('shrink_node'): Any(str)} def shrink_prefix(): """ :returns: {Optional('shrink_prefix', default=''): Any(None, str)} """ return {Optional('shrink_prefix', default=''): Any(None, str)} def shrink_suffix(): """ :returns: {Optional('shrink_suffix', default='-shrink'): Any(None, str)} """ return {Optional('shrink_suffix', default='-shrink'): Any(None, str)} def skip_repo_fs_check(): """ :returns: {Optional('skip_repo_fs_check', default=True): Any(bool, All(Any(str), Boolean()))} """ return { Optional('skip_repo_fs_check', default=True): Any( bool, All(Any(str), Boolean()) ) } def slices(): """ :returns: {Optional('slices', default=1): Any(All(Coerce(int), Range(min=1, max=500)), None)} """ return { Optional('slices', default=1): Any( All(Coerce(int), Range(min=1, max=500)), None ) } def timeout(action): """ :returns: {Optional('timeout', default=defval): Any(Coerce(int), None)} """ # if action == 'reindex': defval = 60 return {Optional('timeout', default=defval): Any(Coerce(int), None)} def timeout_override(action): """ :returns: {Optional('timeout_override', default=defval): Any(Coerce(int), None)} where ``defval`` is determined by the action. ``['forcemerge', 'restore', 'snapshot']`` = ``21600`` ``close`` = ``180`` ``delete_snapshots`` = ``300`` """ if action in ['forcemerge', 'restore', 'snapshot']: defval = 21600 elif action == 'close': defval = 180 elif action == 'delete_snapshots': defval = 300 else: defval = None return {Optional('timeout_override', default=defval): Any(Coerce(int), None)} def value(): """ :returns: {Required('value', default=None): Any(None, str)} """ return {Required('value', default=None): Any(None, str)} def wait_for_active_shards(action): """ :returns: {Optional('wait_for_active_shards', default=defval): Any(Coerce(int), 'all', None)} where ``defval`` defaults to 0, but changes to 1 for the ``reindex`` and ``shrink`` actions. """ defval = 0 if action in ['reindex', 'shrink']: defval = 1 return { Optional('wait_for_active_shards', default=defval): Any( Coerce(int), 'all', None ) } def wait_for_completion(action): """ :returns: {Optional('wait_for_completion', default=defval): Any(bool, All(Any(str), Boolean()))} where ``defval`` defaults to True, but changes to False if action is ``allocation``, ``cluster_routing``, or ``replicas``. """ # if action in ['cold2frozen', 'reindex', 'restore', 'snapshot']: defval = True if action in ['allocation', 'cluster_routing', 'replicas']: defval = False return { Optional('wait_for_completion', default=defval): Any( bool, All(Any(str), Boolean()) ) } def wait_for_rebalance(): """ :returns: {Optional('wait_for_rebalance', default=True): Any(bool, All(Any(str), Boolean()))} """ return { Optional('wait_for_rebalance', default=True): Any( bool, All(Any(str), Boolean()) ) } def wait_interval(action): """ :returns: {Optional('wait_interval', default=defval): Any(All(Coerce(int), Range(min=minval, max=maxval)), None)} where ``minval`` = ``1``, ``maxval`` = ``30``, and ``defval`` is ``3``, unless the action is one of ``['restore', 'snapshot', 'reindex', 'shrink']``, and then ``defval`` is ``9``. """ minval = 1 maxval = 30 # if action in ['allocation', 'cluster_routing', 'replicas']: defval = 3 if action in ['restore', 'snapshot', 'reindex', 'shrink']: defval = 9 return { Optional('wait_interval', default=defval): Any( All(Coerce(int), Range(min=minval, max=maxval)), None ) } def warn_if_no_indices(): """ :returns: {Optional('warn_if_no_indices', default=False): Any(bool, All(Any(str), Boolean()))} """ return { Optional('warn_if_no_indices', default=False): Any( bool, All(Any(str), Boolean()) ) } elasticsearch-curator-8.0.21/curator/defaults/settings.py000066400000000000000000000160651477314666200236210ustar00rootroot00000000000000"""Utilities/Helpers for defaults and schemas""" from os import path from voluptuous import Any, Boolean, Coerce, Optional from curator.exceptions import CuratorException # pylint: disable=E1120 CURATOR_DOCS = 'https://www.elastic.co/guide/en/elasticsearch/client/curator' CLICK_DRYRUN = { 'dry-run': {'help': 'Do not perform any changes.', 'is_flag': True}, } DATA_NODE_ROLES = ['data', 'data_content', 'data_hot', 'data_warm'] EXCLUDE_SYSTEM = ( '-.kibana*,-.security*,-.watch*,-.triggered_watch*,' '-.ml*,-.geoip_databases*,-.logstash*,-.tasks*' ) VERSION_MIN = (7, 14, 0) VERSION_MAX = (8, 99, 99) # Click specifics def footer(version, tail='index.html'): """ Generate a footer linking to Curator docs based on Curator version :param version: The Curator version :type version: str :returns: An epilog/footer suitable for Click """ if not isinstance(version, str): raise CuratorException('Parameter version is not a string: {type(version)}') majmin = '' try: ver = version.split('.') majmin = f'{ver[0]}.{ver[1]}' except Exception as exc: msg = f'Could not determine Curator version from provided value: {version}' raise CuratorException(msg) from exc return f'Learn more at {CURATOR_DOCS}/{majmin}/{tail}' # Default Config file location def default_config_file(): """ :returns: The default configuration file location: path.join(path.expanduser('~'), '.curator', 'curator.yml') """ default = path.join(path.expanduser('~'), '.curator', 'curator.yml') if path.isfile(default): return default # Default filter patterns (regular expressions) def regex_map(): """ :returns: A dictionary of pattern filter 'kind's with their associated regular expression: {'timestring': r'^.*{0}.*$', 'regex': r'{0}', 'prefix': r'^{0}.*$', 'suffix': r'^.*{0}$'} """ return { 'timestring': r'^.*{0}.*$', 'regex': r'{0}', 'prefix': r'^{0}.*$', 'suffix': r'^.*{0}$', } def date_regex(): """ :returns: A dictionary/map of the strftime string characters and their string lengths: {'Y':'4', 'G':'4', 'y':'2', 'm':'2', 'W':'2', 'V':'2', 'U':'2', 'd':'2', 'H':'2', 'M':'2', 'S':'2', 'j':'3'} """ return { 'Y': '4', 'G': '4', 'y': '2', 'm': '2', 'W': '2', 'V': '2', 'U': '2', 'd': '2', 'H': '2', 'M': '2', 'S': '2', 'j': '3', } # Actions def cluster_actions(): """ :returns: A list of supported cluster actions (right now, that's only ['cluster_routing']) """ return ['cluster_routing'] def index_actions(): """ :returns: The list of supported index actions: [ 'alias', 'allocation', 'close', 'create_index', 'delete_indices', 'forcemerge', 'index_settings', 'open', 'reindex', 'replicas', 'rollover', 'shrink', 'snapshot'] """ return [ 'alias', 'allocation', 'close', 'cold2frozen', 'create_index', 'delete_indices', 'forcemerge', 'index_settings', 'open', 'reindex', 'replicas', 'rollover', 'shrink', 'snapshot', ] def snapshot_actions(): """ :returns: The list of supported snapshot actions: ['delete_snapshots', 'restore'] """ return ['delete_snapshots', 'restore'] def all_actions(): """ :returns: A sorted list of all supported actions: cluster, index, and snapshot """ return sorted(cluster_actions() + index_actions() + snapshot_actions()) def index_filtertypes(): """ :returns: The list of supported index filter types: ['alias', 'allocated', 'age', 'closed', 'count', 'empty', 'forcemerged', 'ilm', 'kibana', 'none', 'opened', 'pattern', 'period', 'space', 'shards', 'size'] """ return [ 'alias', 'allocated', 'age', 'closed', 'count', 'empty', 'forcemerged', 'ilm', 'kibana', 'none', 'opened', 'pattern', 'period', 'space', 'shards', 'size', ] def snapshot_filtertypes(): """ :returns: The list of supported snapshot filter types: ['age', 'count', 'none', 'pattern', 'period', 'state'] """ return ['age', 'count', 'none', 'pattern', 'period', 'state'] def all_filtertypes(): """ :returns: A sorted list of all supported filter types (both snapshot and index) """ return sorted(list(set(index_filtertypes() + snapshot_filtertypes()))) def default_options(): """ :returns: The default values for these options: {'allow_ilm_indices': False, 'continue_if_exception': False, 'disable_action': False, 'ignore_empty_list': False, 'include_hidden': False, 'timeout_override': None} """ return { 'allow_ilm_indices': False, 'continue_if_exception': False, 'disable_action': False, 'ignore_empty_list': False, 'include_hidden': False, 'timeout_override': None, } def default_filters(): """ If no filters are set, add a 'none' filter :returns: {'filters': [{'filtertype': 'none'}]} """ return {'filters': [{'filtertype': 'none'}]} def structural_filter_elements(): """ :returns: Barebones schemas for initial validation of filters """ return { Optional('aliases'): Any(list, str), Optional('allocation_type'): Any(str), Optional('count'): Coerce(int), Optional('date_from'): Any(None, str), Optional('date_from_format'): Any(None, str), Optional('date_to'): Any(None, str), Optional('date_to_format'): Any(None, str), Optional('direction'): Any(str), Optional('disk_space'): float, Optional('epoch'): Any(Coerce(int), None), Optional('exclude'): Any(None, bool, int, str), Optional('field'): Any(None, str), Optional('intersect'): Any(None, bool, int, str), Optional('key'): Any(str), Optional('kind'): Any(str), Optional('max_num_segments'): Coerce(int), Optional('number_of_shards'): Coerce(int), Optional('pattern'): Any(str), Optional('period_type'): Any(str), Optional('reverse'): Any(None, bool, int, str), Optional('range_from'): Coerce(int), Optional('range_to'): Coerce(int), Optional('shard_filter_behavior'): Any(str), Optional('size_behavior'): Any(str), Optional('size_threshold'): Any(Coerce(float)), Optional('source'): Any(str), Optional('state'): Any(str), Optional('stats_result'): Any(None, str), Optional('timestring'): Any(None, str), Optional('threshold_behavior'): Any(str), Optional('unit'): Any(str), Optional('unit_count'): Coerce(int), Optional('unit_count_pattern'): Any(str), Optional('use_age'): Boolean(), Optional('value'): Any(int, float, bool, str), Optional('week_starts_on'): Any(None, str), } elasticsearch-curator-8.0.21/curator/exceptions.py000066400000000000000000000042411477314666200223240ustar00rootroot00000000000000"""Curator Exceptions""" class CuratorException(Exception): """ Base class for all exceptions raised by Curator which are not Elasticsearch exceptions. """ class ConfigurationError(CuratorException): """ Exception raised when a misconfiguration is detected """ class MissingArgument(CuratorException): """ Exception raised when a needed argument is not passed. """ class NoIndices(CuratorException): """ Exception raised when an operation is attempted against an empty index_list """ class NoSnapshots(CuratorException): """ Exception raised when an operation is attempted against an empty snapshot_list """ class ActionError(CuratorException): """ Exception raised when an action (against an index_list or snapshot_list) cannot be taken. """ class FailedExecution(CuratorException): """ Exception raised when an action fails to execute for some reason. """ class SnapshotInProgress(ActionError): """ Exception raised when a snapshot is already in progress """ class ActionTimeout(CuratorException): """ Exception raised when an action fails to complete in the allotted time """ class FailedSnapshot(CuratorException): """ Exception raised when a snapshot does not complete with state SUCCESS """ class FailedRestore(CuratorException): """ Exception raised when a Snapshot Restore does not restore all selected indices """ class FailedReindex(CuratorException): """ Exception raised when failures are found in the reindex task response """ class ClientException(CuratorException): """ Exception raised when the Elasticsearch client and/or connection is the source of the problem. """ class LoggingException(CuratorException): """ Exception raised when Curator cannot either log or configure logging """ class RepositoryException(CuratorException): """ Exception raised when Curator cannot verify a snapshot repository """ class SearchableSnapshotException(CuratorException): """ Exception raised when Curator finds something out of order with a Searchable Snapshot """ elasticsearch-curator-8.0.21/curator/helpers/000077500000000000000000000000001477314666200212325ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/helpers/__init__.py000066400000000000000000000003321477314666200233410ustar00rootroot00000000000000"""Curator Helper Modules""" from curator.helpers.date_ops import * from curator.helpers.getters import * from curator.helpers.testers import * from curator.helpers.utils import * from curator.helpers.waiters import * elasticsearch-curator-8.0.21/curator/helpers/date_ops.py000066400000000000000000000613301477314666200234050ustar00rootroot00000000000000"""Curator date and time functions""" import logging import random import re import string import time from datetime import timedelta, datetime, timezone from elasticsearch8.exceptions import NotFoundError from curator.exceptions import ConfigurationError from curator.defaults.settings import date_regex class TimestringSearch: """ An object to allow repetitive search against a string, ``searchme``, without having to repeatedly recreate the regex. :param timestring: An ``strftime`` pattern :type timestring: :py:func:`~.time.strftime` """ def __init__(self, timestring): # pylint: disable=consider-using-f-string regex = r'(?P{0})'.format(get_date_regex(timestring)) #: Object attribute. ``re.compile(regex)`` where #: ``regex = r'(?P{0})'.format(get_date_regex(timestring))``. Uses #: :py:func:`get_date_regex` self.pattern = re.compile(regex) #: Object attribute preserving param ``timestring`` self.timestring = timestring def get_epoch(self, searchme): """ :param searchme: A string to be matched against :py:attr:`pattern` that matches :py:attr:`timestring` :returns: The epoch timestamp extracted from ``searchme`` by regex matching against :py:attr:`pattern` :rtype: int or None """ match = self.pattern.search(searchme) if match: if match.group("date"): timestamp = match.group("date") return datetime_to_epoch(get_datetime(timestamp, self.timestring)) return None return None def absolute_date_range( unit, date_from, date_to, date_from_format=None, date_to_format=None ): """ This function calculates a date range with an absolute time stamp for both the start time and the end time. These dates are converted to epoch time. The parameter ``unit`` is used when the same simplified date is used for both ``date_from`` and ``date_to`` to calculate the duration. For example, if ``unit`` is ``months``, and ``date_from`` and ``date_to`` are both ``2017.01``, then the entire month of January 2017 will be the absolute date range. :param unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param date_from: The simplified date for the start of the range :param date_to: The simplified date for the end of the range. :param date_from_format: The :py:func:`~.time.strftime` string used to parse ``date_from`` :param date_to_format: The :py:func:`~.time.strftime` string used to parse ``date_to`` :type unit: str :type date_from: str :type date_to: str :type date_from_format: str :type date_to_format: str :returns: The epoch start time and end time of a date range :rtype: tuple """ logger = logging.getLogger(__name__) acceptable_units = [ 'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years', ] if unit not in acceptable_units: raise ConfigurationError(f'"unit" must be one of: {acceptable_units}') if not date_from_format or not date_to_format: raise ConfigurationError('Must provide "date_from_format" and "date_to_format"') try: start_epoch = datetime_to_epoch(get_datetime(date_from, date_from_format)) logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch)) except Exception as err: raise ConfigurationError( f'Unable to parse "date_from" {date_from} and "date_from_format" ' f'{date_from_format}. Error: {err}' ) from err try: end_date = get_datetime(date_to, date_to_format) except Exception as err: raise ConfigurationError( f'Unable to parse "date_to" {date_to} and "date_to_format" ' f'{date_to_format}. Error: {err}' ) from err # We have to iterate to one more month, and then subtract a second to get # the last day of the correct month if unit == 'months': month = end_date.month year = end_date.year if month == 12: year += 1 month = 1 else: month += 1 new_end_date = datetime(year, month, 1, 0, 0, 0) end_epoch = datetime_to_epoch(new_end_date) - 1 # Similarly, with years, we need to get the last moment of the year elif unit == 'years': new_end_date = datetime(end_date.year + 1, 1, 1, 0, 0, 0) end_epoch = datetime_to_epoch(new_end_date) - 1 # It's not months or years, which have inconsistent reckoning... else: # This lets us use an existing method to simply add 1 more unit's worth # of seconds to get hours, days, or weeks, as they don't change # We use -1 as point of reference normally subtracts from the epoch # and we need to add to it, so we'll make it subtract a negative value. # Then, as before, subtract 1 to get the end of the period end_epoch = ( get_point_of_reference(unit, -1, epoch=datetime_to_epoch(end_date)) - 1 ) logger.debug('End ISO8601 = %s', epoch2iso(end_epoch)) return (start_epoch, end_epoch) def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'): """ This function calculates a date range with a distinct epoch time stamp of both the start time and the end time in counts of ``unit`` relative to the time at execution, if ``epoch`` is not set. If ``unit`` is ``weeks``, you can also determine when a week begins using ``week_starts_on``, which can be either ``sunday`` or ``monday``. :param unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param range_from: Count of ``unit`` in the past/future is the origin? :param range_to: Count of ``unit`` in the past/future is the end point? :param epoch: An epoch timestamp used to establish a point of reference for calculations. :param week_starts_on: Either ``sunday`` or ``monday``. Default is ``sunday`` :type unit: str :type range_from: int :type range_to: int :type epoch: int :type week_starts_on: str :returns: The epoch start time and end time of a date range :rtype: tuple """ logger = logging.getLogger(__name__) start_date = None start_delta = None acceptable_units = ['hours', 'days', 'weeks', 'months', 'years'] if unit not in acceptable_units: raise ConfigurationError(f'"unit" must be one of: {acceptable_units}') if not range_to >= range_from: raise ConfigurationError( '"range_to" must be greater than or equal to "range_from"' ) if not epoch: epoch = time.time() epoch = fix_epoch(epoch) raw_point_of_ref = datetime.fromtimestamp(epoch, timezone.utc) logger.debug('Raw point of Reference = %s', raw_point_of_ref) # Reverse the polarity, because -1 as last week makes sense when read by # humans, but datetime timedelta math makes -1 in the future. origin = range_from * -1 # These if statements help get the start date or start_delta if unit == 'hours': point_of_ref = datetime( raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, raw_point_of_ref.hour, 0, 0, ) start_delta = timedelta(hours=origin) if unit == 'days': point_of_ref = datetime( raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, 0, 0, 0 ) start_delta = timedelta(days=origin) if unit == 'weeks': point_of_ref = datetime( raw_point_of_ref.year, raw_point_of_ref.month, raw_point_of_ref.day, 0, 0, 0 ) sunday = False if week_starts_on.lower() == 'sunday': sunday = True weekday = point_of_ref.weekday() # Compensate for ISO week starting on Monday by default if sunday: weekday += 1 logger.debug('Weekday = %s', weekday) start_delta = timedelta(days=weekday, weeks=origin) if unit == 'months': point_of_ref = datetime( raw_point_of_ref.year, raw_point_of_ref.month, 1, 0, 0, 0 ) year = raw_point_of_ref.year month = raw_point_of_ref.month if origin > 0: for _ in range(0, origin): if month == 1: year -= 1 month = 12 else: month -= 1 else: for _ in range(origin, 0): if month == 12: year += 1 month = 1 else: month += 1 start_date = datetime(year, month, 1, 0, 0, 0) if unit == 'years': point_of_ref = datetime(raw_point_of_ref.year, 1, 1, 0, 0, 0) start_date = datetime(raw_point_of_ref.year - origin, 1, 1, 0, 0, 0) if unit not in ['months', 'years']: start_date = point_of_ref - start_delta # By this point, we know our start date and can convert it to epoch time start_epoch = datetime_to_epoch(start_date) logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch)) # This is the number of units we need to consider. count = (range_to - range_from) + 1 # We have to iterate to one more month, and then subtract a second to get # the last day of the correct month if unit == 'months': month = start_date.month year = start_date.year for _ in range(0, count): if month == 12: year += 1 month = 1 else: month += 1 end_date = datetime(year, month, 1, 0, 0, 0) end_epoch = datetime_to_epoch(end_date) - 1 # Similarly, with years, we need to get the last moment of the year elif unit == 'years': end_date = datetime((raw_point_of_ref.year - origin) + count, 1, 1, 0, 0, 0) end_epoch = datetime_to_epoch(end_date) - 1 # It's not months or years, which have inconsistent reckoning... else: # This lets us use an existing method to simply add unit * count seconds # to get hours, days, or weeks, as they don't change end_epoch = get_point_of_reference(unit, count * -1, epoch=start_epoch) - 1 logger.debug('End ISO8601 = %s', epoch2iso(end_epoch)) return (start_epoch, end_epoch) def datetime_to_epoch(mydate): """ Converts datetime into epoch seconds :param mydate: A Python datetime :type mydate: :py:class:`~.datetime.datetime` :returns: An epoch timestamp based on ``mydate`` :rtype: int """ tdelta = mydate - datetime(1970, 1, 1) return tdelta.seconds + tdelta.days * 24 * 3600 def epoch2iso(epoch: int) -> str: """ Return an ISO8601 value for epoch :param epoch: An epoch timestamp :type epoch: int :returns: An ISO8601 timestamp :rtype: str """ # Because Python 3.12 now requires non-naive timezone declarations, we must change. # # ## Example: # ## epoch == 1491256800 # ## # ## The old way: # ##datetime.utcfromtimestamp(epoch) # ## datetime.datetime(2017, 4, 3, 22, 0).isoformat() # ## Result: 2017-04-03T22:00:00 # ## # ## The new way: # ## datetime.fromtimestamp(epoch, timezone.utc) # ## datetime.datetime( # ## 2017, 4, 3, 22, 0, tzinfo=datetime.timezone.utc).isoformat() # ## Result: 2017-04-03T22:00:00+00:00 # ## # ## End Example # # Note that the +00:00 is appended now where we affirmatively declare the UTC # timezone # # As a result, we will use this function to prune away the timezone if it is +00:00 # and replace it with Z, which is shorter Zulu notation for UTC (which # Elasticsearch uses) # # We are MANUALLY, FORCEFULLY declaring timezone.utc, so it should ALWAYS be # +00:00, but could in theory sometime show up as a Z, so we test for that. parts = datetime.fromtimestamp(epoch, timezone.utc).isoformat().split('+') if len(parts) == 1: if parts[0][-1] == 'Z': return parts[0] # Our ISO8601 already ends with a Z for Zulu/UTC time return f'{parts[0]}Z' # It doesn't end with a Z so we put one there if parts[1] == '00:00': return f'{parts[0]}Z' # It doesn't end with a Z so we put one there return f'{parts[0]}+{parts[1]}' # Fallback publishes the +TZ, whatever that was def fix_epoch(epoch): """ Fix value of ``epoch`` to be the count since the epoch in seconds only, which should be 10 or fewer digits long. :param epoch: An epoch timestamp, in epoch + milliseconds, or microsecond, or even nanoseconds. :type epoch: int :returns: An epoch timestamp in seconds only, based on ``epoch`` :rtype: int """ try: # No decimals allowed epoch = int(epoch) except Exception as err: raise ValueError( f'Bad epoch value. Unable to convert {epoch} to int. {err}' ) from err # If we're still using this script past January, 2038, we have bigger # problems than my hacky math here... if len(str(epoch)) <= 10: # Epoch is fine, no changes pass elif len(str(epoch)) > 10 and len(str(epoch)) <= 13: epoch = int(epoch / 1000) else: orders_of_magnitude = len(str(epoch)) - 10 powers_of_ten = 10**orders_of_magnitude epoch = int(epoch / powers_of_ten) return epoch def get_date_regex(timestring): """ :param timestring: An ``strftime`` pattern :type timestring: :py:func:`~.time.strftime` :returns: A regex string based on a provided :py:func:`~.time.strftime` ``timestring``. :rtype: str """ logger = logging.getLogger(__name__) prev, regex = ('', '') logger.debug('Provided timestring = "%s"', timestring) for idx, char in enumerate(timestring): logger.debug('Current character: %s Array position: %s', char, idx) if char == '%': pass elif char in date_regex() and prev == '%': regex += r'\d{' + date_regex()[char] + '}' elif char in ['.', '-']: regex += "\\" + char else: regex += char prev = char logger.debug('regex = %s', regex) return regex def get_datemath(client, datemath, random_element=None): """ :param client: A client connection object :param datemath: An elasticsearch datemath string :param random_element: This allows external randomization of the name and is only useful for tests so that you can guarantee the output because you provided the random portion. :type client: :py:class:`~.elasticsearch.Elasticsearch` :type datemath: :py:class:`~.datemath.datemath` :type random_element: str :returns: the parsed index name from ``datemath`` :rtype: str """ logger = logging.getLogger(__name__) if random_element is None: random_prefix = 'curator_get_datemath_function_' + ''.join( random.choice(string.ascii_lowercase) for _ in range(32) ) else: random_prefix = 'curator_get_datemath_function_' + random_element datemath_dummy = f'<{random_prefix}-{datemath}>' # We both want and expect a 404 here (NotFoundError), since we have # created a 32 character random string to definitely be an unknown # index name. logger.debug('Random datemath string for extraction: %s', datemath_dummy) faux_index = '' try: client.indices.get(index=datemath_dummy) except NotFoundError as err: # This is the magic. Elasticsearch still gave us the formatted # index name in the error results. faux_index = err.body['error']['index'] logger.debug('Response index name for extraction: %s', faux_index) # Now we strip the random index prefix back out again # pylint: disable=consider-using-f-string pattern = r'^{0}-(.*)$'.format(random_prefix) regex = re.compile(pattern) try: # And return only the now-parsed date string return regex.match(faux_index).group(1) except AttributeError as exc: raise ConfigurationError( f'The rendered index "{faux_index}" does not contain a valid date pattern ' f'or has invalid index name characters.' ) from exc def get_datetime(index_timestamp, timestring): """ :param index_timestamp: The index timestamp :param timestring: An ``strftime`` pattern :type index_timestamp: str :type timestring: :py:func:`~.time.strftime` :returns: The datetime extracted from the index name, which is the index creation time. :rtype: :py:class:`~.datetime.datetime` """ # Compensate for week of year by appending '%w' to the timestring # and '1' (Monday) to index_timestamp iso_week_number = False if '%W' in timestring or '%U' in timestring or '%V' in timestring: timestring += '%w' index_timestamp += '1' if '%V' in timestring and '%G' in timestring: iso_week_number = True # Fake as so we read Greg format instead. We will process it later timestring = timestring.replace("%G", "%Y").replace("%V", "%W") elif '%m' in timestring: if '%d' not in timestring: timestring += '%d' index_timestamp += '1' mydate = datetime.strptime(index_timestamp, timestring) # Handle ISO time string if iso_week_number: mydate = handle_iso_week_number(mydate, timestring, index_timestamp) return mydate def get_point_of_reference(unit, count, epoch=None): """ :param unit: One of ``seconds``, ``minutes``, ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param unit_count: The number of ``units``. ``unit_count`` * ``unit`` will be calculated out to the relative number of seconds. :param epoch: An epoch timestamp used in conjunction with ``unit`` and ``unit_count`` to establish a point of reference for calculations. :type unit: str :type unit_count: int :type epoch: int :returns: A point-of-reference timestamp in epoch + milliseconds by deriving from a ``unit`` and a ``count``, and an optional reference timestamp, ``epoch`` :rtype: int """ if unit == 'seconds': multiplier = 1 elif unit == 'minutes': multiplier = 60 elif unit == 'hours': multiplier = 3600 elif unit == 'days': multiplier = 3600 * 24 elif unit == 'weeks': multiplier = 3600 * 24 * 7 elif unit == 'months': multiplier = 3600 * 24 * 30 elif unit == 'years': multiplier = 3600 * 24 * 365 else: raise ValueError(f'Invalid unit: {unit}.') # Use this moment as a reference point, if one is not provided. if not epoch: epoch = time.time() epoch = fix_epoch(epoch) return epoch - multiplier * count def get_unit_count_from_name(index_name, pattern): """ :param index_name: An index name :param pattern: A regular expression pattern :type index_name: str :type pattern: str :returns: The unit count, if a match is able to be found in the name :rtype: int """ if pattern is None: return None match = pattern.search(index_name) if match: try: return int(match.group(1)) # pylint: disable=broad-except except Exception: return None else: return None def handle_iso_week_number(mydate, timestring, index_timestamp): """ :param mydate: A Python datetime :param timestring: An ``strftime`` pattern :param index_timestamp: The index timestamp :type mydate: :py:class:`~.datetime.datetime` :type timestring: :py:func:`~.time.strftime` :type index_timestamp: str :returns: The date of the previous week based on ISO week number :rtype: :py:class:`~.datetime.datetime` """ date_iso = mydate.isocalendar() # iso_week_str = "{Y:04d}{W:02d}".format(Y=date_iso[0], W=date_iso[1]) iso_week_str = f'{date_iso[0]:04d}{date_iso[1]:02d}' greg_week_str = datetime.strftime(mydate, "%Y%W") # Edge case 1: ISO week number is bigger than Greg week number. # Ex: year 2014, all ISO week numbers were 1 more than in Greg. if ( iso_week_str > greg_week_str or # Edge case 2: 2010-01-01 in ISO: 2009.W53, in Greg: 2010.W00 # For Greg converting 2009.W53 gives 2010-01-04, converting back # to same timestring gives: 2010.W01. datetime.strftime(mydate, timestring) != index_timestamp ): # Remove one week in this case mydate = mydate - timedelta(days=7) return mydate def isdatemath(data): """ :param data: An expression to validate as being datemath or not :type data: str :returns: ``True`` if ``data`` is a valid datemath expression, else ``False`` :rtype: bool """ logger = logging.getLogger(__name__) initial_check = r'^(.).*(.)$' regex = re.compile(initial_check) opener = regex.match(data).group(1) closer = regex.match(data).group(2) logger.debug('opener = %s, closer = %s', opener, closer) if (opener == '<' and closer != '>') or (opener != '<' and closer == '>'): raise ConfigurationError('Incomplete datemath encapsulation in "< >"') if opener != '<' and closer != '>': return False return True def parse_date_pattern(name): """ Scan and parse ``name`` for :py:func:`~.time.strftime` strings, replacing them with the associated value when found, but otherwise returning lowercase values, as uppercase snapshot names are not allowed. It will detect if the first character is a ``<``, which would indicate ``name`` is going to be using Elasticsearch date math syntax, and skip accordingly. The :py:func:`~.time.strftime` identifiers that Curator currently recognizes as acceptable include: * ``Y``: A 4 digit year * ``y``: A 2 digit year * ``m``: The 2 digit month * ``W``: The 2 digit week of the year * ``d``: The 2 digit day of the month * ``H``: The 2 digit hour of the day, in 24 hour notation * ``M``: The 2 digit minute of the hour * ``S``: The 2 digit number of second of the minute * ``j``: The 3 digit day of the year :param name: A name, which can contain :py:func:`~.time.strftime` strings :type name: str :returns: The parsed date pattern :rtype: str """ logger = logging.getLogger(__name__) prev, rendered = ('', '') logger.debug('Provided index name: %s', name) for idx, char in enumerate(name): logger.debug('Current character in provided name: %s, position: %s', char, idx) if char == '<': logger.info('"%s" is probably using Elasticsearch date math.', name) rendered = name break if char == '%': pass elif char in date_regex() and prev == '%': rendered += str(datetime.now(timezone.utc).strftime(f'%{char}')) else: rendered += char logger.debug('Partially rendered name: %s', rendered) prev = char logger.debug('Fully rendered name: %s', rendered) return rendered def parse_datemath(client, value): """ Validate that ``value`` looks like proper datemath. If it passes this test, then try to ship it to Elasticsearch for real. It may yet fail this test, and if it does, it will raise a :py:exc:`~.curator.exceptions.ConfigurationError` exception. If it passes, return the fully parsed string. :param client: A client connection object :param value: A string to check for datemath :type client: :py:class:`~.elasticsearch.Elasticsearch` :type value: str :returns: A datemath indexname, fully rendered by Elasticsearch :rtype: str """ logger = logging.getLogger(__name__) if not isdatemath(value): return value # if we didn't return here, we can continue, no 'else' needed. logger.debug('Properly encapsulated, proceeding to next evaluation...') # Our pattern has 4 capture groups. # 1. Everything after the initial '<' up to the first '{', which we call ``prefix`` # 2. Everything between the outermost '{' and '}', which we call ``datemath`` # 3. An optional inner '{' and '}' containing a date formatter and potentially a # timezone. Not captured. # 4. Everything after the last '}' up to the closing '>' pattern = r'^<([^\{\}]*)?(\{.*(\{.*\})?\})([^\{\}]*)?>$' regex = re.compile(pattern) try: prefix = regex.match(value).group(1) or '' datemath = regex.match(value).group(2) # formatter = regex.match(value).group(3) or '' (not captured, but counted) suffix = regex.match(value).group(4) or '' except AttributeError as exc: raise ConfigurationError( f'Value "{value}" does not contain a valid datemath pattern.' ) from exc return f'{prefix}{get_datemath(client, datemath)}{suffix}' elasticsearch-curator-8.0.21/curator/helpers/getters.py000066400000000000000000000345561477314666200232760ustar00rootroot00000000000000"""Utility functions that get things""" import logging from elasticsearch8 import exceptions as es8exc from curator.defaults.settings import EXCLUDE_SYSTEM from curator.exceptions import ( ConfigurationError, CuratorException, FailedExecution, MissingArgument, ) def byte_size(num, suffix='B'): """ :param num: The number of byte :param suffix: An arbitrary suffix, like ``Bytes`` :type num: int :type suffix: str :returns: A formatted string indicating the size in bytes, with the proper unit, e.g. KB, MB, GB, TB, etc. :rtype: float """ for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1024.0: return f'{num:3.1f}{unit}{suffix}' num /= 1024.0 return f'{num:.1f}Y{suffix}' def escape_dots(stringval): """ Escape any dots (periods) in ``stringval``. Primarily used for ``filter_path`` where dots are indicators of path nesting :param stringval: A string, ostensibly an index name :type stringval: str :returns: ``stringval``, but with any periods escaped with a backslash :retval: str """ return stringval.replace('.', r'\.') def get_alias_actions(oldidx, newidx, aliases): """ :param oldidx: The old index name :param newidx: The new index name :param aliases: The aliases :type oldidx: str :type newidx: str :type aliases: dict :returns: A list of actions suitable for :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` ``actions`` kwarg. :rtype: list """ actions = [] for alias in aliases.keys(): actions.append({'remove': {'index': oldidx, 'alias': alias}}) actions.append({'add': {'index': newidx, 'alias': alias}}) return actions def get_data_tiers(client): """ Get all valid data tiers from the node roles of each node in the cluster by polling each node :param client: A client connection object :type client: :py:class:`~.elasticsearch.Elasticsearch` :returns: The available data tiers in ``tier: bool`` form. :rtype: dict """ def role_check(role, node_info): if role in node_info['roles']: return True return False info = client.nodes.info()['nodes'] retval = { 'data_hot': False, 'data_warm': False, 'data_cold': False, 'data_frozen': False, } for node in info: for role in ['data_hot', 'data_warm', 'data_cold', 'data_frozen']: # This guarantees we don't overwrite a True with a False. # We only add True values if role_check(role, info[node]): retval[role] = True return retval def get_indices(client, search_pattern='*', include_hidden=False): """ Calls :py:meth:`~.elasticsearch.client.CatClient.indices` Will use the provided ``search_pattern`` to get a list of indices from the cluster. If ``include_hidden`` is ``True``, it will include hidden indices in 'expand_wildcards'. :param client: A client connection object search_pattern: The index search pattern to use include_hidden: Include hidden indices in the list :type client: :py:class:`~.elasticsearch.Elasticsearch` search_pattern: str include_hidden: bool :returns: The matching list of indices from the cluster :rtype: list """ logger = logging.getLogger(__name__) indices = [] expand = 'open,closed,hidden' if include_hidden else 'open,closed' logger.debug('expand = %s', expand) try: # Doing this in two stages because IndexList also calls for these args, # and the unit tests need to Mock this call the same exact way. resp = client.cat.indices( index=search_pattern + ',' + EXCLUDE_SYSTEM, expand_wildcards=expand, h='index,status', format='json', ) except Exception as err: raise FailedExecution(f'Failed to get indices. Error: {err}') from err if not resp: return indices for entry in resp: indices.append(entry['index']) logger.debug('All indices: %s', indices) return indices def get_repository(client, repository=''): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` :param client: A client connection object :param repository: The Elasticsearch snapshot repository to use :type client: :py:class:`~.elasticsearch.Elasticsearch` :type repository: str :returns: Configuration information for ``repository``. :rtype: dict """ try: return client.snapshot.get_repository(name=repository) except (es8exc.TransportError, es8exc.NotFoundError) as err: msg = ( f'Unable to get repository {repository}. Error: {err} Check Elasticsearch ' f'logs for more information.' ) raise CuratorException(msg) from err def get_snapshot(client, repository=None, snapshot=''): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get` :param client: A client connection object :param repository: The Elasticsearch snapshot repository to use :param snapshot: The snapshot name, or a comma-separated list of snapshots :type client: :py:class:`~.elasticsearch.Elasticsearch` :type repository: str :type snapshot: str :returns: Information about the provided ``snapshot``, a snapshot (or a comma-separated list of snapshots). If no snapshot specified, it will collect info for all snapshots. If none exist, an empty :py:class:`dict` will be returned. :rtype: dict """ if not repository: raise MissingArgument('No value for "repository" provided') snapname = '*' if snapshot == '' else snapshot try: return client.snapshot.get(repository=repository, snapshot=snapshot) except (es8exc.TransportError, es8exc.NotFoundError) as err: msg = ( f'Unable to get information about snapshot {snapname} from repository: ' f'{repository}. Error: {err}' ) raise FailedExecution(msg) from err def get_snapshot_data(client, repository=None): """ Get all snapshots from repository and return a list. Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get` :param client: A client connection object :param repository: The Elasticsearch snapshot repository to use :type client: :py:class:`~.elasticsearch.Elasticsearch` :type repository: str :returns: The list of all snapshots from ``repository`` :rtype: list """ if not repository: raise MissingArgument('No value for "repository" provided') try: return client.snapshot.get(repository=repository, snapshot="*")['snapshots'] except (es8exc.TransportError, es8exc.NotFoundError) as err: msg = ( f'Unable to get snapshot information from repository: ' f'{repository}. Error: {err}' ) raise FailedExecution(msg) from err def get_tier_preference(client, target_tier='data_frozen'): """Do the tier preference thing in reverse order from coldest to hottest Based on the value of ``target_tier``, build out the list to use. :param client: A client connection object :param target_tier: The target data tier, e.g. data_warm. :type client: :py:class:`~.elasticsearch.Elasticsearch` :type target_tier: str :returns: A suitable tier preference string in csv format :rtype: str """ tiermap = { 'data_content': 0, 'data_hot': 1, 'data_warm': 2, 'data_cold': 3, 'data_frozen': 4, } tiers = get_data_tiers(client) test_list = [] for tier in ['data_hot', 'data_warm', 'data_cold', 'data_frozen']: if tier in tiers and tiermap[tier] <= tiermap[target_tier]: test_list.insert(0, tier) if target_tier == 'data_frozen': # We're migrating to frozen here. If a frozen tier exists, frozen searchable # snapshot mounts should only ever go to the frozen tier. if 'data_frozen' in tiers and tiers['data_frozen']: return 'data_frozen' # If there are no nodes with the 'data_frozen' role... preflist = [] for key in test_list: # This ordering ensures that colder tiers are prioritized if key in tiers and tiers[key]: preflist.append(key) # If all of these are false, then we have no data tiers and must use 'data_content' if not preflist: return 'data_content' # This will join from coldest to hottest as csv string, # e.g. 'data_cold,data_warm,data_hot' return ','.join(preflist) def get_write_index(client, alias): """ Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` :param client: A client connection object :param alias: An alias name :type client: :py:class:`~.elasticsearch.Elasticsearch` :type alias: str :returns: The the index name associated with the alias that is designated ``is_write_index`` :rtype: str """ try: response = client.indices.get_alias(index=alias) except Exception as exc: raise CuratorException(f'Alias {alias} not found') from exc # If there are more than one in the list, one needs to be the write index # otherwise the alias is a one to many, and can't do rollover. retval = None if len(list(response.keys())) > 1: for index in list(response.keys()): try: if response[index]['aliases'][alias]['is_write_index']: retval = index except KeyError as exc: raise FailedExecution( 'Invalid alias: is_write_index not found in 1 to many alias' ) from exc else: # There's only one, so this is it retval = list(response.keys())[0] return retval def index_size(client, idx, value='total'): """ Calls :py:meth:`~.elasticsearch.client.IndicesClient.stats` :param client: A client connection object :param idx: An index name :param value: One of either ``primaries`` or ``total`` :type client: :py:class:`~.elasticsearch.Elasticsearch` :type idx: str :type value: str :returns: The sum of either ``primaries`` or ``total`` shards for index ``idx`` :rtype: integer """ fpath = f'indices.{escape_dots(idx)}.{value}.store.size_in_bytes' return client.indices.stats(index=idx, filter_path=fpath)['indices'][idx][value][ 'store' ]['size_in_bytes'] def meta_getter(client, idx, get=None): """Meta Getter Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` or :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` :param client: A client connection object :param idx: An Elasticsearch index :param get: The kind of get to perform, e.g. settings or alias :type client: :py:class:`~.elasticsearch.Elasticsearch` :type idx: str :type get: str :returns: The settings from the get call to the named index :rtype: dict """ logger = logging.getLogger(__name__) acceptable = ['settings', 'alias'] if not get: raise ConfigurationError('"get" can not be a NoneType') if get not in acceptable: raise ConfigurationError(f'"get" must be one of {acceptable}') retval = {} try: if get == 'settings': retval = client.indices.get_settings(index=idx)[idx]['settings']['index'] elif get == 'alias': retval = client.indices.get_alias(index=idx)[idx]['aliases'] except es8exc.NotFoundError as missing: logger.error('Index %s was not found!', idx) raise es8exc.NotFoundError from missing except KeyError as err: logger.error('Key not found: %s', err) raise KeyError from err # pylint: disable=broad-except except Exception as exc: logger.error('Exception encountered: %s', exc) return retval def name_to_node_id(client, name): """ Calls :py:meth:`~.elasticsearch.client.NodesClient.info` :param client: A client connection object :param name: The node ``name`` :type client: :py:class:`~.elasticsearch.Elasticsearch` :type name: str :returns: The node_id of the node identified by ``name`` :rtype: str """ logger = logging.getLogger(__name__) fpath = 'nodes' info = client.nodes.info(filter_path=fpath) for node in info['nodes']: if info['nodes'][node]['name'] == name: logger.debug('Found node_id "%s" for name "%s".', node, name) return node logger.error('No node_id found matching name: "%s"', name) return None def node_id_to_name(client, node_id): """ Calls :py:meth:`~.elasticsearch.client.NodesClient.info` :param client: A client connection object :param node_id: The node ``node_id`` :type client: :py:class:`~.elasticsearch.Elasticsearch` :type node_id: str :returns: The name of the node identified by ``node_id`` :rtype: str """ logger = logging.getLogger(__name__) fpath = f'nodes.{node_id}.name' info = client.nodes.info(filter_path=fpath) name = None if node_id in info['nodes']: name = info['nodes'][node_id]['name'] else: logger.error('No node_id found matching: "%s"', node_id) logger.debug('Name associated with node_id "%s": %s', node_id, name) return name def node_roles(client, node_id): """ Calls :py:meth:`~.elasticsearch.client.NodesClient.info` :param client: A client connection object :param node_id: The node ``node_id`` :type client: :py:class:`~.elasticsearch.Elasticsearch` :type node_id: str :returns: The list of roles assigned to the node identified by ``node_id`` :rtype: list """ fpath = f'nodes.{node_id}.roles' return client.nodes.info(filter_path=fpath)['nodes'][node_id]['roles'] def single_data_path(client, node_id): """ In order for a shrink to work, it should be on a single filesystem, as shards cannot span filesystems. Calls :py:meth:`~.elasticsearch.client.NodesClient.stats` :param client: A client connection object :param node_id: The node ``node_id`` :type client: :py:class:`~.elasticsearch.Elasticsearch` :type node_id: str :returns: ``True`` if the node has a single filesystem, else ``False`` :rtype: bool """ fpath = f'nodes.{node_id}.fs.data' response = client.nodes.stats(filter_path=fpath) return len(response['nodes'][node_id]['fs']['data']) == 1 elasticsearch-curator-8.0.21/curator/helpers/testers.py000066400000000000000000000362431477314666200233050ustar00rootroot00000000000000"""Utility functions that get things""" import logging from voluptuous import Schema from elasticsearch8 import Elasticsearch from elasticsearch8.exceptions import NotFoundError from es_client.helpers.schemacheck import SchemaCheck from es_client.helpers.utils import prune_nones from curator.helpers.getters import get_repository, get_write_index from curator.exceptions import ( ConfigurationError, MissingArgument, RepositoryException, SearchableSnapshotException, ) from curator.defaults.settings import ( index_filtertypes, snapshot_actions, snapshot_filtertypes, ) from curator.validators import actions, options from curator.validators.filter_functions import validfilters from curator.helpers.utils import report_failure def has_lifecycle_name(idx_settings): """ :param idx_settings: The settings for an index being tested :type idx_settings: dict :returns: ``True`` if a lifecycle name exists in settings, else ``False`` :rtype: bool """ if 'lifecycle' in idx_settings: if 'name' in idx_settings['lifecycle']: return True return False def is_idx_partial(idx_settings): """ :param idx_settings: The settings for an index being tested :type idx_settings: dict :returns: ``True`` if store.snapshot.partial exists in settings, else ``False`` :rtype: bool """ if 'store' in idx_settings: if 'snapshot' in idx_settings['store']: if 'partial' in idx_settings['store']['snapshot']: if idx_settings['store']['snapshot']['partial']: return True # store.snapshot.partial exists but is False -- Not a frozen tier mount return False # store.snapshot exists, but partial isn't there -- # Possibly a cold tier mount return False raise SearchableSnapshotException('Index not a mounted searchable snapshot') raise SearchableSnapshotException('Index not a mounted searchable snapshot') def ilm_policy_check(client, alias): """Test if alias is associated with an ILM policy Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings` :param client: A client connection object :param alias: The alias name :type client: :py:class:`~.elasticsearch.Elasticsearch` :type alias: str :rtype: bool """ logger = logging.getLogger(__name__) # alias = action_obj.options['name'] write_index = get_write_index(client, alias) try: idx_settings = client.indices.get_settings(index=write_index) if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']: # logger.info('Alias %s is associated with ILM policy.', alias) # logger.info('Skipping action %s because allow_ilm_indices is false.', idx) return True except KeyError: logger.debug('No ILM policies associated with %s', alias) return False def repository_exists(client, repository=None): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` :param client: A client connection object :param repository: The Elasticsearch snapshot repository to use :type client: :py:class:`~.elasticsearch.Elasticsearch` :type repository: str :returns: ``True`` if ``repository`` exists, else ``False`` :rtype: bool """ logger = logging.getLogger(__name__) if not repository: raise MissingArgument('No value for "repository" provided') try: test_result = get_repository(client, repository) if repository in test_result: logger.debug("Repository %s exists.", repository) response = True else: logger.debug("Repository %s not found...", repository) response = False # pylint: disable=broad-except except Exception as err: logger.debug('Unable to find repository "%s": Error: %s', repository, err) response = False return response def rollable_alias(client, alias): """ Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias` :param client: A client connection object :param alias: An Elasticsearch alias :type client: :py:class:`~.elasticsearch.Elasticsearch` :type alias: str :returns: ``True`` or ``False`` depending on whether ``alias`` is an alias that points to an index that can be used by the ``_rollover`` API. :rtype: bool """ logger = logging.getLogger(__name__) try: response = client.indices.get_alias(name=alias) except NotFoundError: logger.error('Alias "%s" not found.', alias) return False # Response should be like: # {'there_should_be_only_one': {'aliases': {'value of "alias" here': {}}}} # where 'there_should_be_only_one' is a single index name that ends in a number, # and 'value of "alias" here' reflects the value of the passed parameter, except # where the ``is_write_index`` setting makes it possible to have more than one # index associated with a rollover index for idx in response: if 'is_write_index' in response[idx]['aliases'][alias]: if response[idx]['aliases'][alias]['is_write_index']: return True # implied ``else``: If not ``is_write_index``, it has to fit the following criteria: if len(response) > 1: logger.error( '"alias" must only reference one index, but points to %s', response ) return False index = list(response.keys())[0] rollable = False # In order for `rollable` to be True, the last 2 digits of the index # must be digits, or a hyphen followed by a digit. # NOTE: This is not a guarantee that the rest of the index name is # necessarily correctly formatted. if index[-2:][1].isdigit(): if index[-2:][0].isdigit(): rollable = True elif index[-2:][0] == '-': rollable = True return rollable def snapshot_running(client): """ Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository` Return ``True`` if a snapshot is in progress, and ``False`` if not :param client: A client connection object :type client: :py:class:`~.elasticsearch.Elasticsearch` :rtype: bool """ try: status = client.snapshot.status()['snapshots'] # pylint: disable=broad-except except Exception as exc: report_failure(exc) # We will only accept a positively identified False. Anything else is # suspect. That's why this statement, rather than just ``return status`` # pylint: disable=simplifiable-if-expression return False if not status else True def validate_actions(data): """ Validate the ``actions`` configuration dictionary, as imported from actions.yml, for example. :param data: The configuration dictionary :type data: dict :returns: The validated and sanitized configuration dictionary. :rtype: dict """ # data is the ENTIRE schema... clean_config = {} # Let's break it down into smaller chunks... # First, let's make sure it has "actions" as a key, with a subdictionary root = SchemaCheck(data, actions.root(), 'Actions File', 'root').result() # We've passed the first step. Now let's iterate over the actions... for action_id in root['actions']: # Now, let's ensure that the basic action structure is correct, with # the proper possibilities for 'action' action_dict = root['actions'][action_id] loc = f'Action ID "{action_id}"' valid_structure = SchemaCheck( action_dict, actions.structure(action_dict, loc), 'structure', loc ).result() # With the basic structure validated, now we extract the action name current_action = valid_structure['action'] # And let's update the location with the action. loc = f'Action ID "{action_id}", action "{current_action}"' clean_options = SchemaCheck( prune_nones(valid_structure['options']), options.get_schema(current_action), 'options', loc, ).result() clean_config[action_id] = { 'action': current_action, 'description': valid_structure['description'], 'options': clean_options, } if current_action == 'alias': add_remove = {} for k in ['add', 'remove']: if k in valid_structure: current_filters = SchemaCheck( valid_structure[k]['filters'], Schema(validfilters(current_action, location=loc)), f'"{k}" filters', f'{loc}, "filters"', ).result() add_remove.update( { k: { 'filters': SchemaCheck( current_filters, Schema(validfilters(current_action, location=loc)), 'filters', f'{loc}, "{k}", "filters"', ).result() } } ) # Add/Remove here clean_config[action_id].update(add_remove) elif current_action in ['cluster_routing', 'create_index', 'rollover']: # neither cluster_routing nor create_index should have filters pass else: # Filters key only appears in non-alias actions valid_filters = SchemaCheck( valid_structure['filters'], Schema(validfilters(current_action, location=loc)), 'filters', f'{loc}, "filters"', ).result() clean_filters = validate_filters(current_action, valid_filters) clean_config[action_id].update({'filters': clean_filters}) # This is a special case for remote reindex if current_action == 'reindex': # Check only if populated with something. if 'remote_filters' in valid_structure['options']: valid_filters = SchemaCheck( valid_structure['options']['remote_filters'], Schema(validfilters(current_action, location=loc)), 'filters', f'{loc}, "filters"', ).result() clean_remote_filters = validate_filters(current_action, valid_filters) clean_config[action_id]['options'].update( {'remote_filters': clean_remote_filters} ) # if we've gotten this far without any Exceptions raised, it's valid! return {'actions': clean_config} def validate_filters(action, myfilters): """ Validate that myfilters are appropriate for the action type, e.g. no index filters applied to a snapshot list. :param action: An action name :param myfilters: A list of filters to test. :type action: str :type myfilters: list :returns: Validated list of filters :rtype: list """ # Define which set of filtertypes to use for testing if action in snapshot_actions(): filtertypes = snapshot_filtertypes() else: filtertypes = index_filtertypes() for fil in myfilters: if fil['filtertype'] not in filtertypes: raise ConfigurationError( f"\"{fil['filtertype']}\" filtertype is not compatible with " f"action \"{action}\"" ) # If we get to this point, we're still valid. Return the original list return myfilters def verify_client_object(test): """ :param test: The variable or object to test :type test: :py:class:`~.elasticsearch.Elasticsearch` :returns: ``True`` if ``test`` is a proper :py:class:`~.elasticsearch.Elasticsearch` client object and raise a :py:exc:`TypeError` exception if it is not. :rtype: bool """ logger = logging.getLogger(__name__) # Ignore mock type for testing if str(type(test)) == "": pass elif not isinstance(test, Elasticsearch): msg = f'Not a valid client object. Type: {type(test)} was passed' logger.error(msg) raise TypeError(msg) def verify_index_list(test): """ :param test: The variable or object to test :type test: :py:class:`~.curator.IndexList` :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.indexlist.IndexList` object, else raise a :py:class:`TypeError` exception. :rtype: None """ # It breaks if this import isn't local to this function: # ImportError: cannot import name 'IndexList' from partially initialized module # 'curator.indexlist' (most likely due to a circular import) # pylint: disable=import-outside-toplevel from curator.indexlist import IndexList logger = logging.getLogger(__name__) if not isinstance(test, IndexList): msg = f'Not a valid IndexList object. Type: {type(test)} was passed' logger.error(msg) raise TypeError(msg) def verify_repository(client, repository=None): """ Do :py:meth:`~.elasticsearch.snapshot.verify_repository` call. If it fails, raise a :py:exc:`~.curator.exceptions.RepositoryException`. :param client: A client connection object :type client: :py:class:`~.elasticsearch.Elasticsearch` :param repository: A repository name :type client: :py:class:`~.elasticsearch.Elasticsearch` :type repository: str :rtype: None """ logger = logging.getLogger(__name__) try: nodes = client.snapshot.verify_repository(name=repository)['nodes'] logger.debug('All nodes can write to the repository') logger.debug('Nodes with verified repository access: %s', nodes) except Exception as err: try: if err.status_code == 404: msg = ( f'--- Repository "{repository}" not found. Error: ' f'{err.meta.status}, {err.error}' ) else: msg = ( f'--- Got a {err.meta.status} response from Elasticsearch. ' f'Error message: {err.error}' ) except AttributeError: msg = f'--- Error message: {err}'.format() report = f'Failed to verify all nodes have repository access: {msg}' raise RepositoryException(report) from err def verify_snapshot_list(test): """ :param test: The variable or object to test :type test: :py:class:`~.curator.SnapshotList` :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.snapshotlist.SnapshotList` object, else raise a :py:class:`TypeError` exception. :rtype: None """ # It breaks if this import isn't local to this function: # ImportError: cannot import name 'SnapshotList' from partially initialized module # 'curator.snapshotlist' (most likely due to a circular import) # pylint: disable=import-outside-toplevel from curator.snapshotlist import SnapshotList logger = logging.getLogger(__name__) if not isinstance(test, SnapshotList): msg = f'Not a valid SnapshotList object. Type: {type(test)} was passed' logger.error(msg) raise TypeError(msg) elasticsearch-curator-8.0.21/curator/helpers/utils.py000066400000000000000000000152121477314666200227450ustar00rootroot00000000000000"""Helper utilities The kind that don't fit in testers, getters, date_ops, or converters """ import re import logging from es_client.helpers.utils import ensure_list from curator.exceptions import FailedExecution logger = logging.getLogger(__name__) def chunk_index_list(indices): """ This utility chunks very large index lists into 3KB chunks. It measures the size as a csv string, then converts back into a list for the return value. :param indices: The list of indices :type indices: list :returns: A list of lists (each a piece of the original ``indices``) :rtype: list """ chunks = [] chunk = "" for index in indices: if len(chunk) < 3072: if not chunk: chunk = index else: chunk += "," + index else: chunks.append(chunk.split(',')) chunk = index chunks.append(chunk.split(',')) return chunks def report_failure(exception): """ Raise a :py:exc:`~.curator.exceptions.FailedExecution` exception and include the original error message. :param exception: The upstream exception. :type exception: :py:exc:Exception :rtype: None """ raise FailedExecution( f'Exception encountered. Rerun with loglevel DEBUG and/or check ' f'Elasticsearch logs for more information. Exception: {exception}' ) def show_dry_run(ilo, action, **kwargs): """ Log dry run output with the action which would have been executed. :param ilo: An IndexList Object :param action: The ``action`` to be performed. :param kwargs: Any other args to show in the log output :type ilo: :py:class:`~.curator.indexlist.IndexList` :type action: str :type kwargs: dict :rtype: None """ logger.info('DRY-RUN MODE. No changes will be made.') msg = ( f'(CLOSED) indices may be shown that may not be acted on by action "{action}".' ) logger.info(msg) indices = sorted(ilo.indices) for idx in indices: # Dry runs need index state, so we collect it here if it's not present. try: index_closed = ilo.index_info[idx]['state'] == 'close' except KeyError: ilo.get_index_state() index_closed = ilo.index_info[idx]['state'] == 'close' var = ' (CLOSED)' if index_closed else '' msg = f'DRY-RUN: {action}: {idx}{var} with arguments: {kwargs}' logger.info(msg) def to_csv(indices): """ :param indices: A list of indices to act on, or a single value, which could be in the format of a csv string already. :type indices: list :returns: A csv string from a list of indices, or a single value if only one value is present :rtype: str """ indices = ensure_list(indices) # in case of a single value passed if indices: return ','.join(sorted(indices)) return None def multitarget_fix(pattern: str) -> str: """ If pattern only has '-' prefixed entries (excludes) prepend a wildcard to pattern :param pattern: The Elasticsearch multi-target syntax pattern :type pattern: str :returns: The pattern, possibly with a wildcard prepended :rtype: str """ # Split pattern into elements elements = pattern.split(',') if len(elements) == 1 and elements[0] == '': return '*' # Initialize positive and negative lists positives = [] negatives = [] # Loop through elements and sort them into positive and negative lists for element in elements: if element.startswith('-'): negatives.append(element) else: positives.append(element) # If there are no positive elements, but there are negative elements, # add a wildcard to the beginning of the pattern if len(positives) == 0 and len(negatives) > 0: logger.debug( "Only negative elements in pattern. Adding '*' so there's something " " to remove" ) return '*,' + pattern # Otherwise, return the original pattern return pattern def regex_loop(matchstr: str, indices: list) -> list: """ Loop through indices, Match against matchstr, return matches :param matchstr: The Python regex pattern to match against :param indices: The list of indices to match against :type matchstr: str :type indices: list :returns: The list of matching indices :rtype: list """ retval = [] for idx in indices: if re.match(matchstr, idx): retval.append(idx) return retval def multitarget_match(pattern: str, index_list: list) -> list: """ Convert Elasticsearch multi-target syntax ``pattern`` into Python regex patterns. Match against ``index_list`` and return the list of matches while excluding any negative matches. :param pattern: The Elasticsearch multi-target syntax pattern :param index_list: The list of indices to match against :type pattern: str :type index_list: list :returns: The final resulting list of indices :rtype: list """ retval = [] excluded = [] includes = [] excludes = [] logger.debug('Multi-target syntax pattern: %s', pattern) elements = multitarget_fix(pattern).split(',') logger.debug('Individual elements of pattern: %s', elements) for element in elements: # Exclude elements are prefixed with '-' exclude = element.startswith('-') # Any index prepended with a . is probably a hidden index, but # we need to escape the . for regex to treat it as a literal matchstr = element.replace('.', '\\.') # Replace Elasticsearch wildcard * with .* for Python regex matchstr = matchstr.replace('*', '.*') # logger.debug('matchstr: %s', matchstr) # If it is not an exclude, add the output of regex_loop to `includes` if not exclude: includes += regex_loop(matchstr, index_list) # If it is an exclude pattern, add the output of regex_loop to `excludes` # Remove the `-` from the matchstr ([1:]) before sending. if exclude: excludes += regex_loop(matchstr[1:], index_list) # Create a unique list of indices to loop through for idx in list(set(includes)): # If idx is not in the unique list of excludes, add it to retval if idx not in list(set(excludes)): retval.append(idx) else: # Otherwise, add it to the excludes excluded.append(idx) # Sort the lists alphabetically retval.sort() excluded.sort() # Log the results logger.debug('Included indices: %s', retval) logger.debug('Excluded indices: %s', excluded) return retval elasticsearch-curator-8.0.21/curator/helpers/waiters.py000066400000000000000000000326711477314666200232730ustar00rootroot00000000000000"""The function that waits ...and its helpers """ import logging import warnings from time import localtime, sleep, strftime from datetime import datetime from elasticsearch8.exceptions import GeneralAvailabilityWarning from curator.exceptions import ( ActionTimeout, ConfigurationError, CuratorException, FailedReindex, MissingArgument, ) from curator.helpers.utils import chunk_index_list def health_check(client, **kwargs): """ This function calls `client.cluster.` :py:meth:`~.elasticsearch.client.ClusterClient.health` and, based on the params provided, will return ``True`` or ``False`` depending on whether that particular keyword appears in the output, and has the expected value. If multiple keys are provided, all must match for a ``True`` response. :param client: A client connection object :type client: :py:class:`~.elasticsearch.Elasticsearch` :rtype: bool """ logger = logging.getLogger(__name__) logger.debug('KWARGS= "%s"', kwargs) klist = list(kwargs.keys()) if not klist: raise MissingArgument('Must provide at least one keyword argument') hc_data = client.cluster.health() response = True for k in klist: # First, verify that all kwargs are in the list if k not in list(hc_data.keys()): raise ConfigurationError('Key "{0}" not in cluster health output') if not hc_data[k] == kwargs[k]: msg = ( f'NO MATCH: Value for key "{kwargs[k]}", ' f'health check data: {hc_data[k]}' ) logger.debug(msg) response = False else: msg = f'MATCH: Value for key "{kwargs[k]}", health check data: {hc_data[k]}' logger.debug(msg) if response: logger.info('Health Check for all provided keys passed.') return response def relocate_check(client, index): """ This function calls `client.cluster.` :py:meth:`~.elasticsearch.client.ClusterClient.state` with a given index to check if all of the shards for that index are in the ``STARTED`` state. It will return ``True`` if all primary and replica shards are in the ``STARTED`` state, and it will return ``False`` if any shard is in a different state. :param client: A client connection object :param index: The index name :type client: :py:class:`~.elasticsearch.Elasticsearch` :type index: str :rtype: bool """ logger = logging.getLogger(__name__) shard_state_data = client.cluster.state(index=index)['routing_table']['indices'][ index ]['shards'] finished_state = all( all(shard['state'] == "STARTED" for shard in shards) for shards in shard_state_data.values() ) if finished_state: logger.info('Relocate Check for index: "%s" has passed.', index) return finished_state def restore_check(client, index_list): """ This function calls `client.indices.` :py:meth:`~.elasticsearch.client.IndicesClient.recovery` with the list of indices to check for complete recovery. It will return ``True`` if recovery of those indices is complete, and ``False`` otherwise. It is designed to fail fast: if a single shard is encountered that is still recovering (not in ``DONE`` stage), it will immediately return ``False``, rather than complete iterating over the rest of the response. :param client: A client connection object :param index_list: The list of indices to verify having been restored. :param kwargs: Any additional keyword arguments to pass to the function :type client: :py:class:`~.elasticsearch.Elasticsearch` :type index_list: list :rtype: bool """ logger = logging.getLogger(__name__) response = {} for chunk in chunk_index_list(index_list): try: chunk_response = client.indices.recovery(index=chunk, human=True) except Exception as err: msg = ( f'Unable to obtain recovery information for specified indices. ' f'Error: {err}' ) raise CuratorException(msg) from err if chunk_response == {}: logger.info('_recovery returned an empty response. Trying again.') return False response.update(chunk_response) logger.info('Provided indices: %s', index_list) logger.info('Found indices: %s', list(response.keys())) # pylint: disable=consider-using-dict-items for index in response: for shard in range(0, len(response[index]['shards'])): stage = response[index]['shards'][shard]['stage'] if stage != 'DONE': logger.info('Index "%s" is still in stage "%s"', index, stage) return False # If we've gotten here, all of the indices have recovered return True def snapshot_check(client, snapshot=None, repository=None): """ This function calls `client.snapshot.` :py:meth:`~.elasticsearch.client.SnapshotClient.get` and tests to see whether the snapshot is complete, and if so, with what status. It will log errors according to the result. If the snapshot is still ``IN_PROGRESS``, it will return ``False``. ``SUCCESS`` will be an ``INFO`` level message, ``PARTIAL`` nets a ``WARNING`` message, ``FAILED`` is an ``ERROR``, message, and all others will be a ``WARNING`` level message. :param client: A client connection object :param snapshot: The snapshot name :param repository: The repository name :type client: :py:class:`~.elasticsearch.Elasticsearch` :type snapshot: str :type repository: str :rtype: bool """ logger = logging.getLogger(__name__) logger.debug('SNAPSHOT: %s', snapshot) logger.debug('REPOSITORY: %s', repository) try: result = client.snapshot.get(repository=repository, snapshot=snapshot) logger.debug('RESULT: %s', result) except Exception as err: raise CuratorException( f'Unable to obtain information for snapshot "{snapshot}" in repository ' f'"{repository}". Error: {err}' ) from err state = result['snapshots'][0]['state'] logger.debug('Snapshot state = %s', state) retval = True if state == 'IN_PROGRESS': logger.info('Snapshot %s still in progress.', snapshot) retval = False elif state == 'SUCCESS': logger.info('Snapshot %s successfully completed.', snapshot) elif state == 'PARTIAL': logger.warning('Snapshot %s completed with state PARTIAL.', snapshot) elif state == 'FAILED': logger.error('Snapshot %s completed with state FAILED.', snapshot) else: logger.warning('Snapshot %s completed with state: %s', snapshot, state) return retval def task_check(client, task_id=None): """ This function calls `client.tasks.` :py:meth:`~.elasticsearch.client.TasksClient.get` with the provided ``task_id``. If the task data contains ``'completed': True``, then it will return ``True``. If the task is not completed, it will log some information about the task and return ``False`` :param client: A client connection object :param task_id: The task id :type client: :py:class:`~.elasticsearch.Elasticsearch` :type task_id: str :rtype: bool """ logger = logging.getLogger(__name__) try: warnings.filterwarnings("ignore", category=GeneralAvailabilityWarning) task_data = client.tasks.get(task_id=task_id) except Exception as err: msg = ( f'Unable to obtain task information for task_id "{task_id}". ' f'Exception {err}' ) raise CuratorException(msg) from err task = task_data['task'] completed = task_data['completed'] if task['action'] == 'indices:data/write/reindex': logger.debug('It\'s a REINDEX TASK') logger.debug('TASK_DATA: %s', task_data) logger.debug('TASK_DATA keys: %s', list(task_data.keys())) if 'response' in task_data: response = task_data['response'] if response['failures']: msg = f'Failures found in reindex response: {response["failures"]}' raise FailedReindex(msg) running_time = 0.000000001 * task['running_time_in_nanos'] logger.debug('Running time: %s seconds', running_time) descr = task['description'] if completed: completion_time = (running_time * 1000) + task['start_time_in_millis'] time_string = strftime('%Y-%m-%dT%H:%M:%SZ', localtime(completion_time / 1000)) logger.info('Task "%s" completed at %s.', descr, time_string) retval = True else: # Log the task status here. logger.debug('Full Task Data: %s', task_data) msg = ( f'Task "{descr}" with task_id "{task_id}" has been running for ' f'{running_time} seconds' ) logger.info(msg) retval = False return retval # pylint: disable=too-many-locals, too-many-arguments def wait_for_it( client, action, task_id=None, snapshot=None, repository=None, index=None, index_list=None, wait_interval=9, max_wait=-1, **kwargs, ): """ This function becomes one place to do all ``wait_for_completion`` type behaviors :param client: A client connection object :param action: The action name that will identify how to wait :param task_id: If the action provided a task_id, this is where it must be declared. :param snapshot: The name of the snapshot. :param repository: The Elasticsearch snapshot repository to use :param wait_interval: Seconds to wait between completion checks. :param max_wait: Maximum number of seconds to ``wait_for_completion`` :param kwargs: Any additional keyword arguments to pass to the function :type client: :py:class:`~.elasticsearch.Elasticsearch` :type action: str :type task_id: str :type snapshot: str :type repository: str :type wait_interval: int :type max_wait: int :type kwargs: dict :rtype: None """ logger = logging.getLogger(__name__) action_map = { 'allocation': {'function': health_check, 'args': {'relocating_shards': 0}}, 'replicas': {'function': health_check, 'args': {'status': 'green'}}, 'cluster_routing': {'function': health_check, 'args': {'relocating_shards': 0}}, 'snapshot': { 'function': snapshot_check, 'args': {'snapshot': snapshot, 'repository': repository}, }, 'restore': { 'function': restore_check, 'args': {'index_list': index_list}, }, 'reindex': {'function': task_check, 'args': {'task_id': task_id}}, 'shrink': {'function': health_check, 'args': {'status': 'green'}}, 'relocate': {'function': relocate_check, 'args': {'index': index}}, } wait_actions = list(action_map.keys()) if action not in wait_actions: raise ConfigurationError(f'"action" must be one of {wait_actions}') if action == 'reindex' and task_id is None: raise MissingArgument(f'A task_id must accompany "action" {action}') if action == 'snapshot' and ((snapshot is None) or (repository is None)): raise MissingArgument( f'A snapshot and repository must accompany "action" {action}. snapshot: ' f'{snapshot}, repository: {repository}' ) if action == 'restore' and index_list is None: raise MissingArgument(f'An index_list must accompany "action" {action}') if action == 'reindex': try: warnings.filterwarnings("ignore", category=GeneralAvailabilityWarning) _ = client.tasks.get(task_id=task_id) except Exception as err: # This exception should only exist in API usage. It should never # occur in regular Curator usage. raise CuratorException( f'Unable to find task_id {task_id}. Exception: {err}' ) from err # Now with this mapped, we can perform the wait as indicated. start_time = datetime.now() result = False while True: elapsed = int((datetime.now() - start_time).total_seconds()) logger.debug('Elapsed time: %s seconds', elapsed) if kwargs: response = action_map[action]['function']( client, **action_map[action]['args'], **kwargs ) else: response = action_map[action]['function']( client, **action_map[action]['args'] ) logger.debug('Response: %s', response) # Success if response: logger.debug( 'Action "%s" finished executing (may or may not have been successful)', action, ) result = True break # Not success, and reached maximum wait (if defined) if (max_wait != -1) and (elapsed >= max_wait): msg = ( f'Unable to complete action "{action}" within max_wait ' f'({max_wait}) seconds.' ) logger.error(msg) break # Not success, so we wait. msg = ( f'Action "{action}" not yet complete, {elapsed} total seconds elapsed. ' f'Waiting {wait_interval} seconds before checking again.' ) logger.debug(msg) sleep(wait_interval) logger.debug('Result: %s', result) if not result: raise ActionTimeout( ( f'Action "{action}" failed to complete in the max_wait period of ' f'{max_wait} seconds' ) ) elasticsearch-curator-8.0.21/curator/indexlist.py000066400000000000000000002101671477314666200221540ustar00rootroot00000000000000"""Index List Class""" # pylint: disable=R0904,R0913,R0917 import re import itertools import logging from elasticsearch8.exceptions import NotFoundError, TransportError from es_client.helpers.schemacheck import SchemaCheck from es_client.helpers.utils import ensure_list from curator.defaults import settings from curator.exceptions import ( ActionError, ConfigurationError, MissingArgument, NoIndices, ) from curator.helpers.date_ops import ( absolute_date_range, date_range, fix_epoch, get_date_regex, get_point_of_reference, get_unit_count_from_name, TimestringSearch, ) from curator.helpers.getters import byte_size, get_indices from curator.helpers.testers import verify_client_object from curator.helpers.utils import chunk_index_list, report_failure, to_csv from curator.validators.filter_functions import filterstructure class IndexList: """IndexList class""" def __init__(self, client, search_pattern='*', include_hidden=False): verify_client_object(client) self.loggit = logging.getLogger('curator.indexlist') #: An :py:class:`~.elasticsearch.Elasticsearch` client object passed from #: param ``client`` self.client = client #: Information extracted from indices, such as segment count, age, etc. #: Populated at instance creation time by private helper methods. #: **Type:** :py:class:`dict` self.index_info = {} #: The running list of indices which will be used by one of the #: :py:mod:`~.curator.actions` classes. Populated at instance creation #: time by private helper methods. **Type:** :py:class:`list` self.indices = [] #: All indices in the cluster at instance creation time. #: **Type:** :py:class:`list` self.all_indices = [] self.__get_indices(search_pattern, include_hidden) self.age_keyfield = None def __actionable(self, idx): self.loggit.debug('Index %s is actionable and remains in the list.', idx) def __not_actionable(self, idx): self.loggit.debug('Index %s is not actionable, removing from list.', idx) self.indices.remove(idx) def __excludify(self, condition, exclude, index, msg=None): if condition is True: if exclude: text = "Removed from actionable list" self.__not_actionable(index) else: text = "Remains in actionable list" self.__actionable(index) else: if exclude: text = "Remains in actionable list" self.__actionable(index) else: text = "Removed from actionable list" self.__not_actionable(index) if msg: self.loggit.debug('%s: %s', text, msg) def __get_indices(self, pattern, include_hidden): """ Pull all indices into ``all_indices``, then populate ``indices`` and ``index_info`` """ self.loggit.debug('Getting indices matching search_pattern: "%s"', pattern) self.all_indices = get_indices( self.client, search_pattern=pattern, include_hidden=include_hidden ) self.indices = self.all_indices[:] # if self.indices: # for index in self.indices: # self.__build_index_info(index) def __build_index_info(self, index): """ Ensure that ``index`` is a key in ``index_info``. If not, create a sub-dictionary structure under that key. """ self.loggit.debug('Building preliminary index metadata for %s', index) if index not in self.index_info: self.index_info[index] = self.__zero_values() def __map_method(self, ftype): methods = { 'alias': self.filter_by_alias, 'age': self.filter_by_age, 'allocated': self.filter_allocated, 'closed': self.filter_closed, 'count': self.filter_by_count, 'empty': self.filter_empty, 'forcemerged': self.filter_forceMerged, 'ilm': self.filter_ilm, 'kibana': self.filter_kibana, 'none': self.filter_none, 'opened': self.filter_opened, 'period': self.filter_period, 'pattern': self.filter_by_regex, 'space': self.filter_by_space, 'shards': self.filter_by_shards, 'size': self.filter_by_size, } return methods[ftype] def __remove_missing(self, err): """ Remove missing index found in ``err`` from self.indices and return that name """ missing = err.info['error']['index'] self.loggit.warning('Index was initiallly present, but now is not: %s', missing) self.loggit.debug('Removing %s from active IndexList', missing) self.indices.remove(missing) return missing def __zero_values(self): """The default values for index metadata""" return { 'age': {'creation_date': 0, 'name': 0}, 'docs': 0, 'number_of_replicas': 0, 'number_of_shards': 0, 'primary_size_in_bytes': 0, 'routing': {}, 'segments': 0, 'size_in_bytes': 0, 'state': '', } def _get_indices_segments(self, data): return self.client.indices.segments(index=to_csv(data))['indices'].copy() def _get_indices_settings(self, data): return self.client.indices.get_settings(index=to_csv(data)) def _get_indices_stats(self, data): return self.client.indices.stats(index=to_csv(data), metric='store,docs')[ 'indices' ] def _bulk_queries(self, data, exec_func): slice_number = 10 query_result = {} loop_number = ( round(len(data) / slice_number) if round(len(data) / slice_number) > 0 else 1 ) self.loggit.debug("Bulk Queries - number requests created: %s", loop_number) for num in range(0, loop_number): if num == (loop_number - 1): data_sliced = data[num * slice_number :] else: data_sliced = data[num * slice_number : (num + 1) * slice_number] query_result.update(exec_func(data_sliced)) return query_result def mitigate_alias(self, index): """ Mitigate when an alias is detected instead of an index name :param index: The index name that is showing up *instead* of what was expected :type index: str :returns: No return value: :rtype: None """ self.loggit.debug('BEGIN mitigate_alias') self.loggit.debug( 'Correcting an instance where an alias name points to index "%s"', index ) data = self.client.indices.get(index=index) aliases = list(data[index]['aliases']) if aliases: for alias in aliases: if alias in self.indices: self.loggit.warning( 'Removing alias "%s" from IndexList.indices', alias ) self.indices.remove(alias) if alias in list(self.index_info): self.loggit.warning( 'Removing alias "%s" from IndexList.index_info', alias ) del self.index_info[alias] self.loggit.debug('Adding "%s" to IndexList.indices', index) self.indices.append(index) self.loggit.debug( 'Adding preliminary metadata for "%s" to IndexList.index_info', index ) self.__build_index_info(index) self.loggit.debug('END mitigate_alias') def alias_index_check(self, data): """ Check each index in data to see if it's an alias. """ # self.loggit.debug('BEGIN alias_index_check') working_list = data[:] for entry in working_list: if self.client.indices.exists_alias(name=entry): index = list(self.client.indices.get_alias(name=entry).keys())[0] self.loggit.warning( '"%s" is actually an alias for index "%s"', entry, index ) self.mitigate_alias(index) # The mitigate_alias step ensures that the class ivars are handled # properly. The following ensure that we pass back a modified list data.remove(entry) data.append(index) # self.loggit.debug('END alias_index_check') return data def indices_exist(self, data, exec_func): """Check if indices exist. If one doesn't, remove it. Loop until all exist""" self.loggit.debug('BEGIN indices_exist') checking = True working_list = {} verified_data = self.alias_index_check(data) while checking: try: working_list.update(exec_func(verified_data)) except NotFoundError as err: data.remove(self.__remove_missing(err)) continue except TransportError as err: if '413' in err.errors: msg = ( 'Huge Payload 413 Err - Trying to get information via ' 'multiple requests' ) self.loggit.debug(msg) working_list.update(self._bulk_queries(verified_data, exec_func)) checking = False # self.loggit.debug('END indices_exist') return working_list def data_getter(self, data, exec_func): """ Function that prevents unnecessary code repetition for different data getter methods """ self.loggit.debug('BEGIN data_getter') checking = True while checking: working_list = self.indices_exist(data, exec_func) if working_list: for index in list(working_list.keys()): try: sii = self.index_info[index] except KeyError: self.loggit.warning( 'Index %s was not present at IndexList initialization, ' ' and may be behind an alias', index, ) self.mitigate_alias(index) sii = self.index_info[index] working_list = {} try: working_list.update(self._bulk_queries(data, exec_func)) except NotFoundError as err: data.remove(self.__remove_missing(err)) continue yield sii, working_list[index], index checking = False # self.loggit.debug('END data_getter') def population_check(self, index, key): """Verify that key is in self.index_info[index], and that it is populated""" retval = True # self.loggit.debug('BEGIN population_check') # self.loggit.debug('population_check: %s, %s', index, key) if index not in self.index_info: # This is just in case the index was somehow not populated self.__build_index_info(index) if key not in self.index_info[index]: self.index_info[index][key] = self.__zero_values()[key] if self.index_info[index][key] == self.__zero_values()[key]: retval = False # self.loggit.debug('END population_check') return retval def needs_data(self, indices, fields): """Check for data population in self.index_info""" self.loggit.debug('Indices: %s, Fields: %s', indices, fields) needful = [] working_list = self.indices_exist(indices, self._get_indices_settings) for idx in working_list: count = 0 for field in fields: # If the return value is True for this field, it means it's populated if self.population_check(idx, field): count += 1 if count == 0: # All values are the default/zero needful.append(idx) if fields == ['state']: self.loggit.debug('Always check open/close for all passed indices') needful = list(working_list.keys()) self.loggit.debug('These indices need data in index_info: %s', needful) return needful def get_index_settings(self): """ For each index in self.indices, populate ``index_info`` with: creation_date number_of_replicas number_of_shards routing information (if present) """ self.loggit.debug('Getting index settings -- BEGIN') self.empty_list_check() fields = ['age', 'number_of_replicas', 'number_of_shards', 'routing'] for lst in chunk_index_list(self.indices): # This portion here is to ensure that we're not polling for data # unless we must needful = self.needs_data(lst, fields) if not needful: # All indices are populated with some data, so we can skip # data collection continue # Now we only need to run on the 'needful' for sii, wli, _ in self.data_getter(needful, self._get_indices_settings): sii['age']['creation_date'] = fix_epoch( wli['settings']['index']['creation_date'] ) sii['number_of_replicas'] = wli['settings']['index'][ 'number_of_replicas' ] sii['number_of_shards'] = wli['settings']['index']['number_of_shards'] if 'routing' in wli['settings']['index']: sii['routing'] = wli['settings']['index']['routing'] self.loggit.debug('Getting index settings -- END') def get_index_state(self): """ For each index in self.indices, populate ``index_info`` with: state (open or closed) from the cat API """ self.loggit.debug('Getting index state -- BEGIN') self.empty_list_check() fields = ['state'] for lst in chunk_index_list(self.indices): # This portion here is to ensure that we're not polling for data # unless we must needful = self.needs_data(lst, fields) # Checking state is _always_ needful. resp = self.client.cat.indices( index=to_csv(needful), format='json', h='index,status' ) for entry in resp: try: self.index_info[entry['index']]['state'] = entry['status'] except KeyError: self.loggit.warning( 'Index %s was not present at IndexList initialization, ' 'and may be behind an alias', entry['index'], ) self.mitigate_alias(entry['index']) self.index_info[entry['index']]['state'] = entry['status'] # self.loggit.debug('Getting index state -- END') def get_index_stats(self): """ Populate ``index_info`` with index ``size_in_bytes``, ``primary_size_in_bytes`` and doc count information for each index. """ self.loggit.debug('Getting index stats -- BEGIN') self.empty_list_check() fields = ['size_in_bytes', 'docs', 'primary_size_in_bytes'] # This ensures that the index state is populated self.get_index_state() # Don't populate working_list until after the get_index state as it # can and will remove missing indices working_list = self.working_list() for index in self.working_list(): if self.index_info[index]['state'] == 'close': working_list.remove(index) if working_list: index_lists = chunk_index_list(working_list) for lst in index_lists: # This portion here is to ensure that we're not polling for data # unless we must needful = self.needs_data(lst, fields) if not needful: # All indices are populated with some data, so we can skip # data collection continue # Now we only need to run on the 'needful' for sii, wli, index in self.data_getter( needful, self._get_indices_stats ): try: size = wli['total']['store']['size_in_bytes'] docs = wli['total']['docs']['count'] primary_size = wli['primaries']['store']['size_in_bytes'] msg = ( f'Index: {index} Size: {byte_size(size)} Docs: {docs} ' f'PrimarySize: {byte_size(primary_size)}' ) self.loggit.debug(msg) sii['size_in_bytes'] = size sii['docs'] = docs sii['primary_size_in_bytes'] = primary_size except KeyError: msg = f'Index stats missing for "{index}" -- might be closed' self.loggit.warning(msg) # self.loggit.debug('Getting index stats -- END') def get_segment_counts(self): """ Populate ``index_info`` with segment information for each index. """ self.loggit.debug('Getting index segment counts') self.empty_list_check() for lst in chunk_index_list(self.indices): for sii, wli, _ in self.data_getter(lst, self._get_indices_segments): shards = wli['shards'] segmentcount = 0 for shardnum in shards: for shard in range(0, len(shards[shardnum])): segmentcount += shards[shardnum][shard]['num_search_segments'] sii['segments'] = segmentcount def empty_list_check(self): """Raise :py:exc:`~.curator.exceptions.NoIndices` if ``indices`` is empty""" self.loggit.debug('Checking for empty list') if not self.indices: msg = 'IndexList object is empty. No indices to act on.' self.loggit.debug(msg) raise NoIndices(msg) def working_list(self): """ Return the current value of ``indices`` as copy-by-value to prevent list stomping during iterations """ # Copy by value, rather than reference to prevent list stomping during # iterations self.loggit.debug('Generating working list of indices') return self.indices[:] def _get_name_based_ages(self, timestring): """ Add indices to ``index_info`` based on the age as indicated by the index name pattern, if it matches ``timestring`` :param timestring: An :py:func:`time.strftime` pattern """ # Check for empty list before proceeding here to prevent non-iterable condition self.loggit.debug('Getting ages of indices by "name"') self.empty_list_check() tstr = TimestringSearch(timestring) for index in self.working_list(): epoch = tstr.get_epoch(index) if isinstance(epoch, int): self.index_info[index]['age']['name'] = epoch else: msg = ( f'Timestring {timestring} was not found in index {index}. ' f'Removing from actionable list' ) self.loggit.debug(msg) self.indices.remove(index) def _get_field_stats_dates(self, field='@timestamp'): """ Add indices to ``index_info`` based on the values the queries return, as determined by the min and max aggregated values of ``field`` :param field: The field with the date value. The field must be mapped in elasticsearch as a date datatype. Default: ``@timestamp`` """ self.loggit.debug('Cannot query closed indices. Omitting any closed indices.') self.filter_closed() self.loggit.debug( 'Cannot use field_stats with empty indices. Omitting any empty indices.' ) self.filter_empty() self.loggit.debug( 'Getting index date by querying indices for min & max value of %s field', field, ) self.empty_list_check() index_lists = chunk_index_list(self.indices) for lst in index_lists: for index in lst: aggs = { 'min': {'min': {'field': field}}, 'max': {'max': {'field': field}}, } response = self.client.search(index=index, size=0, aggs=aggs) self.loggit.debug('RESPONSE: %s', response) if response: try: res = response['aggregations'] self.loggit.debug('res: %s', res) data = self.index_info[index]['age'] data['min_value'] = fix_epoch(res['min']['value']) data['max_value'] = fix_epoch(res['max']['value']) self.loggit.debug('data: %s', data) except KeyError as exc: raise ActionError( f'Field "{field}" not found in index "{index}"' ) from exc def _calculate_ages( self, source=None, timestring=None, field=None, stats_result=None ): """ This method initiates index age calculation based on the given parameters. Exceptions are raised when they are improperly configured. Set instance variable ``age_keyfield`` for use later, if needed. :param source: Source of index age. Can be: ``name``, ``creation_date``, or ``field_stats`` :param timestring: An :py:func:`time.strftime` string to match the datestamp in an index name. Only used for index filtering by ``name``. :param field: A timestamp field name. Only used for ``field_stats`` based calculations. :param stats_result: Either ``min_value`` or ``max_value``. Only used in conjunction with ``source=field_stats`` to choose whether to reference the min or max result value. """ self.age_keyfield = source if source == 'name': if not timestring: raise MissingArgument( 'source "name" requires the "timestring" keyword argument' ) self._get_name_based_ages(timestring) elif source == 'creation_date': # Nothing to do here as this comes from `get_settings` in __init__ pass elif source == 'field_stats': if not field: raise MissingArgument( 'source "field_stats" requires the "field" keyword argument' ) if stats_result not in ['min_value', 'max_value']: raise ValueError(f'Invalid value for "stats_result": {stats_result}') self.age_keyfield = stats_result self._get_field_stats_dates(field=field) else: raise ValueError( f'Invalid source: {source}. Must be one of "name", "creation_date", ' f'"field_stats".' ) def _sort_by_age(self, index_list, reverse=True): """ Take a list of indices and sort them by date. By default, the youngest are first with ``reverse=True``, but the oldest can be first by setting ``reverse=False`` """ # Do the age-based sorting here. # Build an temporary dictionary with just index and age as the key and # value, respectively temp = {} for index in index_list: try: if self.index_info[index]['age'][self.age_keyfield]: temp[index] = self.index_info[index]['age'][self.age_keyfield] else: msg = ( f'No date for "{index}" in IndexList metadata. ' f'Possible timestring mismatch. Excluding index "{index}".' ) self.__excludify(True, True, index, msg) except KeyError: msg = ( f'{index} does not have key "{self.age_keyfield}" in IndexList ' f'metadata' ) self.__excludify(True, True, index, msg) # Sort alphabetically prior to age sort to keep sorting consistent temp_tuple = sorted(temp.items(), key=lambda k: k[0], reverse=reverse) # If reverse is True, this will sort so the youngest indices are first. # However, if you want oldest first, set reverse to False. # Effectively, this should set us up to act on everything older than # meets the other set criteria. It starts as a tuple, but then becomes a list. sorted_tuple = sorted(temp_tuple, key=lambda k: k[1], reverse=reverse) return [x[0] for x in sorted_tuple] def filter_by_regex(self, kind=None, value=None, exclude=False): """ Match indices by regular expression (pattern). :param kind: Can be one of: ``suffix``, ``prefix``, ``regex``, or ``timestring``. This option defines what ``kind`` of filter you will be building. :param value: Depends on ``kind``. It is the :py:func:`time.strftime` string if ``kind`` is ``timestring``. It's used to build the regular expression for other kinds. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering indices by regex') if kind not in ['regex', 'prefix', 'suffix', 'timestring']: raise ValueError(f'{kind}: Invalid value for kind') # Stop here if None or empty value, but zero is okay if value == 0: pass elif not value: raise ValueError( 'Invalid None value for "value". Cannot be "None" type, empty, or False' ) if kind == 'timestring': regex = settings.regex_map()[kind].format(get_date_regex(value)) else: regex = settings.regex_map()[kind].format(value) self.empty_list_check() pattern = re.compile(regex) for index in self.working_list(): self.loggit.debug('Filter by regex: Index: %s', index) match = pattern.search(index) if match: self.__excludify(True, exclude, index) else: self.__excludify(False, exclude, index) def filter_by_age( self, source='name', direction=None, timestring=None, unit=None, unit_count=None, field=None, stats_result='min_value', epoch=None, exclude=False, unit_count_pattern=False, ): """ Match indices by relative age calculations. :param source: Source of index age. Can be one of ``name``, ``creation_date``, or ``field_stats`` :param direction: Time to filter, either ``older`` or ``younger`` :param timestring: An :py:func:`time.strftime` string to match the datestamp in an index name. Only used for index filtering by ``name``. :param unit: One of ``seconds``, ``minutes``, ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param unit_count: The count of ``unit``. ``unit_count`` * ``unit`` will be calculated out to the relative number of seconds. :param unit_count_pattern: A regular expression whose capture group identifies the value for ``unit_count``. :param field: A timestamp field name. Only used for ``field_stats`` based calculations. :param stats_result: Either ``min_value`` or ``max_value``. Only used in conjunction with ``source=field_stats`` to choose whether to reference the minimum or maximum result value. :param epoch: An epoch timestamp used in conjunction with ``unit`` and ``unit_count`` to establish a point of reference for calculations. If not provided, the current time will be used. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude`` is `False`, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering indices by age') # Get timestamp point of reference, por por = get_point_of_reference(unit, unit_count, epoch) if not direction: raise MissingArgument('Must provide a value for "direction"') if direction not in ['older', 'younger']: raise ValueError(f'Invalid value for "direction": {direction}') # This filter requires index settings. self.get_index_settings() self._calculate_ages( source=source, timestring=timestring, field=field, stats_result=stats_result ) if unit_count_pattern: try: unit_count_matcher = re.compile(unit_count_pattern) # pylint: disable=broad-except except Exception as exc: # We got an illegal regex, so won't be able to match anything self.loggit.error( 'Regular expression failure. Will not match unit count. Error: %s', exc, ) unit_count_matcher = None for index in self.working_list(): try: remove_this_index = False age = int(self.index_info[index]['age'][self.age_keyfield]) # if age == 0: # msg = ( # f'Evaluating {index} resulted in an epoch timestamp of ' # f'0, meaning there is no associated date. Removing from ' # f'the actionable list.' # ) # self.loggit.debug(msg) # self.indices.remove(index) # continue msg = ( f'Index "{index}" age ({age}), direction: "{direction}", point of ' f'reference, ({por})' ) # Because time adds to epoch, smaller numbers are actually older # timestamps. if unit_count_pattern: msg = ( f'unit_count_pattern is set, trying to match pattern to ' f'index "{index}"' ) self.loggit.debug(msg) unit_count_from_index = get_unit_count_from_name( index, unit_count_matcher ) if unit_count_from_index: self.loggit.debug( 'Pattern matched, applying unit_count of "%s"', unit_count_from_index, ) adjustedpor = get_point_of_reference( unit, unit_count_from_index, epoch ) msg = ( f'Adjusting point of reference from {por} to {adjustedpor} ' f'based on unit_count of {unit_count_from_index} from ' f'index name' ) self.loggit.debug(msg) elif unit_count == -1: # Unable to match pattern and unit_count is -1, meaning no # fallback, so this index is removed from the list msg = ( f'Unable to match pattern and no fallback value set. ' f'Removing index "{index}" from actionable list' ) self.loggit.debug(msg) remove_this_index = True adjustedpor = por # necessary to avoid exception if the first index is excluded else: # Unable to match the pattern and unit_count is set, so # fall back to using unit_count for determining whether # to keep this index in the list self.loggit.debug( 'Unable to match pattern using fallback value of "%s"', unit_count, ) adjustedpor = por else: adjustedpor = por if direction == 'older': agetest = age < adjustedpor else: agetest = age > adjustedpor self.__excludify(agetest and not remove_this_index, exclude, index, msg) except KeyError: msg = ( f'Index "{index}" does not meet provided criteria. ' f'Removing from list.' ) self.loggit.debug(msg) self.indices.remove(index) def filter_by_space( self, disk_space=None, reverse=True, use_age=False, source='creation_date', timestring=None, field=None, stats_result='min_value', exclude=False, threshold_behavior='greater_than', ): """ Remove indices from the actionable list based on space consumed, sorted reverse-alphabetically by default. If you set ``reverse`` to ``False``, it will be sorted alphabetically. The default is usually what you will want. If only one kind of index is provided--for example, indices matching ``logstash-%Y.%m.%d`` --then reverse alphabetical sorting will mean the oldest will remain in the list, because lower numbers in the dates mean older indices. By setting ``reverse`` to ``False``, then ``index3`` will be deleted before ``index2``, which will be deleted before ``index1`` ``use_age`` allows ordering indices by age. Age is determined by the index creation date by default, but you can specify an ``source`` of ``name``, ``max_value``, or ``min_value``. The ``name`` ``source`` requires the timestring argument. ``threshold_behavior``, when set to ``greater_than`` (default), includes if it the index tests to be larger than ``disk_space``. When set to ``less_than``, it includes if the index is smaller than ``disk_space`` :param disk_space: Filter indices over *n* gigabytes :param threshold_behavior: Size to filter, either ``greater_than`` or ``less_than``. Defaults to ``greater_than`` to preserve backwards compatability. :param reverse: The filtering direction. (default: ``True``). Ignored if ``use_age`` is ``True`` :param use_age: Sort indices by age. ``source`` is required in this case. :param source: Source of index age. Can be one of ``name``, ``creation_date``, or ``field_stats``. Default: ``creation_date`` :param timestring: An :py:func:`time.strftime` string to match the datestamp in an index name. Only used if ``source=name`` is selected. :param field: A timestamp field name. Only used if ``source=field_stats`` is selected. :param stats_result: Either ``min_value`` or ``max_value``. Only used if ``source=field_stats`` is selected. It determines whether to reference the minimum or maximum value of `field` in each index. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering indices by disk space') # Ensure that disk_space is a float if not disk_space: raise MissingArgument('No value for "disk_space" provided') if threshold_behavior not in ['greater_than', 'less_than']: raise ValueError( f'Invalid value for "threshold_behavior": {threshold_behavior}' ) # This filter requires both index stats and index settings self.get_index_stats() self.get_index_settings() disk_space = float(disk_space) disk_usage = 0.0 disk_limit = disk_space * 2**30 msg = ( 'Cannot get disk usage info from closed indices. Omitting any ' 'closed indices.' ) self.loggit.debug(msg) self.filter_closed() if use_age: self._calculate_ages( source=source, timestring=timestring, field=field, stats_result=stats_result, ) # Using default value of reverse=True in self._sort_by_age() self.loggit.debug('SORTING BY AGE') sorted_indices = self._sort_by_age(self.working_list()) else: # Default to sorting by index name sorted_indices = sorted(self.working_list(), reverse=reverse) for index in sorted_indices: disk_usage += self.index_info[index]['size_in_bytes'] msg = ( f'{index}, summed disk usage is {byte_size(disk_usage)} and disk limit ' f'is {byte_size(disk_limit)}.' ) if threshold_behavior == 'greater_than': self.__excludify((disk_usage > disk_limit), exclude, index, msg) elif threshold_behavior == 'less_than': self.__excludify((disk_usage < disk_limit), exclude, index, msg) def filter_kibana(self, exclude=True): """ Match any index named ``.kibana*`` in ``indices``. Older releases addressed index names that no longer exist. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering kibana indices') self.empty_list_check() for index in self.working_list(): pattern = re.compile(r'^\.kibana.*$') if pattern.match(index): self.__excludify(True, exclude, index) else: self.__excludify(False, exclude, index) def filter_forceMerged(self, max_num_segments=None, exclude=True): """ Match any index which has ``max_num_segments`` per shard or fewer in the actionable list. :param max_num_segments: Cutoff number of segments per shard. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering forceMerged indices') if not max_num_segments: raise MissingArgument('Missing value for "max_num_segments"') self.loggit.debug( 'Cannot get segment count of closed indices. Omitting any closed indices.' ) # This filter requires the index state (open/close), and index settings. self.get_index_state() self.get_index_settings() self.filter_closed() self.get_segment_counts() for index in self.working_list(): # Do this to reduce long lines and make it more readable... shards = int(self.index_info[index]['number_of_shards']) replicas = int(self.index_info[index]['number_of_replicas']) segments = int(self.index_info[index]['segments']) msg = ( f'{index} has {shards} shard(s) + {replicas} replica(s) ' f'with a sum total of {segments} segments.' ) expected_count = (shards + (shards * replicas)) * max_num_segments self.__excludify((segments <= expected_count), exclude, index, msg) def filter_closed(self, exclude=True): """ Filter out closed indices from ``indices`` :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering closed indices') # This filter requires index state (open/close) self.get_index_state() self.empty_list_check() for index in self.working_list(): condition = self.index_info[index]['state'] == 'close' self.loggit.debug( 'Index %s state: %s', index, self.index_info[index]['state'] ) self.__excludify(condition, exclude, index) def filter_empty(self, exclude=True): """ Filter indices with a document count of zero. Indices that are closed are automatically excluded from consideration due to closed indices reporting a document count of zero. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering empty indices') # This index requires index state (open/close) and index stats self.get_index_state() self.get_index_stats() self.filter_closed() self.empty_list_check() for index in self.working_list(): condition = self.index_info[index]['docs'] == 0 self.loggit.debug( 'Index %s doc count: %s', index, self.index_info[index]['docs'] ) self.__excludify(condition, exclude, index) def filter_opened(self, exclude=True): """ Filter out opened indices from ``indices`` :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering open indices') # This filter requires index state (open/close) self.get_index_state() self.empty_list_check() for index in self.working_list(): condition = self.index_info[index]['state'] == 'open' self.loggit.debug( 'Index %s state: %s', index, self.index_info[index]['state'] ) self.__excludify(condition, exclude, index) def filter_allocated( self, key=None, value=None, allocation_type='require', exclude=True ): """ Match indices that have the routing allocation rule of ``key=value`` from ``indices`` :param key: The allocation attribute to check for :param value: The value to check for :param allocation_type: Type of allocation to apply :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is `T`rue` """ self.loggit.debug('Filtering indices with shard routing allocation rules') if not key: raise MissingArgument('No value for "key" provided') if not value: raise MissingArgument('No value for "value" provided') if allocation_type not in ['include', 'exclude', 'require']: raise ValueError(f'Invalid "allocation_type": {allocation_type}') # This filter requires index settings self.get_index_settings() self.get_index_state() self.empty_list_check() for lst in chunk_index_list(self.indices): working_list = self._get_indices_settings(lst) if working_list: for index in list(working_list.keys()): try: has_routing = ( working_list[index]['settings']['index']['routing'][ 'allocation' ][allocation_type][key] == value ) except KeyError: has_routing = False # if has_routing: msg = ( f'{index}: Routing (mis)match: ' f'index.routing.allocation.{allocation_type}.{key}={value}.' ) self.__excludify(has_routing, exclude, index, msg) def filter_none(self): """The legendary NULL filter""" self.loggit.debug('"None" filter selected. No filtering will be done.') def filter_by_alias(self, aliases=None, exclude=False): """ Match indices which are associated with the alias or list of aliases identified by ``aliases``. Indices must appear in all aliases in list ``aliases`` or a 404 error will result, leading to no indices being matched. :param aliases: A list of alias names. :type aliases: list :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering indices matching aliases: "%s"', aliases) if not aliases: raise MissingArgument('No value for "aliases" provided') aliases = ensure_list(aliases) self.empty_list_check() for lst in chunk_index_list(self.indices): try: # get_alias will either return {} or a NotFoundError. has_alias = list( self.client.indices.get_alias( index=to_csv(lst), name=to_csv(aliases) ).keys() ) self.loggit.debug('has_alias: %s', has_alias) except NotFoundError: # if we see the NotFoundError, we need to set working_list to {} has_alias = [] for index in lst: if index in has_alias: isness = 'is' condition = True else: isness = 'is not' condition = False msg = f'{index} {isness} associated with aliases: {aliases}' self.__excludify(condition, exclude, index, msg) def filter_by_count( self, count=None, reverse=True, use_age=False, pattern=None, source='creation_date', timestring=None, field=None, stats_result='min_value', exclude=True, ): """ Remove indices from the actionable list beyond the number ``count``, sorted reverse-alphabetically by default. If you set ``reverse=False``, it will be sorted alphabetically. The default is usually what you will want. If only one kind of index is provided--for example, indices matching ``logstash-%Y.%m.%d`` -- then reverse alphabetical sorting will mean the oldest will remain in the list, because lower numbers in the dates mean older indices. By setting ``reverse=False``, then ``index3`` will be deleted before ``index2``, which will be deleted before ``index1`` ``use_age`` allows ordering indices by age. Age is determined by the index creation date by default, but you can specify an ``source`` of ``name``, ``max_value``, or ``min_value``. The ``name`` `source` requires the timestring argument. :param count: Filter indices beyond ``count``. :param reverse: The filtering direction. (default: ``True``). :param use_age: Sort indices by age. ``source`` is required in this case. :param pattern: Select indices to count from a regular expression pattern. This pattern must have one and only one capture group. This can allow a single ``count`` filter instance to operate against any number of matching patterns, and keep ``count`` of each index in that group. For example, given a ``pattern`` of ``'^(.*)-\\d{6}$'``, it will match both ``rollover-000001`` and ``index-999990``, but not ``logstash-2017.10.12``. Following the same example, if my cluster also had ``rollover-000002`` through ``rollover-000010`` and ``index-888888`` through ``index-999999``, it will process both groups of indices, and include or exclude the ``count`` of each. :param source: Source of index age. Can be one of ``name``, ``creation_date``, or ``field_stats``. Default: ``creation_date`` :param timestring: An :py:func:`time.strftime` string to match the datestamp in an index name. Only used if ``source=name``. :param field: A timestamp field name. Only used if ``source=field_stats``. :param stats_result: Either ``min_value`` or ``max_value``. Only used if ``source=field_stats``. It determines whether to reference the minimum or maximum value of ``field`` in each index. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering indices by count') if not count: raise MissingArgument('No value for "count" provided') # This filter requires index state (open/close) and index settings self.get_index_state() self.get_index_settings() # Create a copy-by-value working list working_list = self.working_list() if pattern: try: regex = re.compile(pattern) if regex.groups < 1: raise ConfigurationError( f'No regular expression group found in {pattern}' ) if regex.groups > 1: raise ConfigurationError( f'More than 1 regular expression group found in {pattern}' ) # Prune indices not matching the regular expression the object # (And filtered_indices) We do not want to act on them by accident. prune_these = list( filter(lambda x: regex.match(x) is None, working_list) ) filtered_indices = working_list for index in prune_these: msg = '{index} does not match regular expression {pattern}.' condition = True exclude = True self.__excludify(condition, exclude, index, msg) # also remove it from filtered_indices filtered_indices.remove(index) # Presort these filtered_indices using the lambda presorted = sorted( filtered_indices, key=lambda x: regex.match(x).group(1) ) except Exception as exc: raise ActionError( f'Unable to process pattern: "{pattern}". Error: {exc}' ) from exc # Initialize groups here groups = [] # We have to pull keys k this way, but we don't need to keep them # We only need g for groups for _, g in itertools.groupby( presorted, key=lambda x: regex.match(x).group(1) ): groups.append(list(g)) else: # Since pattern will create a list of lists, and we iterate over that, # we need to put our single list inside a list groups = [working_list] for group in groups: if use_age: if source != 'name': self.loggit.warning( 'Cannot get age information from closed indices unless ' 'source="name". Omitting any closed indices.' ) self.filter_closed() self._calculate_ages( source=source, timestring=timestring, field=field, stats_result=stats_result, ) # Using default value of reverse=True in self._sort_by_age() sorted_indices = self._sort_by_age(group, reverse=reverse) else: # Default to sorting by index name sorted_indices = sorted(group, reverse=reverse) idx = 1 for index in sorted_indices: msg = f'{index} is {idx} of specified count of {count}.' condition = True if idx <= count else False self.__excludify(condition, exclude, index, msg) idx += 1 def filter_by_shards( self, number_of_shards=None, shard_filter_behavior='greater_than', exclude=False ): """ Match ``indices`` with a given shard count. Selects all indices with a shard count ``greater_than`` ``number_of_shards`` by default. Use ``shard_filter_behavior`` to select indices with shard count ``greater_than``, ``greater_than_or_equal``, ``less_than``, ``less_than_or_equal``, or ``equal`` to ``number_of_shards``. :param number_of_shards: shard threshold :param shard_filter_behavior: Do you want to filter on ``greater_than``, ``greater_than_or_equal``, ``less_than``, ``less_than_or_equal``, or ``equal``? :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug("Filtering indices by number of shards") if not number_of_shards: raise MissingArgument('No value for "number_of_shards" provided') if shard_filter_behavior not in [ 'greater_than', 'less_than', 'greater_than_or_equal', 'less_than_or_equal', 'equal', ]: raise ValueError( f'Invalid value for "shard_filter_behavior": {shard_filter_behavior}' ) if number_of_shards < 1 or ( shard_filter_behavior == 'less_than' and number_of_shards == 1 ): raise ValueError( f'Unacceptable value: {number_of_shards} -- "number_of_shards" cannot ' f'be less than 1. A valid index will have at least one shard.' ) # This filter requires index_settings to count shards self.get_index_settings() self.empty_list_check() for index in self.working_list(): self.loggit.debug('Filter by number of shards: Index: %s', index) if shard_filter_behavior == 'greater_than': condition = ( int(self.index_info[index]['number_of_shards']) > number_of_shards ) elif shard_filter_behavior == 'less_than': condition = ( int(self.index_info[index]['number_of_shards']) < number_of_shards ) elif shard_filter_behavior == 'greater_than_or_equal': condition = ( int(self.index_info[index]['number_of_shards']) >= number_of_shards ) elif shard_filter_behavior == 'less_than_or_equal': condition = ( int(self.index_info[index]['number_of_shards']) <= number_of_shards ) else: condition = ( int(self.index_info[index]['number_of_shards']) == number_of_shards ) self.__excludify(condition, exclude, index) def filter_period( self, period_type='relative', source='name', range_from=None, range_to=None, date_from=None, date_to=None, date_from_format=None, date_to_format=None, timestring=None, unit=None, field=None, stats_result='min_value', intersect=False, week_starts_on='sunday', epoch=None, exclude=False, ): """ Match ``indices`` with ages within a given period. :param period_type: Can be either ``absolute`` or ``relative``. Default is ``relative``. ``date_from`` and ``date_to`` are required when using ``period_type='absolute'``. ``range_from`` and ``range_to`` are required with ``period_type='relative'``. :param source: Source of index age. Can be ``name``, ``creation_date``, or ``field_stats`` :param range_from: How many ``unit`` (s) in the past/future is the origin? :param range_to: How many ``unit`` (s) in the past/future is the end point? :param date_from: The simplified date for the start of the range :param date_to: The simplified date for the end of the range. If this value is the same as ``date_from``, the full value of ``unit`` will be extrapolated for the range. For example, if ``unit=months``, and ``date_from`` and ``date_to`` are both ``2017.01``, then the entire month of January 2017 will be the absolute date range. :param date_from_format: The :py:func:`time.strftime` string used to parse ``date_from`` :param date_to_format: The :py:func:`time.strftime` string used to parse ``date_to`` :param timestring: An :py:func:`time.strftime` string to match the datestamp in an index name. Only used for index filtering by ``name``. :param unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param field: A timestamp field name. Only used for ``field_stats`` based calculations. :param stats_result: Either ``min_value`` or ``max_value``. Only used in conjunction with ``source='field_stats'`` to choose whether to reference the min or max result value. :param intersect: Only used when ``source='field_stats'``. If ``True``, only indices where both ``min_value`` and ``max_value`` are within the period will be selected. If ``False``, it will use whichever you specified. Default is ``False`` to preserve expected behavior. :param week_starts_on: Either ``sunday`` or ``monday``. Default is ``sunday`` :param epoch: An epoch timestamp used to establish a point of reference for calculations. If not provided, the current time will be used. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering indices by period') if period_type not in ['absolute', 'relative']: raise ValueError( f'Unacceptable value: {period_type} -- "period_type" must be either ' f'"absolute" or "relative".' ) if period_type == 'relative': func = date_range args = [unit, range_from, range_to, epoch] kwgs = {'week_starts_on': week_starts_on} if (not isinstance(range_from, int)) or (not isinstance(range_to, int)): raise ConfigurationError( '"range_from" and "range_to" must be integer values' ) else: func = absolute_date_range args = [unit, date_from, date_to] kwgs = { 'date_from_format': date_from_format, 'date_to_format': date_to_format, } for reqd in [date_from, date_to, date_from_format, date_to_format]: if not reqd: raise ConfigurationError( 'Must provide "date_from", "date_to", "date_from_format", and ' '"date_to_format" with absolute period_type' ) # This filter requires index settings self.get_index_settings() try: start, end = func(*args, **kwgs) # pylint: disable=broad-except except Exception as exc: report_failure(exc) self._calculate_ages( source=source, timestring=timestring, field=field, stats_result=stats_result ) for index in self.working_list(): try: if source == 'field_stats' and intersect: min_age = int(self.index_info[index]['age']['min_value']) max_age = int(self.index_info[index]['age']['max_value']) msg = ( f'Index "{index}", timestamp field "{field}", min_value ' f'({min_age}), max_value ({max_age}), period start: ' f'"{start}", period end, "{end}"' ) # Because time adds to epoch, smaller numbers are actually older # timestamps. inrange = (min_age >= start) and (max_age <= end) else: age = int(self.index_info[index]['age'][self.age_keyfield]) msg = ( f'Index "{index}" age ({age}), period start: "{start}", period ' f'end, "{end}"' ) # Because time adds to epoch, smaller numbers are actually older # timestamps. inrange = (age >= start) and (age <= end) self.__excludify(inrange, exclude, index, msg) except KeyError: self.loggit.debug( 'Index "%s" does not meet provided criteria. Removing from list.', index, ) self.indices.remove(index) def filter_ilm(self, exclude=True): """ Match indices that have the setting ``index.lifecycle.name`` :param exclude: If ``exclude=True``, this filter will remove matching ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``True`` """ self.loggit.debug('Filtering indices with index.lifecycle.name') index_lists = chunk_index_list(self.indices) if index_lists == [['']]: self.loggit.debug('Empty working list. No ILM indices to filter.') return for lst in index_lists: working_list = self._get_indices_settings(lst) if working_list: for index in list(working_list.keys()): try: subvalue = working_list[index]['settings']['index']['lifecycle'] has_ilm = 'name' in subvalue msg = f"{index} has index.lifecycle.name {subvalue['name']}" except KeyError: has_ilm = False msg = f'index.lifecycle.name is not set for index {index}' self.__excludify(has_ilm, exclude, index, msg) def iterate_filters(self, filter_dict): """ Iterate over the filters defined in ``config`` and execute them. :param filter_dict: The configuration dictionary .. note:: ``filter_dict`` should be a dictionary with the following form: .. code-block:: python { 'filters' : [ { 'filtertype': 'the_filter_type', 'key1' : 'value1', ... 'keyN' : 'valueN' } ] } """ self.loggit.debug('Iterating over a list of filters') # Make sure we actually _have_ filters to act on if 'filters' not in filter_dict or len(filter_dict['filters']) < 1: self.loggit.info('No filters in config. Returning unaltered object.') return self.loggit.debug('All filters: %s', filter_dict['filters']) for fil in filter_dict['filters']: self.loggit.debug('Top of the loop: %s', self.indices) self.loggit.debug('Un-parsed filter args: %s', fil) # Make sure we got at least this much in the configuration chk = SchemaCheck( fil, filterstructure(), 'filter', 'IndexList.iterate_filters' ).result() msg = f'Parsed filter args: {chk}' self.loggit.debug(msg) method = self.__map_method(fil['filtertype']) del fil['filtertype'] # If it's a filtertype with arguments, update the defaults with the # provided settings. if fil: self.loggit.debug('Filter args: %s', fil) self.loggit.debug('Pre-instance: %s', self.indices) method(**fil) self.loggit.debug('Post-instance: %s', self.indices) else: # Otherwise, it's a settingless filter. method() def filter_by_size( self, size_threshold=None, threshold_behavior='greater_than', exclude=False, size_behavior='primary', ): """ Remove indices from the actionable list based on index size. ``threshold_behavior``, when set to ``greater_than`` (default), includes if it the index tests to be larger than ``size_threshold``. When set to ``less_than``, it includes if the index is smaller than ``size_threshold`` :param size_threshold: Filter indices over *n* gigabytes :param threshold_behavior: Size to filter, either ``greater_than`` or ``less_than``. Defaults to ``greater_than`` to preserve backwards compatability. :param size_behavior: Size that used to filter, either ``primary`` or ``total``. Defaults to ``primary`` :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering indices by index size') # Ensure that disk_space is a float if not size_threshold: raise MissingArgument('No value for "size_threshold" provided') if size_behavior not in ['primary', 'total']: raise ValueError(f'Invalid value for "size_behavior": {size_behavior}') if threshold_behavior not in ['greater_than', 'less_than']: raise ValueError( f'Invalid value for "threshold_behavior": {threshold_behavior}' ) index_size_limit = float(size_threshold) * 2**30 msg = ( 'Cannot get disk usage info from closed indices. ' 'Omitting any closed indices.' ) self.loggit.debug(msg) # This filter requires index state (open/close) and index stats self.get_index_state() self.get_index_stats() self.filter_closed() # Create a copy-by-value working list working_list = self.working_list() for index in working_list: if size_behavior == 'primary': index_size = self.index_info[index]['primary_size_in_bytes'] else: index_size = self.index_info[index]['size_in_bytes'] msg = ( f'{index}, index size is {byte_size(index_size)} and ' f'size limit is {byte_size(index_size_limit)}.' ) if threshold_behavior == 'greater_than': self.__excludify((index_size > index_size_limit), exclude, index, msg) elif threshold_behavior == 'less_than': self.__excludify((index_size < index_size_limit), exclude, index, msg) elasticsearch-curator-8.0.21/curator/repomgrcli.py000066400000000000000000000575541477314666200223250ustar00rootroot00000000000000"""es_repo_mgr CLI""" import sys import logging import pprint import click from elasticsearch8 import ApiError, NotFoundError from es_client.defaults import LOGGING_SETTINGS, SHOW_OPTION from es_client.builder import Builder from es_client.helpers.config import ( cli_opts, context_settings, generate_configdict, get_config, ) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper from curator.defaults.settings import ( CLICK_DRYRUN, VERSION_MAX, VERSION_MIN, default_config_file, footer, ) from curator.helpers.getters import get_repository from curator._version import __version__ ONOFF = {'on': '', 'off': 'no-'} click_opt_wrap = option_wrapper() # pylint: disable=unused-argument def delete_callback(ctx, param, value): """Callback if command ``delete`` called If the action is ``delete``, this is the :py:class:`click.Parameter` callback function if you used the ``--yes`` flag. :param ctx: The Click Context :param param: The parameter name :param value: The value :type ctx: :py:class:`~.click.Context` :type param: str :type value: str """ # It complains if ``param`` isn't passed as a positional parameter, but this # is a one-trick pony callback. We don't need to know that you selected # ``--yes`` as the parameter. if not value: ctx.abort() def show_repos(client): """Show all repositories :param client: A client connection object :type client: :py:class:`~.elasticsearch.Elasticsearch` :rtype: None """ logger = logging.getLogger(__name__) for repository in sorted(get_repository(client, '*').keys()): logger.debug('REPOSITORY = %s', repository) click.echo(f'{repository}') sys.exit(0) def get_client(ctx): """ :param ctx: The :py:class:`click` Context :type ctx: :py:class:`~.click.Context` :returns: A client connection object :rtype: :py:class:`~.elasticsearch.Elasticsearch` """ builder = Builder( configdict=ctx.obj['configdict'], version_max=VERSION_MAX, version_min=VERSION_MIN, ) try: builder.connect() # pylint: disable=broad-except except Exception as exc: click.echo(f'Exception encountered: {exc}') sys.exit(1) return builder.client # pylint: disable=broad-except def create_repo(ctx, repo_name=None, repo_type=None, repo_settings=None, verify=False): """ Call :py:meth:`~.elasticsearch.client.SnapshotClient.create_repository` to create a snapshot repository from the provided arguments :param ctx: The :py:class:`click` Context :param repo_name: The repository name :param repo_type: The repository name :param repo_settings: Settings to configure the repository :param verify: Whether to verify repository access :type ctx: :py:class:`~.click.Context` :type repo_name: str :type repo_type: str :type repo_settings: dict :type verify: bool :rtype: None """ logger = logging.getLogger('curator.repomgrcli.create_repo') client = get_client(ctx) request_body = {'type': repo_type, 'settings': repo_settings} try: client.snapshot.create_repository( name=repo_name, body=request_body, verify=verify ) except ApiError as exc: if exc.meta.status >= 500: logger.critical('Server-side error!') if exc.body['error']['type'] == 'repository_exception': logger.critical('Repository exception: %s', exc.body) click.echo(f'Repository exception: {exc.body}') sys.exit(1) except Exception as exc: logger.critical('Encountered exception %s', exc) sys.exit(1) # pylint: disable=unused-argument, invalid-name, line-too-long @click.command(short_help='Azure Repository') @click.option('--name', required=True, type=str, help='Repository name') @click.option( '--client', default='default', show_default=True, type=str, help='Azure named client to use.', ) @click.option( '--container', default='elasticsearch-snapshots', show_default=True, type=str, help=( 'Container name. You must create the Azure container before creating ' 'the repository.' ), ) @click.option( '--base_path', default='', show_default=True, type=str, help=( 'Specifies the path within container to repository data. Defaults to ' 'empty (root directory).' ), ) @click.option( '--chunk_size', type=str, help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]' ) @click.option( '--compress/--no-compress', default=True, show_default=True, help='Enable/Disable metadata compression.', ) @click.option( '--max_restore_rate', type=str, default='20mb', show_default=True, help='Throttles per node restore rate (per second).', ) @click.option( '--max_snapshot_rate', type=str, default='20mb', show_default=True, help='Throttles per node snapshot rate (per second).', ) @click.option('--readonly', is_flag=True, help='If set, the repository is read-only.') @click.option( '--location_mode', default='primary_only', type=click.Choice(['primary_only', 'secondary_only']), help='Note that if you set it to secondary_only, it will force readonly to true.', ) @click.option('--verify', is_flag=True, help='Verify repository after creation.') @click.pass_context def azure( ctx, name, client, container, base_path, chunk_size, compress, max_restore_rate, max_snapshot_rate, readonly, location_mode, verify, ): """Create an Azure repository.""" azure_settings = { 'client': client, 'container': container, 'base_path': base_path, 'chunk_size': chunk_size, 'compress': compress, 'max_restore_bytes_per_sec': max_restore_rate, 'max_snapshot_bytes_per_sec': max_snapshot_rate, 'readonly': readonly, 'location_mode': location_mode, } create_repo( ctx, repo_name=name, repo_type='azure', repo_settings=azure_settings, verify=verify, ) # pylint: disable=unused-argument, invalid-name, line-too-long @click.command(short_help='Google Cloud Storage Repository') @click.option('--name', required=True, type=str, help='Repository name') @click.option( '--bucket', required=True, type=str, help='The name of the bucket to be used for snapshots.', ) @click.option( '--client', default='default', show_default=True, type=str, help='The name of the client to use to connect to Google Cloud Storage.', ) @click.option( '--base_path', default='', show_default=True, type=str, help=( 'Specifies the path within bucket to repository data. ' 'Defaults to the root of the bucket.' ), ) @click.option( '--chunk_size', type=str, help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]' ) @click.option( '--compress/--no-compress', default=True, show_default=True, help='Enable/Disable metadata compression.', ) @click.option( '--max_restore_rate', type=str, default='20mb', show_default=True, help='Throttles per node restore rate (per second).', ) @click.option( '--max_snapshot_rate', type=str, default='20mb', show_default=True, help='Throttles per node snapshot rate (per second).', ) @click.option('--readonly', is_flag=True, help='If set, the repository is read-only.') @click.option('--verify', is_flag=True, help='Verify repository after creation.') @click.pass_context def gcs( ctx, name, bucket, client, base_path, chunk_size, compress, max_restore_rate, max_snapshot_rate, readonly, verify, ): """Create a Google Cloud Storage repository.""" gcs_settings = { 'bucket': bucket, 'client': client, 'base_path': base_path, 'chunk_size': chunk_size, 'compress': compress, 'max_restore_bytes_per_sec': max_restore_rate, 'max_snapshot_bytes_per_sec': max_snapshot_rate, 'readonly': readonly, } create_repo( ctx, repo_name=name, repo_type='gcs', repo_settings=gcs_settings, verify=verify ) # pylint: disable=unused-argument, invalid-name, line-too-long @click.command(short_help='S3 Repository') @click.option('--name', required=True, type=str, help='Repository name') @click.option( '--bucket', required=True, type=str, help='The bucket name must adhere to Amazon’s S3 bucket naming rules.', ) @click.option( '--client', default='default', show_default=True, type=str, help='The name of the S3 client to use to connect to S3.', ) @click.option( '--base_path', default='', show_default=True, type=str, help=( 'Specifies the path to the repository data within its bucket. Defaults ' 'to an empty string, meaning that the repository is at the root of the ' 'bucket. The value of this setting should not start or end with a /.' ), ) @click.option( '--chunk_size', type=str, help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]' ) @click.option( '--compress/--no-compress', default=True, show_default=True, help='Enable/Disable metadata compression.', ) @click.option( '--max_restore_rate', type=str, default='20mb', show_default=True, help='Throttles per node restore rate (per second).', ) @click.option( '--max_snapshot_rate', type=str, default='20mb', show_default=True, help='Throttles per node snapshot rate (per second).', ) @click.option('--readonly', is_flag=True, help='If set, the repository is read-only.') @click.option( '--server_side_encryption', is_flag=True, help='If set, files are encrypted on server side using AES256 algorithm.', ) @click.option( '--buffer_size', default='', type=str, help=( 'Minimum threshold below which the chunk is uploaded using a single ' 'request. Must be between 5mb and 5gb.' ), ) @click.option( '--canned_acl', default='private', type=click.Choice( [ 'private', 'public-read', 'public-read-write', 'authenticated-read', 'log-delivery-write', 'bucket-owner-read', 'bucket-owner-full-control', ] ), help=( 'When the S3 repository creates buckets and objects, it adds the canned ' 'ACL into the buckets and objects.' ), ) @click.option( '--storage_class', default='standard', type=click.Choice( [ 'standard', 'reduced_redundancy', 'standard_ia', 'onezone_ia', 'intelligent_tiering', ] ), help='Sets the S3 storage class for objects stored in the snapshot repository.', ) @click.option('--verify', is_flag=True, help='Verify repository after creation.') @click.pass_context def s3( ctx, name, bucket, client, base_path, chunk_size, compress, max_restore_rate, max_snapshot_rate, readonly, server_side_encryption, buffer_size, canned_acl, storage_class, verify, ): """ Create an S3 repository. """ s3_settings = { 'bucket': bucket, 'client': client, 'base_path': base_path, 'chunk_size': chunk_size, 'compress': compress, 'max_restore_bytes_per_sec': max_restore_rate, 'max_snapshot_bytes_per_sec': max_snapshot_rate, 'readonly': readonly, 'server_side_encryption': server_side_encryption, 'buffer_size': buffer_size, 'canned_acl': canned_acl, 'storage_class': storage_class, } create_repo( ctx, repo_name=name, repo_type='s3', repo_settings=s3_settings, verify=verify ) # pylint: disable=unused-argument, invalid-name @click.command(short_help='Shared Filesystem Repository') @click.option('--name', required=True, type=str, help='Repository name') @click.option( '--location', required=True, type=str, help=( 'Shared file-system location. Must match remote path, & be accessible ' 'to all master & data nodes' ), ) @click.option( '--compress/--no-compress', default=True, show_default=True, help='Enable/Disable metadata compression.', ) @click.option( '--chunk_size', type=str, help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]' ) @click.option( '--max_snapshots', default=2147483647, type=int, help=( 'Maximum number of snapshots the repository can contain. ' 'Defaults to Integer.MAX_VALUE, which is 2147483647.' ), ) @click.option( '--max_restore_rate', type=str, default='20mb', show_default=True, help='Throttles per node restore rate (per second).', ) @click.option( '--max_snapshot_rate', type=str, default='20mb', show_default=True, help='Throttles per node snapshot rate (per second).', ) @click.option('--readonly', is_flag=True, help='If set, the repository is read-only.') @click.option('--verify', is_flag=True, help='Verify repository after creation.') @click.pass_context def fs( ctx, name, location, compress, chunk_size, max_snapshots, max_restore_rate, max_snapshot_rate, readonly, verify, ): """ Create a filesystem repository. """ fs_settings = { 'location': location, 'compress': compress, 'chunk_size': chunk_size, 'max_number_of_snapshots': max_snapshots, 'max_restore_bytes_per_sec': max_restore_rate, 'max_snapshot_bytes_per_sec': max_snapshot_rate, 'readonly': readonly, } create_repo( ctx, repo_name=name, repo_type='fs', repo_settings=fs_settings, verify=verify ) # pylint: disable=unused-argument, invalid-name @click.command(short_help='Read-only URL Repository') @click.option('--name', required=True, type=str, help='Repository name') @click.option( '--chunk_size', type=str, help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]' ) @click.option( '--http_max_retries', type=int, default=5, show_default=True, help='Maximum number of retries for http and https', ) @click.option( '--http_socket_timeout', type=str, default='50s', show_default=True, help='Maximum wait time for data transfers over a connection.', ) @click.option( '--compress/--no-compress', default=True, show_default=True, help='Enable/Disable metadata compression.', ) @click.option( '--max_snapshots', default=2147483647, type=int, help=( 'Maximum number of snapshots the repository can contain. ' 'Defaults to Integer.MAX_VALUE, which is 2147483647.' ), ) @click.option( '--max_restore_rate', type=str, default='20mb', show_default=True, help='Throttles per node restore rate (per second).', ) @click.option( '--shared_filesystem_url', required=True, type=str, help='URL location of the root of the shared filesystem repository.', ) @click.option('--verify', is_flag=True, help='Verify repository after creation.') @click.pass_context def url( ctx, name, chunk_size, http_max_retries, http_socket_timeout, compress, max_snapshots, max_restore_rate, shared_filesystem_url, verify, ): """ Create a read-only url repository. """ url_settings = { 'chunk_size': chunk_size, 'http_max_retries': http_max_retries, 'http_socket_timeout': http_socket_timeout, 'compress': compress, 'max_number_of_snapshots': max_snapshots, 'max_restore_bytes_per_sec': max_restore_rate, 'url': shared_filesystem_url, } create_repo( ctx, repo_name=name, repo_type='url', repo_settings=url_settings, verify=verify ) # pylint: disable=unused-argument, invalid-name @click.command(short_help='Source-Only Repository') @click.option('--name', required=True, type=str, help='Repository name') @click.option( '--delegate_type', required=True, type=click.Choice(['azure', 'gcs', 's3', 'fs']), help='Delegated repository type.', ) @click.option( '--location', required=True, type=str, help='For Source-Only, specify the delegated repository name here', ) @click.option( '--compress/--no-compress', default=True, show_default=True, help='Enable/Disable metadata compression.', ) @click.option( '--chunk_size', type=str, help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]' ) @click.option( '--max_snapshots', default=2147483647, type=int, help=( 'Maximum number of snapshots the repository can contain. ' 'Defaults to Integer.MAX_VALUE, which is 2147483647.' ), ) @click.option( '--max_restore_rate', type=str, default='20mb', show_default=True, help='Throttles per node restore rate (per second).', ) @click.option( '--max_snapshot_rate', type=str, default='20mb', show_default=True, help='Throttles per node snapshot rate (per second).', ) @click.option('--readonly', is_flag=True, help='If set, the repository is read-only.') @click.option('--verify', is_flag=True, help='Verify repository after creation.') @click.pass_context def source( ctx, name, delegate_type, location, compress, chunk_size, max_snapshots, max_restore_rate, max_snapshot_rate, readonly, verify, ): """ Create a filesystem repository. """ source_settings = { 'chunk_size': chunk_size, 'compress': compress, 'delegate_type': delegate_type, 'location': location, 'max_number_of_snapshots': max_snapshots, 'max_restore_bytes_per_sec': max_restore_rate, 'max_snapshot_bytes_per_sec': max_snapshot_rate, 'readonly': readonly, } create_repo( ctx, repo_name=name, repo_type='source', repo_settings=source_settings, verify=verify, ) # pylint: disable=R0913, R0914, W0622 @click.group( 'repo_mgr_cli', context_settings=context_settings(), epilog=footer(__version__) ) @click_opt_wrap(*cli_opts('config')) @click_opt_wrap(*cli_opts('hosts')) @click_opt_wrap(*cli_opts('cloud_id')) @click_opt_wrap(*cli_opts('api_token')) @click_opt_wrap(*cli_opts('id')) @click_opt_wrap(*cli_opts('api_key')) @click_opt_wrap(*cli_opts('username')) @click_opt_wrap(*cli_opts('password')) @click_opt_wrap(*cli_opts('bearer_auth')) @click_opt_wrap(*cli_opts('opaque_id')) @click_opt_wrap(*cli_opts('request_timeout')) @click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF)) @click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF)) @click_opt_wrap(*cli_opts('ca_certs')) @click_opt_wrap(*cli_opts('client_cert')) @click_opt_wrap(*cli_opts('client_key')) @click_opt_wrap(*cli_opts('ssl_assert_hostname')) @click_opt_wrap(*cli_opts('ssl_assert_fingerprint')) @click_opt_wrap(*cli_opts('ssl_version')) @click_opt_wrap(*cli_opts('master-only', onoff=ONOFF)) @click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF)) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) @click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS)) @click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS)) @click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS)) @click.version_option(__version__, '-v', '--version', prog_name="es_repo_mgr") @click.pass_context def repo_mgr_cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, dry_run, loglevel, logfile, logformat, ): """ Repository manager for Elasticsearch Curator The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. Some less-frequently used client configuration options are now hidden. To see the full list, run: es_repo_mgr show-all-options """ ctx.obj = {} ctx.obj['dry_run'] = dry_run ctx.obj['default_config'] = default_config_file() get_config(ctx) configure_logging(ctx) logger = logging.getLogger('curator.repomgrcli') generate_configdict(ctx) logger.debug('Exiting initial command function...') @repo_mgr_cli.command( 'show-all-options', context_settings=context_settings(), short_help='Show all configuration options', epilog=footer(__version__), ) @click_opt_wrap(*cli_opts('config')) @click_opt_wrap(*cli_opts('hosts')) @click_opt_wrap(*cli_opts('cloud_id')) @click_opt_wrap(*cli_opts('api_token')) @click_opt_wrap(*cli_opts('id')) @click_opt_wrap(*cli_opts('api_key')) @click_opt_wrap(*cli_opts('username')) @click_opt_wrap(*cli_opts('password')) @click_opt_wrap(*cli_opts('bearer_auth', override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('opaque_id', override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('request_timeout')) @click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF, override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF)) @click_opt_wrap(*cli_opts('ca_certs')) @click_opt_wrap(*cli_opts('client_cert')) @click_opt_wrap(*cli_opts('client_key')) @click_opt_wrap(*cli_opts('ssl_assert_hostname', override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('ssl_assert_fingerprint', override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('ssl_version', override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('master-only', onoff=ONOFF, override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF, override=SHOW_OPTION)) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) @click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS)) @click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS)) @click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS)) @click.version_option(__version__, '-v', '--version', prog_name="es_repo_mgr") @click.pass_context def show_all_options( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, dry_run, loglevel, logfile, logformat, ): """ ALL CLIENT OPTIONS The following is the full list of settings available for configuring a connection using command-line options. """ ctx = click.get_current_context() click.echo(ctx.get_help()) ctx.exit() @repo_mgr_cli.group('create') @click.pass_context def _create(ctx): """Create an Elasticsearch repository""" _create.add_command(azure) _create.add_command(gcs) _create.add_command(s3) _create.add_command(fs) _create.add_command(url) _create.add_command(source) @repo_mgr_cli.command('show') @click.pass_context def show(ctx): """Show all repositories""" client = get_client(ctx) show_repos(client) @repo_mgr_cli.command('delete') @click.option('--name', required=True, help='Repository name', type=str) @click.option( '--yes', is_flag=True, callback=delete_callback, expose_value=False, prompt='Are you sure you want to delete the repository?', ) @click.pass_context def _delete(ctx, name): """ Delete an Elasticsearch repository """ logger = logging.getLogger('curator.repomgrcli._delete') client = get_client(ctx) try: logger.info('Deleting repository %s...', name) client.snapshot.delete_repository(name=name) except NotFoundError: logger.error('Unable to delete repository: %s Not Found.', name) sys.exit(1) @repo_mgr_cli.command('info') @click.pass_context def info(ctx): """Show cluster info""" client = get_client(ctx) pp = pprint.PrettyPrinter(indent=4) output = dict(client.info()) click.echo(f'{pp.pprint(output)}') elasticsearch-curator-8.0.21/curator/singletons.py000066400000000000000000000050321477314666200223270ustar00rootroot00000000000000"""CLI module for curator_cli""" import click from es_client.defaults import SHOW_EVERYTHING from es_client.helpers.config import ( cli_opts, context_settings, generate_configdict, get_config, options_from_dict, ) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer from curator._version import __version__ from curator.cli_singletons import ( alias, allocation, close, delete_indices, delete_snapshots, forcemerge, open_indices, replicas, restore, rollover, snapshot, shrink, ) from curator.cli_singletons.show import show_indices, show_snapshots click_opt_wrap = option_wrapper() # pylint: disable=R0913, R0914, W0613, W0622, W0718 @click.group( context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html'), ) @options_from_dict(SHOW_EVERYTHING) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) @click.version_option(__version__, '-v', '--version', prog_name='curator_cli') @click.pass_context def curator_cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, loglevel, logfile, logformat, blacklist, dry_run, ): """ Curator CLI (Singleton Tool) Run a single action from the command-line. The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. """ ctx.obj = {} ctx.obj['dry_run'] = dry_run ctx.obj['default_config'] = default_config_file() get_config(ctx) configure_logging(ctx) generate_configdict(ctx) # Add the subcommands curator_cli.add_command(alias) curator_cli.add_command(allocation) curator_cli.add_command(close) curator_cli.add_command(delete_indices) curator_cli.add_command(delete_snapshots) curator_cli.add_command(forcemerge) curator_cli.add_command(open_indices) curator_cli.add_command(replicas) curator_cli.add_command(snapshot) curator_cli.add_command(restore) curator_cli.add_command(rollover) curator_cli.add_command(shrink) curator_cli.add_command(show_indices) curator_cli.add_command(show_snapshots) elasticsearch-curator-8.0.21/curator/snapshotlist.py000066400000000000000000000555721477314666200227130ustar00rootroot00000000000000"""SnapshotList""" import re import logging from es_client.helpers.schemacheck import SchemaCheck from curator.exceptions import ( ConfigurationError, FailedExecution, MissingArgument, NoSnapshots, ) from curator.helpers.date_ops import ( absolute_date_range, date_range, fix_epoch, get_date_regex, get_point_of_reference, TimestringSearch, ) from curator.helpers.getters import get_snapshot_data from curator.helpers.testers import repository_exists, verify_client_object from curator.helpers.utils import report_failure from curator.defaults import settings from curator.validators.filter_functions import filterstructure class SnapshotList: """Snapshot list object""" def __init__(self, client, repository=None): verify_client_object(client) if not repository: raise MissingArgument('No value for "repository" provided') if not repository_exists(client, repository): raise FailedExecution( f'Unable to verify existence of repository {repository}' ) self.loggit = logging.getLogger('curator.snapshotlist') #: An :py:class:`~.elasticsearch.Elasticsearch` client object passed from #: param ``client`` self.client = client #: The value passed as ``delete_aliases`` self.repository = repository #: Information extracted from snapshots, such as age, etc. #: Populated by internal method ``__get_snapshots`` at instance creation #: time. **Type:** :py:class:`dict` self.snapshot_info = {} #: The running list of snapshots which will be used by an Action class. #: Populated by internal methods ``__get_snapshots`` at instance creation #: time. **Type:** :py:class:`list` self.snapshots = [] #: Raw data dump of all snapshots in the repository at instance creation #: time. **Type:** :py:class:`list` of :py:class:`dict` data. self.__get_snapshots() self.age_keyfield = None def __actionable(self, snap): self.loggit.debug('Snapshot %s is actionable and remains in the list.', snap) def __not_actionable(self, snap): self.loggit.debug('Snapshot %s is not actionable, removing from list.', snap) self.snapshots.remove(snap) def __excludify(self, condition, exclude, snap, msg=None): if condition: if exclude: text = "Removed from actionable list" self.__not_actionable(snap) else: text = "Remains in actionable list" self.__actionable(snap) else: if exclude: text = "Remains in actionable list" self.__actionable(snap) else: text = "Removed from actionable list" self.__not_actionable(snap) if msg: self.loggit.debug('%s: %s', text, msg) def __get_snapshots(self): """ Pull all snapshots into `snapshots` and populate ``snapshot_info`` """ self.all_snapshots = get_snapshot_data(self.client, self.repository) for list_item in self.all_snapshots: if 'snapshot' in list_item.keys(): self.snapshots.append(list_item['snapshot']) self.snapshot_info[list_item['snapshot']] = list_item self.empty_list_check() def __map_method(self, ftype): methods = { 'age': self.filter_by_age, 'count': self.filter_by_count, 'none': self.filter_none, 'pattern': self.filter_by_regex, 'period': self.filter_period, 'state': self.filter_by_state, } return methods[ftype] def empty_list_check(self): """Raise exception if ``snapshots`` is empty""" if not self.snapshots: raise NoSnapshots('snapshot_list object is empty.') def working_list(self): """ Return the current value of ``snapshots`` as copy-by-value to prevent list stomping during iterations """ # Copy by value, rather than reference to prevent list stomping during # iterations return self.snapshots[:] def _get_name_based_ages(self, timestring): """ Add a snapshot age to ``snapshot_info`` based on the age as indicated by the snapshot name pattern, if it matches ``timestring``. This is stored at key ``age_by_name``. :param timestring: A :py:func:`time.strftime` pattern """ # Check for empty list before proceeding here to prevent non-iterable # condition self.empty_list_check() tstamp = TimestringSearch(timestring) for snapshot in self.working_list(): epoch = tstamp.get_epoch(snapshot) if epoch: self.snapshot_info[snapshot]['age_by_name'] = epoch else: self.snapshot_info[snapshot]['age_by_name'] = None def _calculate_ages(self, source='creation_date', timestring=None): """ This method initiates snapshot age calculation based on the given parameters. Exceptions are raised when they are improperly configured. Set instance variable ``age_keyfield`` for use later, if needed. :param source: Source of snapshot age. Can be ``name`` or ``creation_date``. :param timestring: An :py:func:`time.strftime` string to match the datestamp in an snapshot name. Only used if ``source=name``. """ if source == 'name': self.age_keyfield = 'age_by_name' if not timestring: raise MissingArgument( 'source "name" requires the "timestring" keyword argument' ) self._get_name_based_ages(timestring) elif source == 'creation_date': self.age_keyfield = 'start_time_in_millis' else: raise ValueError( f'Invalid source: {source}. Must be "name", or "creation_date".' ) def _sort_by_age(self, snapshot_list, reverse=True): """ Take a list of snapshots and sort them by date. By default, the youngest are first with ``reverse=True``, but the oldest can be first by setting ``reverse=False`` """ # Do the age-based sorting here. # First, build an temporary dictionary with just snapshot and age # as the key and value, respectively temp = {} for snap in snapshot_list: if self.age_keyfield in self.snapshot_info[snap]: # This fixes #1366. Catch None is a potential age value. if self.snapshot_info[snap][self.age_keyfield]: temp[snap] = self.snapshot_info[snap][self.age_keyfield] else: msg = f' snapshot {snap} has no age' self.__excludify(True, True, snap, msg) else: msg = ( f'{snap} does not have age key "{self.age_keyfield}" ' f'in SnapshotList metadata' ) self.__excludify(True, True, snap, msg) # If reverse is True, this will sort so the youngest snapshots are # first. However, if you want oldest first, set reverse to False. # Effectively, this should set us up to act on everything older than # meets the other set criteria. # It starts as a tuple, but then becomes a list. sorted_tuple = sorted(temp.items(), key=lambda k: k[1], reverse=reverse) return [x[0] for x in sorted_tuple] def most_recent(self): """ Return the most recent snapshot based on ``start_time_in_millis``. """ self.empty_list_check() most_recent_time = 0 most_recent_snap = '' for snapshot in self.snapshots: snaptime = fix_epoch(self.snapshot_info[snapshot]['start_time_in_millis']) if snaptime > most_recent_time: most_recent_snap = snapshot most_recent_time = snaptime return most_recent_snap def filter_by_regex(self, kind=None, value=None, exclude=False): """ Filter out snapshots not matching the pattern, or in the case of exclude, filter those matching the pattern. :param kind: Can be one of: ``suffix``, ``prefix``, ``regex``, or ``timestring``. This option defines what kind of filter you will be building. :param value: Depends on ``kind``. It is the :py:func:`time.strftime` string if ``kind`` is ``timestring``. It's used to build the regular expression for other kinds. :param exclude: If ``exclude=True``, this filter will remove matching snapshots from ``snapshots``. If ``exclude=False``, then only matching snapshots will be kept in ``snapshots``. Default is ``False`` """ if kind not in ['regex', 'prefix', 'suffix', 'timestring']: raise ValueError(f'{kind}: Invalid value for kind') # Stop here if None or empty value, but zero is okay if value == 0: pass elif not value: raise ValueError( ( f'{value}: Invalid value for "value". Cannot be "None" type, ' f'empty, or False' ) ) if kind == 'timestring': regex = settings.regex_map()[kind].format(get_date_regex(value)) else: regex = settings.regex_map()[kind].format(value) self.empty_list_check() pattern = re.compile(regex) for snapshot in self.working_list(): match = pattern.search(snapshot) self.loggit.debug('Filter by regex: Snapshot: %s', snapshot) if match: self.__excludify(True, exclude, snapshot) else: self.__excludify(False, exclude, snapshot) def filter_by_age( self, source='creation_date', direction=None, timestring=None, unit=None, unit_count=None, epoch=None, exclude=False, ): """ Remove snapshots from ``snapshots`` by relative age calculations. :param source: Source of snapshot age. Can be ``name``, or ``creation_date``. :param direction: Time to filter, either ``older`` or ``younger`` :param timestring: A :py:func:`time.strftime` string to match the datestamp in an snapshot name. Only used for snapshot filtering by ``name``. :param unit: One of ``seconds``, ``minutes``, ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param unit_count: The number of ``unit`` (s). ``unit_count`` * ``unit`` will be calculated out to the relative number of seconds. :param epoch: An epoch timestamp used in conjunction with ``unit`` and ``unit_count`` to establish a point of reference for calculations. If not provided, the current time will be used. :param exclude: If ``exclude=True``, this filter will remove matching snapshots from ``snapshots``. If ``exclude=False``, then only matching snapshots will be kept in ``snapshots``. Default is ``False`` """ self.loggit.debug('Starting filter_by_age') # Get timestamp point of reference, por por = get_point_of_reference(unit, unit_count, epoch) self.loggit.debug('Point of Reference: %s', por) if not direction: raise MissingArgument('Must provide a value for "direction"') if direction not in ['older', 'younger']: raise ValueError(f'Invalid value for "direction": {direction}') self._calculate_ages(source=source, timestring=timestring) for snapshot in self.working_list(): if not self.snapshot_info[snapshot][self.age_keyfield]: self.loggit.debug('Removing snapshot %s for having no age', snapshot) self.snapshots.remove(snapshot) continue age = fix_epoch(self.snapshot_info[snapshot][self.age_keyfield]) msg = ( f'Snapshot "{snapshot}" age ({age}), direction: "{direction}", ' f'point of reference, ({por})' ) # Because time adds to epoch, smaller numbers are actually older # timestamps. snapshot_age = fix_epoch(self.snapshot_info[snapshot][self.age_keyfield]) if direction == 'older': agetest = snapshot_age < por else: # 'younger' agetest = snapshot_age > por self.__excludify(agetest, exclude, snapshot, msg) def filter_by_state(self, state=None, exclude=False): """ Filter out snapshots not matching ``state``, or in the case of exclude, filter those matching ``state``. :param state: The snapshot state to filter for. Must be one of ``SUCCESS``, ``PARTIAL``, ``FAILED``, or ``IN_PROGRESS``. :param exclude: If ``exclude=True``, this filter will remove matching snapshots from ``snapshots``. If ``exclude=False``, then only matching snapshots will be kept in ``snapshots``. Default is ``False`` """ if state.upper() not in ['SUCCESS', 'PARTIAL', 'FAILED', 'IN_PROGRESS']: raise ValueError(f'{state}: Invalid value for state') self.empty_list_check() for snapshot in self.working_list(): self.loggit.debug('Filter by state: Snapshot: %s', snapshot) if self.snapshot_info[snapshot]['state'] == state: self.__excludify(True, exclude, snapshot) else: self.__excludify(False, exclude, snapshot) def filter_none(self): """No filter at all""" self.loggit.debug('"None" filter selected. No filtering will be done.') def filter_by_count( self, count=None, reverse=True, use_age=False, source='creation_date', timestring=None, exclude=True, ): """ Remove snapshots from the actionable list beyond the number ``count``, sorted reverse-alphabetically by default. If you set ``reverse=False``, it will be sorted alphabetically. The default is usually what you will want. If only one kind of snapshot is provided--for example, snapshots matching ``curator-%Y%m%d%H%M%S``--then reverse alphabetical sorting will mean the oldest will remain in the list, because lower numbers in the dates mean older snapshots. By setting ``reverse=False``, then ``snapshot3`` will be acted on before ``snapshot2``, which will be acted on before ``snapshot1`` ``use_age`` allows ordering snapshots by age. Age is determined by the snapshot creation date (as identified by ``start_time_in_millis``) by default, but you can also specify ``source=name``. The ``name`` ``source`` requires the timestring argument. :param count: Filter snapshots beyond ``count``. :param reverse: The filtering direction. (default: ``True``). :param use_age: Sort snapshots by age. ``source`` is required in this case. :param source: Source of snapshot age. Can be one of ``name``, or ``creation_date``. Default: ``creation_date`` :param timestring: A :py:func:`time.strftime` string to match the datestamp in a snapshot name. Only used if ``source=name``. :param exclude: If ``exclude=True``, this filter will remove matching snapshots from ``snapshots``. If ``exclude=False``, then only matching snapshots will be kept in ``snapshots``. Default is ``True`` """ self.loggit.debug('Filtering snapshots by count') if not count: raise MissingArgument('No value for "count" provided') # Create a copy-by-value working list working_list = self.working_list() if use_age: self._calculate_ages(source=source, timestring=timestring) # Using default value of reverse=True in self._sort_by_age() sorted_snapshots = self._sort_by_age(working_list, reverse=reverse) else: # Default to sorting by snapshot name sorted_snapshots = sorted(working_list, reverse=reverse) idx = 1 for snap in sorted_snapshots: msg = f'{snap} is {idx} of specified count of {count}.' condition = True if idx <= count else False self.__excludify(condition, exclude, snap, msg) idx += 1 def filter_period( self, period_type='relative', source='name', range_from=None, range_to=None, date_from=None, date_to=None, date_from_format=None, date_to_format=None, timestring=None, unit=None, week_starts_on='sunday', epoch=None, exclude=False, ): """ Match ``snapshots`` with ages within a given period. :param period_type: Can be either ``absolute`` or ``relative``. Default is ``relative``. ``date_from`` and ``date_to`` are required when using ``period_type='absolute'``. ``range_from`` and ``range_to`` are required with ``period_type='relative'``. :param source: Source of snapshot age. Can be ``name``, or ``creation_date``. :param range_from: How many ``unit`` (s) in the past/future is the origin? :param range_to: How many ``unit`` (s) in the past/future is the end point? :param date_from: The simplified date for the start of the range :param date_to: The simplified date for the end of the range. If this value is the same as ``date_from``, the full value of ``unit`` will be extrapolated for the range. For example, if ``unit=months``, and ``date_from`` and ``date_to`` are both ``2017.01``, then the entire month of January 2017 will be the absolute date range. :param date_from_format: The :py:func:`time.strftime` string used to parse ``date_from`` :param date_to_format: The :py:func:`time.strftime` string used to parse ``date_to`` :param timestring: An :py:func:`time.strftime` string to match the datestamp in an snapshot name. Only used for snapshot filtering by ``name``. :param unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``. :param week_starts_on: Either ``sunday`` or ``monday``. Default is ``sunday`` :param epoch: An epoch timestamp used to establish a point of reference for calculations. If not provided, the current time will be used. :param exclude: If ``exclude=True``, this filter will remove matching indices from ``indices``. If ``exclude=False``, then only matching indices will be kept in ``indices``. Default is ``False`` """ self.loggit.debug('Filtering snapshots by period') if period_type not in ['absolute', 'relative']: raise ValueError( f'Unacceptable value: {period_type} -- "period_type" must be either ' f'"absolute" or "relative".' ) self.loggit.debug('period_type = %s', period_type) if period_type == 'relative': func = date_range args = [unit, range_from, range_to, epoch] kwgs = {'week_starts_on': week_starts_on} try: range_from = int(range_from) range_to = int(range_to) except ValueError as err: raise ConfigurationError( f'"range_from" and "range_to" must be integer values. Error: {err}' ) from err else: func = absolute_date_range args = [unit, date_from, date_to] kwgs = { 'date_from_format': date_from_format, 'date_to_format': date_to_format, } for reqd in [date_from, date_to, date_from_format, date_to_format]: if not reqd: raise ConfigurationError( 'Must provide "date_from", "date_to", "date_from_format", ' 'and "date_to_format" with absolute period_type' ) try: start, end = func(*args, **kwgs) # pylint: disable=broad-except except Exception as err: report_failure(err) self._calculate_ages(source=source, timestring=timestring) for snapshot in self.working_list(): if not self.snapshot_info[snapshot][self.age_keyfield]: self.loggit.debug('Removing snapshot {0} for having no age') self.snapshots.remove(snapshot) continue age = fix_epoch(self.snapshot_info[snapshot][self.age_keyfield]) msg = ( f'Snapshot "{snapshot}" age ({age}), period start: "{start}", period ' f'end, ({end})' ) # Because time adds to epoch, smaller numbers are actually older # timestamps. inrange = (age >= start) and (age <= end) self.__excludify(inrange, exclude, snapshot, msg) def iterate_filters(self, config): """ Iterate over the filters defined in ``config`` and execute them. :param config: A dictionary of filters, as extracted from the YAML configuration file. .. note:: ``config`` should be a dictionary with the following form: .. code-block:: python { 'filters' : [ { 'filtertype': 'the_filter_type', 'key1' : 'value1', ... 'keyN' : 'valueN' } ] } """ # Make sure we actually _have_ filters to act on if 'filters' not in config or not config['filters']: self.loggit.info('No filters in config. Returning unaltered object.') return self.loggit.debug('All filters: %s', config['filters']) for fltr in config['filters']: self.loggit.debug('Top of the loop: %s', self.snapshots) self.loggit.debug('Un-parsed filter args: %s', fltr) filter_result = SchemaCheck( fltr, filterstructure(), 'filter', 'SnapshotList.iterate_filters' ).result() self.loggit.debug('Parsed filter args: %s', filter_result) method = self.__map_method(fltr['filtertype']) # Remove key 'filtertype' from dictionary 'fltr' del fltr['filtertype'] # If it's a filtertype with arguments, update the defaults with the # provided settings. self.loggit.debug('Filter args: %s', fltr) self.loggit.debug('Pre-instance: %s', self.snapshots) method(**fltr) self.loggit.debug('Post-instance: %s', self.snapshots) elasticsearch-curator-8.0.21/curator/validators/000077500000000000000000000000001477314666200217405ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/validators/__init__.py000066400000000000000000000000001477314666200240370ustar00rootroot00000000000000elasticsearch-curator-8.0.21/curator/validators/actions.py000066400000000000000000000054341477314666200237600ustar00rootroot00000000000000"""Validate root ``actions`` and individual ``action`` Schemas""" from voluptuous import Any, In, Schema, Optional, Required from es_client.helpers.schemacheck import SchemaCheck from curator.defaults import settings def root(): """ Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which is a dictionary with ``actions`` :py:class:`~.voluptuous.schema_builder.Required` to be the root key with another dictionary as the value. """ return Schema({Required('actions'): dict}) def valid_action(): """ Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which is that the value of key ``action`` is :py:class:`~.voluptuous.schema_builder.Required` to be :py:class:`~.voluptuous.schema_builder.In` the value returned by :py:func:`~.curator.defaults.settings.all_actions`. """ return { Required('action'): Any( In(settings.all_actions()), msg=f'action must be one of {settings.all_actions()}', ) } def structure(data, location): """ Return a valid :py:class:`~.voluptuous.schema_builder.Schema` definition which tests ``data``, which is ostensibly an individual action dictionary. If it is a :py:func:`~.curator.validators.actions.valid_action`, then it will :py:meth:`~.voluptuous.schema_builder.Schema.update` the base :py:class:`~.voluptuous.schema_builder.Schema` with other options, based on the what the value of ``data['action']`` is. :param data: The configuration dictionary, or sub-dictionary, being validated :type data: dict :param location: A string to report which configuration sub-block is being tested. :type location: str :returns: A :py:class:`~.voluptuous.schema_builder.Schema` object """ _ = SchemaCheck( data, Schema(valid_action(), extra=True), 'action type', location, ).result() # Build a valid schema knowing that the action has already been validated retval = valid_action() retval.update({Optional('description', default='No description given'): Any(str)}) retval.update({Optional('options', default=settings.default_options()): dict}) action = data['action'] if action in ['cluster_routing', 'create_index', 'rollover']: # The cluster_routing, create_index, and rollover actions should not # have a 'filters' block pass elif action == 'alias': # The alias action should not have a filters block, but should have # an add and/or remove block. retval.update( { Optional('add'): dict, Optional('remove'): dict, } ) else: retval.update({Optional('filters', default=settings.default_filters()): list}) return Schema(retval) elasticsearch-curator-8.0.21/curator/validators/filter_functions.py000066400000000000000000000061761477314666200257010ustar00rootroot00000000000000"""Functions validating the ``filter`` Schema of an ``action``""" import logging from voluptuous import Any, In, Required, Schema from es_client.helpers.schemacheck import SchemaCheck from es_client.helpers.utils import prune_nones from curator.defaults import settings, filtertypes from curator.exceptions import ConfigurationError logger = logging.getLogger(__name__) def filtertype(): """ Return a :py:class:`~.voluptuous.schema_builder.Schema` object that uses :py:func:`~.curator.defaults.settings.all_filtertypes` to populate acceptable values :returns: A :py:class:`~.voluptuous.schema_builder.Schema` object """ return { Required('filtertype'): Any( In(settings.all_filtertypes()), msg=f'filtertype must be one of {settings.all_filtertypes()}', ) } def filterstructure(): """ Return a :py:class:`~.voluptuous.schema_builder.Schema` object that uses the return value from :py:func:`~.curator.defaults.settings.structural_filter_elements` to populate acceptable values and updates/merges the Schema object with the return value from :py:func:`filtertype` :returns: A :py:class:`~.voluptuous.schema_builder.Schema` object """ # This is to first ensure that only the possible keys/filter elements are # there, and get a dictionary back to work with. retval = settings.structural_filter_elements() retval.update(filtertype()) return Schema(retval) def singlefilter(action, data): """ Return a :py:class:`~.voluptuous.schema_builder.Schema` object that is created using the return value from :py:func:`filtertype` to create a local variable ``ftype``. The values from ``action`` and ``data`` are used to update ``ftype`` based on matching function names in :py:mod:`~.curator.defaults.filtertypes`. :py:func:`~.curator.defaults.settings.structural_filter_elements` to populate acceptable values and updates/merges the Schema object with the return value from :py:func:`filtertype` :param action: The Curator action name :type action: str :param data: The filter block of the action :returns: A :py:class:`~.voluptuous.schema_builder.Schema` object """ try: ftdata = data['filtertype'] except KeyError as exc: raise ConfigurationError('Missing key "filtertype"') from exc ftype = filtertype() for each in getattr(filtertypes, ftdata)(action, data): ftype.update(each) return Schema(ftype) def validfilters(action, location=None): """Validate the filters in a list""" def func(val): """This validator method simply validates all filters in the list.""" for idx, value in enumerate(val): pruned = prune_nones(value) filter_dict = SchemaCheck( pruned, singlefilter(action, pruned), 'filter', f'{location}, filter #{idx}: {pruned}', ).result() logger.debug('Filter #%s: %s', idx, filter_dict) val[idx] = filter_dict # If we've made it here without raising an Exception, it's valid return val return func elasticsearch-curator-8.0.21/curator/validators/options.py000066400000000000000000000155431477314666200240150ustar00rootroot00000000000000"""Set up voluptuous Schema defaults for various actions""" from voluptuous import Schema from curator.defaults import option_defaults # Methods for building the schema def action_specific(action): """ :param action: The name of an action :type action: str :returns: A :py:class:`list` containing one or more :py:class:`~.voluptuous.schema_builder.Optional` or :py:class:`~.voluptuous.schema_builder.Required` options from :py:mod:`~.curator.defaults.option_defaults`, defining acceptable values for each for the given ``action`` :rtype: list """ options = { 'alias': [ option_defaults.name(action), option_defaults.warn_if_no_indices(), option_defaults.extra_settings(), ], 'allocation': [ option_defaults.search_pattern(), option_defaults.key(), option_defaults.value(), option_defaults.allocation_type(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), ], 'close': [ option_defaults.search_pattern(), option_defaults.delete_aliases(), option_defaults.skip_flush(), ], 'cluster_routing': [ option_defaults.routing_type(), option_defaults.cluster_routing_setting(), option_defaults.cluster_routing_value(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), ], 'cold2frozen': [ option_defaults.search_pattern(), option_defaults.c2f_index_settings(), option_defaults.c2f_ignore_index_settings(), option_defaults.wait_for_completion('cold2frozen'), ], 'create_index': [ option_defaults.name(action), option_defaults.ignore_existing(), option_defaults.extra_settings(), ], 'delete_indices': [ option_defaults.search_pattern(), ], 'delete_snapshots': [ option_defaults.repository(), option_defaults.retry_interval(), option_defaults.retry_count(), ], 'forcemerge': [ option_defaults.search_pattern(), option_defaults.delay(), option_defaults.max_num_segments(), ], 'index_settings': [ option_defaults.search_pattern(), option_defaults.index_settings(), option_defaults.ignore_unavailable(), option_defaults.preserve_existing(), ], 'open': [ option_defaults.search_pattern(), ], 'reindex': [ option_defaults.request_body(), option_defaults.refresh(), option_defaults.requests_per_second(), option_defaults.slices(), option_defaults.timeout(action), option_defaults.wait_for_active_shards(action), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), option_defaults.remote_certificate(), option_defaults.remote_client_cert(), option_defaults.remote_client_key(), option_defaults.remote_filters(), option_defaults.migration_prefix(), option_defaults.migration_suffix(), ], 'replicas': [ option_defaults.search_pattern(), option_defaults.count(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), ], 'rollover': [ option_defaults.name(action), option_defaults.new_index(), option_defaults.conditions(), option_defaults.extra_settings(), option_defaults.wait_for_active_shards(action), ], 'restore': [ option_defaults.repository(), option_defaults.name(action), option_defaults.indices(), option_defaults.ignore_unavailable(), option_defaults.include_aliases(), option_defaults.include_global_state(action), option_defaults.partial(), option_defaults.rename_pattern(), option_defaults.rename_replacement(), option_defaults.extra_settings(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), option_defaults.skip_repo_fs_check(), ], 'snapshot': [ option_defaults.search_pattern(), option_defaults.repository(), option_defaults.name(action), option_defaults.ignore_unavailable(), option_defaults.include_global_state(action), option_defaults.partial(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), option_defaults.skip_repo_fs_check(), ], 'shrink': [ option_defaults.search_pattern(), option_defaults.shrink_node(), option_defaults.node_filters(), option_defaults.number_of_shards(), option_defaults.number_of_replicas(), option_defaults.shrink_prefix(), option_defaults.shrink_suffix(), option_defaults.copy_aliases(), option_defaults.delete_after(), option_defaults.post_allocation(), option_defaults.wait_for_active_shards(action), option_defaults.extra_settings(), option_defaults.wait_for_completion(action), option_defaults.wait_for_rebalance(), option_defaults.wait_interval(action), option_defaults.max_wait(action), ], } return options[action] def get_schema(action): """ Return a :py:class:`~.voluptuous.schema_builder.Schema` of acceptable options and their default values as returned by :py:func:`action_specific`, passing along the value of ``action``. :param action: The name of an action :type action: str :returns: A valid :py:class:`~.voluptuous.schema_builder.Schema` of the options for ``action`` """ options = {} defaults = [ option_defaults.allow_ilm_indices(), option_defaults.continue_if_exception(), option_defaults.disable_action(), option_defaults.ignore_empty_list(), option_defaults.include_hidden(), option_defaults.timeout_override(action), ] for each in defaults: options.update(each) for each in action_specific(action): options.update(each) return Schema(options) elasticsearch-curator-8.0.21/docker_test/000077500000000000000000000000001477314666200204175ustar00rootroot00000000000000elasticsearch-curator-8.0.21/docker_test/scripts/000077500000000000000000000000001477314666200221065ustar00rootroot00000000000000elasticsearch-curator-8.0.21/docker_test/scripts/create.sh000077500000000000000000000105631477314666200237150ustar00rootroot00000000000000#!/bin/bash if [ "x$1" == "x" ]; then echo "Error! No Elasticsearch version provided." echo "VERSION must be in Semver format, e.g. X.Y.Z, 7.17.8" echo "USAGE: $0 VERSION" exit 1 fi unsupported () { echo "Elasticsearch version ${1} is not supported. Exiting." exit 1 } VERSION=$1 JAVA_OPTS="-Xms512m -Xmx512m" MAJOR=$(echo $VERSION | awk -F. '{print $1}') MINOR=$(echo $VERSION | awk -F. '{print $2}') RUNNAME=curator${MAJOR}-es LOCAL_NAME=${RUNNAME}-local REMOTE_NAME=${RUNNAME}-remote LOCAL_PORT=9200 REMOTE_PORT=9201 LOCAL_URL=http://127.0.0.1:${LOCAL_PORT} REMOTE_URL=http://127.0.0.1:${REMOTE_PORT} if [[ "$MAJOR" -eq 8 ]]; then MONITORING="xpack.monitoring.templates.enabled" IMAGE=docker.elastic.co/elasticsearch/elasticsearch elif [[ "$MAJOR" -eq 7 ]]; then if [[ "$MINOR" -lt 14 ]]; then unsupported $VERSION fi IMAGE=docker.elastic.co/elasticsearch/elasticsearch MONITORING="xpack.monitoring.enabled" else unsupported $VERSION fi # Determine local IPs OS=$(uname -a | awk '{print $1}') if [[ "$OS" = "Linux" ]]; then IPLIST=$(ip -4 -o addr show scope global | grep -v docker |awk '{gsub(/\/.*/,"",$4); print $4}') elif [[ "$OS" = "Darwin" ]]; then IPLIST=$(ifconfig | awk -F "[: ]+" '/inet / { if ($2 != "127.0.0.1") print $2 }') else echo "Could not determine local IPs for assigning REMOTE_ES_SERVER env variable..." echo "Please manually determine your local non-loopback IP address and assign it to REMOTE_ES_SERVER" echo "e.g. REMOTE_ES_SERVER=http://A.B.C.D:${REMOTE_PORT} (be sure to use port ${REMOTE_PORT}!)" exit 0 fi WHITELIST="" for IP in $IPLIST; do if [ "x${WHITELIST}" == "x" ]; then WHITELIST="${IP}:${REMOTE_PORT}" else WHITELIST="${WHITELIST},${IP}:${REMOTE_PORT}" fi done # Save original execution path EXECPATH=$(pwd) # Extract the path for the script SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # Navigate to the script, regardless of whether we were there cd $SCRIPTPATH # Go up one directory cd .. # Find out what the last part of this directory is called UPONE=$(pwd | awk -F\/ '{print $NF}') if [[ "$UPONE" = "docker_test" ]]; then REPOPATH=$(pwd)/repo # Nuke it from orbit, just to be sure rm -rf ${REPOPATH} mkdir -p ${REPOPATH} else echo "Unable to correctly locate bind mount repo path. Exiting." exit 1 fi # # Check if the image has been built. If not, build it. # if [[ "$(docker images -q ${IMAGE}:${VERSION} 2> /dev/null)" == "" ]]; then # echo "Docker image ${IMAGE}:${VERSION} not found. Building from Dockerfile..." # cd $SCRIPTPATH # # Create a Dockerfile from the template # cat Dockerfile.tmpl | sed -e "s/ES_VERSION/${VERSION}/" > Dockerfile # docker build . -t ${IMAGE}:${VERSION} # fi ### Launch the containers echo -en "\rStarting ${LOCAL_NAME} container... " docker run -d --name ${LOCAL_NAME} -p ${LOCAL_PORT}:9200 -v ${REPOPATH}:/media \ -e ES_JAVA_OPTS="${JAVA_OPTS}" \ -e "discovery.type=single-node" \ -e "cluster.name=local-cluster" \ -e "node.name=local" \ -e "${MONITORING}=false" \ -e "path.repo=/media" \ -e "xpack.security.enabled=false" \ -e "reindex.remote.whitelist=${WHITELIST}" \ ${IMAGE}:${VERSION} echo -en "\rStarting ${REMOTE_NAME} container... " docker run -d --name ${REMOTE_NAME} -p ${REMOTE_PORT}:9200 -v ${REPOPATH}:/media \ -e ES_JAVA_OPTS="${JAVA_OPTS}" \ -e "discovery.type=single-node" \ -e "cluster.name=remote-cluster" \ -e "node.name=remote" \ -e "${MONITORING}=false" \ -e "path.repo=/media" \ -e "xpack.security.enabled=false" \ ${IMAGE}:${VERSION} ### Check to make sure the ES instances are up and running echo echo "Waiting for Elasticsearch instances to become available..." echo EXPECTED=200 for URL in $LOCAL_URL $REMOTE_URL; do if [[ "$URL" = "$LOCAL_URL" ]]; then NODE="${LOCAL_NAME} instance" else NODE="${REMOTE_NAME} instance" fi ACTUAL=0 while [ $ACTUAL -ne $EXPECTED ]; do ACTUAL=$(curl -o /dev/null -s -w "%{http_code}\n" $URL) echo -en "\rHTTP status code for $NODE is: $ACTUAL" if [ $EXPECTED -eq $ACTUAL ]; then echo " --- $NODE is ready!" fi sleep 1 done done # Done echo echo "Creation complete. ${LOCAL_NAME} and ${REMOTE_NAME} containers are up using image ${IMAGE}:${VERSION}" echo echo "Please select one of these environment variables to prepend your 'pytest --cov=curator' run:" echo for IP in $IPLIST; do echo "REMOTE_ES_SERVER=\"$IP:${REMOTE_PORT}\"" done echo echo "Ready to test!"elasticsearch-curator-8.0.21/docker_test/scripts/destroy.sh000077500000000000000000000026451477314666200241450ustar00rootroot00000000000000#!/bin/bash if [ "x${1}" != "verbose" ]; then DEBUG=0 else DEBUG=1 fi log_out () { # $1 is the log line if [ ${DEBUG} -eq 1 ]; then echo ${1} fi } # Stop running containers echo echo "Stopping all containers..." RUNNING=$(docker ps | egrep 'curator.?-es-(remote|local)' | awk '{print $NF}') for container in ${RUNNING}; do log_out "Stopping container ${container}..." log_out "$(docker stop ${container}) stopped." done # Remove existing containers echo "Removing all containers..." EXISTS=$(docker ps -a | egrep 'curator.?-es-(remote|local)' | awk '{print $NF}') for container in ${EXISTS}; do log_out "Removing container ${container}..." log_out "$(docker rm -f ${container}) deleted." done ### Now begins the repo cleanup phase # Save original execution path EXECPATH=$(pwd) # Extract the path for the script SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # Navigate to the script, regardless of whether we were there cd $SCRIPTPATH # Remove the created Dockerfile rm -f Dockerfile # Go up one directory cd .. # Find out what the last part of this directory is called UPONE=$(pwd | awk -F\/ '{print $NF}') if [[ "$UPONE" = "docker_test" ]]; then rm -rf $(pwd)/repo/* else echo "WARNING: Unable to automatically empty bind mounted repo path." echo "Please manually empty the contents of the repo directory!" fi # Return to origin to be clean cd $EXECPATH echo "Cleanup complete." elasticsearch-curator-8.0.21/docs/000077500000000000000000000000001477314666200170415ustar00rootroot00000000000000elasticsearch-curator-8.0.21/docs/Changelog.rst000066400000000000000000003674231477314666200215010ustar00rootroot00000000000000.. _Changelog: Changelog ========= 8.0.21 (1 April 2025) --------------------- **Bugfix Release** As was reported in #1704, the ``--ignore_empty_list`` option was not being respected. Code changes were made to each singleton that uses the ``--ignore_empty_list`` option to ensure that the option is respected. **Changes** * Fix ``--ignore_empty_list`` option to be respected in all singletons. * Add debug message to IndexList class when an empty list condition is encountered. * Dependency version bump: ``es_client`` to ``8.17.5`` 8.0.20 (21 March 2025) ---------------------- **Patch Release** There are no code changes in this release. A package maintainer at Debian raised an issue with the now outdated ``cx_Freeze`` dependency in ``setup.py``. As ``cx_Freeze`` is now only used in the Docker build, ``setup.py`` has been removed to avoid the unneeded dependency. Docker and package builds were tested to ensure that the changes work as expected before this release. **Changes** * Move ``cx_Freeze`` configuration from ``setup.py`` to ``pyproject.toml``. The configuration will be read during the Docker image build but not otherwise result in ``cx_Freeze`` being a dependency. * Change how ``cx_Freeze`` is invoked in ``Dockerfile``, removing the need for ``setup.py``. * Remove ``setup.py`` as it is no longer needed for package building. * Use ``es_client==8.17.4`` which fixes a logging issue. * Fix copyright dates in ``docs/index.rst`` * Update asciidoc files to reflect the new version numbers. 8.0.19 (5 March 2025) --------------------- **Announcement** The ability to include hidden indices is now available in Curator. This is now an option you can include in the ``options`` section of your configuration file. .. code-block:: yaml options: include_hidden: True This will allow Curator to include hidden indices in its actions. This will not suddenly reveal system indices, but it will allow you to include indices that are hidden, such as indices backing certain data_streams. This is also an option flag for the CLI Singletons, e.g. ``--include-hidden``, with the default value being the equivalent of ``--no-include_hidden``. There's an odd caveat to this, however, and it is probaby a bug in Elasticsearch. If you have an index that starts with a dot, e.g. ``.my_index``, and you set your multi-target search pattern to also start with a dot, e.g. ``.my_*``, and you set the index settings for ``.my_index`` to be ``hidden: true``, the index will not be excluded from the search results when the ``expand_wildcards`` parameter is set to include hidden indices, e.g. ``open,closed,hidden``. This is a bug in Elasticsearch, and not in Curator. I've reported it to the Elasticsearch team at https://github.com/elastic/elasticsearch/issues/124167, but I'm not sure when it will be addressed. In the meantime, if you need to guarantee hidden indices stay excluded, use ``search_pattern`` and filters to exclude anything that needs to stay excluded. All tests pass on versions 7.17.25 and 8.17.2 of Elasticsearch. **Changes** * Updated ``tests/integration/test_integrations.py::TestFilters`` to have a ``filter_closed`` test to ensure functionality is working as expected. This test was added because of #1733, which is technically about Curator v7.0.1, but since the release of Curator 8.0.18, which supports the version of Elasticsearch being used in that issue, a confirmation integration test was added here. * PEP8 formatting changes to ``tests/integration/testvars.py`` as well as adding the ``filter_closed`` YAML sample. * Updated ``docs/conf.py`` to reflect the modules being used. * Add support to include hidden indices. This is done by adding ``hidden`` to the ``expand_wildcards`` keyword arg, which will make it ``open,closed,hidden``. 8.0.18 (27 February 2025) ------------------------- **Announcement** Release 8.0.18 allows Curator v8 to work with Curator v7.14.x and later. All tests pass for v7.14.0, v7.14.2, v7.17.7, v7.17.25, v7.17.27. Due to JVM issues with M-series processors (OpenJDK 16 < FAIL < OpenJDK 19 ), I'm unable to test v7.15.x - v7.17.6 on my MacBook Pro, but I have no reason to believe they won't as there are no API changes between the 7.14 and 7.17 series that would affect Curator in any way. Hopefully this is helpful for those who are using Elasticsearch 7 still, and would like the improvements in Curator v8. Do note that while the action files used in Curator v7 will work with Curator v8, the client configuration files will not. There are a few differences in syntax that are in the documentation. See https://www.elastic.co/guide/en/elasticsearch/client/curator/current/configfile.html for more information. **Changes** * Huge number of comment and code line break changes to be more PEP8 compliant. * This includes updates to Integration tests as well * Update to use ``es_client==8.17.2``, which enables Curator v8 to work with Elasticsearch v7.14.x and later. * Add ``VERSION_MAX`` and ``VERSION_MIN`` to ``curator.defaults`` to allow for version compatibility checks. ``VERSION_MIN`` is set to 7.14.0. * Exclude system indices by default. The list of patterns to exclude is in ``curator.defaults``, and is presently .. code-block:: python EXCLUDE_SYSTEM = ( '-.kibana*,-.security*,-.watch*,-.triggered_watch*,' '-.ml*,-.geoip_databases*,-.logstash*,-.tasks*' ) * Restored and fixed datemath integration tests for patterns that include colons. This was a problem in the past due to Elasticsearch evaluating colon characters as potential indicators of remote indices. This was fixed in version 8.7.0 of Elasticsearch. **Bugfix** * All of the testing for this release revealed a shortcoming in the snapshot restore action class. It was relying on the user to provide the exact index names to restore. The restore API call to Elasticsearch allows for multi-target syntax to select and de-select indices by comma-separated patterns. While not expressly allowed, it was possible to use. The problem was that indices could not be properly matched and verified if patterns rather than exact names were used. Three functions were added to ``helpers.utils`` to make this work properly: ``multitarget_match``, ``multitarget_fix``, and ``regex_loop``. The ``Restore`` class now calls ``multitarget_match`` in the ``_get_expected_output`` method to get the names from the snapshot list object. This now works with patterns, and the tests have been updated to ensure this. 8.0.17 (25 October 2024) ------------------------ **Bugfix** * Reported in #1727 (and I'm relieved that nobody got bit by this sooner), A serious bug was found where if using the ``age`` filter while deriving the age from the index name, it was erroneously ignoring any index that did not have a matching ``timestring`` pattern, which would leave a default epoch time of ``0``, which would definitely be older than your cutoff date! As a result, indices were matched and deleted that should not have been. The fix is to remove indices that do not match the pattern in the ``_get_name_based_ages`` method. The patch is in this release, as are updated tests to replicate the failure scenario. Hat tip to @giom-l for reporting this. **Changes** * Update to use ``es_client==8.15.2``. * Update data node detection to include ``data``, ``data_content``, ``data_hot``, and ``data_warm`` for ``shrink`` action. This was first raised in #1621, but needed to go further than just adding ``data_hot``. Hat tip to @gnobironts for the original pull request. * Add ``docker_test/.env`` to ``.gitignore`` * More formatting changes as suggested by pylint * Improve API calls to ``node.info`` and ``node.stats`` to use ``filter_path`` 8.0.16 (6 August 2024) ---------------------- **Changes** * Update to use ``es_client==8.14.2`` * Formatting changes and improvements * Update CLI to get client using ``ctx.obj['configdict']`` as it's already built by ``es_client``. **Bugfixes** * Fix improper log levels erroneously left in from debugging. Thanks to @boutetnico in #1714 * ``es_client`` version 8.14.2 addresses a problem where Python 3.8 is not officially supported for use with ``voluptuous`` greater than ``0.14.2``. 8.0.15 (10 April 2024) ---------------------- **Announcement** * Python 3.12 support becomes official. A few changes were necessary to ``datetime`` calls which were still using naive timestamps. Tests across all minor Python versions from 3.8 - 3.12 verify everything is working as expected with regards to those changes. Note that Docker builds are still running Python 3.11 as cx_Freeze still does not officially support Python 3.12. * Added infrastructure to test multiple versions of Python against the code base. This requires you to run: * ``pip install -U hatch hatchling`` -- Install prerequisites * ``hatch run docker:create X.Y.Z`` -- where ``X.Y.Z`` is an ES version on Docker Hub * ``hatch run test:pytest`` -- Run the test suite for each supported version of Python * ``hatch run docker:destroy`` -- Cleanup the Docker containers created in ``docker:create`` **Bugfix** * A bug reported in ``es_client`` with Python versions 3.8 and 3.9 has been addressed. Going forward, testing protocol will be to ensure that Curator works with all supported versions of Python, or support will be removed (when 3.8 is EOL, for example). **Changes** * Address deprecation warning in ``get_alias()`` call by limiting indices to only open and closed indices via ``expand_wildcards=['open', 'closed']``. * Address test warnings for an improperly escaped ``\d`` in a docstring in ``indexlist.py`` * Updated Python version in Docker build. See Dockerfile for more information. * Docker test scripts updated to make Hatch matrix testing easier (.env file) 8.0.14 (2 April 2024) --------------------- **Announcement** * A long awaited feature has been added, stealthily. It's fully in the documentation, but I do not yet plan to make a big announcement about it. In actions that search through indices, you can now specify a ``search_pattern`` to limit the number of indices that will be filtered. If no search pattern is specified, the behavior will be the same as it ever was: it will search through ``_all`` indices. The actions that support this option are: allocation, close, cold2frozen, delete_indices, forcemerge, index_settings, open, replicas, shrink, and snapshot. **Bugfix** * A mixup with naming conventions from the PII redacter tool got in the way of the cold2frozen action completing properly. **Changes** * Version bump: ``es_client==8.13.0`` * With the version bump to ``es_client`` comes a necessary change to calls to create a repository. In https://github.com/elastic/elasticsearch-specification/pull/2255 it became clear that using ``type`` and ``settings`` as it has been was insufficient for repository settings, so we go back to using a request ``body`` as in older times. This change affects ``esrepomgr`` in one place, and otherwise only in snapshot/restore testing. * Added the curator.helpers.getters.meta_getter to reduce near duplicate functions. * Changed curator.helpers.getters.get_indices to use the _cat API to pull indices. The primary driver for this is that it avoids pulling in the full mapping and index settings when all we really need to return is a list of index names. This should help keep memory from ballooning quite as much. The function also now allows for a search_pattern kwarg to search only for indices matching a pattern. This will also potentially make the initial index return list much smaller, and the list of indices needing to be filtered that much smaller. * Tests were added to ensure that the changes for ``get_indices`` work everywhere. * Tests were added to ensure that the new ``search_pattern`` did not break anything, and does behave as expected. 8.0.13 (26 March 2024) ---------------------- **Bugfix** * An issue was discovered in ``es_client`` that caused default values from command-line options which should not have been set to override settings in configuration files. ``es_client==8.12.9`` corrects this. Reported in #1708, hat tip to @rgaduput for reporting this bug. 8.0.12 (20 March 2024) ---------------------- **Bugfix** * ``six`` dependency erroneously removed from ``es_client``. It's back in ``es_client==8.12.8`` 8.0.11 (20 March 2024) ---------------------- **Announcement** * With the advent of ``es_client==8.12.5``, environment variables can now be used to automatically populate command-line options. The ``ESCLIENT_`` prefix just needs to prepend the capitalized option name, and any hyphens need to be replaced by underscores. ``--http-compress True`` is automatically settable by having ``ESCLIENT_HTTP_COMPRESS=1``. Boolean values are 1, 0, True, or False (case-insensitive). Options like ``hosts`` which can have multiple values just need to have whitespace between the values, e.g. ``ESCLIENT_HOSTS='http://127.0.0.1:9200 http://localhost:9200'``. It splits perfectly. This is tremendous news for the containerization/k8s community. You won't have to have all of the options spelled out any more. Just have the environment variables assigned. * Also, log blacklisting has made it to the command-line as well. It similarly can be set via environment variable, e.g. ``ESCLIENT_BLACKLIST='elastic_transport urllib3'``, or by multiple ``--blacklist`` entries at the command line. * ``es_client`` has simplified things such that I can clean up arg sprawl in the command line scripts. **Changes** Lots of pending pull requests have been merged. Thank you to the community members who took the time to contribute to Curator's code. * DOCFIX - Update date math section to use ``y`` instead of ``Y`` (#1510) * DOCFIX - Update period filtertype description (#1550) * add .dockerignore to increase build speed (#1604) * DOCFIX - clarification on prefix and suffix kinds (#1558) The provided documentation was adapted and edited. * Use builtin unittest.mock (#1695) * Had to also update ``helpers.testers.verify_client_object``. * Display proper error when mapping incorrect (#1526) - @namreg Also assisting with this is @alexhornblake in #1537 Apologies for needing to adapt the code manually since it's been so long. * Version bumps: * ``es_client==8.12.6`` 8.0.10 (1 February 2024) ------------------------ **Changes** The upstream dependency, ``es_client``, needed to be patched to address a Docker logging permission issue. This release only version bumps that dependency: * ``es_client==8.12.4`` 8.0.9 (31 January 2024) ----------------------- **Announcements** Curator is improving command-line options using new defaults and helpers from from the ``es_client`` module. This will make things appear a bit cleaner at the command-line as well as normalize command-line structure between projects using ``es_client``. No more reimplementing the same code in 5 different projects! **Changes** * Fix Docker logging per #1694. It should detect whether that path exists and that the process has write permissions before blindly attempting to use it. * If ``--config`` is not specified Curator will now assume you either mean to use CLI options exclusively or look for a config in the default location. Curator will not halt on the absence of ``--config`` any more, per #1698 * Increment Dockerfile settings to ``python:3.11.7-alpine3.18`` * Some command-line options are hidden by default now but remain usable. The help output explains how to see the full list, if needed. * Dependency bumps * As ``es_client`` covers all of the same upstream dependencies that were necessary in previous releases, all local dependencies have been erased in favor of that one. For this release, that is ``es_client==8.12.3`` 8.0.8 (21 July 2023) -------------------- **Announcements** Small change to further reduce memory usage by not creating unused data structures. This revealed a glitch with dry-runs that would eventually have been reported. **Changes** * Don't populate IndexList.index_info until required by a filter. In other words, stop populating the zero values as part of instantiation. * This uncovered an oversight with the 8.0.7 release. Certain actions, if taken with no filters, or only a pattern filter, would never ever populate the index_info. This wasn't a huge problem, unless you were using the dry-run flag. The dry-run output requires the index_info to be present for each index. In 8.0.7, where the empty structure was already in place, a dry-run wouldn't fail, but it also wouldn't show any data. This is fixed. * A few tests also needed updating. They were using contrived scenarios to test certain conditions. Now these tests are manually grabbing necessary metadata so they can pass. 8.0.7 (21 July 2023) -------------------- **Announcements** Functionally, there are no changes in this release. However... This release ends the practice of collecting all stats and metadata at IndexList initiation. This should make execution much faster for users with enormous clusters with hundreds to thousands of indices. In the past, this was handled at IndexList instantiation by making a cluster state API call. This is rather heavy, and can be so much data as to slow down Curator for minutes on clusters with hundreds to thousands of shards. This is all changed in this release. For example, the pattern filter requires no index metadata, as it only works against the index name. If you use a pattern filter first, the actionable list of indices is reduced. Then if you need to filter based on age using the ``creation_date``, the age filter will call ``get_index_settings`` to pull the necessary data for that filter to complete. Some filters will not work against closed indices. Those filters will automatically call ``get_index_state`` to get the open/close status of the indices in the actionable list. The disk space filter will require the index state as it won't work on closed indices, and will call ``get_index_stats`` to pull the size_in_bytes stats. Additionally, the cat API is used to get index state (open/close), now, as it is the only API call besides the cluster state which can inform on this matter. Not calling for a huge dump of the entire cluster state should drastically reduce memory requirements, though that may vary for some users still after all of the index data is polled, depending on what filters are used. There is a potential caveat to all this rejoicing, however. Searchable snapshot behavior with ILM policies usually keeps indices out of Curator's vision. You need to manually tell Curator to allow it to work on ILM enabled indices. But for some users who need to restore a snapshot to remove PII or other data from an index, it can't be in ILM anymore. This has caused some headaches. For example, if you are tracking an index in the hot tier named 'index1' and it is in process of being migrated to the cold tier as a searchable snapshot, it may suddenly disappear from the system as 'index1' and suddenly re-appear as 'restored-index1'. The original index may now be an alias that points to the newly mounted cold-tier index. Before this version, Curator would choke if it encountered this scenario. In fact, one user saw it repeatedly. See the last comment of issue 1682 in the GitHub repository for more information. To combat this, many repeated checks for index integrity have become necessary. This also involves verifying that indices in the IndexList are not actually aliases. Elasticsearch provides ``exists`` tests for these, but they cannot be performed in bulk. They are, however, very lightweight. But network turnaround times could make large clusters slower. For this reason, it is highly recommended that regex filters be used first, early, and often, before using any other filters. This will reduce the number of indices Curator has to check and/or verify during execution, which will speed things up drastically. 8.0.6.post1 (18 July 2023) -------------------------- **Breakfix Patch** No code was changed in this release, only Python dependencies. If you are using ``pip`` to install Curator, chances are good you won't need this release. This release was necessary after Docker refused to build a viable container using PyYAML 6.0.0, which will not build with the new Cython 3, released on Friday, July 14, 2023. A speedy fix was released as PyYAML 6.0.1 to address this. The current 8.0.6 Docker image uses these fixes. This version will be published to PyPI, but not otherwise released as its own version. 8.0.6 (18 July 2023) -------------------- **Breakfix Release** * Small breakfix change to catch a similar rare race condition patched in 8.0.5 covering the ``get_index_stats()`` method of IndexList. This patch covers the ``get_metadata()`` method and closes #1682. 8.0.5 (13 July 2023) -------------------- **Announcements** Release for Elasticsearch 8.8.2 **Changes** * Small PEP formatting changes that were found editing code. * Bump Python version in Dockerfile to 3.11.4 * Bump Python dependency versions. * Change ``targetName`` to ``target_name`` in ``setup.py`` for newest version of cx_Freeze. Hat tip to ``@rene-dekker`` in #1681 who made these changes to 5.x and 7.x. * Fix command-line behavior to not fail if the default config file is not present. The newer CLI-based configuration should allow for no config file at all, and now that's fixed. * Initial work done to prevent a race condition where an index is present at IndexList initialization, but is missing by the time index stats collection begins. The resultant 404s were causing Curator to shut down and not complete steps. * When running in a Docker container, make Curator log to ``/proc/1/fd/1`` by default, if no value is provided for ``logfile`` (otherwise, use that). 8.0.4 (28 April 2023) --------------------- **Announcements** Allow single-string, base64 API Key tokens in Curator. To use a base64 API Key token in YAML configuration: :: elasticsearch: client: hosts: https://host.example.tld:9243 other_args: api_key: token: '' To use a base64 API Key token at the command-line: :: curator --hosts https://host.example.tld:9243 --api_token [OTHER ARGS/OPTIONS] **NOTE:** In neither of the above examples are the alligator clips necessary (the ``<`` and ``>`` characters). **Changes** * Update ``es_client`` to 8.7.0, which enables the use of the base64 encoded API Key token. This also fixes #1671 via https://github.com/untergeek/es_client/issues/33 8.0.3 (22 February 2023) ------------------------ **Announcements** A new action called ``cold2frozen`` has been added to Curator. It is not going to be of much use to the vast majority of Elasticsearch users as it serves a very narrow use-case. That is, it migrates searchable snapshot indices from the cold tier to the frozen tier, but only if they are not associated with ILM (Index Lifecycle Management) policies. As escalation into the cold and frozen tiers is usually handled by ILM, this is indeed a rare use case. **Changes** * Fixed instruction display for delete repository action of ``es_repo_mgr`` * Fix unit tests to import more specifically/cleanly * Fixed Hatch build includes (this was speed-released to PyPI as 8.0.2.post1) as Curator did not function after a pip install. * Added ``cold2frozen`` action, and tests. 8.0.2 (15 February 2023) ------------------------ **Changes** * Added the same CLI flags that the singletons offers. This gives much more flexibility with regards to passing configuration settings as command-line options, particularly for Docker. * Re-created the ``get_client`` function. It now resides in ``curator.helpers.getters`` and will eventually see use in the Reindex class for remote connections. * Created a new set of classes to import, validate the schema, and split individual actions into their own sub-object instances. This is primarily to make ``curator/cli.py`` read much more cleanly. No new functionality here, but fewer conditional branches, and hopefully more readable code. * Updated the documentation to show these changes, both the API and the Elastic.co usage docs. 8.0.1 (10 February 2023) ------------------------ **Announcements** The 8.0.0 release was about getting Curator out the door with all of the functionality users were accustomed to in 5.8.4, but with the newer, updated args and methods in ``elasticsearch8``. Very little else was changed that didn't need to be. Now comes a few improvements, and more are coming, which is why I didn't start with 8.6.0 as my release version. * Now offering multi-architecture Docker builds for ``arm64`` (``v8``) and ``amd64``. * This required the addition of two new scripts at the root level of the project: ``alpine4docker.sh`` and ``post4docker.py``. These scripts are used only when building the Dockerfile. They were needed to make multi-architecture Docker images possible. I'm sure you'll be able to see how they work with a cursory glance. **Breaking Changes** * I split ``curator.utils`` into several, separate modules under ``curator.helpers``. I suppose, technically, that this qualifies as a breaking change from 8.0, but I sincerely doubt I have any users using Curator as an API yet, so I made the change. No functions were renamed, so this isn't as breaking so much as a slight shift in module naming. This gave me headaches, but it needed to be done a long time ago. It was always grating to see the Pylint warnings that the file is longer than 1000 lines, and searching for the module you wanted was way too much scrolling. This also gave me the chance to update the tests and the docstring's formatting for rST docs. Most of this release's changes came from this change. **Changes** * Curator has supported ECS logging for a while, but now that there is an official Python module, Curator is going to use it. Welcome, ``ecs-logging``! As before, just use ``logformat: ecs``, but now it has all of the goodness right there! * rST docs are improved and updated. Check out https://curator.readthedocs.io to see. * Logging turned out to be too verbose due to a shift. Now the ``blacklist`` defaults to ``['elastic_transport', 'urllib3']``. Documentation updated accordingly. * Default behavior is now to not verify snapshot repository access for Snapshot and Restore actions. It was a hacky fix for older versions of Elasticsearch that just shouldn't be needed. 8.0.0 (31 January 2023) ----------------------- **Announcement** This release is a *major* refactoring of the Curator code to work with both Elasticsearch 8.x and the Elasticsearch-py Python module of the same major and minor versions. I apologize for the crazy merge messes trying to get this all to work. In the end, I had to delete my fork on github and start over clean. **Breaking Changes** * Curator is now version locked. Curator v8.x will only work with Elasticsearch v8.x * Your old Curator ``config.yml`` file will no longer work as written. There have been more than a few changes necessitated by the updates in the ``elasticsearch8`` Python client library. The client connection code has also been extracted to its own module, ``es_client``. This is actually a good thing, however, as new options for configuring the client connection become possible. * Going forward, Curator will only be released as a tarball via GitHub, as an ``sdist`` or ``wheel`` via ``pip`` on PyPI, and to Docker Hub. There will no longer be RPM, DEB, or Windows ZIP releases. I am sorry if this is inconvenient, but one of the reasons the development and release cycle was delayed so long is because of how painfully difficult it was to do releases. * Curator will only work with Python 3.8+, and will more tightly follow the Python version releases. **Changes** * Last minute doc fixes. Mostly updated links to Elasticsearch documentation. * Python 3.11.1 is fully supported, and all versions of Python 3.8+ should be fully supported. * Use ``hatch`` and ``hatchling`` for package building & publishing * Because of ``hatch`` and ``pyproject.toml``, the release version still only needs to be tracked in ``curator/_version.py``. * Maintain the barest ``setup.py`` for building a binary version of Curator for Docker using ``cx_Freeze``. * Remove ``setup.cfg``, ``requirements.txt``, ``MANIFEST.in``, and other files as functionality is now handled by ``pyproject.toml`` and doing ``pip install .`` to grab dependencies and install them. YAY! Only one place to track dependencies now!!! * Preliminarily updated the docs. * Migrate towards ``pytest`` and away from ``nose`` tests. * Revamped almost every integration test * Scripts provided now that aid in producing and destroying Docker containers for testing. See ``docker_test/scripts/create.sh``. To spin up a numbered version release of Elasticsearch, run ``docker_test/scripts/create.sh 8.6.1``. It will download any necessary images, launch them, and tell you when it's ready, as well as provide ``REMOTE_ES_SERVER`` environment variables for testing the ``reindex`` action, e.g. ``REMOTE_ES_SERVER="http://172.16.0.1:9201" pytest --cov=curator``. These tests are skipped if this value is not provided. To clean up afterwards, run ``docker_test/scripts/destroy.sh`` * The action classes were broken into their own path, ``curator/actions/filename.py``. * ``curator_cli`` has been updated with more client connection settings, like ``cloud_id``. * As Curator 8 is version locked and will not use AWS credentials to connect to any ES 8.x instance, all AWS ES connection settings and references have been removed. 8.0.0rc1 (30 January 2023) -------------------------- **Announcement** This release-candidate is a *major* refactoring of the Curator code to work with both Elasticsearch 8.x and the Elasticsearch-py Python module of the same major and minor versions. **Breaking Changes** * Curator is now version locked. Curator v8.x will only work with Elasticsearch v8.x * Your old Curator ``config.yml`` file will no longer work as written. There have been more than a few changes necessitated by the updates in the ``elasticsearch8`` Python client library. The client connection code has also been extracted to its own module, ``es_client``. This is actually a good thing, however, as new options for configuring the client connection become possible. * Going forward, Curator will only be released as a tarball via GitHub, as an ``sdist`` or ``wheel`` via ``pip`` on PyPI, and to Docker Hub. There will no longer be RPM, DEB, or Windows ZIP releases. I am sorry if this is inconvenient, but one of the reasons the development and release cycle was delayed so long is because of how painfully difficult it was to do releases. * Curator will only work with Python 3.8+, and will more tightly follow the Python version releases. **Changes** * Python 3.11.1 is fully supported, and all versions of Python 3.8+ should be fully supported. * Use ``hatch`` and ``hatchling`` for package building & publishing * Because of ``hatch`` and ``pyproject.toml``, the release version still only needs to be tracked in ``curator/_version.py``. * Maintain the barest ``setup.py`` for building a binary version of Curator for Docker using ``cx_Freeze``. * Remove ``setup.cfg``, ``requirements.txt``, ``MANIFEST.in``, and other files as functionality is now handled by ``pyproject.toml`` and doing ``pip install .`` to grab dependencies and install them. YAY! Only one place to track dependencies now!!! * Preliminarily updated the docs. * Migrate towards ``pytest`` and away from ``nose`` tests. * Revamped almost every integration test * Scripts provided now that aid in producing and destroying Docker containers for testing. See ``docker_test/scripts/create.sh``. To spin up a numbered version release of Elasticsearch, run ``docker_test/scripts/create.sh 8.6.1``. It will download any necessary images, launch them, and tell you when it's ready, as well as provide ``REMOTE_ES_SERVER`` environment variables for testing the ``reindex`` action, e.g. ``REMOTE_ES_SERVER="http://172.16.0.1:9201" pytest --cov=curator``. These tests are skipped if this value is not provided. To clean up afterwards, run ``docker_test/scripts/destroy.sh`` * The action classes were broken into their own path, ``curator/actions/filename.py``. * ``curator_cli`` has been updated with more client connection settings, like ``cloud_id``. * As Curator 8 is version locked and will not use AWS credentials to connect to any ES 8.x instance, all AWS ES connection settings and references have been removed. 8.0.0a1 (26 January 2023) ------------------------- **Announcement** This release-candidate is a *major* refactoring of the Curator code to work with both Elasticsearch 8.x and the Elasticsearch-py Python module of the same major and minor versions. **Breaking Changes** * Curator is now version locked. Curator v8.x will only work with Elasticsearch v8.x * Your old Curator ``config.yml`` file will no longer work as written. There have been more than a few changes necessitated by the updates in the ``elasticsearch8`` Python client library. The client connection code has also been extracted to its own module, ``es_client``. This is actually a good thing, however, as new options for configuring the client connection become possible. * Going forward, Curator will only be released as a tarball via GitHub, as an ``sdist`` or ``wheel`` via ``pip`` on PyPI, and to Docker Hub. There will no longer be RPM, DEB, or Windows ZIP releases. I am sorry if this is inconvenient, but one of the reasons the development and release cycle was delayed so long is because of how painfully difficult it was to do releases. * Curator will only work with Python 3.8+, and will more tightly follow the Python version releases. **Changes** * Python 3.11.1 is fully supported, and all versions of Python 3.8+ should be fully supported. * Use ``hatch`` and ``hatchling`` for package building & publishing * Because of ``hatch`` and ``pyproject.toml``, the release version still only needs to be tracked in ``curator/_version.py``. * Maintain the barest ``setup.py`` for building a binary version of Curator for Docker using ``cx_Freeze``. * Remove ``setup.cfg``, ``requirements.txt``, ``MANIFEST.in``, and other files as functionality is now handled by ``pyproject.toml`` and doing ``pip install .`` to grab dependencies and install them. YAY! Only one place to track dependencies now!!! * Preliminarily updated the docs. * Migrate towards ``pytest`` and away from ``nose`` tests. * Revamped almost every integration test * Scripts provided now that aid in producing and destroying Docker containers for testing. See ``docker_test/scripts/create.sh``. To spin up a numbered version release of Elasticsearch, run ``docker_test/scripts/create.sh 8.6.1``. It will download any necessary images, launch them, and tell you when it's ready, as well as provide ``REMOTE_ES_SERVER`` environment variables for testing the ``reindex`` action, e.g. ``REMOTE_ES_SERVER="http://172.16.0.1:9201" pytest --cov=curator``. These tests are skipped if this value is not provided. To clean up afterwards, run ``docker_test/scripts/destroy.sh`` * The action classes were broken into their own path, ``curator/actions/filename.py``. * ``curator_cli`` has been updated with more client connection settings, like ``cloud_id``. * As Curator 8 is version locked and will not use AWS credentials to connect to any ES 8.x instance, all AWS ES connection settings and references have been removed. 7.0.0 (31 January 2023) ----------------------- **Announcement** * This release is a simplified release for only ``pip`` and Docker. It only works with Elasticsearch 7.x and is functionally identical to 5.8.4 **Breaking Changes** * Curator is now version locked. Curator v7.x will only work with Elasticsearch v7.x * Going forward, Curator will only be released as a tarball via GitHub, as an ``sdist`` or ``wheel`` via ``pip`` on PyPI, and to Docker Hub. There will no longer be RPM, DEB, or Windows ZIP releases. I am sorry if this is inconvenient, but one of the reasons the development and release cycle was delayed so long is because of how painfully difficult it was to do releases. * Curator will only work with Python 3.8+, and will more tightly follow the Python version releases. **New** * Python 3.11.1 is fully supported, and all versions of Python 3.8+ should be fully supported. * Use ``hatch`` and ``hatchling`` for package building & publishing * Because of ``hatch`` and ``pyproject.toml``, the release version still only needs to be tracked in ``curator/_version.py``. * Maintain the barest ``setup.py`` for building a binary version of Curator for Docker using ``cx_Freeze``. * Remove ``setup.cfg``, ``requirements.txt``, ``MANIFEST.in``, and other files as functionality is now handled by ``pyproject.toml`` and doing ``pip install .`` to grab dependencies and install them. YAY! Only one place to track dependencies now!!! * Preliminarily updated the docs. * Migrate towards ``pytest`` and away from ``nose`` tests. * Scripts provided now that aid in producing and destroying Docker containers for testing. See ``docker_test/scripts/create.sh``. To spin up a numbered version release of Elasticsearch, run ``docker_test/scripts/create.sh 7.17.8``. It will download any necessary images, launch them, and tell you when it's ready, as well as provide ``REMOTE_ES_SERVER`` environment variables for testing the ``reindex`` action, e.g. ``REMOTE_ES_SERVER="172.16.0.1:9201" pytest --cov=curator``. These tests are skipped if this value is not provided. To clean up afterwards, run ``docker_test/scripts/destroy.sh`` * Add filter by size feature. #1612 (IndraGunawan) * Update Elasticsearch client to 7.17.8 **Security Fixes** * Use `urllib3` 1.26.5 or higher #1610 (tsaarni) — This dependency is now fully handled by the ``elasticsearch7`` module and not a separate ``urllib3`` import. 6.0.0 (31 January 2023) ----------------------- **Announcement** * This release is a simplified release for only ``pip`` and Docker. It only works with Elasticsearch 6.x and is functionally identical to 5.8.4 **Breaking Changes** * Curator is now version locked. Curator v6.x will only work with Elasticsearch v6.x * Going forward, Curator will only be released as a tarball via GitHub, as an ``sdist`` or ``wheel`` via ``pip`` on PyPI, and to Docker Hub. There will no longer be RPM, DEB, or Windows ZIP releases. I am sorry if this is inconvenient, but one of the reasons the development and release cycle was delayed so long is because of how painfully difficult it was to do releases. * Curator will only work with Python 3.8+, and will more tightly follow the Python version releases. **New** * Python 3.11.1 is fully supported, and all versions of Python 3.8+ should be fully supported. * Use ``hatch`` and ``hatchling`` for package building & publishing * Because of ``hatch`` and ``pyproject.toml``, the release version still only needs to be tracked in ``curator/_version.py``. * Maintain the barest ``setup.py`` for building a binary version of Curator for Docker using ``cx_Freeze``. * Remove ``setup.cfg``, ``requirements.txt``, ``MANIFEST.in``, and other files as functionality is now handled by ``pyproject.toml`` and doing ``pip install .`` to grab dependencies and install them. YAY! Only one place to track dependencies now!!! * Preliminarily updated the docs. * Migrate towards ``pytest`` and away from ``nose`` tests. * Scripts provided now that aid in producing and destroying Docker containers for testing. See ``docker_test/scripts/create.sh``. To spin up a numbered version release of Elasticsearch, run ``docker_test/scripts/create.sh 6.8.23``. It will download any necessary images, launch them, and tell you when it's ready, as well as provide ``REMOTE_ES_SERVER`` environment variables for testing the ``reindex`` action, e.g. ``REMOTE_ES_SERVER="172.16.0.1:9201" pytest --cov=curator``. These tests are skipped if this value is not provided. To clean up afterwards, run ``docker_test/scripts/destroy.sh`` * Add filter by size feature. #1612 (IndraGunawan) * Update Elasticsearch client to 6.8.2 **Security Fixes** * Use `urllib3` 1.26.5 or higher #1610 (tsaarni) — This dependency is now fully handled by the ``elasticsearch7`` module and not a separate ``urllib3`` import. 5.8.4 (27 April 2021) --------------------- **Announcement** * Because Python 2.7 has been EOL for over a year now, many projects are no longer supporting it. This will also be the case for Curator as its dependencies cease to support Python 2.7. With `boto3` having announced it is ceasing support of Python 2.7, deprecated as of 15 Jan 2021, and fully unsupported on 15 Jul 2021, Curator will follow these same dates. This means that you will need to use an older version of Curator to continue using Python 2.7, or upgrade to Python 3.6 or greater. **Breaking** * Normally I would not include breaking changes, but users have asked for Click v7, which changes actions to require hyphens, and not underscores. Options can still have underscores, but actions can't--well, not strictly true. You can have underscores, but Click v7 will convert them to hyphens. This should _only_ affect users of the Curator CLI, and not YAML file users, and only the actions: `show-indices`, `show-snapshots`, `delete-indices`, `delete-snapshots`. The actual actions are still named with underscores, and the code has been updated to work with the hyphenated action names. **New** * Now using `elasticsearch-py` version 7.12.0 * Adding testing for Python 3.9 * Removing testing on Python 3.6 * Tested Elasticsearch versions now include 7.12.0, 7.11.2, 7.10.2, 7.9.3, 7.8.1, 6.8.15, 5.6.16 * Changing `requirements.txt` as follows: - boto3-1.17.57 - certifi-2020.12.5 - click-7.1.2 - elasticsearch-7.12.0 - pyyaml-5.4.1 - requests-2.25.1 - requests-aws4auth-1.0.1 - six-1.15.0 - urllib3-1.26.4 - voluptuous-0.12.1 **Bug Fixes** * Alias integration tests needed updating for newer versions of Elasticsearch that include ILM. * Click 7.0 now reports an exit code of `1` for schema mismatches where it yielded a `-1` in the past. Tests needed updating to correct for this. **Security** * Address multiple `pyyaml` vulnerabilities by bumping to version 5.4.1. Contributed in #1596 (tsaarni) 5.8.3 (25 November 2020) ------------------------ **New** * Determined to test the last 2 major version's final patch releases, plus the last 5 minor releases in the current major version. Travis CI testing needs to go faster, and this should suffice. For now, this means versions 5.6.16, 6.8.13, 7.6.2, 7.7.1, 7.8.1, 7.9.3, and 7.10.0 **Bug Fixes** * Caught a few stale merge failures, and asciidoc documentation problems which needed fixing in the 5.8 branch, which necessitate this tiny bump release. No code changes between 5.8.2 and 5.8.3. 5.8.2 (24 November 2020) ------------------------ **Announcement** * No, Curator isn't going away. But as you can tell, it's not as actively developed as it once was. I am gratified to find there are still users who make it a part of their workflow. I intend to continue development in my spare time. Curator is now a labor of love, not an engineering project I do during work hours. **New** * Testing changes. Only last ES version of 5.x and 6.x are tested, plus the releases of 7.x since 7.2. * ``http_auth`` is now deprecated. You can continue to use it, but it will go away in the next major release. Moving forward, you should use ``username`` and ``password``. This should work in ``curator``, ``curator_cli``, and ``es_repo_mgr``. * Removed tests for all 5.x branches of Elasticsearch but the final (5.6). * Added tests for missing 7.x branches of Elasticsearch * Remove tests for Python 3.5 * Fix hang of Shrink action in ES 7.x in #1528 (jclegras) * Add ``ecs`` as a ``logformat`` option in #1529 (m1keil) **Bug Fixes** * Lots of code cleanup, trying to go PEP-8. All tests are still passing, and the APIs are not changed (yet—-that comes in the next major release). * Dockerfile has been updated to produce a working version with Python 3.7 and Curator 5.8.1 * Pin (for now) Elasticsearch Python module to 7.1.0. This will be updated when an updated release of the module fixes the `cluster.state` API call regression at https://github.com/elastic/elasticsearch-py/issues/1141 * Fix ``client.tasks.get`` API call to be ``client.tasks.list`` when no index name is provided. See https://github.com/elastic/elasticsearch-py/issues/1110 * Pin some pip versions to allow urllib3 and boto to coexist. See #1562 (sethmlarson). **Documentation** * Add Freeze/Unfreeze documentation in #1497 (lucabelluccini) * Update compatibility matrix in #1522 (jibsonline) 5.8.1 (25 September 2019) ------------------------- **Bug Fixes** * ``LD_LIBRARY_PATH`` will now be set in ``/usr/bin/curator`` and the associated scripts rather than set in ``/etc/ld.so.conf.d`` **Other** * Unsaved logging change in ``utils.py`` that got missed is merged. 5.8.0 (24 September 2019) ------------------------- **New** * Require ``elasticsearch-py`` version 7.0.4 * Official support for Python 3.7 — In fact, the pre-built packages are built using Python 3.7 now. * Packages bundle OpenSSL 1.1.1c, removing the need for system OpenSSL * Certifi 2019.9.11 certificates included. * New client configuration option: api_key - used in the X-Api-key header in requests to Elasticsearch when set, which may be required if ReadonlyREST plugin is configured to require api-key. Requested in #1409 (vetler) * Add ``skip_flush`` option to the ``close`` action. This should be useful when trying to close indices with unassigned shards (e.g. before restore). Raised in #1412. (psypuff) * Use ``RequestsHttpConnection`` class, which permits the use of ``HTTP_PROXY`` and ``HTTPS_PROXY`` environment variables. Raised in #510 and addressed by #1259 (raynigon) in August of 2018. Subsequent changes, however, required some adaptation, and re-submission as a different PR. (untergeek) * ``ignore_existing`` option added to ``CreateIndex``. Will not raise an error if the index to be created already exists. Raised by (breml) in #1352. (untergeek) * Add support for ``freeze`` and ``unfreeze`` indexes using curator. Requires Elasticsearch version 6.6 or greater with xpack enabled. Requested in issue #1399 and rasied in PR #1454. (junmuz) * Allow the ``close`` action to ignore synced flush failures with the new ``ignore_sync_failures`` option. Raised in #1248. (untergeek) **Bug Fixes** * Fix kibana filter to match any and all indices starting with ``.kibana``. This addresses #1363, and everyone else upgrading to Elasticsearch 7.x. Update documentation accordingly. (untergeek) * Fix reindex post-action checks. When the filters do not return documents to be reindexed, the post-action check to ensure the target index exists is not needed. This new version will skip that validation if no documents are processed (issue #1170). (afharo) * Prevent the ``empty`` filtertype from incorrectly matching against closed indices #1430 (heyitsmdr) * Fix ``index_size`` function to be able to report either for either the ``total`` of all shards (default) or just ``primaries``. Added as a keyword arg to preserve existing behavior. This was needed to fix sizing calculations for the Shrink action, which should only count ``primaries``. Raised in #1429 (untergeek). * Fix ``allow_ilm_indices`` to work with the ``rollover`` action. Reported in #1418 (untergeek) * Update the client connection logic to be cleaner and log more verbosely in an attempt to address issues like #1418 and others like it more effectively as other failures have appeared to be client failures because the last log message were vague indications that a client connection was attempted. This is a step in the right direction, as it explicitly exits with a 1 exit code for different conditions now. (untergeek) * Catch snapshots without a timestring in the name causing a logic error when using the ``count`` filter and ``use_age`` with ``source: name``. Reported by (nerophon) in #1366. (untergeek) * Ensure authentication (401), authorization (403), and other 400 errors are logged properly. Reported by (rfalke) in #1413. (untergeek) * Fix crashes in restore of "large" number of indices reported by breml in #1360. (anandsinghkunwar) * Do an empty list check before querying indices for field stats. Fixed by (CiXiHuo) in #1448. * Fix "Correctly report task runtime in seconds" while reindexing. Reported by (jkelastic) in #1335 **Documentation** * Grammar correction of ilm.asciidoc #1425 (SlavikCA) * Updates to reflect changes to Elasticsearch 7 documentation #1426 and #1428 (lcawl) and (jrodewig) 5.7.6 (6 May 2019) ------------------ **Security Fix** Evidently, there were some upstream dependencies which required vulnerable versions of ``urllib3`` and ``requests``. These have been addressed. * CVE-2018-20060, CVE-2019-11324, CVE-2018-18074 are addressed by this update. Fixed in #1395 (cburgess) **Bug Fixes** * Allow aliases in Elasticsearch versions >= 6.5.0 to refer to more than one index, if ``is_write_index`` is present and one index has it set to `True`. Requested in #1342 (untergeek) 5.7.5 (26 April 2019) --------------------- This has to be a new record with 5 releases in 3 days, however, as a wonderful aside, this release is the Curator Haiku release (if you don't know why, look up the structure of a Haiku). **Bug Fix** * Persistent ILM filter error has finally been caught. Apparently, in Python, a list of lists ``[[]]`` will evaluate as existing, because it has one array element, even if that element is empty. So, this is my bad, but it is fixed now. (untergeek) 5.7.4 (25 April 2019) --------------------- **Bug Fix** * ILM filter was reading from full index list, rather than the working list Reported in #1389 (untergeek) 5.7.3 (24 April 2019) --------------------- **Bug Fix** * Still further package collisions with ``urllib3`` between ``boto3`` and ``requests``. It was working, but with an unacceptable error, which is addressed in release 5.7.3. (untergeek) 5.7.2 (24 April 2019) --------------------- **Bug Fix** * Fix ``urllib3`` dependency collision on account of ``boto3`` (untergeek) 5.7.1 (24 April 2019) --------------------- We do not speak of 5.7.1 5.7.0 (24 April 2019) --------------------- **New** * Support for ``elasticsearch-py`` 7.0.0 (untergeek) * Support for Elasticsearch 7.0 #1371 (untergeek) * TravisCI testing for Elasticsearch 6.5, 6.6, 6.7, and 7.0 (untergeek) * Allow shrink action to use multiple data paths #1350 (IzekChen) **Bug Fixes** * Fix ``regex`` pattern filter to use ``re.search`` #1355 (matthewdupre) * Report rollover results in both dry-run and regular runs. Requested in #1313 (untergeek) * Hide passwords in DEBUG logs. Requested in #1336 (untergeek) * With ILM fully released, Curator tests now correctly use the ``allow_ilm_indices`` option. (untergeek) **Documentation** * Many thanks to those who submitted documentation fixes, both factual as well as typos! 5.6.0 (13 November 2018) ------------------------ **New** * The ``empty`` filter has been exposed for general use. This filter matches indices with no documents. (jrask) #1264 * Added tests for Elasticsearch 6.3 and 6.4 releases. (untergeek) * Sort indices alphabetically before sorting by age. (tschroeder-zendesk) #1290 * Add ``shards`` filtertype (cushind) #1298 **Bug Fixes** * Fix YAML linting so that YAML errors are caught and displayed on the command line. Reported in #1237 (untergeek) * Pin ``click`` version for compatibility. (Andrewsville) #1280 * Allow much older epoch timestamps (rsteneteg) #1296 * Reindex action respects ``ignore_empty_list`` flag (untergeek) #1297 * Update ILM index version minimum to 6.6.0 (untergeek) * Catch reindex failures properly. Reported in #1260 (untergeek) **Documentation** * Added Reindex example to the sidebar. (Nostalgiac) #1227 * Fix Rollover example text and typos. (untergeek) 5.5.4 (23 May 2018) ------------------- **Bug Fix** * Extra args in show.py prevented show_snapshots from executing (untergeek) 5.5.3 (21 May 2018) ------------------- Short release cycle here specifically to address the Snapshot restore issue raised in #1192 **Changes** * By default, filter out indices with ``index.lifecycle.name`` set. This can be overridden with the option ``allow_ilm_indices`` with the caveat that you are on your own if there are conflicts. NOTE: The Index Lifecycle Management feature will not appear in Elasticsearch until 6.4.0 * Removed some unused files from the repository. **Bug Fixes** * Fix an ambiguously designed Alias test (untergeek) * Snapshot action will now raise an exception if the snapshot does not complete with state ``SUCCESS``. Reported in #1192 (untergeek) * The show_indices and show_snapshots singletons were not working within the new framework. They've been fixed now. 5.5.2 (14 May 2018) ------------------- **Changes** * The ``alias``, ``restore``, ``rollover``, and ``shrink`` actions have been added to ``curator_cli``, along with a revamped method to manage/add actions in the future. * Updated ``certifi`` dependency to ``2018.4.16`` * Added ``six`` dependency * Permit the use of versions 6.1 and greater of the ``elasticsearch`` python module. There are issues with SSL contexts in the 6.0 release that prevent Curator from being able to use this version. Currently the requirement version string is ``elasticsearch>=5.5.2,!=6.0.0,<7.0.0`` * Start of pylint cleanup, and use of `six` `string_types`. (untergeek) **Bug Fixes** * `unit_count_pattern` setting can cause indices to mistakenly be included in an index filter. Fixed in #1206 (soenkeliebau) * Fix rollover _check_max_size() call. Reported in #1202 by @diranged (untergeek). * Update tested versions of Elasticsearch. (untergeek). * Update setup.cfg to install dependencies during source install. (untergeek) * Fix reference to unset variable name in log output at https://github.com/elastic/curator/blob/v5.5.1/curator/actions.py#L2145 It should be `idx` instead of `index`. (untergeek). * Alias action should raise `NoIndices` exception if `warn_if_no_indices` is `True`, and no `add` or `remove` sub-actions are found, rather than raising an `ActionError`. Reported in #1209 (untergeek). **Documentation** * Clarify inclusive filtering for allocated filter. Fixed in #1203 (geekpete) * Fix Kibana filter description. #1199 (quartett-opa) * Add missing documentation about the ``new_name`` option for rollover. Reported in #1197 (untergeek) 5.5.1 (22 March 2018) --------------------- **Bug Fixes** * Fix ``pip`` installation issues for older versions of Python #1183 (untergeek) 5.5.0 (21 March 2018) --------------------- **New Features** * Add ``wait_for_rebalance`` as an option for ``shrink`` action. By default the behavior remains unchanged. You can now set this to False though to allow the shrink action to only check that the index being shrunk has finished being relocated and it will not wait for the cluster to rebalance. #1129 (tschroeder-zendesk) * Work around for extremely large cluster states. #1142 (rewiko) * Add CI tests for Elasticsearch versions 6.1 and 6.2 (untergeek) * Add Elasticsearch datemath support for snapshot names #1078 (untergeek) * Support ``max_size`` as a rollover condition for Elasticsearch versions 6.1.0 and up. #1140 (untergeek) * Skip indices with a document count of 0 when using ``source: field_stats`` to do ``age`` or ``period`` type filtering. #1130 (untergeek) **Bug Fixes** * Fix missing node information in log line. #1142 (untergeek) * Fix default options in code that were causing schema validation errors after ``voluptuous`` upgrade to 0.11.1. Reported in #1149, fixed in #1156 (untergeek) * Disallow empty lists as reindex source. Raise exception if that happens. Reported in #1139 (untergeek) * Set a ``timeout_override`` for ``delete_snapshots`` to catch cases where slower repository network and/or disk access can cause a snapshot delete to take longer than the default 30 second client timeout. #1133 (untergeek) * Add AWS ES 5.1 support. #1172 (wanix) * Add missing ``period`` filter arguments for ``delete_snapshots``. Reported in #1173 (untergeek) * Fix kibana filtertype to catch newer index names. Reported in #1171 (untergeek) * Re-order the closed indices filter for the Replicas action to take place `before` the empty list check. Reported in #1180 by ``@agomerz`` (untergeek) **General** * Deprecate testing for Python 3.4. It is no longer being supported by Python. * Increase logging to show error when ``master_only`` is true and there are multiple hosts. **Documentation** * Correct a misunderstanding about the nature of rollover conditions. #1144 (untergeek) * Correct links to the field_stats API, as it is non-existent in Elasticsearch 6.x. (untergeek) * Add a warning about using forcemerge on active indices. #1153 (untergeek) * Fix select URLs in pip installation from source to not be 404 #1133 (untergeek) * Fix an error in regex filter documentation #1138 (arne-cl) 5.4.1 (6 December 2017) ----------------------- **Bug Fixes** * Improve Dockerfile to build from source and produce slimmer image #1111 (mikn) * Fix ``filter_kibana`` to correctly use ``exclude`` argument #1116 (cjuroz) * Fix `ssl_no_validate` behavior within AWS ES #1118 (igalarzab) * Improve command-line exception management #1119 (4383) * Make ``alias`` action always process ``remove`` before ``add`` to prevent undesired alias removals. #1120 (untergeek) **General** * Bump ES versions in Travis CI **Documentation** * Remove ``unit_count`` parameter doc for parameter that no longer exists #1107 (dashford) * Add missing ``exclude: True`` in ``timestring`` docs #1117 (GregMefford) 5.4.0 (13 November 2017) ------------------------ **Announcement** * Support for Elasticsearch 6.0!!! Yes! **New Features** * The ``field_stats`` API may be gone from Elasticsearch, but its utility cannot be denied. And so, Curator has replaced the ``field_stats`` API call with a small aggregation query. This will be perhaps a bit more costly in performance terms, as this small aggregation query must be made to each index in sequence, rather than as a one-shot call, like the ``field_stats`` API call. But the benefit will remain available, and it's the only major API that did not persevere between Elasticsearch 5.x and 6.x that was needed by Curator. 5.3.0 (31 October 2017) ----------------------- **New Features** * With the period filter and field_stats, it is useful to match indices that fit `within` the period, rather than just their start dates. This is now possible with ``intersect``. See more in the documentation. Requested in #1045. (untergeek) * Add a ``restore`` function to ``curator_cli`` singleton. Mentioned in #851 (alexef) * Add ``pattern`` to the ``count`` filter. This is particularly useful when working with rollover indices. Requested in #1044 (untergeek) * The ``es_repo_mgr create`` command now can take ``skip_repo_fs_check`` as an argument (default is False) #1072 (alexef) * Add ``pattern_type`` feature expansion to the ``period`` filter. The default behavior is ``pattern_type='relative'``, which preserves existing behaviors so users with existing configurations can continue to use them without interruption. The new ``pattern_type`` is ``absolute``, which allows you to specify hard dates for ``date_from`` and ``date_to``, while ``date_from_format`` and ``date_to_format`` are strftime strings to interpret the from and to dates. Requested in #1047 (untergeek) * Add ``copy_aliases`` option to the ``shrink`` action. So this option is only set in the ``shrink`` action. The default value of the option is ``copy_aliases: 'False'`` and it does nothing. If you set to ``copy_aliases: 'True'``, you could copy the aliases from the source index to the target index. Requested in #1060 (monkey3199) * IAM Credentials can now be retrieved from the environment using the Boto3 Credentials provider. #1084 (kobuskc) **Bug Fixes** * Delete the target index (if it exists) in the event that a shrink fails. Requested in #1058 (untergeek) * Fixed an integration test that could fail in the waning days of a month. * Fix build system anomalies for both unix and windows. **Documentation** * Set repository access to be https by default. * Add documentation for ``copy_aliases`` option. 5.2.0 (1 September 2017) ------------------------ **New Features** * Shrink action! Apologies to all who have patiently waited for this feature. It's been a long time coming, but it is hopefully worth the wait. There are a lot of checks and tests associated with this action, as there are many conditions that have to be met in order for a shrink to take place. Curator will try its best to ensure that all of these conditions are met so you can comfortably rest assured that shrink will work properly unattended. See the documentation for more information. * The ``cli`` function has been split into ``cli`` and ``run`` functions. The behavior of ``cli`` will be indistinguishable from previous releases, preserving API integrity. The new ``run`` function allows lambda and other users to `run` Curator from the API with only a client configuration file and action file as arguments. Requested in #1031 (untergeek) * Allow use of time/date string interpolation for Rollover index naming. Added in #1010 (tschroeder-zendesk) * New ``unit_count_pattern`` allows you to derive the ``unit_count`` from the index name itself. This involves regular expressions, so be sure to do lots of testing in ``--dry-run`` mode before deploying to production. Added by (soenkeliebau) in #997 **Bug Fixes** * Reindex ``request_body`` allows for 2 different ``size`` options. One limits the number of documents reindexed. The other is for batch sizing. The batch sizing option was missing from the schema validator. This has been corrected. Reported in #1038 (untergeek) * A few sundry logging and notification changes were made. 5.1.2 (08 August 2017) ---------------------- **Errata** * An update to Elasticsearch 5.5.0 changes the behavior of ``filter_by_aliases``, differing from previous 5.x versions. If a list of aliases is provided, indices must appear in `all` listed aliases or a 404 error will result, leading to no indices being matched. In older versions, if the index was associated with even one of the aliases in aliases, it would result in a match. Tests and documentation have been updated to address these changes. * Debian 9 changed SSL versions, which means that the pre-built debian packages no longer work in Debian 9. In the short term, this requires a new repository. In the long term, I will try to get a better repository system working for these so they all work together, better. Requested in #998 (untergeek) **Bug Fixes** * Support date math in reindex operations better. It did work previously, but would report failure because the test was looking for the index with that name from a list of indices, rather than letting Elasticsearch do the date math. Reported by DPattee in #1008 (untergeek) * Under rare circumstances, snapshot delete (or create) actions could fail, even when there were no snapshots in state ``IN_PROGRESS``. This was tracked down by JD557 as a collision with a previously deleted snapshot that hadn't finished deleting. It could be seen in the tasks API. An additional test for snapshot activity in the tasks API has been added to cover this scenario. Reported in #999 (untergeek) * The ``restore_check`` function did not work properly with wildcard index patterns. This has been rectified, and an integration test added to satisfy this. Reported in #989 (untergeek) * Make Curator report the Curator version, and not just reiterate the elasticsearch version when reporting version incompatibilities. Reported in #992. (untergeek) * Fix repository/snapshot name logging issue. #1005 (jpcarey) * Fix Windows build issue #1014 (untergeek) **Documentation** * Fix/improve rST API documentation. * Thanks to many users who not only found and reported documentation issues, but also submitted corrections. 5.1.1 (8 June 2017) ------------------- **Bug Fixes** * Mock and cx_Freeze don't play well together. Packages weren't working, so I reverted the string-based comparison as before. 5.1.0 (8 June 2017) ------------------- **New Features** * Index Settings are here! First requested as far back as #160, it's been requested in various forms culminating in #656. The official documentation addresses the usage. (untergeek) * Remote reindex now adds the ability to migrate from one cluster to another, preserving the index names, or optionally adding a prefix and/or a suffix. The official documentation shows you how. (untergeek) * Added support for naming rollover indices. #970 (jurajseffer) * Testing against ES 5.4.1, 5.3.3 **Bug Fixes** * Since Curator no longer supports old versions of python, convert tests to use ``isinstance``. #973 (untergeek) * Fix stray instance of ``is not`` comparison instead of ``!=`` #972 (untergeek) * Increase remote client timeout to 180 seconds for remote reindex. #930 (untergeek) **General** * elasticsearch-py dependency bumped to 5.4.0 * Added mock dependency due to isinstance and testing requirements * AWS ES 5.3 officially supports Curator now. Documentation has been updated to reflect this. 5.0.4 (16 May 2017) ------------------- **Bug Fixes** * The ``_recovery`` check needs to compare using ``!=`` instead of ``is not``, which apparently does not accurately compare unicode strings. Reported in #966 (untergeek) 5.0.3 (15 May 2017) ------------------- **Bug Fixes** * Restoring a snapshot on an exceptionally fast cluster/node can create a race condition where a ``_recovery`` check returns an empty dictionary ``{}``, which causes Curator to fail. Added test and code to correct this. Reported in #962. (untergeek) 5.0.2 (4 May 2017) ------------------ **Bug Fixes** * Nasty bug in schema validation fixed where boolean options or filter flags would validate as ``True`` if non-boolean types were submitted. Reported in #945. (untergeek) * Check for presence of alias after reindex, in case the reindex was to an alias. Reported in #941. (untergeek) * Fix an edge case where an index named with `1970.01.01` could not be sorted by index-name age. Reported in #951. (untergeek) * Update tests to include ES 5.3.2 * Bump certifi requirement to 2017.4.17. **Documentation** * Document substitute strftime symbols for doing ISO Week timestrings added in #932. (untergeek) * Document how to include file paths better. Fixes #944. (untergeek) 5.0.1 (10 April 2017) --------------------- **Bug Fixes** * Fixed default values for ``include_global_state`` on the restore action to be in line with defaults in Elasticsearch 5.3 **Documentation** * Huge improvement to documenation, with many more examples. * Address age filter limitations per #859 (untergeek) * Address date matching behavior better per #858 (untergeek) 5.0.0 (5 April 2017) -------------------- The full feature set of 5.0 (including alpha releases) is included here. **New Features** * Reindex is here! The new reindex action has a ton of flexibility. You can even reindex from remote locations, so long as the remote cluster is Elasticsearch 1.4 or newer. * Added the ``period`` filter (#733). This allows you to select indices or snapshots, based on whether they fit within a period of hours, days, weeks, months, or years. * Add dedicated "wait for completion" functionality. This supports health checks, recovery (restore) checks, snapshot checks, and operations which support the new tasks API. All actions which can use this have been refactored to take advantage of this. The benefit of this new feature is that client timeouts will be less likely to happen when performing long operations, like snapshot and restore. NOTE: There is one caveat: forceMerge does not support this, per the Elasticsearch API. A forceMerge call will hold the client until complete, or the client times out. There is no clean way around this that I can discern. * Elasticsearch date math naming is supported and documented for the ``create_index`` action. An integration test is included for validation. * Allow allocation action to unset a key/value pair by using an empty value. Requested in #906. (untergeek) * Added support for the Rollover API. Requested in #898, and by countless others. * Added ``warn_if_no_indices`` option for ``alias`` action in response to #883. Using this option will permit the ``alias`` add or remove to continue with a logged warning, even if the filters result in a ``NoIndices`` condition. Use with care. **General** * Bumped ``click`` (python module) version dependency to 6.7 * Bumped ``urllib3`` (python module) version dependency to 1.20 * Bumped ``elasticsearch`` (python module) version dependency to 5.3 * Refactored a ton of code to be cleaner and hopefully more consistent. **Bug Fixes** * Curator now logs version incompatibilities as an error, rather than just raising an Exception. #874 (untergeek) * The ``get_repository()`` function now properly raises an exception instead of returning `False` if nothing is found. #761 (untergeek) * Check if an index is in an alias before attempting to delete it from the alias. Issue raised in #887. (untergeek) * Fix allocation issues when using Elasticsearch 5.1+. Issue raised in #871 (untergeek) **Documentation** * Add missing repository arg to auto-gen API docs. Reported in #888 (untergeek) * Add all new documentation and clean up for v5 specific. **Breaking Changes** * IndexList no longer checks to see if there are indices on initialization. 5.0.0a1 (23 March 2017) ----------------------- This is the first alpha release of Curator 5. This should not be used for production! There `will` be many more changes before 5.0.0 is released. **New Features** * Allow allocation action to unset a key/value pair by using an empty value. Requested in #906. (untergeek) * Added support for the Rollover API. Requested in #898, and by countless others. * Added ``warn_if_no_indices`` option for ``alias`` action in response to #883. Using this option will permit the ``alias`` add or remove to continue with a logged warning, even if the filters result in a ``NoIndices`` condition. Use with care. **Bug Fixes** * Check if an index is in an alias before attempting to delete it from the alias. Issue raised in #887. (untergeek) * Fix allocation issues when using Elasticsearch 5.1+. Issue raised in #871 (untergeek) **Documentation** * Add missing repository arg to auto-gen API docs. Reported in #888 (untergeek) 4.2.6 (27 January 2016) ----------------------- **General** * Update Curator to use version 5.1 of the ``elasticsearch-py`` python module. With this change, there will be no reverse compatibility with Elasticsearch 2.x. For 2.x versions, continue to use the 4.x branches of Curator. * Tests were updated to reflect the changes in API calls, which were minimal. * Remove "official" support for Python 2.6. If you must use Curator on a system that uses Python 2.6 (RHEL/CentOS 6 users), it is recommended that you use the official RPM package as it is a frozen binary built on Python 3.5.x which will not conflict with your system Python. * Use ``isinstance()`` to verify client object. #862 (cp2587) * Prune older versions from Travis CI tests. * Update ``certifi`` dependency to latest version **Documentation** * Add version compatibility section to official documentation. * Update docs to reflect changes. Remove cruft and references to older versions. 4.2.5 (22 December 2016) ------------------------ **General** * Add and increment test versions for Travis CI. #839 (untergeek) * Make `filter_list` optional in snapshot, show_snapshot and show_indices singleton actions. #853 (alexef) **Bug Fixes** * Fix cli integration test when different host/port are specified. Reported in #843 (untergeek) * Catch empty list condition during filter iteration in singleton actions. Reported in #848 (untergeek) **Documentation** * Add docs regarding how filters are ANDed together, and how to do an OR with the regex pattern filter type. Requested in #842 (untergeek) * Fix typo in Click version in docs. #850 (breml) * Where applicable, replace `[source,text]` with `[source,yaml]` for better formatting in the resulting docs. 4.2.4 (7 December 2016) ----------------------- **Bug Fixes** * ``--wait_for_completion`` should be `True` by default for Snapshot singleton action. Reported in #829 (untergeek) * Increase `version_max` to 5.1.99. Prematurely reported in #832 (untergeek) * Make the '.security' index visible for snapshots so long as proper credentials are used. Reported in #826 (untergeek) 4.2.3.post1 (22 November 2016) ------------------------------ This fix is `only` going in for ``pip``-based installs. There are no other code changes. **Bug Fixes** * Fixed incorrect assumption of PyPI picking up dependency for certifi. It is still a dependency, but should not affect ``pip`` installs with an error any more. Reported in #821 (untergeek) 4.2.3 (21 November 2016) ------------------------ 4.2.2 was pulled immediately after release after it was discovered that the Windows binary distributions were still not including the certifi-provided certificates. This has now been remedied. **General** * ``certifi`` is now officially a requirement. * ``setup.py`` now forcibly includes the ``certifi`` certificate PEM file in the "frozen" distributions (i.e., the compiled versions). The ``get_client`` method was updated to reflect this and catch it for both the Linux and Windows binary distributions. This should `finally` put to rest #810 4.2.2 (21 November 2016) ------------------------ **Bug Fixes** * The certifi-provided certificates were not propagating to the compiled RPM/DEB packages. This has been corrected. Reported in #810 (untergeek) **General** * Added missing ``--ignore_empty_list`` option to singleton actions. Requested in #812 (untergeek) **Documentation** * Add a FAQ entry regarding the click module's need for Unicode when using Python 3. Kind of a bug fix too, as the entry_points were altered to catch this omission and report a potential solution on the command-line. Reported in #814 (untergeek) * Change the "Command-Line" documentation header to be "Running Curator" 4.2.1 (8 November 2016) ----------------------- **Bug Fixes** * In the course of package release testing, an undesirable scenario was caught where boolean flags default values for ``curator_cli`` were improperly overriding values from a yaml config file. **General** * Adding in direct download URLs for the RPM, DEB, tarball and zip packages. 4.2.0 (4 November 2016) ----------------------- **New Features** * Shard routing allocation enable/disable. This will allow you to disable shard allocation routing before performing one or more actions, and then re-enable after it is complete. Requested in #446 (untergeek) * Curator 3.x-style command-line. This is now ``curator_cli``, to differentiate between the current binary. Not all actions are available, but the most commonly used ones are. With the addition in 4.1.0 of schema and configuration validation, there's even a way to still do filter chaining on the command-line! Requested in #767, and by many other users (untergeek) **General** * Update testing to the most recent versions. * Lock elasticsearch-py module version at >= 2.4.0 and <= 3.0.0. There are API changes in the 5.0 release that cause tests to fail. **Bug Fixes** * Guarantee that binary packages are built from the latest Python + libraries. This ensures that SSL/TLS will work without warning messages about insecure connections, unless they actually are insecure. Reported in #780, though the reported problem isn't what was fixed. The fix is needed based on what was discovered while troubleshooting the problem. (untergeek) 4.1.2 (6 October 2016) ---------------------- This release does not actually add any new code to Curator, but instead improves documentation and includes new linux binary packages. **General** * New Curator binary packages for common Linux systems! These will be found in the same repositories that the python-based packages are in, but have no dependencies. All necessary libraries/modules are bundled with the binary, so everything should work out of the box. This feature doesn't change any other behavior, so it's not a major release. These binaries have been tested in: * CentOS 6 & 7 * Ubuntu 12.04, 14.04, 16.04 * Debian 8 They do not work in Debian 7 (library mismatch). They may work in other systems, but that is untested. The script used is in the unix_packages directory. The Vagrantfiles for the various build systems are in the Vagrant directory. **Bug Fixes** * The only bug that can be called a bug is actually a stray ``.exe`` suffix in the binary package creation section (cx_freeze) of ``setup.py``. The Windows binaries should have ``.exe`` extensions, but not unix variants. * Elasticsearch 5.0.0-beta1 testing revealed that a document ID is required during document creation in tests. This has been fixed, and a redundant bit of code in the forcemerge integration test was removed. **Documentation** * The documentation has been updated and improved. Examples and installation are now top-level events, with the sub-sections each having their own link. They also now show how to install and use the binary packages, and the section on installation from source has been improved. The missing section on installing the voluptuous schema verification module has been written and included. #776 (untergeek) 4.1.1 (27 September 2016) ------------------------- **Bug Fixes** * String-based booleans are now properly coerced. This fixes an issue where `True`/`False` were used in environment variables, but not recognized. #765 (untergeek) * Fix missing `count` method in ``__map_method`` in SnapshotList. Reported in #766 (untergeek) **General** * Update es_repo_mgr to use the same client/logging YAML config file. Requested in #752 (untergeek) **Schema Validation** * Cases where ``source`` was not defined in a filter (but should have been) were informing users that a `timestring` field was there that shouldn't have been. This edge case has been corrected. **Documentation** * Added notifications and FAQ entry to explain that AWS ES is not supported. 4.1.0 (6 September 2016) ------------------------ **New Features** * Configuration and Action file schema validation. Requested in #674 (untergeek) * Alias filtertype! With this filter, you can select indices based on whether they are part of an alias. Merged in #748 (untergeek) * Count filtertype! With this filter, you can now configure Curator to only keep the most recent `n` indices (or snapshots!). Merged in #749 (untergeek) * Experimental! Use environment variables in your YAML configuration files. This was a popular request, #697. (untergeek) **General** * New requirement! ``voluptuous`` Python schema validation module * Requirement version bump: Now requires ``elasticsearch-py`` 2.4.0 **Bug Fixes** * ``delete_aliases`` option in ``close`` action no longer results in an error if not all selected indices have an alias. Add test to confirm expected behavior. Reported in #736 (untergeek) **Documentation** * Add information to FAQ regarding indices created before Elasticsearch 1.4. Merged in #747 4.0.6 (15 August 2016) ---------------------- **Bug Fixes** * Update old calls used with ES 1.x to reflect changes in 2.x+. This was necessary to work with Elasticsearch 5.0.0-alpha5. Fixed in #728 (untergeek) **Doc Fixes** * Add section detailing that the value of a ``value`` filter element should be encapsulated in single quotes. Reported in #726. (untergeek) 4.0.5 (3 August 2016) --------------------- **Bug Fixes** * Fix incorrect variable name for AWS Region reported in #679 (basex) * Fix ``filter_by_space()`` to not fail when index age metadata is not present. Indices without the appropriate age metadata will instead be excluded, with a debug-level message. Reported in #724 (untergeek) **Doc Fixes** * Fix documentation for the space filter and the source filter element. 4.0.4 (1 August 2016) --------------------- **Bug Fixes** * Fix incorrect variable name in Allocation action. #706 (lukewaite) * Incorrect error message in ``create_snapshot_body`` reported in #711 (untergeek) * Test for empty index list object should happen in action initialization for snapshot action. Discovered in #711. (untergeek) **Doc Fixes** * Add menus to asciidoc chapters #704 (untergeek) * Add pyyaml dependency #710 (dtrv) 4.0.3 (22 July 2016) -------------------- **General** * 4.0.2 didn't work for ``pip`` installs due to an omission in the MANIFEST.in file. This came up during release testing, but before the release was fully published. As the release was never fully published, this should not have actually affected anyone. **Bug Fixes** * These are the same as 4.0.2, but it was never fully released. * All default settings are now values returned from functions instead of constants. This was resulting in settings getting stomped on. New test addresses the original complaint. This removes the need for ``deepcopy``. See issue #687 (untergeek) * Fix ``host`` vs. ``hosts`` issue in ``get_client()`` rather than the non-functional function in ``repomgrcli.py``. * Update versions being tested. * Community contributed doc fixes. * Reduced logging verbosity by making most messages debug level. #684 (untergeek) * Fixed log whitelist behavior (and switched to blacklisting instead). Default behavior will now filter traffic from the ``elasticsearch`` and ``urllib3`` modules. * Fix Travis CI testing to accept some skipped tests, as needed. #695 (untergeek) * Fix missing empty index test in snapshot action. #682 (sherzberg) 4.0.2 (22 July 2016) -------------------- **Bug Fixes** * All default settings are now values returned from functions instead of constants. This was resulting in settings getting stomped on. New test addresses the original complaint. This removes the need for ``deepcopy``. See issue #687 (untergeek) * Fix ``host`` vs. ``hosts`` issue in ``get_client()`` rather than the non-functional function in ``repomgrcli.py``. * Update versions being tested. * Community contributed doc fixes. * Reduced logging verbosity by making most messages debug level. #684 (untergeek) * Fixed log whitelist behavior (and switched to blacklisting instead). Default behavior will now filter traffic from the ``elasticsearch`` and ``urllib3`` modules. * Fix Travis CI testing to accept some skipped tests, as needed. #695 (untergeek) * Fix missing empty index test in snapshot action. #682 (sherzberg) 4.0.1 (1 July 2016) ------------------- **Bug Fixes** * Coerce Logstash/JSON logformat type timestamp value to always use UTC. #661 (untergeek) * Catch and remove indices from the actionable list if they do not have a `creation_date` field in settings. This field was introduced in ES v1.4, so that indicates a rather old index. #663 (untergeek) * Replace missing ``state`` filter for ``snapshotlist``. #665 (untergeek) * Restore ``es_repo_mgr`` as a stopgap until other CLI scripts are added. It will remain undocumented for now, as I am debating whether to make repository creation its own action in the API. #668 (untergeek) * Fix dry run results for snapshot action. #673 (untergeek) 4.0.0 (24 June 2016) -------------------- It's official! Curator 4.0.0 is released! **Breaking Changes** * New and improved API! * Command-line changes. No more command-line args, except for ``--config``, ``--actions``, and ``--dry-run``: - ``--config`` points to a YAML client and logging configuration file. The default location is ``~/.curator/curator.yml`` - ``--actions`` arg points to a YAML action configuration file - ``--dry-run`` will simulate the action(s) which would have taken place, but not actually make any changes to the cluster or its indices. **New Features** * Snapshot restore is here! * YAML configuration files. Now a single file can define an entire batch of commands, each with their own filters, to be performed in sequence. * Sort by index age not only by index name (as with previous versions of Curator), but also by index `creation_date`, or by calculations from the Field Stats API on a timestamp field. * Atomically add/remove indices from aliases! This is possible by way of the new `IndexList` class and YAML configuration files. * State of indices pulled and stored in `IndexList` instance. Fewer API calls required to serially test for open/close, `size_in_bytes`, etc. * Filter by space now allows sorting by age! * Experimental! Use AWS IAM credentials to sign requests to Elasticsearch. This requires the end user to *manually* install the `requests_aws4auth` python module. * Optionally delete aliases from indices before closing. * An empty index or snapshot list no longer results in an error if you set ``ignore_empty_list`` to `True`. If `True` it will still log that the action was not performed, but will continue to the next action. If 'False' it will log an ERROR and exit with code 1. **API** * Updated API documentation * Class: `IndexList`. This pulls all indices at instantiation, and you apply filters, which are class methods. You can iterate over as many filters as you like, in fact, due to the YAML config file. * Class: `SnapshotList`. This pulls all snapshots from the given repository at instantiation, and you apply filters, which are class methods. You can iterate over as many filters as you like, in fact, due to the YAML config file. * Add `wait_for_completion` to Allocation and Replicas actions. These will use the client timeout, as set by default or `timeout_override`, to determine how long to wait for timeout. These are handled in batches of indices for now. * Allow `timeout_override` option for all actions. This allows for different timeout values per action. * Improve API by giving each action its own `do_dry_run()` method. **General** * Updated use documentation for Elastic main site. * Include example files for ``--config`` and ``--actions``. 4.0.0b2 (16 June 2016) ---------------------- **Second beta release of the 4.0 branch** **New Feature** * An empty index or snapshot list no longer results in an error if you set ``ignore_empty_list`` to `True`. If `True` it will still log that the action was not performed, but will continue to the next action. If 'False' it will log an ERROR and exit with code 1. (untergeek) 4.0.0b1 (13 June 2016) ---------------------- **First beta release of the 4.0 branch!** The release notes will be rehashing the new features in 4.0, rather than the bug fixes done during the alphas. **Breaking Changes** * New and improved API! * Command-line changes. No more command-line args, except for ``--config``, ``--actions``, and ``--dry-run``: - ``--config`` points to a YAML client and logging configuration file. The default location is ``~/.curator/curator.yml`` - ``--actions`` arg points to a YAML action configuration file - ``--dry-run`` will simulate the action(s) which would have taken place, but not actually make any changes to the cluster or its indices. **New Features** * Snapshot restore is here! * YAML configuration files. Now a single file can define an entire batch of commands, each with their own filters, to be performed in sequence. * Sort by index age not only by index name (as with previous versions of Curator), but also by index `creation_date`, or by calculations from the Field Stats API on a timestamp field. * Atomically add/remove indices from aliases! This is possible by way of the new `IndexList` class and YAML configuration files. * State of indices pulled and stored in `IndexList` instance. Fewer API calls required to serially test for open/close, `size_in_bytes`, etc. * Filter by space now allows sorting by age! * Experimental! Use AWS IAM credentials to sign requests to Elasticsearch. This requires the end user to *manually* install the `requests_aws4auth` python module. * Optionally delete aliases from indices before closing. **API** * Updated API documentation * Class: `IndexList`. This pulls all indices at instantiation, and you apply filters, which are class methods. You can iterate over as many filters as you like, in fact, due to the YAML config file. * Class: `SnapshotList`. This pulls all snapshots from the given repository at instantiation, and you apply filters, which are class methods. You can iterate over as many filters as you like, in fact, due to the YAML config file. * Add `wait_for_completion` to Allocation and Replicas actions. These will use the client timeout, as set by default or `timeout_override`, to determine how long to wait for timeout. These are handled in batches of indices for now. * Allow `timeout_override` option for all actions. This allows for different timeout values per action. * Improve API by giving each action its own `do_dry_run()` method. **General** * Updated use documentation for Elastic main site. * Include example files for ``--config`` and ``--actions``. 4.0.0a10 (10 June 2016) ----------------------- **New Features** * Snapshot restore is here! * Optionally delete aliases from indices before closing. Fixes #644 (untergeek) **General** * Add `wait_for_completion` to Allocation and Replicas actions. These will use the client timeout, as set by default or `timeout_override`, to determine how long to wait for timeout. These are handled in batches of indices for now. * Allow `timeout_override` option for all actions. This allows for different timeout values per action. **Bug Fixes** * Disallow use of `master_only` if multiple hosts are used. Fixes #615 (untergeek) * Fix an issue where arguments weren't being properly passed and populated. * ForceMerge replaced Optimize in ES 2.1.0. * Fix prune_nones to work with Python 2.6. Fixes #619 (untergeek) * Fix TimestringSearch to work with Python 2.6. Fixes #622 (untergeek) * Add language classifiers to ``setup.py``. Fixes #640 (untergeek) * Changed references to readthedocs.org to be readthedocs.io. 4.0.0a9 (27 Apr 2016) --------------------- **General** * Changed `create_index` API to use kwarg `extra_settings` instead of `body` * Normalized Alias action to use `name` instead of `alias`. This simplifies documentation by reducing the number of option elements. * Streamlined some code * Made `exclude` a filter element setting for all filters. Updated all examples to show this. * Improved documentation **New Features** * Alias action can now accept `extra_settings` to allow adding filters, and/or routing. 4.0.0a8 (26 Apr 2016) --------------------- **Bug Fixes** * Fix to use `optimize` with versions of Elasticsearch < 5.0 * Fix missing setting in testvars 4.0.0a7 (25 Apr 2016) --------------------- **Bug Fixes** * Fix AWS4Auth error. 4.0.0a6 (25 Apr 2016) --------------------- **General** * Documentation updates. * Improve API by giving each action its own `do_dry_run()` method. **Bug Fixes** * Do not escape characters other than ``.`` and ``-`` in timestrings. Fixes #602 (untergeek) ** New Features** * Added `CreateIndex` action. 4.0.0a4 (21 Apr 2016) --------------------- **Bug Fixes** * Require `pyyaml` 3.10 or better. * In the case that no `options` are in an action, apply the defaults. 4.0.0a3 (21 Apr 2016) --------------------- It's time for Curator 4.0 alpha! **Breaking Changes** * New API! (again?!) * Command-line changes. No more command-line args, except for ``--config``, ``--actions``, and ``--dry-run``: - ``--config`` points to a YAML client and logging configuration file. The default location is ``~/.curator/curator.yml`` - ``--actions`` arg points to a YAML action configuration file - ``--dry-run`` will simulate the action(s) which would have taken place, but not actually make any changes to the cluster or its indices. **General** * Updated API documentation * Updated use documentation for Elastic main site. * Include example files for ``--config`` and ``--actions``. **New Features** * Sort by index age not only by index name (as with previous versions of Curator), but also by index `creation_date`, or by calculations from the Field Stats API on a timestamp field. * Class: `IndexList`. This pulls all indices at instantiation, and you apply filters, which are class methods. You can iterate over as many filters as you like, in fact, due to the YAML config file. * Class: `SnapshotList`. This pulls all snapshots from the given repository at instantiation, and you apply filters, which are class methods. You can iterate over as many filters as you like, in fact, due to the YAML config file. * YAML configuration files. Now a single file can define an entire batch of commands, each with their own filters, to be performed in sequence. * Atomically add/remove indices from aliases! This is possible by way of the new `IndexList` class and YAML configuration files. * State of indices pulled and stored in `IndexList` instance. Fewer API calls required to serially test for open/close, `size_in_bytes`, etc. * Filter by space now allows sorting by age! * Experimental! Use AWS IAM credentials to sign requests to Elasticsearch. This requires the end user to *manually* install the `requests_aws4auth` python module. 3.5.1 (21 March 2016) --------------------- **Bug fixes** * Add more logging information to snapshot delete method #582 (untergeek) * Improve default timeout, logging, and exception handling for `seal` command #583 (untergeek) * Fix use of default snapshot name. #584 (untergeek) 3.5.0 (16 March 2016) --------------------- **General** * Add support for the `--client-cert` and `--client-key` command line parameters and client_cert and client_key parameters to the get_client() call. #520 (richm) **Bug fixes** * Disallow users from creating snapshots with upper-case letters, which is not permitted by Elasticsearch. #562 (untergeek) * Remove `print()` command from ``setup.py`` as it causes issues with command-line retrieval of ``--url``, etc. #568 (thib-ack) * Remove unnecessary argument from `build_filter()` #530 (zzugg) * Allow day of year filter to be made up with 1, 2 or 3 digits #578 (petitout) 3.4.1 (10 February 2016) ------------------------ **General** * Update license copyright to 2016 * Use slim python version with Docker #527 (xaka) * Changed ``--master-only`` exit code to 0 when connected to non-master node #540 (wkruse) * Add ``cx_Freeze`` capability to ``setup.py``, plus a ``binary_release.py`` script to simplify binary package creation. #554 (untergeek) * Set Elastic as author. #555 (untergeek) * Put repository creation methods into API and document them. Requested in #550 (untergeek) **Bug fixes** * Fix sphinx documentation build error #506 (hydrapolic) * Ensure snapshots are found before iterating #507 (garyelephant) * Fix a doc inconsistency #509 (pmoust) * Fix a typo in `show` documentation #513 (pbamba) * Default to trying the cluster state for checking whether indices are closed, and then fall back to using the _cat API (for Amazon ES instances). #519 (untergeek) * Improve logging to show time delay between optimize runs, if selected. #525 (untergeek) * Allow elasticsearch-py module versions through 2.3.0 (a presumption at this point) #524 (untergeek) * Improve logging in snapshot api method to reveal when a repository appears to be missing. Reported in #551 (untergeek) * Test that ``--timestring`` has the correct variable for ``--time-unit``. Reported in #544 (untergeek) * Allocation will exit with exit_code 0 now when there are no indices to work on. Reported in #531 (untergeek) 3.4.0 (28 October 2015) ----------------------- **General** * API change in elasticsearch-py 1.7.0 prevented alias operations. Fixed in #486 (HonzaKral) * During index selection you can now select only closed indices with ``--closed-only``. Does not impact ``--all-indices`` Reported in #476. Fixed in #487 (Basster) * API Changes in Elasticsearch 2.0.0 required some refactoring. All tests pass for ES versions 1.0.3 through 2.0.0-rc1. Fixed in #488 (untergeek) * es_repo_mgr now has access to the same SSL options from #462. #489 (untergeek) * Logging improvements requested in #475. (untergeek) * Added ``--quiet`` flag. #494 (untergeek) * Fixed ``index_closed`` to work with AWS Elasticsearch. #499 (univerio) * Acceptable versions of Elasticsearch-py module are 1.8.0 up to 2.1.0 (untergeek) 3.3.0 (31 August 2015) ---------------------- **Announcement** * Curator is tested in Jenkins. Each commit to the master branch is tested with both Python versions 2.7.6 and 3.4.0 against each of the following Elasticsearch versions: * 1.7_nightly * 1.6_nightly * 1.7.0 * 1.6.1 * 1.5.1 * 1.4.4 * 1.3.9 * 1.2.4 * 1.1.2 * 1.0.3 * If you are using a version different from this, your results may vary. **General** * Allocation type can now also be ``include`` or ``exclude``, in addition to the existing default ``require`` type. Add ``--type`` to the allocation command to specify the type. #443 (steffo) * Bump elasticsearch python module dependency to 1.6.0+ to enable synced_flush API call. Reported in #447 (untergeek) * Add SSL features, ``--ssl-no-validate`` and ``certificate`` to provide other ways to validate SSL connections to Elasticsearch. #436 (untergeek) **Bug fixes** * Delete by space was only reporting space used by primary shards. Fixed to show all space consumed. Reported in #455 (untergeek) * Update exit codes and messages for snapshot selection. Reported in #452 (untergeek) * Fix potential int/float casting issues. Reported in #465 (untergeek) 3.2.3 (16 July 2015) -------------------- **Bug fix** * In order to address customer and community issues with bulk deletes, the ``master_timeout`` is now invoked for delete operations. This should address 503s with 30s timeouts in the debug log, even when ``--timeout`` is set to a much higher value. The ``master_timeout`` is tied to the ``--timeout`` flag value, but will not exceed 300 seconds. #420 (untergeek) **General** * Mixing it up a bit here by putting `General` second! The only other changes are that logging has been improved for deletes so you won't need to have the ``--debug`` flag to see if you have error codes >= 400, and some code documentation improvements. 3.2.2 (13 July 2015) -------------------- **General** * This is a very minor change. The ``mock`` library recently removed support for Python 2.6. As many Curator users are using RHEL/CentOS 6, which is pinned to Python 2.6, this requires the mock version referenced by Curator to also be pinned to a supported version (``mock==1.0.1``). 3.2.1 (10 July 2015) -------------------- **General** * Added delete verification & retry (fixed at 3x) to potentially cover an edge case in #420 (untergeek) * Since GitHub allows rST (reStructuredText) README documents, and that's what PyPI wants also, the README has been rebuilt in rST. (untergeek) **Bug fixes** * If closing indices with ES 1.6+, and all indices are closed, ensure that the seal command does not try to seal all indices. Reported in #426 (untergeek) * Capture AttributeError when sealing indices if a non-TransportError occurs. Reported in #429 (untergeek) 3.2.0 (25 June 2015) -------------------- **New!** * Added support to manually seal, or perform a [synced flush](http://www.elastic.co/guide/en/elasticsearch/reference/current/indices-synced-flush.html) on indices with the ``seal`` command. #394 (untergeek) * Added *experimental* support for SSL certificate validation. In order for this to work, you must install the ``certifi`` python module: ``pip install certifi`` This feature *should* automatically work if the ``certifi`` module is installed. Please report any issues. **General** * Changed logging to go to stdout rather than stderr. Reopened #121 and figured they were right. This is better. (untergeek) * Exit code 99 was unpopular. It has been removed. Reported in #371 and #391 (untergeek) * Add ``--skip-repo-validation`` flag for snapshots. Do not validate write access to repository on all cluster nodes before proceeding. Useful for shared filesystems where intermittent timeouts can affect validation, but won't likely affect snapshot success. Requested in #396 (untergeek) * An alias no longer needs to be pre-existent in order to use the alias command. #317 (untergeek) * es_repo_mgr now passes through upstream errors in the event a repository fails to be created. Requested in #405 (untergeek) **Bug fixes** * In rare cases, ``*`` wildcard would not expand. Replaced with _all. Reported in #399 (untergeek) * Beginning with Elasticsearch 1.6, closed indices cannot have their replica count altered. Attempting to do so results in this error: ``org.elasticsearch.ElasticsearchIllegalArgumentException: Can't update [index.number_of_replicas] on closed indices [[test_index]] - can leave index in an unopenable state`` As a result, the ``change_replicas`` method has been updated to prune closed indices. This change will apply to all versions of Elasticsearch. Reported in #400 (untergeek) * Fixed es_repo_mgr repository creation verification error. Reported in #389 (untergeek) 3.1.0 (21 May 2015) ------------------- **General** * If ``wait_for_completion`` is true, snapshot success is now tested and logged. Reported in #253 (untergeek) * Log & return false if a snapshot is already in progress (untergeek) * Logs individual deletes per index, even though they happen in batch mode. Also log individual snapshot deletions. Reported in #372 (untergeek) * Moved ``chunk_index_list`` from cli to api utils as it's now also used by ``filter.py`` * Added a warning and 10 second timer countdown if you use ``--timestring`` to filter indices, but do not use ``--older-than`` or ``--newer-than`` in conjunction with it. This is to address #348, which behavior isn't a bug, but prevents accidental action against all of your time-series indices. The warning and timer are not displayed for ``show`` and ``--dry-run`` operations. * Added tests for ``es_repo_mgr`` in #350 * Doc fixes **Bug fixes** * delete-by-space needed the same fix used for #245. Fixed in #353 (untergeek) * Increase default client timeout for ``es_repo_mgr`` as node discovery and availability checks for S3 repositories can take a bit. Fixed in #352 (untergeek) * If an index is closed, indicate in ``show`` and ``--dry-run`` output. Reported in #327. (untergeek) * Fix issue where CLI parameters were not being passed to the ``es_repo_mgr`` create sub-command. Reported in #337. (feltnerm) 3.0.3 (27 Mar 2015) ------------------- **Announcement** This is a bug fix release. #319 and #320 are affecting a few users, so this release is being expedited. Test count: 228 Code coverage: 99% **General** * Documentation for the CLI converted to Asciidoc and moved to http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html * Improved logging, and refactored a few methods to help with this. * Dry-run output is now more like v2, with the index or snapshot in the log line, along with the command. Several tests needed refactoring with this change, along with a bit of documentation. **Bug fixes** * Fix links to repository in setup.py. Reported in #318 (untergeek) * No more ``--delay`` with optimized indices. Reported in #319 (untergeek) * ``--request_timeout`` not working as expected. Reinstate the version 2 timeout override feature to prevent default timeouts for ``optimize`` and ``snapshot`` operations. Reported in #320 (untergeek) * Reduce index count to 200 for test.integration.test_cli_commands.TestCLISnapshot.test_cli_snapshot_huge_list in order to reduce or eliminate Jenkins CI test timeouts. Reported in #324 (untergeek) * ``--dry-run`` no longer calls ``show``, but will show output in the log, as in v2. This was a recurring complaint. See #328 (untergeek) 3.0.2 (23 Mar 2015) ------------------- **Announcement** This is a bug fix release. #307 and #309 were big enough to warrant an expedited release. **Bug fixes** * Purge unneeded constants, and clean up config options for snapshot. Reported in #303 (untergeek) * Don't split large index list if performing snapshots. Reported in #307 (untergeek) * Act correctly if a zero value for `--older-than` or `--newer-than` is provided. #309 (untergeek) 3.0.1 (16 Mar 2015) ------------------- **Announcement** The ``regex_iterate`` method was horribly named. It has been renamed to ``apply_filter``. Methods have been added to allow API users to build a filtered list of indices similarly to how the CLI does. This was an oversight. Props to @SegFaultAX for pointing this out. **General** * In conjunction with the rebrand to Elastic, URLs and documentation were updated. * Renamed horribly named `regex_iterate` method to `apply_filter` #298 (untergeek) * Added `build_filter` method to mimic CLI calls. #298 (untergeek) * Added Examples page in the API documentation. #298 (untergeek) **Bug fixes** * Refactored to show `--dry-run` info for `--disk-space` calls. Reported in #290 (untergeek) * Added list chunking so acting on huge lists of indices won't result in a URL bigger than 4096 bytes (Elasticsearch's default limit.) Reported in https://github.com/elastic/curator/issues/245#issuecomment-77916081 * Refactored `to_csv()` method to be simpler. * Added and removed tests according to changes. Code coverage still at 99% 3.0.0 (9 March 2015) -------------------- **Release Notes** The full release of Curator 3.0 is out! Check out all of the changes here! *Note:* This release is _not_ reverse compatible with any previous version. Because 3.0 is a major point release, there have been some major changes to both the API as well as the CLI arguments and structure. Be sure to read the updated command-line specific docs in the [wiki](https://github.com/elasticsearch/curator/wiki) and change your command-line arguments accordingly. The API docs are still at http://curator.readthedocs.io. Be sure to read the latest docs, or select the docs for 3.0.0. **General** * **Breaking changes to the API.** Because this is a major point revision, changes to the API have been made which are non-reverse compatible. Before upgrading, be sure to update your scripts and test them thoroughly. * **Python 3 support** Somewhere along the line, Curator would no longer work with curator. All tests now pass for both Python2 and Python3, with 99% code coverage in both environments. * **New CLI library.** Using Click now. http://click.pocoo.org/3/ This change is especially important as it allows very easy CLI integration testing. * **Pipelined filtering!** You can now use ``--older-than`` & ``--newer-than`` in the same command! You can also provide your own regex via the ``--regex`` parameter. You can use multiple instances of the ``--exclude`` flag. * **Manually include indices!** With the ``--index`` paramter, you can add an index to the working list. You can provide multiple instances of the ``--index`` parameter as well! * **Tests!** So many tests now. Test coverage of the API methods is at 100% now, and at 99% for the CLI methods. This doesn't mean that all of the tests are perfect, or that I haven't missed some scenarios. It does mean, however, that it will be much easier to write tests if something turns up missed. It also means that any new functionality will now need to have tests. * **Iteration changes** Methods now only iterate through each index when appropriate! In fact, the only commands that iterate are `alias` and `optimize`. The `bloom` command will iterate, but only if you have added the `--delay` flag with a value greater than zero. * **Improved packaging!** Methods have been moved into categories of ``api`` and ``cli``, and further broken out into individual modules to help them be easier to find and read. * Check for allocation before potentially re-applying an allocation rule. #273 (ferki) * Assigning replica count and routing allocation rules _can_ be done to closed indices. #283 (ferki) **Bug fixes** * Don't accidentally delete ``.kibana`` index. #261 (malagoli) * Fix segment count for empty indices. #265 (untergeek) * Change bloom filter cutoff Elasticsearch version to 1.4. Reported in #267 (untergeek) 3.0.0rc1 (5 March 2015) ----------------------- **Release Notes** RC1 is here! I'm re-releasing the Changes from all betas here, minus the intra-beta code fixes. Barring any show stoppers, the official release will be soon. **General** * **Breaking changes to the API.** Because this is a major point revision, changes to the API have been made which are non-reverse compatible. Before upgrading, be sure to update your scripts and test them thoroughly. * **Python 3 support** Somewhere along the line, Curator would no longer work with curator. All tests now pass for both Python2 and Python3, with 99% code coverage in both environments. * **New CLI library.** Using Click now. http://click.pocoo.org/3/ This change is especially important as it allows very easy CLI integration testing. * **Pipelined filtering!** You can now use ``--older-than`` & ``--newer-than`` in the same command! You can also provide your own regex via the ``--regex`` parameter. You can use multiple instances of the ``--exclude`` flag. * **Manually include indices!** With the ``--index`` paramter, you can add an index to the working list. You can provide multiple instances of the ``--index`` parameter as well! * **Tests!** So many tests now. Test coverage of the API methods is at 100% now, and at 99% for the CLI methods. This doesn't mean that all of the tests are perfect, or that I haven't missed some scenarios. It does mean, however, that it will be much easier to write tests if something turns up missed. It also means that any new functionality will now need to have tests. * Methods now only iterate through each index when appropriate! * Improved packaging! Hopefully the ``entry_point`` issues some users have had will be addressed by this. Methods have been moved into categories of ``api`` and ``cli``, and further broken out into individual modules to help them be easier to find and read. * Check for allocation before potentially re-applying an allocation rule. #273 (ferki) * Assigning replica count and routing allocation rules _can_ be done to closed indices. #283 (ferki) **Bug fixes** * Don't accidentally delete ``.kibana`` index. #261 (malagoli) * Fix segment count for empty indices. #265 (untergeek) * Change bloom filter cutoff Elasticsearch version to 1.4. Reported in #267 (untergeek) 3.0.0b4 (5 March 2015) ---------------------- **Notes** Integration testing! Because I finally figured out how to use the Click Testing API, I now have a good collection of command-line simulations, complete with a real back-end. This testing found a few bugs (this is why testing exists, right?), and fixed a few of them. **Bug fixes** * HUGE! `curator show snapshots` would _delete_ snapshots. This is fixed. * Return values are now being sent from the commands. * `scripttest` is no longer necessary (click.Test works!) * Calling `get_snapshot` without a snapshot name returns all snapshots 3.0.0b3 (4 March 2015) ---------------------- **Bug fixes** * setup.py was lacking the new packages "curator.api" and "curator.cli" The package works now. * Python3 suggested I had to normalize the beta tag to just b3, so that's also changed. * Cleaned out superfluous imports and logger references from the __init__.py files. 3.0.0-beta2 (3 March 2015) -------------------------- **Bug fixes** * Python3 issues resolved. Tests now pass on both Python2 and Python3 3.0.0-beta1 (3 March 2015) -------------------------- **General** * **Breaking changes to the API.** Because this is a major point revision, changes to the API have been made which are non-reverse compatible. Before upgrading, be sure to update your scripts and test them thoroughly. * **New CLI library.** Using Click now. http://click.pocoo.org/3/ * **Pipelined filtering!** You can now use ``--older-than`` & ``--newer-than`` in the same command! You can also provide your own regex via the ``--regex`` parameter. You can use multiple instances of the ``--exclude`` flag. * **Manually include indices!** With the ``--index`` paramter, you can add an index to the working list. You can provide multiple instances of the ``--index`` parameter as well! * **Tests!** So many tests now. Unit test coverage of the API methods is at 100% now. This doesn't mean that all of the tests are perfect, or that I haven't missed some scenarios. It does mean that any new functionality will need to also have tests, now. * Methods now only iterate through each index when appropriate! * Improved packaging! Hopefully the ``entry_point`` issues some users have had will be addressed by this. Methods have been moved into categories of ``api`` and ``cli``, and further broken out into individual modules to help them be easier to find and read. * Check for allocation before potentially re-applying an allocation rule. #273 (ferki) **Bug fixes** * Don't accidentally delete ``.kibana`` index. #261 (malagoli) * Fix segment count for empty indices. #265 (untergeek) * Change bloom filter cutoff Elasticsearch version to 1.4. Reported in #267 (untergeek) 2.1.2 (22 January 2015) ----------------------- **Bug fixes** * Do not try to set replica count if count matches provided argument. #247 (bobrik) * Fix JSON logging (Logstash format). #250 (magnusbaeck) * Fix bug in `filter_by_space()` which would match all indices if the provided patterns found no matches. Reported in #254 (untergeek) 2.1.1 (30 December 2014) ------------------------ **Bug fixes** * Renamed unnecessarily redundant ``--replicas`` to ``--count`` in args for ``curator_script.py`` 2.1.0 (30 December 2014) ------------------------ **General** * Snapshot name now appears in log output or STDOUT. #178 (untergeek) * Replicas! You can now change the replica count of indices. Requested in #175 (untergeek) * Delay option added to Bloom Filter functionality. #206 (untergeek) * Add 2-digit years as acceptable pattern (y vs. Y). Reported in #209 (untergeek) * Add Docker container definition #226 (christianvozar) * Allow the use of 0 with --older-than, --most-recent and --delete-older-than. See #208. #211 (bobrik) **Bug fixes** * Edge case where 1.4.0.Beta1-SNAPSHOT would break version check. Reported in #183 (untergeek) * Typo fixed. #193 (ferki) * Type fixed. #204 (gheppner) * Shows proper error in the event of concurrent snapshots. #177 (untergeek) * Fixes erroneous index display of ``_, a, l, l`` when --all-indices selected. Reported in #222 (untergeek) * Use json.dumps() to escape exceptions. Reported in #210 (untergeek) * Check if index is closed before adding to alias. Reported in #214 (bt5e) * No longer force-install argparse if pre-installed #216 (whyscream) * Bloom filters have been removed from Elasticsearch 1.5.0. Update methods and tests to act accordingly. #233 (untergeek) 2.0.2 (8 October 2014) ---------------------- **Bug fixes** * Snapshot name not displayed in log or STDOUT #185 (untergeek) * Variable name collision in delete_snapshot() #186 (untergeek) 2.0.1 (1 October 2014) ---------------------- **Bug fix** * Override default timeout when snapshotting --all-indices #179 (untergeek) 2.0.0 (25 September 2014) ------------------------- **General** * New! Separation of Elasticsearch Curator Python API and curator_script.py (untergeek) * New! ``--delay`` after optimize to allow cluster to quiesce #131 (untergeek) * New! ``--suffix`` option in addition to ``--prefix`` #136 (untergeek) * New! Support for wildcards in prefix & suffix #136 (untergeek) * Complete refactor of snapshots. Now supporting incrementals! (untergeek) **Bug fix** * Incorrect error msg if no indices sent to create_snapshot (untergeek) * Correct for API change coming in ES 1.4 #168 (untergeek) * Missing ``"`` in Logstash log format #143 (cassianoleal) * Change non-master node test to exit code 0, log as ``INFO``. #145 (untergeek) * `months` option missing from validate_timestring() (untergeek) 1.2.2 (29 July 2014) -------------------- **Bug fix** * Updated ``README.md`` to briefly explain what curator does #117 (untergeek) * Fixed ``es_repo_mgr`` logging whitelist #119 (untergeek) * Fixed absent ``months`` time-unit #120 (untergeek) * Filter out ``.marvel-kibana`` when prefix is ``.marvel-`` #120 (untergeek) * Clean up arg parsing code where redundancy exists #123 (untergeek) * Properly divide debug from non-debug logging #125 (untergeek) * Fixed ``show`` command bug caused by changes to command structure #126 (michaelweiser) 1.2.1 (24 July 2014) -------------------- **Bug fix** * Fixed the new logging when called by ``curator`` entrypoint. 1.2.0 (24 July 2014) -------------------- **General** * New! Allow user-specified date patterns: ``--timestring`` #111 (untergeek) * New! Curate weekly indices (must use week of year) #111 (untergeek) * New! Log output in logstash format ``--logformat logstash`` #111 (untergeek) * Updated! Cleaner default logs (debug still shows everything) (untergeek) * Improved! Dry runs are more visible in log output (untergeek) Errata * The ``--separator`` option was removed in lieu of user-specified date patterns. * Default ``--timestring`` for days: ``%Y.%m.%d`` (Same as before) * Default ``--timestring`` for hours: ``%Y.%m.%d.%H`` (Same as before) * Default ``--timestring`` for weeks: ``%Y.%W`` 1.1.3 (18 July 2014) -------------------- **Bug fix** * Prefix not passed in ``get_object_list()`` #106 (untergeek) * Use ``os.devnull`` instead of ``/dev/null`` for Windows #102 (untergeek) * The http auth feature was erroneously omitted #100 (bbuchacher) 1.1.2 (13 June 2014) -------------------- **Bug fix** * This was a showstopper bug for anyone using RHEL/CentOS with a Python 2.6 dependency for yum * Python 2.6 does not like format calls without an index. #96 via #95 (untergeek) * We won't talk about what happened to 1.1.1. No really. I hate git today :( 1.1.0 (12 June 2014) -------------------- **General** * Updated! New command structure * New! Snapshot to fs or s3 #82 (untergeek) * New! Add/Remove indices to alias #82 via #86 (cschellenger) * New! ``--exclude-pattern`` #80 (ekamil) * New! (sort of) Restored ``--log-level`` support #73 (xavier-calland) * New! show command-line options #82 via #68 (untergeek) * New! Shard Allocation Routing #82 via #62 (nickethier) **Bug fix** * Fix ``--max_num_segments`` not being passed correctly #74 (untergeek) * Change ``BUILD_NUMBER`` to ``CURATOR_BUILD_NUMBER`` in ``setup.py`` #60 (mohabusama) * Fix off-by-one error in time calculations #66 (untergeek) * Fix testing with python3 #92 (untergeek) Errata * Removed ``optparse`` compatibility. Now requires ``argparse``. 1.0.0 (25 Mar 2014) ------------------- **General** * compatible with ``elasticsearch-py`` 1.0 and Elasticsearch 1.0 (honzakral) * Lots of tests! (honzakral) * Streamline code for 1.0 ES versions (honzakral) **Bug fix** * Fix ``find_expired_indices()`` to not skip closed indices (honzakral) 0.6.2 (18 Feb 2014) ------------------- **General** * Documentation fixes #38 (dharrigan) * Add support for HTTPS URI scheme and ``optparse`` compatibility for Python 2.6 (gelim) * Add elasticsearch module version checking for future compatibility checks (untergeek) 0.6.1 (08 Feb 2014) ------------------- **General** * Added tarball versioning to ``setup.py`` (untergeek) **Bug fix** * Fix ``long_description`` by including ``README.md`` in ``MANIFEST.in`` (untergeek) * Incorrect version number in ``curator.py`` (untergeek) 0.6.0 (08 Feb 2014) ------------------- **General** * Restructured repository to a be a proper python package. (arieb) * Added ``setup.py`` file. (arieb) * Removed the deprecated file ``logstash_index_cleaner.py`` (arieb) * Updated ``README.md`` to fit the new package, most importantly the usage and installation. (arieb) * Fixes and package push to PyPI (untergeek) 0.5.2 (26 Jan 2014) ------------------- **General** * Fix boolean logic determining hours or days for time selection (untergeek) 0.5.1 (20 Jan 2014) ------------------- **General** * Fix ``can_bloom`` to compare numbers (HonzaKral) * Switched ``find_expired_indices()`` to use ``datetime`` and ``timedelta`` * Do not try and catch unrecoverable exceptions. (HonzaKral) * Future proofing the use of the elasticsearch client (i.e. work with version 1.0+ of Elasticsearch) (HonzaKral) Needs more testing, but should work. * Add tests for these scenarios (HonzaKral) 0.5.0 (17 Jan 2014) ------------------- **General** * Deprecated ``logstash_index_cleaner.py`` Use new ``curator.py`` instead (untergeek) * new script change: ``curator.py`` (untergeek) * new add index optimization (Lucene forceMerge) to reduce segments and therefore memory usage. (untergeek) * update refactor of args and several functions to streamline operation and make it more readable (untergeek) * update refactor further to clean up and allow immediate (and future) portability (HonzaKral) 0.4.0 ----- **General** * First version logged in ``CHANGELOG`` * new ``--disable-bloom-days`` feature requires 0.90.9+ http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/index-modules-codec.html#bloom-postings This can save a lot of heap space on cold indexes (i.e. not actively indexing documents) elasticsearch-curator-8.0.21/docs/Makefile000066400000000000000000000152061477314666200205050ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Elasticsearch.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Elasticsearch.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Elasticsearch" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Elasticsearch" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." elasticsearch-curator-8.0.21/docs/about.rst000066400000000000000000000010431477314666200207030ustar00rootroot00000000000000.. _about: About ===== Elasticsearch Curator helps you curate, or manage, your Elasticsearch indices and snapshots by: 1. Obtaining the full list of indices (or snapshots) from the cluster, as the *actionable list* 2. Iterate through a list of user-defined filters to progressively remove indices (or snapshots) from this *actionable list* as needed. 3. Perform various :doc:`actions ` on the items which remain in the *actionable list.* .. toctree:: :maxdepth: 1 examples usage testing contributing Changelogelasticsearch-curator-8.0.21/docs/actions.rst000066400000000000000000000047741477314666200212470ustar00rootroot00000000000000.. _actions: Actions ####### .. seealso:: Each action has a `do_action()` method, which accepts no arguments. This is the means by which all actions are executed. .. _actions_alias: Alias ===== .. autoclass:: curator.actions.Alias :members: :undoc-members: :show-inheritance: .. _action_allocation: Allocation ========== .. autoclass:: curator.actions.Allocation :members: :undoc-members: :show-inheritance: .. _actions_close: Close ===== .. autoclass:: curator.actions.Close :members: :undoc-members: :show-inheritance: .. _actions_clusterrouting: Cluster Routing =============== .. autoclass:: curator.actions.ClusterRouting :members: :undoc-members: :show-inheritance: .. _actions_createindex: Cold2Frozen =========== .. autoclass:: curator.actions.Cold2Frozen :members: :undoc-members: :show-inheritance: Create Index ============ .. autoclass:: curator.actions.CreateIndex :members: :undoc-members: :show-inheritance: .. _actions_deleteindices: Delete Indices ============== .. autoclass:: curator.actions.DeleteIndices :members: :undoc-members: :show-inheritance: .. _actions_deletesnapshots: Delete Snapshots ================ .. autoclass:: curator.actions.DeleteSnapshots :members: :undoc-members: :show-inheritance: .. _actions_forcemerge: Force Merge =========== .. autoclass:: curator.actions.ForceMerge :members: :undoc-members: :show-inheritance: .. _actions_indexsettings: Index Settings ============== .. autoclass:: curator.actions.IndexSettings :members: :undoc-members: :show-inheritance: .. _actions_open: Open ==== .. autoclass:: curator.actions.Open :members: :undoc-members: :show-inheritance: .. _actions_reindex: Reindex ======= .. autoclass:: curator.actions.Reindex :members: :undoc-members: :show-inheritance: .. _actions_replicas: Replicas ======== .. autoclass:: curator.actions.Replicas :members: :undoc-members: :show-inheritance: .. _actions_restore: Restore ======= .. autoclass:: curator.actions.Restore :members: :undoc-members: :show-inheritance: .. _actions_rollover: Rollover ======== .. autoclass:: curator.actions.Rollover :members: :undoc-members: :show-inheritance: .. _actions_shrink: Shrink ====== .. autoclass:: curator.actions.Shrink :members: :undoc-members: :show-inheritance: .. _actions_snapshot: Snapshot ======== .. autoclass:: curator.actions.Snapshot :members: :undoc-members: :show-inheritance: elasticsearch-curator-8.0.21/docs/api.rst000066400000000000000000000003201477314666200203370ustar00rootroot00000000000000.. _api: API and Code Reference ###################### .. toctree:: :maxdepth: 1 actions indexlist snapshotlist classdef helpers validators defaults exceptions other_modules elasticsearch-curator-8.0.21/docs/asciidoc/000077500000000000000000000000001477314666200206175ustar00rootroot00000000000000elasticsearch-curator-8.0.21/docs/asciidoc/about.asciidoc000066400000000000000000000124411477314666200234330ustar00rootroot00000000000000[[about]] = About [partintro] -- Elasticsearch Curator helps you curate, or manage, your Elasticsearch indices and snapshots by: 1. Obtaining the full list of indices (or snapshots) from the cluster, as the _actionable list_ 2. Iterate through a list of user-defined <> to progressively remove indices (or snapshots) from this _actionable list_ as needed. 3. Perform various <> on the items which remain in the _actionable list._ Learn More: * <> * <> * <> * <> * <> * <> * <> -- [[about-origin]] == Origin Curator was first called https://logstash.jira.com/browse/LOGSTASH-211[`clearESindices.py`]. Its sole function was to delete indices. It was almost immediately renamed to https://logstash.jira.com/browse/LOGSTASH-211[`logstash_index_cleaner.py`]. After a time it was briefly relocated under the https://github.com/elastic/logstash[logstash] repository as `expire_logs`, at which point it began to gain new functionality. Soon thereafter, Jordan Sissel was hired by Elastic (then still Elasticsearch), as was the original author of Curator. Not long after that it became Elasticsearch Curator and is now hosted at https://github.com/elastic/curator Curator now performs many operations on your Elasticsearch indices, from delete to snapshot to shard allocation routing. [[about-features]] == Features Curator allows for many different operations to be performed to both indices and snapshots, including: * Add or remove indices (or both!) from an <> * Change shard routing <> * <> indices * <> * <> * <> * <> closed indices * <> indices * <> indices, including from remote clusters * Change the number of <> per shard for indices * <> indices * Take a <> (backup) of indices * <> snapshots * <> indices [[about-cli]] == Command-Line Interface (CLI) Curator has always been a command-line tool. This site provides the documentation for how to use Curator on the command-line. TIP: Learn more about <>. [[about-api]] == Application Program Interface (API) Curator ships with both an API and CLI tool. The API, or Application Program Interface, allows you to write your own scripts to accomplish similar goals--or even new and different things--with the same code that Curator uses. The API documentation is not on this site, but is available at http://curator.readthedocs.io/. The Curator API is built using the http://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html[Elasticsearch Python API]. [[license]] == License Copyright (c) {copyright_years} Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. [[site-corrections]] == Site Corrections All documentation on this site allows for quick correction submission. To do this, use the "Edit" link to the right on any page. * You will need to log in with your GitHub account. * Make your corrections or additions to the documentation. * Please use the option to "Create a new branch for this commit and start a pull request." * Please make sure you have signed our http://www.elastic.co/contributor-agreement/[Contributor License Agreement]. We are not asking you to assign copyright to us, but to give us the right to distribute your code (even documentation corrections) without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. If you are uncomfortable with this, feel free to submit a https://github.com/elastic/curator/issues[GitHub Issue] with your suggested correction instead. * Changes will be reviewed and merged, if acceptable. [[about-contributing]] == Contributing We welcome contributions and bug fixes to Curator's API and CLI. We are grateful for the many https://github.com/elastic/curator/blob/master/CONTRIBUTORS[contributors] who have helped Curator become what it is today. Please read through our https://github.com/elastic/curator/blob/master/CONTRIBUTING.md[contribution] guide, and the Curator https://github.com/elastic/curator/blob/master/README.rst[readme] document. A brief overview of the steps * fork the repo * make changes in your fork * add tests to cover your changes (if necessary) * run tests * sign the http://elastic.co/contributor-agreement/[CLA] * send a pull request! TIP: To submit documentation fixes for this site, see <> elasticsearch-curator-8.0.21/docs/asciidoc/actions.asciidoc000066400000000000000000001115601477314666200237630ustar00rootroot00000000000000[[actions]] = Actions [partintro] -- Actions are the tasks which Curator can perform on your indices. Snapshots, once created, can only be deleted. * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> -- [[alias]] == Alias [source,yaml] ------------- action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name add: filters: - filtertype: ... remove: filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action adds and/or removes indices from the alias identified by <> The <> under the `add` and `remove` directives define which indices will be added and/or removed. This is an atomic action, so adds and removes happen instantaneously. The <> option allows the addition of extra settings with the `add` directive. These settings are ignored for `remove`. An example of how these settings can be used to create a filtered alias might be: [source,yaml] ------------- action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name extra_settings: filter: term: user: kimchy add: filters: - filtertype: ... remove: filters: - filtertype: ... ------------- WARNING: Before creating a filtered alias, first ensure that the fields already exist in the mapping. Learn more about adding filtering and routing to aliases in the {ref}/indices-aliases.html[Elasticsearch Alias API documentation]. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[allocation]] == Allocation [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action changes the shard routing allocation for the selected indices. See {ref}/shard-allocation-filtering.html for more information. You can optionally set `wait_for_completion` to `True` to have Curator wait for the shard routing to complete before continuing: [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: True max_wait: 300 wait_interval: 10 filters: - filtertype: ... ------------- This configuration will wait for a maximum of 300 seconds for shard routing and reallocation to complete before giving up. A `max_wait` value of `-1` will wait indefinitely. Curator will poll for completion at `10` second intervals, as defined by `wait_interval`. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[close]] == Close [source,yaml] ------------- action: close description: "Close selected indices" options: delete_aliases: false skip_flush: false filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action closes the selected indices, and optionally deletes associated aliases beforehand. === Optional settings * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[cluster_routing]] == Cluster Routing [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action changes the shard routing allocation for the selected indices. See {ref}/shards-allocation.html for more information. You can optionally set `wait_for_completion` to `True` to have Curator wait for the shard routing to complete before continuing: [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- This configuration will wait for a maximum of 300 seconds for shard routing and reallocation to complete before giving up. A `max_wait` value of `-1` will wait indefinitely. Curator will poll for completion at `10` second intervals, as defined by `wait_interval`. === Required settings * <> * <> * <> Currently must be set to `enable`. This setting is a placeholder for potential future expansion. === Optional settings * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[cold2frozen]] == Cold2Frozen IMPORTANT: This action is for an unusual case where an index is a mounted, searchable snapshot in the cold tier and is not associated with an ILM policy. This action will not work with an index associated with an ILM policy regardless of the value of `allow_ilm_indices`. [source,yaml] ------------- action: cold2frozen description: "Migrate non-ILM indices from the cold tier to the frozen tier" options: index_settings: {} ignore_index_settings: [] wait_for_completion: True filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action migrates the selected non-ILM indices from the cold tier to the frozen tier. You may well ask why this action is here and why it is limited to non-ILM indices. The answer is "redacted data." If an index must be restored from the cold tier to be live so that sensitive data can be redacted, at present, it must be disassociated from an ILM policy to accomplish this. If you forcemerge and re-snapshot the redacted index, you can still put it in the cold or frozen tier, but it will not be associated with an ILM policy any more. This custom action is for moving that manually re-mounted cold tier index to the frozen tier, preserving the aliases it currently has. === index_settings Settings that should be added to the index when it is mounted. This should be a YAML dictionary containing anything under what would normally appear in `settings`. See {ref}/searchable-snapshots-api-mount-snapshot.html [source,yaml] ------------- action: cold2frozen description: "Migrate non-ILM indices from the cold tier to the frozen tier" options: index_settings: routing: allocation: include: _tier_preference: data_frozen ignore_index_settings: [] wait_for_completion: True filters: - filtertype: ... ------------- NOTE: If unset, the default behavior is to ensure that the `_tier_preference` is `data_frozen`, if available. If it is not, Curator will assess which data tiers are available in your cluster and use those from coldest to warmest, e.g. `data_cold,data_warm,data_hot`. If none of these are available, it will default to `data_content`. === ignore_index_settings This should be a YAML list of index settings the migrated index should ignore after mount. See {ref}/searchable-snapshots-api-mount-snapshot.html [source,yaml] ------------- action: cold2frozen description: "Migrate non-ILM indices from the cold tier to the frozen tier" options: index_settings: ignore_index_settings: - 'index.refresh_interval' wait_for_completion: True filters: - filtertype: ... ------------- NOTE: If unset, the default behavior is to ensure that the `index.refresh_interval` is ignored. === Optional settings * <> * <> * <> * <> * <> * <> [[create_index]] == Create Index [source,yaml] ------------- action: create_index description: "Create index as named" options: name: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action creates the named index. There are multiple different ways to configure how the name is represented. === Manual naming [source,yaml] ------------- action: create_index description: "Create index as named" options: name: myindex # ... ------------- In this case, what you see is what you get. An index named `myindex` will be created === Python strftime [source,yaml] ------------- action: create_index description: "Create index as named" options: name: 'myindex-%Y.%m' # ... ------------- For the `create_index` action, the <> option can contain Python strftime strings. The method for doing so is described in detail, including which strftime strings are acceptable, in the documentation for the <> option. === Date Math [source,yaml] ------------- action: create_index description: "Create index as named" options: name: '' # ... ------------- For the `create_index` action, the <> option can be in Elasticsearch {ref}/api-conventions.html#api-date-math-index-names[date math] format. This allows index names containing dates to use deterministic math to set a date name in the past or the future. For example, if today's date were 2017-03-27, the name `` will create an index named `logstash-2017.03.27`. If you wanted to create _tomorrow's_ index, you would use the name ``, which adds 1 day. This pattern creates an index named `logstash-2017.03.28`. For many more configuration options, read the Elasticsearch {ref}/api-conventions.html#api-date-math-index-names[date math] documentation. === Extra Settings The <> option allows the addition of extra settings, such as index settings and mappings. An example of how these settings can be used to create an index might be: [source,yaml] ------------- action: create_index description: "Create index as named" options: name: myindex # ... extra_settings: settings: number_of_shards: 1 number_of_replicas: 0 mappings: type1: properties: field1: type: string index: not_analyzed ------------- === Required settings * <> === Optional settings * <> No default value. You can add any acceptable index settings and mappings as nested YAML. See the {ref}/indices-create-index.html[Elasticsearch Create Index API documentation] for more information. * <> * <> * <> TIP: See an example of this action in an <> <>. [[delete_indices]] == Delete Indices [source,yaml] ------------- action: delete_indices description: "Delete selected indices" options: continue_if_exception: False filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action deletes the selected indices. In clusters which are overcrowded with indices, or a high number of shards per node, deletes can take a longer time to process. In such cases, it may be helpful to set a higher timeout than is set in the <>. You can override that <> as follows: [source,yaml] ------------- action: delete_indices description: "Delete selected indices" options: timeout_override: 300 continue_if_exception: False filters: - filtertype: ... ------------- === Optional settings * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[delete_snapshots]] == Delete Snapshots [source,yaml] ------------- action: delete_snapshots description: "Delete selected snapshots from 'repository'" options: repository: ... retry_interval: 120 retry_count: 3 filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action deletes the selected snapshots from the selected <>. If a snapshot is currently underway, Curator will retry up to <> times, with a delay of <> seconds between retries. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[forcemerge]] == Forcemerge [source,yaml] ------------- action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action performs a forceMerge on the selected indices, merging them to <> per shard. WARNING: A {ref}/indices-forcemerge.html#indices-forcemerge[`forcemerge`] should never be executed on an index that is actively receiving data. It should only ever be performed on indices where no more documents are ever anticipated to be added in the future. You can optionally pause between each merge for <> seconds to allow the cluster to quiesce: [source,yaml] ------------- action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 delay: 120 filters: - filtertype: ... ------------- === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[index_settings]] == Index Settings [source,yaml] ------------- action: index_settings description: "Change settings for selected indices" options: index_settings: index: refresh_interval: 5s ignore_unavailable: False preserve_existing: False filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action updates the specified index settings for the selected indices. [IMPORTANT] ======================= While Elasticsearch allows for either dotted notation of index settings, such as [source,json] ------------- PUT /indexname/_settings { "index.blocks.read_only": true } ------------- or in nested structure, like this: [source,json] ------------- PUT /indexname/_settings { "index": { "blocks": { "read_only": true } } } ------------- In order to appropriately detect https://www.elastic.co/guide/en/elasticsearch/reference/5.4/index-modules.html#_static_index_settings[static] vs. https://www.elastic.co/guide/en/elasticsearch/reference/5.4/index-modules.html#dynamic-index-settings[dynamic] settings, and to be able to verify configurational integrity in the YAML file, **Curator does not support using dotted notation.** ======================= === Optional settings * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[open]] == Open [source,yaml] ------------- action: open description: "open selected indices" options: continue_if_exception: False filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action opens the selected indices. === Optional settings * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[reindex]] == Reindex [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- There are many options for the reindex option. The best place to start is in the <> to see how to configure this action. All other options are as follows. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. === Compatibility Generally speaking, the Curator should be able to perform a remote reindex from any version of Elasticsearch, 1.4 and newer. Strictly speaking, the Reindex API in Elasticsearch _is_ able to reindex from older clusters, but Curator cannot be used to facilitate this due to Curator's dependency on changes released in 1.4. [[replicas]] == Replicas [source,yaml] ------------- action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action will set the number of replicas per shard to the value of <>. You can optionally set `wait_for_completion` to `True` to have Curator wait for the replication operation to complete before continuing: [source,yaml] ------------- action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ------------- This configuration will wait for a maximum of 600 seconds for all index replicas to be complete before giving up. A `max_wait` value of `-1` will wait indefinitely. Curator will poll for completion at `10` second intervals, as defined by `wait_interval`. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[restore]] == Restore [source,yaml] ------------- actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action will restore indices from the indicated <>, from the most recent snapshot identified by the applied filters, or the snapshot identified by <>. === Renaming indices on restore You can cause indices to be renamed at restore with the <> and <> options: [source,yaml] ------------- actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: rename_pattern: 'index(.+)' rename_replacement: 'restored_index$1' wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- In this configuration, Elasticsearch will capture whatever appears after `index` and put it after `restored_index`. For example, if I was restoring `index-2017.03.01`, the resulting index would be renamed to `restored_index-2017.03.01`. === Extra settings The <> option allows the addition of extra settings, such as index settings. An example of how these settings can be used to change settings for an index being restored might be: [source,yaml] ------------- actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: extra_settings: index_settings: number_of_replicas: 0 wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- In this case, the number of replicas will be applied to the restored indices. For more information see the {ref}/snapshots-restore-snapshot.html[official Elasticsearch Documentation]. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[rollover]] == Rollover [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-YYYY.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 max_size: 5gb ------------- This action uses the {ref}/indices-rollover-index.html[Elasticsearch Rollover API] to create a new index, if any of the described conditions are met. IMPORTANT: When choosing `conditions`, **any** one of <>, <>, <>, **or any combination of the three** may be used. If multiple are used, then the specified condition for any one of them must be matched for the rollover to occur. WARNING: If one or more of the <>, <>, or <> options are present, they must each have a value. Because there are no default values, none of these conditions can be left empty, or Curator will generate an error. === Extra settings The <> option allows the addition of extra index settings (but not mappings). An example of how these settings can be used might be: [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-YYYY.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: False ------------- === Required settings * <> The alias name * <> The maximum age that is allowed before triggering a rollover. This _must_ be nested under `conditions:`. There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. * <> The maximum number of documents allowed in an index before triggering a rollover. This _must_ be nested under `conditions:`. There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. * <> The maximum size the index can be before a rollover is triggered. This _must_ be nested under `conditions:`. There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. === Optional settings * <> No default value. You can add any acceptable index settings (not mappings) as nested YAML. See the {ref}/indices-create-index.html[Elasticsearch Create Index API documentation] for more information. * <> Specify a new index name. * <> * <> * <> TIP: See an example of this action in an <> <>. [[shrink]] == Shrink [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink, then reroute the shrunk index with the provided parameters. options: ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. Shrinking an index is a good way to reduce the total shard count in your cluster. https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-shrink-index.html#_shrinking_an_index[Several conditions need to be met] in order for index shrinking to take place: * The index must be marked as read-only * A (primary or replica) copy of every shard in the index must be relocated to the same node * The cluster must have health `green` * The target index must not exist * The number of primary shards in the target index must be a factor of the number of primary shards in the source index. * The source index must have more primary shards than the target index. * The index must not contain more than 2,147,483,519 documents in total across all shards that will be shrunk into a single shard on the target index as this is the maximum number of docs that can fit into a single shard. * The node handling the shrink process must have sufficient free disk space to accommodate a second copy of the existing index. Curator will try to meet these conditions. If it is unable to meet them all, it will not perform a shrink operation. This action will shrink indices to the target index, the name of which is the value of <> + the source index name + <>. The resulting index will have <> primary shards, and <> replica shards. The shrinking will take place on the node identified by <>, unless `DETERMINISTIC` is specified, in which case Curator will evaluate all of the nodes to determine which one has the most free space. If multiple indices are identified for shrinking by the filter block, and `DETERMINISTIC` is specified, the node selection process will be repeated for each successive index, preventing all of the space being consumed on a single node. By default, Curator will delete the source index after a successful shrink. This can be disabled by setting <> to `False`. If the source index, is not deleted after a successful shrink, Curator will remove the read-only setting and the shard allocation routing applied to the source index to put it on the shrink node. Curator will wait for the shards to stop rerouting before continuing. The <> option applies to the target index after the shrink is complete. If set, this shard allocation routing will be applied (after a successful shrink) and Curator will wait for all shards to stop rerouting before continuing. The only <> which are acceptable are `settings` and `aliases`. Please note that in the example above, while `best_compression` is being applied to the new index, it will not take effect until new writes are made to the index, such as when <> the shard to a single segment. The other options are usually okay to leave at the defaults, but feel free to change them as needed. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. [[snapshot]] == Snapshot [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: ... # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' name: wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. This action will snapshot indices to the indicated <>, with a name, or name pattern, as identified by <>. The other options are usually okay to leave at the defaults, but feel free to read about them and change them accordingly. === Required settings * <> === Optional settings * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> TIP: See an example of this action in an <> <>. elasticsearch-curator-8.0.21/docs/asciidoc/command-line.asciidoc000066400000000000000000000340221477314666200246630ustar00rootroot00000000000000[[cli]] = Running Curator [partintro] -- * <> * <> * <> -- [[command-line]] == Command Line Interface Most common client configuration settings are now available at the command-line. IMPORTANT: While both the configuration file and the command-line arguments can be used together, it is important to note that command-line options will override file-based configuration of the same setting. The most basic command-line arguments are as follows: [source,sh] ------- curator [--config CONFIG.YML] [--dry-run] ACTION_FILE.YML ------- The square braces indicate optional elements. If `--config` and `CONFIG.YML` are not provided, Curator will look in `~/.curator/curator.yml` for the configuration file. `~` is the home directory of the user executing Curator. In a Unix system, this might be `/home/username/.curator/curator.yml`, while on a Windows system, it might be `C:\Users\username\.curator\curator.yml` If `--dry-run` is included, Curator will simulate the action(s) in ACTION_FILE.YML as closely as possible without actually making any changes. The results will be in the logfile, or STDOUT/command-line if no logfile is specified. `ACTION_FILE.YML` is a YAML <>. For other client configuration options, command-line help is never far away: [source,sh] ------- curator --help ------- The help output looks like this: [source,sh] ------- $ curator --help Usage: curator [OPTIONS] ACTION_FILE Curator for Elasticsearch indices The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. Some less-frequently used client configuration options are now hidden. To see the full list, run: curator_cli -h Options: --config PATH Path to configuration file. --hosts TEXT Elasticsearch URL to connect to. --cloud_id TEXT Elastic Cloud instance id --api_token TEXT The base64 encoded API Key token --id TEXT API Key "id" value --api_key TEXT API Key "api_key" value --username TEXT Elasticsearch username --password TEXT Elasticsearch password --request_timeout FLOAT Request timeout in seconds --verify_certs / --no-verify_certs Verify SSL/TLS certificate(s) [default: verify_certs] --ca_certs TEXT Path to CA certificate file or directory --client_cert TEXT Path to client certificate file --client_key TEXT Path to client key file --dry-run Do not perform any changes. --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] Log level --logfile TEXT Log file --logformat [default|ecs] Log output format -v, --version Show the version and exit. -h, --help Show this message and exit. Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/command-line.html ------- You can use <> in your configuration files. === Running Curator from Docker Running Curator from the command-line using Docker requires only a few additional steps. Should you desire to use them, Docker-based Curator requires you to map a volume for your configuration and/or log files. Attempting to read a YAML configuration file if you have neglected to volume map your configuration directory to `/.curator` will not work. It looks like this: [source,sh] ------- docker run [-t] --rm --name myimagename \ -v /PATH/TO/MY/CONFIGS:/.curator \ untergeek/curator:mytag \ --config /.curator/config.yml /.curator/actionfile.yml ------- NOTE: While testing, adding the `-t` flag will allocate a pseudo-tty, allowing you to see terminal output that would otherwise be hidden. Both of the files `config.yml` and `actionfile.yml` should already exist in the path `/PATH/TO/MY/CONFIGS` before run time. The `--rm` in the command means that the container (not the image) will be deleted after completing execution. You definitely want this as there is no reason to keep creating containers for each run. The eventual cleanup from this would be unpleasant.   [[singleton-cli]] == Singleton Command Line Interface The `curator_cli` command allows users to run a single, supported action from the command-line, without needing either the client or action YAML configuration file, though it does support using the client configuration file if you want. As an important bonus, the command-line options allow you to override the settings in the `curator.yml` file! IMPORTANT: While both the configuration file and the command-line arguments can be used together, it is important to note that command-line options will override file-based configuration of the same setting. [source,sh] --------- $ curator_cli --help Usage: curator_cli [OPTIONS] COMMAND [ARGS]... Curator CLI (Singleton Tool) Run a single action from the command-line. The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. Options: --config PATH Path to configuration file. --hosts TEXT Elasticsearch URL to connect to. --cloud_id TEXT Elastic Cloud instance id --api_token TEXT The base64 encoded API Key token --id TEXT API Key "id" value --api_key TEXT API Key "api_key" value --username TEXT Elasticsearch username --password TEXT Elasticsearch password --bearer_auth TEXT Bearer authentication token --opaque_id TEXT X-Opaque-Id HTTP header value --request_timeout FLOAT Request timeout in seconds --http_compress / --no-http_compress Enable HTTP compression [default: no-http_compress] --verify_certs / --no-verify_certs Verify SSL/TLS certificate(s) [default: verify_certs] --ca_certs TEXT Path to CA certificate file or directory --client_cert TEXT Path to client certificate file --client_key TEXT Path to client key file --ssl_assert_hostname TEXT Hostname or IP address to verify on the node's certificate. --ssl_assert_fingerprint TEXT SHA-256 fingerprint of the node's certificate. If this value is given then root-of-trust verification isn't done and only the node's certificate fingerprint is verified. --ssl_version TEXT Minimum acceptable TLS/SSL version --master-only / --no-master-only Only run if the single host provided is the elected master [default: no-master-only] --skip_version_test / --no-skip_version_test Elasticsearch version compatibility check [default: no-skip_version_test] --dry-run Do not perform any changes. --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] Log level --logfile TEXT Log file --logformat [default|ecs] Log output format -v, --version Show the version and exit. -h, --help Show this message and exit. Commands: alias Add/Remove Indices to/from Alias allocation Shard Routing Allocation close Close Indices delete-indices Delete Indices delete-snapshots Delete Snapshots forcemerge forceMerge Indices (reduce segment count) open Open Indices replicas Change Replica Count restore Restore Indices rollover Rollover Index associated with Alias show-indices Show Indices show-snapshots Show Snapshots shrink Shrink Indices to --number_of_shards snapshot Snapshot Indices Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/singleton-cli.html --------- The option flags for the given commands match those used for the same <>. The only difference is how filtering is handled. === Running Curator from Docker Running `curator_cli` from the command-line using Docker requires only a few additional steps. Should you desire to use them, Docker-based `curator_cli` requires you to map a volume for your configuration and/or log files. Attempting to read a YAML configuration file if you have neglected to volume map your configuration directory to `/.curator` will not work. It looks like this: [source,sh] ------- docker run [-t] --rm --name myimagename \ --entrypoint /curator/curator_cli \ -v /PATH/TO/MY/CONFIGS:/.curator \ untergeek/curator:mytag \ --config /.curator/config.yml [OPTIONS] COMMAND [ARGS]... ------- NOTE: While testing, adding the `-t` flag will allocate a pseudo-tty, allowing you to see terminal output that would otherwise be hidden. The `config.yml` file should already exist in the path `/PATH/TO/MY/CONFIGS` before run time. The `--rm` in the command means that the container (not the image) will be deleted after completing execution. You definitely want this as there is no reason to keep creating containers for each run. The eventual cleanup from this would be unpleasant. === Command-line filtering Recent improvements in Curator include schema and setting validation. With these improvements, it is possible to validate filters and their many permutations if passed in a way that Curator can easily digest. [source,sh] ----------- --filter_list TEXT JSON string representing an array of filters. ----------- This means that filters need to be passed as a single object, or an array of objects in JSON format. Single: [source,sh] ----------- --filter_list '{"filtertype":"none"}' ----------- Multiple: [source,sh] ----------- --filter_list '[{"filtertype":"age","source":"creation_date","direction":"older","unit":"days","unit_count":13},{"filtertype":"pattern","kind":"prefix","value":"logstash"}]' ----------- This preserves the power of chained filters, making them available on the command line. NOTE: You may need to escape all of the double quotes on some platforms, or shells like PowerShell, for instance. Caveats to this approach: 1. Only one action can be taken at a time. 2. Not all actions have singleton analogs. For example, <> and + <> do not have singleton actions. === Show Indices/Snapshots One feature that the singleton command offers that the other cannot is to show which indices and snapshots are in the system. It's a great way to visually test your filters without causing any harm to the system. [source,sh] ----------- $ curator_cli show-indices --help Usage: curator_cli show-indices [OPTIONS] Show indices Options: --verbose Show verbose output. --header Print header if --verbose --epoch Print time as epoch if --verbose --filter_list TEXT JSON string representing an array of filters. [required] --help Show this message and exit. Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/singleton-cli.html#_show_indicessnapshots ----------- [source,sh] ----------- $ curator_cli show-snapshots --help Usage: curator_cli show-snapshots [OPTIONS] Show snapshots Options: --repository TEXT Snapshot repository name [required] --filter_list TEXT JSON string representing an array of filters. [required] --help Show this message and exit. Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/singleton-cli.html#_show_indicessnapshots ----------- The `show-snapshots` command will only show snapshots matching the provided filters. The `show-indices` command will also do this, but also offers a few extra features. * `--verbose` adds state, total size of primary and all replicas, the document count, the number of primary and replica shards, and the creation date in ISO8601 format. * `--header` adds a header that shows the column names. This only occurs if `--verbose` is also selected. * `--epoch` changes the date format from ISO8601 to epoch time. If `--header` is also selected, the column header title will change to `creation_date` There are no extra columns or `--verbose` output for the `show-snapshots` command. Without `--epoch` [source,sh] ----------- Index State Size Docs Pri Rep Creation Timestamp logstash-2016.10.20 close 0.0B 0 5 1 2016-10-20T00:00:03Z logstash-2016.10.21 open 763.3MB 5860016 5 1 2016-10-21T00:00:03Z logstash-2016.10.22 open 759.1MB 5858450 5 1 2016-10-22T00:00:04Z logstash-2016.10.23 open 757.8MB 5857456 5 1 2016-10-23T00:00:04Z logstash-2016.10.24 open 771.5MB 5859720 5 1 2016-10-24T00:00:00Z logstash-2016.10.25 open 771.0MB 5860112 5 1 2016-10-25T00:00:01Z logstash-2016.10.27 open 658.3MB 4872830 5 1 2016-10-27T00:00:03Z logstash-2016.10.28 open 655.1MB 5237250 5 1 2016-10-28T00:00:00Z ----------- With `--epoch` [source,sh] ----------- Index State Size Docs Pri Rep creation_date logstash-2016.10.20 close 0.0B 0 5 1 1476921603 logstash-2016.10.21 open 763.3MB 5860016 5 1 1477008003 logstash-2016.10.22 open 759.1MB 5858450 5 1 1477094404 logstash-2016.10.23 open 757.8MB 5857456 5 1 1477180804 logstash-2016.10.24 open 771.5MB 5859720 5 1 1477267200 logstash-2016.10.25 open 771.0MB 5860112 5 1 1477353601 logstash-2016.10.27 open 658.3MB 4872830 5 1 1477526403 logstash-2016.10.28 open 655.1MB 5237250 5 1 1477612800 -----------   [[exit-codes]] == Exit Codes Exit codes will indicate success or failure. * `0` — Success * `1` — Failure * `-1` - Exception raised that does not result in a `1` exit code.   elasticsearch-curator-8.0.21/docs/asciidoc/configuration.asciidoc000066400000000000000000000332421477314666200251720ustar00rootroot00000000000000[[configuration]] = Configuration [partintro] -- These are the higher-level configuration settings used by the configuration files. <> and <> are documented separately. * <> * <> * <> -- [[envvars]] == Environment Variables WARNING: This functionality is experimental and may be changed or removed + completely in a future release. You can use environment variable references in both the <> and the <> to set values that need to be configurable at runtime. To do this, use: [source,sh] ------- ${VAR} ------- Where `VAR` is the name of the environment variable. Each variable reference is replaced at startup by the value of the environment variable. The replacement is case-sensitive and occurs while the YAML file is parsed, but before configuration schema validation. References to undefined variables are replaced by `None` unless you specify a default value. To specify a default value, use: [source,sh] ------- ${VAR:default_value} ------- Where `default_value` is the value to use if the environment variable is undefined. [IMPORTANT] .Unsupported use cases ================================================================= When using environment variables, the value must _only_ be the environment variable. Using extra text, such as: [source,sh] ------- logfile: ${LOGPATH}/extra/path/information/file.log ------- is not supported at this time. ================================================================= === Examples Here are some examples of configurations that use environment variables and what each configuration looks like after replacement: [options="header"] |================================== |Config source |Environment setting |Config after replacement |`unit: ${UNIT}` |`export UNIT=days` |`unit: days` |`unit: ${UNIT}` |no setting |`unit:` |`unit: ${UNIT:days}` |no setting |`unit: days` |`unit: ${UNIT:days}` |`export UNIT=hours` |`unit: hours` |================================== [[actionfile]] == Action File NOTE: You can use <> in your configuration files. An action file has the following structure: [source,sh] ----------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: ACTION1 description: OPTIONAL DESCRIPTION options: option1: value1 ... optionN: valueN continue_if_exception: False disable_action: True filters: - filtertype: *first* filter_element1: value1 ... filter_elementN: valueN - filtertype: *second* filter_element1: value1 ... filter_elementN: valueN 2: action: ACTION2 description: OPTIONAL DESCRIPTION options: option1: value1 ... optionN: valueN continue_if_exception: False disable_action: True filters: - filtertype: *first* filter_element1: value1 ... filter_elementN: valueN - filtertype: *second* filter_element1: value1 ... filter_elementN: valueN 3: action: ACTION3 ... 4: action: ACTION4 ... ----------- It is a YAML configuration file. The root key must be `actions`, after which there can be any number of actions, nested underneath numbers. Actions will be taken in the order they are completed. The high-level elements of each numbered action are: * <> * <> * <> * <> In the case of the <>, there are two additional high-level elements: `add` and `remove`, which are described in the <> documentation. [[description]] === description This is an optional description which can help describe what the action and its filters are supposed to do. [source,yaml] ------------- description: >- I can make the description span multiple lines by putting ">-" at the beginning of the line, as seen above. Subsequent lines must also be indented. options: option1: ... ------------- [[configfile]] == Configuration File NOTE: The default location of the configuration file is `~/.curator/curator.yml`, but another location can be specified using the `--config` flag on the <>. NOTE: You can use <> in your configuration files. The configuration file contains client connection and settings for logging. It looks like this: [source,sh] ----------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" elasticsearch: client: hosts: - http://127.0.0.1:9200 cloud_id: ca_certs: client_cert: client_key: verify_certs: request_timeout: 30 other_settings: master_only: False username: password: api_key: id: api_key: token: logging: loglevel: INFO logfile: logformat: default blacklist: ['elastic_transport', 'urllib3'] ----------- It is a YAML configuration file. The two root keys must be `elasticsearch` and `logging`. The subkeys of each of these will be described here. There are other keys available for the `client` subkey of the `elasticsearch` root key, many of which are listed https://es-client.readthedocs.io/en/latest/defaults.html[here]. The most commonly used ones (listed above) are described as follows: [[hosts]] === hosts IMPORTANT: All hosts must be in `HTTP[S]://FQDN.DOMAIN.TLD:PORT` form or they will be rejected and Curator will exit with an error. The only exception to this is `HTTPS://FQDN.DOMAIN.TLD` (without port), in which case `:443` is implicit, and is, in fact, appended automatically. WARNING: If both `cloud_id` and `hosts` keys are populated an exception will be thrown and Curator will exit. A `hosts` definition can be a single value: [source,sh] ----------- hosts: http://127.0.0.1:9200 ----------- Or multiple values in the 3 acceptable YAML ways to render sequences, or arrays: WARNING: Curator can only work with one cluster at a time. Including clients from multiple clusters in the `hosts` setting will result in errors. Flow: [source,sh] ----------- hosts: [ "http://10.0.0.1:9200", "http://10.0.0.2:9200" ] ----------- Spanning: [source,sh] ----------- hosts: [ "http://10.0.0.1:9200", "http://10.0.0.2:9200" ] ----------- Block: [source,sh] ----------- hosts: - http://10.0.0.1:9200 - http://10.0.0.2:9200 ----------- [[cloud_id]] === cloud_id The value should encapsulated in quotes because of the included colon: [source,sh] ----------- cloud_id: 'deployment_name:BIG_HASH_VALUE' ----------- WARNING: If both `cloud_id` and `hosts` keys are populated an exception will be thrown and Curator will exit. [[ca_certs]] === ca_certs This should be a file path to your CA certificate, or left empty. [source,sh] ----------- ca_certs: ----------- This setting allows the use of a specified CA certificate file to validate the SSL certificate used by Elasticsearch. There is no default. include::inc_filepath.asciidoc[] [[client_cert]] === client_cert This should be a file path to a client certificate (public key), or left empty. [source,sh] ----------- client_cert: ----------- Allows the use of a specified SSL client cert file to authenticate to Elasticsearch. The file may contain both an SSL client certificate and an SSL key, in which case <> is not used. If specifying `client_cert`, and the file specified does not also contain the key, use <> to specify the file containing the SSL key. The file must be in PEM format, and the key part, if used, must be an unencrypted key in PEM format as well. include::inc_filepath.asciidoc[] [[client_key]] === client_key This should be a file path to a client key (private key), or left empty. [source,sh] ----------- client_key: ----------- Allows the use of a specified SSL client key file to authenticate to Elasticsearch. If using <> and the file specified does not also contain the key, use `client_key` to specify the file containing the SSL key. The key file must be an unencrypted key in PEM format. include::inc_filepath.asciidoc[] [[verify_certs]] === verify_certs This should be `True`, `False` or left empty. [source,sh] ----------- verify_certs: ----------- If access to your Elasticsearch instance is protected by SSL encryption, you may set `verify_certs` to `False` to disable SSL certificate verification. Valid use cases for doing so include the use of self-signed certificates that cannot be otherwise verified and would generate error messages. WARNING: Setting `verify_certs` to `False` will likely result in a warning message that your SSL certificates are not trusted. This is expected behavior. The default value is `True`. [[request_timeout]] === request_timeout This should be an integer number of seconds, or left empty. [source,sh] ----------- request_timeout: ----------- You can change the default client connection timeout value with this setting. The default value is `30` (seconds) should typically not be changed to be very large. If a longer timeout is necessary for a given action, such as <>, <>, or <>, the client timeout can be overridden on per action basis by setting <> in the action <>. There are default override values for some of those longer running actions. [[master_only]] === master_only This should be `True`, `False` or left empty. [source,sh] ----------- master_only: ----------- In some situations, primarily with automated deployments, it makes sense to install Curator on every node. But you wouldn’t want it to run on each node. By setting `master_only` to `True`, this is possible. It tests for, and will only continue running on the node that is the elected master. WARNING: If `master_only` is `True`, and <> has more than one value, Curator will raise an Exception. This setting should _only_ be used with a single host in <>, as its utility centers around deploying to all nodes in the cluster. The default value is `False`. [[username]] === username The HTTP Basic Authentication username [[password]] === password The HTTP Basic Authentication password [[id]] === id This should be the `id` portion of an API Key pair. [source,sh] --------- api_key: id: --------- This setting combined with the other subkey `api_key` allows API Key authentication to an Elasticsearch instance. The default is empty. [[api_key]] === api_key This should be the `api_key` portion of an API Key pair. [source,sh] --------- api_key: api_key: --------- This setting combined with the other subkey `id` allows API Key authentication to an Elasticsearch instance. The default is empty. [[token]] === token This should be a base64 encoded representation of an API Key pair. [source,sh] --------- api_key: token: --------- This setting will override any values provided for the `id` or `api_key` subkeys of `api_key`. The default is empty. [[loglevel]] === loglevel This should be `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`, or left empty. [source,sh] ----------- loglevel: ----------- Set the minimum acceptable log severity to display. * `CRITICAL` will only display critical messages. * `ERROR` will only display error and critical messages. * `WARNING` will display error, warning, and critical messages. * `INFO` will display informational, error, warning, and critical messages. * `DEBUG` will display debug messages, in addition to all of the above. The default value is `INFO`. [[logfile]] === logfile This should be a path to a log file, or left empty. [source,sh] ----------- logfile: ----------- include::inc_filepath.asciidoc[] The default value is empty, which will result in logging to `STDOUT`, or the console. [[logformat]] === logformat This should `default`, `json`, `logstash`, `ecs` or left empty. [source,sh] ----------- logformat: ----------- The `default` format looks like: [source,sh] ----------- 2016-04-22 11:53:09,972 INFO Action #1: ACTIONNAME ----------- The `json` or `logstash` formats look like: [source,sh] ----------- {"@timestamp": "2016-04-22T11:54:29.033Z", "function": "cli", "linenum": 178, "loglevel": "INFO", "message": "Action #1: ACTIONNAME", "name": "curator.cli"} ----------- The `ecs` format looks like: [source,sh] ----------- {"@timestamp":"2020-02-22T11:55:00.022Z","log.level":"info","message":"Action #1: ACTIONNAME","ecs":{"version":"1.6.0"},"log":{"logger":"curator.cli","origin": {"file":{"line":178,"name":"cli.py"},"function":"run"},"original":"Action #1: ACTIONNAME"},"process":{"name":"MainProcess","pid":12345,"thread": {"id":123456789886543,"name":"MainThread"}}} ----------- The default value is `default`. [[blacklist]] === blacklist This should be an empty array `[]`, an array of log handler strings, or left empty. [source,sh] ----------- blacklist: ['elastic_transport', 'urllib3'] ----------- The default value is `['elastic_transport', 'urllib3']`, which will result in logs for the `elastic_transport` and `urllib3` Python modules _not_ being output. These can be quite verbose, so unless you need them to debug an issue, you should accept the default value. TIP: If you do need to troubleshoot an issue, set `blacklist` to `[]`, which is an empty array. Leaving it unset will result in the default behavior, which is to filter out `elastic_transport` and `urllib3` log traffic. elasticsearch-curator-8.0.21/docs/asciidoc/examples.asciidoc000066400000000000000000000476561477314666200241570ustar00rootroot00000000000000[[examples]] = Examples [partintro] -- These examples should help illustrate how to build your own <>. You can use <> in your configuration files. * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> -- [[ex_alias]] == alias [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: alias description: >- Alias indices from last week, with a prefix of logstash- to 'last_week', remove indices from the previous week. options: name: last_week warn_if_no_indices: False disable_action: True add: filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday remove: filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: period period_type: relative source: name range_from: -2 range_to: -2 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday ------------- [[ex_allocation]] == allocation [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: allocation description: >- Apply shard allocation routing to 'require' 'tag=cold' for hot/cold node setup for logstash- indices older than 3 days, based on index_creation date options: key: tag value: cold allocation_type: require disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ------------- [[ex_close]] == close [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: close description: >- Close indices older than 30 days (based on index name), for logstash- prefixed indices. options: skip_flush: False delete_aliases: False ignore_sync_failures: True disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 30 ------------- [[ex_cluster_routing]] == cluster_routing [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. # # This action example has a blank spot at action ID 2. This is to show that # Curator can disable allocation before one or more actions, and then re-enable # it afterward. actions: 1: action: cluster_routing description: >- Disable shard routing for the entire cluster. options: routing_type: allocation value: none setting: enable wait_for_completion: True disable_action: True 2: action: (any other action details go here) ... 3: action: cluster_routing description: >- Re-enable shard routing for the entire cluster. options: routing_type: allocation value: all setting: enable wait_for_completion: True disable_action: True ------------- [[ex_create_index]] == create_index [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: create_index description: Create the index as named, with the specified extra settings. options: name: myindex extra_settings: settings: number_of_shards: 2 number_of_replicas: 1 disable_action: True ------------- [[ex_delete_indices]] == delete_indices [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: delete_indices description: >- Delete indices older than 45 days (based on index name), for logstash- prefixed indices. Ignore the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly. options: ignore_empty_list: True disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 45 ------------- [[ex_delete_snapshots]] == delete_snapshots [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: delete_snapshots description: >- Delete snapshots from the selected repository older than 45 days (based on creation_date), for 'curator-' prefixed snapshots. options: repository: disable_action: True filters: - filtertype: pattern kind: prefix value: curator- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 45 ------------- [[ex_forcemerge]] == forcemerge [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: forcemerge description: >- forceMerge logstash- prefixed indices older than 2 days (based on index creation_date) to 2 segments per shard. Delay 120 seconds between each forceMerge operation to allow the cluster to quiesce. Skip indices that have already been forcemerged to the minimum number of segments to avoid reprocessing. options: max_num_segments: 2 delay: 120 timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 2 exclude: - filtertype: forcemerged max_num_segments: 2 exclude: ------------- [[ex_index_settings]] == index_settings [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: index_settings description: >- Set Logstash indices older than 10 days to be read only (block writes) options: disable_action: True index_settings: index: blocks: write: True ignore_unavailable: False preserve_existing: False filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 10 ------------- [[ex_open]] == open [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: open description: >- Open indices older than 30 days but younger than 60 days (based on index name), for logstash- prefixed indices. options: disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 30 - filtertype: age source: name direction: younger timestring: '%Y.%m.%d' unit: days unit_count: 60 ------------- [[ex_reindex]] == reindex === Manually selected reindex of a single index [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: "Reindex index1 into index2" action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- === Manually selected reindex of a multiple indices [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ------------- === Filter-Selected Indices [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: >- 'Reindex all daily logstash indices from March 2017 into logstash-2017.03' action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: index: REINDEX_SELECTION dest: index: logstash-2017.03 filters: - filtertype: pattern kind: prefix value: logstash-2017.03. ------------- === Reindex From Remote [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: >- 'Reindex all daily logstash indices from March 2017 into logstash-2017.03' action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: index1 dest: index: index1 filters: - filtertype: none ------------- === Reindex From Remote With Filter-Selected Indices [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: >- Reindex all remote daily logstash indices from March 2017 into local index logstash-2017.03 action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: REINDEX_SELECTION dest: index: logstash-2017.03 remote_filters: - filtertype: pattern kind: prefix value: logstash-2017.03. filters: - filtertype: none ------------- === Manually selected reindex of a single index with query [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: "Reindex index1 into index2" action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: query: range: timestamp: gte: "now-1h" index: index1 dest: index: index2 filters: - filtertype: none ------------- [[ex_replicas]] == replicas [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: replicas description: >- Reduce the replica count to 0 for logstash- prefixed indices older than 10 days (based on index creation_date) options: count: 0 wait_for_completion: True disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 10 ------------- [[ex_restore]] == restore [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: restore description: >- Restore all indices in the most recent curator-* snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: include_aliases: False ignore_unavailable: False include_global_state: False partial: False rename_pattern: rename_replacement: extra_settings: wait_for_completion: True skip_repo_fs_check: True disable_action: True filters: - filtertype: pattern kind: prefix value: curator- - filtertype: state state: SUCCESS ------------- [[ex_rollover]] == rollover [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the format of prefix-000001 (or similar), or prefix-YYYY.MM.DD-1. options: disable_action: True name: aliasname conditions: max_age: 1d max_docs: 1000000 max_size: 50g extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 ------------- [[ex_shrink]] == shrink [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: shrink description: >- Shrink logstash indices older than 21 days on the node with the most available space, excluding the node named 'not_this_node'. Delete each source index after successful shrink, then reroute the shrunk index with the provided parameters. options: disable_action: True ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 21 ------------- [[ex_snapshot]] == snapshot [source,yaml] ------------- --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: snapshot description: >- Snapshot logstash- prefixed indices older than 1 day (based on index creation_date) with the default snapshot name pattern of 'curator-%Y%m%d%H%M%S'. Wait for the snapshot to complete. Do not skip the repository filesystem access check. Use the other options to create the snapshot. options: repository: # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' name: ignore_unavailable: False include_global_state: True partial: False wait_for_completion: True skip_repo_fs_check: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 1 ------------- elasticsearch-curator-8.0.21/docs/asciidoc/faq.asciidoc000066400000000000000000000141571477314666200230760ustar00rootroot00000000000000[[faq]] = Frequently Asked Questions [partintro] -- This section will be updated as more frequently asked questions arise * <> * <> * <> -- [[faq_doc_error]] == Q: How can I report an error in the documentation? === A: Use the "Edit" link on any page See <>. [[faq_partial_delete]] == Q: Can I delete only certain data from within indices? === A: It's complicated [float] TL;DR: No. Curator can only delete entire indices. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [float] Full answer: ^^^^^^^^^^^^ As a thought exercise, think of Elasticsearch indices as being like databases, or tablespaces within a database. If you had hundreds of millions of rows to delete from your database, would you run a separate `DELETE from TABLE where date> with <> set to `regex`, and <> set to the needed regular expression. [float] The Problem: ^^^^^^^^^^^^ Illegal characters make it hard to delete indices. ------------------ % curl logs.example.com:9200/_cat/indices red }?ebc-2015.04.08.03 sip-request{ 5 1 0 0 632b 316b red }?ebc-2015.04.08.03 sip-response 5 1 0 0 474b 237b red ?ebc-2015.04.08.02 sip-request{ 5 1 0 0 474b 316b red eb 5 1 0 0 632b 316b red ?e 5 1 0 0 632b 316b ------------------   You can see it looks like there are some tab characters and maybe newline characters. This makes it hard to use the HTTP API to delete the indices. Dumping all the index settings out: [source,sh] ------- curl -XGET localhost:9200/*/_settings?pretty -------   ...reveals the index names as the first key in the resulting JSON. In this case, the names were very atypical: ------- }\b?\u0011ebc-2015.04.08.02\u000Bsip-request{ }\u0006?\u0011ebc-2015.04.08.03\u000Bsip-request{ }\u0003?\u0011ebc-2015.04.08.03\fsip-response ... -------   Curator lets you use regular expressions to select indices to perform actions on. WARNING: Before attempting an action, see what will be affected by using the `--dry-run` flag first. To delete the first three from the above example, use `'.*sip.*'` as your regular expression. NOTE: In an <>, regular expressions and strftime date strings _must_ be encapsulated in single-quotes. The next one is trickier. The real name of the index was `\n\u0011eb`. The regular expression `.*b$` did not work, but `'\n.*'` did. The last index can be deleted with a regular expression of `'.*e$'`. The resulting <> might look like this: [source,yaml] -------- actions: 1: description: Delete indices with strange characters that match regex '.*sip.*' action: delete_indices options: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: regex value: '.*sip.*' 2: description: Delete indices with strange characters that match regex '\n.*' action: delete_indices options: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: regex value: '\n.*' 3: description: Delete indices with strange characters that match regex '.*e$' action: delete_indices options: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: regex value: '.*e$' -------- elasticsearch-curator-8.0.21/docs/asciidoc/filter_elements.asciidoc000066400000000000000000000665511477314666200255150ustar00rootroot00000000000000 [[filter_elements]] = Filter Elements [partintro] -- * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> You can use <> in your configuration files. -- [[fe_aliases]] == aliases include::inc_filter_by_aliases.asciidoc[] NOTE: This setting is used only when using the <> filter. The value of this setting must be a single alias name, or a list of alias names. This can be done in any of the ways YAML allows for lists or arrays. Here are a few examples. **Single** [source,txt] ------- filters: - filtertype: alias aliases: my_alias exclude: False ------- **List** - Flow style: + [source,txt] ------- filters: - filtertype: alias aliases: [ my_alias, another_alias ] exclude: False ------- - Block style: + [source,txt] ------- filters: - filtertype: alias aliases: - my_alias - another_alias exclude: False ------- There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[fe_allocation_type]] == allocation_type NOTE: This setting is used only when using the <> filter. [source,yaml] ------------- - filtertype: allocated key: ... value: ... allocation_type: require exclude: True ------------- The value of this setting must be one of `require`, `include`, or `exclude`. Read more about these settings in the {ref}/shard-allocation-filtering.html[Elasticsearch documentation]. The default value for this setting is `require`. [[fe_count]] == count NOTE: This setting is only used with the <> filtertype + and is a required setting. [source,yaml] ------------- - filtertype: count count: 10 ------------- The value for this setting is a number of indices or snapshots to match. Items will remain in the actionable list depending on the value of <>, and <>. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[fe_date_from]] == date_from NOTE: This setting is only used with the <> filtertype + when <> is `absolute`. [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ------------- The value for this setting should be a date which can be easily parsed using the strftime string in <>: include::inc_strftime_table.asciidoc[] [[fe_date_from_format]] == date_from_format NOTE: This setting is only used with the <> filtertype + when <> is `absolute`. [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ------------- The value for this setting should be an strftime string which corresponds to the date in <>: include::inc_strftime_table.asciidoc[] [[fe_date_to]] == date_to NOTE: This setting is only used with the <> filtertype + when <> is `absolute`. [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ------------- The value for this setting should be a date which can be easily parsed using the strftime string in <>: include::inc_strftime_table.asciidoc[] [[fe_date_to_format]] == date_to_format NOTE: This setting is only used with the <> filtertype + when <> is `absolute`. [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ------------- The value for this setting should be an strftime string which corresponds to the date in <>: include::inc_strftime_table.asciidoc[] [[fe_direction]] == direction NOTE: This setting is only used with the <> filtertype. [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ------------- This setting must be either `older` or `younger`. This setting is used to determine whether indices or snapshots are `older` or `younger` than the reference point in time determined by <>, <>, and optionally, <>. If `direction` is `older`, then indices (or snapshots) which are _older_ than the reference point in time will be matched. Likewise, if `direction` is `younger`, then indices (or snapshots) which are _younger_ than the reference point in time will be matched. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[fe_disk_space]] == disk_space NOTE: This setting is only used with the <> filtertype + and is a required setting. [source,yaml] ------------- - filtertype: space disk_space: 100 ------------- The value for this setting is a number of gigabytes. Indices in excess of this number of gigabytes will be matched. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[fe_epoch]] == epoch NOTE: This setting is available in the <> filtertype, and any filter which has the <> setting. This setting is strictly optional. TIP: This setting is not common. It is most frequently used for testing. <>, <>, and optionally, <>, are used by Curator to establish the moment in time point of reference with this formula: [source,sh] ----------- point_of_reference = epoch - ((number of seconds in unit) * unit_count) ----------- If <> is unset, the current time is used. It is possible to set a point of reference in the future by using a negative value for <>. === Example [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 epoch: 1491577200 ------------- The value for this setting must be an epoch timestamp. In this example, the given epoch time of `1491577200` is 2017-04-04T15:00:00Z (UTC). This will use 3 days older than that timestamp as the point of reference for age comparisons. [[fe_exclude]] == exclude NOTE: This setting is available in _all_ filter types. If `exclude` is `True`, the filter will remove matches from the actionable list. If `exclude` is `False`, then only matches will be kept in the actionable list. The default value for this setting is different for each filter type. === Examples [source,yaml] ------------- - filtertype: opened exclude: True ------------- This filter will result in only `closed` indices being in the actionable list. [source,yaml] ------------- - filtertype: opened exclude: False ------------- This filter will result in only `open` indices being in the actionable list. [[fe_field]] == field NOTE: This setting is available in the <> filtertype, and any filter which has the <> setting. This setting is strictly optional. [source,yaml] ------------- - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ------------- The value of this setting must be a timestamp field name. This field must be present in the indices being filtered or an exception will be raised, and execution will halt. In Curator 5.3 and older, source `field_stats` uses the http://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.html[Field Stats API] to calculate either the `min_value` or the `max_value` of the <> as the <>, and then use that value for age comparisons. In 5.4 and above, even though it is still called `field_stats`, it uses an aggregation to calculate the same values, as the `field_stats` API is no longer used in Elasticsearch 6.x and up. This setting is only used when <> is `field_stats`. The default value for this setting is `@timestamp`. [[fe_intersect]] == intersect NOTE: This setting is only available in the <> filtertype. This setting is strictly optional. [source,yaml] ------------- - filtertype: period source: field_stats direction: older intersect: true unit: weeks range_from: -1 range_to: -1 field: '@timestamp' stats_result: min_value ------------- The value of this setting must be `True` or `False`. `field_stats` uses an aggregation query to calculate either the `min_value` and the `max_value` of the <> as the <>. If `intersect` is `True`, then only indices where the `min_value` _and_ the `max_value` are within the `range_from` and `range_to` (relative to `unit`) will match. This means that either `min_value` or `max_value` can be used for <> when `intersect` is `True` with identical results. This setting is only used when <> is `field_stats`. The default value for this setting is `False`. [[fe_key]] == key NOTE: This setting is required when using the <>. [source,yaml] ------------- - filtertype: allocated key: ... value: ... allocation_type: exclude: True ------------- The value of this setting should correspond to a node setting on one or more nodes in your cluster. For example, you might have set [source,sh] ----------- node.tag: myvalue ----------- in your `elasticsearch.yml` file for one or more of your nodes. To match allocation in this case, set key to `tag`. These special attributes are also supported: [cols="2*", options="header"] |=== |attribute |description |`_name` |Match nodes by node name |`_host_ip` |Match nodes by host IP address (IP associated with hostname) |`_publish_ip` |Match nodes by publish IP address |`_ip` |Match either `_host_ip` or `_publish_ip` |`_host` |Match nodes by hostname |=== There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[fe_kind]] == kind NOTE: This setting is only used with the <> + filtertype and is a required setting. This setting tells the <> what pattern type to match. Acceptable values for this setting are `prefix`, `suffix`, `timestring`, and `regex`. include::inc_filter_chaining.asciidoc[] There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. include::inc_kinds.asciidoc[] [[fe_max_num_segments]] == max_num_segments NOTE: This setting is only used with the <> filtertype. [source,yaml] ------------- - filtertype: forcemerged max_num_segments: 2 exclude: True ------------- The value for this setting is the cutoff number of segments per shard. Indices which have this number of segments per shard, or fewer, will be actionable depending on the value of <>, which is `True` by default for the <> filter type. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[fe_pattern]] == pattern NOTE: This setting is only used with the <> filtertype [source,yaml] ------------- - filtertype: count count: 1 pattern: '^(.*)-\d{6}$' reverse: true ------------- This particular example will match indices following the basic rollover pattern of `indexname-######`, and keep the highest numbered index for each group. For example, given indices `a-000001`, `a-000002`, `a-000003` and `b-000006`, and `b-000007`, the indices will would be matched are `a-000003` and `b-000007`. Indices that do not match the regular expression in `pattern` will be automatically excluded. This is particularly useful with indices created and managed using the {ref}/indices-rollover-index.html[Rollover API], as you can select only the active indices with the above example (<> defaults to `False`). Setting <> to `True` with the above example will _remove_ the active rollover indices, leaving only those which have been rolled-over. While this is perhaps most useful for the aforementioned scenario, it can also be used with age-based indices as well. Items will remain in the actionable list depending on the value of <>, and <>. There is no default value. The value must include a capture group, defined by parenthesis, or left empty. If a value is provided, and there is no capture group, and exception will be raised and execution will halt. [[fe_period_type]] == period_type NOTE: This setting is only used with the <> filtertype [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ------------- The value for this setting must be either `relative` or `absolute`. The default value is `relative`. [[fe_range_from]] == range_from NOTE: This setting is only used with the <> filtertype [source,yaml] ------------- - filtertype: period source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: days ------------- <> and <> are counters of whole <>. A negative number indicates a whole unit in the past, while a positive number indicates a whole unit in the future. A `0` indicates the present unit. Read more about this setting in context in the <>, including examples. [[fe_range_to]] == range_to NOTE: This setting is only used with the <> filtertype [source,yaml] ------------- - filtertype: period source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: days ------------- <> and <> are counters of whole <>. A negative number indicates a whole unit in the past, while a positive number indicates a whole unit in the future. A `0` indicates the present unit. Read more about this setting in context in the <>, including examples. [[fe_reverse]] == reverse NOTE: This setting is used in the <> and <> filtertypes This setting affects the sort order of the indices. `True` means reverse-alphabetical. This means that if all index names share the same pattern with a date--e.g. index-2016.03.01--older indices will be selected first. The default value of this setting is `True`. This setting is ignored if <> is `True`. TIP: There are context-specific examples of how `reverse` works in the <> and <> documentation. [[fe_source]] == source The _source_ from which to derive the index or snapshot age. Can be one of `name`, `creation_date`, or `field_stats`. NOTE: This setting is only used with the <> filtertype, or + with the <> filtertype when <> is set to `True`. NOTE: When using the <> filtertype, source requires + <>, <>, <>, + and additionally, the optional setting, <>. include::inc_sources.asciidoc[] [[fe_state]] == state NOTE: This setting is only used with the <> filtertype. [source,yaml] ------------- - filtertype: state state: SUCCESS ------------- The value for this setting must be one of `SUCCESS`, `PARTIAL`, `FAILED`, or `IN_PROGRESS`. This setting determines what kind of snapshots will be passed. The default value for this setting is `SUCCESS`. [[fe_stats_result]] == stats_result NOTE: This setting is only used with the <> filtertype. [source,yaml] ------------- - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ------------- The value for this setting can be either `min_value` or `max_value`. This setting is only used when <> is `field_stats`, and determines whether Curator will use the minimum or maximum value of <> for time calculations. The default value for this setting is `min_value`. [[fe_timestring]] == timestring NOTE: This setting is only used with the <> filtertype, or + with the <> filtertype if <> is set to `True`. === strftime This setting must be a valid Python strftime string. It is used to match and extract the timestamp in an index or snapshot name. include::inc_strftime_table.asciidoc[] These identifiers may be combined with each other, and/or separated from each other with hyphens `-`, periods `.`, underscores `_`, or other characters valid in an index name. Each identifier must be preceded by a `%` character in the timestring. For example, an index like `index-2016.04.01` would use a timestring of `'%Y.%m.%d'`. When <> is `name`, this setting must be set by the user or an exception will be raised, and execution will halt. There is no default value. include::inc_timestring_regex.asciidoc[] [[fe_threshold_behavior]] == threshold_behavior NOTE: This setting is only available in the <> filtertype. This setting is optional, and defaults to `greater_than` to preserve backwards compatability. [source,yaml] ------------- - filtertype: space disk_space: 5 threshold_behavior: less_than ------------- The value for this setting is `greater_than` (default) or `less_than`. When set to `less_than`, indices in less than `disk_space` gigabytes will be matched. When set to `greater_than` (default), indices larger than `disk_space` will be matched. [[fe_unit]] == unit NOTE: This setting is used with the <> filtertype, with the <> filtertype, or with the <> filtertype if <> is set to `True`. [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ------------- This setting must be one of `seconds`, `minutes`, `hours`, `days`, `weeks`, `months`, or `years`. The values `seconds` and `minutes` are not allowed with the <> filtertype and will result in an error condition if used there. For the <> filtertype, or when <> is set to `True`, <>, <>, and optionally, <>, are used by Curator to establish the moment in time point of reference with this formula: [source,sh] ----------- point_of_reference = epoch - ((number of seconds in unit) * unit_count) ----------- include::inc_unit_table.asciidoc[] If <> is unset, the current time is used. It is possible to set a point of reference in the future by using a negative value for <>. This setting must be set by the user or an exception will be raised, and execution will halt. TIP: See the <> for more information about time calculation. [[fe_unit_count]] == unit_count NOTE: This setting is only used with the <> filtertype, or + with the <> filtertype if <> is set to `True`. [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ------------- The value of this setting will be used as a multiplier for <>. <>, <>, and optionally, <>, are used by Curator to establish the moment in time point of reference with this formula: [source,sh] ----------- point_of_reference = epoch - ((number of seconds in unit) * unit_count) ----------- include::inc_unit_table.asciidoc[] If <> is unset, the current time is used. It is possible to set a point of reference in the future by using a negative value for <>. This setting must be set by the user or an exception will be raised, and execution will halt. If this setting is used in conjunction with <>, the configured value will only be used as a fallback value in case the pattern could not be matched. The value _-1_ has a special meaning in this case and causes the index to be ignored when pattern matching fails. TIP: See the <> for more information about time calculation. [[fe_unit_count_pattern]] == unit_count_pattern NOTE: This setting is only used with the age filtertype to define, whether the <> value is taken from the configuration or read from the index name via a regular expression. [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 unit_count_pattern: -([0-9]+)- ------------- This setting can be used in cases where the value against which index age should be assessed is not a static value but can be different for every index. For this case, there is the option of extracting the index specific value from the index names via a regular expression defined in this parameter. Consider for example the following index name patterns that contain the retention time in their name: _logstash-30-yyyy.mm.dd_, _logstash-12-yyyy.mm_, __3_logstash-yyyy.mm.dd_. To extract a value from the index names, this setting will be compiled as a regular expression and matched against index names, for a successful match, the value of the first capture group from the regular expression is used as the value for <>. If there is any error during compiling or matching the expression, or the expression does not contain a capture group, the value configured in <> is used as a fallback value, unless it is set to _-1_, in which case the index will be skipped. TIP: Regular expressions and match groups are not explained here as they are a fairly large and complex topic, but there are numerous resources online that will help. Using an online tool for testing regular expressions like https://regex101.com/[regex101.com] will help a lot when developing patterns. *Examples* * _logstash-30-yyyy.mm.dd_: Daily index that should be deleted after 30 days, indices that don't match the pattern will be deleted after 365 days [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 365 unit_count_pattern: -([0-9]+)- ------------- * _logstash-12-yyyy.mm_: Monthly index that should be deleted after 12 months, indices that don't match the pattern will be deleted after 3 months [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: months unit_count: 3 unit_count_pattern: -([0-9]+)- ------------- * __3_logstash-yyyy.mm.dd_: Daily index that should be deleted after 3 years, indices that don't match the pattern will be ignored [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: years unit_count: -1 unit_count_pattern: ^_([0-9]+)_ ------------- IMPORTANT: Be sure to pay attention to the interaction of this parameter and <>! [[fe_use_age]] == use_age [source,yaml] ------------- - filtertype: count count: 10 use_age: True source: creation_date ------------- This setting allows filtering of indices by their age _after_ other considerations. The default value of this setting is `False` NOTE: Use of this setting requires the additional setting, <>. TIP: There are context-specific examples using `use_age` in the <> and <> documentation. [[fe_value]] == value NOTE: This setting is only used with the <> filtertype and is a required setting. There is a separate <> associated with the <>, and the <>. The value of this setting is used by <> as follows: * `prefix`: Search the first part of an index name for the provided value * `suffix`: Search the last part of an index name for the provided value * `regex`: Provide your own regular expression, and Curator will find the matches. * `timestring`: An strftime string to extrapolate and find indices that match. For example, given a `timestring` of `'%Y.%m.%d'`, matching indices would include `logstash-2016.04.01` and `.marvel-2016.04.01`, but not `myindex-2016-04-01`, as the pattern is different. IMPORTANT: Whatever you provide for `value` is always going to be a part of a + regular expression. The safest practice is to always encapsulate within single quotes. For example: `value: '-suffix'`, or `value: 'prefix-'` There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. TIP: There are context-specific examples using `value` in the <> documentation. [[fe_week_starts_on]] == week_starts_on NOTE: This setting is only used with the <> filtertype + when <> is `relative`. [source,yaml] ------------- - filtertype: period source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday ------------- The value of this setting indicates whether weeks should be measured starting on `sunday` or `monday`. Though Monday is the ISO standard, Sunday is frequently preferred. This setting is only used when <> is set to `weeks`. The default value for this setting is `sunday`. elasticsearch-curator-8.0.21/docs/asciidoc/filters.asciidoc000066400000000000000000000726531477314666200240040ustar00rootroot00000000000000[[filters]] = Filters [partintro] -- Filters are the way to select only the indices (or snapshots) you want. include::inc_filter_chaining.asciidoc[] The index filtertypes are: * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> The snapshot filtertypes are: * <> * <> * <> * <> * <> * <> You can use <> in your configuration files. -- [[filtertype]] == filtertype Each filter is defined first by a `filtertype`. Each filtertype has its own settings, or no settings at all. In a configuration file, filters are defined as follows: [source,yaml] ------------- - filtertype: *first* setting1: ... ... settingN: ... - filtertype: *second* setting1: ... ... settingN: ... - filtertype: *third* ------------- The `-` indicates in the YAML that this is an array element. Each filtertype declaration must be preceded by a `-` for the filters to be read properly. This is how Curator can chain filters together. Anywhere filters can be used, multiple can be chained together in this manner. The index filtertypes are: * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> The snapshot filtertypes are: * <> * <> * <> * <> * <> * <> [[filtertype_age]] == age NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list and match indices based on their age. They will remain in, or be removed from the actionable list based on the value of <>. === Age calculation include::inc_unit_table.asciidoc[] All calculations are in epoch time, which is the number of seconds elapsed since 1 Jan 1970. If no <> is specified in the filter, then the current epoch time-which is always UTC-is used as the basis for comparison. As epoch time is always increasing, lower numbers indicate dates and times in the past. When age is calculated, <> is multiplied by <> to obtain a total number of seconds to use as a differential. For example, if the time at execution were 2017-04-07T15:00:00Z (UTC), then the epoch timestamp would be `1491577200`. If I had an age filter defined like this: [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ------------- The time differential would be `3*24*60*60` seconds, which is `259200` seconds. Subtracting this value from `1491577200` gives us `1491318000`, which is 2017-04-04T15:00:00Z (UTC), exactly 3 days in the past. The `creation_date` of indices or snapshots is compared to this timestamp. If it is `older`, it stays in the actionable list, otherwise it is removed from the actionable list. [IMPORTANT] .`age` filter vs. `period` filter ================================= The time differential means of calculation can lead to frustration. Setting `unit` to `months`, and `unit_count` to `3` will actually calculate the age as `3*30*24*60*60`, which is `7776000` seconds. This may be a big deal. If the date is 2017-01-01T02:30:00Z, or `1483237800` in epoch time. Subtracting `7776000` seconds makes `1475461800`, which is 2016-10-03T02:30:00Z. If you were to try to match monthly indices, `index-2016.12`, `index-2016.11`, `2016.10`, `2016.09`, etc., then both `index-2016.09` _and_ `index-2016.10` will be _older_ than the cutoff date. This may result in unintended behavior. Another way this can cause issues is with weeks. Weekly indices may start on Sunday or Monday. The age filter's calculation doesn't take this into consideration, and merely tests the difference between execution time and the timestamp on the index (from any `source`). Another means of selecting indices and snapshots is the <> filter, which is perhaps a better choice for selecting weeks and months as it compensates for these differences. ================================= include::inc_sources.asciidoc[] === Required settings * <> * <> * <> * <> === Dependent settings * <> (required if `source` is `name`) * <> (required if `source` is `field_stats`) [Indices only] * <> (only used if `source` is `field_stats`) [Indices only] === Optional settings * <> * <> * <> (default is `False`) [[filtertype_alias]] == alias [source,yaml] ------------- - filtertype: alias aliases: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list and match indices based on whether they are associated with the given <>, which can be a single value, or an array. They will remain in, or be removed from the actionable list based on the value of <>. include::inc_filter_by_aliases.asciidoc[] === Required settings * <> === Optional settings * <> [[filtertype_allocated]] == allocated [source,yaml] ------------- - filtertype: allocated key: ... value: ... allocation_type: exclude: True ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list and match indices based on their shard routing allocation settings. They will remain in, or be removed from the actionable list based on the value of <>. By default the indices matched by the `allocated` filter will be excluded since the `exclude` setting defaults to `True`. To include matching indices rather than exclude, set the `exclude` setting to `False`. === Required settings * <> * <> === Optional settings * <> * <> (default is `True`) [[filtertype_closed]] == closed [source,yaml] ------------- - filtertype: closed exclude: True ------------- This <> will iterate over the actionable list and match indices which are closed. They will remain in, or be removed from the actionable list based on the value of <>. === Optional settings * <> (default is `True`) [[filtertype_count]] == count [source,yaml] ------------- - filtertype: count count: 10 ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list of indices _or_ snapshots. They are ordered by age, or by alphabet, so as to guarantee that the correct items will remain in, or be removed from the actionable list based on the values of <>, <>, and <>. === Age-based sorting For use cases where "like" items are being counted, and their name pattern guarantees date sorting is equal to alphabetical sorting, it is unnecessary to set <> to `True`, as item names will be sorted in <> order by default. This means that the item count will start beginning with the _newest_ indices or snapshots, and proceed through to the oldest. Where this is not the case, the <> setting can be used to ensure that index or snapshot ages are properly considered for sorting: [source,yaml] ------------- - filtertype: count count: 10 use_age: True source: creation_date ------------- All of the age-related settings from the <> filter are supported, and the same restrictions apply with regard to filtering indices vs. snapshots. === Pattern-based sorting [source,yaml] ------------- - filtertype: count count: 1 pattern: '^(.*)-\d{6}$' reverse: true ------------- This particular example will match indices following the basic rollover pattern of `indexname-######`, and keep the highest numbered index for each group. For example, given indices `a-000001`, `a-000002`, `a-000003` and `b-000006`, and `b-000007`, the indices will would be matched are `a-000003` and `b-000007`. Indices that do not match the regular expression in `pattern` will be automatically excluded. This is particularly useful with indices created and managed using the {ref}/indices-rollover-index.html[Rollover API], as you can select only the active indices with the above example (<> defaults to `False`). Setting <> to `True` with the above example will _remove_ the active rollover indices, leaving only those which have been rolled-over. While this is perhaps most useful for the aforementioned scenario, it can also be used with age-based indices as well. === Reversing sorting Using the default configuration, <> is `True`. Given These indices: [source,sh] ------------- index1 index2 index3 index4 index5 ------------- And this filter: [source,yaml] ------------- - filtertype: count count: 2 ------------- Indices `index5` and `index4` will be recognized as the `2` _most recent,_ and will be removed from the actionable list, leaving `index1`, `index2`, and `index3` to be acted on by the given <>. Similarly, given these indices: [source,sh] ------------- index-2017.03.01 index-2017.03.02 index-2017.03.03 index-2017.03.04 index-2017.03.05 ------------- And this filter: [source,yaml] ------------- - filtertype: count count: 2 use_age: True source: name timestring: '%Y.%m.%d' ------------- The result will be similar. Indices `index-2017.03.05` and `index-2017.03.04` will be recognized as the `2` _most recent,_ and will be removed from the actionable list, leaving `index-2017.03.01`, `index-2017.03.02`, and `index-2017.03.03` to be acted on by the given <>. In some cases, you may wish to filter for the most recent indices. To accomplish this you set <> to `False`: [source,yaml] ------------- - filtertype: count count: 2 reverse: False ------------- This time indices `index1` and `index2` will be the `2` which will be removed from the actionable list, leaving `index3`, `index4`, and `index5` to be acted on by the given <>. Likewise with the age sorted indices: [source,yaml] ------------- - filtertype: count count: 2 use_age: True source: name timestring: '%Y.%m.%d' reverse: True ------------- Indices `index-2017.03.01` and `index-2017.03.02` will be the `2` which will be removed from the actionable list, leaving `index-2017.03.03`, `index-2017.03.04`, and `index-2017.03.05` to be acted on by the given <>. === Required settings * <> === Optional settings * <> * <> * <> * <> (required if `use_age` is `True`) * <> (required if `source` is `name`) * <> (default is `True`) === Index-only settings * <> (required if `source` is `field_stats`) * <> (only used if `source` is `field_stats`) [[filtertype_empty]] == empty [source,yaml] ------------- - filtertype: empty exclude: False ------------- This <> will iterate over the actionable list and match indices which do not contain any documents. Indices that are closed are automatically removed from consideration. They will remain in, or be removed from the actionable list based on the value of <>. By default the indices matched by the empty filter will be excluded since the exclude setting defaults to True. To include matching indices rather than exclude, set the exclude setting to False. === Optional settings * <> (default is `True`) [[filtertype_forcemerged]] == forcemerged [source,yaml] ------------- - filtertype: forcemerged max_num_segments: 2 exclude: True ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list and match indices which have `max_num_segments` segments per shard, or fewer. They will remain in, or be removed from the actionable list based on the value of <>. === Required settings * <> === Optional settings * <> (default is `True`) [[filtertype_kibana]] == kibana [source,yaml] ------------- - filtertype: kibana exclude: True ------------- This <> will remove any index matching the regular expression `^\.kibana.*$` from the list of indices, if present. This <> will iterate over the actionable list and match indices matching the regular expression `^\.kibana.*$`. They will remain in, or be removed from the actionable list based on the value of <>. === Optional settings * <> (default is `True`) [[filtertype_none]] == none [source,yaml] ------------- - filtertype: none ------------- This <> will not filter anything, returning the full list of indices or snapshots. There are no settings for this <>. [[filtertype_opened]] == opened [source,yaml] ------------- - filtertype: opened exclude: True ------------- This <> will iterate over the actionable list and match indices which are opened. They will remain in, or be removed from the actionable list based on the value of <>. === Optional settings * <> (default is `True`) [[filtertype_pattern]] == pattern [source,yaml] ------------- - filtertype: pattern kind: ... value: ... ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list and match indices matching a given pattern. They will remain in, or be removed from the actionable list based on the value of <>. include::inc_filter_chaining.asciidoc[] include::inc_kinds.asciidoc[] === Required settings * <> * <> === Optional settings * <> (default is `False`) [[filtertype_period]] == period This <> will iterate over the actionable list and match indices or snapshots based on whether they fit within the given time range. They will remain in, or be removed from the actionable list based on the value of <>. [source,yaml] ------------- - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. === Relative Periods A relative period will be reckoned relative to execution time, unless an <> timestamp is provided. Reckoning is truncated to the most recent whole unit, where a <> can be one of `hours`, `days`, `weeks`, `months`, or `years`. For example, if I selected `hours` as my `unit`, and I began execution at 02:35, then the point of reckoning would be 02:00. This is relatively easy with `days`, `months`, and `years`, but slightly more complicated with `weeks`. Some users may wish to reckon weeks by the ISO standard, which starts weeks on Monday. Others may wish to use Sunday as the first day of the week. Both are acceptable options with the `period` filter. The default behavior for `weeks` is to have Sunday be the start of the week. This can be overridden with <> as follows: [source,yaml] ------------- - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: monday ------------- <> and <> are counters of whole <>. A negative number indicates a whole unit in the past, while a positive number indicates a whole unit in the future. A `0` indicates the present unit. With such a timeline mentality, it is relatively easy to create a date range that will meet your needs. If the time of execution time is *2017-04-03T13:45:23.831*, this table will help you figure out what the previous whole unit, current unit, and next whole unit will be, in ISO8601 format. [frame="topbot",options="header"] |====================================================================== |unit |-1 |0 |+1 |hours |2017-04-03T12:00:00|2017-04-03T13:00:00|2017-04-03T14:00:00 |days |2017-04-02T00:00:00|2017-04-03T00:00:00|2017-04-04T00:00:00 |weeks sun |2017-03-26T00:00:00|2017-04-02T00:00:00|2017-04-09T00:00:00 |weeks mon |2017-03-27T00:00:00|2017-04-03T00:00:00|2017-04-10T00:00:00 |months |2017-03-01T00:00:00|2017-04-01T00:00:00|2017-05-01T00:00:00 |years |2016-01-01T00:00:00|2017-01-01T00:00:00|2018-01-01T00:00:00 |====================================================================== Ranges must be from older dates to newer dates, or smaller numbers (including negative numbers) to larger numbers or Curator will return an exception. An example `period` filter demonstrating how to select all daily indices by timestring found in the index name from last month might look like this: [source,yaml] ------------- - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: months ------------- Having `range_from` and `range_to` both be the same value will mean that only that whole unit will be selected, in this case, a month. IMPORTANT: `range_from` and `range_to` are required for the `relative` pattern type. === Absolute Periods [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ------------- In addition to relative periods, you can define absolute periods. These are defined as the period between the <> and the <>. For example, if `date_from` and `date_to` are both `2017.01`, and <> is `months`, all indices with a `name`, `creation_date`, or `stats_result` (depending on the value of <>) within the month of January 2017 will match. The `date_from` is used to establish the beginning of the time period, regardless of whether `date_from_format` is in hours, and the indices you are trying to filter are in weeks or months. The format and date of `date_from` will simply set the beginning of the time period. The `date_to`, `date_to_format`, and `unit` work in conjunction to select the end date. For example, if my `date_to` were `2017.04`, and `date_to_format` is `%Y.%m` to properly parse that date, it would follow that `unit` would be `months`. [IMPORTANT] ===================================== The period filter is smart enough to calculate `months` and `years` properly. **If `unit` is not `months` or `years`,** then your date range will be `unit` seconds more than the beginning of the `date_from` date, minus 1 second, according to this table: include::inc_unit_table.asciidoc[] ===================================== Specific date ranges can be captured with up to whole second resolution: [source,yaml] ------------- - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d.%H' unit: seconds date_from: 2017-01-01T00:00:00 date_from_format: '%Y-%m-%dT%H:%M:%S' date_to: 2017-01-01T12:34:56 date_to_format: '%Y-%m-%dT%H:%M:%S' ------------- This example will capture indices with an hourly timestamp in their name that fit between the `date_from` and `date_to` timestamps. include::inc_strftime_table.asciidoc[] === Required settings * <> * <> === Dependent settings * <> * <> * <> * <> * <> * <> * <> (required if `source` is `name`) * <> (required if `source` is `field_stats`) [Indices only] * <> (only used if `source` is `field_stats`) [Indices only] * <> (optional if `source` is `field_stats`) [Indices only] === Optional settings * <> * <> (default is `False`) * <> [[filtertype_space]] == space This <> will iterate over the actionable list and match indices when their cumulative disk consumption is `greater_than` (default) or `less_than` than <> gigabytes. They are first ordered by age, or by alphabet, so as to guarantee the oldest indices are deleted first. They will remain in, or be removed from the actionable list based on the value of <>. === Deleting Indices By Space [source,yaml] ------------- - filtertype: space disk_space: 100 ------------- This <> is for those who want to retain indices based on disk consumption, rather than by a set number of days. There are some important caveats regarding this choice: * Elasticsearch cannot calculate the size of closed indices. Elasticsearch does not keep tabs on how much disk-space closed indices consume. If you close indices, your space calculations will be inaccurate. * Indices consume resources just by existing. You could run into performance and/or operational snags in Elasticsearch as the count of indices climbs. * You need to manually calculate how much space across all nodes. The total you give will be the sum of all space consumed across all nodes in your cluster. If you use shard allocation to put more shards or indices on a single node, it will not affect the total space reported by the cluster, but you may still run out of space on that node. These are only a few of the caveats. This is still a valid use-case, especially for those running a single-node test box. For use cases where "like" indices are being counted, and their name pattern guarantees date sorting is equal to alphabetical sorting, it is unnecessary to set <> to `True`, as index names will be sorted in <> order by default. For this case, this means that disk space calculations will start beginning with the _newest_ indices, and proceeding through to the oldest. === Age-based sorting [source,yaml] ------------- - filtertype: space disk_space: 100 use_age: True source: creation_date ------------- For use cases where "like" indices are being counted, and their name pattern guarantees date sorting is equal to alphabetical sorting, it is unnecessary to set <> to `True`, as index names will be sorted in <> order by default. For this case, this means that disk space calculations will start beginning with the _newest_ indices, and proceeding through to the oldest. Where this is not the case, the <> setting can be used to ensure that index or snapshot ages are properly considered for sorting: All of the age-related settings from the <> filter are supported. === Reversing sorting IMPORTANT: The <> setting is ignored when <> is `True`. When <> is `True`, sorting is _always_ from newest to oldest, ensuring that old indices are always selected first. Using the default configuration, <> is `True`. Given These indices: [source,sh] ------------- index1 10g index2 10g index3 10g index4 10g index5 10g ------------- And this filter: [source,yaml] ------------- - filtertype: space disk_space: 21 ------------- The indices will be sorted alphabetically and iterated over in the indicated order (the value of <>) and the total disk space compared after adding the size of each successive index. In this example, that means that `index5` will be added first, and the running total of consumed disk space will be `10g`. Since `10g` is less than the indicated threshold of `21`, `index5` will be removed from the actionable list. On the next iteration, the amount of space consumed by `index4` will be added, which brings the running total to `20g`, which is still less than the `21` threshold, so `index4` is also removed from the actionable list. This process changes when the iteration adds the disk space consumed by `index3`. Now the running total crosses the `21` threshold established by <> (the running total is now `30g`). Even though it is only `1g` in excess of the total, `index3` will remain in the actionable list. The threshold is absolute. The remaining indices, `index2` and `index1` will also be in excess of the threshold, so they will also remain in the actionable list. So in this example `index1`, `index2`, and `index3` will be acted on by the <> for this block. If you were to run this with <> set to `DEBUG`, you might see messages like these in the output: [source,sh] ------------- ...Removed from actionable list: index5, summed disk usage is 10GB and disk limit is 21.0GB. ...Removed from actionable list: index4, summed disk usage is 20GB and disk limit is 21.0GB. ...Remains in actionable list: index3, summed disk usage is 30GB and disk limit is 21.0GB. ...Remains in actionable list: index2, summed disk usage is 40GB and disk limit is 21.0GB. ...Remains in actionable list: index1, summed disk usage is 50GB and disk limit is 21.0GB. ------------- In some cases, you may wish to filter in the reverse order. To accomplish this, you set <> to `False`: [source,yaml] ------------- - filtertype: space disk_space: 21 reverse: False ------------- This time indices `index1` and `index2` will be the ones removed from the actionable list, leaving `index3`, `index4`, and `index5` to be acted on by the given <>. If you were to run this with <> set to `DEBUG`, you might see messages like these in the output: [source,sh] ------------- ...Removed from actionable list: index1, summed disk usage is 10GB and disk limit is 21.0GB. ...Removed from actionable list: index2, summed disk usage is 20GB and disk limit is 21.0GB. ...Remains in actionable list: index3, summed disk usage is 30GB and disk limit is 21.0GB. ...Remains in actionable list: index4, summed disk usage is 40GB and disk limit is 21.0GB. ...Remains in actionable list: index5, summed disk usage is 50GB and disk limit is 21.0GB. ------------- === Required settings * <> === Optional settings * <> * <> * <> (required if `use_age` is `True`) * <> (required if `source` is `name`) * <> (default is `greater_than`) * <> (required if `source` is `field_stats`) * <> (only used if `source` is `field_stats`) * <> (default is `False`) [[filtertype_state]] == state [source,yaml] ------------- - filtertype: state state: SUCCESS ------------- NOTE: Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given <>, it may generate an error. This <> will iterate over the actionable list and match snapshots based on the value of <>. They will remain in, or be removed from the actionable list based on the value of <>. === Required settings * <> === Optional settings * <> (default is `False`) elasticsearch-curator-8.0.21/docs/asciidoc/ilm.asciidoc000066400000000000000000000062111477314666200231000ustar00rootroot00000000000000[[ilm]] = Curator and Index Lifecycle Management [partintro] -- Beginning with Elasticsearch version 6.6, Elasticsearch has provided {ref}/index-lifecycle-management.html[Index Lifecycle Management] (or, ILM) to users with at least a Basic license. ILM provides users with many of the most common index management features as a matter of policy, rather than execution time analysis (which is how Curator works). -- [[ilm-actions]] == ILM Actions ILM applies policy actions as indices enter time-oriented phases: * Hot * Warm * Cold * Delete The policy actions include: * {ref}/ilm-set-priority.html[Set Priority] * {ref}/ilm-rollover.html[Rollover] * {ref}/ilm-unfollow.html[Unfollow] * {ref}/ilm-allocate.html[Allocate] * {ref}/ilm-readonly.html[Read-Only] * {ref}/ilm-forcemerge.html[Force Merge] * {ref}/ilm-shrink.html[Shrink] * {ref}/ilm-delete.html[Delete] [[ilm-or-curator]] == ILM or Curator? If ILM provides the functionality to manage your index lifecycle, and you have at least a Basic license, consider using ILM in place of Curator. Many of the Stack components make use of ILM by default. [[ilm-beats]] === Beats NOTE: All Beats share a similar ILM configuration. Filebeats is used as a reference here. Starting with version 7.0, Filebeat uses index lifecycle management by default when it connects to a cluster that supports lifecycle management. Filebeat loads the default policy automatically and applies it to any indices created by Filebeat. You can view and edit the policy in the Index lifecycle policies UI in Kibana. For more information about working with the UI, see {esref}/index-lifecycle-management.html[Index lifecyle policies]. Read more about Filebeat and ILM {fbref}/ilm.html[here]. [[ilm-logstash]] === Logstash NOTE: The Index Lifecycle Management feature requires version 9.3.1 or higher of the `logstash-output-elasticsearch` plugin. Logstash can use [Index Lifecycle Management](docs-content://manage-data/lifecycle/index-lifecycle-management/index-lifecycle.md) to automate the management of indices over time. The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting. By default, this will automatically detect whether the Elasticsearch instance supports ILM, and will use it if it is available. `ilm_enabled` can also be set to `true` or `false` to override the automatic detection, or disable ILM. Read more about [Logstash and ILM](logstash-docs-md://lsr/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ilm). [[ilm-and-curator]] == ILM and Curator! WARNING: Curator will not act on any index associated with an ILM policy without setting `allow_ilm_indices` to `true`. Curator and ILM _can_ coexist. However, to prevent Curator from accidentally interfering, or colliding with ILM policies, any index associated with an ILM policy name is excluded by default. This is true whether you have a Basic license or not, or whether the ILM policy is enabled or not. Curator can be configured to work with ILM-enabled indices by setting the <> option to `true` for any action. Learn more about Index Lifecycle Management {ref}/index-lifecycle-management.html[here]. elasticsearch-curator-8.0.21/docs/asciidoc/inc_datemath.asciidoc000066400000000000000000000020511477314666200247350ustar00rootroot00000000000000This setting may be a valid {ref}/api-conventions.html#api-date-math-index-names[Elasticsearch date math string]. A date math name takes the following form: [source,sh] ------------- ------------- [width="50%", cols="| logstash-2024.03.22 || logstash-2024.03.01 || logstash-2024.03 || logstash-2024.02 | | logstash-2024.03.23 |=== elasticsearch-curator-8.0.21/docs/asciidoc/inc_filepath.asciidoc000066400000000000000000000007031477314666200247440ustar00rootroot00000000000000[TIP] .File paths ===================================================================== File paths can be specified as follows: *For Windows:* [source,sh] ------------- 'C:\path\to\file' ------------- *For Linux, BSD, Mac OS:* [source,sh] ------------- '/path/to/file' ------------- Using single-quotes around your file path is encouraged, especially with Windows file paths. ===================================================================== elasticsearch-curator-8.0.21/docs/asciidoc/inc_filter_by_aliases.asciidoc000066400000000000000000000011041477314666200266240ustar00rootroot00000000000000[IMPORTANT] .Matching Indices and Aliases ============================ https://www.elastic.co/guide/en/elasticsearch/reference/5.5/breaking-changes-5.5.html#breaking_55_rest_changes[Indices must be in all aliases to match]. If a list of <> is provided (instead of only one), indices must appear in _all_ listed <> or a 404 error will result, leading to no indices being matched. In older versions, if the index was associated with even one of the aliases in <>, it would result in a match. ============================ elasticsearch-curator-8.0.21/docs/asciidoc/inc_filter_chaining.asciidoc000066400000000000000000000017141477314666200263000ustar00rootroot00000000000000[NOTE] .Filter chaining ===================================================================== It is important to note that while filters can be chained, each is linked by an implied logical *AND* operation. If you want to match from one of several different patterns, as with a logical *OR* operation, you can do so with the <> filtertype using _regex_ as the <>. This example shows how to select multiple indices based on them beginning with either `alpha-`, `bravo-`, or `charlie-`: [source,yaml] ------------- filters: - filtertype: pattern kind: regex value: '^(alpha-|bravo-|charlie-).*$' ------------- Explaining all of the different ways in which regular expressions can be used is outside the scope of this document, but hopefully this gives you some idea of how a regular expression pattern can be used when a logical *OR* is desired. ===================================================================== elasticsearch-curator-8.0.21/docs/asciidoc/inc_kinds.asciidoc000066400000000000000000000037421477314666200242660ustar00rootroot00000000000000The different <> are described as follows: === prefix To match all indices starting with `logstash-`: [source,yaml] ------------- - filtertype: pattern kind: prefix value: logstash- ------------- To match all indices _except_ those starting with `logstash-`: [source,yaml] ------------- - filtertype: pattern kind: prefix value: logstash- exclude: True ------------- NOTE: Internally, the `prefix` value is used to create a _regex_ pattern: `^{0}.*$`. Any special characters should be escaped with a backslash to match literally. === suffix To match all indices ending with `-prod`: [source,yaml] ------------- - filtertype: pattern kind: suffix value: -prod ------------- To match all indices _except_ those ending with `-prod`: [source,yaml] ------------- - filtertype: pattern kind: suffix value: -prod exclude: True ------------- NOTE: Internally, the `suffix` value is used to create a _regex_ pattern: `^.*{0}$`. Any special characters should be escaped with a backslash to match literally. === timestring IMPORTANT: No age calculation takes place here. It is strictly a pattern match. To match all indices with a Year.month.day pattern, like `index-2017.04.01`: [source,yaml] ------------- - filtertype: pattern kind: timestring value: '%Y.%m.%d' ------------- To match all indices _except_ those with a Year.month.day pattern, like `index-2017.04.01`: [source,yaml] ------------- - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ------------- include::inc_timestring_regex.asciidoc[] === regex This <> allows you to design a regular-expression to match indices or snapshots: To match all indices starting with `a-`, `b-`, or `c-`: [source,yaml] ------------- - filtertype: pattern kind: regex value: '^a-|^b-|^c-' ------------- To match all indices _except_ those starting with `a-`, `b-`, or `c-`: [source,yaml] ------------- - filtertype: pattern kind: regex value: '^a-|^b-|^c-' exclude: True ------------- elasticsearch-curator-8.0.21/docs/asciidoc/inc_sources.asciidoc000066400000000000000000000027461477314666200246440ustar00rootroot00000000000000=== `name`-based ages Using `name` as the `source` tells Curator to look for a <> within the index or snapshot name, and convert that into an epoch timestamp (epoch implies UTC). [source,yaml] ------------- - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 3 ------------- include::inc_timestring_regex.asciidoc[] === `creation_date`-based ages `creation_date` extracts the epoch time of index or snapshot creation. [source,yaml] ------------- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ------------- === `field_stats`-based ages NOTE: `source` can only be `field_stats` when filtering indices. In Curator 5.3 and older, source `field_stats` uses the http://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.html[Field Stats API] to calculate either the `min_value` or the `max_value` of the <> as the <>, and then use that value for age comparisons. In 5.4 and above, even though it is still called `field_stats`, it uses an aggregation to calculate the same values, as the `field_stats` API is no longer used in Elasticsearch 6.x and up. <> must be of type `date` in Elasticsearch. [source,yaml] ------------- - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ------------- elasticsearch-curator-8.0.21/docs/asciidoc/inc_strftime_table.asciidoc000066400000000000000000000007711477314666200261610ustar00rootroot00000000000000The identifiers that Curator currently recognizes include: [width="50%", cols="> are calculated as follows: [width="50%", cols=">, the easiest way to use and upgrade. * <> * <> -- [[pip]] == pip This installation procedure requires a functional Python `pip` executable and requires that the target machine has internet connectivity for downloading Curator and the dependencies from https://pypi.org[The Python Package Index]. --------------------------------- pip install elasticsearch-curator --------------------------------- === Upgrading with pip If you already have Elasticsearch Curator installed, and want to upgrade to the latest version, use the `-U` flag: ------------------------------------ pip install -U elasticsearch-curator ------------------------------------ === Installing a specific version with pip The `-U` flag uninstalls the current version (if any), then installs the latest version, or a specified one. Specify a specific version by adding `==` followed by the version you'd like to install, like this: ------------------------------------------- pip install -U elasticsearch-curator==X.Y.Z ------------------------------------------- For this release, you would type: `pip install -U elasticsearch-curator==`+pass:attributes[{curator_version}]+ === System-wide vs. User-only installation The above commands each imply a system-wide installation. This usually requires super-user access, or the `sudo` command. There is a way to install Curator into a path for just the current user, using the `--user` flag. ---------------------------------------- pip install --user elasticsearch-curator ---------------------------------------- This will result in the `curator` end-point being installed in the current user's home directory, in the `.local` directory, in the `bin` subdirectory. The full path might look something like this: ----------------------------- /home/user/.local/bin/curator ----------------------------- You can make an alias or a symlink to this so you can call it more easily. The `--user` flag can also be used in conjunction with the `-U` flag: ---------------------------------------- pip install -U --user elasticsearch-curator==X.Y.Z ----------------------------------------   [[python-source]] == Installation from source Installing or Curator from source tarball (rather than doing a `git clone`) is also possible. Download and install Curator from tarball: . `wget https://github.com/elastic/curator/archive/v`+pass:attributes[{curator_version}].tar.gz -O elasticsearch-curator.tar.gz+ . `pip install elasticsearch-curator.tar.gz`   [[docker]] == Docker Curator is periodically published to Docker Hub at https://hub.docker.com/repository/docker/untergeek/curator/general[`untergeek/curator`]. Download Curator Docker image: . `docker pull untergeek/curator:`+pass:attributes[{curator_version}]+ elasticsearch-curator-8.0.21/docs/asciidoc/options.asciidoc000066400000000000000000002362731477314666200240270ustar00rootroot00000000000000[[options]] = Options [partintro] -- Options are settings used by <>. * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> * <> You can use <> in your configuration files. -- [[option_allocation_type]] == allocation_type NOTE: This setting is used only when using the <> [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... filters: - filtertype: ... ------------- The value of this setting must be one of `require`, `include`, or `exclude`. Read more about these settings at {ref}/shard-allocation-filtering.html The default value for this setting is `require`. [[option_allow_ilm]] == allow_ilm_indices This option allows Curator to manage ILM-enabled indices. Exercise caution that ILM policies and Curator actions will not interfere with each other. IMPORTANT: Read more about Curator and Index Lifecycle Management <>. [source,yaml] ------------- action: delete_indices description: "Delete the specified indices" options: allow_ilm_indices: true filters: - filtertype: ... ------------- The value of this setting must be either `true` or `false`. The default value for this setting is `false`. [[option_include_hidden]] == include_hidden This option allows Curator to act on indices with the setting `hidden: true`, which is common with data_streams. IMPORTANT: If data_stream backing indices are matched by the `search_pattern` and/or after the filters, any attempt to delete the active backing index will result in an error code. The only way to delete all of a data_stream is via the data_stream API. [source,yaml] ------------- action: delete_indices description: "Delete the specified indices" options: include_hidden: true filters: - filtertype: ... ------------- The value of this setting must be either `true` or `false`. The default value for this setting is `false`. [[option_continue]] == continue_if_exception [IMPORTANT] .Using `ignore_empty_list` rather than `continue_if_exception` ==================================== Curator has two general classifications of exceptions: Empty list exceptions, and everything else. The empty list conditions are `curator.exception.NoIndices` and `curator.exception.NoSnapshots`. The `continue_if_exception` option _only_ catches conditions _other_ than empty list conditions. In most cases, you will want to use `ignore_empty_list` instead of `continue_if_exception`. So why are there two kinds of exceptions? When Curator 4 was released, the ability to continue in the event of any exception was covered by the `continue_if_exception` option. However, an empty list is a _benign_ condition. In fact, it's expected with brand new clusters, or when new index patterns are added. The decision was made to split the exceptions, and have a new option catch the empty lists. See <> for more information. ==================================== NOTE: This setting is available in all actions. [source,yaml] ------------- action: delete_indices description: "Delete selected indices" options: continue_if_exception: False filters: - filtertype: ... ------------- If `continue_if_exception` is set to `True`, Curator will attempt to continue on to the next action, if any, even if an exception is encountered. Curator will log but ignore the exception that was raised. The default value for this setting is `False` [[option_copy_aliases]] == copy_aliases NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Reimplement source index aliases on target index after successful shrink. options: shrink_node: DETERMINISTIC copy_aliases: True filters: - filtertype: ... ------------- The default value of this setting is `False`. If `True`, after an index has been successfully shrunk, any aliases associated with the source index will be copied to the target index. [[option_count]] == count NOTE: This setting is required when using the <>. [source,yaml] ------------- action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... filters: - filtertype: ... ------------- The value for this setting is the number of replicas to assign to matching indices. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[option_delay]] == delay NOTE: This setting is only used by the <>, and is optional. [source,yaml] ------------- action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 delay: 120 filters: - filtertype: ... ------------- The value for this setting is the number of seconds to delay between forceMerging indices, to allow the cluster to quiesce. There is no default value. [[option_delete_after]] == delete_after NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink. options: shrink_node: DETERMINISTIC delete_after: True filters: - filtertype: ... ------------- The default value of this setting is `True`. After an index has been successfully shrunk, the source index will be deleted or preserved based on the value of this setting. [[option_delete_aliases]] == delete_aliases NOTE: This setting is only used by the <>, and is optional. [source,yaml] ------------- action: close description: "Close selected indices" options: delete_aliases: False filters: - filtertype: ... ------------- The value for this setting determines whether aliases will be deleted from indices before closing. The default value is `False`. [[option_skip_flush]] == skip_flush NOTE: This setting is only used by the <>, and is optional. [source,yaml] ------------- action: close description: "Close selected indices" options: skip_flush: False filters: - filtertype: ... ------------- If `skip_flush` is set to `True`, Curator will not flush indices to disk before closing. This may be useful for closing red indices before restoring. The default value is `False`. [[option_disable]] == disable_action NOTE: This setting is available in all actions. [source,yaml] ------------- action: delete_indices description: "Delete selected indices" options: disable_action: False filters: - filtertype: ... ------------- If `disable_action` is set to `True`, Curator will ignore the current action. This may be useful for temporarily disabling actions in a large configuration file. The default value for this setting is `False` [[option_extra_settings]] == extra_settings This setting should be nested YAML. The values beneath `extra_settings` will be used by whichever action uses the option. === <> [source,yaml] ------------- action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name extra_settings: filter: term: user: kimchy add: filters: - filtertype: ... remove: filters: - filtertype: ... ------------- === <> [source,yaml] ------------- action: create_index description: "Create index as named" options: name: myindex # ... extra_settings: settings: number_of_shards: 1 number_of_replicas: 0 mappings: type1: properties: field1: type: string index: not_analyzed ------------- === <> See the {ref}/snapshots-restore-snapshot.html[official Elasticsearch Documentation]. [source,yaml] ------------- actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: extra_settings: index_settings: number_of_replicas: 0 wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- === <> [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'name', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: False ------------- === <> NOTE: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-shrink-index.html#_shrinking_an_index[Only `settings` and `aliases` are acceptable] when used in <>. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink, then reroute the shrunk index with the provided parameters. options: shrink_node: DETERMINISTIC extra_settings: settings: index.codec: best_compression aliases: my_alias: {} filters: - filtertype: ... ------------- There is no default value. [[option_ignore_empty]] == ignore_empty_list This setting must be either `True` or `False`. [source,yaml] ------------- action: delete_indices description: "Delete selected indices" options: ignore_empty_list: True filters: - filtertype: ... ------------- Depending on your indices, and how you've filtered them, an empty list may be presented to the action. This results in an error condition. When the ignore_empty_list option is set to `True`, the action will exit with an INFO level log message indicating such. If ignore_empty_list is set to `False`, an ERROR level message will be logged, and Curator will exit with code 1. The default value of this setting is `False` [[option_ignore]] == ignore_unavailable NOTE: This setting is used by the <>, <>, and <> actions. This setting must be either `True` or `False`. The default value of this setting is `False` === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index ignore_unavailable: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- When the `ignore_unavailable` option is `False` and an index is missing the restore request will fail. === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot ignore_unavailable: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- When the `ignore_unavailable` option is `False` and an index is missing, the snapshot request will fail. This is not frequently a concern in Curator, as it should only ever find indices that exist. === <> [source,yaml] ------------- action: index_settings description: "Change settings for selected indices" options: index_settings: index: refresh_interval: 5s ignore_unavailable: False preserve_existing: False filters: - filtertype: ... ------------- When the `ignore_unavailable` option is `False` and an index is missing, or if the request is to apply a {esref}/index-modules.html#_static_index_settings[static] setting and the index is opened, the index setting request will fail. The `ignore_unavailable` option allows these indices to be skipped, when set to `True`. NOTE: {esref}/index-modules.html#dynamic-index-settings[Dynamic] index settings can be applied to either open or closed indices. [[option_include_aliases]] == include_aliases NOTE: This setting is only used by the <> action. [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index include_aliases: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- This setting must be either `True` or `False`. The value of this setting determines whether Elasticsearch should include index aliases when restoring the snapshot. The default value of this setting is `False` [[option_include_gs]] == include_global_state NOTE: This setting is used by the <> and <> actions. This setting must be either `True` or `False`. The value of this setting determines whether Elasticsearch should include the global cluster state with the snapshot or restore. When performing a <>, the default value of this setting is `True`. When performing a <>, the default value of this setting is `False`. === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index include_global_state: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot include_global_state: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- [[option_indices]] == indices NOTE: This setting is only used by the <> action. === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- This setting must be a list of indices to restore. Any valid YAML format for lists are acceptable here. If `indices` is left empty, or unset, all indices in the snapshot will be restored. The default value of this setting is an empty setting. [[option_key]] == key NOTE: This setting is required when using the <>. [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... filters: - filtertype: ... ------------- The value of this setting should correspond to a node setting on one or more nodes in your cluster. For example, you might have set [source,sh] ----------- node.tag: myvalue ----------- in your `elasticsearch.yml` file for one or more of your nodes. To match allocation in this case, set key to `tag`. These special attributes are also supported: [cols="2*", options="header"] |=== |attribute |description |`_name` |Match nodes by node name |`_host_ip` |Match nodes by host IP address (IP associated with hostname) |`_publish_ip` |Match nodes by publish IP address |`_ip` |Match either `_host_ip` or `_publish_ip` |`_host` |Match nodes by hostname |=== There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[option_max_age]] == max_age [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_age: 1d ------------- NOTE: At least one of <>, <>, <> or any combinations of the three are required as `conditions:` for the <> action. The maximum age that is allowed before triggering a rollover. It _must_ be nested under `conditions:` There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. Ages such as `1d` for one day, or `30s` for 30 seconds can be used. [[option_max_docs]] == max_docs [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_docs: 1000000 ------------- NOTE: At least one of <>, <>, <> or any combinations of the three are required as `conditions:` for the <> action. The maximum number of documents allowed in an index before triggering a rollover. It _must_ be nested under `conditions:` There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. [[option_max_size]] == max_size [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_size: 5gb ------------- NOTE: At least one of <>, <>, <> or any combinations of the three are required as `conditions:` for the <> action. The maximum approximate size an index is allowed to be before triggering a rollover. Sizes must use Elasticsearch approved {ref}/common-options.html[human-readable byte units]. It _must_ be nested under `conditions:` There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. [[option_mns]] == max_num_segments NOTE: This setting is required when using the <>. [source,yaml] ------------- action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 filters: - filtertype: ... ------------- The value for this setting is the cutoff number of segments per shard. Indices which have more than this number of segments per shard will remain in the index list. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[option_max_wait]] == max_wait NOTE: This setting is used by the <>, <>, <>, <>, <>, and <> actions. This setting must be a positive integer, or `-1`. This setting specifies how long in seconds to wait to see if the action has completed before giving up. This option is used in conjunction with <>, which is the number of seconds to wait between checking to see if the given action is complete. The default value for this setting is `-1`, meaning that Curator will wait indefinitely for the action to complete. === <> [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: True max_wait: 300 wait_interval: 10 filters: - filtertype: ... ------------- === <> [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- === <> [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- === <> [source,yaml] ------------- action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ------------- === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index include_global_state: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot include_global_state: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- [[option_migration_prefix]] == migration_prefix NOTE: This setting is used by the <> action. If the destination index is set to `MIGRATION`, Curator will reindex all selected indices one by one until they have all been reindexed. By configuring `migration_prefix`, a value can prepend each of those index names. For example, if I were reindexing `index1`, `index2`, and `index3`, and `migration_prefix` were set to `new-`, then the reindexed indices would be named `new-index1`, `new-index2`, and `new-index3`: [source,yaml] ------------- actions: 1: description: >- Reindex index1, index2, and index3 with a prefix of new-, resulting in indices named new-index1, new-index2, and new-index3 action: reindex options: wait_interval: 9 max_wait: -1 migration_prefix: new- request_body: source: index: ["index1", "index2", "index3"] dest: index: MIGRATION filters: - filtertype: none ------------- `migration_prefix` can be used in conjunction with <> [[option_migration_suffix]] == migration_suffix NOTE: This setting is used by the <> action. If the destination index is set to `MIGRATION`, Curator will reindex all selected indices one by one until they have all been reindexed. By configuring `migration_suffix`, a value can be appended to each of those index names. For example, if I were reindexing `index1`, `index2`, and `index3`, and `migration_suffix` were set to `-new`, then the reindexed indices would be named `index1-new`, `index2-new`, and `index3-new`: [source,yaml] ------------- actions: 1: description: >- Reindex index1, index2, and index3 with a suffix of -new, resulting in indices named index1-new, index2-new, and index3-new action: reindex options: wait_interval: 9 max_wait: -1 migration_suffix: -new request_body: source: index: ["index1", "index2", "index3"] dest: index: MIGRATION filters: - filtertype: none ------------- `migration_suffix` can be used in conjunction with <> [[option_name]] == name NOTE: This setting is used by the <>, <> and <>, actions. The value of this setting is the name of the alias, snapshot, or index, depending on which action makes use of `name`. === date math include::inc_datemath.asciidoc[] === strftime This setting may alternately contain a valid Python strftime string. Curator will extract the strftime identifiers and replace them with the corresponding values. The identifiers that Curator currently recognizes include: [width="50%", cols="> [source,yaml] ------------- action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name add: filters: - filtertype: ... remove: filters: - filtertype: ... ------------- This option is required by the <> action, and has no default setting in that context. === <> For the <> action, there is no default setting, but you can use strftime: [source,yaml] ------------- action: create_index description: "Create index as named" options: name: 'myindex-%Y.%m' # ... ------------- or use Elasticsearch {ref}/api-conventions.html#api-date-math-index-names[date math] [source,yaml] ------------- action: create_index description: "Create index as named" options: name: '' # ... ------------- to name your indices. See more in the <> documenation. === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: include_global_state: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- For the <> action, the default value of this setting is `curator-%Y%m%d%H%M%S` [[option_new_index]] == new_index NOTE: This optional setting is only used by the <> action. [source,yaml] ------------- description: >- Rollover the index associated with alias 'name'. Specify new index name using date math. options: name: aliasname new_index: "" conditions: max_age: 1d wait_for_active_shards: 1 ------------- IMPORTANT: A new index name for rollover should still end with a dash followed by an incrementable number, e.g. `my_new_index-1`, or if using date math, `` or `` === date_math include::inc_datemath.asciidoc[] There is no default value for `new_index`. [[option_node_filters]] == node_filters NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Allow master/data nodes to be potential shrink targets, but exclude 'named_node' from potential selection. options: shrink_node: DETERMINISTIC node_filters: permit_masters: True exclude_nodes: ['named_node'] filters: - filtertype: ... ------------- There is no default value for `node_filters`. The current sub-options are as follows: === permit_masters This option indicates whether the shrink action can select master eligible nodes when using `DETERMINISTIC` as the value for <>. The default value is `False`. Please note that this will exclude the elected master, as well as other master-eligible nodes. [IMPORTANT] ===================================== If you have a small cluster with only master/data nodes, you must set `permit_masters` to `True` in order to select one of those nodes as a potential <>. ===================================== === exclude_nodes This option provides means to exclude nodes from selection when using `DETERMINISTIC` as the value for <>. It should be noted that you _can_ use a named node for <> and then exclude it here, and it will prevent a shrink from occurring. [[option_number_of_replicas]] == number_of_replicas NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Set the number of replicas to 0. options: shrink_node: DETERMINISTIC number_of_replicas: 0 filters: - filtertype: ... ------------- The value of this setting determines the number of replica shards per primary shard in the target index. The default value is `1`. [[option_number_of_shards]] == number_of_shards NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Set the number of shards to 2. options: shrink_node: DETERMINISTIC number_of_shards: 2 filters: - filtertype: ... ------------- The value of this setting determines the number of primary shards in the target index. The default value is `1`. [IMPORTANT] =========================== The value for `number_of_shards` must meet the following criteria: * It must be lower than the number of primary shards in the source index. * It must be a factor of the number of primary shards in the source index. =========================== [[option_partial]] == partial NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: ... partial: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- This setting must be either `True` or `False`. The entire snapshot will fail if one or more indices being added to the snapshot do not have all primary shards available. This behavior can be changed by setting partial to `True`. The default value of this setting is `False` [[option_post_allocation]] == post_allocation NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Apply shard routing allocation to target indices. options: shrink_node: DETERMINISTIC post_allocation: allocation_type: include key: node_tag value: cold filters: - filtertype: ... ------------- The only permitted subkeys for `post_allocation` are the same options used by the <> action: <>, <>, and <>. If present, these values will be use to apply shard routing allocation to the target index after shrinking. There is no default value for `post_allocation`. [[option_preserve_existing]] == preserve_existing [source,yaml] ------------- action: index_settings description: "Change settings for selected indices" options: index_settings: index: refresh_interval: 5s ignore_unavailable: False preserve_existing: False filters: - filtertype: ... ------------- This setting must be either `True` or `False`. If `preserve_existing` is set to `True`, and the configuration attempts to push a setting to an index that already has any value for that setting, the existing setting will be preserved, and remain unchanged. The default value of this setting is `False` [[option_refresh]] == refresh NOTE: This setting is only used by the <> action. [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 refresh: True request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- Setting `refresh` to `True` will cause all re-indexed indexes to be refreshed. This differs from the Index API’s refresh parameter which causes just the _shard_ that received the new data to be refreshed. Read more about this setting at {ref}/docs-reindex.html The default value is `True`. [[option_remote_certificate]] == remote_certificate This should be a file path to a CA certificate, or left empty. [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_certificate: /path/to/my/ca.cert remote_client_cert: /path/to/my/client.cert remote_client_key: /path/to/my/client.key request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ------------- NOTE: This option is only used by the <> when performing a remote reindex operation. This setting allows the use of a specified CA certificate file to validate the SSL certificate used by Elasticsearch. There is no default. [[option_remote_client_cert]] == remote_client_cert NOTE: This option is only used by the <> when performing a remote reindex operation. This should be a file path to a client certificate (public key), or left empty. [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_certificate: /path/to/my/ca.cert remote_client_cert: /path/to/my/client.cert remote_client_key: /path/to/my/client.key request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ------------- Allows the use of a specified SSL client cert file to authenticate to Elasticsearch. The file may contain both an SSL client certificate and an SSL key, in which case <> is not used. If specifying `client_cert`, and the file specified does not also contain the key, use <> to specify the file containing the SSL key. The file must be in PEM format, and the key part, if used, must be an unencrypted key in PEM format as well. [[option_remote_client_key]] == remote_client_key NOTE: This option is only used by the <> when performing a remote reindex operation. This should be a file path to a client key (private key), or left empty. [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_certificate: /path/to/my/ca.cert remote_client_cert: /path/to/my/client.cert remote_client_key: /path/to/my/client.key request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ------------- Allows the use of a specified SSL client key file to authenticate to Elasticsearch. If using <> and the file specified does not also contain the key, use `client_key` to specify the file containing the SSL key. The key file must be an unencrypted key in PEM format. [[option_remote_filters]] == remote_filters NOTE: This option is only used by the <> when performing a remote reindex operation. This is an array of <>, exactly as with regular index selection: [source,yaml] ------------- actions: 1: description: "Reindex matching indices into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: https://otherhost:9200 index: REINDEX_SELECTION dest: index: index2 remote_filters: - filtertype: *first* setting1: ... ... settingN: ... - filtertype: *second* setting1: ... ... settingN: ... - filtertype: *third* filters: - filtertype: none ------------- This feature will only work when the `source` `index` is set to `REINDEX_SELECTION`. It will select _remote_ indices matching the filters provided, and reindex them to the _local_ cluster as the name provided in the `dest` `index`. In this example, that is `index2`. [[option_remote_url_prefix]] == remote_url_prefix NOTE: This option is only used by the <> when performing a remote reindex operation. This should be a single value or left empty. [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_url_prefix: my_prefix request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ------------- In some cases you may be obliged to connect to a remote Elasticsearch cluster through a proxy of some kind. There may be a URL prefix before the API URI items, e.g. http://example.com/elasticsearch/ as opposed to http://localhost:9200. In such a case, set the `remote_url_prefix` to the appropriate value, 'elasticsearch' in this example. The default is an empty string. [[option_rename_pattern]] == rename_pattern NOTE: This setting is only used by the <> action. [TIP] .from the Elasticsearch documentation ====================================== The <> and <> options can be also used to rename indices on restore using regular expression that supports referencing the original text as explained http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here]. ====================================== [source,yaml] ------------- actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: rename_pattern: 'index(.+)' rename_replacement: 'restored_index$1' wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- In this configuration, Elasticsearch will capture whatever appears after `index` and put it after `restored_index`. For example, if I was restoring `index-2017.03.01`, the resulting index would be renamed to `restored_index-2017.03.01`. Read more about this setting at {ref}/snapshots-restore-snapshot.html There is no default value. [[option_rename_replacement]] == rename_replacement NOTE: This setting is only used by the <> action. [TIP] .From the Elasticsearch documentation ====================================== The <> and <> options can be also used to rename indices on restore using regular expression that supports referencing the original text as explained http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here]. ====================================== [source,yaml] ------------- actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: rename_pattern: 'index(.+)' rename_replacement: 'restored_index$1' wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- In this configuration, Elasticsearch will capture whatever appears after `index` and put it after `restored_index`. For example, if I was restoring `index-2017.03.01`, the resulting index would be renamed to `restored_index-2017.03.01`. Read more about this setting at {ref}/snapshots-restore-snapshot.html There is no default value. [[option_repository]] == repository NOTE: This setting is only used by the <>, and <> actions. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- [[option_requests_per_second]] == requests_per_second NOTE: This option is only used by the <> [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 requests_per_second: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- `requests_per_second` can be set to any positive decimal number (1.4, 6, 1000, etc) and throttles the number of requests per second that the reindex issues or it can be set to `-1` to disable throttling. The default value for this is option is `-1`. [[option_request_body]] == request_body NOTE: This setting is only used by the <> action. === Manual index selection The `request_body` option is the heart of the reindex action. In here, using YAML syntax, you recreate the body sent to Elasticsearch as described in {ref}/docs-reindex.html[the official documentation.] You can manually select indices as with this example: [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- You can also select multiple indices to reindex by making a list in acceptable YAML syntax: [source,yaml] ------------- actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ------------- IMPORTANT: You _must_ set at least a <> filter, or the action will fail. Do not worry. If you've manually specified your indices, those are the only ones which will be reindexed. === Filter-Selected Indices Curator allows you to use all indices found by the `filters` section by setting the `source` index to `REINDEX_SELECTION`, like this: [source,yaml] ------------- actions: 1: description: >- Reindex all daily logstash indices from March 2017 into logstash-2017.03 action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: REINDEX_SELECTION dest: index: logstash-2017.03 filters: - filtertype: pattern kind: prefix value: logstash-2017.03. ------------- === Reindex From Remote You can also reindex from remote: [source,yaml] ------------- actions: 1: description: "Reindex remote index1 to local index1" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: index1 dest: index: index1 filters: - filtertype: none ------------- IMPORTANT: You _must_ set at least a <> filter, or the action will fail. Do not worry. Only the indices you specified in `source` will be reindexed. Curator will create a connection to the host specified as the `host` key in the above example. It will determine which port to connect to, and whether to use SSL by parsing the URL entered there. Because this `host` is specifically used by Elasticsearch, and Curator is making a separate connection, it is important to ensure that both Curator _and_ your Elasticsearch cluster have access to the remote host. If you do not whitelist the remote cluster, you will not be able to reindex. This can be done by adding the following line to your `elasticsearch.yml` file: [source,yaml] ------------- reindex.remote.whitelist: remote_host_or_IP1:9200, remote_host_or_IP2:9200 ------------- or by adding this flag to the command-line when starting Elasticsearch: [source,sh] ------------- bin/elasticsearch -Edefault.reindex.remote.whitelist="remote_host_or_IP:9200" ------------- Of course, be sure to substitute the correct host, IP, or port. Other client connection arguments can also be supplied in the form of action options: * <> * <> * <> * <> === Reindex From Remote With Filter-Selected Indices You can even reindex from remote with filter-selected indices on the remote side: [source,yaml] ------------- actions: 1: description: >- Reindex all remote daily logstash indices from March 2017 into local index logstash-2017.03 action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: REINDEX_SELECTION dest: index: logstash-2017.03 remote_filters: - filtertype: pattern kind: prefix value: logstash-2017.03. filters: - filtertype: none ------------- IMPORTANT: Even though you are reindexing from remote, you _must_ set at least a <> filter, or the action will fail. Do not worry. Only the indices specified in `source` will be reindexed. Curator will create a connection to the host specified as the `host` key in the above example. It will determine which port to connect to, and whether to use SSL by parsing the URL entered there. Because this `host` is specifically used by Elasticsearch, and Curator is making a separate connection, it is important to ensure that both Curator _and_ your Elasticsearch cluster have access to the remote host. If you do not whitelist the remote cluster, you will not be able to reindex. This can be done by adding the following line to your `elasticsearch.yml` file: [source,yaml] ------------- reindex.remote.whitelist: remote_host_or_IP1:9200, remote_host_or_IP2:9200 ------------- or by adding this flag to the command-line when starting Elasticsearch: [source,sh] ------------- bin/elasticsearch -Edefault.reindex.remote.whitelist="remote_host_or_IP:9200" ------------- Of course, be sure to substitute the correct host, IP, or port. Other client connection arguments can also be supplied in the form of action options: * <> * <> * <> * <> === Reindex - Migration Curator allows reindexing, particularly from remote, as a migration path. This can be a very useful feature for migrating an older cluster (1.4+) to a new cluster, on different hardware. It can also be a useful tool for serially reindexing indices into newer mappings in an automatable way. Ordinarily, a reindex operation is from either one, or many indices, to a single, named index. Assigning the `dest` `index` to `MIGRATION` tells Curator to treat this reindex differently. [IMPORTANT] ============================= **If it is a _local_ reindex,** you _must_ set either <>, or <>, or both. This prevents collisions and other bad things from happening. By assigning a prefix or a suffix (or both), you can reindex any local indices to new versions of themselves, but named differently. It is true the Reindex API already has this functionality. Curator includes this same functionality for convenience. ============================= This example will reindex all of the remote indices matching `logstash-2017.03.` into the local cluster, but preserve the original index names, rather than merge all of the contents into a single index. Internal to Curator, this results in multiple reindex actions: one per index. All other available options and settings are available. [source,yaml] ------------- actions: 1: description: >- Reindex all remote daily logstash indices from March 2017 into local versions with the same index names. action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: REINDEX_SELECTION dest: index: MIGRATION remote_filters: - filtertype: pattern kind: prefix value: logstash-2017.03. filters: - filtertype: none ------------- IMPORTANT: Even though you are reindexing from remote, you _must_ set at least a <> filter, or the action will fail. Do not worry. Only the indices specified in `source` will be reindexed. Curator will create a connection to the host specified as the `host` key in the above example. It will determine which port to connect to, and whether to use SSL by parsing the URL entered there. Because this `host` is specifically used by Elasticsearch, and Curator is making a separate connection, it is important to ensure that both Curator _and_ your Elasticsearch cluster have access to the remote host. If you do not whitelist the remote cluster, you will not be able to reindex. This can be done by adding the following line to your `elasticsearch.yml` file: [source,yaml] ------------- reindex.remote.whitelist: remote_host_or_IP1:9200, remote_host_or_IP2:9200 ------------- or by adding this flag to the command-line when starting Elasticsearch: [source,sh] ------------- bin/elasticsearch -Edefault.reindex.remote.whitelist="remote_host_or_IP:9200" ------------- Of course, be sure to substitute the correct host, IP, or port. Other client connection arguments can also be supplied in the form of action options: * <> * <> * <> * <> * <> * <> === Other scenarios and options Nearly all scenarios supported by the reindex API are supported in the request_body, including (but not limited to): * Pipelines * Scripting * Queries * Conflict resolution * Limiting by count * Versioning * Reindexing operation type (for example, create-only) Read more about these, and more, at {ref}/docs-reindex.html Notable exceptions include: * You cannot manually specify slices. Instead, use the <> option for automated sliced reindexing. [[option_retry_count]] == retry_count NOTE: This setting is only used by the <>. [source,yaml] ------------- action: delete_snapshots description: "Delete selected snapshots from 'repository'" options: repository: ... retry_interval: 120 retry_count: 3 filters: - filtertype: ... ------------- The value of this setting is the number of times to retry deleting a snapshot. The default for this setting is `3`. [[option_retry_interval]] == retry_interval NOTE: This setting is only used by the <>. [source,yaml] ------------- action: delete_snapshots description: "Delete selected snapshots from 'repository'" options: repository: ... retry_interval: 120 retry_count: 3 filters: - filtertype: ... ------------- The value of this setting is the number of seconds to delay between retries. The default for this setting is `120`. [[option_routing_type]] == routing_type NOTE: This setting is only used by the <>. [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- The value of this setting must be either `allocation` or `rebalance` There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[option_search_pattern]] == search_pattern NOTE: This setting is only used by the <>, <>, <>, <>, <>, <>, <>, <>, <>, and <> actions. [source,yaml] ------------- action: delete_indices description: "Delete selected indices" options: search_pattern: 'index-*,-index-00002,named-1' filters: - filtertype: ... ------------- Supports {ref}/api-conventions.html#api-multi-index[Elasticsearch multi-target syntax]. The value of this option can be a comma-separated list of data streams, indices, and aliases used to limit the request. To target all data streams and indices, omit this parameter or use *. If using wildcards it is highly recommended to encapsulate the entire search pattern in single quotes, e.g. `search_pattern: 'a*,b*,c*'` The default value is `*`. Even though `*` is the default, system indices are filtered by default. This appends the pattern: [source,python] ------------- EXCLUDE_SYSTEM = ( '-.kibana*,-.security*,-.watch*,-.triggered_watch*,' '-.ml*,-.geoip_databases*,-.logstash*,-.tasks*' ) ------------- [[option_setting]] == setting NOTE: This setting is only used by the <>. [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- The value of this must be `enable` at present. It is a placeholder for future expansion. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[option_shrink_node]] == shrink_node NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space, excluding master nodes and the node named 'not_this_node' options: shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] shrink_suffix: '-shrink' filters: - filtertype: ... ------------- This setting is required. There is no default value. The value of this setting must be the valid name of a node in your Elasticsearch cluster, or `DETERMINISTIC`. If the value is `DETERMINISTIC`, Curator will automatically select the data node with the most available free space and make that the target node. Curator will repeat this process for each successive index when the value is `DETERMINISTIC`. If <>, such as `exclude_nodes` are defined, those nodes will not be considered as potential target nodes. [[option_shrink_prefix]] == shrink_prefix NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Prepend target index names with 'foo-' and append a suffix of '-shrink' options: shrink_node: DETERMINISTIC shrink_prefix: 'foo-' shrink_suffix: '-shrink' filters: - filtertype: ... ------------- There is no default value for this setting. The value of this setting will be prepended to target index names. If the source index were `index`, and the `shrink_prefix` were `foo-`, and the `shrink_suffix` were `-shrink`, the resulting target index name would be `foo-index-shrink`. [[option_shrink_suffix]] == shrink_suffix NOTE: This setting is only used by the <> action. [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Prepend target index names with 'foo-' and append a suffix of '-shrink' options: shrink_node: DETERMINISTIC shrink_prefix: 'foo-' shrink_suffix: '-shrink' filters: - filtertype: ... ------------- The default value for this setting is `-shrink`. The value of this setting will be appended to target index names. If the source index were `index`, and the `shrink_prefix` were `foo-`, and the `shrink_suffix` were `-shrink`, the resulting target index name would be `foo-index-shrink`. [[option_slices]] == slices NOTE: This setting is only used by the <> action. This setting can speed up reindexing operations by using {ref}/paginate-search-results.html#slice-scroll[Sliced Scroll] to slice on the \_uid. [source,yaml] ------------- actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 slices: 3 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ------------- === Picking the number of slices Here are a few recommendations around the number of `slices` to use: * Don’t use large numbers. `500` creates fairly massive CPU thrash, so Curator will not allow a number larger than this. * It is more efficient from a query performance standpoint to use some multiple of the number of shards in the source index. * Using exactly as many shards as are in the source index is the most efficient from a query performance standpoint. * Indexing performance should scale linearly across available resources with the number of slices. * Whether indexing or query performance dominates that process depends on lots of factors like the documents being reindexed and the cluster doing the reindexing. [[option_skip_fsck]] == skip_repo_fs_check NOTE: This setting is used by the <> and <> actions. This setting must be either `True` or `False`. The default value of this setting is `False` === <> Each master and data node in the cluster _must_ have write access to the shared filesystem used by the repository for a snapshot to be 100% valid. For the purposes of a <>, this is a lightweight attempt to ensure that all nodes are _still_ actively able to write to the repository, in hopes that snapshots were from all nodes. It is not otherwise necessary for the purposes of a restore. Some filesystems may take longer to respond to a check, which results in a false positive for the filesystem access check. For these cases, it is desirable to bypass this verification step, by setting this to `True.` [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index skip_repo_fs_check: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- === <> Each master and data node in the cluster _must_ have write access to the shared filesystem used by the repository for a snapshot to be 100% valid. Some filesystems may take longer to respond to a check, which results in a false positive for the filesystem access check. For these cases, it is desirable to bypass this verification step, by setting this to `True.` [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot skip_repo_fs_check: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- [[option_timeout]] == timeout NOTE: This setting is only used by the <> action. The `timeout` is the length in seconds each individual bulk request should wait for shards that are unavailable. The default value is `60`, meaning 60 seconds. [source,yaml] ------------- actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 timeout: 90 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ------------- [[option_timeout_override]] == timeout_override NOTE: This setting is available in all actions. [source,yaml] ------------- action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 filters: - filtertype: ... ------------- If `timeout_override` is unset in your configuration, some actions will try to set a sane default value. The following table shows these default values: [cols="m,", options="header"] |=== |Action Name |Default `timeout_override` Value |close |180 |delete_snapshots |300 |forcemerge |21600 |restore |21600 |snapshot |21600 |=== All other actions have no default value for `timeout_override`. This setting must be an integer number of seconds, or an error will result. This setting is particularly useful for the <> action, as all other actions have a new polling behavior when using <> that should reduce or prevent client timeouts. [[option_value]] == value NOTE: This setting is optional when using the <> and required when using the <>. === <> For the <>, the value of this setting should correspond to a node setting on one or more nodes in your cluster For example, you might have set [source,sh] ----------- node.tag: myvalue ----------- in your `elasticsearch.yml` file for one or more of your nodes. To match allocation in this case, set value to `myvalue`. Additonally, if you used one of the special attribute names `_ip`, `_name`, `_id`, or `_host` for <>, value can match the one of the node ip addresses, names, ids, or host names, respectively. NOTE: To remove a routing allocation, the value of this setting should be left empty, or the `value` setting not even included as an option. For example, you might have set [source,sh] ----------- PUT test/_settings { "index.routing.allocation.exclude.size": "small" } ----------- to keep index `test` from allocating shards on nodes that have `node.tag: small`. To remove this shard routing allocation setting, you might use an action file similar to this: [source,yaml] ----------- --- actions: 1: action: allocation description: -> Unset 'index.routing.allocation.exclude.size' for index 'test' by passing an empty value. options: key: size value: ... allocation_type: exclude filters: - filtertype: pattern kind: regex value: '^test$' ----------- === <> For the <>, the acceptable values for this setting depend on the value of <>. [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: ... value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- Acceptable values when <> is either `allocation` or `rebalance` are `all`, `primaries`, and `none` (string, not `NoneType`). If `routing_type` is `allocation`, this can also be `new_primaries`. If `routing_type` is `rebalance`, then the value can also be `replicas`. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. [[option_wait_for_active_shards]] == wait_for_active_shards NOTE: This setting is used by the <>, <>, and <> actions. Each use it similarly. This setting determines the number of shard copies that must be active before the client returns. The default value is 1, which implies only the primary shards. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) Read {ref}/docs-index_.html#index-wait-for-active-shards[the Elasticsearch documentation] for more information. === Reindex [source,yaml] ------------- actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 wait_for_active_shards: 2 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ------------- === Rollover [source,yaml] ------------- action: rollover description: >- Rollover the index associated with alias 'name', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 wait_for_active_shards: 1 extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: False ------------- === Shrink [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Prepend target index names with 'foo-' and append a suffix of '-shrink' options: shrink_node: DETERMINISTIC wait_for_active_shards: all filters: - filtertype: ... ------------- [[option_wfc]] == wait_for_completion NOTE: This setting is used by the <>, <>, <>, <>, <>, and <> actions. This setting must be either `True` or `False`. This setting specifies whether or not the request should return immediately or wait for the operation to complete before returning. === <> [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: False max_wait: 300 wait_interval: 10 filters: - filtertype: ... ------------- The default value for the <> action is `False`. === <> [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- The default value for the <> action is `False`. === <> [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- The default value for the <> action is `False`. === <> [source,yaml] ------------- action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ------------- The default value for the <> action is `False`. === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- The default value for the <> action is `True`. === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- The default value for the <> action is `True`. TIP: During snapshot initialization, information about all previous snapshots is loaded into the memory, which means that in large repositories it may take several seconds (or even minutes) for this command to return even if the `wait_for_completion` setting is set to `False`. [[option_wait_for_rebalance]] == wait_for_rebalance [source,yaml] ------------- action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink, then reroute the shrunk index with the provided parameters. options: ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: ... ------------- NOTE: This setting is used by the <> action. This setting must be `true` or `false`. Setting this to `false` will result in the <> action only checking that the index being shrunk has finished being relocated, and not continue to wait for the cluster to fully rebalance all shards. The default value for this setting is `false`. [[option_wait_interval]] == wait_interval NOTE: This setting is used by the <>, <>, <>, <>, <>, and <> actions. This setting must be a positive integer between 1 and 30. This setting specifies how long to wait between checks to see if the action has completed or not. This number should not be larger than the client <> or the <>. As the default client <> value for is 30, this should be uncommon. The default value for this setting is `9`, meaning 9 seconds between checks. This option is generally used in conjunction with <>, which is the maximum amount of time in seconds to wait for the given action to complete. === <> [source,yaml] ------------- action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: False max_wait: 300 wait_interval: 10 filters: - filtertype: ... ------------- === <> [source,yaml] ------------- action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ------------- === <> [source,yaml] ------------- actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ------------- === <> [source,yaml] ------------- action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ------------- === <> [source,yaml] ------------- actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ------------- === <> [source,yaml] ------------- action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ------------- [[option_warn_if_no_indices]] == warn_if_no_indices NOTE: This setting is only used by the <> action. This setting must be either `True` or `False`. The default value for this setting is `False`. [source,yaml] ------------- action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name warn_if_no_indices: False add: filters: - filtertype: ... remove: filters: - filtertype: ... ------------- This setting specifies whether or not the alias action should continue with a warning or return immediately in the event that the filters in the add or remove section result in an empty index list. [WARNING] .Improper use of this setting can yield undesirable results ===================================================================== *Ideal use case:* For example, you want to add the most recent seven days of time-series indices into a _lastweek_ alias, and remove indices older than seven days from this same alias. If you do not not yet have any indices older than seven days, this will result in an empty index list condition which will prevent the entire alias action from completing successfully. If `warn_if_no_indices` were set to `True`, however, it would avert that potential outcome. *Potentially undesirable outcome:* A _non-beneficial_ case would be where if `warn_if_no_indices` is set to `True`, and a misconfiguration results in indices not being found, and therefore not being disassociated from the alias. As a result, an alias that should only query one week now references multiple weeks of data. If `warn_if_no_indices` were set to `False`, this scenario would have been averted because the empty list condition would have resulted in an error. ===================================================================== elasticsearch-curator-8.0.21/docs/asciidoc/versions.asciidoc000066400000000000000000000015671477314666200242000ustar00rootroot00000000000000[[versions]] = Versions [partintro] -- Elasticsearch Curator has been around for many different versions of Elasticsearch. Earlier releases of Curator supported multiple versions of Elasticsearch, but this is no longer the case. Curator is now major version locked with Elasticsearch, which means that if Curator's major version is {curator_major}, it should support any Elasticsearch {curator_major}.x release. However, in many cases, the API calls that Curator makes do not change between the current and previous major releases, or at least the last few minor releases of the previous major release. This has proven to be the case, and starting with Curator 8.0.18, Curator v8 has been tested to work with Elasticsearch 7.14+. * <> -- [[current_release]] == Current Release The current version of Curator v{curator_major} is {curator_version}. elasticsearch-curator-8.0.21/docs/classdef.rst000066400000000000000000000007151477314666200213620ustar00rootroot00000000000000.. _classdef: Other Class Definitions ####################### .. _wrapper: Wrapper ======= .. autoclass:: curator.classdef.Wrapper :members: :undoc-members: :show-inheritance: .. _actionfile: ActionsFile =========== .. autoclass:: curator.classdef.ActionsFile :members: :undoc-members: :show-inheritance: .. _actiondef: ActionDef ========= .. autoclass:: curator.classdef.ActionDef :members: :undoc-members: :show-inheritance: elasticsearch-curator-8.0.21/docs/conf.py000066400000000000000000000053331477314666200203440ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # pylint: disable=redefined-builtin, invalid-name import sys import os import re from datetime import datetime VERSIONFILE = '../curator/_version.py' COPYRIGHT_YEARS = f'2011-{datetime.now().year}' verstrline = open(VERSIONFILE, "rt", encoding='utf-8').read() VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" mo = re.search(VSRE, verstrline, re.M) if mo: verstr = mo.group(1) else: raise RuntimeError(f"Unable to find version string in {VERSIONFILE}.") # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../')) extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx'] autoclass_content = "both" # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = 'Elasticsearch Curator' copyright = f'{COPYRIGHT_YEARS}, Elasticsearch' release = verstr version = '.'.join(release.split('.')[:2]) exclude_patterns = ['_build'] pygments_style = "sphinx" on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] intersphinx_mapping = { 'python': ('https://docs.python.org/3.12', None), 'es_client': ('https://es-client.readthedocs.io/en/v8.17.5', None), 'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.17.2', None), 'voluptuous': ('http://alecthomas.github.io/voluptuous/docs/_build/html', None), 'click': ('https://click.palletsprojects.com/en/8.1.x', None), } elasticsearch-curator-8.0.21/docs/contributing.rst000066400000000000000000000013541477314666200223050ustar00rootroot00000000000000.. _contributing: Contributing ############ We welcome contributions and bug fixes to Curator's API and CLI. We are grateful for the many `contributors`_ who have helped Curator become what it is today. Please read through our `contribution`_ guide, and the Curator `readme`_ document. .. _contributors: https://github.com/elastic/curator/blob/master/CONTRIBUTORS .. _contribution: https://github.com/elastic/curator/blob/master/CONTRIBUTING.md .. _readme: https://github.com/elastic/curator/blob/master/README.rst A brief overview of the steps * fork the repo * make changes in your fork * add tests to cover your changes (if necessary) * run tests * sign the `CLA`_ * send a pull request! .. _CLA: http://elastic.co/contributor-agreement/elasticsearch-curator-8.0.21/docs/defaults.rst000066400000000000000000000007321477314666200214040ustar00rootroot00000000000000.. _defaults: Defaults ######## .. _defaults_fe: Filter Elements =============== .. automodule:: curator.defaults.filter_elements :members: .. _defaults_ftypes: Filter Types ============ .. automodule:: curator.defaults.filtertypes :members: .. _defaults_options: Option Defaults =============== .. automodule:: curator.defaults.option_defaults :members: .. _defaults_settings: Settings ======== .. automodule:: curator.defaults.settings :members: elasticsearch-curator-8.0.21/docs/docset.yml000066400000000000000000000002751477314666200210510ustar00rootroot00000000000000project: 'Curator index management' cross_links: - beats - docs-content - elasticsearch - logstash-docs-md toc: - toc: reference subs: curator_version: 8.0.18 curator_major: 8elasticsearch-curator-8.0.21/docs/examples.rst000066400000000000000000000046411477314666200214160ustar00rootroot00000000000000.. _examples: Examples ======== Each of these examples presupposes that the requisite modules have been imported and an instance of the Elasticsearch client object has been created: .. code-block:: python import elasticsearch import curator client = elasticsearch.Elasticsearch() Filter indices by prefix ++++++++++++++++++++++++ .. code-block:: python ilo = curator.IndexList(client) ilo.filter_by_regex(kind='prefix', value='logstash-') The contents of ``ilo.indices`` will be only indices prefixed by ``logstash-``. Filter indices by suffix ++++++++++++++++++++++++ .. code-block:: python ilo = curator.IndexList(client) ilo.filter_by_regex(kind='suffix', value='-prod') The contents of ``ilo.indices`` will be only indices suffixed by ``-prod``. Filter indices by age (name) ++++++++++++++++++++++++++++ This example will match indices with the following criteria: * Have a date string of ``%Y.%m.%d`` * Use ``days`` as the unit of time measurement * Filter indices ``older`` than 5 ``days`` .. code-block:: python ilo = curator.IndexList(client) ilo.filter_by_age( source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=5 ) The contents of ``ilo.indices`` would then only be indices matching these criteria. Filter indices by age (creation_date) +++++++++++++++++++++++++++++++++++++ This example will match indices with the following criteria: * Use ``months`` as the unit of time measurement * Filter indices where the index creation date is ``older`` than 2 ``months`` from this moment. .. code-block:: python ilo = curator.IndexList(client) ilo.filter_by_age( source='creation_date', direction='older', unit='months', unit_count=2 ) The contents of ``ilo.indices`` would then only be indices matching these criteria. Filter indices by age (field_stats) +++++++++++++++++++++++++++++++++++ This example will match indices with the following criteria: * Use ``days`` as the unit of time measurement * Filter indices where the ``timestamp`` field's ``min_value`` is a date ``older`` than 3 ``weeks`` from this moment. .. code-block:: python ilo = curator.IndexList(client) ilo.filter_by_age( source='field_stats', direction='older', unit='weeks', unit_count=3, field='timestamp', stats_result='min_value' ) The contents of ``ilo.indices`` would then only be indices matching these criteria. elasticsearch-curator-8.0.21/docs/exceptions.rst000066400000000000000000000001311477314666200217470ustar00rootroot00000000000000.. _exceptions: Exceptions ########## .. automodule:: curator.exceptions :members: elasticsearch-curator-8.0.21/docs/helpers.rst000066400000000000000000000042321477314666200212360ustar00rootroot00000000000000.. _helpers: Helpers ####### .. _helpers_date_ops: Date Ops ======== .. py:module:: curator.helpers.date_ops .. autoclass:: TimestringSearch :members: :undoc-members: :show-inheritance: .. autofunction:: absolute_date_range .. autofunction:: date_range .. autofunction:: datetime_to_epoch .. autofunction:: fix_epoch .. autofunction:: get_date_regex .. autofunction:: get_datemath .. autofunction:: get_datetime .. autofunction:: get_point_of_reference .. autofunction:: get_unit_count_from_name .. autofunction:: handle_iso_week_number .. autofunction:: isdatemath .. autofunction:: parse_date_pattern .. autofunction:: parse_datemath .. _helpers_getters: Getters ======= .. py:module:: curator.helpers.getters .. autofunction:: byte_size .. autofunction:: get_alias_actions .. autofunction:: get_data_tiers .. autofunction:: get_indices .. autofunction:: get_repository .. autofunction:: get_snapshot .. autofunction:: get_snapshot_data .. autofunction:: get_tier_preference .. autofunction:: get_write_index .. autofunction:: index_size .. autofunction:: name_to_node_id .. autofunction:: node_id_to_name .. autofunction:: node_roles .. autofunction:: single_data_path .. _helpers_testers: Testers ======= .. py:module:: curator.helpers.testers .. autofunction:: ilm_policy_check .. autofunction:: repository_exists .. autofunction:: rollable_alias .. autofunction:: snapshot_running .. autofunction:: validate_actions .. autofunction:: validate_filters .. autofunction:: verify_client_object .. autofunction:: verify_repository .. autofunction:: verify_snapshot_list .. _helpers_utils: Utils ===== .. py:module:: curator.helpers.utils .. autofunction:: chunk_index_list .. autofunction:: report_failure .. autofunction:: show_dry_run .. autofunction:: to_csv .. autofunction:: multitarget_fix .. autofunction:: regex_loop .. autofunction:: multitarget_match .. _helpers_waiters: Waiters ======= .. py:module:: curator.helpers.waiters .. autofunction:: health_check .. autofunction:: relocate_check .. autofunction:: restore_check .. autofunction:: snapshot_check .. autofunction:: task_check .. autofunction:: wait_for_it elasticsearch-curator-8.0.21/docs/index.rst000066400000000000000000000024041477314666200207020ustar00rootroot00000000000000Elasticsearch Curator ##################### The Elasticsearch Curator helps you manage your indices and snapshots. .. note:: This documentation is for Elasticsearch Curator. Documentation specifically for use of the command-line interface -- which uses this API and is installed as an ``entry_point`` as part of the package -- is available in the `Elastic guide`_. .. _Elastic guide: http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html Contents ======== .. toctree:: :maxdepth: 2 about usage api testing examples Changelog genindex License ======= Copyright (c) 2011–2025 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Indices and Tables ================== * :ref:`genindex` * :ref:`search` elasticsearch-curator-8.0.21/docs/indexlist.rst000066400000000000000000000003331477314666200215750ustar00rootroot00000000000000.. _indexlist: Index List ########## Index filtering is performed using the ``filter_`` methods of :py:class:`curator.IndexList` .. autoclass:: curator.IndexList :members: :undoc-members: :show-inheritance: elasticsearch-curator-8.0.21/docs/other_modules.rst000066400000000000000000000333621477314666200224530ustar00rootroot00000000000000.. _other_modules: Other Modules ############# ``curator.cli`` =============== .. py:module:: curator.cli .. autofunction:: process_action .. autofunction:: ilm_action_skip .. autofunction:: exception_handler .. autofunction:: run .. py:function:: cli(ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, dry_run, loglevel, logfile, logformat, action_file) This is the :py:class:`click.Command` that initiates everything and connects the command-line to the rest of Curator. :param ctx: The Click Context :param config: Path to configuration file. :param hosts: Elasticsearch URL to connect to :param cloud_id: Shorthand to connect to Elastic Cloud instance :param id: API Key "id" value :param api_key: API Key "api_key" value :param username: Username used to create "basic_auth" tuple :param password: Password used to create "basic_auth" tuple :param bearer_auth: Bearer Auth Token :param opaque_id: Opaque ID string :param request_timeout: Request timeout in seconds :param http_compress: Enable HTTP compression :param verify_certs: Verify SSL/TLS certificate(s) :param ca_certs: Path to CA certificate file or directory :param client_cert: Path to client certificate file :param client_key: Path to client certificate key :param ssl_assert_hostname: Hostname or IP address to verify on the node's certificate. :param ssl_assert_fingerprint: SHA-256 fingerprint of the node's certificate. If this value is given then root-of-trust verification isn't done and only the node's certificate fingerprint is verified. :param ssl_version: Minimum acceptable TLS/SSL version :param master_only: Only run if the single host provided is the elected master :param skip_version_test: Do not check the host version :param dry_run: Do not perform any changes. :param loglevel: Log level :param logfile: Path to log file :param logformat: Log output format :param action_file: Path to action file :type ctx: :py:class:`~.click.Context` :type config: str :type hosts: list :type cloud_id: str :type id: str :type api_key: str :type username: str :type password: str :type bearer_auth: str :type opaque_id: str :type request_timeout: int :type http_compress: bool :type verify_certs: bool :type ca_certs: str :type client_cert: str :type client_key: str :type ssl_assert_hostname: str :type ssl_assert_fingerprint: str :type ssl_version: str :type master_only: bool :type skip_version_test: bool :type dry_run: bool :type loglevel: str :type logfile: str :type logformat: str :type action_file: str ``curator.repomgrcli`` ====================== .. py:module:: curator.repomgrcli .. autofunction:: delete_callback .. autofunction:: show_repos .. autofunction:: get_client .. autofunction:: create_repo .. py:function:: azure(ctx, name, client, container, base_path, chunk_size, compress, max_restore_rate, max_snapshot_rate, readonly, location_mode, verify) :param ctx: The Click Context :param name: The repository name :param client: The named client (Azure) :param container: Container name. You must create the Azure container before creating the repository. :param base_path: Specifies the path within container to repository data. Defaults to empty (root directory). :param chunk_size: Chunk size, e.g. ``1g``, ``10m``, ``5k``. Default is unbounded. :param compress: Enable/Disable metadata compression. :param max_restore_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param max_snapshot_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param readonly: Make repsitory read-only. :param location_mode: Either ``primary_only`` or ``secondary_only``. Note that if you set it to ``secondary_only``, it will force ``readonly`` to ``True``. :param verify: Verify repository after creation. :type ctx: :py:class:`~.click.Context` :type name: str :type client: str :type container: str :type base_path: str :type chunk_size: str :type compress: bool :type max_restore_rate: str :type max_snapshot_rate: str :type readonly: bool :type location_mode: str :type verify: bool .. py:function:: gcs(ctx, name, bucket, client, base_path, chunk_size, compress, max_restore_rate, max_snapshot_rate, readonly, verify) :param ctx: The Click Context :param name: The repository name :param client: The name of the client to use to connect to Google Cloud Storage. :param bucket: The name of the bucket to be used for snapshots. :param base_path: Specifies the path within bucket to repository data. Defaults to the root of the bucket. :param chunk_size: Chunk size, e.g. ``1g``, ``10m``, ``5k``. Default is unbounded. :param compress: Enable/Disable metadata compression. :param max_restore_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param max_snapshot_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param readonly: Make repsitory read-only. :param verify: Verify repository after creation. :type ctx: :py:class:`~.click.Context` :type name: str :type client: str :type bucket: str :type base_path: str :type chunk_size: str :type compress: bool :type max_restore_rate: str :type max_snapshot_rate: str :type readonly: bool :type verify: bool .. py:function:: s3(ctx, name, bucket, client, base_path, chunk_size, compress, max_restore_rate, max_snapshot_rate, readonly, server_side_encryption, buffer_size, canned_acl, storage_class, verify) :param ctx: The Click Context :param name: The repository name :param bucket: The bucket name must adhere to Amazon's S3 bucket naming rules. :param client: The name of the S3 client to use to connect to S3. :param base_path: Specifies the path to the repository data within its bucket. Defaults to an empty string, meaning that the repository is at the root of the bucket. The value of this setting should not start or end with a /. :param chunk_size: Chunk size, e.g. ``1g``, ``10m``, ``5k``. Default is unbounded. :param compress: Enable/Disable metadata compression. :param max_restore_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param max_snapshot_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param readonly: Make repsitory read-only. :param server_side_encryption: If set, files are encrypted on server side using AES256 algorithm. :param buffer_size: Minimum threshold below which the chunk is uploaded using a single request. Must be between 5mb and 5gb. :param canned_acl: When the S3 repository creates buckets and objects, it adds the canned ACL into the buckets and objects. :param storage_class: Sets the S3 storage class for objects stored in the snapshot repository. :param verify: Verify repository after creation. :type ctx: :py:class:`~.click.Context` :type name: str :type bucket: str :type client: str :type base_path: str :type chunk_size: str :type compress: bool :type max_restore_rate: str :type max_snapshot_rate: str :type readonly: bool :type server_side_encryption: bool :type buffer_size: str :type canned_acl: str :type storage_class: str :type verify: bool .. py:function:: fs(ctx, name, location, compress, chunk_size, max_snapshots, max_restore_rate, max_snapshot_rate, readonly, verify) :param ctx: The Click Context :param name: The repository name :param location: Shared file-system location. Must match remote path, & be accessible to all master & data nodes :param compress: Enable/Disable metadata compression. :param chunk_size: Chunk size, e.g. ``1g``, ``10m``, ``5k``. Default is unbounded. :param max_snapshots: Maximum number of snapshots the repository can contain. Defaults to ``Integer.MAX_VALUE``, which is 2147483647. :param max_restore_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param max_snapshot_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param readonly: Make repsitory read-only. :param verify: Verify repository after creation. :type ctx: :py:class:`~.click.Context` :type name: str :type location: str :type compress: bool :type chunk_size: str :type max_snapshots: int :type max_restore_rate: str :type max_snapshot_rate: str :type readonly: bool :type verify: bool .. py:function:: url(ctx, name, chunk_size, http_max_retries, http_socket_timeout, compress, max_snapshots, max_restore_rate, shared_filesystem_url, verify) :param ctx: The Click Context :param name: The repository name :param chunk_size: Chunk size, e.g. ``1g``, ``10m``, ``5k``. Default is unbounded. :param http_max_retries: Maximum number of retries for http and https :param http_socket_timeout: Maximum wait time for data transfers over a connection. :param compress: Enable/Disable metadata compression. :param max_snapshots: Maximum number of snapshots the repository can contain. Defaults to ``Integer.MAX_VALUE``, which is 2147483647. :param max_restore_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param shared_filesystem_url: URL location of the root of the shared filesystem repository. :param verify: Verify repository after creation. :type ctx: :py:class:`~.click.Context` :type name: str :type chunk_size: str :type http_max_retries: int :type http_socket_timeout: int :type compress: bool :type max_snapshots: int :type max_restore_rate: str :type shared_filesystem_url: str :type verify: bool .. py:function:: source(ctx, name, delegate_type, location, compress, chunk_size, max_snapshots, max_restore_rate, max_snapshot_rate, readonly, verify) :param ctx: The Click Context :param name: The repository name :param delegate_type: Delegated repository type. :param location: Shared file-system location. Must match remote path, & be accessible to all master & data nodes :param compress: Enable/Disable metadata compression. :param chunk_size: Chunk size, e.g. ``1g``, ``10m``, ``5k``. Default is unbounded. :param max_snapshots: Maximum number of snapshots the repository can contain. Defaults to ``Integer.MAX_VALUE``, which is 2147483647. :param max_restore_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param max_snapshot_rate: Throttles per node restore rate (per second). Default is ``20mb``. :param readonly: Make repsitory read-only. :param verify: Verify repository after creation. :type ctx: :py:class:`~.click.Context` :type name: str :type delegate_type: str :type location: str :type compress: bool :type chunk_size: str :type max_snapshots: int :type max_restore_rate: str :type max_snapshot_rate: str :type readonly: bool :type verify: bool .. py:function:: repo_mgr_cli(ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, dry_run, loglevel, logfile, logformat) :param ctx: The Click Context :param config: Path to configuration file. :param hosts: Elasticsearch URL to connect to :param cloud_id: Shorthand to connect to Elastic Cloud instance :param id: API Key "id" value :param api_key: API Key "api_key" value :param username: Username used to create "basic_auth" tuple :param password: Password used to create "basic_auth" tuple :param bearer_auth: Bearer Auth Token :param opaque_id: Opaque ID string :param request_timeout: Request timeout in seconds :param http_compress: Enable HTTP compression :param verify_certs: Verify SSL/TLS certificate(s) :param ca_certs: Path to CA certificate file or directory :param client_cert: Path to client certificate file :param client_key: Path to client certificate key :param ssl_assert_hostname: Hostname or IP address to verify on the node's certificate. :param ssl_assert_fingerprint: SHA-256 fingerprint of the node's certificate. If this value is given then root-of-trust verification isn't done and only the node's certificate fingerprint is verified. :param ssl_version: Minimum acceptable TLS/SSL version :param master_only: Only run if the single host provided is the elected master :param skip_version_test: Do not check the host version :param dry_run: Do not perform any changes. NON-FUNCTIONAL PLACEHOLDER! DO NOT USE! :param loglevel: Log level :param logfile: Path to log file :param logformat: Log output format :type ctx: :py:class:`~.click.Context` :type config: str :type hosts: list :type cloud_id: str :type id: str :type api_key: str :type username: str :type password: str :type bearer_auth: str :type opaque_id: str :type request_timeout: int :type http_compress: bool :type verify_certs: bool :type ca_certs: str :type client_cert: str :type client_key: str :type ssl_assert_hostname: str :type ssl_assert_fingerprint: str :type ssl_version: str :type master_only: bool :type skip_version_test: bool :type dry_run: bool :type loglevel: str :type logfile: str :type logformat: str elasticsearch-curator-8.0.21/docs/reference/000077500000000000000000000000001477314666200207775ustar00rootroot00000000000000elasticsearch-curator-8.0.21/docs/reference/about-api.md000066400000000000000000000012411477314666200232000ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about-api.html --- # Application Program Interface (API) [about-api] Curator ships with both an API and CLI tool. The API, or Application Program Interface, allows you to write your own scripts to accomplish similar goals—​or even new and different things—​with the same code that Curator uses. The API documentation is not on this site, but is available at [http://curator.readthedocs.io/](http://curator.readthedocs.io/). The Curator API is built using the [Elasticsearch Python API](http://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.md). elasticsearch-curator-8.0.21/docs/reference/about-cli.md000066400000000000000000000005661477314666200232070ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about-cli.html --- # Command-Line Interface (CLI) [about-cli] Curator has always been a command-line tool. This site provides the documentation for how to use Curator on the command-line. ::::{tip} Learn more about [the command-line interface](/reference/command-line.md). :::: elasticsearch-curator-8.0.21/docs/reference/about-contributing.md000066400000000000000000000016311477314666200251410ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about-contributing.html --- # Contributing [about-contributing] We welcome contributions and bug fixes to Curator’s API and CLI. We are grateful for the many [contributors](https://github.com/elastic/curator/blob/master/CONTRIBUTORS) who have helped Curator become what it is today. Please read through our [contribution](https://github.com/elastic/curator/blob/master/CONTRIBUTING.md) guide, and the Curator [readme](https://github.com/elastic/curator/blob/master/README.rst) document. A brief overview of the steps * fork the repo * make changes in your fork * add tests to cover your changes (if necessary) * run tests * sign the [CLA](http://elastic.co/contributor-agreement/) * send a pull request! ::::{tip} To submit documentation fixes for this site, see [Site Corrections](/reference/site-corrections.md) :::: elasticsearch-curator-8.0.21/docs/reference/about-features.md000066400000000000000000000017651477314666200242600ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about-features.html --- # Features [about-features] Curator allows for many different operations to be performed to both indices and snapshots, including: * Add or remove indices (or both!) from an [alias](/reference/alias.md) * Change shard routing [allocation](/reference/allocation.md) * [Close](/reference/close.md) indices * [Create index](/reference/create_index.md) * [Delete indices](/reference/delete_indices.md) * [Delete snapshots](/reference/delete_snapshots.md) * [Open](/reference/open.md) closed indices * [forceMerge](/reference/forcemerge.md) indices * [reindex](/reference/reindex.md) indices, including from remote clusters * Change the number of [replicas](/reference/replicas.md) per shard for indices * [rollover](/reference/rollover.md) indices * Take a [snapshot](/reference/snapshot.md) (backup) of indices * [Restore](/reference/restore.md) snapshots * [Shrink](/reference/shrink.md) indices elasticsearch-curator-8.0.21/docs/reference/about-origin.md000066400000000000000000000016751477314666200237310ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about-origin.html --- # Origin [about-origin] Curator was first called [`clearESindices.py`](https://logstash.jira.com/browse/LOGSTASH-211). Its sole function was to delete indices. It was almost immediately renamed to [`logstash_index_cleaner.py`](https://logstash.jira.com/browse/LOGSTASH-211). After a time it was briefly relocated under the [logstash](https://github.com/elastic/logstash) repository as `expire_logs`, at which point it began to gain new functionality. Soon thereafter, Jordan Sissel was hired by Elastic (then still Elasticsearch), as was the original author of Curator. Not long after that it became Elasticsearch Curator and is now hosted at [https://github.com/elastic/curator](https://github.com/elastic/curator) Curator now performs many operations on your Elasticsearch indices, from delete to snapshot to shard allocation routing. elasticsearch-curator-8.0.21/docs/reference/about.md000066400000000000000000000016551477314666200224420ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html --- # About [about] Elasticsearch Curator helps you curate, or manage, your Elasticsearch indices and snapshots by: 1. Obtaining the full list of indices (or snapshots) from the cluster, as the *actionable list* 2. Iterate through a list of user-defined [filters](/reference/filters.md) to progressively remove indices (or snapshots) from this *actionable list* as needed. 3. Perform various [actions](/reference/actions.md) on the items which remain in the *actionable list.* Learn More: * [Origin](/reference/about-origin.md) * [Features](/reference/about-features.md) * [Command-Line Interface (CLI)](/reference/about-cli.md) * [Application Program Interface (API)](/reference/about-api.md) * [License](/reference/license.md) * [Site Corrections](/reference/site-corrections.md) * [Contributing](/reference/about-contributing.md) elasticsearch-curator-8.0.21/docs/reference/actionfile.md000066400000000000000000000045221477314666200234410ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/actionfile.html --- # Action File [actionfile] ::::{note} You can use [environment variables](/reference/envvars.md) in your configuration files. :::: An action file has the following structure: ```sh --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: ACTION1 description: OPTIONAL DESCRIPTION options: option1: value1 ... optionN: valueN continue_if_exception: False disable_action: True filters: - filtertype: *first* filter_element1: value1 ... filter_elementN: valueN - filtertype: *second* filter_element1: value1 ... filter_elementN: valueN 2: action: ACTION2 description: OPTIONAL DESCRIPTION options: option1: value1 ... optionN: valueN continue_if_exception: False disable_action: True filters: - filtertype: *first* filter_element1: value1 ... filter_elementN: valueN - filtertype: *second* filter_element1: value1 ... filter_elementN: valueN 3: action: ACTION3 ... 4: action: ACTION4 ... ``` It is a YAML configuration file. The root key must be `actions`, after which there can be any number of actions, nested underneath numbers. Actions will be taken in the order they are completed. The high-level elements of each numbered action are: * [action](/reference/actions.md) * [description](#description) * [options](/reference/options.md) * [filters](/reference/filters.md) In the case of the [alias action](/reference/alias.md), there are two additional high-level elements: `add` and `remove`, which are described in the [alias action](/reference/alias.md) documentation. ## description [description] This is an optional description which can help describe what the action and its filters are supposed to do. ```yaml description: >- I can make the description span multiple lines by putting ">-" at the beginning of the line, as seen above. Subsequent lines must also be indented. options: option1: ... ``` elasticsearch-curator-8.0.21/docs/reference/actions.md000066400000000000000000000016311477314666200227620ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/actions.html --- # Actions [actions] Actions are the tasks which Curator can perform on your indices. Snapshots, once created, can only be deleted. * [Alias](/reference/alias.md) * [Allocation](/reference/allocation.md) * [Close](/reference/close.md) * [Cluster Routing](/reference/cluster_routing.md) * [Cold2Frozen](/reference/cold2frozen.md) * [Create Index](/reference/create_index.md) * [Delete Indices](/reference/delete_indices.md) * [Delete Snapshots](/reference/delete_snapshots.md) * [forceMerge](/reference/forcemerge.md) * [Index Settings](/reference/index_settings.md) * [Open](/reference/open.md) * [Reindex](/reference/reindex.md) * [Replicas](/reference/replicas.md) * [Restore](/reference/restore.md) * [Rollover](/reference/rollover.md) * [Shrink](/reference/shrink.md) * [Snapshot](/reference/snapshot.md) elasticsearch-curator-8.0.21/docs/reference/alias.md000066400000000000000000000042661477314666200224220ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/alias.html --- # Alias [alias] ```yaml action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name add: filters: - filtertype: ... remove: filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action adds and/or removes indices from the alias identified by [name](/reference/option_name.md) The [filters](/reference/filters.md) under the `add` and `remove` directives define which indices will be added and/or removed. This is an atomic action, so adds and removes happen instantaneously. The [extra_settings](/reference/option_extra_settings.md) option allows the addition of extra settings with the `add` directive. These settings are ignored for `remove`. An example of how these settings can be used to create a filtered alias might be: ```yaml action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name extra_settings: filter: term: user: kimchy add: filters: - filtertype: ... remove: filters: - filtertype: ... ``` ::::{warning} Before creating a filtered alias, first ensure that the fields already exist in the mapping. :::: Learn more about adding filtering and routing to aliases in the [Elasticsearch Alias API documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-aliases.md). ## Required settings [_required_settings] * [name](/reference/option_name.md) ## Optional settings [_optional_settings] * [warn_if_no_indices](/reference/option_warn_if_no_indices.md) * [extra_settings](/reference/option_extra_settings.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_alias.md). :::: elasticsearch-curator-8.0.21/docs/reference/allocation.md000066400000000000000000000043011477314666200234440ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/allocation.html --- # Allocation [allocation] ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action changes the shard routing allocation for the selected indices. See [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shard-allocation-filtering.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shard-allocation-filtering.html) for more information. You can optionally set `wait_for_completion` to `True` to have Curator wait for the shard routing to complete before continuing: ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: True max_wait: 300 wait_interval: 10 filters: - filtertype: ... ``` This configuration will wait for a maximum of 300 seconds for shard routing and reallocation to complete before giving up. A `max_wait` value of `-1` will wait indefinitely. Curator will poll for completion at `10` second intervals, as defined by `wait_interval`. ## Required settings [_required_settings_2] * [key](/reference/option_key.md) ## Optional settings [_optional_settings_2] * [search_pattern](/reference/option_search_pattern.md) * [allocation_type](/reference/option_allocation_type.md) * [value](/reference/option_value.md) * [wait_for_completion](/reference/option_wfc.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_allocation.md). :::: elasticsearch-curator-8.0.21/docs/reference/cli.md000066400000000000000000000004451477314666200220730ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/cli.html --- # Running Curator [cli] * [Command Line Interface](/reference/command-line.md) * [Singleton Command Line Interface](/reference/singleton-cli.md) * [Exit Codes](/reference/exit-codes.md) elasticsearch-curator-8.0.21/docs/reference/close.md000066400000000000000000000021161477314666200224260ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/close.html --- # Close [close] ```yaml action: close description: "Close selected indices" options: delete_aliases: false skip_flush: false filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action closes the selected indices, and optionally deletes associated aliases beforehand. ## Optional settings [_optional_settings_3] * [search_pattern](/reference/option_search_pattern.md) * [delete_aliases](/reference/option_delete_aliases.md) * [skip_flush](/reference/option_skip_flush.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_close.md). :::: elasticsearch-curator-8.0.21/docs/reference/cluster_routing.md000066400000000000000000000041341477314666200245530ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/cluster_routing.html --- # Cluster Routing [cluster_routing] ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action changes the shard routing allocation for the selected indices. See [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shards-allocation.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shards-allocation.md) for more information. You can optionally set `wait_for_completion` to `True` to have Curator wait for the shard routing to complete before continuing: ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` This configuration will wait for a maximum of 300 seconds for shard routing and reallocation to complete before giving up. A `max_wait` value of `-1` will wait indefinitely. Curator will poll for completion at `10` second intervals, as defined by `wait_interval`. ## Required settings [_required_settings_3] * [routing_type](/reference/option_routing_type.md) * [value](/reference/option_value.md) * [setting](/reference/option_setting.md) Currently must be set to `enable`. This setting is a placeholder for potential future expansion. ## Optional settings [_optional_settings_4] * [wait_for_completion](/reference/option_wfc.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_cluster_routing.md). :::: elasticsearch-curator-8.0.21/docs/reference/cold2frozen.md000066400000000000000000000071311477314666200235520ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/cold2frozen.html --- # Cold2Frozen [cold2frozen] ::::{important} This action is for an unusual case where an index is a mounted, searchable snapshot in the cold tier and is not associated with an ILM policy. This action will not work with an index associated with an ILM policy regardless of the value of `allow_ilm_indices`. :::: ```yaml action: cold2frozen description: "Migrate non-ILM indices from the cold tier to the frozen tier" options: index_settings: {} ignore_index_settings: [] wait_for_completion: True filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action migrates the selected non-ILM indices from the cold tier to the frozen tier. You may well ask why this action is here and why it is limited to non-ILM indices. The answer is "redacted data." If an index must be restored from the cold tier to be live so that sensitive data can be redacted, at present, it must be disassociated from an ILM policy to accomplish this. If you forcemerge and re-snapshot the redacted index, you can still put it in the cold or frozen tier, but it will not be associated with an ILM policy any more. This custom action is for moving that manually re-mounted cold tier index to the frozen tier, preserving the aliases it currently has. ## index_settings [_index_settings] Settings that should be added to the index when it is mounted. This should be a YAML dictionary containing anything under what would normally appear in `settings`. See [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/searchable-snapshots-api-mount-snapshot.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/searchable-snapshots-api-mount-snapshot.md) ```yaml action: cold2frozen description: "Migrate non-ILM indices from the cold tier to the frozen tier" options: index_settings: routing: allocation: include: _tier_preference: data_frozen ignore_index_settings: [] wait_for_completion: True filters: - filtertype: ... ``` ::::{note} If unset, the default behavior is to ensure that the `_tier_preference` is `data_frozen`, if available. If it is not, Curator will assess which data tiers are available in your cluster and use those from coldest to warmest, e.g. `data_cold,data_warm,data_hot`. If none of these are available, it will default to `data_content`. :::: ## ignore_index_settings [_ignore_index_settings] This should be a YAML list of index settings the migrated index should ignore after mount. See [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/searchable-snapshots-api-mount-snapshot.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/searchable-snapshots-api-mount-snapshot.md) ```yaml action: cold2frozen description: "Migrate non-ILM indices from the cold tier to the frozen tier" options: index_settings: ignore_index_settings: - 'index.refresh_interval' wait_for_completion: True filters: - filtertype: ... ``` ::::{note} If unset, the default behavior is to ensure that the `index.refresh_interval` is ignored. :::: ## Optional settings [_optional_settings_5] * [search_pattern](/reference/option_search_pattern.md) * [wait_for_completion](/reference/option_wfc.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) elasticsearch-curator-8.0.21/docs/reference/command-line.md000066400000000000000000000104641477314666200236710ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/command-line.html --- # Command Line Interface [command-line] Most common client configuration settings are now available at the command-line. ::::{important} While both the configuration file and the command-line arguments can be used together, it is important to note that command-line options will override file-based configuration of the same setting. :::: The most basic command-line arguments are as follows: ```sh curator [--config CONFIG.YML] [--dry-run] ACTION_FILE.YML ``` The square braces indicate optional elements. If `--config` and `CONFIG.YML` are not provided, Curator will look in `~/.curator/curator.yml` for the configuration file. `~` is the home directory of the user executing Curator. In a Unix system, this might be `/home/username/.curator/curator.yml`, while on a Windows system, it might be `C:\Users\username\.curator\curator.yml` If `--dry-run` is included, Curator will simulate the action(s) in ACTION_FILE.YML as closely as possible without actually making any changes. The results will be in the logfile, or STDOUT/command-line if no logfile is specified. `ACTION_FILE.YML` is a YAML [actionfile](/reference/actionfile.md). For other client configuration options, command-line help is never far away: ```sh curator --help ``` The help output looks like this: ```sh $ curator --help Usage: curator [OPTIONS] ACTION_FILE Curator for Elasticsearch indices The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. Some less-frequently used client configuration options are now hidden. To see the full list, run: curator_cli -h Options: --config PATH Path to configuration file. --hosts TEXT Elasticsearch URL to connect to. --cloud_id TEXT Elastic Cloud instance id --api_token TEXT The base64 encoded API Key token --id TEXT API Key "id" value --api_key TEXT API Key "api_key" value --username TEXT Elasticsearch username --password TEXT Elasticsearch password --request_timeout FLOAT Request timeout in seconds --verify_certs / --no-verify_certs Verify SSL/TLS certificate(s) [default: verify_certs] --ca_certs TEXT Path to CA certificate file or directory --client_cert TEXT Path to client certificate file --client_key TEXT Path to client key file --dry-run Do not perform any changes. --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] Log level --logfile TEXT Log file --logformat [default|ecs] Log output format -v, --version Show the version and exit. -h, --help Show this message and exit. Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/command-line.html ``` You can use [environment variables](/reference/envvars.md) in your configuration files. ## Running Curator from Docker [_running_curator_from_docker] Running Curator from the command-line using Docker requires only a few additional steps. Should you desire to use them, Docker-based Curator requires you to map a volume for your configuration and/or log files. Attempting to read a YAML configuration file if you have neglected to volume map your configuration directory to `/.curator` will not work. It looks like this: ```sh docker run [-t] --rm --name myimagename \ -v /PATH/TO/MY/CONFIGS:/.curator \ untergeek/curator:mytag \ --config /.curator/config.yml /.curator/actionfile.yml ``` ::::{note} While testing, adding the `-t` flag will allocate a pseudo-tty, allowing you to see terminal output that would otherwise be hidden. :::: Both of the files `config.yml` and `actionfile.yml` should already exist in the path `/PATH/TO/MY/CONFIGS` before run time. The `--rm` in the command means that the container (not the image) will be deleted after completing execution. You definitely want this as there is no reason to keep creating containers for each run. The eventual cleanup from this would be unpleasant.   elasticsearch-curator-8.0.21/docs/reference/configfile.md000066400000000000000000000240061477314666200234300ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/configfile.html --- # Configuration File [configfile] ::::{note} The default location of the configuration file is `~/.curator/curator.yml`, but another location can be specified using the `--config` flag on the [command-line](/reference/command-line.md). :::: ::::{note} You can use [environment variables](/reference/envvars.md) in your configuration files. :::: The configuration file contains client connection and settings for logging. It looks like this: ```sh --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" elasticsearch: client: hosts: - http://127.0.0.1:9200 cloud_id: ca_certs: client_cert: client_key: verify_certs: request_timeout: 30 other_settings: master_only: False username: password: api_key: id: api_key: token: logging: loglevel: INFO logfile: logformat: default blacklist: ['elastic_transport', 'urllib3'] ``` It is a YAML configuration file. The two root keys must be `elasticsearch` and `logging`. The subkeys of each of these will be described here. There are other keys available for the `client` subkey of the `elasticsearch` root key, many of which are listed [here](https://es-client.readthedocs.io/en/latest/defaults.html). The most commonly used ones (listed above) are described as follows: ## hosts [hosts] ::::{important} All hosts must be in `HTTP[S]://FQDN.DOMAIN.TLD:PORT` form or they will be rejected and Curator will exit with an error. The only exception to this is `HTTPS://FQDN.DOMAIN.TLD` (without port), in which case `:443` is implicit, and is, in fact, appended automatically. :::: ::::{warning} If both `cloud_id` and `hosts` keys are populated an exception will be thrown and Curator will exit. :::: A `hosts` definition can be a single value: ```sh hosts: http://127.0.0.1:9200 ``` Or multiple values in the 3 acceptable YAML ways to render sequences, or arrays: ::::{warning} Curator can only work with one cluster at a time. Including clients from multiple clusters in the `hosts` setting will result in errors. :::: Flow: ```sh hosts: [ "http://10.0.0.1:9200", "http://10.0.0.2:9200" ] ``` Spanning: ```sh hosts: [ "http://10.0.0.1:9200", "http://10.0.0.2:9200" ] ``` Block: ```sh hosts: - http://10.0.0.1:9200 - http://10.0.0.2:9200 ``` ## cloud_id [cloud_id] The value should encapsulated in quotes because of the included colon: ```sh cloud_id: 'deployment_name:BIG_HASH_VALUE' ``` ::::{warning} If both `cloud_id` and `hosts` keys are populated an exception will be thrown and Curator will exit. :::: ## ca_certs [ca_certs] This should be a file path to your CA certificate, or left empty. ```sh ca_certs: ``` This setting allows the use of a specified CA certificate file to validate the SSL certificate used by Elasticsearch. There is no default. ::::{admonition} File paths :class: tip File paths can be specified as follows: **For Windows:** ```sh 'C:\path\to\file' ``` **For Linux, BSD, Mac OS:** ```sh '/path/to/file' ``` Using single-quotes around your file path is encouraged, especially with Windows file paths. :::: ## client_cert [client_cert] This should be a file path to a client certificate (public key), or left empty. ```sh client_cert: ``` Allows the use of a specified SSL client cert file to authenticate to Elasticsearch. The file may contain both an SSL client certificate and an SSL key, in which case [client_key](#client_key) is not used. If specifying `client_cert`, and the file specified does not also contain the key, use [client_key](#client_key) to specify the file containing the SSL key. The file must be in PEM format, and the key part, if used, must be an unencrypted key in PEM format as well. ::::{admonition} File paths :class: tip File paths can be specified as follows: **For Windows:** ```sh 'C:\path\to\file' ``` **For Linux, BSD, Mac OS:** ```sh '/path/to/file' ``` Using single-quotes around your file path is encouraged, especially with Windows file paths. :::: ## client_key [client_key] This should be a file path to a client key (private key), or left empty. ```sh client_key: ``` Allows the use of a specified SSL client key file to authenticate to Elasticsearch. If using [client_cert](#client_cert) and the file specified does not also contain the key, use `client_key` to specify the file containing the SSL key. The key file must be an unencrypted key in PEM format. ::::{admonition} File paths :class: tip File paths can be specified as follows: **For Windows:** ```sh 'C:\path\to\file' ``` **For Linux, BSD, Mac OS:** ```sh '/path/to/file' ``` Using single-quotes around your file path is encouraged, especially with Windows file paths. :::: ## verify_certs [verify_certs] This should be `True`, `False` or left empty. ```sh verify_certs: ``` If access to your Elasticsearch instance is protected by SSL encryption, you may set `verify_certs` to `False` to disable SSL certificate verification. Valid use cases for doing so include the use of self-signed certificates that cannot be otherwise verified and would generate error messages. ::::{warning} Setting `verify_certs` to `False` will likely result in a warning message that your SSL certificates are not trusted. This is expected behavior. :::: The default value is `True`. ## request_timeout [request_timeout] This should be an integer number of seconds, or left empty. ```sh request_timeout: ``` You can change the default client connection timeout value with this setting. The default value is `30` (seconds) should typically not be changed to be very large. If a longer timeout is necessary for a given action, such as [snapshot](/reference/snapshot.md), [restore](/reference/restore.md), or [forcemerge](/reference/forcemerge.md), the client timeout can be overridden on per action basis by setting [timeout_override](/reference/option_timeout_override.md) in the action [options](/reference/options.md). There are default override values for some of those longer running actions. ## master_only [master_only] This should be `True`, `False` or left empty. ```sh master_only: ``` In some situations, primarily with automated deployments, it makes sense to install Curator on every node. But you wouldn’t want it to run on each node. By setting `master_only` to `True`, this is possible. It tests for, and will only continue running on the node that is the elected master. ::::{warning} If `master_only` is `True`, and [hosts](#hosts) has more than one value, Curator will raise an Exception. This setting should *only* be used with a single host in [hosts](#hosts), as its utility centers around deploying to all nodes in the cluster. :::: The default value is `False`. ## username [username] The HTTP Basic Authentication username ## password [password] The HTTP Basic Authentication password ## id [id] This should be the `id` portion of an API Key pair. ```sh api_key: id: ``` This setting combined with the other subkey `api_key` allows API Key authentication to an Elasticsearch instance. The default is empty. ## api_key [api_key] This should be the `api_key` portion of an API Key pair. ```sh api_key: api_key: ``` This setting combined with the other subkey `id` allows API Key authentication to an Elasticsearch instance. The default is empty. ## token [token] This should be a base64 encoded representation of an API Key pair. ```sh api_key: token: ``` This setting will override any values provided for the `id` or `api_key` subkeys of `api_key`. The default is empty. ## loglevel [loglevel] This should be `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`, or left empty. ```sh loglevel: ``` Set the minimum acceptable log severity to display. * `CRITICAL` will only display critical messages. * `ERROR` will only display error and critical messages. * `WARNING` will display error, warning, and critical messages. * `INFO` will display informational, error, warning, and critical messages. * `DEBUG` will display debug messages, in addition to all of the above. The default value is `INFO`. ## logfile [logfile] This should be a path to a log file, or left empty. ```sh logfile: ``` ::::{admonition} File paths :class: tip File paths can be specified as follows: **For Windows:** ```sh 'C:\path\to\file' ``` **For Linux, BSD, Mac OS:** ```sh '/path/to/file' ``` Using single-quotes around your file path is encouraged, especially with Windows file paths. :::: The default value is empty, which will result in logging to `STDOUT`, or the console. ## logformat [logformat] This should `default`, `json`, `logstash`, `ecs` or left empty. ```sh logformat: ``` The `default` format looks like: ```sh 2016-04-22 11:53:09,972 INFO Action #1: ACTIONNAME ``` The `json` or `logstash` formats look like: ```sh {"@timestamp": "2016-04-22T11:54:29.033Z", "function": "cli", "linenum": 178, "loglevel": "INFO", "message": "Action #1: ACTIONNAME", "name": "curator.cli"} ``` The `ecs` format looks like: ```sh {"@timestamp":"2020-02-22T11:55:00.022Z","log.level":"info","message":"Action #1: ACTIONNAME","ecs":{"version":"1.6.0"},"log":{"logger":"curator.cli","origin": {"file":{"line":178,"name":"cli.py"},"function":"run"},"original":"Action #1: ACTIONNAME"},"process":{"name":"MainProcess","pid":12345,"thread": {"id":123456789886543,"name":"MainThread"}}} ``` The default value is `default`. ## blacklist [blacklist] This should be an empty array `[]`, an array of log handler strings, or left empty. ```sh blacklist: ['elastic_transport', 'urllib3'] ``` The default value is `['elastic_transport', 'urllib3']`, which will result in logs for the `elastic_transport` and `urllib3` Python modules *not* being output. These can be quite verbose, so unless you need them to debug an issue, you should accept the default value. ::::{tip} If you do need to troubleshoot an issue, set `blacklist` to `[]`, which is an empty array. Leaving it unset will result in the default behavior, which is to filter out `elastic_transport` and `urllib3` log traffic. :::: elasticsearch-curator-8.0.21/docs/reference/configuration.md000066400000000000000000000007271477314666200241760ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/configuration.html --- # Configuration [configuration] These are the higher-level configuration settings used by the configuration files. [Actions](/reference/actions.md) and [filters](/reference/filters.md) are documented separately. * [Environment Variables](/reference/envvars.md) * [Action File](/reference/actionfile.md) * [Configuration File](/reference/configfile.md) elasticsearch-curator-8.0.21/docs/reference/create_index.md000066400000000000000000000066331477314666200237630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/create_index.html --- # Create Index [create_index] ```yaml action: create_index description: "Create index as named" options: name: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action creates the named index. There are multiple different ways to configure how the name is represented. ## Manual naming [_manual_naming] ```yaml action: create_index description: "Create index as named" options: name: myindex # ... ``` In this case, what you see is what you get. An index named `myindex` will be created ## Python strftime [_python_strftime] ```yaml action: create_index description: "Create index as named" options: name: 'myindex-%Y.%m' # ... ``` For the `create_index` action, the [name](/reference/option_name.md) option can contain Python strftime strings. The method for doing so is described in detail, including which strftime strings are acceptable, in the documentation for the [name](/reference/option_name.md) option. ## Date Math [_date_math] ```yaml action: create_index description: "Create index as named" options: name: '' # ... ``` For the `create_index` action, the [name](/reference/option_name.md) option can be in Elasticsearch [date math](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/api-conventions.md#api-date-math-index-names) format. This allows index names containing dates to use deterministic math to set a date name in the past or the future. For example, if today’s date were 2017-03-27, the name `` will create an index named `logstash-2017.03.27`. If you wanted to create *tomorrow’s* index, you would use the name ``, which adds 1 day. This pattern creates an index named `logstash-2017.03.28`. For many more configuration options, read the Elasticsearch [date math](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/api-conventions.md#api-date-math-index-names) documentation. ## Extra Settings [_extra_settings] The [extra_settings](/reference/option_extra_settings.md) option allows the addition of extra settings, such as index settings and mappings. An example of how these settings can be used to create an index might be: ```yaml action: create_index description: "Create index as named" options: name: myindex # ... extra_settings: settings: number_of_shards: 1 number_of_replicas: 0 mappings: type1: properties: field1: type: string index: not_analyzed ``` ## Required settings [_required_settings_4] * [name](/reference/option_name.md) ## Optional settings [_optional_settings_6] * [extra_settings](/reference/option_extra_settings.md) No default value. You can add any acceptable index settings and mappings as nested YAML. See the [Elasticsearch Create Index API documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-create-index.md) for more information. * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_create_index.md). :::: elasticsearch-curator-8.0.21/docs/reference/curator-ilm.md000066400000000000000000000011741477314666200235620ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ilm.html - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html --- # Curator and index lifecycle management [ilm] Beginning with Elasticsearch version 6.6, Elasticsearch has provided [Index Lifecycle Management](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/index-lifecycle-management.md) (ILM) to users with at least a Basic license. ILM provides users with many of the most common index management features as a matter of policy, rather than execution time analysis (which is how Curator works). elasticsearch-curator-8.0.21/docs/reference/delete_indices.md000066400000000000000000000027251477314666200242670ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/delete_indices.html --- # Delete Indices [delete_indices] ```yaml action: delete_indices description: "Delete selected indices" options: continue_if_exception: False filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action deletes the selected indices. In clusters which are overcrowded with indices, or a high number of shards per node, deletes can take a longer time to process. In such cases, it may be helpful to set a higher timeout than is set in the [configuration file](/reference/configfile.md). You can override that [request_timeout](/reference/configfile.md#request_timeout) as follows: ```yaml action: delete_indices description: "Delete selected indices" options: timeout_override: 300 continue_if_exception: False filters: - filtertype: ... ``` ## Optional settings [_optional_settings_7] * [search_pattern](/reference/option_search_pattern.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_delete_indices.md). :::: elasticsearch-curator-8.0.21/docs/reference/delete_snapshots.md000066400000000000000000000026621477314666200246730ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/delete_snapshots.html --- # Delete Snapshots [delete_snapshots] ```yaml action: delete_snapshots description: "Delete selected snapshots from 'repository'" options: repository: ... retry_interval: 120 retry_count: 3 filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action deletes the selected snapshots from the selected [repository](/reference/option_repository.md). If a snapshot is currently underway, Curator will retry up to [retry_count](/reference/option_retry_count.md) times, with a delay of [retry_interval](/reference/option_retry_interval.md) seconds between retries. ## Required settings [_required_settings_5] * [repository](/reference/option_repository.md) ## Optional settings [_optional_settings_8] * [retry_interval](/reference/option_retry_interval.md) * [retry_count](/reference/option_retry_count.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_delete_snapshots.md). :::: elasticsearch-curator-8.0.21/docs/reference/docker.md000066400000000000000000000005511477314666200225710ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/docker.html --- # Docker [docker] Curator is periodically published to Docker Hub at [`untergeek/curator`](https://hub.docker.com/repository/docker/untergeek/curator/general). Download Curator Docker image: ``` docker pull untergeek/curator:{{curator_version}} ``` elasticsearch-curator-8.0.21/docs/reference/envvars.md000066400000000000000000000033251477314666200230100ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/envvars.html --- # Environment Variables [envvars] ::::{warning} This functionality is experimental and may be changed or removed
completely in a future release. :::: You can use environment variable references in both the [configuration file](/reference/configfile.md) and the [action file](/reference/actionfile.md) to set values that need to be configurable at runtime. To do this, use: ```sh ${VAR} ``` Where `VAR` is the name of the environment variable. Each variable reference is replaced at startup by the value of the environment variable. The replacement is case-sensitive and occurs while the YAML file is parsed, but before configuration schema validation. References to undefined variables are replaced by `None` unless you specify a default value. To specify a default value, use: ```sh ${VAR:default_value} ``` Where `default_value` is the value to use if the environment variable is undefined. ::::{admonition} Unsupported use cases :class: important When using environment variables, the value must *only* be the environment variable. Using extra text, such as: ```sh logfile: ${LOGPATH}/extra/path/information/file.log ``` is not supported at this time. :::: ## Examples [_examples] Here are some examples of configurations that use environment variables and what each configuration looks like after replacement: | Config source | Environment setting | Config after replacement | | --- | --- | --- | | `unit: ${UNIT}` | `export UNIT=days` | `unit: days` | | `unit: ${UNIT}` | no setting | `unit:` | | `unit: ${UNIT:days}` | no setting | `unit: days` | | `unit: ${UNIT:days}` | `export UNIT=hours` | `unit: hours` | elasticsearch-curator-8.0.21/docs/reference/ex_alias.md000066400000000000000000000024401477314666200231060ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_alias.html --- # alias [ex_alias] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: alias description: >- Alias indices from last week, with a prefix of logstash- to 'last_week', remove indices from the previous week. options: name: last_week warn_if_no_indices: False disable_action: True add: filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday remove: filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: period period_type: relative source: name range_from: -2 range_to: -2 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday ``` elasticsearch-curator-8.0.21/docs/reference/ex_allocation.md000066400000000000000000000016701477314666200241460ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_allocation.html --- # allocation [ex_allocation] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: allocation description: >- Apply shard allocation routing to 'require' 'tag=cold' for hot/cold node setup for logstash- indices older than 3 days, based on index_creation date options: key: tag value: cold allocation_type: require disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` elasticsearch-curator-8.0.21/docs/reference/ex_close.md000066400000000000000000000016131477314666200231230ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_close.html --- # close [ex_close] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: close description: >- Close indices older than 30 days (based on index name), for logstash- prefixed indices. options: skip_flush: False delete_aliases: False ignore_sync_failures: True disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 30 ``` elasticsearch-curator-8.0.21/docs/reference/ex_cluster_routing.md000066400000000000000000000022451477314666200252500ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_cluster_routing.html --- # cluster_routing [ex_cluster_routing] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. # # This action example has a blank spot at action ID 2. This is to show that # Curator can disable allocation before one or more actions, and then re-enable # it afterward. actions: 1: action: cluster_routing description: >- Disable shard routing for the entire cluster. options: routing_type: allocation value: none setting: enable wait_for_completion: True disable_action: True 2: action: (any other action details go here) ... 3: action: cluster_routing description: >- Re-enable shard routing for the entire cluster. options: routing_type: allocation value: all setting: enable wait_for_completion: True disable_action: True ``` elasticsearch-curator-8.0.21/docs/reference/ex_create_index.md000066400000000000000000000013171477314666200244510ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_create_index.html --- # create_index [ex_create_index] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: create_index description: Create the index as named, with the specified extra settings. options: name: myindex extra_settings: settings: number_of_shards: 2 number_of_replicas: 1 disable_action: True ``` elasticsearch-curator-8.0.21/docs/reference/ex_delete_indices.md000066400000000000000000000017651477314666200247660ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_delete_indices.html --- # delete_indices [ex_delete_indices] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: delete_indices description: >- Delete indices older than 45 days (based on index name), for logstash- prefixed indices. Ignore the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly. options: ignore_empty_list: True disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 45 ``` elasticsearch-curator-8.0.21/docs/reference/ex_delete_snapshots.md000066400000000000000000000016241477314666200253640ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_delete_snapshots.html --- # delete_snapshots [ex_delete_snapshots] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: delete_snapshots description: >- Delete snapshots from the selected repository older than 45 days (based on creation_date), for 'curator-' prefixed snapshots. options: repository: disable_action: True filters: - filtertype: pattern kind: prefix value: curator- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 45 ``` elasticsearch-curator-8.0.21/docs/reference/ex_forcemerge.md000066400000000000000000000023511477314666200241340ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_forcemerge.html --- # forcemerge [ex_forcemerge] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: forcemerge description: >- forceMerge logstash- prefixed indices older than 2 days (based on index creation_date) to 2 segments per shard. Delay 120 seconds between each forceMerge operation to allow the cluster to quiesce. Skip indices that have already been forcemerged to the minimum number of segments to avoid reprocessing. options: max_num_segments: 2 delay: 120 timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 2 exclude: - filtertype: forcemerged max_num_segments: 2 exclude: ``` elasticsearch-curator-8.0.21/docs/reference/ex_index_settings.md000066400000000000000000000017401477314666200250460ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_index_settings.html --- # index_settings [ex_index_settings] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: index_settings description: >- Set Logstash indices older than 10 days to be read only (block writes) options: disable_action: True index_settings: index: blocks: write: True ignore_unavailable: False preserve_existing: False filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 10 ``` elasticsearch-curator-8.0.21/docs/reference/ex_open.md000066400000000000000000000017361477314666200227650ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_open.html --- # open [ex_open] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: open description: >- Open indices older than 30 days but younger than 60 days (based on index name), for logstash- prefixed indices. options: disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 30 - filtertype: age source: name direction: younger timestring: '%Y.%m.%d' unit: days unit_count: 60 ``` elasticsearch-curator-8.0.21/docs/reference/ex_reindex.md000066400000000000000000000114741477314666200234620ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_reindex.html --- # reindex [ex_reindex] ## Manually selected reindex of a single index [_manually_selected_reindex_of_a_single_index] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: "Reindex index1 into index2" action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` ## Manually selected reindex of a multiple indices [_manually_selected_reindex_of_a_multiple_indices] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ``` ## Filter-Selected Indices [_filter_selected_indices_2] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: >- 'Reindex all daily logstash indices from March 2017 into logstash-2017.03' action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: index: REINDEX_SELECTION dest: index: logstash-2017.03 filters: - filtertype: pattern kind: prefix value: logstash-2017.03. ``` ## Reindex From Remote [_reindex_from_remote_2] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: >- 'Reindex all daily logstash indices from March 2017 into logstash-2017.03' action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: index1 dest: index: index1 filters: - filtertype: none ``` ## Reindex From Remote With Filter-Selected Indices [_reindex_from_remote_with_filter_selected_indices_2] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: >- Reindex all remote daily logstash indices from March 2017 into local index logstash-2017.03 action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: REINDEX_SELECTION dest: index: logstash-2017.03 remote_filters: - filtertype: pattern kind: prefix value: logstash-2017.03. filters: - filtertype: none ``` ## Manually selected reindex of a single index with query [_manually_selected_reindex_of_a_single_index_with_query] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: description: "Reindex index1 into index2" action: reindex options: disable_action: True wait_interval: 9 max_wait: -1 request_body: source: query: range: timestamp: gte: "now-1h" index: index1 dest: index: index2 filters: - filtertype: none ``` elasticsearch-curator-8.0.21/docs/reference/ex_replicas.md000066400000000000000000000015641477314666200236250ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_replicas.html --- # replicas [ex_replicas] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: replicas description: >- Reduce the replica count to 0 for logstash- prefixed indices older than 10 days (based on index creation_date) options: count: 0 wait_for_completion: True disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 10 ``` elasticsearch-curator-8.0.21/docs/reference/ex_restore.md000066400000000000000000000025311477314666200235010ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_restore.html --- # restore [ex_restore] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: restore description: >- Restore all indices in the most recent curator-* snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: include_aliases: False ignore_unavailable: False include_global_state: False partial: False rename_pattern: rename_replacement: extra_settings: wait_for_completion: True skip_repo_fs_check: True disable_action: True filters: - filtertype: pattern kind: prefix value: curator- - filtertype: state state: SUCCESS ``` elasticsearch-curator-8.0.21/docs/reference/ex_rollover.md000066400000000000000000000015511477314666200236630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_rollover.html --- # rollover [ex_rollover] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the format of prefix-000001 (or similar), or prefix-YYYY.MM.DD-1. options: disable_action: True name: aliasname conditions: max_age: 1d max_docs: 1000000 max_size: 50g extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 ``` elasticsearch-curator-8.0.21/docs/reference/ex_shrink.md000066400000000000000000000030401477314666200233100ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_shrink.html --- # shrink [ex_shrink] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: shrink description: >- Shrink logstash indices older than 21 days on the node with the most available space, excluding the node named 'not_this_node'. Delete each source index after successful shrink, then reroute the shrunk index with the provided parameters. options: disable_action: True ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 21 ``` elasticsearch-curator-8.0.21/docs/reference/ex_snapshot.md000066400000000000000000000024061477314666200236560ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ex_snapshot.html --- # snapshot [ex_snapshot] ```yaml --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: snapshot description: >- Snapshot logstash- prefixed indices older than 1 day (based on index creation_date) with the default snapshot name pattern of 'curator-%Y%m%d%H%M%S'. Wait for the snapshot to complete. Do not skip the repository filesystem access check. Use the other options to create the snapshot. options: repository: # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' name: ignore_unavailable: False include_global_state: True partial: False wait_for_completion: True skip_repo_fs_check: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- - filtertype: age source: creation_date direction: older unit: days unit_count: 1 ``` elasticsearch-curator-8.0.21/docs/reference/examples.md000066400000000000000000000016641477314666200231460ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/examples.html --- # Examples [examples] These examples should help illustrate how to build your own [actions](/reference/actions.md). You can use [environment variables](/reference/envvars.md) in your configuration files. * [alias](/reference/ex_alias.md) * [allocation](/reference/ex_allocation.md) * [close](/reference/ex_close.md) * [cluster_routing](/reference/ex_cluster_routing.md) * [create_index](/reference/ex_create_index.md) * [delete_indices](/reference/ex_delete_indices.md) * [delete_snapshots](/reference/ex_delete_snapshots.md) * [forcemerge](/reference/ex_forcemerge.md) * [open](/reference/ex_open.md) * [reindex](/reference/ex_reindex.md) * [replicas](/reference/ex_replicas.md) * [restore](/reference/ex_restore.md) * [rollover](/reference/ex_rollover.md) * [shrink](/reference/ex_shrink.md) * [snapshot](/reference/ex_snapshot.md) elasticsearch-curator-8.0.21/docs/reference/exit-codes.md000066400000000000000000000004451477314666200233700ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/exit-codes.html --- # Exit Codes [exit-codes] Exit codes will indicate success or failure. * `0` — Success * `1` — Failure * `-1` - Exception raised that does not result in a `1` exit code.   elasticsearch-curator-8.0.21/docs/reference/faq.md000066400000000000000000000007351477314666200220750ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/faq.html --- # Frequently Asked Questions [faq] This section will be updated as more frequently asked questions arise * [How can I report an error in the documentation?](/reference/faq_doc_error.md) * [Can I delete only certain data from within indices?](/reference/faq_partial_delete.md) * [Can Curator handle index names with strange characters?](/reference/faq_strange_chars.md) elasticsearch-curator-8.0.21/docs/reference/faq_doc_error.md000066400000000000000000000004751477314666200241340ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/faq_doc_error.html --- # Q: How can I report an error in the documentation? [faq_doc_error] ## A: Use the "Edit" link on any page [_a_use_the_edit_link_on_any_page] See [Site Corrections](/reference/site-corrections.md). elasticsearch-curator-8.0.21/docs/reference/faq_partial_delete.md000066400000000000000000000056071477314666200251360ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/faq_partial_delete.html --- # Q: Can I delete only certain data from within indices? [faq_partial_delete] ## A: It’s complicated [_a_its_complicated] #### TL;DR: No. Curator can only delete entire indices. [_tldr_no_curator_can_only_delete_entire_indices] #### Full answer: [_full_answer] As a thought exercise, think of Elasticsearch indices as being like databases, or tablespaces within a database. If you had hundreds of millions of rows to delete from your database, would you run a separate `DELETE from TABLE where date[1](http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.md) and watch what happens to your segments when you delete data.], nor best practices, it is still possible to perform these search & delete operations yourself, using the [Delete-by-Query API](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/docs-delete-by-query.md). Curator will not be modified to perform operations such as these, however. Curator is meant to manage at the index level, rather than the data level.
elasticsearch-curator-8.0.21/docs/reference/faq_strange_chars.md000066400000000000000000000060371477314666200250010ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/faq_strange_chars.html --- # Q: Can Curator handle index names with strange characters? [faq_strange_chars] ## A: Yes! [_a_yes] This problem can be resolved by using the [pattern filtertype](/reference/filtertype_pattern.md) with [kind](/reference/fe_kind.md) set to `regex`, and [value](/reference/fe_value.md) set to the needed regular expression. #### The Problem: [_the_problem] Illegal characters make it hard to delete indices. ``` % curl logs.example.com:9200/_cat/indices red }?ebc-2015.04.08.03 sip-request{ 5 1 0 0 632b 316b red }?ebc-2015.04.08.03 sip-response 5 1 0 0 474b 237b red ?ebc-2015.04.08.02 sip-request{ 5 1 0 0 474b 316b red eb 5 1 0 0 632b 316b red ?e 5 1 0 0 632b 316b ```   You can see it looks like there are some tab characters and maybe newline characters. This makes it hard to use the HTTP API to delete the indices. Dumping all the index settings out: ```sh curl -XGET localhost:9200/*/_settings?pretty ```   …​reveals the index names as the first key in the resulting JSON. In this case, the names were very atypical: ``` }\b?\u0011ebc-2015.04.08.02\u000Bsip-request{ }\u0006?\u0011ebc-2015.04.08.03\u000Bsip-request{ }\u0003?\u0011ebc-2015.04.08.03\fsip-response ... ```   Curator lets you use regular expressions to select indices to perform actions on. ::::{warning} Before attempting an action, see what will be affected by using the `--dry-run` flag first. :::: To delete the first three from the above example, use `'.*sip.*'` as your regular expression. ::::{note} In an [actionfile](/reference/actionfile.md), regular expressions and strftime date strings *must* be encapsulated in single-quotes. :::: The next one is trickier. The real name of the index was `\n\u0011eb`. The regular expression `.*b$` did not work, but `'\n.*'` did. The last index can be deleted with a regular expression of `'.*e$'`. The resulting [actionfile](/reference/actionfile.md) might look like this: ```yaml actions: 1: description: Delete indices with strange characters that match regex '.*sip.*' action: delete_indices options: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: regex value: '.*sip.*' 2: description: Delete indices with strange characters that match regex '\n.*' action: delete_indices options: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: regex value: '\n.*' 3: description: Delete indices with strange characters that match regex '.*e$' action: delete_indices options: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: regex value: '.*e$' ``` elasticsearch-curator-8.0.21/docs/reference/fe_aliases.md000066400000000000000000000026461477314666200234240ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_aliases.html --- # aliases [fe_aliases] ::::{admonition} Matching Indices and Aliases :class: important [Indices must be in all aliases to match](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/breaking-changes-5.5.html#breaking_55_rest_changes). If a list of `aliases` is provided (instead of only one), indices must appear in *all* listed `aliases` or a 404 error will result, leading to no indices being matched. In older versions, if the index was associated with even one of the aliases in `aliases`, it would result in a match. :::: ::::{note} This setting is used only when using the [alias](/reference/filtertype_alias.md) filter. :::: The value of this setting must be a single alias name, or a list of alias names. This can be done in any of the ways YAML allows for lists or arrays. Here are a few examples. **Single** ```txt filters: - filtertype: alias aliases: my_alias exclude: False ``` **List** * Flow style: ```txt filters: - filtertype: alias aliases: [ my_alias, another_alias ] exclude: False ``` * Block style: ```txt filters: - filtertype: alias aliases: - my_alias - another_alias exclude: False ``` There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_allocation_type.md000066400000000000000000000012361477314666200251630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_allocation_type.html --- # allocation_type [fe_allocation_type] ::::{note} This setting is used only when using the [allocated](/reference/filtertype_allocated.md) filter. :::: ```yaml - filtertype: allocated key: ... value: ... allocation_type: require exclude: True ``` The value of this setting must be one of `require`, `include`, or `exclude`. Read more about these settings in the [Elasticsearch documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shard-allocation-filtering.html). The default value for this setting is `require`. elasticsearch-curator-8.0.21/docs/reference/fe_count.md000066400000000000000000000012141477314666200231210ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_count.html --- # count [fe_count] ::::{note} This setting is only used with the [count](/reference/filtertype_count.md) filtertype
and is a required setting. :::: ```yaml - filtertype: count count: 10 ``` The value for this setting is a number of indices or snapshots to match. Items will remain in the actionable list depending on the value of [exclude](/reference/fe_exclude.md), and [reverse](/reference/fe_reverse.md). There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_date_from.md000066400000000000000000000023601477314666200237340ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_date_from.html --- # date_from [fe_date_from] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype
when [period_type](/reference/fe_period_type.md) is `absolute`. :::: ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ``` The value for this setting should be a date which can be easily parsed using the strftime string in [`date_from_format`](/reference/fe_date_from_format.md): The identifiers that Curator currently recognizes include: | Unit | Value | Note | | --- | --- | --- | | `%Y` | 4 digit year | | | `%G` | 4 digit year | use instead of `%Y` when doing ISO Week calculations | | `%y` | 2 digit year | | | `%m` | 2 digit month | | | `%W` | 2 digit week of the year | | | `%V` | 2 digit week of the year | use instead of `%W` when doing ISO Week calculations | | `%d` | 2 digit day of the month | | | `%H` | 2 digit hour | 24 hour notation | | `%M` | 2 digit minute | | | `%S` | 2 digit second | | | `%j` | 3 digit day of the year | | elasticsearch-curator-8.0.21/docs/reference/fe_date_from_format.md000066400000000000000000000023541477314666200253070ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_date_from_format.html --- # date_from_format [fe_date_from_format] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype
when [period_type](/reference/fe_period_type.md) is `absolute`. :::: ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ``` The value for this setting should be an strftime string which corresponds to the date in [`date_from`](/reference/fe_date_from.md): The identifiers that Curator currently recognizes include: | Unit | Value | Note | | --- | --- | --- | | `%Y` | 4 digit year | | | `%G` | 4 digit year | use instead of `%Y` when doing ISO Week calculations | | `%y` | 2 digit year | | | `%m` | 2 digit month | | | `%W` | 2 digit week of the year | | | `%V` | 2 digit week of the year | use instead of `%W` when doing ISO Week calculations | | `%d` | 2 digit day of the month | | | `%H` | 2 digit hour | 24 hour notation | | `%M` | 2 digit minute | | | `%S` | 2 digit second | | | `%j` | 3 digit day of the year | | elasticsearch-curator-8.0.21/docs/reference/fe_date_to.md000066400000000000000000000023461477314666200234170ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_date_to.html --- # date_to [fe_date_to] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype
when [period_type](/reference/fe_period_type.md) is `absolute`. :::: ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ``` The value for this setting should be a date which can be easily parsed using the strftime string in [`date_to_format`](/reference/fe_date_to_format.md): The identifiers that Curator currently recognizes include: | Unit | Value | Note | | --- | --- | --- | | `%Y` | 4 digit year | | | `%G` | 4 digit year | use instead of `%Y` when doing ISO Week calculations | | `%y` | 2 digit year | | | `%m` | 2 digit month | | | `%W` | 2 digit week of the year | | | `%V` | 2 digit week of the year | use instead of `%W` when doing ISO Week calculations | | `%d` | 2 digit day of the month | | | `%H` | 2 digit hour | 24 hour notation | | `%M` | 2 digit minute | | | `%S` | 2 digit second | | | `%j` | 3 digit day of the year | | elasticsearch-curator-8.0.21/docs/reference/fe_date_to_format.md000066400000000000000000000023421477314666200247630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_date_to_format.html --- # date_to_format [fe_date_to_format] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype
when [period_type](/reference/fe_period_type.md) is `absolute`. :::: ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ``` The value for this setting should be an strftime string which corresponds to the date in [`date_to`](/reference/fe_date_to.md): The identifiers that Curator currently recognizes include: | Unit | Value | Note | | --- | --- | --- | | `%Y` | 4 digit year | | | `%G` | 4 digit year | use instead of `%Y` when doing ISO Week calculations | | `%y` | 2 digit year | | | `%m` | 2 digit month | | | `%W` | 2 digit week of the year | | | `%V` | 2 digit week of the year | use instead of `%W` when doing ISO Week calculations | | `%d` | 2 digit day of the month | | | `%H` | 2 digit hour | 24 hour notation | | `%M` | 2 digit minute | | | `%S` | 2 digit second | | | `%j` | 3 digit day of the year | | elasticsearch-curator-8.0.21/docs/reference/fe_direction.md000066400000000000000000000020251477314666200237520ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_direction.html --- # direction [fe_direction] ::::{note} This setting is only used with the [age](/reference/filtertype_age.md) filtertype. :::: ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` This setting must be either `older` or `younger`. This setting is used to determine whether indices or snapshots are `older` or `younger` than the reference point in time determined by [unit](/reference/fe_unit.md), [unit_count](/reference/fe_unit_count.md), and optionally, [epoch](/reference/fe_epoch.md). If `direction` is `older`, then indices (or snapshots) which are *older* than the reference point in time will be matched. Likewise, if `direction` is `younger`, then indices (or snapshots) which are *younger* than the reference point in time will be matched. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_disk_space.md000066400000000000000000000010731477314666200241010ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_disk_space.html --- # disk_space [fe_disk_space] ::::{note} This setting is only used with the [space](/reference/filtertype_space.md) filtertype
and is a required setting. :::: ```yaml - filtertype: space disk_space: 100 ``` The value for this setting is a number of gigabytes. Indices in excess of this number of gigabytes will be matched. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_epoch.md000066400000000000000000000023561477314666200230770ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_epoch.html --- # epoch [fe_epoch] ::::{note} This setting is available in the [age](/reference/filtertype_age.md) filtertype, and any filter which has the [`use_age`](/reference/fe_use_age.md) setting. This setting is strictly optional. :::: ::::{tip} This setting is not common. It is most frequently used for testing. :::: [unit](/reference/fe_unit.md), [unit_count](/reference/fe_unit_count.md), and optionally, epoch, are used by Curator to establish the moment in time point of reference with this formula: ```sh point_of_reference = epoch - ((number of seconds in unit) * unit_count) ``` If epoch is unset, the current time is used. It is possible to set a point of reference in the future by using a negative value for [unit_count](/reference/fe_unit_count.md). ## Example [_example] ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 epoch: 1491577200 ``` The value for this setting must be an epoch timestamp. In this example, the given epoch time of `1491577200` is 2017-04-04T15:00:00Z (UTC). This will use 3 days older than that timestamp as the point of reference for age comparisons. elasticsearch-curator-8.0.21/docs/reference/fe_exclude.md000066400000000000000000000013231477314666200234230ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_exclude.html --- # exclude [fe_exclude] ::::{note} This setting is available in *all* filter types. :::: If `exclude` is `True`, the filter will remove matches from the actionable list. If `exclude` is `False`, then only matches will be kept in the actionable list. The default value for this setting is different for each filter type. ## Examples [_examples_2] ```yaml - filtertype: opened exclude: True ``` This filter will result in only `closed` indices being in the actionable list. ```yaml - filtertype: opened exclude: False ``` This filter will result in only `open` indices being in the actionable list. elasticsearch-curator-8.0.21/docs/reference/fe_field.md000066400000000000000000000024551477314666200230640ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_field.html --- # field [fe_field] ::::{note} This setting is available in the [age](/reference/filtertype_age.md) filtertype, and any filter which has the [`use_age`](/reference/fe_use_age.md) setting. This setting is strictly optional. :::: ```yaml - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ``` The value of this setting must be a timestamp field name. This field must be present in the indices being filtered or an exception will be raised, and execution will halt. In Curator 5.3 and older, source `field_stats` uses the [Field Stats API](http://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.md) to calculate either the `min_value` or the `max_value` of the `field` as the [`stats_result`](/reference/fe_stats_result.md), and then use that value for age comparisons. In 5.4 and above, even though it is still called `field_stats`, it uses an aggregation to calculate the same values, as the `field_stats` API is no longer used in Elasticsearch 6.x and up. This setting is only used when [source](/reference/fe_source.md) is `field_stats`. The default value for this setting is `@timestamp`. elasticsearch-curator-8.0.21/docs/reference/fe_intersect.md000066400000000000000000000022471477314666200240000ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_intersect.html --- # intersect [fe_intersect] ::::{note} This setting is only available in the [period](/reference/filtertype_age.md) filtertype. This setting is strictly optional. :::: ```yaml - filtertype: period source: field_stats direction: older intersect: true unit: weeks range_from: -1 range_to: -1 field: '@timestamp' stats_result: min_value ``` The value of this setting must be `True` or `False`. `field_stats` uses an aggregation query to calculate either the `min_value` and the `max_value` of the [`field`](/reference/fe_field.md) as the [`stats_result`](/reference/fe_stats_result.md). If `intersect` is `True`, then only indices where the `min_value` *and* the `max_value` are within the `range_from` and `range_to` (relative to `unit`) will match. This means that either `min_value` or `max_value` can be used for [`stats_result`](/reference/fe_stats_result.md) when `intersect` is `True` with identical results. This setting is only used when [source](/reference/fe_source.md) is `field_stats`. The default value for this setting is `False`. elasticsearch-curator-8.0.21/docs/reference/fe_key.md000066400000000000000000000021041477314666200225600ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_key.html --- # key [fe_key] ::::{note} This setting is required when using the [allocated filtertype](/reference/filtertype_allocated.md). :::: ```yaml - filtertype: allocated key: ... value: ... allocation_type: exclude: True ``` The value of this setting should correspond to a node setting on one or more nodes in your cluster. For example, you might have set ```sh node.tag: myvalue ``` in your `elasticsearch.yml` file for one or more of your nodes. To match allocation in this case, set key to `tag`. These special attributes are also supported: | attribute | description | | --- | --- | | `_name` | Match nodes by node name | | `_host_ip` | Match nodes by host IP address (IP associated with hostname) | | `_publish_ip` | Match nodes by publish IP address | | `_ip` | Match either `_host_ip` or `_publish_ip` | | `_host` | Match nodes by hostname | There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_kind.md000066400000000000000000000105551477314666200227260ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_kind.html --- # kind [fe_kind] ::::{note} This setting is only used with the [pattern](/reference/filtertype_pattern.md)
filtertype and is a required setting. :::: This setting tells the [pattern](/reference/filtertype_pattern.md) what pattern type to match. Acceptable values for this setting are `prefix`, `suffix`, `timestring`, and `regex`. ::::{admonition} Filter chaining :class: note It is important to note that while filters can be chained, each is linked by an implied logical **AND** operation. If you want to match from one of several different patterns, as with a logical **OR** operation, you can do so with the [pattern](/reference/filtertype_pattern.md) filtertype using *regex* as the `kind`. This example shows how to select multiple indices based on them beginning with either `alpha-`, `bravo-`, or `charlie-`: ```yaml filters: - filtertype: pattern kind: regex value: '^(alpha-|bravo-|charlie-).*$' ``` Explaining all of the different ways in which regular expressions can be used is outside the scope of this document, but hopefully this gives you some idea of how a regular expression pattern can be used when a logical **OR** is desired. :::: There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. The different `kinds` are described as follows: ## prefix [_prefix_2] To match all indices starting with `logstash-`: ```yaml - filtertype: pattern kind: prefix value: logstash- ``` To match all indices *except* those starting with `logstash-`: ```yaml - filtertype: pattern kind: prefix value: logstash- exclude: True ``` ::::{note} Internally, the `prefix` value is used to create a *regex* pattern: `^{{0}}.*$`. Any special characters should be escaped with a backslash to match literally. :::: ## suffix [_suffix_2] To match all indices ending with `-prod`: ```yaml - filtertype: pattern kind: suffix value: -prod ``` To match all indices *except* those ending with `-prod`: ```yaml - filtertype: pattern kind: suffix value: -prod exclude: True ``` ::::{note} Internally, the `suffix` value is used to create a *regex* pattern: `^.*{{0}}$`. Any special characters should be escaped with a backslash to match literally. :::: ## timestring [_timestring_2] ::::{important} No age calculation takes place here. It is strictly a pattern match. :::: To match all indices with a Year.month.day pattern, like `index-2017.04.01`: ```yaml - filtertype: pattern kind: timestring value: '%Y.%m.%d' ``` To match all indices *except* those with a Year.month.day pattern, like `index-2017.04.01`: ```yaml - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` ::::{admonition} A word about regular expression matching with timestrings :class: warning Timestrings are parsed from strftime patterns, like `%Y.%m.%d`, into regular expressions. For example, `%Y` is 4 digits, so the regular expression for that looks like `\d{{4}}`, and `%m` is 2 digits, so the regular expression is `\d{{2}}`. What this means is that a simple timestring to match year and month, `%Y.%m` will result in a regular expression like this: `^.*\d{{4}}\.\d{{2}}.*$`. This pattern will match any 4 digits, followed by a period `.`, followed by 2 digits, occurring anywhere in the index name. This means it *will* match monthly indices, like `index-2016.12`, as well as daily indices, like `index-2017.04.01`, which may not be the intended behavior. To compensate for this, when selecting indices matching a subset of another pattern, use a second filter with `exclude` set to `True` ```yaml - filtertype: pattern kind: timestring value: '%Y.%m' - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` This will prevent the `%Y.%m` pattern from matching the `%Y.%m` part of the daily indices. **This applies whether using `timestring` as a mere pattern match, or as part of date calculations.** :::: ## regex [_regex_2] This `kind` allows you to design a regular-expression to match indices or snapshots: To match all indices starting with `a-`, `b-`, or `c-`: ```yaml - filtertype: pattern kind: regex value: '^a-|^b-|^c-' ``` To match all indices *except* those starting with `a-`, `b-`, or `c-`: ```yaml - filtertype: pattern kind: regex value: '^a-|^b-|^c-' exclude: True ``` elasticsearch-curator-8.0.21/docs/reference/fe_max_num_segments.md000066400000000000000000000014361477314666200253500ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_max_num_segments.html --- # max_num_segments [fe_max_num_segments] ::::{note} This setting is only used with the [forcemerged](/reference/filtertype_forcemerged.md) filtertype. :::: ```yaml - filtertype: forcemerged max_num_segments: 2 exclude: True ``` The value for this setting is the cutoff number of segments per shard. Indices which have this number of segments per shard, or fewer, will be actionable depending on the value of [exclude](/reference/fe_exclude.md), which is `True` by default for the [forcemerged](/reference/filtertype_forcemerged.md) filter type. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_pattern.md000066400000000000000000000032321477314666200234500ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_pattern.html --- # pattern [fe_pattern] ::::{note} This setting is only used with the [count](/reference/filtertype_count.md) filtertype :::: ```yaml - filtertype: count count: 1 pattern: '^(.*)-\d{6}$' reverse: true ``` This particular example will match indices following the basic rollover pattern of `indexname-######`, and keep the highest numbered index for each group. For example, given indices `a-000001`, `a-000002`, `a-000003` and `b-000006`, and `b-000007`, the indices will would be matched are `a-000003` and `b-000007`. Indices that do not match the regular expression in `pattern` will be automatically excluded. This is particularly useful with indices created and managed using the [Rollover API](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-rollover-index.md), as you can select only the active indices with the above example ([`exclude`](/reference/fe_exclude.md) defaults to `False`). Setting [`exclude`](/reference/fe_exclude.md) to `True` with the above example will *remove* the active rollover indices, leaving only those which have been rolled-over. While this is perhaps most useful for the aforementioned scenario, it can also be used with age-based indices as well. Items will remain in the actionable list depending on the value of [exclude](/reference/fe_exclude.md), and [reverse](/reference/fe_reverse.md). There is no default value. The value must include a capture group, defined by parenthesis, or left empty. If a value is provided, and there is no capture group, and exception will be raised and execution will halt. elasticsearch-curator-8.0.21/docs/reference/fe_period_type.md000066400000000000000000000010741477314666200243200ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_period_type.html --- # period_type [fe_period_type] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype :::: ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ``` The value for this setting must be either `relative` or `absolute`. The default value is `relative`. elasticsearch-curator-8.0.21/docs/reference/fe_range_from.md000066400000000000000000000014061477314666200241130ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_range_from.html --- # range_from [fe_range_from] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype :::: ```yaml - filtertype: period source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: days ``` `range_from` and [`range_to`](/reference/fe_range_to.md) are counters of whole [units](/reference/fe_unit.md). A negative number indicates a whole unit in the past, while a positive number indicates a whole unit in the future. A `0` indicates the present unit. Read more about this setting in context in the [period filtertype documentation](/reference/filtertype_period.md), including examples. elasticsearch-curator-8.0.21/docs/reference/fe_range_to.md000066400000000000000000000013761477314666200236000ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_range_to.html --- # range_to [fe_range_to] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype :::: ```yaml - filtertype: period source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: days ``` [range_from](/reference/fe_range_from.md) and range_to are counters of whole [units](/reference/fe_unit.md). A negative number indicates a whole unit in the past, while a positive number indicates a whole unit in the future. A `0` indicates the present unit. Read more about this setting in context in the [period filtertype documentation](/reference/filtertype_period.md), including examples. elasticsearch-curator-8.0.21/docs/reference/fe_reverse.md000066400000000000000000000014531477314666200234510ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_reverse.html --- # reverse [fe_reverse] ::::{note} This setting is used in the [count](/reference/filtertype_count.md) and [space](/reference/filtertype_space.md) filtertypes :::: This setting affects the sort order of the indices. `True` means reverse-alphabetical. This means that if all index names share the same pattern with a date—​e.g. index-2016.03.01—​older indices will be selected first. The default value of this setting is `True`. This setting is ignored if [use_age](/reference/fe_use_age.md) is `True`. ::::{tip} There are context-specific examples of how `reverse` works in the [count](/reference/filtertype_count.md) and [space](/reference/filtertype_space.md) documentation. :::: elasticsearch-curator-8.0.21/docs/reference/fe_source.md000066400000000000000000000067251477314666200233050ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_source.html --- # source [fe_source] The *source* from which to derive the index or snapshot age. Can be one of `name`, `creation_date`, or `field_stats`. ::::{note} This setting is only used with the [age](/reference/filtertype_age.md) filtertype, or
with the [space](/reference/filtertype_space.md) filtertype when [use_age](/reference/fe_use_age.md) is set to `True`. :::: ::::{note} When using the [age](/reference/filtertype_age.md) filtertype, source requires
[direction](/reference/fe_direction.md), [unit](/reference/fe_unit.md), [unit_count](/reference/fe_unit_count.md),
and additionally, the optional setting, [epoch](/reference/fe_epoch.md). :::: ## `name`-based ages [_name_based_ages_2] Using `name` as the `source` tells Curator to look for a [`timestring`](/reference/fe_timestring.md) within the index or snapshot name, and convert that into an epoch timestamp (epoch implies UTC). ```yaml - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 3 ``` ::::{admonition} A word about regular expression matching with timestrings :class: warning Timestrings are parsed from strftime patterns, like `%Y.%m.%d`, into regular expressions. For example, `%Y` is 4 digits, so the regular expression for that looks like `\d{{4}}`, and `%m` is 2 digits, so the regular expression is `\d{{2}}`. What this means is that a simple timestring to match year and month, `%Y.%m` will result in a regular expression like this: `^.*\d{{4}}\.\d{{2}}.*$`. This pattern will match any 4 digits, followed by a period `.`, followed by 2 digits, occurring anywhere in the index name. This means it *will* match monthly indices, like `index-2016.12`, as well as daily indices, like `index-2017.04.01`, which may not be the intended behavior. To compensate for this, when selecting indices matching a subset of another pattern, use a second filter with `exclude` set to `True` ```yaml - filtertype: pattern kind: timestring value: '%Y.%m' - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` This will prevent the `%Y.%m` pattern from matching the `%Y.%m` part of the daily indices. **This applies whether using `timestring` as a mere pattern match, or as part of date calculations.** :::: ## `creation_date`-based ages [_creation_date_based_ages_2] `creation_date` extracts the epoch time of index or snapshot creation. ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` ## `field_stats`-based ages [_field_stats_based_ages_2] ::::{note} `source` can only be `field_stats` when filtering indices. :::: In Curator 5.3 and older, source `field_stats` uses the [Field Stats API](http://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.md) to calculate either the `min_value` or the `max_value` of the [`field`](/reference/fe_field.md) as the [`stats_result`](/reference/fe_stats_result.md), and then use that value for age comparisons. In 5.4 and above, even though it is still called `field_stats`, it uses an aggregation to calculate the same values, as the `field_stats` API is no longer used in Elasticsearch 6.x and up. [`field`](/reference/fe_field.md) must be of type `date` in Elasticsearch. ```yaml - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ``` elasticsearch-curator-8.0.21/docs/reference/fe_state.md000066400000000000000000000007541477314666200231210ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_state.html --- # state [fe_state] ::::{note} This setting is only used with the [state](/reference/filtertype_state.md) filtertype. :::: ```yaml - filtertype: state state: SUCCESS ``` The value for this setting must be one of `SUCCESS`, `PARTIAL`, `FAILED`, or `IN_PROGRESS`. This setting determines what kind of snapshots will be passed. The default value for this setting is `SUCCESS`. elasticsearch-curator-8.0.21/docs/reference/fe_stats_result.md000066400000000000000000000013451477314666200245320ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_stats_result.html --- # stats_result [fe_stats_result] ::::{note} This setting is only used with the [age](/reference/filtertype_age.md) filtertype. :::: ```yaml - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ``` The value for this setting can be either `min_value` or `max_value`. This setting is only used when [source](/reference/fe_source.md) is `field_stats`, and determines whether Curator will use the minimum or maximum value of [field](/reference/fe_field.md) for time calculations. The default value for this setting is `min_value`. elasticsearch-curator-8.0.21/docs/reference/fe_threshold_behavior.md000066400000000000000000000012731477314666200256510ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_threshold_behavior.html --- # threshold_behavior [fe_threshold_behavior] ::::{note} This setting is only available in the [space](/reference/filtertype_space.md) filtertype. This setting is optional, and defaults to `greater_than` to preserve backwards compatability. :::: ```yaml - filtertype: space disk_space: 5 threshold_behavior: less_than ``` The value for this setting is `greater_than` (default) or `less_than`. When set to `less_than`, indices in less than `disk_space` gigabytes will be matched. When set to `greater_than` (default), indices larger than `disk_space` will be matched. elasticsearch-curator-8.0.21/docs/reference/fe_timestring.md000066400000000000000000000054531477314666200241670ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_timestring.html --- # timestring [fe_timestring] ::::{note} This setting is only used with the [age](/reference/filtertype_age.md) filtertype, or
with the [space](/reference/filtertype_space.md) filtertype if [use_age](/reference/fe_use_age.md) is set to `True`. :::: ## strftime [_strftime_2] This setting must be a valid Python strftime string. It is used to match and extract the timestamp in an index or snapshot name. The identifiers that Curator currently recognizes include: | Unit | Value | Note | | --- | --- | --- | | `%Y` | 4 digit year | | | `%G` | 4 digit year | use instead of `%Y` when doing ISO Week calculations | | `%y` | 2 digit year | | | `%m` | 2 digit month | | | `%W` | 2 digit week of the year | | | `%V` | 2 digit week of the year | use instead of `%W` when doing ISO Week calculations | | `%d` | 2 digit day of the month | | | `%H` | 2 digit hour | 24 hour notation | | `%M` | 2 digit minute | | | `%S` | 2 digit second | | | `%j` | 3 digit day of the year | | These identifiers may be combined with each other, and/or separated from each other with hyphens `-`, periods `.`, underscores `_`, or other characters valid in an index name. Each identifier must be preceded by a `%` character in the timestring. For example, an index like `index-2016.04.01` would use a timestring of `'%Y.%m.%d'`. When [source](/reference/fe_source.md) is `name`, this setting must be set by the user or an exception will be raised, and execution will halt. There is no default value. ::::{admonition} A word about regular expression matching with timestrings :class: warning Timestrings are parsed from strftime patterns, like `%Y.%m.%d`, into regular expressions. For example, `%Y` is 4 digits, so the regular expression for that looks like `\d{{4}}`, and `%m` is 2 digits, so the regular expression is `\d{{2}}`. What this means is that a simple timestring to match year and month, `%Y.%m` will result in a regular expression like this: `^.*\d{{4}}\.\d{{2}}.*$`. This pattern will match any 4 digits, followed by a period `.`, followed by 2 digits, occurring anywhere in the index name. This means it *will* match monthly indices, like `index-2016.12`, as well as daily indices, like `index-2017.04.01`, which may not be the intended behavior. To compensate for this, when selecting indices matching a subset of another pattern, use a second filter with `exclude` set to `True` ```yaml - filtertype: pattern kind: timestring value: '%Y.%m' - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` This will prevent the `%Y.%m` pattern from matching the `%Y.%m` part of the daily indices. **This applies whether using `timestring` as a mere pattern match, or as part of date calculations.** :::: elasticsearch-curator-8.0.21/docs/reference/fe_unit.md000066400000000000000000000040111477314666200227460ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_unit.html --- # unit [fe_unit] ::::{note} This setting is used with the [age](/reference/filtertype_age.md) filtertype, with the [period](/reference/filtertype_period.md) filtertype, or with the [space](/reference/filtertype_space.md) filtertype if [use_age](/reference/fe_use_age.md) is set to `True`. :::: ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` This setting must be one of `seconds`, `minutes`, `hours`, `days`, `weeks`, `months`, or `years`. The values `seconds` and `minutes` are not allowed with the [period](/reference/filtertype_period.md) filtertype and will result in an error condition if used there. For the [age](/reference/filtertype_age.md) filtertype, or when [use_age](/reference/fe_use_age.md) is set to `True`, unit, [unit_count](/reference/fe_unit_count.md), and optionally, [epoch](/reference/fe_epoch.md), are used by Curator to establish the moment in time point of reference with this formula: ```sh point_of_reference = epoch - ((number of seconds in unit) * unit_count) ``` `units` are calculated as follows: | Unit | Seconds | Note | | --- | --- | --- | | `seconds` | `1` | One second | | `minutes` | `60` | Calculated as 60 seconds | | `hours` | `3600` | Calculated as 60 minutes (60*60) | | `days` | `86400` | Calculated as 24 hours (24*60*60) | | `weeks` | `604800` | Calculated as 7 days (7*24*60*60) | | `months` | `2592000` | Calculated as 30 days (30*24*60*60) | | `years` | `31536000` | Calculated as 365 days (365*24*60*60) | If [epoch](/reference/fe_epoch.md) is unset, the current time is used. It is possible to set a point of reference in the future by using a negative value for [unit_count](/reference/fe_unit_count.md). This setting must be set by the user or an exception will be raised, and execution will halt. ::::{tip} See the [age filter documentation](/reference/filtertype_age.md) for more information about time calculation. :::: elasticsearch-curator-8.0.21/docs/reference/fe_unit_count.md000066400000000000000000000037641477314666200241740ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_unit_count.html --- # unit_count [fe_unit_count] ::::{note} This setting is only used with the [age](/reference/filtertype_age.md) filtertype, or
with the [space](/reference/filtertype_space.md) filtertype if [use_age](/reference/fe_use_age.md) is set to `True`. :::: ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` The value of this setting will be used as a multiplier for [unit](/reference/fe_unit.md). [unit](/reference/fe_unit.md), unit_count, and optionally, [epoch](/reference/fe_epoch.md), are used by Curator to establish the moment in time point of reference with this formula: ```sh point_of_reference = epoch - ((number of seconds in unit) * unit_count) ``` [`units`](/reference/fe_unit.md) are calculated as follows: | Unit | Seconds | Note | | --- | --- | --- | | `seconds` | `1` | One second | | `minutes` | `60` | Calculated as 60 seconds | | `hours` | `3600` | Calculated as 60 minutes (60*60) | | `days` | `86400` | Calculated as 24 hours (24*60*60) | | `weeks` | `604800` | Calculated as 7 days (7*24*60*60) | | `months` | `2592000` | Calculated as 30 days (30*24*60*60) | | `years` | `31536000` | Calculated as 365 days (365*24*60*60) | If [epoch](/reference/fe_epoch.md) is unset, the current time is used. It is possible to set a point of reference in the future by using a negative value for unit_count. This setting must be set by the user or an exception will be raised, and execution will halt. If this setting is used in conjunction with [unit_count_pattern](/reference/fe_unit_count_pattern.md), the configured value will only be used as a fallback value in case the pattern could not be matched. The value *-1* has a special meaning in this case and causes the index to be ignored when pattern matching fails. ::::{tip} See the [age filter documentation](/reference/filtertype_age.md) for more information about time calculation. :::: elasticsearch-curator-8.0.21/docs/reference/fe_unit_count_pattern.md000066400000000000000000000055341477314666200257260ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_unit_count_pattern.html --- # unit_count_pattern [fe_unit_count_pattern] ::::{note} This setting is only used with the age filtertype to define, whether the [unit_count](/reference/fe_unit_count.md) value is taken from the configuration or read from the index name via a regular expression. :::: ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 unit_count_pattern: -([0-9]+)- ``` This setting can be used in cases where the value against which index age should be assessed is not a static value but can be different for every index. For this case, there is the option of extracting the index specific value from the index names via a regular expression defined in this parameter. Consider for example the following index name patterns that contain the retention time in their name: *logstash-30-yyyy.mm.dd*, *logstash-12-yyyy.mm*, *_3_logstash-yyyy.mm.dd*. To extract a value from the index names, this setting will be compiled as a regular expression and matched against index names, for a successful match, the value of the first capture group from the regular expression is used as the value for [unit_count](/reference/fe_unit_count.md). If there is any error during compiling or matching the expression, or the expression does not contain a capture group, the value configured in [unit_count](/reference/fe_unit_count.md) is used as a fallback value, unless it is set to *-1*, in which case the index will be skipped. ::::{tip} Regular expressions and match groups are not explained here as they are a fairly large and complex topic, but there are numerous resources online that will help. Using an online tool for testing regular expressions like [regex101.com](https://regex101.com/) will help a lot when developing patterns. :::: **Examples** * *logstash-30-yyyy.mm.dd*: Daily index that should be deleted after 30 days, indices that don’t match the pattern will be deleted after 365 days ``` - filtertype: age source: creation_date direction: older unit: days unit_count: 365 unit_count_pattern: -([0-9]+)- ``` * *logstash-12-yyyy.mm*: Monthly index that should be deleted after 12 months, indices that don’t match the pattern will be deleted after 3 months ```yaml - filtertype: age source: creation_date direction: older unit: months unit_count: 3 unit_count_pattern: -([0-9]+)- ``` * *_3_logstash-yyyy.mm.dd*: Daily index that should be deleted after 3 years, indices that don’t match the pattern will be ignored ```yaml - filtertype: age source: creation_date direction: older unit: years unit_count: -1 unit_count_pattern: ^_([0-9]+)_ ``` ::::{important} Be sure to pay attention to the interaction of this parameter and [unit_count](/reference/fe_unit_count.md)! :::: elasticsearch-curator-8.0.21/docs/reference/fe_use_age.md000066400000000000000000000011711477314666200234030ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_use_age.html --- # use_age [fe_use_age] ```yaml - filtertype: count count: 10 use_age: True source: creation_date ``` This setting allows filtering of indices by their age *after* other considerations. The default value of this setting is `False` ::::{note} Use of this setting requires the additional setting, [source](/reference/fe_source.md). :::: ::::{tip} There are context-specific examples using `use_age` in the [count](/reference/filtertype_count.md) and [space](/reference/filtertype_space.md) documentation. :::: elasticsearch-curator-8.0.21/docs/reference/fe_value.md000066400000000000000000000027621477314666200231160ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_value.html --- # value [fe_value] ::::{note} This setting is only used with the [pattern](/reference/filtertype_pattern.md) filtertype and is a required setting. There is a separate [value option](/reference/option_value.md) associated with the [allocation action](/reference/allocation.md), and the [allocated filtertype](/reference/filtertype_allocated.md). :::: The value of this setting is used by [kind](/reference/fe_kind.md) as follows: * `prefix`: Search the first part of an index name for the provided value * `suffix`: Search the last part of an index name for the provided value * `regex`: Provide your own regular expression, and Curator will find the matches. * `timestring`: An strftime string to extrapolate and find indices that match. For example, given a `timestring` of `'%Y.%m.%d'`, matching indices would include `logstash-2016.04.01` and `.marvel-2016.04.01`, but not `myindex-2016-04-01`, as the pattern is different. ::::{important} Whatever you provide for `value` is always going to be a part of a
regular expression. The safest practice is to always encapsulate within single quotes. For example: `value: '-suffix'`, or `value: 'prefix-'` :::: There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. ::::{tip} There are context-specific examples using `value` in the [kind](/reference/fe_kind.md) documentation. :::: elasticsearch-curator-8.0.21/docs/reference/fe_week_starts_on.md000066400000000000000000000014211477314666200250200ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_week_starts_on.html --- # week_starts_on [fe_week_starts_on] ::::{note} This setting is only used with the [period](/reference/filtertype_period.md) filtertype
when [period_type](/reference/fe_period_type.md) is `relative`. :::: ```yaml - filtertype: period source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday ``` The value of this setting indicates whether weeks should be measured starting on `sunday` or `monday`. Though Monday is the ISO standard, Sunday is frequently preferred. This setting is only used when [unit](/reference/fe_unit.md) is set to `weeks`. The default value for this setting is `sunday`. elasticsearch-curator-8.0.21/docs/reference/filter_elements.md000066400000000000000000000030761477314666200245100ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filter_elements.html --- # Filter Elements [filter_elements] * [aliases](/reference/fe_aliases.md) * [allocation_type](/reference/fe_allocation_type.md) * [count](/reference/fe_count.md) * [date_from](/reference/fe_date_from.md) * [date_from_format](/reference/fe_date_from_format.md) * [date_to](/reference/fe_date_to.md) * [date_to_format](/reference/fe_date_to_format.md) * [direction](/reference/fe_direction.md) * [disk_space](/reference/fe_disk_space.md) * [epoch](/reference/fe_epoch.md) * [exclude](/reference/fe_exclude.md) * [field](/reference/fe_field.md) * [intersect](/reference/fe_intersect.md) * [key](/reference/fe_key.md) * [kind](/reference/fe_kind.md) * [max_num_segments](/reference/fe_max_num_segments.md) * [pattern](/reference/fe_pattern.md) * [period_type](/reference/fe_period_type.md) * [range_from](/reference/fe_range_from.md) * [range_to](/reference/fe_range_to.md) * [reverse](/reference/fe_reverse.md) * [source](/reference/fe_source.md) * [state](/reference/fe_state.md) * [stats_result](/reference/fe_stats_result.md) * [timestring](/reference/fe_timestring.md) * [threshold_behavior](/reference/fe_threshold_behavior.md) * [unit](/reference/fe_unit.md) * [unit_count](/reference/fe_unit_count.md) * [unit_count_pattern](/reference/fe_unit_count_pattern.md) * [use_age](/reference/fe_use_age.md) * [value](/reference/fe_value.md) * [week_starts_on](/reference/fe_week_starts_on.md) You can use [environment variables](/reference/envvars.md) in your configuration files. elasticsearch-curator-8.0.21/docs/reference/filters.md000066400000000000000000000037721477314666200230020ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filters.html --- # Filters [filters] Filters are the way to select only the indices (or snapshots) you want. ::::{admonition} Filter chaining :class: note It is important to note that while filters can be chained, each is linked by an implied logical **AND** operation. If you want to match from one of several different patterns, as with a logical **OR** operation, you can do so with the [pattern](/reference/filtertype_pattern.md) filtertype using *regex* as the [kind](/reference/fe_kind.md). This example shows how to select multiple indices based on them beginning with either `alpha-`, `bravo-`, or `charlie-`: ```yaml filters: - filtertype: pattern kind: regex value: '^(alpha-|bravo-|charlie-).*$' ``` Explaining all of the different ways in which regular expressions can be used is outside the scope of this document, but hopefully this gives you some idea of how a regular expression pattern can be used when a logical **OR** is desired. :::: The index filtertypes are: * [age](/reference/filtertype_age.md) * [alias](/reference/filtertype_alias.md) * [allocated](/reference/filtertype_allocated.md) * [closed](/reference/filtertype_closed.md) * [count](/reference/filtertype_count.md) * [empty](/reference/filtertype_empty.md) * [forcemerged](/reference/filtertype_forcemerged.md) * [kibana](/reference/filtertype_kibana.md) * [none](/reference/filtertype_none.md) * [opened](/reference/filtertype_opened.md) * [pattern](/reference/filtertype_pattern.md) * [period](/reference/filtertype_period.md) * [space](/reference/filtertype_space.md) The snapshot filtertypes are: * [age](/reference/filtertype_age.md) * [count](/reference/filtertype_count.md) * [none](/reference/filtertype_none.md) * [pattern](/reference/filtertype_pattern.md) * [period](/reference/filtertype_period.md) * [state](/reference/filtertype_state.md) You can use [environment variables](/reference/envvars.md) in your configuration files. elasticsearch-curator-8.0.21/docs/reference/filtertype.md000066400000000000000000000031401477314666200235060ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype.html --- # filtertype [filtertype] Each filter is defined first by a `filtertype`. Each filtertype has its own settings, or no settings at all. In a configuration file, filters are defined as follows: ```yaml - filtertype: *first* setting1: ... ... settingN: ... - filtertype: *second* setting1: ... ... settingN: ... - filtertype: *third* ``` The `-` indicates in the YAML that this is an array element. Each filtertype declaration must be preceded by a `-` for the filters to be read properly. This is how Curator can chain filters together. Anywhere filters can be used, multiple can be chained together in this manner. The index filtertypes are: * [age](/reference/filtertype_age.md) * [alias](/reference/filtertype_alias.md) * [allocated](/reference/filtertype_allocated.md) * [closed](/reference/filtertype_closed.md) * [count](/reference/filtertype_count.md) * [empty](/reference/filtertype_empty.md) * [forcemerged](/reference/filtertype_forcemerged.md) * [kibana](/reference/filtertype_kibana.md) * [none](/reference/filtertype_none.md) * [opened](/reference/filtertype_opened.md) * [pattern](/reference/filtertype_pattern.md) * [period](/reference/filtertype_period.md) * [space](/reference/filtertype_space.md) The snapshot filtertypes are: * [age](/reference/filtertype_age.md) * [count](/reference/filtertype_count.md) * [none](/reference/filtertype_none.md) * [pattern](/reference/filtertype_pattern.md) * [period](/reference/filtertype_period.md) * [state](/reference/filtertype_state.md) elasticsearch-curator-8.0.21/docs/reference/filtertype_age.md000066400000000000000000000154601477314666200243320ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_age.html --- # age [filtertype_age] ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices based on their age. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Age calculation [_age_calculation] [`units`](/reference/fe_unit.md) are calculated as follows: | Unit | Seconds | Note | | --- | --- | --- | | `seconds` | `1` | One second | | `minutes` | `60` | Calculated as 60 seconds | | `hours` | `3600` | Calculated as 60 minutes (60*60) | | `days` | `86400` | Calculated as 24 hours (24*60*60) | | `weeks` | `604800` | Calculated as 7 days (7*24*60*60) | | `months` | `2592000` | Calculated as 30 days (30*24*60*60) | | `years` | `31536000` | Calculated as 365 days (365*24*60*60) | All calculations are in epoch time, which is the number of seconds elapsed since 1 Jan 1970. If no [`epoch`](/reference/fe_epoch.md) is specified in the filter, then the current epoch time-which is always UTC-is used as the basis for comparison. As epoch time is always increasing, lower numbers indicate dates and times in the past. When age is calculated, [`unit`](/reference/fe_unit.md) is multiplied by [`unit_count`](/reference/fe_unit_count.md) to obtain a total number of seconds to use as a differential. For example, if the time at execution were 2017-04-07T15:00:00Z (UTC), then the epoch timestamp would be `1491577200`. If I had an age filter defined like this: ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` The time differential would be `3*24*60*60` seconds, which is `259200` seconds. Subtracting this value from `1491577200` gives us `1491318000`, which is 2017-04-04T15:00:00Z (UTC), exactly 3 days in the past. The `creation_date` of indices or snapshots is compared to this timestamp. If it is `older`, it stays in the actionable list, otherwise it is removed from the actionable list. ::::{admonition} `age` filter vs. `period` filter :class: important The time differential means of calculation can lead to frustration. Setting `unit` to `months`, and `unit_count` to `3` will actually calculate the age as `3*30*24*60*60`, which is `7776000` seconds. This may be a big deal. If the date is 2017-01-01T02:30:00Z, or `1483237800` in epoch time. Subtracting `7776000` seconds makes `1475461800`, which is 2016-10-03T02:30:00Z. If you were to try to match monthly indices, `index-2016.12`, `index-2016.11`, `2016.10`, `2016.09`, etc., then both `index-2016.09` *and* `index-2016.10` will be *older* than the cutoff date. This may result in unintended behavior. Another way this can cause issues is with weeks. Weekly indices may start on Sunday or Monday. The age filter’s calculation doesn’t take this into consideration, and merely tests the difference between execution time and the timestamp on the index (from any `source`). Another means of selecting indices and snapshots is the [period](/reference/filtertype_period.md) filter, which is perhaps a better choice for selecting weeks and months as it compensates for these differences. :::: ## `name`-based ages [_name_based_ages] Using `name` as the `source` tells Curator to look for a [`timestring`](/reference/fe_timestring.md) within the index or snapshot name, and convert that into an epoch timestamp (epoch implies UTC). ```yaml - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 3 ``` ::::{admonition} A word about regular expression matching with timestrings :class: warning Timestrings are parsed from strftime patterns, like `%Y.%m.%d`, into regular expressions. For example, `%Y` is 4 digits, so the regular expression for that looks like `\d{{4}}`, and `%m` is 2 digits, so the regular expression is `\d{{2}}`. What this means is that a simple timestring to match year and month, `%Y.%m` will result in a regular expression like this: `^.*\d{{4}}\.\d{{2}}.*$`. This pattern will match any 4 digits, followed by a period `.`, followed by 2 digits, occurring anywhere in the index name. This means it *will* match monthly indices, like `index-2016.12`, as well as daily indices, like `index-2017.04.01`, which may not be the intended behavior. To compensate for this, when selecting indices matching a subset of another pattern, use a second filter with `exclude` set to `True` ```yaml - filtertype: pattern kind: timestring value: '%Y.%m' - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` This will prevent the `%Y.%m` pattern from matching the `%Y.%m` part of the daily indices. **This applies whether using `timestring` as a mere pattern match, or as part of date calculations.** :::: ## `creation_date`-based ages [_creation_date_based_ages] `creation_date` extracts the epoch time of index or snapshot creation. ```yaml - filtertype: age source: creation_date direction: older unit: days unit_count: 3 ``` ## `field_stats`-based ages [_field_stats_based_ages] ::::{note} `source` can only be `field_stats` when filtering indices. :::: In Curator 5.3 and older, source `field_stats` uses the [Field Stats API](http://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.md) to calculate either the `min_value` or the `max_value` of the [`field`](/reference/fe_field.md) as the [`stats_result`](/reference/fe_stats_result.md), and then use that value for age comparisons. In 5.4 and above, even though it is still called `field_stats`, it uses an aggregation to calculate the same values, as the `field_stats` API is no longer used in Elasticsearch 6.x and up. [`field`](/reference/fe_field.md) must be of type `date` in Elasticsearch. ```yaml - filtertype: age source: field_stats direction: older unit: days unit_count: 3 field: '@timestamp' stats_result: min_value ``` ## Required settings [_required_settings_13] * [source](/reference/fe_source.md) * [direction](/reference/fe_direction.md) * [unit](/reference/fe_unit.md) * [unit_count](/reference/fe_unit_count.md) ## Dependent settings [_dependent_settings] * [timestring](/reference/fe_timestring.md) (required if `source` is `name`) * [field](/reference/fe_field.md) (required if `source` is `field_stats`) [Indices only] * [stats_result](/reference/fe_stats_result.md) (only used if `source` is `field_stats`) [Indices only] ## Optional settings [_optional_settings_18] * [unit_count_pattern](/reference/fe_unit_count_pattern.md) * [epoch](/reference/fe_epoch.md) * [exclude](/reference/fe_exclude.md) (default is `False`) elasticsearch-curator-8.0.21/docs/reference/filtertype_alias.md000066400000000000000000000027751477314666200246740ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_alias.html --- # alias [filtertype_alias] ```yaml - filtertype: alias aliases: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices based on whether they are associated with the given [aliases](/reference/fe_aliases.md), which can be a single value, or an array. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ::::{admonition} Matching Indices and Aliases :class: important [Indices must be in all aliases to match](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/breaking-changes-5.5.html#breaking_55_rest_changes). If a list of [aliases](/reference/fe_aliases.md) is provided (instead of only one), indices must appear in *all* listed [aliases](/reference/fe_aliases.md) or a 404 error will result, leading to no indices being matched. In older versions, if the index was associated with even one of the aliases in [aliases](/reference/fe_aliases.md), it would result in a match. :::: ## Required settings [_required_settings_14] * [aliases](/reference/fe_aliases.md) ## Optional settings [_optional_settings_19] * [exclude](/reference/fe_exclude.md) elasticsearch-curator-8.0.21/docs/reference/filtertype_allocated.md000066400000000000000000000023151477314666200255210ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_allocated.html --- # allocated [filtertype_allocated] ```yaml - filtertype: allocated key: ... value: ... allocation_type: exclude: True ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices based on their shard routing allocation settings. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). By default the indices matched by the `allocated` filter will be excluded since the `exclude` setting defaults to `True`. To include matching indices rather than exclude, set the `exclude` setting to `False`. ## Required settings [_required_settings_15] * [key](/reference/fe_key.md) * [value](/reference/fe_value.md) ## Optional settings [_optional_settings_20] * [allocation_type](/reference/fe_allocation_type.md) * [exclude](/reference/fe_exclude.md) (default is `True`) elasticsearch-curator-8.0.21/docs/reference/filtertype_closed.md000066400000000000000000000010401477314666200250340ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_closed.html --- # closed [filtertype_closed] ```yaml - filtertype: closed exclude: True ``` This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices which are closed. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Optional settings [_optional_settings_21] * [exclude](/reference/fe_exclude.md) (default is `True`) elasticsearch-curator-8.0.21/docs/reference/filtertype_count.md000066400000000000000000000123321477314666200247210ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_count.html --- # count [filtertype_count] ```yaml - filtertype: count count: 10 ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list of indices *or* snapshots. They are ordered by age, or by alphabet, so as to guarantee that the correct items will remain in, or be removed from the actionable list based on the values of [count](/reference/fe_count.md), [exclude](/reference/fe_exclude.md), and [reverse](/reference/fe_reverse.md). ## Age-based sorting [_age_based_sorting] For use cases where "like" items are being counted, and their name pattern guarantees date sorting is equal to alphabetical sorting, it is unnecessary to set [use_age](/reference/fe_use_age.md) to `True`, as item names will be sorted in [reverse](/reference/fe_reverse.md) order by default. This means that the item count will start beginning with the *newest* indices or snapshots, and proceed through to the oldest. Where this is not the case, the [`use_age`](/reference/fe_use_age.md) setting can be used to ensure that index or snapshot ages are properly considered for sorting: ```yaml - filtertype: count count: 10 use_age: True source: creation_date ``` All of the age-related settings from the [`age`](/reference/filtertype_age.md) filter are supported, and the same restrictions apply with regard to filtering indices vs. snapshots. ## Pattern-based sorting [_pattern_based_sorting] ```yaml - filtertype: count count: 1 pattern: '^(.*)-\d{6}$' reverse: true ``` This particular example will match indices following the basic rollover pattern of `indexname-######`, and keep the highest numbered index for each group. For example, given indices `a-000001`, `a-000002`, `a-000003` and `b-000006`, and `b-000007`, the indices will would be matched are `a-000003` and `b-000007`. Indices that do not match the regular expression in `pattern` will be automatically excluded. This is particularly useful with indices created and managed using the [Rollover API](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-rollover-index.md), as you can select only the active indices with the above example ([`exclude`](/reference/fe_exclude.md) defaults to `False`). Setting [`exclude`](/reference/fe_exclude.md) to `True` with the above example will *remove* the active rollover indices, leaving only those which have been rolled-over. While this is perhaps most useful for the aforementioned scenario, it can also be used with age-based indices as well. ## Reversing sorting [_reversing_sorting] Using the default configuration, [`reverse`](/reference/fe_reverse.md) is `True`. Given These indices: ```sh index1 index2 index3 index4 index5 ``` And this filter: ```yaml - filtertype: count count: 2 ``` Indices `index5` and `index4` will be recognized as the `2` *most recent,* and will be removed from the actionable list, leaving `index1`, `index2`, and `index3` to be acted on by the given [action](/reference/actions.md). Similarly, given these indices: ```sh index-2017.03.01 index-2017.03.02 index-2017.03.03 index-2017.03.04 index-2017.03.05 ``` And this filter: ```yaml - filtertype: count count: 2 use_age: True source: name timestring: '%Y.%m.%d' ``` The result will be similar. Indices `index-2017.03.05` and `index-2017.03.04` will be recognized as the `2` *most recent,* and will be removed from the actionable list, leaving `index-2017.03.01`, `index-2017.03.02`, and `index-2017.03.03` to be acted on by the given [action](/reference/actions.md). In some cases, you may wish to filter for the most recent indices. To accomplish this you set [`reverse`](/reference/fe_reverse.md) to `False`: ```yaml - filtertype: count count: 2 reverse: False ``` This time indices `index1` and `index2` will be the `2` which will be removed from the actionable list, leaving `index3`, `index4`, and `index5` to be acted on by the given [action](/reference/actions.md). Likewise with the age sorted indices: ```yaml - filtertype: count count: 2 use_age: True source: name timestring: '%Y.%m.%d' reverse: True ``` Indices `index-2017.03.01` and `index-2017.03.02` will be the `2` which will be removed from the actionable list, leaving `index-2017.03.03`, `index-2017.03.04`, and `index-2017.03.05` to be acted on by the given [action](/reference/actions.md). ## Required settings [_required_settings_16] * [count](/reference/fe_count.md) ## Optional settings [_optional_settings_22] * [reverse](/reference/fe_reverse.md) * [use_age](/reference/fe_use_age.md) * [pattern](/reference/fe_pattern.md) * [source](/reference/fe_source.md) (required if `use_age` is `True`) * [timestring](/reference/fe_timestring.md) (required if `source` is `name`) * [exclude](/reference/fe_exclude.md) (default is `True`) ## Index-only settings [_index_only_settings] * [field](/reference/fe_field.md) (required if `source` is `field_stats`) * [stats_result](/reference/fe_stats_result.md) (only used if `source` is `field_stats`) elasticsearch-curator-8.0.21/docs/reference/filtertype_empty.md000066400000000000000000000014701477314666200247300ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_empty.html --- # empty [filtertype_empty] ```yaml - filtertype: empty exclude: False ``` This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices which do not contain any documents. Indices that are closed are automatically removed from consideration. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). By default the indices matched by the empty filter will be excluded since the exclude setting defaults to True. To include matching indices rather than exclude, set the exclude setting to False. ## Optional settings [_optional_settings_23] * [exclude](/reference/fe_exclude.md) (default is `True`) elasticsearch-curator-8.0.21/docs/reference/filtertype_forcemerged.md000066400000000000000000000016711477314666200260570ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_forcemerged.html --- # forcemerged [filtertype_forcemerged] ```yaml - filtertype: forcemerged max_num_segments: 2 exclude: True ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices which have `max_num_segments` segments per shard, or fewer. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Required settings [_required_settings_17] * [max_num_segments](/reference/fe_max_num_segments.md) ## Optional settings [_optional_settings_24] * [exclude](/reference/fe_exclude.md) (default is `True`) elasticsearch-curator-8.0.21/docs/reference/filtertype_kibana.md000066400000000000000000000013251477314666200250160ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_kibana.html --- # kibana [filtertype_kibana] ```yaml - filtertype: kibana exclude: True ``` This [filtertype](/reference/filtertype.md) will remove any index matching the regular expression `^\.kibana.*$` from the list of indices, if present. This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices matching the regular expression `^\.kibana.*$`. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Optional settings [_optional_settings_25] * [exclude](/reference/fe_exclude.md) (default is `True`) elasticsearch-curator-8.0.21/docs/reference/filtertype_none.md000066400000000000000000000005571477314666200245360ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_none.html --- # none [filtertype_none] ```yaml - filtertype: none ``` This [filtertype](/reference/filtertype.md) will not filter anything, returning the full list of indices or snapshots. There are no settings for this [filtertype](/reference/filtertype.md). elasticsearch-curator-8.0.21/docs/reference/filtertype_opened.md000066400000000000000000000010401477314666200250350ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_opened.html --- # opened [filtertype_opened] ```yaml - filtertype: opened exclude: True ``` This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices which are opened. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Optional settings [_optional_settings_26] * [exclude](/reference/fe_exclude.md) (default is `True`) elasticsearch-curator-8.0.21/docs/reference/filtertype_pattern.md000066400000000000000000000113231477314666200252450ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_pattern.html --- # pattern [filtertype_pattern] ```yaml - filtertype: pattern kind: ... value: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices matching a given pattern. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ::::{admonition} Filter chaining :class: note It is important to note that while filters can be chained, each is linked by an implied logical **AND** operation. If you want to match from one of several different patterns, as with a logical **OR** operation, you can do so with the pattern filtertype using *regex* as the [kind](/reference/fe_kind.md). This example shows how to select multiple indices based on them beginning with either `alpha-`, `bravo-`, or `charlie-`: ```yaml filters: - filtertype: pattern kind: regex value: '^(alpha-|bravo-|charlie-).*$' ``` Explaining all of the different ways in which regular expressions can be used is outside the scope of this document, but hopefully this gives you some idea of how a regular expression pattern can be used when a logical **OR** is desired. :::: The different [`kinds`](/reference/fe_kind.md) are described as follows: ## prefix [_prefix] To match all indices starting with `logstash-`: ```yaml - filtertype: pattern kind: prefix value: logstash- ``` To match all indices *except* those starting with `logstash-`: ```yaml - filtertype: pattern kind: prefix value: logstash- exclude: True ``` ::::{note} Internally, the `prefix` value is used to create a *regex* pattern: `^{{0}}.*$`. Any special characters should be escaped with a backslash to match literally. :::: ## suffix [_suffix] To match all indices ending with `-prod`: ```yaml - filtertype: pattern kind: suffix value: -prod ``` To match all indices *except* those ending with `-prod`: ```yaml - filtertype: pattern kind: suffix value: -prod exclude: True ``` ::::{note} Internally, the `suffix` value is used to create a *regex* pattern: `^.*{{0}}$`. Any special characters should be escaped with a backslash to match literally. :::: ## timestring [_timestring] ::::{important} No age calculation takes place here. It is strictly a pattern match. :::: To match all indices with a Year.month.day pattern, like `index-2017.04.01`: ```yaml - filtertype: pattern kind: timestring value: '%Y.%m.%d' ``` To match all indices *except* those with a Year.month.day pattern, like `index-2017.04.01`: ```yaml - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` ::::{admonition} A word about regular expression matching with timestrings :class: warning Timestrings are parsed from strftime patterns, like `%Y.%m.%d`, into regular expressions. For example, `%Y` is 4 digits, so the regular expression for that looks like `\d{{4}}`, and `%m` is 2 digits, so the regular expression is `\d{{2}}`. What this means is that a simple timestring to match year and month, `%Y.%m` will result in a regular expression like this: `^.*\d{{4}}\.\d{{2}}.*$`. This pattern will match any 4 digits, followed by a period `.`, followed by 2 digits, occurring anywhere in the index name. This means it *will* match monthly indices, like `index-2016.12`, as well as daily indices, like `index-2017.04.01`, which may not be the intended behavior. To compensate for this, when selecting indices matching a subset of another pattern, use a second filter with `exclude` set to `True` ```yaml - filtertype: pattern kind: timestring value: '%Y.%m' - filtertype: pattern kind: timestring value: '%Y.%m.%d' exclude: True ``` This will prevent the `%Y.%m` pattern from matching the `%Y.%m` part of the daily indices. **This applies whether using `timestring` as a mere pattern match, or as part of date calculations.** :::: ## regex [_regex] This [`kind`](/reference/fe_kind.md) allows you to design a regular-expression to match indices or snapshots: To match all indices starting with `a-`, `b-`, or `c-`: ```yaml - filtertype: pattern kind: regex value: '^a-|^b-|^c-' ``` To match all indices *except* those starting with `a-`, `b-`, or `c-`: ```yaml - filtertype: pattern kind: regex value: '^a-|^b-|^c-' exclude: True ``` ## Required settings [_required_settings_18] * [kind](/reference/fe_kind.md) * [value](/reference/fe_value.md) ## Optional settings [_optional_settings_27] * [exclude](/reference/fe_exclude.md) (default is `False`) elasticsearch-curator-8.0.21/docs/reference/filtertype_period.md000066400000000000000000000171641477314666200250630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_period.html --- # period [filtertype_period] This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices or snapshots based on whether they fit within the given time range. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ```yaml - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: sunday ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: ## Relative Periods [_relative_periods] A relative period will be reckoned relative to execution time, unless an [epoch](/reference/fe_epoch.md) timestamp is provided. Reckoning is truncated to the most recent whole unit, where a [unit](/reference/fe_unit.md) can be one of `hours`, `days`, `weeks`, `months`, or `years`. For example, if I selected `hours` as my `unit`, and I began execution at 02:35, then the point of reckoning would be 02:00. This is relatively easy with `days`, `months`, and `years`, but slightly more complicated with `weeks`. Some users may wish to reckon weeks by the ISO standard, which starts weeks on Monday. Others may wish to use Sunday as the first day of the week. Both are acceptable options with the `period` filter. The default behavior for `weeks` is to have Sunday be the start of the week. This can be overridden with [week_starts_on](/reference/fe_week_starts_on.md) as follows: ```yaml - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: weeks week_starts_on: monday ``` [range_from](/reference/fe_range_from.md) and [range_to](/reference/fe_range_to.md) are counters of whole [units](/reference/fe_unit.md). A negative number indicates a whole unit in the past, while a positive number indicates a whole unit in the future. A `0` indicates the present unit. With such a timeline mentality, it is relatively easy to create a date range that will meet your needs. If the time of execution time is **2017-04-03T13:45:23.831**, this table will help you figure out what the previous whole unit, current unit, and next whole unit will be, in ISO8601 format. | unit | -1 | 0 | +1 | | --- | --- | --- | --- | | hours | 2017-04-03T12:00:00 | 2017-04-03T13:00:00 | 2017-04-03T14:00:00 | | days | 2017-04-02T00:00:00 | 2017-04-03T00:00:00 | 2017-04-04T00:00:00 | | weeks sun | 2017-03-26T00:00:00 | 2017-04-02T00:00:00 | 2017-04-09T00:00:00 | | weeks mon | 2017-03-27T00:00:00 | 2017-04-03T00:00:00 | 2017-04-10T00:00:00 | | months | 2017-03-01T00:00:00 | 2017-04-01T00:00:00 | 2017-05-01T00:00:00 | | years | 2016-01-01T00:00:00 | 2017-01-01T00:00:00 | 2018-01-01T00:00:00 | Ranges must be from older dates to newer dates, or smaller numbers (including negative numbers) to larger numbers or Curator will return an exception. An example `period` filter demonstrating how to select all daily indices by timestring found in the index name from last month might look like this: ```yaml - filtertype: period period_type: relative source: name range_from: -1 range_to: -1 timestring: '%Y.%m.%d' unit: months ``` Having `range_from` and `range_to` both be the same value will mean that only that whole unit will be selected, in this case, a month. ::::{important} `range_from` and `range_to` are required for the `relative` pattern type. :::: ## Absolute Periods [_absolute_periods] ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d' unit: months date_from: 2017.01 date_from_format: '%Y.%m' date_to: 2017.01 date_to_format: '%Y.%m' ``` In addition to relative periods, you can define absolute periods. These are defined as the period between the [`date_from`](/reference/fe_date_from.md) and the [`date_to`](/reference/fe_date_to.md). For example, if `date_from` and `date_to` are both `2017.01`, and [`unit`](/reference/fe_unit.md) is `months`, all indices with a `name`, `creation_date`, or `stats_result` (depending on the value of [`source`](/reference/fe_source.md)) within the month of January 2017 will match. The `date_from` is used to establish the beginning of the time period, regardless of whether `date_from_format` is in hours, and the indices you are trying to filter are in weeks or months. The format and date of `date_from` will simply set the beginning of the time period. The `date_to`, `date_to_format`, and `unit` work in conjunction to select the end date. For example, if my `date_to` were `2017.04`, and `date_to_format` is `%Y.%m` to properly parse that date, it would follow that `unit` would be `months`. ::::{important} The period filter is smart enough to calculate `months` and `years` properly. **If `unit` is not `months` or `years`,** then your date range will be `unit` seconds more than the beginning of the `date_from` date, minus 1 second, according to this table: [`units`](/reference/fe_unit.md) are calculated as follows: | Unit | Seconds | Note | | --- | --- | --- | | `seconds` | `1` | One second | | `minutes` | `60` | Calculated as 60 seconds | | `hours` | `3600` | Calculated as 60 minutes (60*60) | | `days` | `86400` | Calculated as 24 hours (24*60*60) | | `weeks` | `604800` | Calculated as 7 days (7*24*60*60) | | `months` | `2592000` | Calculated as 30 days (30*24*60*60) | | `years` | `31536000` | Calculated as 365 days (365*24*60*60) | :::: Specific date ranges can be captured with up to whole second resolution: ```yaml - filtertype: period period_type: absolute source: name timestring: '%Y.%m.%d.%H' unit: seconds date_from: 2017-01-01T00:00:00 date_from_format: '%Y-%m-%dT%H:%M:%S' date_to: 2017-01-01T12:34:56 date_to_format: '%Y-%m-%dT%H:%M:%S' ``` This example will capture indices with an hourly timestamp in their name that fit between the `date_from` and `date_to` timestamps. The identifiers that Curator currently recognizes include: | Unit | Value | Note | | --- | --- | --- | | `%Y` | 4 digit year | | | `%G` | 4 digit year | use instead of `%Y` when doing ISO Week calculations | | `%y` | 2 digit year | | | `%m` | 2 digit month | | | `%W` | 2 digit week of the year | | | `%V` | 2 digit week of the year | use instead of `%W` when doing ISO Week calculations | | `%d` | 2 digit day of the month | | | `%H` | 2 digit hour | 24 hour notation | | `%M` | 2 digit minute | | | `%S` | 2 digit second | | | `%j` | 3 digit day of the year | | ## Required settings [_required_settings_19] * [source](/reference/fe_source.md) * [unit](/reference/fe_unit.md) ## Dependent settings [_dependent_settings_2] * [range_from](/reference/fe_range_from.md) * [range_to](/reference/fe_range_to.md) * [date_from](/reference/fe_date_from.md) * [date_to](/reference/fe_date_to.md) * [date_from_format](/reference/fe_date_from_format.md) * [date_to_format](/reference/fe_date_to_format.md) * [timestring](/reference/fe_timestring.md) (required if `source` is `name`) * [field](/reference/fe_field.md) (required if `source` is `field_stats`) [Indices only] * [stats_result](/reference/fe_stats_result.md) (only used if `source` is `field_stats`) [Indices only] * [intersect](/reference/fe_intersect.md) (optional if `source` is `field_stats`) [Indices only] ## Optional settings [_optional_settings_28] * [epoch](/reference/fe_epoch.md) * [exclude](/reference/fe_exclude.md) (default is `False`) * [week_starts_on](/reference/fe_week_starts_on.md) elasticsearch-curator-8.0.21/docs/reference/filtertype_space.md000066400000000000000000000160151477314666200246660ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_space.html --- # space [filtertype_space] This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match indices when their cumulative disk consumption is `greater_than` (default) or `less_than` than [disk_space](/reference/fe_disk_space.md) gigabytes. They are first ordered by age, or by alphabet, so as to guarantee the oldest indices are deleted first. They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Deleting Indices By Space [_deleting_indices_by_space] ```yaml - filtertype: space disk_space: 100 ``` This [filtertype](/reference/filtertype.md) is for those who want to retain indices based on disk consumption, rather than by a set number of days. There are some important caveats regarding this choice: * Elasticsearch cannot calculate the size of closed indices. Elasticsearch does not keep tabs on how much disk-space closed indices consume. If you close indices, your space calculations will be inaccurate. * Indices consume resources just by existing. You could run into performance and/or operational snags in Elasticsearch as the count of indices climbs. * You need to manually calculate how much space across all nodes. The total you give will be the sum of all space consumed across all nodes in your cluster. If you use shard allocation to put more shards or indices on a single node, it will not affect the total space reported by the cluster, but you may still run out of space on that node. These are only a few of the caveats. This is still a valid use-case, especially for those running a single-node test box. For use cases where "like" indices are being counted, and their name pattern guarantees date sorting is equal to alphabetical sorting, it is unnecessary to set [use_age](/reference/fe_use_age.md) to `True`, as index names will be sorted in [reverse](/reference/fe_reverse.md) order by default. For this case, this means that disk space calculations will start beginning with the *newest* indices, and proceeding through to the oldest. ## Age-based sorting [_age_based_sorting_2] ```yaml - filtertype: space disk_space: 100 use_age: True source: creation_date ``` For use cases where "like" indices are being counted, and their name pattern guarantees date sorting is equal to alphabetical sorting, it is unnecessary to set [use_age](/reference/fe_use_age.md) to `True`, as index names will be sorted in [reverse](/reference/fe_reverse.md) order by default. For this case, this means that disk space calculations will start beginning with the *newest* indices, and proceeding through to the oldest. Where this is not the case, the [`use_age`](/reference/fe_use_age.md) setting can be used to ensure that index or snapshot ages are properly considered for sorting: All of the age-related settings from the [`age`](/reference/filtertype_age.md) filter are supported. ## Reversing sorting [_reversing_sorting_2] ::::{important} The [`reverse`](/reference/fe_reverse.md) setting is ignored when [`use_age`](/reference/fe_use_age.md) is `True`. When [`use_age`](/reference/fe_use_age.md) is `True`, sorting is *always* from newest to oldest, ensuring that old indices are always selected first. :::: Using the default configuration, [`reverse`](/reference/fe_reverse.md) is `True`. Given These indices: ```sh index1 10g index2 10g index3 10g index4 10g index5 10g ``` And this filter: ```yaml - filtertype: space disk_space: 21 ``` The indices will be sorted alphabetically and iterated over in the indicated order (the value of [`reverse`](/reference/fe_reverse.md)) and the total disk space compared after adding the size of each successive index. In this example, that means that `index5` will be added first, and the running total of consumed disk space will be `10g`. Since `10g` is less than the indicated threshold of `21`, `index5` will be removed from the actionable list. On the next iteration, the amount of space consumed by `index4` will be added, which brings the running total to `20g`, which is still less than the `21` threshold, so `index4` is also removed from the actionable list. This process changes when the iteration adds the disk space consumed by `index3`. Now the running total crosses the `21` threshold established by [`disk_space`](/reference/fe_disk_space.md) (the running total is now `30g`). Even though it is only `1g` in excess of the total, `index3` will remain in the actionable list. The threshold is absolute. The remaining indices, `index2` and `index1` will also be in excess of the threshold, so they will also remain in the actionable list. So in this example `index1`, `index2`, and `index3` will be acted on by the [action](/reference/actions.md) for this block. If you were to run this with [loglevel](/reference/configfile.md#loglevel) set to `DEBUG`, you might see messages like these in the output: ```sh ...Removed from actionable list: index5, summed disk usage is 10GB and disk limit is 21.0GB. ...Removed from actionable list: index4, summed disk usage is 20GB and disk limit is 21.0GB. ...Remains in actionable list: index3, summed disk usage is 30GB and disk limit is 21.0GB. ...Remains in actionable list: index2, summed disk usage is 40GB and disk limit is 21.0GB. ...Remains in actionable list: index1, summed disk usage is 50GB and disk limit is 21.0GB. ``` In some cases, you may wish to filter in the reverse order. To accomplish this, you set [`reverse`](/reference/fe_reverse.md) to `False`: ```yaml - filtertype: space disk_space: 21 reverse: False ``` This time indices `index1` and `index2` will be the ones removed from the actionable list, leaving `index3`, `index4`, and `index5` to be acted on by the given [action](/reference/actions.md). If you were to run this with [loglevel](/reference/configfile.md#loglevel) set to `DEBUG`, you might see messages like these in the output: ```sh ...Removed from actionable list: index1, summed disk usage is 10GB and disk limit is 21.0GB. ...Removed from actionable list: index2, summed disk usage is 20GB and disk limit is 21.0GB. ...Remains in actionable list: index3, summed disk usage is 30GB and disk limit is 21.0GB. ...Remains in actionable list: index4, summed disk usage is 40GB and disk limit is 21.0GB. ...Remains in actionable list: index5, summed disk usage is 50GB and disk limit is 21.0GB. ``` ## Required settings [_required_settings_20] * [disk_space](/reference/fe_disk_space.md) ## Optional settings [_optional_settings_29] * [reverse](/reference/fe_reverse.md) * [use_age](/reference/fe_use_age.md) * [source](/reference/fe_source.md) (required if `use_age` is `True`) * [timestring](/reference/fe_timestring.md) (required if `source` is `name`) * [threshold_behavior](/reference/fe_threshold_behavior.md) (default is `greater_than`) * [field](/reference/fe_field.md) (required if `source` is `field_stats`) * [stats_result](/reference/fe_stats_result.md) (only used if `source` is `field_stats`) * [exclude](/reference/fe_exclude.md) (default is `False`) elasticsearch-curator-8.0.21/docs/reference/filtertype_state.md000066400000000000000000000015641477314666200247160ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/filtertype_state.html --- # state [filtertype_state] ```yaml - filtertype: state state: SUCCESS ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given [filtertype](/reference/filtertype.md), it may generate an error. :::: This [filtertype](/reference/filtertype.md) will iterate over the actionable list and match snapshots based on the value of [state](/reference/fe_state.md). They will remain in, or be removed from the actionable list based on the value of [exclude](/reference/fe_exclude.md). ## Required settings [_required_settings_21] * [state](/reference/fe_state.md) ## Optional settings [_optional_settings_30] * [exclude](/reference/fe_exclude.md) (default is `False`) elasticsearch-curator-8.0.21/docs/reference/forcemerge.md000066400000000000000000000036011477314666200234370ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/forcemerge.html --- # Forcemerge [forcemerge] ```yaml action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action performs a forceMerge on the selected indices, merging them to [max_num_segments](/reference/option_mns.md) per shard. ::::{warning} A [`forcemerge`](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-forcemerge.md#indices-forcemerge) should never be executed on an index that is actively receiving data. It should only ever be performed on indices where no more documents are ever anticipated to be added in the future. :::: You can optionally pause between each merge for [delay](/reference/option_delay.md) seconds to allow the cluster to quiesce: ```yaml action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 delay: 120 filters: - filtertype: ... ``` ## Required settings [_required_settings_6] * [max_num_segments](/reference/option_mns.md) ## Optional settings [_optional_settings_9] * [search_pattern](/reference/option_search_pattern.md) * [delay](/reference/option_delay.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_forcemerge.md). :::: elasticsearch-curator-8.0.21/docs/reference/ilm-actions.md000066400000000000000000000017431477314666200235450ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ilm-actions.html --- # ILM Actions [ilm-actions] ILM applies policy actions as indices enter time-oriented phases: * Hot * Warm * Cold * Delete The policy actions include: * [Set Priority](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-set-priority.md) * [Rollover](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-rollover.md) * [Unfollow](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-unfollow.md) * [Allocate](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-allocate.md) * [Read-Only](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-readonly.md) * [Force Merge](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-forcemerge.md) * [Shrink](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-shrink.md) * [Delete](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/ilm-delete.md) elasticsearch-curator-8.0.21/docs/reference/ilm-and-curator.md000066400000000000000000000015641477314666200243250ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ilm-and-curator.html --- # ILM and Curator! [ilm-and-curator] ::::{warning} Curator will not act on any index associated with an ILM policy without setting `allow_ilm_indices` to `true`. :::: Curator and ILM *can* coexist. However, to prevent Curator from accidentally interfering, or colliding with ILM policies, any index associated with an ILM policy name is excluded by default. This is true whether you have a Basic license or not, or whether the ILM policy is enabled or not. Curator can be configured to work with ILM-enabled indices by setting the [`allow_ilm_indices`](/reference/option_allow_ilm.md) option to `true` for any action. Learn more about Index Lifecycle Management [here](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/index-lifecycle-management.md). elasticsearch-curator-8.0.21/docs/reference/ilm-or-curator.md000066400000000000000000000034151477314666200242000ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/ilm-or-curator.html --- # ILM or Curator? [ilm-or-curator] If ILM provides the functionality to manage your index lifecycle, and you have at least a Basic license, consider using ILM in place of Curator. Many of the Stack components make use of ILM by default. ## Beats [ilm-beats] ::::{note} All Beats share a similar ILM configuration. Filebeats is used as a reference here. :::: Starting with version 7.0, Filebeat uses index lifecycle management by default when it connects to a cluster that supports lifecycle management. Filebeat loads the default policy automatically and applies it to any indices created by Filebeat. You can view and edit the policy in the Index lifecycle policies UI in Kibana. For more information about working with the UI, see [Index lifecyle policies](docs-content://manage-data/lifecycle/index-lifecycle-management.md). Read more about Filebeat and ILM in [](beats://reference/filebeat/ilm.md). ## Logstash [ilm-logstash] ::::{note} The Index Lifecycle Management feature requires version 9.3.1 or higher of the `logstash-output-elasticsearch` plugin. :::: Logstash can use [index lifecycle management](docs-content://manage-data/lifecycle/index-lifecycle-management.md) to automate the management of indices over time. The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting. By default, this will automatically detect whether the Elasticsearch instance supports ILM, and will use it if it is available. `ilm_enabled` can also be set to `true` or `false` to override the automatic detection, or disable ILM. Read more about Logstash and ILM in [](logstash-docs-md://lsr/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-ilm). elasticsearch-curator-8.0.21/docs/reference/index.md000066400000000000000000000016041477314666200224310ustar00rootroot00000000000000--- navigation_title: "Curator" mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/versions.html - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/current_release.html --- # Curator index management This section provides reference information for Elasticsearch Curator. ## Versions [versions] Elasticsearch Curator has been around for many different versions of Elasticsearch. Earlier releases of Curator supported multiple versions of Elasticsearch, but this is no longer the case. Curator is now major version locked with Elasticsearch, which means that if Curator’s major version is 8, it should support any Elasticsearch 8.x release. ### Current release [current_release] The current version of Curator {{curator_major}} is {{curator_version}}.elasticsearch-curator-8.0.21/docs/reference/index_settings.md000066400000000000000000000034011477314666200243460ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index_settings.html --- # Index Settings [index_settings] ```yaml action: index_settings description: "Change settings for selected indices" options: index_settings: index: refresh_interval: 5s ignore_unavailable: False preserve_existing: False filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action updates the specified index settings for the selected indices. ::::{important} While Elasticsearch allows for either dotted notation of index settings, such as ```json PUT /indexname/_settings { "index.blocks.read_only": true } ``` or in nested structure, like this: ```json PUT /indexname/_settings { "index": { "blocks": { "read_only": true } } } ``` In order to appropriately detect static vs. dynamic [index settings](elasticsearch://reference/elasticsearch/index-settings/index.md) and to be able to verify configurational integrity in the YAML file, **Curator does not support using dotted notation.** :::: ## Optional settings [_optional_settings_10] * [search_pattern](/reference/option_search_pattern.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) * [ignore_unavailable](/reference/option_ignore.md) * [preserve_existing](/reference/option_preserve_existing.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_index_settings.md). :::: elasticsearch-curator-8.0.21/docs/reference/installation.md000066400000000000000000000010541477314666200240220ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/installation.html --- # Installation [installation] Curator can be installed in a variety of ways, depending on what meets your needs. It is important to note that Curator only requires access to a client node in the Elasticsearch cluster to work. It does not need to be installed on one of the nodes in the cluster. * [pip](/reference/pip.md), the easiest way to use and upgrade. * [Source Code](/reference/python-source.md) * [Docker](/reference/docker.md) elasticsearch-curator-8.0.21/docs/reference/license.md000066400000000000000000000013401477314666200227410ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/license.html --- # License [license] Copyright (c) 2011-2024 Elasticsearch [http://elastic.co](http://elastic.co) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at ``` http://www.apache.org/licenses/LICENSE-2.0 ``` Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. elasticsearch-curator-8.0.21/docs/reference/open.md000066400000000000000000000016351477314666200222670ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/open.html --- # Open [open] ```yaml action: open description: "open selected indices" options: continue_if_exception: False filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action opens the selected indices. ## Optional settings [_optional_settings_11] * [search_pattern](/reference/option_search_pattern.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_open.md). :::: elasticsearch-curator-8.0.21/docs/reference/option_allocation_type.md000066400000000000000000000014571477314666200261060ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_allocation_type.html --- # allocation_type [option_allocation_type] ::::{note} This setting is used only when using the [allocation action](/reference/allocation.md) :::: ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... filters: - filtertype: ... ``` The value of this setting must be one of `require`, `include`, or `exclude`. Read more about these settings at [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shard-allocation-filtering.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/shard-allocation-filtering.md) The default value for this setting is `require`. elasticsearch-curator-8.0.21/docs/reference/option_allow_ilm.md000066400000000000000000000012321477314666200246660ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_allow_ilm.html --- # allow_ilm_indices [option_allow_ilm] This option allows Curator to manage ILM-enabled indices. Exercise caution that ILM policies and Curator actions will not interfere with each other. ::::{important} Read more about Curator and Index Lifecycle Management [here](/reference/index.md). :::: ```yaml action: delete_indices description: "Delete the specified indices" options: allow_ilm_indices: true filters: - filtertype: ... ``` The value of this setting must be either `true` or `false`. The default value for this setting is `false`. elasticsearch-curator-8.0.21/docs/reference/option_continue.md000066400000000000000000000030751477314666200245420ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_continue.html --- # continue_if_exception [option_continue] ::::{admonition} Using `ignore_empty_list` rather than `continue_if_exception` :class: important Curator has two general classifications of exceptions: Empty list exceptions, and everything else. The empty list conditions are `curator.exception.NoIndices` and `curator.exception.NoSnapshots`. The `continue_if_exception` option *only* catches conditions *other* than empty list conditions. In most cases, you will want to use `ignore_empty_list` instead of `continue_if_exception`. So why are there two kinds of exceptions? When Curator 4 was released, the ability to continue in the event of any exception was covered by the `continue_if_exception` option. However, an empty list is a *benign* condition. In fact, it’s expected with brand new clusters, or when new index patterns are added. The decision was made to split the exceptions, and have a new option catch the empty lists. See [`ignore_empty_list`](/reference/option_ignore_empty.md) for more information. :::: ::::{note} This setting is available in all actions. :::: ```yaml action: delete_indices description: "Delete selected indices" options: continue_if_exception: False filters: - filtertype: ... ``` If `continue_if_exception` is set to `True`, Curator will attempt to continue on to the next action, if any, even if an exception is encountered. Curator will log but ignore the exception that was raised. The default value for this setting is `False` elasticsearch-curator-8.0.21/docs/reference/option_copy_aliases.md000066400000000000000000000013071477314666200253650ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_copy_aliases.html --- # copy_aliases [option_copy_aliases] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Reimplement source index aliases on target index after successful shrink. options: shrink_node: DETERMINISTIC copy_aliases: True filters: - filtertype: ... ``` The default value of this setting is `False`. If `True`, after an index has been successfully shrunk, any aliases associated with the source index will be copied to the target index. elasticsearch-curator-8.0.21/docs/reference/option_count.md000066400000000000000000000011461477314666200240430ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_count.html --- # count [option_count] ::::{note} This setting is required when using the [replicas action](/reference/replicas.md). :::: ```yaml action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... filters: - filtertype: ... ``` The value for this setting is the number of replicas to assign to matching indices. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/option_delay.md000066400000000000000000000011621477314666200240070ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_delay.html --- # delay [option_delay] ::::{note} This setting is only used by the [forceMerge action](/reference/forcemerge.md), and is optional. :::: ```yaml action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 delay: 120 filters: - filtertype: ... ``` The value for this setting is the number of seconds to delay between forceMerging indices, to allow the cluster to quiesce. There is no default value. elasticsearch-curator-8.0.21/docs/reference/option_delete_after.md000066400000000000000000000012371477314666200253370ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_delete_after.html --- # delete_after [option_delete_after] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink. options: shrink_node: DETERMINISTIC delete_after: True filters: - filtertype: ... ``` The default value of this setting is `True`. After an index has been successfully shrunk, the source index will be deleted or preserved based on the value of this setting. elasticsearch-curator-8.0.21/docs/reference/option_delete_aliases.md000066400000000000000000000010161477314666200256520ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_delete_aliases.html --- # delete_aliases [option_delete_aliases] ::::{note} This setting is only used by the [close action](/reference/close.md), and is optional. :::: ```yaml action: close description: "Close selected indices" options: delete_aliases: False filters: - filtertype: ... ``` The value for this setting determines whether aliases will be deleted from indices before closing. The default value is `False`. elasticsearch-curator-8.0.21/docs/reference/option_disable.md000066400000000000000000000010541477314666200243140ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_disable.html --- # disable_action [option_disable] ::::{note} This setting is available in all actions. :::: ```yaml action: delete_indices description: "Delete selected indices" options: disable_action: False filters: - filtertype: ... ``` If `disable_action` is set to `True`, Curator will ignore the current action. This may be useful for temporarily disabling actions in a large configuration file. The default value for this setting is `False` elasticsearch-curator-8.0.21/docs/reference/option_extra_settings.md000066400000000000000000000066241477314666200257640ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_extra_settings.html --- # extra_settings [option_extra_settings] This setting should be nested YAML. The values beneath `extra_settings` will be used by whichever action uses the option. ## [alias](/reference/alias.md) [_alias/curator/docs/reference/elasticsearch/elasticsearch-client-curator/alias.md] ```yaml action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name extra_settings: filter: term: user: kimchy add: filters: - filtertype: ... remove: filters: - filtertype: ... ``` ## [create_index](/reference/create_index.md) [_create_index/curator/docs/reference/elasticsearch/elasticsearch-client-curator/create_index.md] ```yaml action: create_index description: "Create index as named" options: name: myindex # ... extra_settings: settings: number_of_shards: 1 number_of_replicas: 0 mappings: type1: properties: field1: type: string index: not_analyzed ``` ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md] See the [official Elasticsearch Documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/snapshots-restore-snapshot.md). ```yaml actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: extra_settings: index_settings: number_of_replicas: 0 wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ## [rollover](/reference/rollover.md) [_rollover/curator/docs/reference/elasticsearch/elasticsearch-client-curator/rollover.md] ```yaml action: rollover description: >- Rollover the index associated with alias 'name', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: False ``` ## [shrink](/reference/shrink.md) [_shrink/curator/docs/reference/elasticsearch/elasticsearch-client-curator/shrink.md] ::::{note} [Only `settings` and `aliases` are acceptable](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-shrink) when used in [shrink](/reference/shrink.md). :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink, then reroute the shrunk index with the provided parameters. options: shrink_node: DETERMINISTIC extra_settings: settings: index.codec: best_compression aliases: my_alias: {} filters: - filtertype: ... ``` There is no default value. elasticsearch-curator-8.0.21/docs/reference/option_ignore.md000066400000000000000000000052221477314666200241750ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_ignore.html --- # ignore_unavailable [option_ignore] ::::{note} This setting is used by the [snapshot](/reference/snapshot.md), [restore](/reference/restore.md), and [index_settings](/reference/index_settings.md) actions. :::: This setting must be either `True` or `False`. The default value of this setting is `False` ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_2] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index ignore_unavailable: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` When the `ignore_unavailable` option is `False` and an index is missing the restore request will fail. ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot ignore_unavailable: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` When the `ignore_unavailable` option is `False` and an index is missing, the snapshot request will fail. This is not frequently a concern in Curator, as it should only ever find indices that exist. ## [index_settings](/reference/index_settings.md) [_index_settings/curator/docs/reference/elasticsearch/elasticsearch-client-curator/index_settings.md] ```yaml action: index_settings description: "Change settings for selected indices" options: index_settings: index: refresh_interval: 5s ignore_unavailable: False preserve_existing: False filters: - filtertype: ... ``` When the `ignore_unavailable` option is `False` and an index is missing, or if the request is to apply a [static](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/index-modules.md#_static_index_settings) setting and the index is opened, the index setting request will fail. The `ignore_unavailable` option allows these indices to be skipped, when set to `True`. ::::{note} [Dynamic](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/index-modules.md#dynamic-index-settings) index settings can be applied to either open or closed indices. :::: elasticsearch-curator-8.0.21/docs/reference/option_ignore_empty.md000066400000000000000000000014071477314666200254140ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_ignore_empty.html --- # ignore_empty_list [option_ignore_empty] This setting must be either `True` or `False`. ```yaml action: delete_indices description: "Delete selected indices" options: ignore_empty_list: True filters: - filtertype: ... ``` Depending on your indices, and how you’ve filtered them, an empty list may be presented to the action. This results in an error condition. When the ignore_empty_list option is set to `True`, the action will exit with an INFO level log message indicating such. If ignore_empty_list is set to `False`, an ERROR level message will be logged, and Curator will exit with code 1. The default value of this setting is `False` elasticsearch-curator-8.0.21/docs/reference/option_include_aliases.md000066400000000000000000000015601477314666200260370ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_include_aliases.html --- # include_aliases [option_include_aliases] ::::{note} This setting is only used by the [restore](/reference/restore.md) action. :::: ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index include_aliases: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` This setting must be either `True` or `False`. The value of this setting determines whether Elasticsearch should include index aliases when restoring the snapshot. The default value of this setting is `False` elasticsearch-curator-8.0.21/docs/reference/option_include_gs.md000066400000000000000000000032401477314666200250240ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_include_gs.html --- # include_global_state [option_include_gs] ::::{note} This setting is used by the [snapshot](/reference/snapshot.md) and [restore](/reference/restore.md) actions. :::: This setting must be either `True` or `False`. The value of this setting determines whether Elasticsearch should include the global cluster state with the snapshot or restore. When performing a [snapshot](/reference/snapshot.md), the default value of this setting is `True`. When performing a [restore](/reference/restore.md), the default value of this setting is `False`. ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_3] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index include_global_state: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_2] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot include_global_state: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` elasticsearch-curator-8.0.21/docs/reference/option_indices.md000066400000000000000000000017371477314666200243370ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_indices.html --- # indices [option_indices] ::::{note} This setting is only used by the [restore](/reference/restore.md) action. :::: ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_4] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` This setting must be a list of indices to restore. Any valid YAML format for lists are acceptable here. If `indices` is left empty, or unset, all indices in the snapshot will be restored. The default value of this setting is an empty setting. elasticsearch-curator-8.0.21/docs/reference/option_key.md000066400000000000000000000022411477314666200235000ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_key.html --- # key [option_key] ::::{note} This setting is required when using the [allocation action](/reference/allocation.md). :::: ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... filters: - filtertype: ... ``` The value of this setting should correspond to a node setting on one or more nodes in your cluster. For example, you might have set ```sh node.tag: myvalue ``` in your `elasticsearch.yml` file for one or more of your nodes. To match allocation in this case, set key to `tag`. These special attributes are also supported: | attribute | description | | --- | --- | | `_name` | Match nodes by node name | | `_host_ip` | Match nodes by host IP address (IP associated with hostname) | | `_publish_ip` | Match nodes by publish IP address | | `_ip` | Match either `_host_ip` or `_publish_ip` | | `_host` | Match nodes by hostname | There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/option_max_age.md000066400000000000000000000016271477314666200243200ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_max_age.html --- # max_age [option_max_age] ```yaml action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_age: 1d ``` ::::{note} At least one of max_age, [max_docs](/reference/option_max_docs.md), [max_size](/reference/option_max_size.md) or any combinations of the three are required as `conditions:` for the [Rollover](/reference/rollover.md) action. :::: The maximum age that is allowed before triggering a rollover. It *must* be nested under `conditions:` There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. Ages such as `1d` for one day, or `30s` for 30 seconds can be used. elasticsearch-curator-8.0.21/docs/reference/option_max_docs.md000066400000000000000000000015561477314666200245150ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_max_docs.html --- # max_docs [option_max_docs] ```yaml action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_docs: 1000000 ``` ::::{note} At least one of [max_age](/reference/option_max_age.md), max_docs, [max_size](/reference/option_max_size.md) or any combinations of the three are required as `conditions:` for the [Rollover](/reference/rollover.md) action. :::: The maximum number of documents allowed in an index before triggering a rollover. It *must* be nested under `conditions:` There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. elasticsearch-curator-8.0.21/docs/reference/option_max_size.md000066400000000000000000000020001477314666200245200ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_max_size.html --- # max_size [option_max_size] ```yaml action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_size: 5gb ``` ::::{note} At least one of [max_age](/reference/option_max_age.md), [max_docs](/reference/option_max_docs.md), max_size or any combinations of the three are required as `conditions:` for the [Rollover](/reference/rollover.md) action. :::: The maximum approximate size an index is allowed to be before triggering a rollover. Sizes must use Elasticsearch approved [human-readable byte units](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/common-options.md). It *must* be nested under `conditions:` There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. elasticsearch-curator-8.0.21/docs/reference/option_max_wait.md000066400000000000000000000066111477314666200245260ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_max_wait.html --- # max_wait [option_max_wait] ::::{note} This setting is used by the [allocation](/reference/allocation.md), [cluster_routing](/reference/cluster_routing.md), [reindex](/reference/reindex.md), [replicas](/reference/replicas.md), [restore](/reference/restore.md), and [snapshot](/reference/snapshot.md) actions. :::: This setting must be a positive integer, or `-1`. This setting specifies how long in seconds to wait to see if the action has completed before giving up. This option is used in conjunction with [wait_interval](/reference/option_wait_interval.md), which is the number of seconds to wait between checking to see if the given action is complete. The default value for this setting is `-1`, meaning that Curator will wait indefinitely for the action to complete. ## [allocation](/reference/allocation.md) [_allocation/curator/docs/reference/elasticsearch/elasticsearch-client-curator/allocation.md] ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: True max_wait: 300 wait_interval: 10 filters: - filtertype: ... ``` ## [cluster_routing](/reference/cluster_routing.md) [_cluster_routing/curator/docs/reference/elasticsearch/elasticsearch-client-curator/cluster_routing.md] ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` ## [reindex](/reference/reindex.md) [_reindex/curator/docs/reference/elasticsearch/elasticsearch-client-curator/reindex.md] ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` ## [replicas](/reference/replicas.md) [_replicas/curator/docs/reference/elasticsearch/elasticsearch-client-curator/replicas.md] ```yaml action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ``` ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_5] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index include_global_state: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_3] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot include_global_state: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` elasticsearch-curator-8.0.21/docs/reference/option_migration_prefix.md000066400000000000000000000023271477314666200262630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_migration_prefix.html --- # migration_prefix [option_migration_prefix] ::::{note} This setting is used by the [reindex](/reference/reindex.md) action. :::: If the destination index is set to `MIGRATION`, Curator will reindex all selected indices one by one until they have all been reindexed. By configuring `migration_prefix`, a value can prepend each of those index names. For example, if I were reindexing `index1`, `index2`, and `index3`, and `migration_prefix` were set to `new-`, then the reindexed indices would be named `new-index1`, `new-index2`, and `new-index3`: ```yaml actions: 1: description: >- Reindex index1, index2, and index3 with a prefix of new-, resulting in indices named new-index1, new-index2, and new-index3 action: reindex options: wait_interval: 9 max_wait: -1 migration_prefix: new- request_body: source: index: ["index1", "index2", "index3"] dest: index: MIGRATION filters: - filtertype: none ``` `migration_prefix` can be used in conjunction with [*migration_suffix*](/reference/option_migration_suffix.md) elasticsearch-curator-8.0.21/docs/reference/option_migration_suffix.md000066400000000000000000000023361477314666200262720ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_migration_suffix.html --- # migration_suffix [option_migration_suffix] ::::{note} This setting is used by the [reindex](/reference/reindex.md) action. :::: If the destination index is set to `MIGRATION`, Curator will reindex all selected indices one by one until they have all been reindexed. By configuring `migration_suffix`, a value can be appended to each of those index names. For example, if I were reindexing `index1`, `index2`, and `index3`, and `migration_suffix` were set to `-new`, then the reindexed indices would be named `index1-new`, `index2-new`, and `index3-new`: ```yaml actions: 1: description: >- Reindex index1, index2, and index3 with a suffix of -new, resulting in indices named index1-new, index2-new, and index3-new action: reindex options: wait_interval: 9 max_wait: -1 migration_suffix: -new request_body: source: index: ["index1", "index2", "index3"] dest: index: MIGRATION filters: - filtertype: none ``` `migration_suffix` can be used in conjunction with [*migration_prefix*](/reference/option_migration_prefix.md) elasticsearch-curator-8.0.21/docs/reference/option_mns.md000066400000000000000000000013521477314666200235070ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_mns.html --- # max_num_segments [option_mns] ::::{note} This setting is required when using the [forceMerge action](/reference/forcemerge.md). :::: ```yaml action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 filters: - filtertype: ... ``` The value for this setting is the cutoff number of segments per shard. Indices which have more than this number of segments per shard will remain in the index list. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/option_name.md000066400000000000000000000075201477314666200236350ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_name.html --- # name [option_name] ::::{note} This setting is used by the [alias](/reference/alias.md), [create_index](/reference/create_index.md) and [snapshot](/reference/snapshot.md), actions. :::: The value of this setting is the name of the alias, snapshot, or index, depending on which action makes use of `name`. ## date math [_date_math_2] This setting may be a valid [Elasticsearch date math string](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/api-conventions.md#api-date-math-index-names). A date math name takes the following form: ```sh ``` | | | | --- | --- | | `static_name` | is the static text part of the name | | `date_math_expr` | is a dynamic date math expression that computes the date dynamically | | `date_format` | is the optional format in which the computed date should be rendered. Defaults to `yyyy.MM.dd`. | | `time_zone` | is the optional time zone . Defaults to `utc`. | The following example shows different forms of date math names and the final form they resolve to given the current time is 22rd March 2024 noon utc. | Expression | Resolves to | | --- | --- | | `` | `logstash-2024.03.22` | | `` | `logstash-2024.03.01` | | `` | `logstash-2024.03` | | `` | `logstash-2024.02` | | `` | `logstash-2024.03.23` | ## strftime [_strftime] This setting may alternately contain a valid Python strftime string. Curator will extract the strftime identifiers and replace them with the corresponding values. The identifiers that Curator currently recognizes include: | Unit | Value | | --- | --- | | `%Y` | 4 digit year | | `%y` | 2 digit year | | `%m` | 2 digit month | | `%W` | 2 digit week of the year | | `%d` | 2 digit day of the month | | `%H` | 2 digit hour of the day, in 24 hour notation | | `%M` | 2 digit minute of the hour | | `%S` | 2 digit second of the minute | | `%j` | 3 digit day of the year | ## [alias](/reference/alias.md) [_alias/curator/docs/reference/elasticsearch/elasticsearch-client-curator/alias.md_2] ```yaml action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name add: filters: - filtertype: ... remove: filters: - filtertype: ... ``` This option is required by the [alias](/reference/alias.md) action, and has no default setting in that context. ## [create_index](/reference/create_index.md) [_create_index/curator/docs/reference/elasticsearch/elasticsearch-client-curator/create_index.md_2] For the [create_index](/reference/create_index.md) action, there is no default setting, but you can use strftime: ```yaml action: create_index description: "Create index as named" options: name: 'myindex-%Y.%m' # ... ``` or use Elasticsearch [date math](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/api-conventions.md#api-date-math-index-names) ```yaml action: create_index description: "Create index as named" options: name: '' # ... ``` to name your indices. See more in the [create_index](/reference/create_index.md) documenation. ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_4] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: include_global_state: True wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` For the [snapshot](/reference/snapshot.md) action, the default value of this setting is `curator-%Y%m%d%H%M%S` elasticsearch-curator-8.0.21/docs/reference/option_new_index.md000066400000000000000000000035521477314666200246760ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_new_index.html --- # new_index [option_new_index] ::::{note} This optional setting is only used by the [rollover](/reference/rollover.md) action. :::: ```yaml description: >- Rollover the index associated with alias 'name'. Specify new index name using date math. options: name: aliasname new_index: "" conditions: max_age: 1d wait_for_active_shards: 1 ``` ::::{important} A new index name for rollover should still end with a dash followed by an incrementable number, e.g. `my_new_index-1`, or if using date math, `` or `` :::: ## date_math [_date_math_3] This setting may be a valid [Elasticsearch date math string](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/api-conventions.md#api-date-math-index-names). A date math name takes the following form: ```sh ``` | | | | --- | --- | | `static_name` | is the static text part of the name | | `date_math_expr` | is a dynamic date math expression that computes the date dynamically | | `date_format` | is the optional format in which the computed date should be rendered. Defaults to `yyyy.MM.dd`. | | `time_zone` | is the optional time zone . Defaults to `utc`. | The following example shows different forms of date math names and the final form they resolve to given the current time is 22rd March 2024 noon utc. | Expression | Resolves to | | --- | --- | | `` | `logstash-2024.03.22` | | `` | `logstash-2024.03.01` | | `` | `logstash-2024.03` | | `` | `logstash-2024.02` | | `` | `logstash-2024.03.23` | There is no default value for `new_index`. elasticsearch-curator-8.0.21/docs/reference/option_node_filters.md000066400000000000000000000031341477314666200253670ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_node_filters.html --- # node_filters [option_node_filters] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Allow master/data nodes to be potential shrink targets, but exclude 'named_node' from potential selection. options: shrink_node: DETERMINISTIC node_filters: permit_masters: True exclude_nodes: ['named_node'] filters: - filtertype: ... ``` There is no default value for `node_filters`. The current sub-options are as follows: ## permit_masters [_permit_masters] This option indicates whether the shrink action can select master eligible nodes when using `DETERMINISTIC` as the value for [shrink_node](/reference/option_shrink_node.md). The default value is `False`. Please note that this will exclude the elected master, as well as other master-eligible nodes. ::::{important} If you have a small cluster with only master/data nodes, you must set `permit_masters` to `True` in order to select one of those nodes as a potential [shrink_node](/reference/option_shrink_node.md). :::: ## exclude_nodes [_exclude_nodes] This option provides means to exclude nodes from selection when using `DETERMINISTIC` as the value for [shrink_node](/reference/option_shrink_node.md). It should be noted that you *can* use a named node for [shrink_node](/reference/option_shrink_node.md) and then exclude it here, and it will prevent a shrink from occurring. elasticsearch-curator-8.0.21/docs/reference/option_number_of_replicas.md000066400000000000000000000011771477314666200265550ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_number_of_replicas.html --- # number_of_replicas [option_number_of_replicas] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Set the number of replicas to 0. options: shrink_node: DETERMINISTIC number_of_replicas: 0 filters: - filtertype: ... ``` The value of this setting determines the number of replica shards per primary shard in the target index. The default value is `1`. elasticsearch-curator-8.0.21/docs/reference/option_number_of_shards.md000066400000000000000000000015241477314666200262330ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_number_of_shards.html --- # number_of_shards [option_number_of_shards] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Set the number of shards to 2. options: shrink_node: DETERMINISTIC number_of_shards: 2 filters: - filtertype: ... ``` The value of this setting determines the number of primary shards in the target index. The default value is `1`. ::::{important} The value for `number_of_shards` must meet the following criteria: * It must be lower than the number of primary shards in the source index. * It must be a factor of the number of primary shards in the source index. :::: elasticsearch-curator-8.0.21/docs/reference/option_partial.md000066400000000000000000000015221477314666200243450ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_partial.html --- # partial [option_partial] ::::{note} This setting is only used by the [snapshot](/reference/snapshot.md) action. :::: ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: ... partial: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` This setting must be either `True` or `False`. The entire snapshot will fail if one or more indices being added to the snapshot do not have all primary shards available. This behavior can be changed by setting partial to `True`. The default value of this setting is `False` elasticsearch-curator-8.0.21/docs/reference/option_post_allocation.md000066400000000000000000000017401477314666200261050ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_post_allocation.html --- # post_allocation [option_post_allocation] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Apply shard routing allocation to target indices. options: shrink_node: DETERMINISTIC post_allocation: allocation_type: include key: node_tag value: cold filters: - filtertype: ... ``` The only permitted subkeys for `post_allocation` are the same options used by the [allocation](/reference/allocation.md) action: [allocation_type](/reference/option_allocation_type.md), [key](/reference/option_key.md), and [value](/reference/option_value.md). If present, these values will be use to apply shard routing allocation to the target index after shrinking. There is no default value for `post_allocation`. elasticsearch-curator-8.0.21/docs/reference/option_preserve_existing.md000066400000000000000000000013111477314666200264520ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_preserve_existing.html --- # preserve_existing [option_preserve_existing] ```yaml action: index_settings description: "Change settings for selected indices" options: index_settings: index: refresh_interval: 5s ignore_unavailable: False preserve_existing: False filters: - filtertype: ... ``` This setting must be either `True` or `False`. If `preserve_existing` is set to `True`, and the configuration attempts to push a setting to an index that already has any value for that setting, the existing setting will be preserved, and remain unchanged. The default value of this setting is `False` elasticsearch-curator-8.0.21/docs/reference/option_refresh.md000066400000000000000000000017101477314666200243460ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_refresh.html --- # refresh [option_refresh] ::::{note} This setting is only used by the [reindex](/reference/reindex.md) action. :::: ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 refresh: True request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` Setting `refresh` to `True` will cause all re-indexed indexes to be refreshed. This differs from the Index API’s refresh parameter which causes just the *shard* that received the new data to be refreshed. Read more about this setting at [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/docs-reindex.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/docs-reindex.html) The default value is `True`. elasticsearch-curator-8.0.21/docs/reference/option_remote_certificate.md000066400000000000000000000017511477314666200265520ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_remote_certificate.html --- # remote_certificate [option_remote_certificate] This should be a file path to a CA certificate, or left empty. ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_certificate: /path/to/my/ca.cert remote_client_cert: /path/to/my/client.cert remote_client_key: /path/to/my/client.key request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ``` ::::{note} This option is only used by the [Reindex action](/reference/reindex.md) when performing a remote reindex operation. :::: This setting allows the use of a specified CA certificate file to validate the SSL certificate used by Elasticsearch. There is no default. elasticsearch-curator-8.0.21/docs/reference/option_remote_client_cert.md000066400000000000000000000025661477314666200265700ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_remote_client_cert.html --- # remote_client_cert [option_remote_client_cert] ::::{note} This option is only used by the [Reindex action](/reference/reindex.md) when performing a remote reindex operation. :::: This should be a file path to a client certificate (public key), or left empty. ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_certificate: /path/to/my/ca.cert remote_client_cert: /path/to/my/client.cert remote_client_key: /path/to/my/client.key request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ``` Allows the use of a specified SSL client cert file to authenticate to Elasticsearch. The file may contain both an SSL client certificate and an SSL key, in which case [client_key](/reference/configfile.md#client_key) is not used. If specifying `client_cert`, and the file specified does not also contain the key, use [client_key](/reference/configfile.md#client_key) to specify the file containing the SSL key. The file must be in PEM format, and the key part, if used, must be an unencrypted key in PEM format as well. elasticsearch-curator-8.0.21/docs/reference/option_remote_client_key.md000066400000000000000000000022371477314666200264160ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_remote_client_key.html --- # remote_client_key [option_remote_client_key] ::::{note} This option is only used by the [Reindex action](/reference/reindex.md) when performing a remote reindex operation. :::: This should be a file path to a client key (private key), or left empty. ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_certificate: /path/to/my/ca.cert remote_client_cert: /path/to/my/client.cert remote_client_key: /path/to/my/client.key request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ``` Allows the use of a specified SSL client key file to authenticate to Elasticsearch. If using [client_cert](/reference/configfile.md#client_cert) and the file specified does not also contain the key, use `client_key` to specify the file containing the SSL key. The key file must be an unencrypted key in PEM format. elasticsearch-curator-8.0.21/docs/reference/option_remote_filters.md000066400000000000000000000023361477314666200257400ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_remote_filters.html --- # remote_filters [option_remote_filters] ::::{note} This option is only used by the [Reindex action](/reference/reindex.md) when performing a remote reindex operation. :::: This is an array of [filters](/reference/filters.md), exactly as with regular index selection: ```yaml actions: 1: description: "Reindex matching indices into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: https://otherhost:9200 index: REINDEX_SELECTION dest: index: index2 remote_filters: - filtertype: *first* setting1: ... ... settingN: ... - filtertype: *second* setting1: ... ... settingN: ... - filtertype: *third* filters: - filtertype: none ``` This feature will only work when the `source` `index` is set to `REINDEX_SELECTION`. It will select *remote* indices matching the filters provided, and reindex them to the *local* cluster as the name provided in the `dest` `index`. In this example, that is `index2`. elasticsearch-curator-8.0.21/docs/reference/option_remote_url_prefix.md000066400000000000000000000022151477314666200264430ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_remote_url_prefix.html --- # remote_url_prefix [option_remote_url_prefix] ::::{note} This option is only used by the [Reindex action](/reference/reindex.md) when performing a remote reindex operation. :::: This should be a single value or left empty. ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 remote_url_prefix: my_prefix request_body: source: remote: host: https://otherhost:9200 index: index1 dest: index: index2 filters: - filtertype: none ``` In some cases you may be obliged to connect to a remote Elasticsearch cluster through a proxy of some kind. There may be a URL prefix before the API URI items, e.g. [http://example.com/elasticsearch/](http://example.com/elasticsearch/) as opposed to [http://localhost:9200](http://localhost:9200). In such a case, set the `remote_url_prefix` to the appropriate value, *elasticsearch* in this example. The default is an empty string. elasticsearch-curator-8.0.21/docs/reference/option_rename_pattern.md000066400000000000000000000036671477314666200257310ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_rename_pattern.html --- # rename_pattern [option_rename_pattern] ::::{note} This setting is only used by the [restore](/reference/restore.md) action. :::: ::::{admonition} from the Elasticsearch documentation :class: tip The rename_pattern and [rename_replacement](/reference/option_rename_replacement.md) options can be also used to rename indices on restore using regular expression that supports referencing the original text as explained [here](http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.md#appendReplacement(java.lang.StringBuffer,%20java.lang.String)). :::: ```yaml actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: rename_pattern: 'index(.+)' rename_replacement: 'restored_index$1' wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` In this configuration, Elasticsearch will capture whatever appears after `index` and put it after `restored_index`. For example, if I was restoring `index-2017.03.01`, the resulting index would be renamed to `restored_index-2017.03.01`. Read more about this setting at [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/snapshots-restore-snapshot.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/snapshots-restore-snapshot.md) There is no default value. elasticsearch-curator-8.0.21/docs/reference/option_rename_replacement.md000066400000000000000000000036771477314666200265540ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_rename_replacement.html --- # rename_replacement [option_rename_replacement] ::::{note} This setting is only used by the [restore](/reference/restore.md) action. :::: ::::{admonition} From the Elasticsearch documentation :class: tip The [rename_pattern](/reference/option_rename_pattern.md) and rename_replacement options can be also used to rename indices on restore using regular expression that supports referencing the original text as explained [here](http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.md#appendReplacement(java.lang.StringBuffer,%20java.lang.String)). :::: ```yaml actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: rename_pattern: 'index(.+)' rename_replacement: 'restored_index$1' wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` In this configuration, Elasticsearch will capture whatever appears after `index` and put it after `restored_index`. For example, if I was restoring `index-2017.03.01`, the resulting index would be renamed to `restored_index-2017.03.01`. Read more about this setting at [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/snapshots-restore-snapshot.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/snapshots-restore-snapshot.md) There is no default value. elasticsearch-curator-8.0.21/docs/reference/option_repository.md000066400000000000000000000025601477314666200251330ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_repository.html --- # repository [option_repository] ::::{note} This setting is only used by the [snapshot](/reference/snapshot.md), and [delete snapshots](/reference/delete_snapshots.md) actions. :::: There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_6] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_5] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` elasticsearch-curator-8.0.21/docs/reference/option_request_body.md000066400000000000000000000245341477314666200254260ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_request_body.html --- # request_body [option_request_body] ::::{note} This setting is only used by the [reindex](/reference/reindex.md) action. :::: ## Manual index selection [_manual_index_selection] The `request_body` option is the heart of the reindex action. In here, using YAML syntax, you recreate the body sent to Elasticsearch as described in [the official documentation](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-reindex). You can manually select indices as with this example: ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` You can also select multiple indices to reindex by making a list in acceptable YAML syntax: ```yaml actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ``` ::::{important} You *must* set at least a [none](/reference/filtertype_none.md) filter, or the action will fail. Do not worry. If you’ve manually specified your indices, those are the only ones which will be reindexed. :::: ## Filter-Selected Indices [_filter_selected_indices] Curator allows you to use all indices found by the `filters` section by setting the `source` index to `REINDEX_SELECTION`, like this: ```yaml actions: 1: description: >- Reindex all daily logstash indices from March 2017 into logstash-2017.03 action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: REINDEX_SELECTION dest: index: logstash-2017.03 filters: - filtertype: pattern kind: prefix value: logstash-2017.03. ``` ## Reindex From Remote [_reindex_from_remote] You can also reindex from remote: ```yaml actions: 1: description: "Reindex remote index1 to local index1" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: index1 dest: index: index1 filters: - filtertype: none ``` ::::{important} You *must* set at least a [none](/reference/filtertype_none.md) filter, or the action will fail. Do not worry. Only the indices you specified in `source` will be reindexed. :::: Curator will create a connection to the host specified as the `host` key in the above example. It will determine which port to connect to, and whether to use SSL by parsing the URL entered there. Because this `host` is specifically used by Elasticsearch, and Curator is making a separate connection, it is important to ensure that both Curator *and* your Elasticsearch cluster have access to the remote host. If you do not whitelist the remote cluster, you will not be able to reindex. This can be done by adding the following line to your `elasticsearch.yml` file: ```yaml reindex.remote.whitelist: remote_host_or_IP1:9200, remote_host_or_IP2:9200 ``` or by adding this flag to the command-line when starting Elasticsearch: ```sh bin/elasticsearch -Edefault.reindex.remote.whitelist="remote_host_or_IP:9200" ``` Of course, be sure to substitute the correct host, IP, or port. Other client connection arguments can also be supplied in the form of action options: * [remote_url_prefix](/reference/option_remote_url_prefix.md) * [remote_certificate](/reference/option_remote_certificate.md) * [remote_client_cert](/reference/option_remote_client_cert.md) * [remote_client_key](/reference/option_remote_client_key.md) ## Reindex From Remote With Filter-Selected Indices [_reindex_from_remote_with_filter_selected_indices] You can even reindex from remote with filter-selected indices on the remote side: ```yaml actions: 1: description: >- Reindex all remote daily logstash indices from March 2017 into local index logstash-2017.03 action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: REINDEX_SELECTION dest: index: logstash-2017.03 remote_filters: - filtertype: pattern kind: prefix value: logstash-2017.03. filters: - filtertype: none ``` ::::{important} Even though you are reindexing from remote, you *must* set at least a [none](/reference/filtertype_none.md) filter, or the action will fail. Do not worry. Only the indices specified in `source` will be reindexed. :::: Curator will create a connection to the host specified as the `host` key in the above example. It will determine which port to connect to, and whether to use SSL by parsing the URL entered there. Because this `host` is specifically used by Elasticsearch, and Curator is making a separate connection, it is important to ensure that both Curator *and* your Elasticsearch cluster have access to the remote host. If you do not whitelist the remote cluster, you will not be able to reindex. This can be done by adding the following line to your `elasticsearch.yml` file: ```yaml reindex.remote.whitelist: remote_host_or_IP1:9200, remote_host_or_IP2:9200 ``` or by adding this flag to the command-line when starting Elasticsearch: ```sh bin/elasticsearch -Edefault.reindex.remote.whitelist="remote_host_or_IP:9200" ``` Of course, be sure to substitute the correct host, IP, or port. Other client connection arguments can also be supplied in the form of action options: * [remote_url_prefix](/reference/option_remote_url_prefix.md) * [remote_certificate](/reference/option_remote_certificate.md) * [remote_client_cert](/reference/option_remote_client_cert.md) * [remote_client_key](/reference/option_remote_client_key.md) ## Reindex - Migration [_reindex_migration] Curator allows reindexing, particularly from remote, as a migration path. This can be a very useful feature for migrating an older cluster (1.4+) to a new cluster, on different hardware. It can also be a useful tool for serially reindexing indices into newer mappings in an automatable way. Ordinarily, a reindex operation is from either one, or many indices, to a single, named index. Assigning the `dest` `index` to `MIGRATION` tells Curator to treat this reindex differently. ::::{important} **If it is a *local* reindex,** you *must* set either [migration_prefix](/reference/option_migration_prefix.md), or [migration_suffix](/reference/option_migration_suffix.md), or both. This prevents collisions and other bad things from happening. By assigning a prefix or a suffix (or both), you can reindex any local indices to new versions of themselves, but named differently. It is true the Reindex API already has this functionality. Curator includes this same functionality for convenience. :::: This example will reindex all of the remote indices matching `logstash-2017.03.` into the local cluster, but preserve the original index names, rather than merge all of the contents into a single index. Internal to Curator, this results in multiple reindex actions: one per index. All other available options and settings are available. ```yaml actions: 1: description: >- Reindex all remote daily logstash indices from March 2017 into local versions with the same index names. action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: remote: host: http://otherhost:9200 username: myuser password: mypass index: REINDEX_SELECTION dest: index: MIGRATION remote_filters: - filtertype: pattern kind: prefix value: logstash-2017.03. filters: - filtertype: none ``` ::::{important} Even though you are reindexing from remote, you *must* set at least a [none](/reference/filtertype_none.md) filter, or the action will fail. Do not worry. Only the indices specified in `source` will be reindexed. :::: Curator will create a connection to the host specified as the `host` key in the above example. It will determine which port to connect to, and whether to use SSL by parsing the URL entered there. Because this `host` is specifically used by Elasticsearch, and Curator is making a separate connection, it is important to ensure that both Curator *and* your Elasticsearch cluster have access to the remote host. If you do not whitelist the remote cluster, you will not be able to reindex. This can be done by adding the following line to your `elasticsearch.yml` file: ```yaml reindex.remote.whitelist: remote_host_or_IP1:9200, remote_host_or_IP2:9200 ``` or by adding this flag to the command-line when starting Elasticsearch: ```sh bin/elasticsearch -Edefault.reindex.remote.whitelist="remote_host_or_IP:9200" ``` Of course, be sure to substitute the correct host, IP, or port. Other client connection arguments can also be supplied in the form of action options: * [remote_url_prefix](/reference/option_remote_url_prefix.md) * [remote_certificate](/reference/option_remote_certificate.md) * [remote_client_cert](/reference/option_remote_client_cert.md) * [remote_client_key](/reference/option_remote_client_key.md) * [migration_prefix](/reference/option_migration_prefix.md) * [migration_suffix](/reference/option_migration_suffix.md) ## Other scenarios and options [_other_scenarios_and_options] Nearly all scenarios supported by the reindex API are supported in the request_body, including (but not limited to): * Pipelines * Scripting * Queries * Conflict resolution * Limiting by count * Versioning * Reindexing operation type (for example, create-only) Read more about these, and more, at [http://www.elastic.co/guide/en/elasticsearch/reference/8.15/docs-reindex.html](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/docs-reindex.html) Notable exceptions include: * You cannot manually specify slices. Instead, use the [slices](/reference/option_slices.md) option for automated sliced reindexing. elasticsearch-curator-8.0.21/docs/reference/option_requests_per_second.md000066400000000000000000000015021477314666200267630ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_requests_per_second.html --- # requests_per_second [option_requests_per_second] ::::{note} This option is only used by the [Reindex action](/reference/reindex.md) :::: ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 requests_per_second: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` `requests_per_second` can be set to any positive decimal number (1.4, 6, 1000, etc) and throttles the number of requests per second that the reindex issues or it can be set to `-1` to disable throttling. The default value for this is option is `-1`. elasticsearch-curator-8.0.21/docs/reference/option_retry_count.md000066400000000000000000000010761477314666200252720ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_retry_count.html --- # retry_count [option_retry_count] ::::{note} This setting is only used by the [delete snapshots action](/reference/delete_snapshots.md). :::: ```yaml action: delete_snapshots description: "Delete selected snapshots from 'repository'" options: repository: ... retry_interval: 120 retry_count: 3 filters: - filtertype: ... ``` The value of this setting is the number of times to retry deleting a snapshot. The default for this setting is `3`. elasticsearch-curator-8.0.21/docs/reference/option_retry_interval.md000066400000000000000000000011071477314666200257610ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_retry_interval.html --- # retry_interval [option_retry_interval] ::::{note} This setting is only used by the [delete snapshots action](/reference/delete_snapshots.md). :::: ```yaml action: delete_snapshots description: "Delete selected snapshots from 'repository'" options: repository: ... retry_interval: 120 retry_count: 3 filters: - filtertype: ... ``` The value of this setting is the number of seconds to delay between retries. The default for this setting is `120`. elasticsearch-curator-8.0.21/docs/reference/option_routing_type.md000066400000000000000000000012411477314666200254370ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_routing_type.html --- # routing_type [option_routing_type] ::::{note} This setting is only used by the [cluster_routing action](/reference/cluster_routing.md). :::: ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` The value of this setting must be either `allocation` or `rebalance` There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/option_search_pattern.md000066400000000000000000000022051477314666200257120ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_search_pattern.html --- # search_pattern [option_search_pattern] ::::{note} This setting is only used by the [*Allocation*](/reference/allocation.md), [*Close*](/reference/close.md), [*Cold2Frozen*](/reference/cold2frozen.md), [*Delete Indices*](/reference/delete_indices.md), [*Forcemerge*](/reference/forcemerge.md), [*Index Settings*](/reference/index_settings.md), [*Open*](/reference/open.md), [*Replicas*](/reference/replicas.md), [*Shrink*](/reference/shrink.md), and [*Snapshot*](/reference/snapshot.md) actions. :::: ```yaml action: delete_indices description: "Delete selected indices" options: search_pattern: 'index-*' filters: - filtertype: ... ``` The value of this option can be a comma-separated list of data streams, indices, and aliases used to limit the request. Supports wildcards (*). To target all data streams and indices, omit this parameter or use * or _all. If using wildcards it is highly recommended to encapsulate the entire search pattern in single quotes, e.g. `search_pattern: 'a*,b*,c*'` The default value is `_all`. elasticsearch-curator-8.0.21/docs/reference/option_setting.md000066400000000000000000000012471477314666200243720ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_setting.html --- # setting [option_setting] ::::{note} This setting is only used by the [cluster_routing action](/reference/cluster_routing.md). :::: ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` The value of this must be `enable` at present. It is a placeholder for future expansion. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/option_shrink_node.md000066400000000000000000000022021477314666200252100ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_shrink_node.html --- # shrink_node [option_shrink_node] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space, excluding master nodes and the node named 'not_this_node' options: shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] shrink_suffix: '-shrink' filters: - filtertype: ... ``` This setting is required. There is no default value. The value of this setting must be the valid name of a node in your Elasticsearch cluster, or `DETERMINISTIC`. If the value is `DETERMINISTIC`, Curator will automatically select the data node with the most available free space and make that the target node. Curator will repeat this process for each successive index when the value is `DETERMINISTIC`. If [node_filters](/reference/option_node_filters.md), such as `exclude_nodes` are defined, those nodes will not be considered as potential target nodes. elasticsearch-curator-8.0.21/docs/reference/option_shrink_prefix.md000066400000000000000000000015131477314666200255640ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_shrink_prefix.html --- # shrink_prefix [option_shrink_prefix] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Prepend target index names with 'foo-' and append a suffix of '-shrink' options: shrink_node: DETERMINISTIC shrink_prefix: 'foo-' shrink_suffix: '-shrink' filters: - filtertype: ... ``` There is no default value for this setting. The value of this setting will be prepended to target index names. If the source index were `index`, and the `shrink_prefix` were `foo-`, and the `shrink_suffix` were `-shrink`, the resulting target index name would be `foo-index-shrink`. elasticsearch-curator-8.0.21/docs/reference/option_shrink_suffix.md000066400000000000000000000015171477314666200255770ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_shrink_suffix.html --- # shrink_suffix [option_shrink_suffix] ::::{note} This setting is only used by the [shrink](/reference/shrink.md) action. :::: ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Prepend target index names with 'foo-' and append a suffix of '-shrink' options: shrink_node: DETERMINISTIC shrink_prefix: 'foo-' shrink_suffix: '-shrink' filters: - filtertype: ... ``` The default value for this setting is `-shrink`. The value of this setting will be appended to target index names. If the source index were `index`, and the `shrink_prefix` were `foo-`, and the `shrink_suffix` were `-shrink`, the resulting target index name would be `foo-index-shrink`. elasticsearch-curator-8.0.21/docs/reference/option_skip_flush.md000066400000000000000000000010611477314666200250560ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_skip_flush.html --- # skip_flush [option_skip_flush] ::::{note} This setting is only used by the [close action](/reference/close.md), and is optional. :::: ```yaml action: close description: "Close selected indices" options: skip_flush: False filters: - filtertype: ... ``` If `skip_flush` is set to `True`, Curator will not flush indices to disk before closing. This may be useful for closing red indices before restoring. The default value is `False`. elasticsearch-curator-8.0.21/docs/reference/option_skip_fsck.md000066400000000000000000000045521477314666200246730ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_skip_fsck.html --- # skip_repo_fs_check [option_skip_fsck] ::::{note} This setting is used by the [snapshot](/reference/snapshot.md) and [restore](/reference/restore.md) actions. :::: This setting must be either `True` or `False`. The default value of this setting is `False` ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_7] Each master and data node in the cluster *must* have write access to the shared filesystem used by the repository for a snapshot to be 100% valid. For the purposes of a [restore](/reference/restore.md), this is a lightweight attempt to ensure that all nodes are *still* actively able to write to the repository, in hopes that snapshots were from all nodes. It is not otherwise necessary for the purposes of a restore. Some filesystems may take longer to respond to a check, which results in a false positive for the filesystem access check. For these cases, it is desirable to bypass this verification step, by setting this to `True.` ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index skip_repo_fs_check: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_6] Each master and data node in the cluster *must* have write access to the shared filesystem used by the repository for a snapshot to be 100% valid. Some filesystems may take longer to respond to a check, which results in a false positive for the filesystem access check. For these cases, it is desirable to bypass this verification step, by setting this to `True.` ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot skip_repo_fs_check: False wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` elasticsearch-curator-8.0.21/docs/reference/option_slices.md000066400000000000000000000030011477314666200241650ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_slices.html --- # slices [option_slices] ::::{note} This setting is only used by the [reindex](/reference/reindex.md) action. :::: This setting can speed up reindexing operations by using [Sliced Scroll](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/paginate-search-results.md#slice-scroll) to slice on the \_uid. ```yaml actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 slices: 3 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ``` ## Picking the number of slices [_picking_the_number_of_slices] Here are a few recommendations around the number of `slices` to use: * Don’t use large numbers. `500` creates fairly massive CPU thrash, so Curator will not allow a number larger than this. * It is more efficient from a query performance standpoint to use some multiple of the number of shards in the source index. * Using exactly as many shards as are in the source index is the most efficient from a query performance standpoint. * Indexing performance should scale linearly across available resources with the number of slices. * Whether indexing or query performance dominates that process depends on lots of factors like the documents being reindexed and the cluster doing the reindexing. elasticsearch-curator-8.0.21/docs/reference/option_timeout.md000066400000000000000000000013451477314666200244020ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_timeout.html --- # timeout [option_timeout] ::::{note} This setting is only used by the [reindex](/reference/reindex.md) action. :::: The `timeout` is the length in seconds each individual bulk request should wait for shards that are unavailable. The default value is `60`, meaning 60 seconds. ```yaml actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 timeout: 90 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ``` elasticsearch-curator-8.0.21/docs/reference/option_timeout_override.md000066400000000000000000000022151477314666200262760ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_timeout_override.html --- # timeout_override [option_timeout_override] ::::{note} This setting is available in all actions. :::: ```yaml action: forcemerge description: >- Perform a forceMerge on selected indices to 'max_num_segments' per shard options: max_num_segments: 2 timeout_override: 21600 filters: - filtertype: ... ``` If `timeout_override` is unset in your configuration, some actions will try to set a sane default value. The following table shows these default values: | Action Name | Default `timeout_override` Value | | --- | --- | | `close` | 180 | | `delete_snapshots` | 300 | | `forcemerge` | 21600 | | `restore` | 21600 | | `snapshot` | 21600 | All other actions have no default value for `timeout_override`. This setting must be an integer number of seconds, or an error will result. This setting is particularly useful for the [forceMerge](/reference/forcemerge.md) action, as all other actions have a new polling behavior when using [wait_for_completion](/reference/option_wfc.md) that should reduce or prevent client timeouts. elasticsearch-curator-8.0.21/docs/reference/option_value.md000066400000000000000000000054231477314666200240310ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_value.html --- # value [option_value] ::::{note} This setting is optional when using the [allocation action](/reference/allocation.md) and required when using the [cluster_routing action](/reference/cluster_routing.md). :::: ## [allocation](/reference/allocation.md) [_allocation/curator/docs/reference/elasticsearch/elasticsearch-client-curator/allocation.md_2] For the [allocation action](/reference/allocation.md), the value of this setting should correspond to a node setting on one or more nodes in your cluster For example, you might have set ```sh node.tag: myvalue ``` in your `elasticsearch.yml` file for one or more of your nodes. To match allocation in this case, set value to `myvalue`. Additonally, if you used one of the special attribute names `_ip`, `_name`, `_id`, or `_host` for [key](/reference/option_key.md), value can match the one of the node ip addresses, names, ids, or host names, respectively. ::::{note} To remove a routing allocation, the value of this setting should be left empty, or the `value` setting not even included as an option. :::: For example, you might have set ```sh PUT test/_settings { "index.routing.allocation.exclude.size": "small" } ``` to keep index `test` from allocating shards on nodes that have `node.tag: small`. To remove this shard routing allocation setting, you might use an action file similar to this: ```yaml --- actions: 1: action: allocation description: -> Unset 'index.routing.allocation.exclude.size' for index 'test' by passing an empty value. options: key: size value: ... allocation_type: exclude filters: - filtertype: pattern kind: regex value: '^test$' ``` ## [cluster_routing](/reference/cluster_routing.md) [_cluster_routing/curator/docs/reference/elasticsearch/elasticsearch-client-curator/cluster_routing.md_2] For the [cluster_routing action](/reference/cluster_routing.md), the acceptable values for this setting depend on the value of [routing_type](/reference/option_routing_type.md). ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: ... value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` Acceptable values when [routing_type](/reference/option_routing_type.md) is either `allocation` or `rebalance` are `all`, `primaries`, and `none` (string, not `NoneType`). If `routing_type` is `allocation`, this can also be `new_primaries`. If `routing_type` is `rebalance`, then the value can also be `replicas`. There is no default value. This setting must be set by the user or an exception will be raised, and execution will halt. elasticsearch-curator-8.0.21/docs/reference/option_wait_for_active_shards.md000066400000000000000000000037501477314666200274270ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_wait_for_active_shards.html --- # wait_for_active_shards [option_wait_for_active_shards] ::::{note} This setting is used by the [Reindex](/reference/reindex.md), [Rollover](/reference/rollover.md), and [Shrink](/reference/shrink.md) actions. Each use it similarly. :::: This setting determines the number of shard copies that must be active before the client returns. The default value is 1, which implies only the primary shards. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1) Read [the Elasticsearch documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/docs-index_.md#index-wait-for-active-shards) for more information. ## Reindex [_reindex] ```yaml actions: 1: description: "Reindex index1,index2,index3 into new_index" action: reindex options: wait_interval: 9 max_wait: -1 wait_for_active_shards: 2 request_body: source: index: ['index1', 'index2', 'index3'] dest: index: new_index filters: - filtertype: none ``` ## Rollover [_rollover] ```yaml action: rollover description: >- Rollover the index associated with alias 'name', which should be in the form of prefix-000001 (or similar), or prefix-yyyy.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 wait_for_active_shards: 1 extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: False ``` ## Shrink [_shrink] ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Prepend target index names with 'foo-' and append a suffix of '-shrink' options: shrink_node: DETERMINISTIC wait_for_active_shards: all filters: - filtertype: ... ``` elasticsearch-curator-8.0.21/docs/reference/option_wait_for_rebalance.md000066400000000000000000000025051477314666200265210ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_wait_for_rebalance.html --- # wait_for_rebalance [option_wait_for_rebalance] ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink, then reroute the shrunk index with the provided parameters. options: ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: ... ``` ::::{note} This setting is used by the [shrink](/reference/shrink.md) action. :::: This setting must be `true` or `false`. Setting this to `false` will result in the [shrink](/reference/shrink.md) action only checking that the index being shrunk has finished being relocated, and not continue to wait for the cluster to fully rebalance all shards. The default value for this setting is `false`. elasticsearch-curator-8.0.21/docs/reference/option_wait_interval.md000066400000000000000000000071431477314666200255660ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_wait_interval.html --- # wait_interval [option_wait_interval] ::::{note} This setting is used by the [allocation](/reference/allocation.md), [cluster_routing](/reference/cluster_routing.md), [reindex](/reference/reindex.md), [replicas](/reference/replicas.md), [restore](/reference/restore.md), and [snapshot](/reference/snapshot.md) actions. :::: This setting must be a positive integer between 1 and 30. This setting specifies how long to wait between checks to see if the action has completed or not. This number should not be larger than the client [request_timeout](/reference/configfile.md#request_timeout) or the [timeout_override](/reference/option_timeout_override.md). As the default client [request_timeout](/reference/configfile.md#request_timeout) value for is 30, this should be uncommon. The default value for this setting is `9`, meaning 9 seconds between checks. This option is generally used in conjunction with [max_wait](/reference/option_max_wait.md), which is the maximum amount of time in seconds to wait for the given action to complete. ## [allocation](/reference/allocation.md) [_allocation/curator/docs/reference/elasticsearch/elasticsearch-client-curator/allocation.md_4] ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: False max_wait: 300 wait_interval: 10 filters: - filtertype: ... ``` ## [cluster_routing](/reference/cluster_routing.md) [_cluster_routing/curator/docs/reference/elasticsearch/elasticsearch-client-curator/cluster_routing.md_4] ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` ## [reindex](/reference/reindex.md) [_reindex/curator/docs/reference/elasticsearch/elasticsearch-client-curator/reindex.md_3] ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` ## [replicas](/reference/replicas.md) [_replicas/curator/docs/reference/elasticsearch/elasticsearch-client-curator/replicas.md_3] ```yaml action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ``` ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_9] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_8] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` elasticsearch-curator-8.0.21/docs/reference/option_warn_if_no_indices.md000066400000000000000000000034511477314666200265330ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_warn_if_no_indices.html --- # warn_if_no_indices [option_warn_if_no_indices] ::::{note} This setting is only used by the [alias](/reference/alias.md) action. :::: This setting must be either `True` or `False`. The default value for this setting is `False`. ```yaml action: alias description: "Add/Remove selected indices to or from the specified alias" options: name: alias_name warn_if_no_indices: False add: filters: - filtertype: ... remove: filters: - filtertype: ... ``` This setting specifies whether or not the alias action should continue with a warning or return immediately in the event that the filters in the add or remove section result in an empty index list. ::::{admonition} Improper use of this setting can yield undesirable results :class: warning **Ideal use case:** For example, you want to add the most recent seven days of time-series indices into a *lastweek* alias, and remove indices older than seven days from this same alias. If you do not not yet have any indices older than seven days, this will result in an empty index list condition which will prevent the entire alias action from completing successfully. If `warn_if_no_indices` were set to `True`, however, it would avert that potential outcome. **Potentially undesirable outcome:** A *non-beneficial* case would be where if `warn_if_no_indices` is set to `True`, and a misconfiguration results in indices not being found, and therefore not being disassociated from the alias. As a result, an alias that should only query one week now references multiple weeks of data. If `warn_if_no_indices` were set to `False`, this scenario would have been averted because the empty list condition would have resulted in an error. :::: elasticsearch-curator-8.0.21/docs/reference/option_wfc.md000066400000000000000000000075161477314666200235010ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/option_wfc.html --- # wait_for_completion [option_wfc] ::::{note} This setting is used by the [allocation](/reference/allocation.md), [cluster_routing](/reference/cluster_routing.md), [reindex](/reference/reindex.md), [replicas](/reference/replicas.md), [restore](/reference/restore.md), and [snapshot](/reference/snapshot.md) actions. :::: This setting must be either `True` or `False`. This setting specifies whether or not the request should return immediately or wait for the operation to complete before returning. ## [allocation](/reference/allocation.md) [_allocation/curator/docs/reference/elasticsearch/elasticsearch-client-curator/allocation.md_3] ```yaml action: allocation description: "Apply shard allocation filtering rules to the specified indices" options: key: ... value: ... allocation_type: ... wait_for_completion: False max_wait: 300 wait_interval: 10 filters: - filtertype: ... ``` The default value for the [allocation](/reference/allocation.md) action is `False`. ## [cluster_routing](/reference/cluster_routing.md) [_cluster_routing/curator/docs/reference/elasticsearch/elasticsearch-client-curator/cluster_routing.md_3] ```yaml action: cluster_routing description: "Apply routing rules to the entire cluster" options: routing_type: value: ... setting: enable wait_for_completion: True max_wait: 300 wait_interval: 10 ``` The default value for the [cluster_routing](/reference/cluster_routing.md) action is `False`. ## [reindex](/reference/reindex.md) [_reindex/curator/docs/reference/elasticsearch/elasticsearch-client-curator/reindex.md_2] ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` The default value for the [reindex](/reference/reindex.md) action is `False`. ## [replicas](/reference/replicas.md) [_replicas/curator/docs/reference/elasticsearch/elasticsearch-client-curator/replicas.md_2] ```yaml action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ``` The default value for the [replicas](/reference/replicas.md) action is `False`. ## [restore](/reference/restore.md) [_restore/curator/docs/reference/elasticsearch/elasticsearch-client-curator/restore.md_8] ```yaml actions: 1: action: restore description: Restore my_index from my_snapshot in my_repository options: repository: my_repository name: my_snapshot indices: my_index wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` The default value for the [restore](/reference/restore.md) action is `True`. ## [snapshot](/reference/snapshot.md) [_snapshot/curator/docs/reference/elasticsearch/elasticsearch-client-curator/snapshot.md_7] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: my_repository name: my_snapshot wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` The default value for the [snapshot](/reference/snapshot.md) action is `True`. ::::{tip} During snapshot initialization, information about all previous snapshots is loaded into the memory, which means that in large repositories it may take several seconds (or even minutes) for this command to return even if the `wait_for_completion` setting is set to `False`. :::: elasticsearch-curator-8.0.21/docs/reference/options.md000066400000000000000000000061311477314666200230150ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/options.html --- # Options [options] Options are settings used by [actions](/reference/actions.md). * [allocation_type](/reference/option_allocation_type.md) * [allow_ilm_indices](/reference/option_allow_ilm.md) * [continue_if_exception](/reference/option_continue.md) * [count](/reference/option_count.md) * [delay](/reference/option_delay.md) * [delete_aliases](/reference/option_delete_aliases.md) * [skip_flush](/reference/option_skip_flush.md) * [disable_action](/reference/option_disable.md) * [extra_settings](/reference/option_extra_settings.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [ignore_unavailable](/reference/option_ignore.md) * [include_aliases](/reference/option_include_aliases.md) * [include_global_state](/reference/option_include_gs.md) * [indices](/reference/option_indices.md) * [key](/reference/option_key.md) * [max_age](/reference/option_max_age.md) * [max_docs](/reference/option_max_docs.md) * [max_size](/reference/option_max_size.md) * [max_num_segments](/reference/option_mns.md) * [max_wait](/reference/option_max_wait.md) * [migration_prefix](/reference/option_migration_prefix.md) * [migration_suffix](/reference/option_migration_suffix.md) * [name](/reference/option_name.md) * [new_index](/reference/option_new_index.md) * [node_filters](/reference/option_node_filters.md) * [number_of_replicas](/reference/option_number_of_replicas.md) * [number_of_shards](/reference/option_number_of_shards.md) * [partial](/reference/option_partial.md) * [refresh](/reference/option_refresh.md) * [remote_certificate](/reference/option_remote_certificate.md) * [remote_client_cert](/reference/option_remote_client_cert.md) * [remote_client_key](/reference/option_remote_client_key.md) * [remote_filters](/reference/option_remote_filters.md) * [remote_url_prefix](/reference/option_remote_url_prefix.md) * [rename_pattern](/reference/option_rename_pattern.md) * [rename_replacement](/reference/option_rename_replacement.md) * [repository](/reference/option_repository.md) * [request_body](/reference/option_request_body.md) * [requests_per_second](/reference/option_requests_per_second.md) * [retry_count](/reference/option_retry_count.md) * [retry_interval](/reference/option_retry_interval.md) * [routing_type](/reference/option_routing_type.md) * [search_pattern](/reference/option_search_pattern.md) * [setting](/reference/option_setting.md) * [shrink_node](/reference/option_shrink_node.md) * [slices](/reference/option_slices.md) * [skip_repo_fs_check](/reference/option_skip_fsck.md) * [timeout](/reference/option_timeout.md) * [timeout_override](/reference/option_timeout_override.md) * [value](/reference/option_value.md) * [wait_for_active_shards](/reference/option_wait_for_active_shards.md) * [wait_for_completion](/reference/option_wfc.md) * [wait_for_rebalance](/reference/option_wait_for_rebalance.md) * [wait_interval](/reference/option_wait_interval.md) * [warn_if_no_indices](/reference/option_warn_if_no_indices.md) You can use [environment variables](/reference/envvars.md) in your configuration files. elasticsearch-curator-8.0.21/docs/reference/pip.md000066400000000000000000000035001477314666200221070ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/pip.html sub: curator_v: 8.0.18 --- # pip [pip] This installation procedure requires a functional Python `pip` executable and requires that the target machine has internet connectivity for downloading Curator and the dependencies from [The Python Package Index](https://pypi.org). ``` pip install elasticsearch-curator ``` ## Upgrading with pip [_upgrading_with_pip] If you already have Elasticsearch Curator installed, and want to upgrade to the latest version, use the `-U` flag: ``` pip install -U elasticsearch-curator ``` ## Installing a specific version with pip [_installing_a_specific_version_with_pip] The `-U` flag uninstalls the current version (if any), then installs the latest version, or a specified one. Specify a specific version by adding `==` followed by the version you’d like to install, like this: ``` pip install -U elasticsearch-curator==X.Y.Z ``` For example: ``` pip install -U elasticsearch-curator=={{curator_v}} ``` ## System-wide vs. User-only installation [_system_wide_vs_user_only_installation] The above commands each imply a system-wide installation. This usually requires super-user access, or the `sudo` command. There is a way to install Curator into a path for just the current user, using the `--user` flag. ``` pip install --user elasticsearch-curator ``` This will result in the `curator` end-point being installed in the current user’s home directory, in the `.local` directory, in the `bin` subdirectory. The full path might look something like this: ``` /home/user/.local/bin/curator ``` You can make an alias or a symlink to this so you can call it more easily. The `--user` flag can also be used in conjunction with the `-U` flag: ``` pip install -U --user elasticsearch-curator==X.Y.Z ```elasticsearch-curator-8.0.21/docs/reference/python-source.md000066400000000000000000000007031477314666200241400ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/python-source.html --- # Installation from source [python-source] Installing or Curator from source tarball (rather than doing a `git clone`) is also possible. Download and install Curator from tarball: 1. `wget https://github.com/elastic/curator/archive/v``8.0.17.tar.gz -O elasticsearch-curator.tar.gz` 2. `pip install elasticsearch-curator.tar.gz`   elasticsearch-curator-8.0.21/docs/reference/reindex.md000066400000000000000000000045111477314666200227600ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/reindex.html --- # Reindex [reindex] ```yaml actions: 1: description: "Reindex index1 into index2" action: reindex options: wait_interval: 9 max_wait: -1 request_body: source: index: index1 dest: index: index2 filters: - filtertype: none ``` There are many options for the reindex option. The best place to start is in the [request_body documentation](/reference/option_request_body.md) to see how to configure this action. All other options are as follows. ## Required settings [_required_settings_7] * [request_body](/reference/option_request_body.md) ## Optional settings [_optional_settings_12] * [refresh](/reference/option_refresh.md) * [remote_certificate](/reference/option_remote_certificate.md) * [remote_client_cert](/reference/option_remote_client_cert.md) * [remote_client_key](/reference/option_remote_client_key.md) * [remote_filters](/reference/option_remote_filters.md) * [remote_url_prefix](/reference/option_remote_url_prefix.md) * [request_body](/reference/option_request_body.md) * [requests_per_second](/reference/option_requests_per_second.md) * [slices](/reference/option_slices.md) * [timeout](/reference/option_timeout.md) * [wait_for_active_shards](/reference/option_wait_for_active_shards.md) * [wait_for_completion](/reference/option_wfc.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) * [migration_prefix](/reference/option_migration_prefix.md) * [migration_suffix](/reference/option_migration_suffix.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_reindex.md). :::: ## Compatibility [_compatibility] Generally speaking, the Curator should be able to perform a remote reindex from any version of Elasticsearch, 1.4 and newer. Strictly speaking, the Reindex API in Elasticsearch *is* able to reindex from older clusters, but Curator cannot be used to facilitate this due to Curator’s dependency on changes released in 1.4. elasticsearch-curator-8.0.21/docs/reference/replicas.md000066400000000000000000000035631477314666200231320ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/replicas.html --- # Replicas [replicas] ```yaml action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action will set the number of replicas per shard to the value of [count](/reference/option_count.md). You can optionally set `wait_for_completion` to `True` to have Curator wait for the replication operation to complete before continuing: ```yaml action: replicas description: >- Set the number of replicas per shard for selected indices to 'count' options: count: ... wait_for_completion: True max_wait: 600 wait_interval: 10 filters: - filtertype: ... ``` This configuration will wait for a maximum of 600 seconds for all index replicas to be complete before giving up. A `max_wait` value of `-1` will wait indefinitely. Curator will poll for completion at `10` second intervals, as defined by `wait_interval`. ## Required settings [_required_settings_8] * [count](/reference/option_count.md) ## Optional settings [_optional_settings_13] * [search_pattern](/reference/option_search_pattern.md) * [wait_for_completion](/reference/option_wfc.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_replicas.md). :::: elasticsearch-curator-8.0.21/docs/reference/restore.md000066400000000000000000000116311477314666200230060ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/restore.html --- # Restore [restore] ```yaml actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action will restore indices from the indicated [repository](/reference/option_repository.md), from the most recent snapshot identified by the applied filters, or the snapshot identified by [name](/reference/option_name.md). ## Renaming indices on restore [_renaming_indices_on_restore] You can cause indices to be renamed at restore with the [rename_pattern](/reference/option_rename_pattern.md) and [rename_replacement](/reference/option_rename_replacement.md) options: ```yaml actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: rename_pattern: 'index(.+)' rename_replacement: 'restored_index$1' wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` In this configuration, Elasticsearch will capture whatever appears after `index` and put it after `restored_index`. For example, if I was restoring `index-2017.03.01`, the resulting index would be renamed to `restored_index-2017.03.01`. ## Extra settings [_extra_settings_2] The [extra_settings](/reference/option_extra_settings.md) option allows the addition of extra settings, such as index settings. An example of how these settings can be used to change settings for an index being restored might be: ```yaml actions: 1: action: restore description: >- Restore all indices in the most recent snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # If name is blank, the most recent snapshot by age will be selected name: # If indices is blank, all indices in the snapshot will be restored indices: extra_settings: index_settings: number_of_replicas: 0 wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: state state: SUCCESS exclude: - filtertype: ... ``` In this case, the number of replicas will be applied to the restored indices. For more information see the [official Elasticsearch Documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/snapshots-restore-snapshot.md). ## Required settings [_required_settings_9] * [repository](/reference/option_repository.md) ## Optional settings [_optional_settings_14] * [name](/reference/option_name.md) * [include_aliases](/reference/option_include_aliases.md) * [indices](/reference/option_indices.md) * [ignore_unavailable](/reference/option_ignore.md) * [include_global_state](/reference/option_include_gs.md) * [partial](/reference/option_partial.md) * [rename_pattern](/reference/option_rename_pattern.md) * [rename_replacement](/reference/option_rename_replacement.md) * [extra_settings](/reference/option_extra_settings.md) * [wait_for_completion](/reference/option_wfc.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) * [skip_repo_fs_check](/reference/option_skip_fsck.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_restore.md). :::: elasticsearch-curator-8.0.21/docs/reference/rollover.md000066400000000000000000000070231477314666200231670ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/rollover.html --- # Rollover [rollover] ```yaml action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-YYYY.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 max_size: 5gb ``` This action uses the [Elasticsearch Rollover API](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-rollover-index.md) to create a new index, if any of the described conditions are met. ::::{important} When choosing `conditions`, **any** one of [max_age](/reference/option_max_age.md), [max_docs](/reference/option_max_docs.md), [max_size](/reference/option_max_size.md), **or any combination of the three** may be used. If multiple are used, then the specified condition for any one of them must be matched for the rollover to occur. :::: ::::{warning} If one or more of the [max_age](/reference/option_max_age.md), [max_docs](/reference/option_max_docs.md), or [max_size](/reference/option_max_size.md) options are present, they must each have a value. Because there are no default values, none of these conditions can be left empty, or Curator will generate an error. :::: ## Extra settings [_extra_settings_3] The [extra_settings](/reference/option_extra_settings.md) option allows the addition of extra index settings (but not mappings). An example of how these settings can be used might be: ```yaml action: rollover description: >- Rollover the index associated with alias 'aliasname', which should be in the form of prefix-000001 (or similar), or prefix-YYYY.MM.DD-1. options: name: aliasname conditions: max_age: 1d max_docs: 1000000 extra_settings: index.number_of_shards: 3 index.number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: False ``` ## Required settings [_required_settings_10] * [name](/reference/option_name.md) The alias name * [max_age](/reference/option_max_age.md) The maximum age that is allowed before triggering a rollover. This *must* be nested under `conditions:`. There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. * [max_docs](/reference/option_max_docs.md) The maximum number of documents allowed in an index before triggering a rollover. This *must* be nested under `conditions:`. There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. * [max_size](/reference/option_max_size.md) The maximum size the index can be before a rollover is triggered. This *must* be nested under `conditions:`. There is no default value. If this condition is specified, it must have a value, or Curator will generate an error. ## Optional settings [_optional_settings_15] * [extra_settings](/reference/option_extra_settings.md) No default value. You can add any acceptable index settings (not mappings) as nested YAML. See the [Elasticsearch Create Index API documentation](http://www.elastic.co/guide/en/elasticsearch/reference/8.15/indices-create-index.md) for more information. * [new_index](/reference/option_new_index.md) Specify a new index name. * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_rollover.md). :::: elasticsearch-curator-8.0.21/docs/reference/shrink.md000066400000000000000000000127521477314666200226260ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/shrink.html --- # Shrink [shrink] ```yaml action: shrink description: >- Shrink selected indices on the node with the most available space. Delete source index after successful shrink, then reroute the shrunk index with the provided parameters. options: ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: Shrinking an index is a good way to reduce the total shard count in your cluster. [Several conditions need to be met](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-shrink) in order for index shrinking to take place: * The index must be marked as read-only * A (primary or replica) copy of every shard in the index must be relocated to the same node * The cluster must have health `green` * The target index must not exist * The number of primary shards in the target index must be a factor of the number of primary shards in the source index. * The source index must have more primary shards than the target index. * The index must not contain more than 2,147,483,519 documents in total across all shards that will be shrunk into a single shard on the target index as this is the maximum number of docs that can fit into a single shard. * The node handling the shrink process must have sufficient free disk space to accommodate a second copy of the existing index. Curator will try to meet these conditions. If it is unable to meet them all, it will not perform a shrink operation. This action will shrink indices to the target index, the name of which is the value of [shrink_prefix](/reference/option_shrink_prefix.md) + the source index name + [shrink_suffix](/reference/option_shrink_suffix.md). The resulting index will have [number_of_shards](/reference/option_number_of_shards.md) primary shards, and [number_of_replicas](/reference/option_number_of_replicas.md) replica shards. The shrinking will take place on the node identified by [shrink_node](/reference/option_shrink_node.md), unless `DETERMINISTIC` is specified, in which case Curator will evaluate all of the nodes to determine which one has the most free space. If multiple indices are identified for shrinking by the filter block, and `DETERMINISTIC` is specified, the node selection process will be repeated for each successive index, preventing all of the space being consumed on a single node. By default, Curator will delete the source index after a successful shrink. This can be disabled by setting [delete_after](/reference/option_delete_after.md) to `False`. If the source index, is not deleted after a successful shrink, Curator will remove the read-only setting and the shard allocation routing applied to the source index to put it on the shrink node. Curator will wait for the shards to stop rerouting before continuing. The [post_allocation](/reference/option_post_allocation.md) option applies to the target index after the shrink is complete. If set, this shard allocation routing will be applied (after a successful shrink) and Curator will wait for all shards to stop rerouting before continuing. The only [extra_settings](/reference/option_extra_settings.md) which are acceptable are `settings` and `aliases`. Please note that in the example above, while `best_compression` is being applied to the new index, it will not take effect until new writes are made to the index, such as when [force-merging](/reference/forcemerge.md) the shard to a single segment. The other options are usually okay to leave at the defaults, but feel free to change them as needed. ## Required settings [_required_settings_11] * [shrink_node](/reference/option_shrink_node.md) ## Optional settings [_optional_settings_16] * [search_pattern](/reference/option_search_pattern.md) * [continue_if_exception](/reference/option_continue.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [copy_aliases](/reference/option_copy_aliases.md) * [delete_after](/reference/option_delete_after.md) * [disable_action](/reference/option_disable.md) * [extra_settings](/reference/option_extra_settings.md) * [node_filters](/reference/option_node_filters.md) * [number_of_shards](/reference/option_number_of_shards.md) * [number_of_replicas](/reference/option_number_of_replicas.md) * [post_allocation](/reference/option_post_allocation.md) * [shrink_prefix](/reference/option_shrink_prefix.md) * [shrink_suffix](/reference/option_shrink_suffix.md) * [timeout_override](/reference/option_timeout_override.md) * [wait_for_active_shards](/reference/option_wait_for_active_shards.md) * [wait_for_completion](/reference/option_wfc.md) * [wait_for_rebalance](/reference/option_wait_for_rebalance.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_shrink.md). :::: elasticsearch-curator-8.0.21/docs/reference/singleton-cli.md000066400000000000000000000230051477314666200240700ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/singleton-cli.html --- # Singleton Command Line Interface [singleton-cli] The `curator_cli` command allows users to run a single, supported action from the command-line, without needing either the client or action YAML configuration file, though it does support using the client configuration file if you want. As an important bonus, the command-line options allow you to override the settings in the `curator.yml` file! ::::{important} While both the configuration file and the command-line arguments can be used together, it is important to note that command-line options will override file-based configuration of the same setting. :::: ```sh $ curator_cli --help Usage: curator_cli [OPTIONS] COMMAND [ARGS]... Curator CLI (Singleton Tool) Run a single action from the command-line. The default $HOME/.curator/curator.yml configuration file (--config) can be used but is not needed. Command-line settings will always override YAML configuration settings. Options: --config PATH Path to configuration file. --hosts TEXT Elasticsearch URL to connect to. --cloud_id TEXT Elastic Cloud instance id --api_token TEXT The base64 encoded API Key token --id TEXT API Key "id" value --api_key TEXT API Key "api_key" value --username TEXT Elasticsearch username --password TEXT Elasticsearch password --bearer_auth TEXT Bearer authentication token --opaque_id TEXT X-Opaque-Id HTTP header value --request_timeout FLOAT Request timeout in seconds --http_compress / --no-http_compress Enable HTTP compression [default: no-http_compress] --verify_certs / --no-verify_certs Verify SSL/TLS certificate(s) [default: verify_certs] --ca_certs TEXT Path to CA certificate file or directory --client_cert TEXT Path to client certificate file --client_key TEXT Path to client key file --ssl_assert_hostname TEXT Hostname or IP address to verify on the node's certificate. --ssl_assert_fingerprint TEXT SHA-256 fingerprint of the node's certificate. If this value is given then root-of-trust verification isn't done and only the node's certificate fingerprint is verified. --ssl_version TEXT Minimum acceptable TLS/SSL version --master-only / --no-master-only Only run if the single host provided is the elected master [default: no-master-only] --skip_version_test / --no-skip_version_test Elasticsearch version compatibility check [default: no-skip_version_test] --dry-run Do not perform any changes. --loglevel [DEBUG|INFO|WARNING|ERROR|CRITICAL] Log level --logfile TEXT Log file --logformat [default|ecs] Log output format -v, --version Show the version and exit. -h, --help Show this message and exit. Commands: alias Add/Remove Indices to/from Alias allocation Shard Routing Allocation close Close Indices delete-indices Delete Indices delete-snapshots Delete Snapshots forcemerge forceMerge Indices (reduce segment count) open Open Indices replicas Change Replica Count restore Restore Indices rollover Rollover Index associated with Alias show-indices Show Indices show-snapshots Show Snapshots shrink Shrink Indices to --number_of_shards snapshot Snapshot Indices Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/singleton-cli.html ``` The option flags for the given commands match those used for the same [actions](/reference/actions.md). The only difference is how filtering is handled. ## Running Curator from Docker [_running_curator_from_docker_2] Running `curator_cli` from the command-line using Docker requires only a few additional steps. Should you desire to use them, Docker-based `curator_cli` requires you to map a volume for your configuration and/or log files. Attempting to read a YAML configuration file if you have neglected to volume map your configuration directory to `/.curator` will not work. It looks like this: ```sh docker run [-t] --rm --name myimagename \ --entrypoint /curator/curator_cli \ -v /PATH/TO/MY/CONFIGS:/.curator \ untergeek/curator:mytag \ --config /.curator/config.yml [OPTIONS] COMMAND [ARGS]... ``` ::::{note} While testing, adding the `-t` flag will allocate a pseudo-tty, allowing you to see terminal output that would otherwise be hidden. :::: The `config.yml` file should already exist in the path `/PATH/TO/MY/CONFIGS` before run time. The `--rm` in the command means that the container (not the image) will be deleted after completing execution. You definitely want this as there is no reason to keep creating containers for each run. The eventual cleanup from this would be unpleasant. ## Command-line filtering [_command_line_filtering] Recent improvements in Curator include schema and setting validation. With these improvements, it is possible to validate filters and their many permutations if passed in a way that Curator can easily digest. ```sh --filter_list TEXT JSON string representing an array of filters. ``` This means that filters need to be passed as a single object, or an array of objects in JSON format. Single: ```sh --filter_list '{"filtertype":"none"}' ``` Multiple: ```sh --filter_list '[{"filtertype":"age","source":"creation_date","direction":"older","unit":"days","unit_count":13},{"filtertype":"pattern","kind":"prefix","value":"logstash"}]' ``` This preserves the power of chained filters, making them available on the command line. ::::{note} You may need to escape all of the double quotes on some platforms, or shells like PowerShell, for instance. :::: Caveats to this approach: 1. Only one action can be taken at a time. 2. Not all actions have singleton analogs. For example, [Alias](/reference/alias.md) and
[Restore](/reference/restore.md) do not have singleton actions. ## Show Indices/Snapshots [_show_indicessnapshots] One feature that the singleton command offers that the other cannot is to show which indices and snapshots are in the system. It’s a great way to visually test your filters without causing any harm to the system. ```sh $ curator_cli show-indices --help Usage: curator_cli show-indices [OPTIONS] Show indices Options: --verbose Show verbose output. --header Print header if --verbose --epoch Print time as epoch if --verbose --filter_list TEXT JSON string representing an array of filters. [required] --help Show this message and exit. Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/singleton-cli.html#_show_indicessnapshots ``` ```sh $ curator_cli show-snapshots --help Usage: curator_cli show-snapshots [OPTIONS] Show snapshots Options: --repository TEXT Snapshot repository name [required] --filter_list TEXT JSON string representing an array of filters. [required] --help Show this message and exit. Learn more at https://www.elastic.co/guide/en/elasticsearch/client/curator/8.0/singleton-cli.html#_show_indicessnapshots ``` The `show-snapshots` command will only show snapshots matching the provided filters. The `show-indices` command will also do this, but also offers a few extra features. * `--verbose` adds state, total size of primary and all replicas, the document count, the number of primary and replica shards, and the creation date in ISO8601 format. * `--header` adds a header that shows the column names. This only occurs if `--verbose` is also selected. * `--epoch` changes the date format from ISO8601 to epoch time. If `--header` is also selected, the column header title will change to `creation_date` There are no extra columns or `--verbose` output for the `show-snapshots` command. Without `--epoch` ```sh Index State Size Docs Pri Rep Creation Timestamp logstash-2016.10.20 close 0.0B 0 5 1 2016-10-20T00:00:03Z logstash-2016.10.21 open 763.3MB 5860016 5 1 2016-10-21T00:00:03Z logstash-2016.10.22 open 759.1MB 5858450 5 1 2016-10-22T00:00:04Z logstash-2016.10.23 open 757.8MB 5857456 5 1 2016-10-23T00:00:04Z logstash-2016.10.24 open 771.5MB 5859720 5 1 2016-10-24T00:00:00Z logstash-2016.10.25 open 771.0MB 5860112 5 1 2016-10-25T00:00:01Z logstash-2016.10.27 open 658.3MB 4872830 5 1 2016-10-27T00:00:03Z logstash-2016.10.28 open 655.1MB 5237250 5 1 2016-10-28T00:00:00Z ``` With `--epoch` ```sh Index State Size Docs Pri Rep creation_date logstash-2016.10.20 close 0.0B 0 5 1 1476921603 logstash-2016.10.21 open 763.3MB 5860016 5 1 1477008003 logstash-2016.10.22 open 759.1MB 5858450 5 1 1477094404 logstash-2016.10.23 open 757.8MB 5857456 5 1 1477180804 logstash-2016.10.24 open 771.5MB 5859720 5 1 1477267200 logstash-2016.10.25 open 771.0MB 5860112 5 1 1477353601 logstash-2016.10.27 open 658.3MB 4872830 5 1 1477526403 logstash-2016.10.28 open 655.1MB 5237250 5 1 1477612800 ```   elasticsearch-curator-8.0.21/docs/reference/site-corrections.md000066400000000000000000000021401477314666200246120ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/site-corrections.html --- # Site Corrections [site-corrections] All documentation on this site allows for quick correction submission. To do this, use the "Edit" link to the right on any page. * You will need to log in with your GitHub account. * Make your corrections or additions to the documentation. * Please use the option to "Create a new branch for this commit and start a pull request." * Please make sure you have signed our [Contributor License Agreement](http://www.elastic.co/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code (even documentation corrections) without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. If you are uncomfortable with this, feel free to submit a [GitHub Issue](https://github.com/elastic/curator/issues) with your suggested correction instead. * Changes will be reviewed and merged, if acceptable. elasticsearch-curator-8.0.21/docs/reference/snapshot.md000066400000000000000000000036231477314666200231640ustar00rootroot00000000000000--- mapped_pages: - https://www.elastic.co/guide/en/elasticsearch/client/curator/current/snapshot.html --- # Snapshot [snapshot] ```yaml action: snapshot description: >- Snapshot selected indices to 'repository' with the snapshot name or name pattern in 'name'. Use all other options as assigned options: repository: ... # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' name: wait_for_completion: True max_wait: 3600 wait_interval: 10 filters: - filtertype: ... ``` ::::{note} Empty values and commented lines will result in the default value, if any, being selected. If a setting is set, but not used by a given action, it will be ignored. :::: This action will snapshot indices to the indicated [repository](/reference/option_repository.md), with a name, or name pattern, as identified by [name](/reference/option_name.md). The other options are usually okay to leave at the defaults, but feel free to read about them and change them accordingly. ## Required settings [_required_settings_12] * [repository](/reference/option_repository.md) ## Optional settings [_optional_settings_17] * [search_pattern](/reference/option_search_pattern.md) * [name](/reference/option_name.md) * [ignore_unavailable](/reference/option_ignore.md) * [include_global_state](/reference/option_include_gs.md) * [partial](/reference/option_partial.md) * [wait_for_completion](/reference/option_wfc.md) * [max_wait](/reference/option_max_wait.md) * [wait_interval](/reference/option_wait_interval.md) * [skip_repo_fs_check](/reference/option_skip_fsck.md) * [ignore_empty_list](/reference/option_ignore_empty.md) * [timeout_override](/reference/option_timeout_override.md) * [continue_if_exception](/reference/option_continue.md) * [disable_action](/reference/option_disable.md) ::::{tip} See an example of this action in an [actionfile](/reference/actionfile.md) [here](/reference/ex_snapshot.md). :::: elasticsearch-curator-8.0.21/docs/reference/toc.yml000066400000000000000000000132061477314666200223110ustar00rootroot00000000000000project: 'Curator reference' toc: - file: index.md - file: curator-ilm.md children: - file: ilm-actions.md - file: ilm-or-curator.md - file: ilm-and-curator.md - file: about.md children: - file: about-origin.md - file: about-features.md - file: about-cli.md - file: about-api.md - file: license.md - file: site-corrections.md - file: about-contributing.md - file: installation.md children: - file: pip.md - file: python-source.md - file: docker.md - file: cli.md children: - file: command-line.md - file: singleton-cli.md - file: exit-codes.md - file: configuration.md children: - file: envvars.md - file: actionfile.md - file: configfile.md - file: actions.md children: - file: alias.md - file: allocation.md - file: close.md - file: cluster_routing.md - file: cold2frozen.md - file: create_index.md - file: delete_indices.md - file: delete_snapshots.md - file: forcemerge.md - file: index_settings.md - file: open.md - file: reindex.md - file: replicas.md - file: restore.md - file: rollover.md - file: shrink.md - file: snapshot.md - file: options.md children: - file: option_allocation_type.md - file: option_allow_ilm.md - file: option_continue.md - file: option_copy_aliases.md - file: option_count.md - file: option_delay.md - file: option_delete_after.md - file: option_delete_aliases.md - file: option_skip_flush.md - file: option_disable.md - file: option_extra_settings.md - file: option_ignore_empty.md - file: option_ignore.md - file: option_include_aliases.md - file: option_include_gs.md - file: option_indices.md - file: option_key.md - file: option_max_age.md - file: option_max_docs.md - file: option_max_size.md - file: option_mns.md - file: option_max_wait.md - file: option_migration_prefix.md - file: option_migration_suffix.md - file: option_name.md - file: option_new_index.md - file: option_node_filters.md - file: option_number_of_replicas.md - file: option_number_of_shards.md - file: option_partial.md - file: option_post_allocation.md - file: option_preserve_existing.md - file: option_refresh.md - file: option_remote_certificate.md - file: option_remote_client_cert.md - file: option_remote_client_key.md - file: option_remote_filters.md - file: option_remote_url_prefix.md - file: option_rename_pattern.md - file: option_rename_replacement.md - file: option_repository.md - file: option_requests_per_second.md - file: option_request_body.md - file: option_retry_count.md - file: option_retry_interval.md - file: option_routing_type.md - file: option_search_pattern.md - file: option_setting.md - file: option_shrink_node.md - file: option_shrink_prefix.md - file: option_shrink_suffix.md - file: option_slices.md - file: option_skip_fsck.md - file: option_timeout.md - file: option_timeout_override.md - file: option_value.md - file: option_wait_for_active_shards.md - file: option_wfc.md - file: option_wait_for_rebalance.md - file: option_wait_interval.md - file: option_warn_if_no_indices.md - file: filters.md children: - file: filtertype.md - file: filtertype_age.md - file: filtertype_alias.md - file: filtertype_allocated.md - file: filtertype_closed.md - file: filtertype_count.md - file: filtertype_empty.md - file: filtertype_forcemerged.md - file: filtertype_kibana.md - file: filtertype_none.md - file: filtertype_opened.md - file: filtertype_pattern.md - file: filtertype_period.md - file: filtertype_space.md - file: filtertype_state.md - file: filter_elements.md children: - file: fe_aliases.md - file: fe_allocation_type.md - file: fe_count.md - file: fe_date_from.md - file: fe_date_from_format.md - file: fe_date_to.md - file: fe_date_to_format.md - file: fe_direction.md - file: fe_disk_space.md - file: fe_epoch.md - file: fe_exclude.md - file: fe_field.md - file: fe_intersect.md - file: fe_key.md - file: fe_kind.md - file: fe_max_num_segments.md - file: fe_pattern.md - file: fe_period_type.md - file: fe_range_from.md - file: fe_range_to.md - file: fe_reverse.md - file: fe_source.md - file: fe_state.md - file: fe_stats_result.md - file: fe_timestring.md - file: fe_threshold_behavior.md - file: fe_unit.md - file: fe_unit_count.md - file: fe_unit_count_pattern.md - file: fe_use_age.md - file: fe_value.md - file: fe_week_starts_on.md - file: examples.md children: - file: ex_alias.md - file: ex_allocation.md - file: ex_close.md - file: ex_cluster_routing.md - file: ex_create_index.md - file: ex_delete_indices.md - file: ex_delete_snapshots.md - file: ex_forcemerge.md - file: ex_index_settings.md - file: ex_open.md - file: ex_reindex.md - file: ex_replicas.md - file: ex_restore.md - file: ex_rollover.md - file: ex_shrink.md - file: ex_snapshot.md - file: faq.md children: - file: faq_doc_error.md - file: faq_partial_delete.md - file: faq_strange_chars.mdelasticsearch-curator-8.0.21/docs/snapshotlist.rst000066400000000000000000000003551477314666200223310ustar00rootroot00000000000000.. _snapshotlist: Snapshot List ############# Snapshot filtering is performed using the ``filter_`` methods of :py:class:`curator.SnapshotList` .. autoclass:: curator.SnapshotList :members: :undoc-members: :show-inheritance: elasticsearch-curator-8.0.21/docs/testing.rst000066400000000000000000000267461477314666200212670ustar00rootroot00000000000000.. _testing: Testing ####### Ensuring that code changes work with new Elasticsearch versions, ``elasticsearch-py`` Python module versions, and even new Python versions can be daunting. I've tried to make it easy to verify that changes will work. Setup Testing ************* Since ``nose`` testing basically died somewhere during Curator's early days, a new testing framework has become necessary. This is where ``pytest`` comes in. Install ``pytest`` ================== From where your ``git`` cloned or forked repository is, you need to install Curator and its dependencies, including for testing: .. code-block:: shell pip install -U '.[test]' Manually install testing dependencies ------------------------------------- These are indicated in ``pyproject.toml`` in the ``[project.optional-dependencies]`` subsection. An example is listed below: .. code-block:: [project.optional-dependencies] test = [ "requests", "pytest >=7.2.1", "pytest-cov", ] These may change with time, and this document not be updated, so double check dependencies here before running the following: .. code-block:: shell pip install -U requests pytest pytest-cov It should be simpler to run the regular method, but if you have some reason to do this manually, those are the steps. Elasticsearch as a testing dependency ===================================== .. warning:: Not using a dedicated instance or instances for testing will result in deleted data! The tests perform setup and teardown functions which will delete anything in your cluster between each test. .. important:: Integration tests will at least require Elasticsearch running on ``http://127.0.0.1:9200`` or ``TEST_ES_SERVER`` being set. The few tests that require a remote cluster to be configured will need ``REMOTE_ES_SERVER`` to be set as well. I will not cover how to install Elasticsearch locally in this document. It can be done, but it is much easier to use Docker containers instead. If you host a dedicated instance somewhere else (and it must be unsecured for testing), you can specify this as an environment variable: .. code-block:: shell TEST_ES_SERVER="http://10.0.0.1:9201" \ pytest --cov=curator --cov-report html:cov_html Additionally, four tests will be skipped if no value for ``REMOTE_ES_SERVER`` is provided. .. code-block:: shell TEST_ES_SERVER="http://10.0.0.1:9201" \ REMOTE_ES_SERVER="http://10.0.0.2:9201" \ pytest --cov=curator --cov-report html:cov_html The ``REMOTE_ES_SERVER`` must be a separate instance altogether, and the main instance must whitelist that instance for reindexing operations. If that sounds complicated, you're not wrong. There are remedies for this, and Curator comes with the necessary tools Using Docker ------------ Fortunately, Curator provides an out-of-the-box, ready to go set of scripts for setting up not only one container, but both containers necessary for testing the remote reindex functionality. .. warning:: Do not use anything but ``create.sh`` and ``destroy.sh``, or edit the ``Dockerfile.tmpl`` or ``small.options`` files unless you're actively trying to improve these scripts. These keep the Elasticsearch containers lean. Do examine ``create.sh`` to see which Elasticsearch startup flags are being used. Create Docker containers for testing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Replace ``X.Y.Z`` with an Elasticsearch version: .. code-block:: shell $ cd /path/to/curator_code/docker_test/scripts $ ./create.sh X.Y.Z Docker image curator_estest:8.6.1 not found. Building from Dockerfile... ... Waiting for Elasticsearch instances to become available... This will create both Docker containers, and will print out the ``REMOTE_ES_SERVER`` line to use: .. code-block:: shell Please select one of these environment variables to prepend your 'pytest' run: REMOTE_ES_SERVER="http://10.0.0.2:9201" Clean up Docker containers used for testing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. note:: The container names ``curator8-es-local`` and ``curator8-es-remote`` are hard coded in both scripts so that ``destroy.sh`` will clean up exactly what ``create.sh`` made. .. code-block:: shell $ cd /path/to/curator_code/docker_test/scripts $ ./destroy.sh curator8-es-local curator8-es-remote curator8-es-local curator8-es-remote Cleanup complete. The ``repo`` directory ^^^^^^^^^^^^^^^^^^^^^^ ``/path/to/curator_code/docker_test/repo`` will be created by ``create.sh`` and deleted by ``destroy.sh``. This is used for snapshot testing and will only ever contain a few files. Anything snapshotted there temporarily is cleaned by the ``teardown`` between tests. Running Tests ************* Using ``pytest`` ================ Using the value of ``REMOTE_ES_SERVER`` you got from ``create.sh``, or your own "remote" Elasticsearch instance, testing is as simple as running: .. note:: All of these examples presume that you are at the base directory of Curator's code such that the ``tests`` direcory is visible. .. code-block:: shell REMOTE_ES_SERVER="http://10.0.0.2:9201" pytest Generating coverage reports --------------------------- .. code-block:: shell $ REMOTE_ES_SERVER="http://10.0.0.2:9201" pytest --cov=curator ............................................................................ [ 12%] ............................................................................ [ 24%] ............................................................................ [ 36%] ............................................................................ [ 48%] ............................................................................ [ 60%] ............................................................................ [ 72%] ............................................................................ [ 84%] ............................................................................ [ 96%] ........................ [100%] ---------- coverage: platform darwin, python 3.11.1-final-0 ---------- Name Stmts Miss Cover ------------------------------------------------------------ curator/__init__.py 10 0 100% curator/_version.py 1 0 100% curator/actions/__init__.py 14 0 100% ... curator/validators/schemacheck.py 42 0 100% ------------------------------------------------------------ TOTAL 4023 1018 75% 475 passed in 4.92s Generating an HTML coverage report ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: shell $ REMOTE_ES_SERVER="http://10.0.0.2:9201" pytest --cov=curator --cov-reporthtml:cov_html ............................................................................ [ 12%] ............................................................................ [ 24%] ............................................................................ [ 36%] ............................................................................ [ 48%] ............................................................................ [ 60%] ............................................................................ [ 72%] ............................................................................ [ 84%] ............................................................................ [ 96%] ........................ [100%] ---------- coverage: platform darwin, python 3.11.1-final-0 ---------- Coverage HTML written to dir cov_html 475 passed in 5.24s At this point, you can view ``/path/to/curator_code/cov_html/index.html`` in your web browser. On macOS, this is as simple as running: .. code-block:: shell $ open cov_html.index.html It will open in your default browser. Testing only unit tests ----------------------- As unit tests do not require a remote Elasticsearch instance, adding the ``REMOTE_ES_SERVER`` environment variable is unnecessary: .. code-block:: shell $ pytest tests/unit You can also add ``--cov=curator`` and/or ``--cov=curator html:cov_html`` options. Testing only integration tests ------------------------------ Most integration tests do not require a remote Elasticsearch instance, so adding the ``REMOTE_ES_SERVER`` environment variable is unnecessary. Having a functional instance of Elasticsearch at ``http://127.0.0.1:9200`` or the ``TEST_ES_SERVER`` environment variable set is required. .. code-block:: shell $ pytest tests/integration You can also add ``--cov=curator`` and/or ``--cov=curator html:cov_html`` options. This will result in 4 skipped tests: .. code-block:: shell $ pytest tests/integration .......................................................................... [ 47%] ...............................s.s...ss................................... [ 94%] ......... [100%] ============================ short test summary info ============================= SKIPPED [1] tests/integration/test_reindex.py:110: REMOTE_ES_SERVER is not defined SKIPPED [1] tests/integration/test_reindex.py:275: REMOTE_ES_SERVER is not defined SKIPPED [1] tests/integration/test_reindex.py:157: REMOTE_ES_SERVER is not defined SKIPPED [1] tests/integration/test_reindex.py:206: REMOTE_ES_SERVER is not defined 153 passed, 4 skipped, 7 warnings in 217.76s (0:03:37) You can see the ``s`` in the test output. The message for each skipped test also clearly explains that ``REMOTE_ES_SERVER`` is undefined. If you were to run this with ``REMOTE_ES_SERVER``, it would clear up the skipped tests. Running specific tests ---------------------- These examples are all derived from unit tests, but the same formatting applies to integration tests as well. The path for those will just be ``tests/integration/test_file.py``. .. important:: Integration tests will at least require Elasticsearch running on ``http://127.0.0.1:9200`` or ``TEST_ES_SERVER`` being set. The few tests that require a remote cluster to be configured will need ``REMOTE_ES_SERVER`` to be set as well. Testing all tests within a given file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This will test every method of every class in ``test_file.py`` .. code-block:: shell $ pytest tests/unit/test_file.py ................................................... [100%] 51 passed in 0.32s Testing all tests within a given class ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This will test every method of class ``TestClass`` in ``test_file.py`` .. code-block:: shell $ pytest tests/unit/test_file.py::TestClass .............. [100%] 14 passed in 0.35s Testing one test within a given class ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This will test method ``test_method`` of class ``TestClass`` in ``test_file.py`` .. code-block:: shell $ pytest tests/unit/test_file.py::TestClass::test_method . [100%] 1 passed in 0.31s elasticsearch-curator-8.0.21/docs/usage.rst000066400000000000000000000060501477314666200207000ustar00rootroot00000000000000.. _usage: Using Curator ############# Compatibility ============= Elasticsearch Curator version 8 is compatible with Elasticsearch version 8.x, and supports Python versions 3.8, 3.9, 3.10, 3.11, and 3.12 officially. Installation ============ Install the ``elasticsearch-curator`` package with `pip `_:: pip install elasticsearch-curator Command-Line Usage ================== The documentation for this is on `Elastic's Website `_. Example API Usage ================= .. code-block:: python import elasticsearch8 import curator client = elasticsearch8.Elasticsearch() ilo = curator.IndexList(client, search_pattern='logstash-*', include_hidden=False) ilo.filter_by_regex(kind='prefix', value='logstash-') ilo.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=30) delete_indices = curator.DeleteIndices(ilo) delete_indices.do_action() .. TIP:: See more examples in the :doc:`Examples ` page. Logging ======= Elasticsearch Curator uses the standard `logging library`_ from Python. It inherits the ``ecs-logging`` formatting module from ``es_client``, which inherits the ``elastic_transport`` logger from ``elasticsearch8``. Clients use the ``elastic_transport`` logger to log standard activity, depending on the log level. It is recommended to use :py:class:`~.es_client.helpers.logging.set_logging` to enable logging, as this has been provided for you. This is quite simple: .. code-block:: python from es_client.helpers.logging import set_logging import logging LOG = { 'loglevel': 'INFO', 'logfile': None, 'logformat': 'default', 'blacklist': ['elastic_transport', 'urllib3'] } set_logging(LOG) logger = logging.getLogger(__name__) logger.info('Sample log message') That's it! If you were to save this file and run it at the command-line, you would see: .. code-block:: shell $ python logtest.py 2023-02-10 20:26:52,262 INFO Sample log message Log Settings ------------ Available settings for ``loglevel`` are: ``NOTSET``, ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``, and ``CRITICAL``. The setting ``logfile`` must be ``None`` or a path to a writeable file. If ``None``, it will log to ``STDOUT``. Available settings for ``logformat`` are: ``default``, ``json``, and ``ecs``. The ``ecs`` option uses `the Python ECS Log Formatter`_ and is great if you plan on ingesting your logs into Elasticsearch. Blacklisting logs by way of the ``blacklist`` setting should remain configured with the defaults (``['elastic_transport', 'urllib3']``), unless you are troubleshooting a connection issue. The ``elastic_transport`` and ``urllib3`` modules logging is exceptionally chatty for inclusion with Curator action tracing. .. _the Python ECS Log Formatter: https://www.elastic.co/guide/en/ecs-logging/python/current/index.html .. _logging library: http://docs.python.org/3.12/library/logging.htmlelasticsearch-curator-8.0.21/docs/validators.rst000066400000000000000000000004401477314666200217410ustar00rootroot00000000000000.. _validators: Validators ########## Actions ======= .. automodule:: curator.validators.actions :members: Options ======= .. automodule:: curator.validators.options :members: Filter Functions ================ .. automodule:: curator.validators.filter_functions :members: elasticsearch-curator-8.0.21/examples/000077500000000000000000000000001477314666200177275ustar00rootroot00000000000000elasticsearch-curator-8.0.21/examples/actions/000077500000000000000000000000001477314666200213675ustar00rootroot00000000000000elasticsearch-curator-8.0.21/examples/actions/alias.yml000066400000000000000000000024351477314666200232070ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: alias description: >- Alias indices older than 7 days but newer than 14 days, with a prefix of logstash- to 'last_week', remove indices older than 14 days. options: name: last_week extra_settings: timeout_override: continue_if_exception: False disable_action: True add: filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 7 exclude: - filtertype: age direction: younger timestring: '%Y.%m.%d' unit: days unit_count: 14 exclude: remove: filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 14 exclude: elasticsearch-curator-8.0.21/examples/actions/allocation.yml000066400000000000000000000015741477314666200242460ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: allocation description: >- Apply shard allocation routing to 'require' 'tag=cold' for hot/cold node setup for logstash- indices older than 3 days, based on index_creation date options: key: tag value: cold allocation_type: require wait_for_completion: False continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 3 exclude: elasticsearch-curator-8.0.21/examples/actions/close.yml000066400000000000000000000014341477314666200232210ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: close description: >- Close indices older than 30 days (based on index name), for logstash- prefixed indices. options: delete_aliases: False timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 30 exclude: elasticsearch-curator-8.0.21/examples/actions/create_index.yml000066400000000000000000000011461477314666200245460ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: create_index description: Create the index as named, with the specified extra settings. options: name: myindex extra_settings: settings: number_of_shards: 2 number_of_replicas: 1 timeout_override: continue_if_exception: False disable_action: True elasticsearch-curator-8.0.21/examples/actions/delete_indices.yml000066400000000000000000000016441477314666200250570ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: delete_indices description: >- Delete indices older than 45 days (based on index name), for logstash- prefixed indices. Ignore the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly. options: ignore_empty_list: True timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 45 exclude: elasticsearch-curator-8.0.21/examples/actions/delete_snapshots.yml000066400000000000000000000014561477314666200254640ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: delete_snapshots description: >- Delete snapshots from the selected repository older than 45 days (based on creation_date), for 'curator-' prefixed snapshots. options: repository: timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: curator- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 45 exclude: elasticsearch-curator-8.0.21/examples/actions/forcemerge.yml000066400000000000000000000020531477314666200242300ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: forcemerge description: >- forceMerge logstash- prefixed indices older than 2 days (based on index creation_date) to 2 segments per shard. Delay 120 seconds between each forceMerge operation to allow the cluster to quiesce. This action will ignore indices already forceMerged to the same or fewer number of segments per shard, so the 'forcemerged' filter is unneeded. options: max_num_segments: 2 delay: 120 timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 2 exclude: elasticsearch-curator-8.0.21/examples/actions/open.yml000066400000000000000000000016531477314666200230600ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: open description: >- Open indices older than 30 days but younger than 60 days (based on index name), for logstash- prefixed indices. options: timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days unit_count: 30 exclude: - filtertype: age source: name direction: younger timestring: '%Y.%m.%d' unit: days unit_count: 60 exclude: elasticsearch-curator-8.0.21/examples/actions/replicas.yml000066400000000000000000000014661477314666200237230ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: replicas description: >- Reduce the replica count to 0 for logstash- prefixed indices older than 10 days (based on index creation_date) options: count: 0 wait_for_completion: False timeout_override: continue_if_exception: False disable_action: True filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 10 exclude: elasticsearch-curator-8.0.21/examples/actions/restore.yml000066400000000000000000000024601477314666200235770ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: restore description: >- Restore all indices in the most recent curator-* snapshot with state SUCCESS. Wait for the restore to complete before continuing. Do not skip the repository filesystem access check. Use the other options to define the index/shard settings for the restore. options: repository: # Leaving name blank will result in restoring the most recent snapshot by age name: # Leaving indices blank will result in restoring all indices in the snapshot indices: include_aliases: False ignore_unavailable: False include_global_state: True partial: False rename_pattern: rename_replacement: extra_settings: wait_for_completion: True skip_repo_fs_check: False timeout_override: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: prefix value: curator- exclude: - filtertype: state state: SUCCESS exclude: elasticsearch-curator-8.0.21/examples/actions/shrink.yml000066400000000000000000000026511477314666200234140ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: shrink description: >- Shrink logstash indices older than 2 days on the node with the most available space, excluding the node named 'not_this_node'. Delete each source index after successful shrink, then reroute the shrunk index with the provided parameters. options: disable_action: False ignore_empty_list: True shrink_node: DETERMINISTIC node_filters: permit_masters: False exclude_nodes: ['not_this_node'] number_of_shards: 1 number_of_replicas: 1 shrink_prefix: shrink_suffix: '-shrink' copy_aliases: True delete_after: True post_allocation: allocation_type: include key: node_tag value: cold wait_for_active_shards: 1 extra_settings: settings: index.codec: best_compression wait_for_completion: True wait_for_rebalance: True wait_interval: 9 max_wait: -1 filters: - filtertype: pattern kind: prefix value: test_shrink- - filtertype: age source: creation_date direction: older unit: days unit_count: 2 elasticsearch-curator-8.0.21/examples/actions/snapshot.yml000066400000000000000000000023101477314666200237450ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" # # Also remember that all examples have 'disable_action' set to True. If you # want to use this action as a template, be sure to set this to False after # copying it. actions: 1: action: snapshot description: >- Snapshot logstash- prefixed indices older than 1 day (based on index creation_date) with the default snapshot name pattern of 'curator-%Y%m%d%H%M%S'. Wait for the snapshot to complete. Do not skip the repository filesystem access check. Use the other options to create the snapshot. options: repository: # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' name: ignore_unavailable: False include_global_state: True partial: False wait_for_completion: True skip_repo_fs_check: False timeout_override: continue_if_exception: False disable_action: False filters: - filtertype: pattern kind: prefix value: logstash- exclude: - filtertype: age source: creation_date direction: older unit: days unit_count: 1 exclude: elasticsearch-curator-8.0.21/examples/curator.yml000066400000000000000000000011761477314666200221360ustar00rootroot00000000000000--- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" elasticsearch: client: hosts: https://10.11.12.13:9200 cloud_id: bearer_auth: opaque_id: request_timeout: 60 http_compress: verify_certs: ca_certs: client_cert: client_key: ssl_assert_hostname: ssl_assert_fingerprint: ssl_version: other_settings: master_only: skip_version_test: username: password: api_key: id: api_key: logging: loglevel: INFO logfile: /path/to/file.log logformat: default blacklist: ['elastic_transport', 'urllib3'] elasticsearch-curator-8.0.21/mypy.ini000066400000000000000000000000651477314666200176110ustar00rootroot00000000000000[mypy] plugins = returns.contrib.mypy.returns_plugin elasticsearch-curator-8.0.21/post4docker.py000066400000000000000000000006321477314666200207250ustar00rootroot00000000000000#!/usr/bin/env python3 import shutil from platform import machine, system, python_version MAJOR, MINOR = tuple(python_version().split('.')[:-1]) SYSTEM = system().lower() BUILD = f'build/exe.{system().lower()}-{machine()}-{MAJOR}.{MINOR}' TARGET = 'curator_build' # Rename the path of BUILD to be generic enough for Dockerfile to get # In other words, rename it to 'curator_build' shutil.move(BUILD, TARGET) elasticsearch-curator-8.0.21/pyproject.toml000066400000000000000000000070241477314666200210300ustar00rootroot00000000000000[build-system] requires = ["hatchling"] build-backend = "hatchling.build" [project] name = "elasticsearch-curator" authors = [{ name="Elastic", email="info@elastic.co" }] dynamic = ["version"] description = "Tending your Elasticsearch indices and snapshots" license = {file = "LICENSE"} readme = "README.rst" requires-python = ">=3.8" classifiers = [ "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] keywords = [ 'elasticsearch', 'time-series', 'indexed', 'index-expiry' ] dependencies = [ "es_client==8.17.5" ] [project.optional-dependencies] test = [ "requests", "pytest >=7.2.1", "pytest-cov", ] doc = ["sphinx", "sphinx_rtd_theme"] [project.scripts] curator = "curator.cli:cli" curator_cli = "curator.singletons:curator_cli" es_repo_mgr = "curator.repomgrcli:repo_mgr_cli" [project.urls] "Homepage" = "https://github.com/elastic/curator" "Bug Tracker" = "https://github.com/elastic/curator/issues" [tool.hatch.version] path = "curator/_version.py" [tool.hatch.module] name = "curator" [tool.hatch.build] include = [ "curator/*.py", "curator/actions/*.py", "curator/cli_singletons/*.py", "curator/defaults/*.py", "curator/helpers/*.py", "curator/validators/*.py", ] [tool.hatch.build.targets.sdist] exclude = [ "dist", "docs", "docker_test", "examples", "html_docs", "tests", ] ### Docker Environment [tool.hatch.envs.docker] platforms = ["linux", "macos"] [tool.hatch.envs.docker.scripts] create = "docker_test/scripts/create.sh {args}" destroy = "docker_test/scripts/destroy.sh" ### Lint environment [tool.hatch.envs.lint] detached = true dependencies = [ 'black>=23.1.0', 'mypy>=1.0.0', 'ruff>=0.0.243', ] [tool.hatch.envs.lint.scripts] run-pyright = "pyright {args:.}" run-black = "black --quiet --check --diff {args:.}" run-ruff = "ruff check --quiet {args:.}" run-curlylint = "curlylint {args:.}" python = ["run-pyright", "run-black", "run-ruff"] templates = ["run-curlylint"] all = ["python", "templates"] [tool.pylint.format] max-line-length = "88" [tool.black] target-version = ['py38'] line-length = 88 skip-string-normalization = true include = '\.pyi?$' ### Test environment [tool.hatch.envs.test] platforms = ["linux", "macos"] dependencies = [ "requests", "pytest >=7.2.1", "pytest-cov" ] [[tool.hatch.envs.test.matrix]] python = ["3.8", "3.9", "3.10", "3.11", "3.12"] [tool.hatch.envs.test.scripts] pytest = "source docker_test/.env; pytest" pytest-cov = "source docker_test/.env; pytest --cov=curator" pytest-cov-report = "source docker_test/.env; pytest --cov=curator --cov-report=term-missing" [tool.pytest.ini_options] pythonpath = [".", "curator"] minversion = "7.2" addopts = "-ra -q" testpaths = [ "tests/unit", "tests/integration", ] [tool.distutils.build_exe] excludes = ["tcltk", "tkinter", "unittest"] zip_include_packages = ["certifi"] [tool.cxfreeze] executables = [ {script="run_curator.py", target_name="curator"}, {script="run_singleton.py", target_name="curator_cli"}, {script="run_es_repo_mgr.py", target_name="es_repo_mgr"}, ] [tool.cxfreeze.build_exe] excludes = ["tcltk", "tkinter", "unittest"] zip_include_packages = ["certifi"] elasticsearch-curator-8.0.21/pytest.ini000066400000000000000000000001451477314666200201420ustar00rootroot00000000000000[pytest] log_format = %(asctime)s %(levelname)-9s %(name)22s %(funcName)22s:%(lineno)-4d %(message)s elasticsearch-curator-8.0.21/renovate.json000066400000000000000000000001751477314666200206320ustar00rootroot00000000000000{ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "local>elastic/renovate-config" ] } elasticsearch-curator-8.0.21/run_curator.py000077500000000000000000000014371477314666200210360ustar00rootroot00000000000000#!/usr/bin/env python # pylint: disable=broad-except, no-value-for-parameter """ Wrapper for running curator from source. When used with Python 3 Curator requires the locale to be unicode. Any unicode definitions are acceptable. To set the locale to be unicode, try: $ export LC_ALL=en_US.utf8 $ curator [ARGS] Alternately, you should be able to specify the locale on the command-line: $ LC_ALL=en_US.utf8 curator [ARGS] Be sure to substitute your unicode variant for en_US.utf8 """ import sys import click from curator.cli import cli if __name__ == '__main__': try: cli() except RuntimeError as err: click.echo(f'{err}') sys.exit(1) except Exception as err: if 'ASCII' in str(err): click.echo(f'{err}') click.echo(__doc__) elasticsearch-curator-8.0.21/run_es_repo_mgr.py000077500000000000000000000015041477314666200216530ustar00rootroot00000000000000#!/usr/bin/env python # pylint: disable=broad-except, no-value-for-parameter """ Wrapper for running es_repo_mgr from source. When used with Python 3 Curator requires the locale to be unicode. Any unicode definitions are acceptable. To set the locale to be unicode, try: $ export LC_ALL=en_US.utf8 $ es_repo_mgr [ARGS] Alternately, you should be able to specify the locale on the command-line: $ LC_ALL=en_US.utf8 es_repo_mgr [ARGS] Be sure to substitute your unicode variant for en_US.utf8 """ import sys import click from curator.repomgrcli import repo_mgr_cli if __name__ == '__main__': try: repo_mgr_cli() except RuntimeError as err: click.echo(f'{err}') sys.exit(1) except Exception as err: if 'ASCII' in str(err): click.echo(f'{err}') click.echo(__doc__) elasticsearch-curator-8.0.21/run_singleton.py000077500000000000000000000015011477314666200213510ustar00rootroot00000000000000#!/usr/bin/env python # pylint: disable=no-value-for-parameter, broad-except """ Wrapper for running singletons from source. When used with Python 3 Curator requires the locale to be unicode. Any unicode definitions are acceptable. To set the locale to be unicode, try: $ export LC_ALL=en_US.utf8 $ curator_cli [ARGS] Alternately, you should be able to specify the locale on the command-line: $ LC_ALL=en_US.utf8 curator_cli [ARGS] Be sure to substitute your unicode variant for en_US.utf8 """ import sys import click from curator.singletons import curator_cli if __name__ == '__main__': try: curator_cli() except RuntimeError as err: click.echo(f'{err}') sys.exit(1) except Exception as err: if 'ASCII' in str(err): click.echo(f'{err}') click.echo(__doc__) elasticsearch-curator-8.0.21/tests/000077500000000000000000000000001477314666200172535ustar00rootroot00000000000000elasticsearch-curator-8.0.21/tests/__init__.py000066400000000000000000000000001477314666200213520ustar00rootroot00000000000000elasticsearch-curator-8.0.21/tests/integration/000077500000000000000000000000001477314666200215765ustar00rootroot00000000000000elasticsearch-curator-8.0.21/tests/integration/__init__.py000066400000000000000000000256431477314666200237210ustar00rootroot00000000000000"""Test setup""" # pylint: disable=C0115, C0116 import logging import os import random import shutil import string import sys import tempfile import time import json import warnings from datetime import timedelta, datetime, date, timezone from subprocess import Popen, PIPE from unittest import SkipTest, TestCase from elasticsearch8 import Elasticsearch from elasticsearch8.exceptions import ConnectionError as ESConnectionError from elasticsearch8.exceptions import ElasticsearchWarning, NotFoundError from click import testing as clicktest from es_client.helpers.utils import get_version from curator.cli import cli from . import testvars client = None DATEMAP = { 'months': '%Y.%m', 'weeks': '%Y.%W', 'days': '%Y.%m.%d', 'hours': '%Y.%m.%d.%H', } HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') def random_directory(): dirname = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8) ) directory = tempfile.mkdtemp(suffix=dirname) if not os.path.exists(directory): os.makedirs(directory) return directory def get_client(): # pylint: disable=global-statement, invalid-name global client if client is not None: return client client = Elasticsearch(hosts=HOST, request_timeout=300) # wait for yellow status for _ in range(100): time.sleep(0.1) try: # pylint: disable=E1123 client.cluster.health(wait_for_status='yellow') return client except ESConnectionError: continue # timeout raise SkipTest("Elasticsearch failed to start.") def setup(): get_client() class Args(dict): def __getattr__(self, att_name): return self.get(att_name, None) class CuratorTestCase(TestCase): def setUp(self): super(CuratorTestCase, self).setUp() self.logger = logging.getLogger('CuratorTestCase.setUp') self.client = get_client() args = {} args['HOST'] = HOST args['time_unit'] = 'days' args['prefix'] = 'logstash-' self.args = args # dirname = ''.join(random.choice(string.ascii_uppercase + string.digits) # for _ in range(8)) # This will create a psuedo-random temporary directory on the machine # which runs the unit tests, but NOT on the machine where elasticsearch # is running. This means tests may fail if run against remote instances # unless you explicitly set `self.args['location']` to a proper spot # on the target machine. # self.args['location'] = random_directory() nodesinfo = self.client.nodes.info() nodename = list(nodesinfo['nodes'].keys())[0] if 'repo' in nodesinfo['nodes'][nodename]['settings']['path']: if isinstance( nodesinfo['nodes'][nodename]['settings']['path']['repo'], list ): self.args['location'] = nodesinfo['nodes'][nodename]['settings'][ 'path' ]['repo'][0] else: self.args['location'] = nodesinfo['nodes'][nodename]['settings'][ 'path' ]['repo'] else: # Use a random directory if repo is not specified, but log it self.logger.warning('path.repo is not configured!') self.args['location'] = random_directory() self.args['configdir'] = random_directory() self.args['configfile'] = os.path.join(self.args['configdir'], 'curator.yml') self.args['actionfile'] = os.path.join(self.args['configdir'], 'actions.yml') self.args['repository'] = 'test_repository' # if not os.path.exists(self.args['location']): # os.makedirs(self.args['location']) self.logger.debug('setUp completed...') self.runner = clicktest.CliRunner() self.runner_args = [ '--config', self.args['configfile'], self.args['actionfile'], ] self.result = None def get_version(self): return get_version(self.client) def tearDown(self): self.logger = logging.getLogger('CuratorTestCase.tearDown') self.logger.debug('tearDown initiated...') # re-enable shard allocation for next tests enable_allocation = json.loads('{"cluster.routing.allocation.enable":null}') self.client.cluster.put_settings(transient=enable_allocation) self.delete_repositories() # 8.0 removes our ability to purge with wildcards... # ElasticsearchWarning: this request accesses system indices: [.tasks], # but in a future major version, direct access to system indices will be # prevented by default warnings.filterwarnings("ignore", category=ElasticsearchWarning) indices = list( self.client.indices.get(index="*", expand_wildcards='open,closed').keys() ) if len(indices) > 0: # ElasticsearchWarning: this request accesses system indices: [.tasks], # but in a future major version, direct access to system indices will be # prevented by default warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.delete(index=','.join(indices)) for path_arg in ['location', 'configdir']: if os.path.exists(self.args[path_arg]): shutil.rmtree(self.args[path_arg]) def parse_args(self): return Args(self.args) def create_indices(self, count, unit=None, ilm_policy=None): now = datetime.now(timezone.utc) unit = unit if unit else self.args['time_unit'] fmt = DATEMAP[unit] if not unit == 'months': step = timedelta(**{unit: 1}) for _ in range(count): self.create_index( self.args['prefix'] + now.strftime(fmt), wait_for_yellow=False, ilm_policy=ilm_policy, ) now -= step else: # months now = date.today() d = date(now.year, now.month, 1) self.create_index( self.args['prefix'] + now.strftime(fmt), wait_for_yellow=False, ilm_policy=ilm_policy, ) for _ in range(1, count): if d.month == 1: d = date(d.year - 1, 12, 1) else: d = date(d.year, d.month - 1, 1) self.create_index( self.args['prefix'] + datetime(d.year, d.month, 1).strftime(fmt), wait_for_yellow=False, ilm_policy=ilm_policy, ) # pylint: disable=E1123 self.client.cluster.health(wait_for_status='yellow') def wfy(self): # pylint: disable=E1123 self.client.cluster.health(wait_for_status='yellow') def create_index( self, name, shards=1, wait_for_yellow=True, ilm_policy=None, wait_for_active_shards=1, ): request_body = {'index': {'number_of_shards': shards, 'number_of_replicas': 0}} if ilm_policy is not None: request_body['index']['lifecycle'] = {'name': ilm_policy} # ElasticsearchWarning: index name [.shouldbehidden] starts with a dot '.', # in the next major version, index names starting with a dot are reserved # for hidden indices and system indices warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.create( index=name, settings=request_body, wait_for_active_shards=wait_for_active_shards, ) if wait_for_yellow: self.wfy() def add_docs(self, idx): for i in ["1", "2", "3"]: self.client.create(index=idx, id=i, document={"doc" + i: 'TEST DOCUMENT'}) # This should force each doc to be in its own segment. # pylint: disable=E1123 self.client.indices.flush(index=idx, force=True) self.client.indices.refresh(index=idx) def create_snapshot(self, name, csv_indices): self.create_repository() self.client.snapshot.create( repository=self.args['repository'], snapshot=name, ignore_unavailable=False, include_global_state=True, partial=False, indices=csv_indices, wait_for_completion=True, ) def delete_snapshot(self, name): try: self.client.snapshot.delete( repository=self.args['repository'], snapshot=name ) except NotFoundError: pass def create_repository(self): request_body = {'type': 'fs', 'settings': {'location': self.args['location']}} self.client.snapshot.create_repository( name=self.args['repository'], body=request_body ) def delete_repositories(self): result = [] try: result = self.client.snapshot.get_repository(name='*') except NotFoundError: pass for repo in result: try: cleanup = self.client.snapshot.get(repository=repo, snapshot='*') # pylint: disable=broad-except except Exception: cleanup = {'snapshots': []} for listitem in cleanup['snapshots']: self.delete_snapshot(listitem['snapshot']) self.client.snapshot.delete_repository(name=repo) def close_index(self, name): self.client.indices.close(index=name) def write_config(self, fname, data): with open(fname, 'w', encoding='utf-8') as fhandle: fhandle.write(data) def get_runner_args(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) runner = os.path.join(os.getcwd(), 'run_singleton.py') return [sys.executable, runner] def run_subprocess(self, args, logname='subprocess'): local_logger = logging.getLogger(logname) p = Popen(args, stderr=PIPE, stdout=PIPE) stdout, stderr = p.communicate() local_logger.debug('STDOUT = %s', stdout.decode('utf-8')) local_logger.debug('STDERR = %s', stderr.decode('utf-8')) return p.returncode def invoke_runner(self, dry_run=False): if dry_run: self.result = self.runner.invoke( cli, [ '--config', self.args['configfile'], '--dry-run', self.args['actionfile'], ], ) return self.result = self.runner.invoke(cli, self.runner_args) def invoke_runner_alt(self, **kwargs): myargs = [] if kwargs: for key, value in kwargs.items(): myargs.append(f'--{key}') myargs.append(value) myargs.append(self.args['actionfile']) self.result = self.runner.invoke(cli, myargs) elasticsearch-curator-8.0.21/tests/integration/test_alias.py000066400000000000000000000275311477314666200243100ustar00rootroot00000000000000"""Test Alias action""" # pylint: disable=C0115, C0116, invalid-name import os import logging from datetime import datetime, timedelta, timezone import pytest from elasticsearch8.exceptions import NotFoundError from . import CuratorTestCase from . import testvars LOGGER = logging.getLogger('test_alias') HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestActionFileAlias(CuratorTestCase): def test_add_only(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_only.format(alias) ) self.create_index('my_index') self.create_index('dummy') self.client.indices.put_alias(index='dummy', name=alias) self.invoke_runner() assert 2 == len(self.client.indices.get_alias(name=alias)) def test_add_only_with_extra_settings(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_only_with_extra_settings.format(alias), ) self.create_index('my_index') self.invoke_runner() expected = { 'my_index': { 'aliases': {'testalias': {'filter': {'term': {'user': 'kimchy'}}}} } } assert expected == self.client.indices.get_alias(name=alias) def test_alias_remove_only(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_remove_only.format(alias) ) idx1, idx2 = ('my_index', 'dummy') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx2, name=alias) self.invoke_runner() expected = {idx2: {'aliases': {}}, idx1: {'aliases': {}}} assert expected == self.client.indices.get_alias(index=f'{idx1},{idx2}') def test_add_only_skip_closed(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_only.format(alias) ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx2) self.client.indices.close(index=idx2, wait_for_active_shards=0) self.create_index(idx1) self.client.indices.put_alias(index=idx1, name=alias) self.invoke_runner() assert 2 == len(self.client.indices.get_alias(name=alias)) def test_add_and_remove(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_remove.format(alias) ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias) self.invoke_runner() expected = {idx2: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) def test_add_and_remove_datemath(self): alias = '' _ = (datetime.now(timezone.utc) - timedelta(days=1)).strftime('%Y.%m.%d') alias_parsed = f"testalias-{_}" self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_remove.format(alias) ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias_parsed) self.invoke_runner() expected = {idx2: {'aliases': {alias_parsed: {}}}} assert expected == self.client.indices.get_alias(name=alias_parsed) def test_add_with_empty_remove(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_with_empty_remove.format(alias) ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias) self.invoke_runner() expected = {idx1: {'aliases': {alias: {}}}, idx2: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) def test_remove_with_empty_add(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_remove_with_empty_add.format(alias) ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=f'{idx1},{idx2}', name=alias) self.invoke_runner() expected = {idx2: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) def test_add_with_empty_list(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_remove_empty.format(alias, 'du', 'rickroll'), ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias) self.invoke_runner() expected = {idx1: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) def test_remove_with_empty_list(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_add_remove_empty.format(alias, 'rickroll', 'my'), ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias) self.invoke_runner() expected = {idx1: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) def test_remove_index_not_in_alias(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_remove_index_not_there.format(alias, 'my'), ) idx1, idx2 = ('my_index1', 'my_index2') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias) self.invoke_runner() with pytest.raises(NotFoundError): self.client.indices.get_alias(name=alias) assert 0 == self.result.exit_code def test_no_add_remove(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.alias_no_add_remove.format(alias) ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert 1 == self.result.exit_code def test_no_alias(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.alias_no_alias) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert 1 == self.result.exit_code def test_extra_options(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('alias') ) self.invoke_runner() assert 1 == self.result.exit_code def test_add_and_remove_sorted(self): alias = 'testalias' alias_add_remove = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add/remove specified indices from designated alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' add:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: dum\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], alias_add_remove.format(alias)) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx2, name=alias) self.invoke_runner() expected = {idx1: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) class TestCLIAlias(CuratorTestCase): """Test CLI Singleton Alias action""" def test_add_and_remove_alias(self): """test_add_and_remove_alias""" alias = 'testalias' idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx2, name=alias) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'alias', '--name', alias, '--add', '{"filtertype":"pattern","kind":"prefix","value":"dum"}', '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] assert 0 == self.run_subprocess(args) expected = {idx1: {'aliases': {alias: {}}}} assert expected == self.client.indices.get_alias(name=alias) def test_warn_if_no_indices(self): """test_warn_if_no_indices""" alias = 'testalias' idx1, idx2 = ('dummy1', 'dummy2') self.create_index(idx1) self.create_index(idx2) self.client.indices.put_alias(index=idx1, name=alias) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'alias', '--name', alias, '--add', '{"filtertype":"none"}', '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my"}', '--warn_if_no_indices', ] LOGGER.debug('ARGS = %s', args) assert 0 == self.run_subprocess(args) expected = {idx1: {'aliases': {alias: {}}}, idx2: {'aliases': {alias: {}}}} assert expected == dict(self.client.indices.get_alias(name=alias)) def test_exit_1_on_empty_list(self): """test_exit_1_on_empty_list""" alias = 'testalias' self.create_index('dummy') args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'alias', '--name', alias, '--add', '{"filtertype":"pattern","kind":"prefix","value":"dum","exclude":false}', '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my","exclude":false}', ] assert 1 == self.run_subprocess( args, logname='TestCLIAlias.test_exit_1_on_empty_list' ) elasticsearch-curator-8.0.21/tests/integration/test_allocation.py000066400000000000000000000171641477314666200253450ustar00rootroot00000000000000"""Test Allocation action""" # pylint: disable=C0115, C0116, invalid-name import os from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') TIEREDROUTING = {'allocation': {'include': {'_tier_preference': 'data_content'}}} KEY = 'tag' VALUE = 'value' class TestActionFileAllocation(CuratorTestCase): def test_include(self): alloc = 'include' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, VALUE, alloc, False), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert ( VALUE == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ]['allocation'][alloc][KEY] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) def test_require(self): alloc = 'require' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, VALUE, alloc, False), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert ( VALUE == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ]['allocation'][alloc][KEY] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) def test_exclude(self): alloc = 'exclude' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, VALUE, alloc, False), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert ( VALUE == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ]['allocation'][alloc][KEY] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) def test_remove_exclude_with_none_value(self): empty = '' # EMPTYVALUE alloc = 'exclude' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, empty, alloc, False), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) # Put a setting in place before we start the test. self.client.indices.put_settings( index=idx1, settings={f'index.routing.allocation.{alloc}.{KEY}': 'bar'} ) # Ensure we _have_ it here first. assert ( 'bar' == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ]['allocation'][alloc][KEY] ) self.invoke_runner() assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) def test_invalid_allocation_type(self): alloc = 'invalid' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, VALUE, alloc, False), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert 1 == self.result.exit_code def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('allocation') ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert 1 == self.result.exit_code def test_skip_closed(self): alloc = 'include' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, VALUE, alloc, False), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.client.indices.close(index=idx1, wait_for_active_shards=0) self.create_index(idx2) self.invoke_runner() assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) def test_wait_for_completion(self): alloc = 'require' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_test.format(KEY, VALUE, alloc, True), ) idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) self.invoke_runner() assert ( VALUE == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ]['allocation'][alloc][KEY] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) class TestCLIAllocation(CuratorTestCase): def test_include(self): alloc = 'include' idx1, idx2 = ('my_index', 'not_my_index') self.create_index(idx1) self.create_index(idx2) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'allocation', '--key', KEY, '--value', VALUE, '--allocation_type', alloc, '--wait_for_completion', '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] assert 0 == self.run_subprocess(args, logname='TestCLIAllocation.test_include') assert ( VALUE == self.client.indices.get_settings(index=idx1)[idx1]['settings']['index'][ 'routing' ]['allocation'][alloc][KEY] ) assert ( TIEREDROUTING == self.client.indices.get_settings(index=idx2)[idx2]['settings']['index'][ 'routing' ] ) elasticsearch-curator-8.0.21/tests/integration/test_cli.py000066400000000000000000000151551477314666200237650ustar00rootroot00000000000000"""Test CLI functionality""" # pylint: disable=C0115, C0116, invalid-name import os from curator.exceptions import ConfigurationError from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestCLIMethods(CuratorTestCase): def test_bad_client_config(self): self.create_indices(10) self.write_config( self.args['configfile'], testvars.bad_client_config.format(HOST) ) self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner() assert 1 == self.result.exit_code def test_cli_client_config(self): self.create_indices(10) self.write_config( self.args['configfile'], testvars.bad_client_config.format(HOST) ) self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner_alt( hosts='http://127.0.0.1:9200', loglevel='DEBUG', logformat='ecs' ) assert 0 == self.result.exit_code def test_cli_unreachable_cloud_id(self): self.create_indices(10) self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner_alt(cloud_id='abc:def', username='user', password='pass') assert 1 == self.result.exit_code def test_no_config(self): # This test checks whether localhost:9200 is provided if no hosts or # port are in the configuration. But in testing, sometimes # TEST_ES_SERVER is set to something other than localhost:9200. In this # case, the test here would fail. The if statement at the end now # compensates. See https://github.com/elastic/curator/issues/843 localtest = False if HOST == 'http://127.0.0.1:9200': localtest = True self.create_indices(10) self.write_config(self.args['configfile'], '---\n') # Empty YAML file. self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner() if localtest: assert 0 == self.result.exit_code else: assert -1 == self.result.exit_code def test_no_logging_config(self): self.create_indices(10) self.write_config( self.args['configfile'], testvars.no_logging_config.format(HOST) ) self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner() assert 0 == self.result.exit_code def test_logging_none(self): self.create_indices(10) self.write_config( self.args['configfile'], testvars.none_logging_config.format(HOST) ) self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner() assert 0 == self.result.exit_code def test_invalid_action(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.optionless_proto.format('invalid_action') ) self.invoke_runner() assert 1 == self.result.exit_code def test_action_is_none(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.optionless_proto.format(' ') ) self.invoke_runner() assert isinstance(self.result.exception, ConfigurationError) def test_no_action(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.actionless_proto) self.invoke_runner() assert isinstance(self.result.exception, ConfigurationError) def test_dry_run(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' ), ) self.invoke_runner(dry_run=True) assert 10 == len(get_indices(self.client)) def test_action_disabled(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.disabled_proto.format('close', 'delete_indices'), ) self.invoke_runner() assert 0 == len(get_indices(self.client)) assert 0 == self.result.exit_code # I'll have to think up another way to create an exception. # The exception that using "alias" created, a missing argument, # is caught too early for this to actually run the test now :/ def test_continue_if_exception(self): name = 'log1' self.create_index(name) self.create_index('log2') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.continue_proto.format(name, True, 'delete_indices', False), ) self.invoke_runner() assert 0 == len(get_indices(self.client)) assert 0 == self.result.exit_code def test_continue_if_exception_false(self): name = 'log1' self.create_index(name) self.create_index('log2') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.continue_proto.format(name, False, 'delete_indices', False), ) self.invoke_runner() assert 2 == len(get_indices(self.client)) assert 1 == self.result.exit_code def test_no_options_in_action(self): self.create_indices(10) self.create_index('my_index') # Added for the ILM filter's sake self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.no_options_proto.format('delete_indices') ) self.invoke_runner(dry_run=True) assert 0 == self.result.exit_code elasticsearch-curator-8.0.21/tests/integration/test_close.py000066400000000000000000000143451477314666200243230ustar00rootroot00000000000000"""Integration tests of the Close action class""" # pylint: disable=C0115, C0116, invalid-name import os from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') MET = 'metadata' IDC = 'indices' STA = 'state' def get_state(client, data): """Return cluster state data""" return client.cluster.state(index=data, metric=MET) class TestActionFileClose(CuratorTestCase): def test_close_opened(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.optionless_proto.format('close') ) idx1, idx2 = ('dummy', 'my_index') indices = f'{idx1},{idx2}' self.create_index(idx1) self.create_index(idx2) self.invoke_runner() state = get_state(self.client, indices) assert 'close' != state[MET][IDC][idx1][STA] assert 'close' == state[MET][IDC][idx2][STA] def test_close_closed(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.optionless_proto.format('close') ) idx1, idx2 = ('dummy', 'my_index') indices = f'{idx1},{idx2}' self.create_index(idx2) # pylint: disable=E1123 self.client.indices.close( index=idx2, ignore_unavailable=True, wait_for_active_shards=0 ) self.create_index(idx1) self.invoke_runner() state = get_state(self.client, indices) assert 'close' != state[MET][IDC][idx1][STA] assert 'close' == state[MET][IDC][idx2][STA] def test_close_delete_aliases(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.close_delete_aliases) # Create aliases first alias = 'testalias' idxtuple = ('dummy', 'my_index', 'my_other') idx1, idx2, idx3 = idxtuple indices = None for idx in idxtuple: self.create_index(idx) indices = f'{indices},{idx}' if indices else f'{idx}' self.client.indices.put_alias(index=f'{idx1},{idx2}', name=alias) precheck = {idx1: {"aliases": {alias: {}}}, idx2: {"aliases": {alias: {}}}} assert precheck == self.client.indices.get_alias(name=alias) # Now close `index` with delete_aliases=True (dummy stays open) self.invoke_runner() state = get_state(self.client, indices) for idx in (idx2, idx3): assert 'close' == state[MET][IDC][idx][STA] # Now open the indices and verify that the alias is still gone. self.client.indices.open(index=idx2) assert {idx1: {"aliases": {alias: {}}}} == self.client.indices.get_alias( name=alias ) def test_close_skip_flush(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.close_skip_flush) idx1, idx2 = ('dummy', 'my_index') indices = f'{idx1},{idx2}' self.create_index(idx1) # Disable shard allocation to make my_index go red disable_allocation = {"cluster.routing.allocation.enable": "none"} self.client.cluster.put_settings(transient=disable_allocation) self.create_index(idx2, wait_for_yellow=False, wait_for_active_shards=0) self.invoke_runner() state = get_state(self.client, indices) assert 'close' != state[MET][IDC][idx1][STA] assert 'close' == state[MET][IDC][idx2][STA] def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('close') ) idx1, idx2 = ('dummy', 'my_index') indices = f'{idx1},{idx2}' self.create_index(idx1) self.create_index(idx2) self.invoke_runner() state = get_state(self.client, indices) for idx in (idx1, idx2): assert 'close' != state[MET][IDC][idx][STA] assert 1 == self.result.exit_code class TestCLIClose(CuratorTestCase): def test_close_delete_aliases(self): # Create aliases first alias = 'testalias' idx1, idx2, idx3 = ('my_index', 'dummy', 'my_other') indices = f'{idx1},{idx2},{idx3}' self.create_index(idx1) self.create_index(idx2) self.create_index(idx3) self.client.indices.put_alias(index=f'{idx1},{idx2}', name=alias) precheck = {idx1: {"aliases": {alias: {}}}, idx2: {"aliases": {alias: {}}}} assert precheck == self.client.indices.get_alias(name=alias) # Now close `index` with delete_aliases=True idx2 stays open) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'close', '--delete_aliases', '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] assert 0 == self.run_subprocess( args, logname='TestCLIClose.test_close_delete_aliases' ) state = get_state(self.client, indices) for idx in (idx1, idx3): assert 'close' == state[MET][IDC][idx][STA] # Now open the indices and verify that the alias is still gone. self.client.indices.open(index=idx1) assert {idx2: {"aliases": {alias: {}}}} == self.client.indices.get_alias( name=alias ) def test_close_skip_flush(self): args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'close', '--skip_flush', '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] idx1, idx2 = ('my_index', 'dummy') indices = f'{idx1},{idx2}' self.create_index(idx1) self.create_index(idx2) assert 0 == self.run_subprocess( args, logname='TestCLIClose.test_close_skip_flush' ) state = get_state(self.client, indices) assert 'close' == state[MET][IDC][idx1][STA] elasticsearch-curator-8.0.21/tests/integration/test_clusterrouting.py000066400000000000000000000022371477314666200263040ustar00rootroot00000000000000"""Test the Cluster Routing Action""" # pylint: disable=C0115, C0116, invalid-name import os from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestCLIClusterRouting(CuratorTestCase): def test_allocation_all(self): routing_type = 'allocation' value = 'all' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.cluster_routing_test.format(routing_type, value), ) self.create_index('my_index') self.create_index('not_my_index') self.invoke_runner() assert testvars.CRA_all == self.client.cluster.get_settings() def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('cluster_routing'), ) self.create_index('my_index') self.create_index('not_my_index') self.invoke_runner() assert 1 == self.result.exit_code elasticsearch-curator-8.0.21/tests/integration/test_cold2frozen.py000066400000000000000000000064621477314666200254460ustar00rootroot00000000000000""" Test cold2frozen functionality This cannot work with the open-source release, so any testing must be manual """ # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long # import os # from time import sleep # from curator import IndexList # from curator.actions import Cold2Frozen, Snapshot # from curator.exceptions import ( # CuratorException, FailedExecution, SearchableSnapshotException) # from curator.helpers.getters import get_snapshot # from . import CuratorTestCase # from . import testvars # HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') # ' repository: {0}\n' # ' - filtertype: {1}\n' # ' source: {2}\n' # ' direction: {3}\n' # ' timestring: {4}\n' # ' unit: {5}\n' # ' unit_count: {6}\n' # ' epoch: {7}\n') # class TestActionFileCold2Frozen(CuratorTestCase): # SNAP = 'cold2frozen' # ORIG = 'myindex-000001' # ALIAS = 'myindex' # AL1 = [{'add': {'index': ORIG, 'alias': ALIAS}}] # AL2 = [ # {'remove': {'index': ORIG, 'alias': ALIAS}}, # {'add': {'index': f'restored-{ORIG}', 'alias': ALIAS}} # ] # COLD = {'routing': {'allocation': {'include': {'_tier_preference': 'data_cold'}}}} # def test_cold2frozen1(self): # ### Create snapshots to delete and verify them # self.create_repository() # self.add_docs(self.ORIG) # # Assign an alias # self.client.indices.update_aliases(actions=self.AL1) # # Forcemerge the index down # self.client.indices.forcemerge(index=self.ORIG, max_num_segments=1) # # Get an indexlist object # ilo = IndexList(self.client) # # Snapshot the single index in it # snp = Snapshot( # ilo, repository=self.args['repository'], name=self.SNAP, # wait_interval=0.5 # ) # snp.do_action() # # Verify the snapshot # snapshot = get_snapshot(self.client, self.args['repository'], self.SNAP) # assert 1 == len(snapshot['snapshots']) # sleep(2.0) # # Mount the index in the snapshot in the cold tier # self.client.searchable_snapshots.mount( # repository=self.args['repository'], snapshot=self.SNAP, index=self.ORIG, # renamed_index=f'restored-{self.ORIG}', index_settings=self.COLD, # storage='full_copy') # # Update the new index to have the aliases of the original one # self.client.indices.update_aliases(actions=self.AL2) # # Delete the original index # self.client.indices.delete(index=self.ORIG) # # Get a new indexlist object # ilo = IndexList(self.client) # # Verify it only has one index in it # assert 1 == len(ilo.indices) # # ...and that the index is our restored- cold-tier mount # settings = self.client.indices.get( # index=f'restored-{self.ORIG}')[f'restored-{self.ORIG}'] # # ...and that it has our alias # assert {self.ALIAS: {}} == settings['aliases'] # # ...and that it has the appropriate settings # snapchk = settings['settings']['index']['store']['snapshot'] # assert self.SNAP == snapchk['snapshot_name'] # assert self.ORIG == snapchk['index_name'] # assert self.args['repository'] == snapchk['repository_name'] elasticsearch-curator-8.0.21/tests/integration/test_count_pattern.py000066400000000000000000000072271477314666200261040ustar00rootroot00000000000000"""Test count pattern""" # pylint: disable=C0115, C0116, invalid-name import os from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') TIEREDROUTING = {'allocation': {'include': {'_tier_preference': 'data_content'}}} DELETE_COUNT_PATTERN = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: count\n' ' pattern: {0}\n' ' use_age: {1}\n' ' source: {2}\n' ' timestring: {3}\n' ' reverse: {4}\n' ' count: {5}\n' ) class TestCLICountPattern(CuratorTestCase): def test_match_proper_indices(self): for i in range(1, 4): self.create_index(f'a-{i}') for i in range(4, 7): self.create_index(f'b-{i}') for i in range(5, 9): self.create_index(f'c-{i}') self.create_index('not_a_match') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], DELETE_COUNT_PATTERN.format( r'^(a|b|c)-\d$', 'false', 'name', '\'%Y.%m.%d\'', 'true', 1 ), ) self.invoke_runner() indices = sorted(list(self.client.indices.get(index='*'))) assert ['a-3', 'b-6', 'c-8', 'not_a_match'] == indices def test_match_proper_indices_by_age(self): self.create_index('a-2017.10.01') self.create_index('a-2017.10.02') self.create_index('a-2017.10.03') self.create_index('b-2017.09.01') self.create_index('b-2017.09.02') self.create_index('b-2017.09.03') self.create_index('not_a_match') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], DELETE_COUNT_PATTERN.format( r'^(a|b)-\d{4}\.\d{2}\.\d{2}$', 'true', 'name', '\'%Y.%m.%d\'', 'true', 1, ), ) self.invoke_runner() indices = sorted(list(self.client.indices.get(index='*'))) assert ['a-2017.10.03', 'b-2017.09.03', 'not_a_match'] == indices def test_count_indices_by_age_same_age(self): key = 'tag' value = 'value' alloc = 'include' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.allocation_count_test.format(key, value, alloc, False), ) idx1, idx2 = ('c-2017.10.03', 'd-2017.10.03') idxlist = [ 'a-2017.10.01', 'a-2017.10.02', 'a-2017.10.03', 'b-2017.10.01', 'b-2017.10.02', 'b-2017.10.03', 'c-2017.10.01', 'c-2017.10.02', 'd-2017.10.01', 'd-2017.10.02', ] for idx in idxlist: self.create_index(idx) self.create_index(idx1) self.create_index(idx2) self.invoke_runner() response = self.client.indices.get_settings(index='*') for idx in (idx1, idx2): assert ( value == response[idx]['settings']['index']['routing']['allocation'][alloc][ key ] ) for idx in idxlist: assert TIEREDROUTING == response[idx]['settings']['index']['routing'] elasticsearch-curator-8.0.21/tests/integration/test_create_index.py000066400000000000000000000120111477314666200256340ustar00rootroot00000000000000"""Test the Create Index action""" # pylint: disable=C0115, C0116, invalid-name import os from curator import IndexList from curator.helpers.date_ops import parse_date_pattern from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestCLICreateIndex(CuratorTestCase): def test_plain(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.create_index.format('testing') ) assert not get_indices(self.client) self.invoke_runner() self.assertEqual(['testing'], get_indices(self.client)) assert ['testing'] == get_indices(self.client) def test_with_extra_settings(self): idx = 'testing' alias = 'aliasname' mapkey1 = 'meep' mapval1 = 'integer' mapkey2 = 'beep' mapval2 = 'keyword' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.create_index_with_extra_settings.format( idx, alias, mapkey1, mapval1, mapkey2, mapval2 ), ) assert not get_indices(self.client) self.invoke_runner() ilo = IndexList(self.client) ilo.get_index_settings() aliases = self.client.indices.get_alias(name=alias) mapping = self.client.indices.get_mapping(index=idx) assert [idx] == ilo.indices assert '1' == ilo.index_info[idx]['number_of_shards'] assert '0' == ilo.index_info[idx]['number_of_replicas'] assert mapping[idx]['mappings']['properties'][mapkey1] == {'type': mapval1} assert mapping[idx]['mappings']['properties'][mapkey2] == {'type': mapval2} assert aliases[idx]['aliases'] == {alias: {'is_write_index': True}} def test_with_strftime(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.create_index.format('testing-%Y.%m.%d') ) assert not get_indices(self.client) idx = parse_date_pattern('testing-%Y.%m.%d') self.invoke_runner() assert [idx] == get_indices(self.client) def test_with_date_math(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.create_index.format('') ) assert not get_indices(self.client) idx = parse_date_pattern('testing-%Y.%m.%d') self.invoke_runner() assert [idx] == get_indices(self.client) def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('create_index'), ) self.invoke_runner() assert not get_indices(self.client) assert 1 == self.result.exit_code def test_already_existing_fail(self): idx = 'testing' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.create_index.format(idx)) self.create_index(idx) self.invoke_runner() assert [idx] == get_indices(self.client) assert 1 == self.result.exit_code def test_already_existing_pass(self): config = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Create index as named"\n' ' action: create_index\n' ' options:\n' ' name: {0}\n' ' ignore_existing: true\n' ) idx = 'testing' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], config.format(idx)) self.create_index(idx) self.invoke_runner() assert [idx] == get_indices(self.client) assert 0 == self.result.exit_code def test_incorrect_mapping_fail_with_propper_error(self): config = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Create index as named"\n' ' action: create_index\n' ' options:\n' ' name: {0}\n' ' extra_settings:\n' ' mappings:\n' ' properties:\n' ' name: ["test"]\n' ) idx = 'testing' self.write_config( self.args['configfile'], testvars.none_logging_config.format(HOST) ) self.write_config(self.args['actionfile'], config.format(idx)) self.invoke_runner() assert not get_indices(self.client) assert 'mapper_parsing_exception' in self.result.stdout assert 1 == self.result.exit_code elasticsearch-curator-8.0.21/tests/integration/test_datemath.py000066400000000000000000000041231477314666200247760ustar00rootroot00000000000000"""Test date math with indices""" # pylint: disable=C0115, C0116, invalid-name from datetime import datetime, timedelta, timezone from unittest import SkipTest from curator.helpers.date_ops import parse_datemath from . import CuratorTestCase class TestParseDateMath(CuratorTestCase): colonblow = ( 'Date math expressions with colons are not supported in Elasticsearch ' 'versions 8.0 - 8.6' ) def test_assorted_datemaths(self): now = datetime.now(timezone.utc) oneday = timedelta(days=1) ymd = '%Y.%m.%d' for test_string, expected in [ ( '', f"prefix-{now.strftime(ymd)}-suffix", ), ( '', f"prefix-{(now-oneday).strftime(ymd)}", ), ( '<{now+1d/d}>', f"{(now+oneday).strftime(ymd)}", ), ( '<{now+1d/d}>', f"{(now+oneday).strftime(ymd)}", ), ( '<{now+10d/d{yyyy-MM}}>', f"{(now+timedelta(days=10)).strftime('%Y-%m')}", ), ]: assert expected == parse_datemath(self.client, test_string) def test_complex_datemath(self): v = self.get_version() if (8, 0, 0) < v < (8, 7, 0): raise SkipTest(f'{self.colonblow} (current version: {v})') now = datetime.now(timezone.utc) _ = now + timedelta(days=10) - timedelta(hours=7) test_string = '<{now+10d/h{yyyy-MM-dd-HH|-07:00}}>' expected = f"{_.strftime('%Y-%m-%d-%H')}" assert expected == parse_datemath(self.client, test_string) def test_very_complex_datemaths(self): v = self.get_version() if (8, 0, 0) < v < (8, 7, 0): raise SkipTest(f'{self.colonblow} (current version: {v})') test_string = '<.prefix-{2001-01-01-13||+1h/h{yyyy-MM-dd-HH|-07:00}}-suffix>' expected = '.prefix-2001-01-01-14-suffix' assert expected == parse_datemath(self.client, test_string) elasticsearch-curator-8.0.21/tests/integration/test_delete_indices.py000066400000000000000000000511301477314666200261470ustar00rootroot00000000000000"""Test the Delete Indices action""" # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long import os import time import requests from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') # ' - filtertype: {0}\n' # ' source: {1}\n' # ' direction: {2}\n' # ' timestring: {3}\n' # ' unit: {4}\n' # ' unit_count: {5}\n' # ' field: {6}\n' # ' stats_result: {7}\n' # ' epoch: {8}\n') ILM_KEYS = ['ilm-history-1-000001', 'ilm-history-1'] def exclude_ilm_history(index_list): """Remove any values from ILM_KEYS found in index_list""" for val in ILM_KEYS: if val in index_list: index_list.remove(val) return index_list class TestActionFileDeleteIndices(CuratorTestCase): def test_retention_from_name_days(self): # Test extraction of unit_count from index name # Create indices for 10 days with retention time of 5 days in index name # Expected: 5 oldest indices are deleted, 5 remain self.args['prefix'] = 'logstash_5_' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_' ), ) self.invoke_runner() self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) def test_retention_from_name_days_ignore_failed_match(self): # Test extraction of unit_count from index name # Create indices for 10 days with retention time of 5 days in index name # Create indices for 10 days with no retention time in index name # Expected: 5 oldest indices are deleted, 5 remain - 10 indices without # retention time are ignored and remain self.args['prefix'] = 'logstash_5_' self.create_indices(10) self.args['prefix'] = 'logstash_' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' ', ), ) self.invoke_runner() self.assertEqual(15, len(exclude_ilm_history(get_indices(self.client)))) def test_retention_from_name_days_keep_exclude_false_after_failed_match(self): # Test extraction of unit_count from index name and confirm correct # behavior after a failed regex match with no fallback time - see gh issue 1206 # Create indices for 30 days with retention time of 5 days in index name # # Create indices for 10 days with no retention time in index name # that alphabetically sort before the 10 with retention time # # Create indices for 10 days with no retention time in index name # that sort after the 10 with retention time # Expected: 45 oldest matching indices are deleted, 5 matching indices remain # 20 indices without retention time are ignored and remain # overall 25 indices should remain self.args['prefix'] = 'logstash-aanomatch-' self.create_indices(10) self.args['prefix'] = 'logstash-match-5-' self.create_indices(30) self.args['prefix'] = 'logstash-zznomatch-' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', -1, r'logstash-\w+-([0-9]+)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}', ' ', ' ', ' ', ), ) self.invoke_runner() self.assertEqual(25, len(exclude_ilm_history(get_indices(self.client)))) def test_retention_from_name_days_failed_match_with_fallback(self): # Test extraction of unit_count from index name # Create indices for 10 days with retention time of 5 days in index name # Create indices for 10 days with no retention time in index name but # configure fallback value of 7 # Expected: 5 oldest indices are deleted, 5 remain - 7 indices without # retention time are ignored and remain due to the fallback value self.args['prefix'] = 'logstash_5_' self.create_indices(10) self.args['prefix'] = 'logstash_' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 7, '_([0-9]+)_', ' ', ' ', ' ', ), ) self.invoke_runner() self.assertEqual(12, len(exclude_ilm_history(get_indices(self.client)))) def test_retention_from_name_no_capture_group(self): # Test extraction of unit_count from index name when pattern contains no # capture group # Create indices for 10 months with retention time of 2 months in index name # Expected: all indices remain as the pattern cannot be used to extract a # retention time self.args['prefix'] = 'logstash_2_' self.args['time_unit'] = 'months' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m\'', 'months', -1, '_[0-9]+_', ' ', ' ', ' ', ), ) self.invoke_runner() self.assertEqual(10, len(exclude_ilm_history(get_indices(self.client)))) def test_retention_from_name_illegal_regex_no_fallback(self): # Test extraction of unit_count from index name when pattern contains an # illegal regular expression # Create indices for 10 months with retention time of 2 months in index name # Expected: all indices remain as the pattern cannot be used to extract a # retention time self.args['prefix'] = 'logstash_2_' self.args['time_unit'] = 'months' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m\'', 'months', -1, '_[0-9+_', ' ', ' ', ' ', ), ) self.invoke_runner() self.assertEqual(10, len(exclude_ilm_history(get_indices(self.client)))) def test_retention_from_name_illegal_regex_with_fallback(self): # Test extraction of unit_count from index name when pattern contains an # illegal regular expression # Create indices for 10 days with retention time of 2 days in index name # Expected: Fallback value of 3 is used and 3 most recent indices remain # in place self.args['prefix'] = 'logstash_2_' self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_pattern_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 3, '_[0-9+_', ' ', ' ', ' ', ), ) self.invoke_runner() self.assertEqual(3, len(exclude_ilm_history(get_indices(self.client)))) def test_name_older_than_now(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' ), ) self.invoke_runner() self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) def test_name_based_age_match_five(self): self.create_indices(10) self.create_index(name='foobert') self.create_index(name='.shouldbehidden') self.create_index(name='this_has_no_date') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' ), ) self.invoke_runner() self.assertEqual(8, len(exclude_ilm_history(get_indices(self.client)))) def test_creation_date_newer_than_epoch(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'creation_date', 'younger', ' ', 'seconds', 0, ' ', ' ', int(time.time()) - 60, ), ) self.invoke_runner() self.assertEqual(0, len(exclude_ilm_history(get_indices(self.client)))) def test_delete_in_period(self): # filtertype: {0} # source: {1} # range_from: {2} # range_to: {3} # timestring: {4} # unit: {5} # field: {6} # stats_result: {7} # intersect: {8} # epoch: {9} # week_starts_on: {10} self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_period_proto.format( 'period', 'name', '-5', '-1', "'%Y.%m.%d'", 'days', ' ', ' ', ' ', ' ', 'monday', ), ) self.invoke_runner() self.assertEqual(0, self.result.exit_code) self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) def test_delete_in_period_absolute_date(self): delete_period_abs = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: {0}\n' ' period_type: absolute\n' ' source: {1}\n' ' date_from: {2}\n' ' date_to: {3}\n' ' timestring: {4}\n' ' unit: {5}\n' ' date_from_format: {6}\n' ' date_to_format: {7}\n' ) expected = 'index-2017.02.02' self.create_index('index-2017.01.02') self.create_index('index-2017.01.03') self.create_index(expected) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], delete_period_abs.format( 'period', 'name', '2017.01.01', '2017.01.10', "'%Y.%m.%d'", 'days', "'%Y.%m.%d'", "'%Y.%m.%d'", ), ) self.invoke_runner() indices = exclude_ilm_history(get_indices(self.client)) self.assertEqual(0, self.result.exit_code) self.assertEqual(1, len(indices)) self.assertEqual(expected, indices[0]) def test_delete_in_period_intersect(self): # filtertype: {0} # source: {1} # range_from: {2} # range_to: {3} # timestring: {4} # unit: {5} # field: {6} # stats_result: {7} # intersect: {8} # epoch: {9} # week_starts_on: {10} # 2017-09-01T01:00:00 = 1504227600 # 2017-09-25T01:00:00 = 1506301200 # 2017-09-29T01:00:00 = 1506646800 idx1, idx2 = ('intersecting', 'notintersecting') self.create_index(idx1) self.create_index(idx2) self.client.index( index=idx1, id='1', document={'@timestamp': '2017-09-25T01:00:00Z', 'doc': 'Earliest'}, ) self.client.index( index=idx1, id='2', document={'@timestamp': '2017-09-29T01:00:00Z', 'doc': 'Latest'}, ) self.client.index( index=idx2, id='1', document={'@timestamp': '2017-09-01T01:00:00Z', 'doc': 'Earliest'}, ) self.client.index( index=idx2, id='2', document={'@timestamp': '2017-09-29T01:00:00Z', 'doc': 'Latest'}, ) # Decorators cause this pylint error # pylint: disable=E1123 self.client.indices.flush(index='*', force=True) self.client.indices.refresh(index=f'{idx1},{idx2}') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_period_proto.format( 'period', 'field_stats', '0', '0', ' ', 'weeks', "'@timestamp'", 'min_value', 'true', 1506716040, 'sunday', ), ) self.invoke_runner() self.assertEqual(0, self.result.exit_code) indices = exclude_ilm_history(get_indices(self.client)) self.assertEqual(1, len(indices)) self.assertEqual(idx2, indices[0]) def test_empty_list(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'creation_date', 'older', ' ', 'days', 90, ' ', ' ', int(time.time()), ), ) self.invoke_runner() self.assertEqual(10, len(exclude_ilm_history(get_indices(self.client)))) self.assertEqual(1, self.result.exit_code) def test_ignore_empty_list(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_ignore_proto.format( 'age', 'creation_date', 'older', ' ', 'days', 90, ' ', ' ', int(time.time()), ), ) self.invoke_runner() self.assertEqual(10, len(exclude_ilm_history(get_indices(self.client)))) self.assertEqual(0, self.result.exit_code) def test_extra_options(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('delete_indices'), ) self.invoke_runner() self.assertEqual(1, self.result.exit_code) def test_945(self): self.create_indices(10) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.test_945) self.invoke_runner() self.assertEqual(1, self.result.exit_code) def test_name_epoch_zero(self): self.create_index('epoch_zero-1970.01.01') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' ), ) self.invoke_runner() self.assertEqual(0, len(exclude_ilm_history(get_indices(self.client)))) def test_name_negative_epoch(self): self.create_index('index-1969.12.31') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' ), ) self.invoke_runner() self.assertEqual(0, len(exclude_ilm_history(get_indices(self.client)))) def test_allow_ilm_indices_true(self): name = 'test' policy = { 'policy': { 'phases': { 'hot': { 'min_age': '0ms', 'actions': {'rollover': {'max_age': '2h', 'max_docs': 4}}, } } } } url = f'{HOST}/_ilm/policy/{name}' _ = requests.put(url, json=policy, timeout=30) # print(r.text) # logging reminder self.create_indices(10, ilm_policy=name) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.ilm_delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'true' ), ) self.invoke_runner() self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) def test_allow_ilm_indices_false(self): name = 'test' policy = { 'policy': { 'phases': { 'hot': { 'min_age': '0ms', 'actions': {'rollover': {'max_age': '2h', 'max_docs': 4}}, } } } } url = f'{HOST}/_ilm/policy/{name}' _ = requests.put(url, json=policy, timeout=30) # print(r.text) # logging reminder self.create_indices(10, ilm_policy=name) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.ilm_delete_proto.format( 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false', ), ) self.invoke_runner() self.assertEqual(10, len(exclude_ilm_history(get_indices(self.client)))) class TestCLIDeleteIndices(CuratorTestCase): def test_name_older_than_now_cli(self): self.create_indices(10) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'delete-indices', '--filter_list', ( '{"filtertype":"age","source":"name","direction":"older",' '"timestring":"%Y.%m.%d","unit":"days","unit_count":5}' ), ] self.assertEqual( 0, self.run_subprocess( args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli' ), ) self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) elasticsearch-curator-8.0.21/tests/integration/test_delete_snapshots.py000066400000000000000000000131001477314666200265460ustar00rootroot00000000000000"""Test delete snapshot functionality""" # pylint: disable=C0115, C0116, invalid-name import os import time from curator import IndexList from curator.actions.snapshot import Snapshot from curator.helpers.getters import get_snapshot from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') # ' repository: {0}\n' # ' - filtertype: {1}\n' # ' source: {2}\n' # ' direction: {3}\n' # ' timestring: {4}\n' # ' unit: {5}\n' # ' unit_count: {6}\n' # ' epoch: {7}\n') class TestActionFileDeleteSnapshots(CuratorTestCase): def test_deletesnapshot(self): # Create snapshots to delete and verify them self.create_repository() timestamps = [] for i in range(1, 4): self.add_docs(f'my_index{i}') ilo = IndexList(self.client) snap = Snapshot( ilo, repository=self.args['repository'], name='curator-%Y%m%d%H%M%S', wait_interval=0.5, ) snap.do_action() snapshot = get_snapshot(self.client, self.args['repository'], '*') assert i == len(snapshot['snapshots']) time.sleep(1.0) timestamps.append(int(time.time())) time.sleep(1.0) # Setup the actual delete self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_snap_proto.format( self.args['repository'], 'age', 'creation_date', 'older', ' ', 'seconds', '0', timestamps[0], ), ) self.invoke_runner() snapshot = get_snapshot(self.client, self.args['repository'], '*') assert 2 == len(snapshot['snapshots']) def test_no_repository(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.delete_snap_proto.format( ' ', 'age', 'creation_date', 'older', ' ', 'seconds', '0', ' ' ), ) self.invoke_runner() assert 1 == self.result.exit_code def test_extra_options(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('delete_snapshots'), ) self.invoke_runner() assert 1 == self.result.exit_code class TestCLIDeleteSnapshots(CuratorTestCase): def test_deletesnapshot(self): # Create snapshots to delete and verify them self.create_repository() timestamps = [] for i in range(1, 4): self.add_docs(f'my_index{i}') ilo = IndexList(self.client) snap = Snapshot( ilo, repository=self.args['repository'], name='curator-%Y%m%d%H%M%S', wait_interval=0.5, ) snap.do_action() snapshot = get_snapshot(self.client, self.args['repository'], '*') assert i == len(snapshot['snapshots']) time.sleep(1.0) timestamps.append(int(time.time())) time.sleep(1.0) filter_list = ( '{"filtertype":"age","source":"creation_date","direction":"older",' '"unit":"seconds","unit_count":0,"epoch":' + str(timestamps[0]) + '}' ) # Setup the actual delete args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'delete-snapshots', '--repository', self.args['repository'], '--filter_list', filter_list, ] assert 0 == self.run_subprocess( args, logname='TestCLIDeleteSnapshots.test_deletesnapshot' ) snapshot = get_snapshot(self.client, self.args['repository'], '*') assert 2 == len(snapshot['snapshots']) def test_count_by_age(self): self.create_repository() timestamps = [] def add_snap(num, name): self.add_docs(f'my_index{num}') ilo = IndexList(self.client) snap = Snapshot( ilo, repository=self.args['repository'], name=name, wait_interval=0.5 ) snap.do_action() snapshot = get_snapshot(self.client, self.args['repository'], '*') assert num == len(snapshot['snapshots']) time.sleep(1.0) timestamps.append(int(time.time())) time.sleep(1.0) name = 'curator-%Y%m%d%H%M%S' for i in range(1, 4): add_snap(i, name) add_snap(4, 'kibana-index') # Setup the actual delete filter_list = ( '{"filtertype":"count","count":2,"use_age":true,"source":"name",' '"timestring":"%Y%m%d%H%M%S"}' ) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'delete-snapshots', '--repository', self.args['repository'], '--filter_list', filter_list, ] assert 0 == self.run_subprocess( args, logname='TestCLIDeleteSnapshots.test_deletesnapshot' ) snapshot = get_snapshot(self.client, self.args['repository'], '*') assert 3 == len(snapshot['snapshots']) elasticsearch-curator-8.0.21/tests/integration/test_es_repo_mgr.py000066400000000000000000000143051477314666200255130ustar00rootroot00000000000000"""Test es_repo_mgr script and functions""" # pylint: disable=C0115, C0116, invalid-name import logging import os from click import testing as clicktest from curator import repo_mgr_cli from curator.helpers.testers import repository_exists from . import CuratorTestCase from . import testvars LOGGER = logging.getLogger(__name__) REPO_PATH = '/media' HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') # class TestLoggingModules(CuratorTestCase): # def test_logger_without_null_handler(self): # from unittest.mock import patch, Mock # mock = Mock() # modules = {'logger': mock, 'logger.NullHandler': mock.module} # self.write_config( # self.args['configfile'], # testvars.client_conf_logfile.format(HOST, os.devnull) # ) # with patch.dict('sys.modules', modules): # self.create_repository() # test = clicktest.CliRunner() # result = test.invoke( # repo_mgr_cli, # ['--config', self.args['configfile'], 'show'] # ) # self.assertEqual(self.args['repository'], result.output.rstrip()) class TestCLIRepositoryCreate(CuratorTestCase): def test_create_fs_repository_success(self): self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'create', 'fs', '--name', self.args['repository'], '--location', REPO_PATH, '--verify', ], ) assert 1 == len( self.client.snapshot.get_repository(name=self.args['repository']) ) assert 0 == result.exit_code def test_create_fs_repository_fail(self): self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'create', 'fs', '--name', self.args['repository'], '--location', os.devnull, '--verify', ], ) assert 1 == result.exit_code def test_create_s3_repository_fail(self): self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'create', 's3', '--bucket', 'mybucket', '--name', self.args['repository'], '--verify', ], ) assert 1 == result.exit_code def test_create_azure_repository_fail(self): self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'create', 'azure', '--container', 'mybucket', '--name', self.args['repository'], '--verify', ], ) assert 1 == result.exit_code def test_create_gcs_repository_fail(self): self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'create', 'gcs', '--bucket', 'mybucket', '--name', self.args['repository'], '--verify', ], ) assert 1 == result.exit_code class TestCLIDeleteRepository(CuratorTestCase): def test_delete_repository_success(self): self.create_repository() self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() _ = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'delete', '--yes', # This ensures no prompting will happen '--name', self.args['repository'], ], ) assert not repository_exists(self.client, self.args['repository']) def test_delete_repository_notfound(self): self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, [ '--config', self.args['configfile'], 'delete', '--yes', # This ensures no prompting will happen '--name', self.args['repository'], ], ) assert 1 == result.exit_code class TestCLIShowRepositories(CuratorTestCase): def test_show_repository(self): self.create_repository() self.write_config( self.args['configfile'], testvars.client_conf_logfile.format(HOST, os.devnull), ) test = clicktest.CliRunner() result = test.invoke( repo_mgr_cli, ['--config', self.args['configfile'], 'show'] ) # The splitlines()[-1] allows me to only capture the last line and ignore # other output assert self.args['repository'] == result.output.splitlines()[-1] elasticsearch-curator-8.0.21/tests/integration/test_forcemerge.py000066400000000000000000000106061477314666200253300ustar00rootroot00000000000000"""Test forcemerge functionality""" # pylint: disable=C0115, C0116, invalid-name, protected-access import os from time import sleep from curator import IndexList from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestActionFileforceMerge(CuratorTestCase): def test_merge(self): count = 1 idx = 'my_index' self.create_index(idx) self.add_docs(idx) ilo1 = IndexList(self.client) ilo1.get_segment_counts() assert 3 == ilo1.index_info[idx]['segments'] self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.forcemerge_test.format(count, 0.9) ) self.invoke_runner() ilo2 = IndexList(self.client) # This stupid block is only because it can finish testing before # the segments have _reported_ as fully merged. This is forcing # 3 checks before giving up and reporting the result. for _ in range(0, 3): self.client.indices.refresh(index=idx) ilo2.get_segment_counts() if ilo2.index_info[idx]['segments'] == count: break sleep(1) assert count == ilo2.index_info[idx]['segments'] def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('forcemerge') ) self.invoke_runner() assert 1 == self.result.exit_code class TestCLIforceMerge(CuratorTestCase): COUNT = 1 IDX = 'my_index' ILO = None CLI_ARGS = None def setup(self): self.COUNT = 1 self.IDX = 'my_index' self.create_index(self.IDX) self.add_docs(self.IDX) self.ILO = IndexList(self.client) self.ILO.get_segment_counts() assert 3 == self.ILO.index_info[self.IDX]['segments'] self.CLI_ARGS = self.get_runner_args() self.CLI_ARGS += [ '--config', self.args['configfile'], 'forcemerge', '--max_num_segments', str(self.COUNT), '--delay', '0.9', '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] def wait_for_merge(self): # This stupid block is only for the benefit of testing # It apparently can finish testing before the segments have _reported_ # as fully merged. # This is forcing 3 checks before giving up and reporting the result. for _ in range(0, 3): self.client.indices.refresh(index=self.IDX) self.ILO.get_segment_counts() if self.ILO.index_info[self.IDX]['segments'] == self.COUNT: break sleep(1) def test_merge(self): self.setup() assert 0 == self.run_subprocess( self.CLI_ARGS, logname='TestCLIforceMerge.test_merge' ) self.wait_for_merge() assert self.COUNT == self.ILO.index_info[self.IDX]['segments'] self.ILO = None self.CLI_ARGS = None def test_empty_list1(self): """Test with an empty list and ignore_empty_list unset""" self.setup() self.client.indices.forcemerge(index=self.IDX, max_num_segments=1) self.wait_for_merge() assert 1 == self.run_subprocess( self.CLI_ARGS, logname='TestCLIforceMerge.test_empty_list1' ) self.ILO = None self.CLI_ARGS = None def test_empty_list2(self): """Test with an empty list and ignore_empty_list = True""" self.setup() self.client.indices.forcemerge(index=self.IDX, max_num_segments=1) self.wait_for_merge() self.CLI_ARGS = self.get_runner_args() self.CLI_ARGS += [ '--config', self.args['configfile'], 'forcemerge', '--max_num_segments', str(self.COUNT), '--delay', '0.9', '--ignore_empty_list', '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] assert 0 == self.run_subprocess( self.CLI_ARGS, logname='TestCLIforceMerge.test_empty_list2' ) self.ILO = None self.CLI_ARGS = None elasticsearch-curator-8.0.21/tests/integration/test_integrations.py000066400000000000000000000266451477314666200257320ustar00rootroot00000000000000"""Test integrations""" # pylint: disable=C0115, C0116, invalid-name import os import warnings import pytest from elasticsearch8 import Elasticsearch from elasticsearch8.exceptions import ElasticsearchWarning, NotFoundError from curator.exceptions import ConfigurationError from curator.helpers.getters import get_indices from curator import IndexList from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestFilters(CuratorTestCase): def test_filter_by_alias(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.filter_by_alias.format('testalias', False) ) self.create_index('my_index') self.create_index('dummy') self.client.indices.put_alias(index='dummy', name=alias) self.invoke_runner() assert 1 == len(get_indices(self.client)) def test_filter_by_array_of_aliases(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.filter_by_alias.format(' [ testalias, foo ]', False), ) self.create_index('my_index') self.create_index('dummy') self.client.indices.put_alias(index='dummy', name=alias) self.invoke_runner() assert 2 == len(get_indices(self.client)) def test_filter_by_alias_bad_aliases(self): alias = 'testalias' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.filter_by_alias.format('{"this":"isadict"}', False), ) self.create_index('my_index') self.create_index('dummy') self.client.indices.put_alias(index='dummy', name=alias) self.invoke_runner() assert isinstance(self.result.exception, ConfigurationError) assert 2 == len(get_indices(self.client)) def test_filter_closed(self): idx1 = 'dummy' idx2 = 'my_index' ptrn = f'{idx1}*,{idx2}*' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.filter_closed.format("True") ) self.create_index(idx1) self.create_index(idx2) # This one will be closed, ergo not deleted self.client.indices.close(index=idx2) self.invoke_runner() assert 1 == len(get_indices(self.client)) result = self.client.indices.get(index=ptrn, expand_wildcards='open,closed') assert idx2 == list(dict(result).keys())[0] def test_field_stats_skips_empty_index(self): delete_field_stats = ( '---\n' 'actions:\n' ' 1:\n' ' action: delete_indices\n' ' filters:\n' ' - filtertype: age\n' ' source: field_stats\n' ' direction: older\n' ' field: "{0}"\n' ' unit: days\n' ' unit_count: 1\n' ' stats_result: min_value\n' ) idx = 'my_index' zero = 'zero' field = '@timestamp' time = '2017-12-31T23:59:59.999Z' # Create idx with a single, @timestamped doc self.client.create(index=idx, id=1, document={field: time}) # Flush to ensure it's written # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=idx, force=True) self.client.indices.refresh(index=idx) # Create zero with no docs self.create_index(zero) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], delete_field_stats.format(field)) self.invoke_runner() # It should skip deleting 'zero', as it has 0 docs assert [zero] == get_indices(self.client) class TestIndexList(CuratorTestCase): """Test some of the IndexList particulars using a live ES instance/cluster""" IDX1 = 'dummy1' IDX2 = 'dummy2' IDX3 = 'my_index' @pytest.fixture(autouse=True) def inject_fixtures(self, caplog): # pylint: disable=attribute-defined-outside-init self._caplog = caplog def test_get_index_stats_with_404(self): """ Check to ensure that index_stats are being collected if one index is missing """ # expected = f'Index was initiallly present, but now is not: {self.IDX2}' self.create_index(self.IDX1) self.create_index(self.IDX2) self.create_index(self.IDX3) ilo = IndexList(self.client) assert ilo.indices == [self.IDX1, self.IDX2, self.IDX3] self.client.indices.delete(index=f'{self.IDX1},{self.IDX2}') ilo.get_index_stats() # with self._caplog.at_level(logging.WARNING): # ilo.get_index_stats() # # Guarantee we're getting the expected WARNING level message # assert self._caplog.records[-1].message == expected assert ilo.indices == [self.IDX3] def test_get_index_state(self): """Check to ensure that open/close status is properly being recognized""" self.create_index(self.IDX1) self.create_index(self.IDX2) ilo = IndexList(self.client) ilo.get_index_state() assert ilo.indices == [self.IDX1, self.IDX2] assert ilo.index_info[self.IDX1]['state'] == 'open' assert ilo.index_info[self.IDX2]['state'] == 'open' # ElasticsearchWarning: the default value for the wait_for_active_shards # parameter will change from '0' to 'index-setting' in version 8; # specify 'wait_for_active_shards=index-setting' to adopt the future default # behaviour, or 'wait_for_active_shards=0' to preserve today's behaviour warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.close(index=self.IDX2) ilo.get_index_state() assert ilo.index_info[self.IDX2]['state'] == 'close' def test_get_index_state_alias(self): """Check to ensure that open/close status catches an alias""" alias = {self.IDX2: {}} self.create_index(self.IDX1) self.create_index(self.IDX2) ilo = IndexList(self.client) assert ilo.indices == [self.IDX1, self.IDX2] self.client.indices.delete(index=self.IDX2) self.client.indices.create(index=self.IDX3, aliases=alias) ilo.get_index_state() assert ilo.indices == [self.IDX1, self.IDX3] def test_population_check_missing_index(self): """ If index_info is missing an index, test to ensure it is populated with the zero value """ key = 'docs' self.create_index(self.IDX1) ilo = IndexList(self.client) ilo.population_check(self.IDX1, key) assert ilo.index_info[self.IDX1][key] == 0 def test_population_check_missing_key(self): """ If index_info is missing a key, test to ensure it is populated with a zero value """ key = 'docs' self.create_index(self.IDX1) ilo = IndexList(self.client) ilo.get_index_stats() assert ilo.indices == [self.IDX1] assert ilo.index_info[self.IDX1][key] == 0 del ilo.index_info[self.IDX1][key] ilo.population_check(self.IDX1, key) assert ilo.index_info[self.IDX1][key] == 0 def test_not_needful(self): """ Check if get_index_stats can be skipped if already populated THIS IS LITERALLY FOR CODE COVERAGE, so a ``continue`` line in the function is tested. """ key = 'docs' self.create_index(self.IDX1) ilo = IndexList(self.client) ilo.get_index_stats() ilo.get_index_stats() # index_info is already populated and it will skip assert ilo.index_info[self.IDX1][key] == 0 def test_search_pattern_1(self): """Check to see if providing a search_pattern limits the index list""" pattern = 'd*' self.create_index(self.IDX1) self.create_index(self.IDX2) self.create_index(self.IDX3) ilo1 = IndexList(self.client) ilo2 = IndexList(self.client, search_pattern=pattern) assert ilo1.indices == [self.IDX1, self.IDX2, self.IDX3] assert ilo2.indices == [self.IDX1, self.IDX2] def cleanup(client, idx): """Cleanup indices""" try: client.indices.delete(index=idx) except NotFoundError: pass @pytest.mark.parametrize( 'idx, pattern, hidden, response', [ ('my_index', 'my_*', True, ['my_index']), ('my_index', 'my_*', False, []), ('my_index', '*_index', True, ['my_index']), ('my_index', '*_index', False, []), ('.my_index', '.my_*', True, ['.my_index']), # ('.my_index', '.my_*', False, []), # This is a weird behavior or a bug. # Check in test__bug__include_hidden_dot_prefix_index ('.my_index', '*_index', True, ['.my_index']), ('.my_index', '*_index', False, []), ], ) def test_include_hidden_index(idx, pattern, hidden, response): """Test that a hidden index is included when include_hidden is True""" host = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') client = Elasticsearch(hosts=host, request_timeout=300) warnings.filterwarnings("ignore", category=ElasticsearchWarning) cleanup(client, idx) client.indices.create(index=idx) client.indices.put_settings(index=idx, body={'index': {'hidden': True}}) ilo = IndexList(client, search_pattern=pattern, include_hidden=hidden) assert ilo.indices == response # Manual teardown because hidden indices are not returned by get_indices del ilo cleanup(client, idx) @pytest.mark.parametrize( 'idx, pattern, hidden, response', [ ('.my_index', '.my_*', False, ['.my_index']), ('.your_index', '.your_*', False, ['.your_index']), ], ) def test__bug__include_hidden_dot_prefix_index(idx, pattern, hidden, response): """ Test that a hidden index is included when include_hidden is True when the multi-target search pattern starts with a leading dot and includes a wildcard, e.g. '.my_*' or '.your_*' This is a bug, and the test is to confirm that behavior until it's fixed. https://github.com/elastic/elasticsearch/issues/124167 """ host = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') client = Elasticsearch(hosts=host, request_timeout=300) warnings.filterwarnings("ignore", category=ElasticsearchWarning) cleanup(client, idx) client.indices.create(index=idx) client.indices.put_settings(index=idx, body={'index': {'hidden': True}}) hidden_test = client.indices.get_settings( index=idx, filter_path=f'\\{idx}.settings.index.hidden', expand_wildcards='all' ) # We're confirming that our index that starts with a leading dot is hidden assert hidden_test[idx]['settings']['index']['hidden'] == 'true' ilo = IndexList(client, search_pattern=pattern, include_hidden=hidden) # And now we assert that the index is still included in the list # WHICH SHOULD NOT HAPPEN -- THIS IS A BUG assert ilo.indices == response # Manual teardown because hidden indices are not returned by get_indices del ilo cleanup(client, idx) elasticsearch-curator-8.0.21/tests/integration/test_open.py000066400000000000000000000067251477314666200241620ustar00rootroot00000000000000"""Test index opening functionality""" # pylint: disable=C0115, C0116, invalid-name import os import warnings from elasticsearch8.exceptions import ElasticsearchWarning from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') MET = 'metadata' class TestActionFileOpenClosed(CuratorTestCase): def test_open_closed(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.optionless_proto.format('open') ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) # ElasticsearchWarning: the default value for the wait_for_active_shards # parameter will change from '0' to 'index-setting' in version 8; # specify 'wait_for_active_shards=index-setting' to adopt the future default # behaviour, or 'wait_for_active_shards=0' to preserve today's behaviour warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.close(index=idx2, ignore_unavailable=True) self.invoke_runner() csi = self.client.cluster.state(metric=MET)[MET]['indices'] for idx in (idx1, idx2): assert 'close' != csi[idx]['state'] def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('open') ) idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) # ElasticsearchWarning: the default value for the wait_for_active_shards # parameter will change from '0' to 'index-setting' in version 8; # specify 'wait_for_active_shards=index-setting' to adopt the future default # behaviour, or 'wait_for_active_shards=0' to preserve today's behaviour warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.close(index=idx2, ignore_unavailable=True) self.invoke_runner() csi = self.client.cluster.state(metric=MET)[MET]['indices'] assert 'close' != csi[idx1]['state'] assert 'close' == csi[idx2]['state'] assert 1 == self.result.exit_code class TestCLIOpenClosed(CuratorTestCase): def test_open_closed(self): idx1, idx2 = ('dummy', 'my_index') self.create_index(idx1) self.create_index(idx2) # ElasticsearchWarning: the default value for the wait_for_active_shards # parameter will change from '0' to 'index-setting' in version 8; # specify 'wait_for_active_shards=index-setting' to adopt the future default # behaviour, or 'wait_for_active_shards=0' to preserve today's behaviour warnings.filterwarnings("ignore", category=ElasticsearchWarning) self.client.indices.close(index=idx2, ignore_unavailable=True) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'open', '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] assert 0 == self.run_subprocess( args, logname='TestCLIOpenClosed.test_open_closed' ) csi = self.client.cluster.state(metric=MET)[MET]['indices'] for idx in (idx1, idx2): assert 'close' != csi[idx]['state'] elasticsearch-curator-8.0.21/tests/integration/test_reindex.py000066400000000000000000000410661477314666200246540ustar00rootroot00000000000000"""Test reindex action functionality""" # pylint: disable=C0115, C0116, invalid-name import os from unittest.case import SkipTest import pytest from dotmap import DotMap # type: ignore from es_client.builder import Builder from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars UNDEF = 'undefined' HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') RHOST = os.environ.get('REMOTE_ES_SERVER', UNDEF) WAIT_INTERVAL = 1 MAX_WAIT = 3 class TestActionFileReindex(CuratorTestCase): """Test file-based reindex operations""" def test_reindex_manual(self): """Test that manual reindex results in proper count of documents""" source = 'my_source' dest = 'my_dest' expected = 3 self.create_index(source) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] def test_reindex_selected(self): """Reindex selected indices""" source = 'my_source' dest = 'my_dest' expected = 3 self.create_index(source) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, 'REINDEX_SELECTION', dest), ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] def test_reindex_empty_list(self): """ This test raises : Reindex failed. The index or alias identified by "my_dest" was not found. The source is never created, so the dest is also not able to be created This means that we expect an empty list. """ source = 'my_source' dest = 'my_dest' expected = [] self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), ) self.invoke_runner() assert expected == get_indices(self.client) def test_reindex_selected_many_to_one(self): """Test reindexing many indices to one destination""" source1 = 'my_source1' source2 = 'my_source2' dest = 'my_dest' expected = 6 self.create_index(source1) self.add_docs(source1) self.create_index(source2) for i in ["4", "5", "6"]: self.client.create( index=source2, id=i, document={"doc" + i: 'TEST DOCUMENT'} ) # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=source2, force=True) self.client.indices.refresh(index=source2) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, 'REINDEX_SELECTION', dest), ) self.invoke_runner() self.client.indices.refresh(index=dest) assert expected == self.client.count(index=dest)['count'] def test_reindex_selected_empty_list_fail(self): """Ensure an empty list results in an exit code 1""" source1 = 'my_source1' source2 = 'my_source2' dest = 'my_dest' self.create_index(source1) self.add_docs(source1) self.create_index(source2) for i in ["4", "5", "6"]: self.client.create( index=source2, id=i, document={"doc" + i: 'TEST DOCUMENT'} ) # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=source2, force=True) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex_empty_list.format('false', WAIT_INTERVAL, MAX_WAIT, dest), ) self.invoke_runner() assert 1 == self.result.exit_code def test_reindex_selected_empty_list_pass(self): """Ensure an empty list results in an exit code 0""" source1 = 'my_source1' source2 = 'my_source2' dest = 'my_dest' self.create_index(source1) self.add_docs(source1) self.create_index(source2) for i in ["4", "5", "6"]: self.client.create( index=source2, id=i, document={"doc" + i: 'TEST DOCUMENT'} ) # Decorators make this pylint exception necessary # pylint: disable=E1123 self.client.indices.flush(index=source2, force=True) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex_empty_list.format('true', WAIT_INTERVAL, MAX_WAIT, dest), ) self.invoke_runner() assert 0 == self.result.exit_code @pytest.mark.skipif((RHOST == UNDEF), reason='REMOTE_ES_SERVER is not defined') def test_reindex_from_remote(self): """Test remote reindex functionality""" diff_wait = 6 source1 = 'my_source1' source2 = 'my_source2' prefix = 'my_' dest = 'my_dest' expected = 6 # Build remote client try: remote_args = DotMap({'hosts': RHOST}) remote_config = {'elasticsearch': {'client': remote_args.toDict()}} builder = Builder(configdict=remote_config) builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): rclient.create( index=rindex, id=str(counter + 1), document={"doc" + str(i): 'TEST DOCUMENT'}, ) counter += 1 # Decorators make this pylint exception necessary # pylint: disable=E1123 rclient.indices.flush(index=rindex, force=True) rclient.indices.refresh(index=rindex) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.remote_reindex.format( WAIT_INTERVAL, diff_wait, RHOST, 'REINDEX_SELECTION', dest, prefix ), ) self.invoke_runner() # Do our own cleanup here. rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) assert expected == self.client.count(index=dest)['count'] @pytest.mark.skipif((RHOST == UNDEF), reason='REMOTE_ES_SERVER is not defined') def test_reindex_migrate_from_remote(self): """Test remote reindex migration""" source1 = 'my_source1' source2 = 'my_source2' prefix = 'my_' dest = 'MIGRATION' expected = 3 # Build remote client try: remote_args = DotMap({'hosts': RHOST}) remote_config = {'elasticsearch': {'client': remote_args.toDict()}} builder = Builder(configdict=remote_config) builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): rclient.create( index=rindex, id=str(counter + 1), document={"doc" + str(i): 'TEST DOCUMENT'}, ) counter += 1 # Decorators make this pylint exception necessary # pylint: disable=E1123 rclient.indices.flush(index=rindex, force=True) rclient.indices.refresh(index=rindex) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.remote_reindex.format( WAIT_INTERVAL, MAX_WAIT, RHOST, 'REINDEX_SELECTION', dest, prefix ), ) self.invoke_runner() # Do our own cleanup here. rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) # And now the neat trick of verifying that the reindex worked to both # indices, and they preserved their names assert expected == self.client.count(index=source1)['count'] assert expected == self.client.count(index=source2)['count'] @pytest.mark.skipif((RHOST == UNDEF), reason='REMOTE_ES_SERVER is not defined') def test_reindex_migrate_from_remote_with_pre_suf_fixes(self): """Ensure migrate from remote with prefixes and suffixes works""" source1 = 'my_source1' source2 = 'my_source2' prefix = 'my_' dest = 'MIGRATION' expected = 3 mpfx = 'pre-' msfx = '-fix' # Build remote client try: remote_args = DotMap({'hosts': RHOST}) remote_config = {'elasticsearch': {'client': remote_args.toDict()}} builder = Builder(configdict=remote_config) builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): rclient.create( index=rindex, id=str(counter + 1), document={"doc" + str(i): 'TEST DOCUMENT'}, ) counter += 1 # Decorators make this pylint exception necessary # pylint: disable=E1123 rclient.indices.flush(index=rindex, force=True) rclient.indices.refresh(index=rindex) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.migration_reindex.format( WAIT_INTERVAL, MAX_WAIT, mpfx, msfx, RHOST, 'REINDEX_SELECTION', dest, prefix, ), ) self.invoke_runner() # Do our own cleanup here. rclient.indices.delete(index=f'{source1},{source2}') # And now the neat trick of verifying that the reindex worked to both # indices, and they preserved their names assert expected == self.client.count(index=f'{mpfx}{source1}{msfx}')['count'] def test_reindex_from_remote_no_connection(self): """Ensure that the inability to connect to the remote cluster fails""" dest = 'my_dest' bad_remote = 'http://127.0.0.1:9601' expected = 1 self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.remote_reindex.format( WAIT_INTERVAL, MAX_WAIT, bad_remote, 'REINDEX_SELECTION', dest, 'my_' ), ) self.invoke_runner() assert expected == self.result.exit_code @pytest.mark.skipif((RHOST == UNDEF), reason='REMOTE_ES_SERVER is not defined') def test_reindex_from_remote_no_indices(self): """ Test that attempting to reindex remotely with an empty list exits with a fail """ source1 = 'wrong1' source2 = 'wrong2' prefix = 'my_' dest = 'my_dest' expected = 1 # Build remote client try: remote_args = DotMap({'hosts': RHOST}) remote_config = {'elasticsearch': {'client': remote_args.toDict()}} builder = Builder(configdict=remote_config) builder.version_min = (5, 0, 0) builder.connect() rclient = builder.client rclient.info() except Exception as exc: raise SkipTest(f'Unable to connect to host at {RHOST}: {exc}') from exc # Build indices remotely. counter = 0 # Force remove my_source1 and my_source2 to prevent false positives rclient.indices.delete( index=f"{'my_source1'},{'my_source2'}", ignore_unavailable=True ) rclient.indices.delete(index=f'{source1},{source2}', ignore_unavailable=True) for rindex in [source1, source2]: rclient.indices.create(index=rindex) for i in range(0, 3): rclient.create( index=rindex, id=str(counter + 1), document={"doc" + str(i): 'TEST DOCUMENT'}, ) counter += 1 rclient.indices.flush(index=rindex, force=True) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.remote_reindex.format( WAIT_INTERVAL, MAX_WAIT, f'{RHOST}', 'REINDEX_SELECTION', dest, prefix ), ) self.invoke_runner() # Do our own cleanup here. rclient.indices.delete(index=f'{source1},{source2}') assert expected == self.result.exit_code def test_reindex_into_alias(self): """Ensure that reindexing into an alias works as expected""" source = 'my_source' dest = 'my_dest' expected = 3 alias_dict = {dest: {}} self.client.indices.create(index='dummy', aliases=alias_dict) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] def test_reindex_manual_date_math(self): """Ensure date math is functional with reindex calls""" source = '' dest = '' expected = 3 self.create_index(source) self.add_docs(source) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), ) self.invoke_runner() assert expected == self.client.count(index=dest)['count'] def test_reindex_bad_mapping(self): """This test addresses GitHub issue #1260""" source = 'my_source' dest = 'my_dest' expected = 1 settings = {"number_of_shards": 1, "number_of_replicas": 0} mappings1 = {"properties": {"doc1": {"type": "keyword"}}} mappings2 = {"properties": {"doc1": {"type": "integer"}}} self.client.indices.create(index=source, settings=settings, mappings=mappings1) self.add_docs(source) # Create the dest index with a different mapping. self.client.indices.create(index=dest, settings=settings, mappings=mappings2) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.reindex.format(WAIT_INTERVAL, MAX_WAIT, source, dest), ) self.invoke_runner() assert expected == self.result.exit_code elasticsearch-curator-8.0.21/tests/integration/test_replicas.py000066400000000000000000000041511477314666200250120ustar00rootroot00000000000000"""Test replica count changing functionality""" # pylint: disable=C0115, C0116, invalid-name import os from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestActionFileReplicas(CuratorTestCase): def test_increase_count(self): count = 2 idx = 'my_index' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.replicas_test.format(count)) self.create_index(idx) self.invoke_runner() assert count == int( self.client.indices.get_settings(index=idx)[idx]['settings']['index'][ 'number_of_replicas' ] ) def test_no_count(self): self.create_index('foo') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], testvars.replicas_test.format(' ')) self.invoke_runner() assert 1 == self.result.exit_code def test_extra_option(self): self.create_index('foo') self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('replicas') ) self.invoke_runner() assert 1 == self.result.exit_code class TestCLIReplicas(CuratorTestCase): def test_increase_count(self): count = 2 idx = 'my_index' self.create_index(idx) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'replicas', '--count', str(count), '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', ] assert 0 == self.run_subprocess( args, logname='TestCLIOpenClosed.test_open_closed' ) assert count == int( self.client.indices.get_settings(index=idx)[idx]['settings']['index'][ 'number_of_replicas' ] ) elasticsearch-curator-8.0.21/tests/integration/test_restore.py000066400000000000000000000170731477314666200247020ustar00rootroot00000000000000"""Test snapshot restore functionality""" # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long import os import time from curator.helpers.getters import get_indices, get_snapshot from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') # ' repository: {0}\n' # ' name: {1}\n' # ' indices: {2}\n' # ' include_aliases: {3}\n' # ' ignore_unavailable: {4}\n' # ' include_global_state: {5}\n' # ' partial: {6}\n' # ' rename_pattern: {7}\n' # ' rename_replacement: {8}\n' # ' extra_settings: {9}\n' # ' wait_for_completion: {10}\n' # ' skip_repo_fs_check: {11}\n' # ' timeout_override: {12}\n' # ' wait_interval: {13}\n' # ' max_wait: {14}\n' class TestActionFileRestore(CuratorTestCase): """Test file-based configuration restore operations""" def test_restore(self): """Test restore action""" indices = [] for i in range(1, 4): self.add_docs(f'my_index{i}') indices.append(f'my_index{i}') snap_name = 'snapshot1' self.create_snapshot(snap_name, ','.join(indices)) snapshot = get_snapshot(self.client, self.args['repository'], '*') self.assertEqual(1, len(snapshot['snapshots'])) assert 1 == len(snapshot['snapshots']) self.client.indices.delete(index=','.join(indices)) self.assertEqual([], get_indices(self.client)) assert not get_indices(self.client) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.restore_snapshot_proto.format( self.args['repository'], snap_name, indices, False, False, True, False, ' ', ' ', ' ', True, False, 301, 1, 3, ), ) self.invoke_runner() restored_indices = sorted(get_indices(self.client)) self.assertEqual(indices, restored_indices) assert indices == restored_indices # The test runs so fast that it tries to execute the cleanup step # and delete the repository before Elasticsearch is actually ready time.sleep(0.5) def test_restore_with_rename(self): """Test restore action with renaming enabled""" indices = [] for i in range(1, 4): self.add_docs(f'my_index{i}') indices.append(f'my_index{i}') snap_name = 'snapshot1' self.create_snapshot(snap_name, ','.join(indices)) snapshot = get_snapshot(self.client, self.args['repository'], '*') time.sleep(1) self.assertEqual(1, len(snapshot['snapshots'])) assert 1 == len(snapshot['snapshots']) self.client.indices.delete(index=','.join(indices)) self.assertEqual([], get_indices(self.client)) assert not get_indices(self.client) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.restore_snapshot_proto.format( self.args['repository'], snap_name, indices, False, False, True, False, 'my_index(.+)', 'new_index$1', ' ', True, False, 301, 1, 3, ), ) self.invoke_runner() time.sleep(1) restored_indices = sorted(get_indices(self.client)) self.assertEqual(['new_index1', 'new_index2', 'new_index3'], restored_indices) assert ['new_index1', 'new_index2', 'new_index3'] == restored_indices # The test runs so fast that it tries to execute the cleanup step # and delete the repository before Elasticsearch is actually ready time.sleep(1) def test_restore_wildcard(self): """Test restore with wildcard""" indices = [] my_indices = [] wildcard = ['my_*'] for i in range(1, 4): for prefix in ['my_', 'not_my_']: self.add_docs(f'{prefix}index{i}') indices.append(f'{prefix}index{i}') if prefix == 'my_': my_indices.append(f'{prefix}index{i}') snap_name = 'snapshot1' self.create_snapshot(snap_name, ','.join(indices)) snapshot = get_snapshot(self.client, self.args['repository'], '*') self.assertEqual(1, len(snapshot['snapshots'])) assert 1 == len(snapshot['snapshots']) self.client.indices.delete(index=','.join(indices)) self.assertEqual([], get_indices(self.client)) assert not get_indices(self.client) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.restore_snapshot_proto.format( self.args['repository'], snap_name, wildcard, False, False, True, False, ' ', ' ', ' ', True, False, 301, 1, 3, ), ) self.invoke_runner() restored_indices = sorted(get_indices(self.client)) self.assertEqual(my_indices, restored_indices) assert my_indices == restored_indices # The test runs so fast that it tries to execute the cleanup step # and delete the repository before Elasticsearch is actually ready time.sleep(0.5) class TestCLIRestore(CuratorTestCase): def test_restore(self): indices = [] for i in range(1, 4): self.add_docs(f'my_index{i}') indices.append(f'my_index{i}') snap_name = 'snapshot1' self.create_snapshot(snap_name, ','.join(indices)) snapshot = get_snapshot(self.client, self.args['repository'], '*') self.assertEqual(1, len(snapshot['snapshots'])) assert 1 == len(snapshot['snapshots']) self.client.indices.delete(index=f'{",".join(indices)}') self.assertEqual([], get_indices(self.client)) assert not get_indices(self.client) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'restore', '--repository', self.args['repository'], '--name', snap_name, '--index', indices[0], '--index', indices[1], '--index', indices[2], '--wait_interval', '1', '--max_wait', '3', '--filter_list', '{"filtertype":"none"}', ] # self.assertEqual( # 0, self.run_subprocess(args, logname='TestCLIRestore.test_restore') # ) assert 0 == self.run_subprocess(args, logname='TestCLIRestore.test_restore') restored_indices = sorted(get_indices(self.client)) self.assertEqual(indices, restored_indices) assert indices == restored_indices # The test runs so fast that it tries to execute the cleanup step # and delete the repository before Elasticsearch is actually ready time.sleep(0.5) elasticsearch-curator-8.0.21/tests/integration/test_rollover.py000066400000000000000000000243321477314666200250570ustar00rootroot00000000000000"""Test rollover action functionality""" # pylint: disable=C0115, C0116, invalid-name import os import time from curator.helpers.date_ops import parse_date_pattern from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') OLDINDEX = 'rolltome-000001' NEWINDEX = 'rolltome-000002' ALIAS = 'delamitri' class TestActionFileRollover(CuratorTestCase): def test_max_age_true(self): condition = 'max_age' value = '1s' expected = {NEWINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_one.format(ALIAS, condition, value), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_max_age_false(self): condition = 'max_age' value = '10s' expected = {OLDINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_one.format(ALIAS, condition, value), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_max_docs_true(self): condition = 'max_docs' value = '2' expecto = {'aliases': {ALIAS: {}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) self.add_docs(OLDINDEX) self.client.indices.rollover( alias=ALIAS, conditions={condition: value}, dry_run=True ) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_one.format(ALIAS, condition, value), ) self.invoke_runner() response = self.client.indices.get_alias(name=ALIAS) assert 1 == len(list(response.keys())) assert NEWINDEX == list(response.keys())[0] assert expecto == response[NEWINDEX] def test_max_docs_false(self): condition = 'max_docs' value = '5' expected = {OLDINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) self.add_docs(OLDINDEX) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_one.format(ALIAS, condition, value), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_conditions_both_false(self): max_age = '10s' max_docs = '5' expected = {OLDINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) self.add_docs(OLDINDEX) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_both.format(ALIAS, max_age, max_docs), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_conditions_both_true(self): max_age = '1s' max_docs = '2' expected = {NEWINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) time.sleep(1) self.add_docs(OLDINDEX) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_both.format(ALIAS, max_age, max_docs), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_conditions_one_false_one_true(self): max_age = '10s' max_docs = '2' expected = {NEWINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) self.add_docs(OLDINDEX) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_both.format(ALIAS, max_age, max_docs), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_conditions_one_empty_one_true(self): max_age = ' ' max_docs = '2' expected = {OLDINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) self.add_docs(OLDINDEX) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_both.format(ALIAS, max_age, max_docs), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) assert 1 == self.result.exit_code def test_bad_settings(self): max_age = '10s' max_docs = '2' expected = {OLDINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) self.add_docs(OLDINDEX) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_bad_settings.format(ALIAS, max_age, max_docs), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) assert 1 == self.result.exit_code def test_extra_option(self): self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_rollover_test.format('rollover'), ) before = get_indices(self.client) self.invoke_runner() assert before == get_indices(self.client) assert 1 == self.result.exit_code def test_max_age_with_new_name(self): newindex = 'crazy_test' condition = 'max_age' value = '1s' expected = {newindex: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_with_name.format(ALIAS, condition, value, newindex), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_max_age_with_new_name_with_date(self): newindex = 'crazy_test-%Y.%m.%d' condition = 'max_age' value = '1s' expected = {parse_date_pattern(newindex): {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_with_name.format(ALIAS, condition, value, newindex), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_max_age_old_index_with_date_with_new_index(self): oldindex = 'crazy_test-2017.01.01' newindex = 'crazy_test-%Y.%m.%d' condition = 'max_age' value = '1s' expected = {f"{parse_date_pattern(newindex)}": {'aliases': {ALIAS: {}}}} self.client.indices.create(index=oldindex, aliases={ALIAS: {}}) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_with_name.format(ALIAS, condition, value, newindex), ) self.invoke_runner() assert expected == self.client.indices.get_alias(name=ALIAS) def test_is_write_alias(self): condition = 'max_age' value = '1s' request_body = {'aliases': {ALIAS: {'is_write_index': True}}} expected = 2 self.client.indices.create(index=OLDINDEX, aliases=request_body['aliases']) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_one.format(ALIAS, condition, value), ) self.invoke_runner() assert expected == len(self.client.indices.get_alias(name=ALIAS)) def test_no_rollover_ilm_associated(self): condition = 'max_age' value = '1s' expected = 1 self.client.indices.create( index=OLDINDEX, settings={'index': {'lifecycle': {'name': 'generic'}}}, aliases={ALIAS: {}}, ) time.sleep(1) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.rollover_one.format(ALIAS, condition, value), ) self.invoke_runner() assert 0 == self.result.exit_code assert expected == len(self.client.indices.get_alias(name=ALIAS)) assert OLDINDEX == list(self.client.indices.get_alias(name=ALIAS).keys())[0] class TestCLIRollover(CuratorTestCase): def test_max_age_true(self): value = '1s' expected = {NEWINDEX: {'aliases': {ALIAS: {}}}} self.client.indices.create(index=OLDINDEX, aliases={ALIAS: {}}) time.sleep(1) args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'rollover', '--name', ALIAS, '--max_age', value, ] assert 0 == self.run_subprocess( args, logname='TestCLIRollover.test_max_age_true' ) assert expected == self.client.indices.get_alias(name=ALIAS) elasticsearch-curator-8.0.21/tests/integration/test_shrink.py000066400000000000000000000253361477314666200245160ustar00rootroot00000000000000"""Test shrink action""" # pylint: disable=C0115, C0116, invalid-name, attribute-defined-outside-init import os import logging from curator.helpers.getters import get_indices from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') SHRINK = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: shrink\n' ' options:\n' ' shrink_node: {0}\n' ' node_filters:\n' ' {1}: {2}\n' ' number_of_shards: {3}\n' ' number_of_replicas: {4}\n' ' shrink_prefix: {5}\n' ' shrink_suffix: {6}\n' ' delete_after: {7}\n' ' wait_for_rebalance: {8}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) NO_PERMIT_MASTERS = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: shrink\n' ' options:\n' ' shrink_node: {0}\n' ' number_of_shards: {1}\n' ' number_of_replicas: {2}\n' ' shrink_prefix: {3}\n' ' shrink_suffix: {4}\n' ' delete_after: {5}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) WITH_EXTRA_SETTINGS = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: shrink\n' ' options:\n' ' shrink_node: {0}\n' ' node_filters:\n' ' {1}: {2}\n' ' number_of_shards: {3}\n' ' number_of_replicas: {4}\n' ' shrink_prefix: {5}\n' ' shrink_suffix: {6}\n' ' delete_after: {7}\n' ' extra_settings:\n' ' settings:\n' ' {8}: {9}\n' ' aliases:\n' ' my_alias: {10}\n' ' post_allocation:\n' ' allocation_type: {11}\n' ' key: {12}\n' ' value: {13}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) COPY_ALIASES = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: shrink\n' ' options:\n' ' shrink_node: {0}\n' ' node_filters:\n' ' {1}: {2}\n' ' number_of_shards: {3}\n' ' number_of_replicas: {4}\n' ' shrink_prefix: {5}\n' ' shrink_suffix: {6}\n' ' copy_aliases: {7}\n' ' delete_after: {8}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) SHRINK_FILTER_BY_SHARDS = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: shrink\n' ' options:\n' ' shrink_node: {0}\n' ' node_filters:\n' ' {1}: {2}\n' ' number_of_shards: {3}\n' ' number_of_replicas: {4}\n' ' shrink_prefix: {5}\n' ' shrink_suffix: {6}\n' ' delete_after: {7}\n' ' wait_for_rebalance: {8}\n' ' filters:\n' ' - filtertype: shards\n' ' number_of_shards: {9}\n' ' shard_filter_behavior: {10}\n' ) class TestActionFileShrink(CuratorTestCase): def builder(self, action_args): self.idx = 'my_index' suffix = '-shrunken' self.target = f'{self.idx}{suffix}' self.create_index(self.idx, shards=2) self.add_docs(self.idx) # add alias in the source index self.alias = 'my_alias' alias_actions = [] alias_actions.append({'add': {'index': self.idx, 'alias': self.alias}}) self.client.indices.update_aliases(actions=alias_actions) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config(self.args['actionfile'], action_args) self.invoke_runner() def test_shrink(self): suffix = '-shrunken' self.builder( SHRINK.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'True', 'True', ) ) indices = get_indices(self.client) assert 1 == len(indices) assert indices[0] == self.target def test_permit_masters_false(self): suffix = '-shrunken' self.builder( NO_PERMIT_MASTERS.format('DETERMINISTIC', 1, 0, '', suffix, 'True') ) indices = get_indices(self.client) assert 1 == len(indices) assert indices[0] == self.idx assert 1 == self.result.exit_code def test_shrink_non_green(self): suffix = '-shrunken' self.client.indices.create( index='just_another_index', settings={'number_of_shards': 1, 'number_of_replicas': 1}, ) self.builder( SHRINK.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'True', 'True', ) ) indices = get_indices(self.client) assert 2 == len(indices) assert 1 == self.result.exit_code def test_shrink_with_extras(self): suffix = '-shrunken' allocation_type = 'exclude' key = '_name' value = 'not_this_node' self.builder( WITH_EXTRA_SETTINGS.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'False', 'index.codec', 'best_compression', dict(), allocation_type, key, value, ) ) indices = get_indices(self.client) assert 2 == len(indices) settings = self.client.indices.get_settings() assert ( value == settings[self.target]['settings']['index']['routing']['allocation'][ allocation_type ][key] ) assert ( '' == settings[self.idx]['settings']['index']['routing']['allocation'][ 'require' ]['_name'] ) assert 'best_compression' == settings[self.target]['settings']['index']['codec'] # This was erroneously testing for True in previous releases. But the target # index should never have had the alias as `copy_aliases` was never set # to True for this test. def test_shrink_with_copy_alias(self): suffix = '-shrunken' self.builder( COPY_ALIASES.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'True', 'True', ) ) indices = get_indices(self.client) assert 1 == len(indices) assert indices[0] == self.target assert self.client.indices.exists_alias(index=self.target, name=self.alias) def test_shrink_without_rebalance(self): suffix = '-shrunken' self.builder( SHRINK.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'True', 'False', ) ) indices = get_indices(self.client) assert 1 == len(indices) assert indices[0] == self.target def test_shrink_implicit_shard_filter(self): self.create_index('my_invalid_shrink_index', shards=1) self.create_index('my_valid_shrink_index', shards=5) suffix = '-shrunken' self.builder( SHRINK.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'True', 'False', ) ) indices = get_indices(self.client) assert 3 == len(indices) assert 'my_invalid_shrink_index-shrunken' not in indices assert 'my_valid_shrink_index-shrunken' in indices def test_shrink_explicit_shard_filter(self): self.create_index('my_invalid_shrink_index', shards=3) self.create_index('my_valid_shrink_index', shards=5) suffix = '-shrunken' self.builder( SHRINK_FILTER_BY_SHARDS.format( 'DETERMINISTIC', 'permit_masters', 'True', 1, 0, '', suffix, 'True', 'False', 5, 'greater_than_or_equal', ) ) indices = get_indices(self.client) assert 3 == len(indices) assert 'my_invalid_shrink_index-shrunken' not in indices assert 'my_valid_shrink_index-shrunken' in indices assert 'my_index-shrunken' not in indices class TestCLIShrink(CuratorTestCase): def builder(self): self.loogger = logging.getLogger('TestCLIShrink.builder') self.idx = 'my_index' self.suffix = '-shrunken' self.target = f'{self.idx}{self.suffix}' self.create_index(self.idx, shards=2) self.add_docs(self.idx) # add alias in the source index self.alias = 'my_alias' alias_actions = [] alias_actions.append({'add': {'index': self.idx, 'alias': self.alias}}) self.client.indices.update_aliases(actions=alias_actions) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.loogger.debug('Test pre-execution build phase complete.') def test_shrink(self): self.builder() args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'shrink', '--shrink_node', 'DETERMINISTIC', '--node_filters', '{"permit_masters":"true"}', '--number_of_shards', "1", '--number_of_replicas', "0", '--shrink_suffix', self.suffix, '--delete_after', '--wait_for_rebalance', '--filter_list', '{"filtertype":"none"}', ] assert 0 == self.run_subprocess(args, logname='TestCLIShrink.test_shrink') indices = get_indices(self.client) assert 1 == len(indices) assert indices[0] == self.target elasticsearch-curator-8.0.21/tests/integration/test_snapshot.py000066400000000000000000000112461477314666200250520ustar00rootroot00000000000000"""Test snapshot action functionality""" # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long import os from datetime import datetime, timedelta, timezone from curator.helpers.getters import get_indices, get_snapshot from curator.exceptions import FailedExecution from . import CuratorTestCase from . import testvars HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') class TestActionFileSnapshot(CuratorTestCase): def test_snapshot(self): self.create_indices(5) self.create_repository() snap_name = 'snapshot1' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.snapshot_test.format(self.args['repository'], snap_name, 1, 30), ) self.invoke_runner() snapshot = get_snapshot(self.client, self.args['repository'], '*') assert 1 == len(snapshot['snapshots']) assert snap_name == snapshot['snapshots'][0]['snapshot'] def test_snapshot_datemath(self): self.create_indices(5) self.create_repository() snap_name = '' _ = (datetime.now(timezone.utc) - timedelta(days=1)).strftime('%Y.%m.%d') snap_name_parsed = f"snapshot-{_}" self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.snapshot_test.format(self.args['repository'], snap_name, 1, 30), ) self.invoke_runner() snapshot = get_snapshot(self.client, self.args['repository'], '*') assert 1 == len(snapshot['snapshots']) assert snap_name_parsed == snapshot['snapshots'][0]['snapshot'] def test_snapshot_ignore_empty_list(self): self.create_indices(5) self.create_repository() snap_name = 'snapshot1' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.test_682.format(self.args['repository'], snap_name, True, 1, 30), ) self.invoke_runner() snapshot = {'snapshots': []} try: snapshot = get_snapshot(self.client, self.args['repository'], '*') except FailedExecution: pass assert 0 == len(snapshot['snapshots']) assert 0 == len(get_indices(self.client)) def test_snapshot_do_not_ignore_empty_list(self): self.create_indices(5) self.create_repository() snap_name = 'snapshot1' self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.test_682.format(self.args['repository'], snap_name, False, 1, 30), ) self.invoke_runner() snapshot = {'snapshots': []} try: snapshot = get_snapshot(self.client, self.args['repository'], '*') except FailedExecution: pass assert 0 == len(snapshot['snapshots']) assert 5 == len(get_indices(self.client)) def test_no_repository(self): self.create_indices(5) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.snapshot_test.format(' ', 'snap_name', 1, 30), ) self.invoke_runner() assert 1 == self.result.exit_code def test_extra_option(self): self.create_indices(5) self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) self.write_config( self.args['actionfile'], testvars.bad_option_proto_test.format('snapshot') ) self.invoke_runner() assert 1 == self.result.exit_code class TestCLISnapshot(CuratorTestCase): def test_snapshot(self): self.create_indices(5) self.create_repository() snap_name = 'snapshot1' args = self.get_runner_args() args += [ '--config', self.args['configfile'], 'snapshot', '--repository', self.args['repository'], '--name', snap_name, '--wait_interval', '1', '--max_wait', '30', '--filter_list', '{"filtertype":"none"}', ] assert 0 == self.run_subprocess(args, logname='TestCLISnapshot.test_snapshot') snapshot = get_snapshot(self.client, self.args['repository'], '*') assert 1 == len(snapshot['snapshots']) assert snap_name == snapshot['snapshots'][0]['snapshot'] elasticsearch-curator-8.0.21/tests/integration/testvars.py000066400000000000000000000637321477314666200240360ustar00rootroot00000000000000"""Test variables""" # pylint: disable=C0103, C0302 client_config = ( '---\n' 'elasticsearch:\n' ' client:\n' ' hosts: {0}\n' ' request_timeout: 30\n' ' other_settings:\n' ' master_only: False\n' '\n' 'logging:\n' ' loglevel: DEBUG\n' ' logfile:\n' ' logformat: default\n' ' blacklist: []\n' ) client_conf_logfile = ( '---\n' 'elasticsearch:\n' ' client:\n' ' hosts: {0}\n' '\n' 'logging:\n' ' loglevel: DEBUG\n' ' logfile: {1}\n' ) client_config_envvars = ( '---\n' 'elasticsearch:\n' ' client:\n' ' hosts: {0}\n' ' request_timeout: {1}\n' ' other_settings:\n' ' master_only: False\n' '\n' 'logging:\n' ' loglevel: DEBUG\n' ' logfile:\n' ' logformat: default\n' ' blacklist: []\n' ) bad_client_config = ( '---\n' 'elasticsearch:\n' ' misspelled:\n' ' hosts: {0}\n' ' certificate:\n' ' client_cert:\n' ' client_key:\n' ' request_timeout: 30\n' ' other_settings:\n' ' master_only: False\n' ) no_logging_config = ( '---\n' 'elasticsearch:\n' ' client:\n' ' hosts: {0}\n' ' certificate:\n' ' client_cert:\n' ' client_key:\n' ' request_timeout: 30\n' ' other_settings:\n' ' master_only: False\n' ) none_logging_config = ( '---\n' 'elasticsearch:\n' ' client:\n' ' hosts: {0}\n' ' certificate:\n' ' client_cert:\n' ' client_key:\n' ' request_timeout: 30\n' ' other_settings:\n' ' master_only: False\n' '\n' 'logging: \n' '\n' ) alias_add_only = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add all indices to specified alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' add:\n' ' filters:\n' ' - filtertype: none\n' ) alias_add_only_with_extra_settings = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add all indices to specified alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' extra_settings:\n' ' filter:\n' ' term:\n' ' user: kimchy\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' add:\n' ' filters:\n' ' - filtertype: none\n' ) alias_remove_only = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Remove all indices from specified alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: none\n' ) alias_add_remove = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add/remove specified indices from designated alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: du\n' ' add:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) alias_remove_index_not_there = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add/remove specified indices from designated alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: {1}\n' ) alias_add_with_empty_remove = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add/remove specified indices from designated alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' warn_if_no_indices: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: insertrickrollhere\n' ' add:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) alias_remove_with_empty_add = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add/remove specified indices from designated alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' warn_if_no_indices: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: du\n' ' add:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: insertrickrollhere\n' ) alias_add_remove_empty = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Add/remove specified indices from designated alias"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: {1}\n' ' add:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: {2}\n' ) alias_no_add_remove = ( '---\n' 'actions:\n' ' 1:\n' ' description: "No add or remove should raise an exception"\n' ' action: alias\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ) alias_no_alias = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Removing alias from options should result in an exception"\n' ' action: alias\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: du\n' ' add:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) allocation_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Allocate by key/value/allocation_type"\n' ' action: allocation\n' ' options:\n' ' key: {0}\n' ' value: {1}\n' ' allocation_type: {2}\n' ' wait_for_completion: {3}\n' ' wait_interval: 1\n' ' max_wait: -1\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) allocation_count_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Allocate by key/value/allocation_type"\n' ' action: allocation\n' ' options:\n' ' key: {0}\n' ' value: {1}\n' ' allocation_type: {2}\n' ' wait_for_completion: {3}\n' ' wait_interval: 1\n' ' max_wait: -1\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: count\n' ' use_age: True\n' ' source: name\n' ' timestring: \'%Y.%m.%d\'\n' ' count: 2\n' ' exclude: False\n' ) cluster_routing_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Alter cluster routing by routing_type/value"\n' ' action: cluster_routing\n' ' options:\n' ' routing_type: {0}\n' ' value: {1}\n' ' setting: enable\n' ) optionless_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: {0}\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) no_options_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: {0}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) actionless_proto = ( '---\n' 'actions:\n' ' 1:\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ) disabled_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: {0}\n' ' options:\n' ' continue_if_exception: False\n' ' ignore_empty_list: True\n' ' disable_action: True\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ' 2:\n' ' description: "Act on indices as filtered"\n' ' action: {1}\n' ' options:\n' ' continue_if_exception: False\n' ' ignore_empty_list: True\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: log\n' ) continue_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Create named index"\n' ' action: create_index\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: {1}\n' ' disable_action: False\n' ' 2:\n' ' description: "Act on indices as filtered"\n' ' action: {2}\n' ' options:\n' ' continue_if_exception: {3}\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: log\n' ) close_delete_aliases = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Close indices as filtered"\n' ' action: close\n' ' options:\n' ' delete_aliases: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) close_skip_flush = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Close indices as filtered"\n' ' action: close\n' ' options:\n' ' skip_flush: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) close_ignore_sync = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Close indices as filtered"\n' ' action: close\n' ' options:\n' ' ignore_sync_failures: {0}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) delete_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: {0}\n' ' source: {1}\n' ' direction: {2}\n' ' timestring: {3}\n' ' unit: {4}\n' ' unit_count: {5}\n' ' field: {6}\n' ' stats_result: {7}\n' ' epoch: {8}\n' ) delete_pattern_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: {0}\n' ' source: {1}\n' ' direction: {2}\n' ' timestring: {3}\n' ' unit: {4}\n' ' unit_count: {5}\n' ' unit_count_pattern: {6}\n' ) delete_period_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: ilm-history-\n' ' exclude: True\n' ' - filtertype: {0}\n' ' source: {1}\n' ' range_from: {2}\n' ' range_to: {3}\n' ' timestring: {4}\n' ' unit: {5}\n' ' field: {6}\n' ' stats_result: {7}\n' ' intersect: {8}\n' ' epoch: {9}\n' ' week_starts_on: {10}\n' ) delete_ignore_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' ignore_empty_list: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: {0}\n' ' source: {1}\n' ' direction: {2}\n' ' timestring: {3}\n' ' unit: {4}\n' ' unit_count: {5}\n' ' field: {6}\n' ' stats_result: {7}\n' ' epoch: {8}\n' ) filter_by_alias = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' ignore_empty_list: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: alias\n' ' aliases: {0}\n' ' exclude: {1}\n' ) filter_closed = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' ignore_empty_list: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: closed\n' ' exclude: {0}\n' ) bad_option_proto_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Should raise exception due to extra option"\n' ' action: {0}\n' ' options:\n' ' invalid: this_should_not_be_here\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: none\n' ) bad_option_rollover_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Should raise exception due to extra option"\n' ' action: {0}\n' ' options:\n' ' invalid: this_should_not_be_here\n' ' continue_if_exception: False\n' ' disable_action: False\n' ) replicas_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Increase replica count to provided value"\n' ' action: replicas\n' ' options:\n' ' count: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) forcemerge_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: >-\n' ' forceMerge segment count per shard to provided value with optional delay\n' ' action: forcemerge\n' ' options:\n' ' max_num_segments: {0}\n' ' delay: {1}\n' ' timeout_override: 300\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) snapshot_test = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Snapshot selected indices"\n' ' action: snapshot\n' ' options:\n' ' repository: {0}\n' ' name: {1}\n' ' wait_interval: {2}\n' ' max_wait: {3}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: none\n' ) delete_snap_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete snapshots as filtered"\n' ' action: delete_snapshots\n' ' options:\n' ' repository: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: {1}\n' ' source: {2}\n' ' direction: {3}\n' ' timestring: {4}\n' ' unit: {5}\n' ' unit_count: {6}\n' ' epoch: {7}\n' ) create_index = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Create index as named"\n' ' action: create_index\n' ' options:\n' ' name: {0}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ) create_index_with_extra_settings = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Create index as named with extra settings"\n' ' action: create_index\n' ' options:\n' ' name: {0}\n' ' extra_settings:\n' ' aliases:\n' ' {1}:\n' ' is_write_index: True\n' ' settings:\n' ' number_of_shards: 1\n' ' number_of_replicas: 0\n' ' mappings:\n' ' properties:\n' ' {2}:\n' ' type: {3}\n' ' {4}:\n' ' type: {5}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ) restore_snapshot_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: Restore snapshot as configured\n' ' action: restore\n' ' options:\n' ' repository: {0}\n' ' name: {1}\n' ' indices: {2}\n' ' include_aliases: {3}\n' ' ignore_unavailable: {4}\n' ' include_global_state: {5}\n' ' partial: {6}\n' ' rename_pattern: {7}\n' ' rename_replacement: {8}\n' ' extra_settings: {9}\n' ' wait_for_completion: {10}\n' ' skip_repo_fs_check: {11}\n' ' timeout_override: {12}\n' ' wait_interval: {13}\n' ' max_wait: {14}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: none\n' ) test_687 = ( '---\n' 'actions:\n' ' 1:\n' ' action: snapshot\n' ' description: >-\n' ' Create a snapshot with the last week index.\n' ' options:\n' ' repository: {0}\n' ' name: {1}\n' ' ignore_unavailable: False\n' ' include_global_state: True\n' ' partial: False\n' ' wait_for_completion: True\n' ' skip_repo_fs_check: False\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: logstash-\n' ' exclude:\n' ' - filtertype: age\n' ' source: creation_date\n' ' direction: younger\n' ' epoch: 1467020729\n' ' unit: weeks\n' ' unit_count: 2\n' ' exclude:\n' ' - filtertype: age\n' ' source: creation_date\n' ' direction: younger\n' ' epoch: 1467020729\n' ' unit: weeks\n' ' unit_count: 1\n' ' exclude: True\n' ' 2:\n' ' action: delete_indices\n' ' description: >-\n' ' Remove indices starting with logstash- older than 5 weeks\n' ' options:\n' ' ignore_empty_list: True\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: logstash-\n' ' exclude:\n' ' - filtertype: age\n' ' source: creation_date\n' ' epoch: 1467020729\n' ' direction: older\n' ' unit: weeks\n' ' unit_count: 5\n' ' exclude:\n' ) test_682 = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Snapshot selected indices"\n' ' action: snapshot\n' ' options:\n' ' repository: {0}\n' ' name: {1}\n' ' ignore_empty_list: {2}\n' ' wait_interval: {3}\n' ' max_wait: {4}\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: notlogstash-\n' ' exclude:\n' ' 2:\n' ' description: "Delete selected indices"\n' ' action: delete_indices\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: logstash-\n' ' exclude:\n' ) CRA_all = { 'persistent': {}, 'transient': {'cluster': {'routing': {'allocation': {'enable': 'all'}}}}, } rollover_one = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Rollover selected alias/index"\n' ' action: rollover\n' ' options:\n' ' name: {0}\n' ' conditions: \n' ' {1}: {2}\n' ' extra_settings:\n' ' index.number_of_shards: 1\n' ' index.number_of_replicas: 0\n' ) rollover_both = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Rollover selected alias/index"\n' ' action: rollover\n' ' options:\n' ' name: {0}\n' ' conditions: \n' ' max_age: {1}\n' ' max_docs: {2}\n' ' extra_settings:\n' ' index.number_of_shards: 1\n' ' index.number_of_replicas: 0\n' ) rollover_bad_settings = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Rollover selected alias/index"\n' ' action: rollover\n' ' options:\n' ' name: {0}\n' ' conditions: \n' ' max_age: {1}\n' ' max_docs: {2}\n' ' extra_settings:\n' ' foo: 1\n' ' bar: 0\n' ) rollover_with_name = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Rollover selected alias/index"\n' ' action: rollover\n' ' options:\n' ' name: {0}\n' ' conditions: \n' ' {1}: {2}\n' ' new_index: {3}\n' ' extra_settings:\n' ' index.number_of_shards: 1\n' ' index.number_of_replicas: 0\n' ) reindex = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Reindex"\n' ' action: reindex\n' ' options:\n' ' allow_ilm_indices: true\n' ' wait_interval: {0}\n' ' max_wait: {1}\n' ' request_body:\n' ' source:\n' ' index: {2}\n' ' dest:\n' ' index: {3}\n' ' filters:\n' ' - filtertype: none\n' ) reindex_empty_list = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Reindex"\n' ' action: reindex\n' ' options:\n' ' ignore_empty_list: {0}\n' ' wait_interval: {1}\n' ' max_wait: {2}\n' ' request_body:\n' ' source:\n' ' index: REINDEX_SELECTED\n' ' dest:\n' ' index: {3}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: notfound\n' ) remote_reindex = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Reindex from remote"\n' ' action: reindex\n' ' options:\n' ' wait_interval: {0}\n' ' max_wait: {1}\n' ' request_body:\n' ' source:\n' ' remote:\n' ' host: {2}\n' ' index: {3}\n' ' dest:\n' ' index: {4}\n' ' remote_filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: {5}\n' ' filters:\n' ' - filtertype: none\n' ) migration_reindex = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Reindex from remote"\n' ' action: reindex\n' ' options:\n' ' wait_interval: {0}\n' ' max_wait: {1}\n' ' migration_prefix: {2}\n' ' migration_suffix: {3}\n' ' request_body:\n' ' source:\n' ' remote:\n' ' host: {4}\n' ' index: {5}\n' ' dest:\n' ' index: {6}\n' ' remote_filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: {7}\n' ' filters:\n' ' - filtertype: none\n' ) test_945 = ( '---\n' 'actions:\n' ' 1:\n' ' action: delete_indices\n' ' description: >-\n' ' Delete indices older than 7 days\n' ' options:\n' ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: logstash-\n' ' exclude:\n' ' - filtertype: age\n' ' source: name\n' ' direction: older\n' ' unit: days\n' ' unit_count: 7\n' ) index_settings = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Act on indices as filtered"\n' ' action: index_settings\n' ' options:\n' ' index_settings:\n' ' index:\n' ' {0}: {1}\n' ' ignore_unavailable: {2}\n' ' preserve_existing: {3}\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' ) ilm_delete_proto = ( '---\n' 'actions:\n' ' 1:\n' ' description: "Delete indices as filtered"\n' ' action: delete_indices\n' ' options:\n' ' allow_ilm_indices: {9}\n' ' filters:\n' ' - filtertype: {0}\n' ' source: {1}\n' ' direction: {2}\n' ' timestring: {3}\n' ' unit: {4}\n' ' unit_count: {5}\n' ' field: {6}\n' ' stats_result: {7}\n' ' epoch: {8}\n' ) elasticsearch-curator-8.0.21/tests/unit/000077500000000000000000000000001477314666200202325ustar00rootroot00000000000000elasticsearch-curator-8.0.21/tests/unit/__init__.py000066400000000000000000000031511477314666200223430ustar00rootroot00000000000000import os import shutil import tempfile import random import string from unittest import SkipTest, TestCase from unittest.mock import Mock from .testvars import * class CLITestCase(TestCase): def setUp(self): super(CLITestCase, self).setUp() self.args = {} dirname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) ymlname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) badyaml = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) # This will create a psuedo-random temporary directory on the machine # which runs the unit tests, but NOT on the machine where elasticsearch # is running. This means tests may fail if run against remote instances # unless you explicitly set `self.args['location']` to a proper spot # on the target machine. self.args['tmpdir'] = tempfile.mkdtemp(suffix=dirname) if not os.path.exists(self.args['tmpdir']): os.makedirs(self.args['tmpdir']) self.args['yamlfile'] = os.path.join(self.args['tmpdir'], ymlname) self.args['invalid_yaml'] = os.path.join(self.args['tmpdir'], badyaml) self.args['no_file_here'] = os.path.join(self.args['tmpdir'], 'not_created') with open(self.args['yamlfile'], 'w') as f: f.write(testvars.yamlconfig) with open(self.args['invalid_yaml'], 'w') as f: f.write('gobbledeygook: @failhere\n') def tearDown(self): if os.path.exists(self.args['tmpdir']): shutil.rmtree(self.args['tmpdir']) elasticsearch-curator-8.0.21/tests/unit/test_action_alias.py000066400000000000000000000103341477314666200242720ustar00rootroot00000000000000"""Alias unit tests""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator import IndexList from curator.exceptions import ActionError, FailedExecution, MissingArgument, NoIndices from curator.actions.alias import Alias from . import testvars class TestActionAlias(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def builder2(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_two self.client.indices.get_settings.return_value = testvars.settings_two self.client.indices.stats.return_value = testvars.stats_two self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_init_raise(self): self.assertRaises(MissingArgument, Alias) def test_add_raises_on_missing_parameter(self): self.builder() ao = Alias(name='alias') self.assertRaises(TypeError, ao.add) def test_add_raises_on_invalid_parameter(self): self.builder() ao = Alias(name='alias') self.assertRaises(TypeError, ao.add, []) def test_add_single(self): self.builder() ao = Alias(name='alias') ao.add(self.ilo) self.assertEqual(testvars.alias_one_add, ao.actions) def test_add_single_with_extra_settings(self): self.builder() esd = { 'filter' : { 'term' : { 'user' : 'kimchy' } } } ao = Alias(name='alias', extra_settings=esd) ao.add(self.ilo) self.assertEqual(testvars.alias_one_add_with_extras, ao.actions) def test_remove_single(self): self.builder() self.client.indices.get_alias.return_value = testvars.settings_1_get_aliases ao = Alias(name='my_alias') ao.remove(self.ilo) self.assertEqual(testvars.alias_one_rm, ao.actions) def test_add_multiple(self): self.builder2() ao = Alias(name='alias') ao.add(self.ilo) cmp = sorted(ao.actions, key=lambda k: k['add']['index']) self.assertEqual(testvars.alias_two_add, cmp) def test_remove_multiple(self): self.builder2() self.client.indices.get_alias.return_value = testvars.settings_2_get_aliases ao = Alias(name='my_alias') ao.remove(self.ilo) cmp = sorted(ao.actions, key=lambda k: k['remove']['index']) self.assertEqual(testvars.alias_two_rm, cmp) def test_raise_action_error_on_empty_body(self): self.builder() ao = Alias(name='alias') self.assertRaises(ActionError, ao.check_actions) def test_raise_no_indices_on_empty_body_when_warn_if_no_indices(self): self.builder() # empty it, so there can be no body self.ilo.indices = [] ao = Alias(name='alias') ao.add(self.ilo, warn_if_no_indices=True) self.assertRaises(NoIndices, ao.check_actions) def test_do_dry_run(self): self.builder() self.client.indices.update_aliases.return_value = testvars.alias_success ao = Alias(name='alias') ao.add(self.ilo) self.assertIsNone(ao.do_dry_run()) def test_do_action(self): self.builder() self.client.indices.update_aliases.return_value = testvars.alias_success ao = Alias(name='alias') ao.add(self.ilo) self.assertIsNone(ao.do_action()) def test_do_action_raises_exception(self): self.builder() self.client.indices.update_aliases.return_value = testvars.alias_success self.client.indices.update_aliases.side_effect = testvars.four_oh_one ao = Alias(name='alias') ao.add(self.ilo) self.assertRaises(FailedExecution, ao.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_allocation.py000066400000000000000000000057071477314666200253360ustar00rootroot00000000000000"""Alias unit tests""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator import IndexList from curator.exceptions import MissingArgument from curator.actions.allocation import Allocation from . import testvars class TestActionAllocation(TestCase): VERSION = {'version': {'number': '5.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.client.indices.put_settings.return_value = None self.ilo = IndexList(self.client) def test_init_raise(self): self.assertRaises(TypeError, Allocation, 'invalid') def test_init(self): self.builder() ao = Allocation(self.ilo, key='key', value='value') self.assertEqual(self.ilo, ao.index_list) self.assertEqual(self.client, ao.client) def test_create_body_no_key(self): self.builder() self.assertRaises(MissingArgument, Allocation, self.ilo) def test_create_body_invalid_allocation_type(self): self.builder() self.assertRaises( ValueError, Allocation, self.ilo, key='key', value='value', allocation_type='invalid' ) def test_create_body_valid(self): self.builder() ao = Allocation(self.ilo, key='key', value='value') self.assertEqual({'index.routing.allocation.require.key': 'value'}, ao.settings) def test_do_action_raise_on_put_settings(self): self.builder() self.client.indices.put_settings.side_effect = testvars.fake_fail ao = Allocation(self.ilo, key='key', value='value') self.assertRaises(Exception, ao.do_action) def test_do_dry_run(self): self.builder() alo = Allocation(self.ilo, key='key', value='value') self.assertIsNone(alo.do_dry_run()) def test_do_action(self): self.builder() alo = Allocation(self.ilo, key='key', value='value') self.assertIsNone(alo.do_action()) def test_do_action_wait_v50(self): self.builder() self.client.cluster.health.return_value = {'relocating_shards':0} alo = Allocation( self.ilo, key='key', value='value', wait_for_completion=True) self.assertIsNone(alo.do_action()) def test_do_action_wait_v51(self): self.builder() self.client.info.return_value = {'version': {'number': '5.1.1'} } self.client.cluster.health.return_value = {'relocating_shards':0} alo = Allocation( self.ilo, key='key', value='value', wait_for_completion=True) self.assertIsNone(alo.do_action()) elasticsearch-curator-8.0.21/tests/unit/test_action_close.py000066400000000000000000000051651477314666200243140ustar00rootroot00000000000000"""Alias unit tests""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator import IndexList from curator.exceptions import FailedExecution from curator.actions.close import Close from . import testvars class TestActionClose(TestCase): VERSION = {'version': {'number': '5.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.flush_synced.return_value = testvars.synced_pass self.client.indices.exists_alias.return_value = False self.client.indices.close.return_value = None self.ilo = IndexList(self.client) def test_init_raise(self): self.assertRaises(TypeError, Close, 'invalid') def test_init(self): self.builder() self.client.indices.flush_synced.return_value = None self.client.indices.close.return_value = None clo = Close(self.ilo) self.assertEqual(self.ilo, clo.index_list) self.assertEqual(self.client, clo.client) def test_do_dry_run(self): self.builder() self.ilo = IndexList(self.client) clo = Close(self.ilo) self.assertIsNone(clo.do_dry_run()) def test_do_action(self): self.builder() self.ilo = IndexList(self.client) clo = Close(self.ilo) self.assertIsNone(clo.do_action()) def test_do_action_with_delete_aliases(self): self.builder() self.ilo = IndexList(self.client) clo = Close(self.ilo, delete_aliases=True) self.assertIsNone(clo.do_action()) def test_do_action_with_skip_flush(self): self.builder() self.ilo = IndexList(self.client) clo = Close(self.ilo, skip_flush=True) self.assertIsNone(clo.do_action()) def test_do_action_raises_exception(self): self.builder() self.client.indices.close.side_effect = testvars.fake_fail self.ilo = IndexList(self.client) clo = Close(self.ilo) self.assertRaises(FailedExecution, clo.do_action) def test_do_action_delete_aliases_with_exception(self): self.builder() self.ilo = IndexList(self.client) self.client.indices.delete_alias.side_effect = testvars.fake_fail clo = Close(self.ilo, delete_aliases=True) self.assertIsNone(clo.do_action()) elasticsearch-curator-8.0.21/tests/unit/test_action_clusterrouting.py000066400000000000000000000045241477314666200262760ustar00rootroot00000000000000"""test_action_clusterrouting""" from unittest import TestCase from unittest.mock import Mock from curator.actions import ClusterRouting # Get test variables and constants from a single source from . import testvars class TestActionAllocation(TestCase): def test_bad_client(self): self.assertRaises(TypeError, ClusterRouting, 'invalid', setting='enable') def test_bad_setting(self): client = Mock() self.assertRaises( ValueError, ClusterRouting, client, setting='invalid' ) def test_bad_routing_type(self): client = Mock() self.assertRaises( ValueError, ClusterRouting, client, routing_type='invalid', setting='enable' ) def test_bad_value_with_allocation(self): client = Mock() self.assertRaises( ValueError, ClusterRouting, client, routing_type='allocation', setting='enable', value='invalid' ) def test_bad_value_with_rebalance(self): client = Mock() self.assertRaises( ValueError, ClusterRouting, client, routing_type='rebalance', setting='enable', value='invalid' ) def test_do_dry_run(self): client = Mock() cro = ClusterRouting( client, routing_type='allocation', setting='enable', value='all' ) self.assertIsNone(cro.do_dry_run()) def test_do_action_raise_on_put_settings(self): client = Mock() client.cluster.put_settings.return_value = None client.cluster.put_settings.side_effect = testvars.fake_fail cro = ClusterRouting( client, routing_type='allocation', setting='enable', value='all' ) self.assertRaises(Exception, cro.do_action) def test_do_action_wait(self): client = Mock() client.cluster.put_settings.return_value = None client.cluster.health.return_value = {'relocating_shards':0} cro = ClusterRouting( client, routing_type='allocation', setting='enable', value='all', wait_for_completion=True ) self.assertIsNone(cro.do_action()) elasticsearch-curator-8.0.21/tests/unit/test_action_cold2frozen.py000066400000000000000000000112471477314666200254340ustar00rootroot00000000000000"""test_action_cold2frozen""" # pylint: disable=attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock import pytest from curator.actions import Cold2Frozen from curator.exceptions import CuratorException, SearchableSnapshotException from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionCold2Frozen(TestCase): """TestActionCold2Frozen""" VERSION = {'version': {'number': '8.0.0'} } def builder(self): """Environment builder""" self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_init_raise_bad_index_list(self): """test_init_raise_bad_index_list""" self.assertRaises(TypeError, Cold2Frozen, 'invalid') with pytest.raises(TypeError): Cold2Frozen('not_an_IndexList') def test_init_add_kwargs(self): """test_init_add_kwargs""" self.builder() testval = {'key': 'value'} c2f = Cold2Frozen(self.ilo, index_settings=testval) assert c2f.index_settings == testval def test_action_generator1(self): """test_action_generator1""" self.builder() settings_ss = { testvars.named_index: { 'aliases': {'my_alias': {}}, 'settings': { 'index': { 'creation_date': '1456963200172', 'refresh_interval': '5s', 'lifecycle': { 'indexing_complete': True }, 'store': { 'type': 'snapshot', 'snapshot': { 'snapshot_name': 'snapname', 'index_name': testvars.named_index, 'repository_name': 'reponame', } } } } } } self.client.indices.get_settings.return_value = settings_ss self.client.indices.get_alias.return_value = settings_ss roles = ['data_content'] self.client.nodes.info.return_value = {'nodes': {'nodename': {'roles': roles}}} c2f = Cold2Frozen(self.ilo) snap = 'snapname' repo = 'reponame' renamed = f'partial-{testvars.named_index}' settings = { "routing": { "allocation": { "include": { "_tier_preference": roles[0] } } } } expected = { 'repository': repo, 'snapshot': snap, 'index': testvars.named_index, 'renamed_index': renamed, 'index_settings': settings, 'ignore_index_settings': ['index.refresh_interval'], 'storage': 'shared_cache', 'wait_for_completion': True, 'aliases': {'my_alias': {}}, 'current_idx': testvars.named_index } for result in c2f.action_generator(): assert result == expected c2f.do_dry_run() # Do this here as it uses the same generator output. def test_action_generator2(self): """test_action_generator2""" self.builder() settings_ss = { testvars.named_index: { 'settings': {'index': {'lifecycle': {'name': 'guaranteed_fail'}}} } } self.client.indices.get_settings.return_value = settings_ss c2f = Cold2Frozen(self.ilo) with pytest.raises(CuratorException, match='associated with an ILM policy'): for result in c2f.action_generator(): _ = result def test_action_generator3(self): """test_action_generator3""" self.builder() settings_ss = { testvars.named_index: { 'settings': { 'index': { 'lifecycle': {'indexing_complete': True}, 'store': {'snapshot': {'partial': True}} } } } } self.client.indices.get_settings.return_value = settings_ss c2f = Cold2Frozen(self.ilo) with pytest.raises(SearchableSnapshotException, match='Index is already in frozen tier'): for result in c2f.action_generator(): _ = result elasticsearch-curator-8.0.21/tests/unit/test_action_create_index.py000066400000000000000000000025441477314666200256370ustar00rootroot00000000000000"""Unit tests for create_index action""" from unittest import TestCase from unittest.mock import Mock from curator.actions import CreateIndex from curator.exceptions import ConfigurationError, FailedExecution # Get test variables and constants from a single source from . import testvars class TestActionCreate_index(TestCase): def test_init_raise(self): self.assertRaises(TypeError, CreateIndex, name='name') def test_init_raise_no_name(self): client = Mock() self.assertRaises(ConfigurationError, CreateIndex, client, name=None) def test_init(self): client = Mock() co = CreateIndex(client, name='name') self.assertEqual('name', co.name) self.assertEqual(client, co.client) def test_do_dry_run(self): client = Mock() co = CreateIndex(client, name='name') self.assertIsNone(co.do_dry_run()) def test_do_action(self): client = Mock() client.indices.create.return_value = None co = CreateIndex(client, name='name') self.assertIsNone(co.do_action()) def test_do_action_raises_exception(self): client = Mock() client.indices.create.return_value = None client.indices.create.side_effect = testvars.fake_fail co = CreateIndex(client, name='name') self.assertRaises(FailedExecution, co.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_delete_indices.py000066400000000000000000000051221477314666200261400ustar00rootroot00000000000000"""test_action_delete_indices""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import DeleteIndices from curator.exceptions import FailedExecution from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionDeleteIndices(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def builder4(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_four self.client.indices.get_settings.return_value = testvars.settings_four self.client.indices.stats.return_value = testvars.stats_four self.client.indices.exists_alias.return_value = False self.client.indices.delete.return_value = None self.ilo = IndexList(self.client) def test_init_raise(self): self.assertRaises(TypeError, DeleteIndices, 'invalid') def test_init_raise_bad_master_timeout(self): self.builder() self.assertRaises(TypeError, DeleteIndices, self.ilo, 'invalid') def test_init(self): self.builder() dio = DeleteIndices(self.ilo) self.assertEqual(self.ilo, dio.index_list) self.assertEqual(self.client, dio.client) def test_do_dry_run(self): self.builder4() dio = DeleteIndices(self.ilo) self.assertIsNone(dio.do_dry_run()) def test_do_action(self): self.builder4() dio = DeleteIndices(self.ilo) self.assertIsNone(dio.do_action()) def test_do_action_not_successful(self): self.builder4() dio = DeleteIndices(self.ilo) self.assertIsNone(dio.do_action()) def test_do_action_raises_exception(self): self.builder4() self.client.indices.delete.side_effect = testvars.fake_fail dio = DeleteIndices(self.ilo) self.assertRaises(FailedExecution, dio.do_action) def test_verify_result_positive(self): self.builder4() dio = DeleteIndices(self.ilo) self.assertTrue(dio._verify_result([],2)) elasticsearch-curator-8.0.21/tests/unit/test_action_delete_snapshots.py000066400000000000000000000063421477314666200265510ustar00rootroot00000000000000"""test_action_delete_snapshots""" from unittest import TestCase from unittest.mock import Mock from curator.actions import DeleteSnapshots from curator.exceptions import FailedExecution from curator import SnapshotList # Get test variables and constants from a single source from . import testvars as testvars class TestActionDeleteSnapshots(TestCase): def test_init_raise(self): self.assertRaises(TypeError, DeleteSnapshots, 'invalid') def test_init(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) do = DeleteSnapshots(slo) self.assertEqual(slo, do.snapshot_list) self.assertEqual(client, do.client) def test_do_dry_run(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.tasks.get.return_value = testvars.no_snap_tasks client.snapshot.delete.return_value = None slo = SnapshotList(client, repository=testvars.repo_name) do = DeleteSnapshots(slo) self.assertIsNone(do.do_dry_run()) def test_do_action(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.tasks.list.return_value = testvars.no_snap_tasks client.snapshot.delete.return_value = None slo = SnapshotList(client, repository=testvars.repo_name) do = DeleteSnapshots(slo) self.assertIsNone(do.do_action()) def test_do_action_raises_exception(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.delete.return_value = None client.tasks.list.return_value = testvars.no_snap_tasks client.snapshot.delete.side_effect = testvars.fake_fail slo = SnapshotList(client, repository=testvars.repo_name) do = DeleteSnapshots(slo) self.assertRaises(FailedExecution, do.do_action) ### This check is not necessary after ES 7.16 as it is possible to have ### up to 1000 concurrent snapshots ### ### https://www.elastic.co/guide/en/elasticsearch/reference/8.6/snapshot-settings.html ### snapshot.max_concurrent_operations ### (Dynamic, integer) Maximum number of concurrent snapshot operations. Defaults to 1000. ### ### This limit applies in total to all ongoing snapshot creation, cloning, and deletion ### operations. Elasticsearch will reject any operations that would exceed this limit. # def test_not_safe_to_snap_raises_exception(self): # client = Mock() # client.snapshot.get.return_value = testvars.inprogress # client.snapshot.get_repository.return_value = testvars.test_repo # client.tasks.list.return_value = testvars.no_snap_tasks # slo = SnapshotList(client, repository=testvars.repo_name) # do = DeleteSnapshots(slo, retry_interval=0, retry_count=1) # self.assertRaises(curator.FailedExecution, do.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_forcemerge.py000066400000000000000000000051031477314666200253150ustar00rootroot00000000000000"""test_action_forcemerge""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import ForceMerge from curator.exceptions import FailedExecution, MissingArgument from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionForceMerge(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.client.indices.segments.return_value = testvars.shards self.ilo = IndexList(self.client) def test_init_raise_bad_client(self): self.assertRaises( TypeError, ForceMerge, 'invalid', max_num_segments=2) def test_init_raise_no_segment_count(self): self.builder() self.assertRaises(MissingArgument, ForceMerge, self.ilo) def test_init(self): self.builder() fmo = ForceMerge(self.ilo, max_num_segments=2) self.assertEqual(self.ilo, fmo.index_list) self.assertEqual(self.client, fmo.client) def test_do_dry_run(self): self.builder() self.client.indices.forcemerge.return_value = None self.client.indices.optimize.return_value = None fmo = ForceMerge(self.ilo, max_num_segments=2) self.assertIsNone(fmo.do_dry_run()) def test_do_action(self): self.builder() self.client.indices.forcemerge.return_value = None fmo = ForceMerge(self.ilo, max_num_segments=2) self.assertIsNone(fmo.do_action()) def test_do_action_with_delay(self): self.builder() self.client.indices.forcemerge.return_value = None fmo = ForceMerge(self.ilo, max_num_segments=2, delay=0.050) self.assertIsNone(fmo.do_action()) def test_do_action_raises_exception(self): self.builder() self.client.indices.forcemerge.return_value = None self.client.indices.optimize.return_value = None self.client.indices.forcemerge.side_effect = testvars.fake_fail self.client.indices.optimize.side_effect = testvars.fake_fail fmo = ForceMerge(self.ilo, max_num_segments=2) self.assertRaises(FailedExecution, fmo.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_indexsettings.py000066400000000000000000000102201477314666200260630ustar00rootroot00000000000000"""test_action_indexsettings""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import IndexSettings from curator.exceptions import ActionError, ConfigurationError, MissingArgument from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionIndexSettings(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_init_raise_bad_index_list(self): self.assertRaises(TypeError, IndexSettings, 'invalid') def test_init_no_index_settings(self): self.builder() _ = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertRaises(MissingArgument, IndexSettings, self.ilo, {}) def test_init_bad_index_settings(self): self.builder() _ = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertRaises(ConfigurationError, IndexSettings, self.ilo, {'a':'b'}) def test_init(self): self.builder() iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertEqual(self.ilo, iso.index_list) self.assertEqual(self.client, iso.client) def test_static_settings(self): static = [ 'number_of_shards', 'shard', 'codec', 'routing_partition_size', ] self.builder() iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertEqual(static, iso._static_settings()) def test_dynamic_settings(self): self.builder() dynamic = [ 'number_of_replicas', 'auto_expand_replicas', 'refresh_interval', 'max_result_window', 'max_rescore_window', 'blocks', 'max_refresh_listeners', 'mapping', 'merge', 'translog', ] iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertEqual(dynamic, iso._dynamic_settings()) def test_settings_check_raises_with_opened(self): self.builder() self.ilo.get_index_state() self.ilo.get_index_settings() iso = IndexSettings(self.ilo, {'index':{'codec':'best_compression'}}) self.assertRaises(ActionError, iso._settings_check) def test_settings_check_no_raise_with_ignore_unavailable(self): self.builder() iso = IndexSettings( self.ilo, {'index':{'codec':'best_compression'}}, ignore_unavailable=True ) self.assertIsNone(iso._settings_check()) def test_settings_check_no_raise_with_dynamic_settings(self): self.builder() iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertIsNone(iso._settings_check()) def test_settings_check_no_raise_with_unknown(self): self.builder() iso = IndexSettings(self.ilo, {'index':{'foobar':'1s'}}) self.assertIsNone(iso._settings_check()) def test_settings_dry_run(self): self.builder() iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertIsNone(iso.do_dry_run()) def test_settings_do_action(self): self.builder() self.client.indices.put_settings.return_value = {"acknowledged":True} iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertIsNone(iso.do_action()) def test_settings_do_action_raises(self): self.builder() self.client.indices.put_settings.side_effect = testvars.fake_fail iso = IndexSettings(self.ilo, {'index':{'refresh_interval':'1s'}}) self.assertRaises(Exception, iso.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_open.py000066400000000000000000000035411477314666200241440ustar00rootroot00000000000000"""test_action_open""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import Open from curator.exceptions import FailedExecution from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionOpen(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_four self.client.indices.get_settings.return_value = testvars.settings_four self.client.indices.stats.return_value = testvars.stats_four self.client.indices.exists_alias.return_value = False self.client.indices.open.return_value = None self.ilo = IndexList(self.client) def test_init_raise(self): self.assertRaises(TypeError, Open, 'invalid') def test_init(self): self.builder() opn = Open(self.ilo) self.assertEqual(self.ilo, opn.index_list) self.assertEqual(self.client, opn.client) def test_do_dry_run(self): self.builder() self.ilo.filter_opened() opn = Open(self.ilo) self.assertEqual(['c-2016.03.05'], opn.index_list.indices) self.assertIsNone(opn.do_dry_run()) def test_do_action(self): self.builder() self.ilo.filter_opened() opn = Open(self.ilo) self.assertEqual(['c-2016.03.05'], opn.index_list.indices) self.assertIsNone(opn.do_action()) def test_do_action_raises_exception(self): self.builder() self.client.indices.open.side_effect = testvars.fake_fail opn = Open(self.ilo) self.assertRaises(FailedExecution, opn.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_reindex.py000066400000000000000000000152771477314666200246520ustar00rootroot00000000000000"""test_action_reindex""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import Reindex from curator.exceptions import ConfigurationError, CuratorException, FailedExecution, NoIndices from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionReindex(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_four self.client.indices.get_settings.return_value = testvars.settings_four self.client.indices.stats.return_value = testvars.stats_four self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_init_bad_ilo(self): self.assertRaises(TypeError, Reindex, 'foo', 'invalid') def test_init_raise_bad_request_body(self): self.builder() self.assertRaises(ConfigurationError, Reindex, self.ilo, 'invalid') def test_init_raise_local_migration_no_prefix_or_suffix(self): self.builder() self.assertRaises(ConfigurationError, Reindex, self.ilo, testvars.reindex_migration) def test_init(self): self.builder() rio = Reindex(self.ilo, testvars.reindex_basic) self.assertEqual(self.ilo, rio.index_list) self.assertEqual(self.client, rio.client) def test_do_dry_run(self): self.builder() rio = Reindex(self.ilo, testvars.reindex_basic) self.assertIsNone(rio.do_dry_run()) def test_replace_index_list(self): self.builder() rio = Reindex(self.ilo, testvars.reindex_replace) self.assertEqual(rio.index_list.indices, rio.body['source']['index']) def test_reindex_with_wait(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} rio = Reindex(self.ilo, testvars.reindex_basic) self.assertIsNone(rio.do_action()) def test_reindex_with_wait_zero_total(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task_zero_total # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} rio = Reindex(self.ilo, testvars.reindex_basic) self.assertIsNone(rio.do_action()) def test_reindex_with_wait_zero_total_fail(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.side_effect = testvars.fake_fail # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} rio = Reindex(self.ilo, testvars.reindex_basic) self.assertRaises(CuratorException, rio.do_action) def test_reindex_without_wait(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task rio = Reindex(self.ilo, testvars.reindex_basic, wait_for_completion=False) self.assertIsNone(rio.do_action()) def test_reindex_timedout(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.incomplete_task rio = Reindex(self.ilo, testvars.reindex_basic, max_wait=1, wait_interval=1) self.assertRaises(FailedExecution, rio.do_action) def test_remote_with_no_host_key(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} badval = { 'source': { 'index': 'irrelevant', 'remote': {'wrong': 'invalid'} }, 'dest': { 'index': 'other_index' } } self.assertRaises( ConfigurationError, Reindex, self.ilo, badval) def test_remote_with_bad_host(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} badval = { 'source': { 'index': 'irrelevant', 'remote': {'host': 'invalid'} }, 'dest': { 'index': 'other_index' } } self.assertRaises( ConfigurationError, Reindex, self.ilo, badval) def test_remote_with_bad_url(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} badval = { 'source': { 'index': 'irrelevant', 'remote': {'host': 'asdf://hostname:1234'} }, 'dest': { 'index': 'other_index' } } self.assertRaises( ConfigurationError, Reindex, self.ilo, badval) def test_remote_with_bad_connection(self): self.builder() self.client.reindex.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task # After building ilo, we need a different return value self.client.indices.get_settings.return_value = {'other_index':{}} badval = { 'source': { 'index': 'REINDEX_SELECTION', 'remote': {'host': 'https://example.org:XXXX'} }, 'dest': { 'index': 'other_index' } } urllib3 = Mock() urllib3.util.retry.side_effect = testvars.fake_fail self.assertRaises(Exception, Reindex, self.ilo, badval) def test_init_raise_empty_source_list(self): self.builder() badval = { 'source': { 'index': [] }, 'dest': { 'index': 'other_index' } } rio = Reindex(self.ilo, badval) self.assertRaises(NoIndices, rio.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_replicas.py000066400000000000000000000042771477314666200250140ustar00rootroot00000000000000"""test_action_replicas""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import Replicas from curator.exceptions import FailedExecution, MissingArgument from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionReplicas(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.client.indices.put_settings.return_value = None self.ilo = IndexList(self.client) def test_init_raise_bad_client(self): self.assertRaises(TypeError, Replicas, 'invalid', count=2) def test_init_raise_no_count(self): self.builder() self.assertRaises(MissingArgument, Replicas, self.ilo) def test_init(self): self.builder() rpo = Replicas(self.ilo, count=2) self.assertEqual(self.ilo, rpo.index_list) self.assertEqual(self.client, rpo.client) def test_do_dry_run(self): self.builder() rpo = Replicas(self.ilo, count=0) self.assertIsNone(rpo.do_dry_run()) def test_do_action(self): self.builder() rpo = Replicas(self.ilo, count=0) self.assertIsNone(rpo.do_action()) def test_do_action_wait(self): self.builder() self.client.cluster.health.return_value = {'status':'green'} rpo = Replicas(self.ilo, count=1, wait_for_completion=True) self.assertIsNone(rpo.do_action()) def test_do_action_raises_exception(self): self.builder() self.client.indices.segments.return_value = testvars.shards self.client.indices.put_settings.side_effect = testvars.fake_fail rpo = Replicas(self.ilo, count=2) self.assertRaises(FailedExecution, rpo.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_restore.py000066400000000000000000000203531477314666200246660ustar00rootroot00000000000000"""test_action_restore""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import Restore from curator.exceptions import CuratorException, FailedExecution, FailedRestore, SnapshotInProgress from curator import SnapshotList # Get test variables and constants from a single source from . import testvars class TestActionRestore(TestCase): def test_init_raise_bad_snapshot_list(self): self.assertRaises(TypeError, Restore, 'invalid') def test_init_raise_unsuccessful_snapshot_list(self): client = Mock() client.snapshot.get.return_value = testvars.partial client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(CuratorException, Restore, slo) def test_snapshot_derived_name(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo) self.assertEqual('snapshot-2015.03.01', ro.name) def test_provided_name(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, name=testvars.snap_name) self.assertEqual(testvars.snap_name, ro.name) def test_partial_snap(self): client = Mock() client.snapshot.get.return_value = testvars.partial client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, partial=True) self.assertEqual(testvars.snap_name, ro.name) def test_provided_indices(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, indices=testvars.named_indices) self.assertEqual('snapshot-2015.03.01', ro.name) def test_extra_settings(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, extra_settings={'foo':'bar'}) self.assertEqual(ro.body['foo'], 'bar') def test_bad_extra_settings(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, extra_settings='invalid') self.assertEqual(ro.body, { 'ignore_unavailable': False, 'partial': False, 'include_aliases': False, 'rename_replacement': '', 'rename_pattern': '', 'indices': ['index-2015.01.01', 'index-2015.02.01'], 'include_global_state': False } ) def test_get_expected_output(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore( slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertEqual( ro.expected_output, ['new_index-2015.01.01', 'new_index-2015.02.01'] ) def test_do_dry_run(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo) self.assertIsNone(ro.do_dry_run()) def test_do_dry_run_with_renames(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore( slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertIsNone(ro.do_dry_run()) def test_report_state_all(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.snapshot.get.return_value = testvars.snapshot client.snapshot.get_repository.return_value = testvars.test_repo client.cat.indices.return_value = testvars.state_named client.indices.get_settings.return_value = testvars.settings_named slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo) self.assertIsNone(ro.report_state()) def test_report_state_not_all(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.cat.indices.return_value = testvars.state_one client.indices.get_settings.return_value = testvars.settings_one slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore( slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertRaises(FailedRestore, ro.report_state) def test_do_action_success(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.cat.indices.return_value = testvars.state_named client.indices.get_settings.return_value = testvars.settings_named client.indices.recovery.return_value = testvars.recovery_output slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, wait_interval=0.5, max_wait=1) self.assertIsNone(ro.do_action()) def test_do_action_snap_in_progress(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.snap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo) self.assertRaises(SnapshotInProgress, ro.do_action) def test_do_action_success_no_wfc(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo, wait_for_completion=False) self.assertIsNone(ro.do_action()) def test_do_action_report_on_failure(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named client.snapshot.restore.side_effect = testvars.fake_fail slo = SnapshotList(client, repository=testvars.repo_name) ro = Restore(slo) self.assertRaises(FailedExecution, ro.do_action) elasticsearch-curator-8.0.21/tests/unit/test_action_rollover.py000066400000000000000000000036011477314666200250440ustar00rootroot00000000000000"""test_action_rollover""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import Rollover from curator.exceptions import ConfigurationError # Get test variables and constants from a single source from . import testvars class TestActionRollover(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION def test_init_raise_bad_client(self): self.assertRaises(TypeError, Rollover, 'invalid', 'name', {}) def test_init_raise_bad_conditions(self): self.builder() self.assertRaises(ConfigurationError, Rollover, self.client, 'name', 'string') def test_init_raise_bad_extra_settings(self): self.builder() self.assertRaises( ConfigurationError, Rollover, self.client, 'name', {'a':'b'}, None, 'string') def test_init_raise_non_rollable_index(self): self.builder() self.client.indices.get_alias.return_value = testvars.alias_retval self.assertRaises(ValueError, Rollover, self.client, testvars.named_alias, {'a':'b'}) def test_do_dry_run(self): self.builder() self.client.indices.get_alias.return_value = testvars.rollable_alias self.client.indices.rollover.return_value = testvars.dry_run_rollover rlo = Rollover(self.client, testvars.named_alias, testvars.rollover_conditions) self.assertIsNone(rlo.do_dry_run()) def test_max_size_in_acceptable_verion(self): self.builder() self.client.indices.get_alias.return_value = testvars.rollable_alias conditions = { 'max_size': '1g' } rlo = Rollover(self.client, testvars.named_alias, conditions) self.assertEqual(conditions, rlo.conditions) elasticsearch-curator-8.0.21/tests/unit/test_action_shrink.py000066400000000000000000000315251477314666200245040ustar00rootroot00000000000000"""test_action_shrink""" # pylint: disable=C0103,C0115,C0116,W0201,W0212 from unittest import TestCase from unittest.mock import Mock from curator.actions import Shrink from curator.exceptions import ActionError, ConfigurationError from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionShrink_ilo(TestCase): def test_init_raise_bad_client(self): self.assertRaises(TypeError, Shrink, 'invalid') class TestActionShrink_extra_settings(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_extra_settings_1(self): self.builder() self.assertRaises( ConfigurationError, Shrink, self.ilo, extra_settings={'settings': {'foobar'}}, ) def test_extra_settings_2(self): self.builder() self.assertRaises( ConfigurationError, Shrink, self.ilo, extra_settings={'foobar'} ) class TestActionShrink_data_node(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.node_name = 'node_name' self.node_id = 'my_node' self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'name': self.node_name}} } self.ilo = IndexList(self.client) self.shrink = Shrink(self.ilo) def test_non_data_node(self): self.builder() self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['ingest'], 'name': self.node_name}} } self.assertFalse(self.shrink._data_node(self.node_id)) def test_data_node(self): self.builder() self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['data'], 'name': self.node_name}} } self.assertTrue(self.shrink._data_node(self.node_id)) class TestActionShrink_exclude_node(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.node_name = 'node_name' self.ilo = IndexList(self.client) def test_positive(self): self.builder() node_filters = {'exclude_nodes': [self.node_name]} shrink = Shrink(self.ilo, node_filters=node_filters) self.assertTrue(shrink._exclude_node(self.node_name)) def test_negative(self): self.builder() node_filters = {'exclude_nodes': ['not_this_node']} shrink = Shrink(self.ilo, node_filters=node_filters) self.assertFalse(shrink._exclude_node(self.node_name)) class TestActionShrink_qualify_single_node(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.node_name = 'node_name' self.node_id = 'my_node' self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'name': self.node_name}} } self.ilo = IndexList(self.client) def test_positive(self): self.builder() byte_count = 123456 self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['data'], 'name': self.node_name}} } self.client.nodes.stats.return_value = { 'nodes': { self.node_id: { 'fs': { 'data': ['one'], 'total': {'available_in_bytes': byte_count}, }, 'name': self.node_name, } } } shrink = Shrink(self.ilo, shrink_node=self.node_name) shrink.qualify_single_node() self.assertEqual(byte_count, shrink.shrink_node_avail) def test_not_found(self): self.builder() shrink = Shrink(self.ilo, shrink_node='not_me') self.assertRaises(ConfigurationError, shrink.qualify_single_node) def test_excluded(self): self.builder() node_filters = {'exclude_nodes': [self.node_name]} shrink = Shrink(self.ilo, shrink_node=self.node_name, node_filters=node_filters) self.assertRaises(ConfigurationError, shrink.qualify_single_node) def test_non_data_node(self): self.builder() self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['ingest'], 'name': self.node_name}} } shrink = Shrink(self.ilo, shrink_node=self.node_name) self.assertRaises(ActionError, shrink.qualify_single_node) class TestActionShrink_most_available_node(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.node_name = 'node_name' self.node_id = 'my_node' self.byte_count = 123456 self.client.nodes.stats.return_value = { 'nodes': {self.node_id: {'name': self.node_name}} } self.ilo = IndexList(self.client) def test_excluded(self): self.builder() self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['data']}} } self.client.nodes.stats.return_value = { 'nodes': { self.node_id: { 'fs': { 'data': ['one'], 'total': {'available_in_bytes': self.byte_count}, }, 'name': self.node_name, } } } node_filters = {'exclude_nodes': [self.node_name]} shrink = Shrink(self.ilo, node_filters=node_filters) shrink.most_available_node() self.assertIsNone(shrink.shrink_node_name) class TestActionShrink_route_index(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) self.shrink = Shrink(self.ilo) def test_raises(self): self.builder() self.client.indices.put_settings.side_effect = testvars.fake_fail self.assertRaises( Exception, self.shrink.route_index, 'index', 'exclude', '_name', 'not_my_node', ) class TestActionShrink_dry_run(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.node_name = 'node_name' self.node_id = 'my_node' self.byte_count = 123456 self.client.nodes.stats.return_value = { 'nodes': { self.node_id: { 'fs': { 'data': ['one'], 'total': {'available_in_bytes': self.byte_count}, }, 'name': self.node_name, } } } self.ilo = IndexList(self.client) def test_dry_run(self): self.builder() self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['data'], 'name': self.node_name}} } self.client.indices.get.return_value = { testvars.named_index: {'settings': {'index': {'number_of_shards': 2}}} } shrink = Shrink( self.ilo, shrink_node=self.node_name, post_allocation={ 'allocation_type': 'require', 'key': '_name', 'value': self.node_name, }, ) self.assertIsNone(shrink.do_dry_run()) def test_dry_run_raises(self): self.builder() self.client.nodes.info.return_value = { 'nodes': {self.node_id: {'roles': ['data'], 'name': self.node_name}} } self.client.indices.get.side_effect = testvars.fake_fail shrink = Shrink(self.ilo, shrink_node=self.node_name) self.assertRaises(Exception, shrink.do_dry_run) class TestActionShrink_various(TestCase): def builder(self): self.client = Mock() self.client.info.return_value = {'version': {'number': '8.0.0'}} self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.node_name = 'node_name' self.node_id = 'my_node' self.byte_count = 1239132959 self.client.nodes.stats.return_value = { 'nodes': { self.node_id: { 'fs': { 'data': ['one'], 'total': {'available_in_bytes': self.byte_count}, }, 'name': self.node_name, } } } self.ilo = IndexList(self.client) def test_target_exists(self): self.builder() self.client.indices.exists.return_value = True shrink = Shrink(self.ilo) self.assertRaises( ActionError, shrink._check_target_exists, testvars.named_index ) def test_doc_count(self): self.builder() too_many = 2147483520 self.client.indices.stats.return_value = { 'indices': { testvars.named_index: {'primaries': {'docs': {'count': too_many}}} } } shrink = Shrink(self.ilo) self.assertRaises(ActionError, shrink._check_doc_count, testvars.named_index) def test_shard_count(self): self.builder() src_shards = 2 shrink = Shrink(self.ilo, number_of_shards=src_shards) self.assertRaises( ActionError, shrink._check_shard_count, testvars.named_index, src_shards ) def test_shard_factor(self): self.builder() src_shards = 5 shrink = Shrink(self.ilo, number_of_shards=3) self.assertRaises( ActionError, shrink._check_shard_factor, testvars.named_index, src_shards ) def test_check_all_shards(self): self.builder() self.client.cluster.state.return_value = { 'routing_table': { 'indices': { testvars.named_index: { 'shards': { '0': [ { 'index': testvars.named_index, 'node': 'not_this_node', 'primary': True, 'shard': 0, 'state': 'STARTED', } ] } } } } } shrink = Shrink(self.ilo, shrink_node=self.node_name) shrink.shrink_node_id = self.node_id self.assertRaises(ActionError, shrink._check_all_shards, testvars.named_index) elasticsearch-curator-8.0.21/tests/unit/test_action_snapshot.py000066400000000000000000000124511477314666200250420ustar00rootroot00000000000000"""test_action_snapshot""" # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long, protected-access, attribute-defined-outside-init from unittest import TestCase from unittest.mock import Mock from curator.actions import Snapshot from curator.exceptions import ActionError, CuratorException, FailedExecution, FailedSnapshot, MissingArgument, SnapshotInProgress from curator import IndexList # Get test variables and constants from a single source from . import testvars class TestActionSnapshot(TestCase): VERSION = {'version': {'number': '8.0.0'} } def builder(self): self.client = Mock() self.client.info.return_value = self.VERSION self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.exists_alias.return_value = False self.client.snapshot.get_repository.return_value = testvars.test_repo self.client.snapshot.get.return_value = testvars.snapshots self.client.tasks.get.return_value = testvars.no_snap_tasks self.ilo = IndexList(self.client) def test_init_raise_bad_index_list(self): self.assertRaises(TypeError, Snapshot, 'invalid') def test_init_no_repo_arg_exception(self): self.builder() self.assertRaises(MissingArgument, Snapshot, self.ilo) def test_init_no_repo_exception(self): self.builder() self.client.snapshot.get_repository.return_value = {'repo':{'foo':'bar'}} self.assertRaises(ActionError, Snapshot, self.ilo, repository='notfound') def test_init_no_name_exception(self): self.builder() self.assertRaises(MissingArgument, Snapshot, self.ilo, repository=testvars.repo_name) def test_init_success(self): self.builder() sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertEqual(testvars.repo_name, sso.repository) self.assertIsNone(sso.state) def test_get_state_success(self): self.builder() sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) sso.get_state() self.assertEqual('SUCCESS', sso.state) def test_get_state_fail(self): self.builder() self.client.snapshot.get.return_value = {'snapshots':[]} sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertRaises(CuratorException, sso.get_state) def test_report_state_success(self): self.builder() sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) sso.report_state() self.assertEqual('SUCCESS', sso.state) def test_report_state_other(self): self.builder() self.client.snapshot.get.return_value = testvars.highly_unlikely sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertRaises(FailedSnapshot, sso.report_state) def test_do_dry_run(self): self.builder() self.client.snapshot.create.return_value = None self.client.snapshot.status.return_value = testvars.nosnap_running self.client.snapshot.verify_repository.return_value = testvars.verified_nodes sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertIsNone(sso.do_dry_run()) def test_do_action_success(self): self.builder() self.client.snapshot.create.return_value = testvars.generic_task self.client.tasks.get.return_value = testvars.completed_task self.client.snapshot.status.return_value = testvars.nosnap_running self.client.snapshot.verify_repository.return_value = testvars.verified_nodes sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertIsNone(sso.do_action()) def test_do_action_raise_snap_in_progress(self): self.builder() self.client.snapshot.create.return_value = None self.client.snapshot.status.return_value = testvars.snap_running self.client.snapshot.verify_repository.return_value = testvars.verified_nodes sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertRaises(SnapshotInProgress, sso.do_action) def test_do_action_no_wait_for_completion(self): self.builder() self.client.snapshot.create.return_value = testvars.generic_task self.client.snapshot.status.return_value = testvars.nosnap_running self.client.snapshot.verify_repository.return_value = testvars.verified_nodes sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name, wait_for_completion=False) self.assertIsNone(sso.do_action()) def test_do_action_raise_on_failure(self): self.builder() self.client.snapshot.create.return_value = None self.client.snapshot.create.side_effect = testvars.fake_fail self.client.snapshot.status.return_value = testvars.nosnap_running self.client.snapshot.verify_repository.return_value = testvars.verified_nodes sso = Snapshot(self.ilo, repository=testvars.repo_name, name=testvars.snap_name) self.assertRaises(FailedExecution, sso.do_action) elasticsearch-curator-8.0.21/tests/unit/test_class_index_list.py000066400000000000000000001304541477314666200252010ustar00rootroot00000000000000"""Test index_list class""" # pylint: disable=C0115, C0116, C0302, W0201, W0212 from copy import deepcopy from unittest import TestCase from unittest.mock import Mock import yaml from es_client.exceptions import FailedValidation from curator.exceptions import ( ActionError, ConfigurationError, FailedExecution, MissingArgument, NoIndices, ) from curator.helpers.date_ops import fix_epoch from curator import IndexList # Get test variables and constants from a single source from . import testvars def get_es_ver(): return {'version': {'number': '8.0.0'}} def get_testvals(number, key): """Return the appropriate value per the provided key number""" data = { "1": { "settings": testvars.settings_one, "state": testvars.state_one, "stats": testvars.stats_one, "fieldstats": testvars.fieldstats_one, }, "2": { "settings": testvars.settings_two, "state": testvars.state_two, "stats": testvars.stats_two, "fieldstats": testvars.fieldstats_two, }, "4": { "settings": testvars.settings_four, "state": testvars.state_four, "stats": testvars.stats_four, "fieldstats": testvars.fieldstats_four, }, } return data[number][key] class TestIndexListClientAndInit(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_init_bad_client(self): client = 'not a real client' self.assertRaises(TypeError, IndexList, client) def test_init_get_indices_exception(self): self.builder() self.client.cat.indices.side_effect = testvars.fake_fail self.assertRaises(FailedExecution, IndexList, self.client) def test_init(self): self.builder() self.ilo.get_index_stats() self.ilo.get_index_settings() self.assertEqual( testvars.stats_two['indices']['index-2016.03.03']['total']['store'][ 'size_in_bytes' ], self.ilo.index_info['index-2016.03.03']['size_in_bytes'], ) self.assertEqual( testvars.state_two[1]['status'], self.ilo.index_info['index-2016.03.04']['state'], ) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_for_closed_index(self): self.builder() self.client.cat.indices.return_value = testvars.state_2_closed self.client.indices.get_settings.return_value = testvars.settings_2_closed ilo2 = IndexList(self.client) ilo2.get_index_state() self.assertEqual('close', ilo2.index_info['index-2016.03.03']['state']) class TestIndexListOtherMethods(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_empty_list(self): self.builder() self.client.indices.exists_alias.return_value = False self.assertEqual(2, len(self.ilo.indices)) self.ilo.indices = [] self.assertRaises(NoIndices, self.ilo.empty_list_check) def test_get_segmentcount(self): self.builder(key='1') self.client.indices.segments.return_value = testvars.shards # Ordinarily get_index_state is run before get_segment_counts, so we do # so manually here. self.ilo.get_index_state() self.ilo.get_segment_counts() self.assertEqual(71, self.ilo.index_info[testvars.named_index]['segments']) class TestIndexListAgeFilterName(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_get_name_based_ages_match(self): self.builder() self.ilo.get_index_state() self.ilo.get_index_settings() self.ilo._get_name_based_ages('%Y.%m.%d') self.assertEqual( 1456963200, self.ilo.index_info['index-2016.03.03']['age']['name'] ) def test_get_name_based_ages_no_match(self): self.builder() self.ilo.get_index_settings() self.ilo._get_name_based_ages('%Y-%m-%d') self.assertEqual( fix_epoch( testvars.settings_two['index-2016.03.03']['settings']['index'][ 'creation_date' ] ), self.ilo.index_info['index-2016.03.03']['age']['creation_date'], ) class TestIndexListAgeFilterStatsAPI(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_get_field_stats_dates_negative(self): self.builder() self.client.search.return_value = testvars.fieldstats_query self.client.field_stats.return_value = testvars.fieldstats_two self.ilo._get_field_stats_dates(field='timestamp') self.assertNotIn('not_an_index_name', list(self.ilo.index_info.keys())) def test_get_field_stats_dates_field_not_found(self): self.builder() self.client.search.return_value = {'aggregations': {'foo': 'bar'}} self.assertRaises( ActionError, self.ilo._get_field_stats_dates, field='not_in_index' ) class TestIndexListRegexFilters(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_filter_by_regex_prefix(self): self.builder() self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='prefix', value='ind') self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='prefix', value='ind', exclude=True) self.assertEqual([], self.ilo.indices) def test_filter_by_regex_middle(self): self.builder() self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='regex', value='dex') self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='regex', value='dex', exclude=True) self.assertEqual([], self.ilo.indices) def test_filter_by_regex_timestring(self): self.builder() self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='timestring', value='%Y.%m.%d') self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='timestring', value='%Y.%m.%d', exclude=True) self.assertEqual([], self.ilo.indices) def test_filter_by_regex_no_match_exclude(self): self.builder() self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='prefix', value='invalid', exclude=True) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_filter_by_regex_no_value(self): self.builder() self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.assertRaises( ValueError, self.ilo.filter_by_regex, kind='prefix', value=None ) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.ilo.filter_by_regex(kind='prefix', value=0) self.assertEqual([], self.ilo.indices) def test_filter_by_regex_bad_kind(self): self.builder() self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) self.assertRaises( ValueError, self.ilo.filter_by_regex, kind='invalid', value=None ) class TestIndexListFilterByAge(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_missing_direction(self): self.builder() self.assertRaises( MissingArgument, self.ilo.filter_by_age, unit='days', unit_count=1 ) def test_bad_direction(self): self.builder() self.assertRaises( ValueError, self.ilo.filter_by_age, unit='days', unit_count=1, direction="invalid", ) def test_name_no_timestring(self): self.builder() self.assertRaises( MissingArgument, self.ilo.filter_by_age, source='name', unit='days', unit_count=1, direction='older', ) def test_name_older_than_now(self): self.builder() self.ilo.filter_by_age( source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=1, ) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_name_older_than_now_exclude(self): self.builder() self.ilo.filter_by_age( source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=1, exclude=True, ) self.assertEqual([], sorted(self.ilo.indices)) def test_name_younger_than_now(self): self.builder() self.ilo.filter_by_age( source='name', direction='younger', timestring='%Y.%m.%d', unit='days', unit_count=1, ) self.assertEqual([], sorted(self.ilo.indices)) def test_name_younger_than_now_exclude(self): self.builder() self.ilo.filter_by_age( source='name', direction='younger', timestring='%Y.%m.%d', unit='days', unit_count=1, exclude=True, ) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_name_younger_than_past_date(self): self.builder() self.ilo.filter_by_age( source='name', direction='younger', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1457049599, ) self.assertEqual(['index-2016.03.04'], sorted(self.ilo.indices)) def test_name_older_than_past_date(self): self.builder() self.ilo.filter_by_age( source='name', direction='older', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1456963201, ) self.assertEqual(['index-2016.03.03'], sorted(self.ilo.indices)) def test_creation_date_older_than_now(self): self.builder() self.ilo.filter_by_age( source='creation_date', direction='older', unit='days', unit_count=1 ) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_creation_date_older_than_now_raises(self): self.builder() self.ilo.get_index_state() self.ilo.get_index_settings() self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date') self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date') self.ilo.filter_by_age( source='creation_date', direction='older', unit='days', unit_count=1 ) self.assertEqual([], self.ilo.indices) def test_creation_date_younger_than_now(self): self.builder() self.ilo.filter_by_age( source='creation_date', direction='younger', unit='days', unit_count=1 ) self.assertEqual([], sorted(self.ilo.indices)) def test_creation_date_younger_than_now_raises(self): self.builder() self.ilo.get_index_state() self.ilo.get_index_settings() self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date') self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date') self.ilo.filter_by_age( source='creation_date', direction='younger', unit='days', unit_count=1 ) self.assertEqual([], self.ilo.indices) def test_creation_date_younger_than_past_date(self): self.builder() self.ilo.filter_by_age( source='creation_date', direction='younger', unit='seconds', unit_count=0, epoch=1457049599, ) self.assertEqual(['index-2016.03.04'], sorted(self.ilo.indices)) def test_creation_date_older_than_past_date(self): self.builder() self.ilo.filter_by_age( source='creation_date', direction='older', unit='seconds', unit_count=0, epoch=1456963201, ) self.assertEqual(['index-2016.03.03'], sorted(self.ilo.indices)) def test_field_stats_missing_field(self): self.builder() self.assertRaises( MissingArgument, self.ilo.filter_by_age, source='field_stats', direction='older', unit='days', unit_count=1, ) def test_field_stats_invalid_stats_result(self): self.builder() self.assertRaises( ValueError, self.ilo.filter_by_age, field='timestamp', source='field_stats', direction='older', unit='days', unit_count=1, stats_result='invalid', ) def test_field_stats_invalid_source(self): self.builder() self.assertRaises( ValueError, self.ilo.filter_by_age, source='invalid', direction='older', unit='days', unit_count=1, ) class TestIndexListFilterBySpace(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.search.return_value = get_testvals(key, 'fieldstats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_missing_disk_space_value(self): self.builder() self.assertRaises(MissingArgument, self.ilo.filter_by_space) def test_filter_result_by_name(self): self.builder() self.ilo.filter_by_space(disk_space=1.1) self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_filter_result_by_name_reverse_order(self): self.builder() self.ilo.filter_by_space(disk_space=1.1, reverse=False) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_filter_result_by_name_exclude(self): self.builder() self.ilo.filter_by_space(disk_space=1.1, exclude=True) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_filter_result_by_date_raise(self): self.builder(key='4') self.assertRaises( ValueError, self.ilo.filter_by_space, disk_space=2.1, use_age=True, source='invalid', ) def test_filter_result_by_date_timestring_raise(self): self.builder(key='4') self.assertRaises( MissingArgument, self.ilo.filter_by_space, disk_space=2.1, use_age=True, source='name', ) def test_filter_result_by_date_timestring(self): self.builder(key='4') self.ilo.filter_by_space( disk_space=2.1, use_age=True, source='name', timestring='%Y.%m.%d' ) self.assertEqual(['a-2016.03.03'], sorted(self.ilo.indices)) def test_filter_result_by_date_non_matching_timestring(self): self.builder(key='4') self.ilo.filter_by_space( disk_space=2.1, use_age=True, source='name', timestring='%Y.%m.%d.%H' ) self.assertEqual([], sorted(self.ilo.indices)) def test_filter_threshold_behavior(self): self.builder() il_a = IndexList(self.client) # less than il_a.filter_by_space( disk_space=1.5, use_age=True, threshold_behavior='less_than' ) self.assertEqual(['index-2016.03.04'], sorted(il_a.indices)) # greater than il_b = IndexList(self.client) il_b.filter_by_space( disk_space=1.5, use_age=True, threshold_behavior='greater_than' ) self.assertEqual(['index-2016.03.03'], sorted(il_b.indices)) # default case il_c = IndexList(self.client) il_c.filter_by_space(disk_space=1.5, use_age=True) self.assertEqual(['index-2016.03.03'], sorted(il_c.indices)) def test_filter_bad_threshold_behavior(self): self.builder() # less than self.assertRaises( ValueError, self.ilo.filter_by_space, disk_space=1.5, threshold_behavior='invalid', ) def test_filter_result_by_date_field_stats_raise(self): self.builder(key='4') self.client.search.return_value = testvars.fieldstats_query self.assertRaises( ValueError, self.ilo.filter_by_space, disk_space=2.1, use_age=True, source='min_value', ) def test_filter_result_by_date_no_field_raise(self): self.builder(key='4') self.assertRaises( MissingArgument, self.ilo.filter_by_space, disk_space=2.1, use_age=True, source='field_stats', ) def test_filter_result_by_date_invalid_stats_result_raise(self): self.builder(key='4') self.assertRaises( ValueError, self.ilo.filter_by_space, disk_space=2.1, use_age=True, source='field_stats', field='timestamp', stats_result='invalid', ) def test_filter_result_by_creation_date(self): self.builder(key='4') self.ilo.filter_by_space(disk_space=2.1, use_age=True) self.assertEqual(['a-2016.03.03'], self.ilo.indices) class TestIndexListFilterKibana(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.search.return_value = get_testvals(key, 'fieldstats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_filter_kibana_positive(self): self.builder() # Establish the object per requirements, then overwrite self.ilo.indices = ['.kibana', '.kibana-5', '.kibana-6', 'dummy'] self.ilo.filter_kibana() self.assertEqual(['dummy'], self.ilo.indices) def test_filter_kibana_positive_include(self): self.builder() # Establish the object per requirements, then overwrite self.ilo.indices = ['.kibana', '.kibana-5', '.kibana-6', 'dummy'] self.ilo.filter_kibana(exclude=False) self.assertEqual(['.kibana', '.kibana-5', '.kibana-6'], self.ilo.indices) def test_filter_kibana_positive_exclude(self): self.builder() # Establish the object per requirements, then overwrite kibana_indices = ['.kibana', '.kibana-5', '.kibana-6'] self.ilo.indices = kibana_indices self.ilo.indices.append('dummy') self.ilo.filter_kibana(exclude=True) self.assertEqual(kibana_indices, self.ilo.indices) def test_filter_kibana_negative(self): self.builder() # Establish the object per requirements, then overwrite self.ilo.indices = [ 'kibana', 'marvel-kibana', 'cabana-int', 'marvel-es-data', 'dummy', ] self.ilo.filter_kibana() self.assertEqual( ['kibana', 'marvel-kibana', 'cabana-int', 'marvel-es-data', 'dummy'], self.ilo.indices, ) class TestIndexListFilterForceMerged(TestCase): VERSION = {'version': {'number': '8.0.0'}} def builder(self): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = testvars.state_one self.client.indices.get_settings.return_value = testvars.settings_one self.client.indices.stats.return_value = testvars.stats_one self.client.indices.segments.return_value = testvars.shards self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_filter_forcemerge_raise(self): self.builder() self.assertRaises(MissingArgument, self.ilo.filter_forceMerged) def test_filter_forcemerge_positive(self): self.builder() self.ilo.filter_forceMerged(max_num_segments=2) self.assertEqual([testvars.named_index], self.ilo.indices) def test_filter_forcemerge_negative(self): self.builder() self.client.indices.segments.return_value = testvars.fm_shards self.ilo.filter_forceMerged(max_num_segments=2) self.assertEqual([], self.ilo.indices) class TestIndexListFilterOpened(TestCase): def test_filter_opened(self): client = Mock() client.info.return_value = {'version': {'number': '8.0.0'}} client.cat.indices.return_value = testvars.state_four client.indices.get_settings.return_value = testvars.settings_four client.indices.stats.return_value = testvars.stats_four client.field_stats.return_value = testvars.fieldstats_four client.indices.exists_alias.return_value = False ilo = IndexList(client) ilo.filter_opened() self.assertEqual(['c-2016.03.05'], ilo.indices) class TestIndexListFilterAllocated(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_missing_key(self): self.builder() self.assertRaises( MissingArgument, self.ilo.filter_allocated, value='foo', allocation_type='invalid', ) def test_missing_value(self): self.builder() self.assertRaises( MissingArgument, self.ilo.filter_allocated, key='tag', allocation_type='invalid', ) def test_invalid_allocation_type(self): self.builder() self.assertRaises( ValueError, self.ilo.filter_allocated, key='tag', value='foo', allocation_type='invalid', ) def test_success(self): self.builder() self.ilo.filter_allocated(key='tag', value='foo', allocation_type='include') self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_invalid_tag(self): self.builder() self.ilo.filter_allocated(key='invalid', value='foo', allocation_type='include') self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) class TestIterateFiltersIndex(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_no_filters(self): self.builder(key='4') self.ilo.iterate_filters({}) self.assertEqual( ['a-2016.03.03', 'b-2016.03.04', 'c-2016.03.05', 'd-2016.03.06'], sorted(self.ilo.indices), ) def test_no_filtertype(self): self.builder(key='4') config = {'filters': [{'no_filtertype': 'fail'}]} self.assertRaises(FailedValidation, self.ilo.iterate_filters, config) def test_invalid_filtertype(self): self.builder(key='4') config = {'filters': [{'filtertype': 12345.6789}]} self.assertRaises(FailedValidation, self.ilo.iterate_filters, config) def test_pattern_filtertype(self): self.builder(key='4') config = yaml.load(testvars.pattern_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['a-2016.03.03'], self.ilo.indices) def test_age_filtertype(self): self.builder() config = yaml.load(testvars.age_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_space_filtertype(self): self.builder(key='4') self.client.field_stats.return_value = testvars.fieldstats_four config = yaml.load(testvars.space_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['a-2016.03.03'], self.ilo.indices) def test_forcemerge_filtertype(self): self.builder(key='1') self.client.indices.segments.return_value = testvars.shards config = yaml.load(testvars.forcemerge_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual([testvars.named_index], self.ilo.indices) def test_allocated_filtertype(self): self.builder() config = yaml.load(testvars.allocated_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_kibana_filtertype(self): self.builder() self.client.field_stats.return_value = testvars.fieldstats_two # Establish the object per requirements, then overwrite self.ilo.indices = ['.kibana', '.kibana-5', '.kibana-6', 'dummy'] config = yaml.load(testvars.kibana_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['dummy'], self.ilo.indices) def test_opened_filtertype(self): self.builder(key='4') self.client.field_stats.return_value = testvars.fieldstats_four config = yaml.load(testvars.opened_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['c-2016.03.05'], self.ilo.indices) def test_closed_filtertype(self): self.builder(key='4') self.client.field_stats.return_value = testvars.fieldstats_four config = yaml.load(testvars.closed_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual( ['a-2016.03.03', 'b-2016.03.04', 'd-2016.03.06'], sorted(self.ilo.indices) ) def test_none_filtertype(self): self.builder() config = yaml.load(testvars.none_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_unknown_filtertype_raises(self): self.builder() config = yaml.load(testvars.invalid_ft, Loader=yaml.FullLoader)['actions'][1] self.assertRaises(FailedValidation, self.ilo.iterate_filters, config) def test_ilm_filtertype_exclude(self): self.builder() # If we don't deepcopy, then it munges the settings for future references. with_ilm = deepcopy(testvars.settings_two) with_ilm['index-2016.03.03']['settings']['index']['lifecycle'] = { 'name': 'mypolicy' } self.client.indices.get_settings.return_value = with_ilm config = {'filters': [{'filtertype': 'ilm', 'exclude': True}]} self.ilo.iterate_filters(config) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_ilm_filtertype_no_setting(self): self.builder() config = {'filters': [{'filtertype': 'ilm', 'exclude': True}]} self.ilo.iterate_filters(config) self.assertEqual( ['index-2016.03.03', 'index-2016.03.04'], sorted(self.ilo.indices) ) def test_size_filtertype(self): self.builder() config = yaml.load(testvars.size_ft, Loader=yaml.FullLoader)['actions'][1] self.ilo.iterate_filters(config) self.assertEqual(['index-2016.03.03'], self.ilo.indices) class TestIndexListFilterAlias(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_raise(self): self.builder(key='1') self.assertRaises(MissingArgument, self.ilo.filter_by_alias) def test_positive(self): self.builder() self.client.indices.get_alias.return_value = testvars.settings_2_get_aliases self.ilo.filter_by_alias(aliases=['my_alias']) self.assertEqual( sorted(list(testvars.settings_two.keys())), sorted(self.ilo.indices) ) def test_negative(self): self.builder() self.client.indices.get_alias.return_value = {} self.ilo.filter_by_alias(aliases=['not_my_alias']) self.assertEqual(sorted([]), sorted(self.ilo.indices)) def test_get_alias_raises(self): self.builder() self.client.indices.get_alias.side_effect = testvars.get_alias_fail self.client.indices.get_alias.return_value = testvars.settings_2_get_aliases self.ilo.filter_by_alias(aliases=['my_alias']) self.assertEqual(sorted([]), sorted(self.ilo.indices)) class TestIndexListFilterCount(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.client.indices.get_alias.return_value = testvars.settings_2_get_aliases self.ilo = IndexList(self.client) def test_raise(self): self.builder() self.assertRaises(MissingArgument, self.ilo.filter_by_count) def test_without_age(self): self.builder() self.ilo.filter_by_count(count=1) self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_without_age_reversed(self): self.builder() self.ilo.filter_by_count(count=1, reverse=False) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_with_age(self): self.builder() self.ilo.filter_by_count( count=1, use_age=True, source='name', timestring='%Y.%m.%d' ) self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_with_age_creation_date(self): self.builder() self.ilo.filter_by_count(count=1, use_age=True) self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_with_age_reversed(self): self.builder() self.ilo.filter_by_count( count=1, use_age=True, source='name', timestring='%Y.%m.%d', reverse=False ) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_pattern_no_regex_group(self): self.builder() self.assertRaises( ActionError, self.ilo.filter_by_count, count=1, use_age=True, pattern=' ', source='name', timestring='%Y.%m.%d', ) def test_pattern_multiple_regex_groups(self): self.builder() self.assertRaises( ActionError, self.ilo.filter_by_count, count=1, use_age=True, pattern=r'^(\ )foo(\ )$', source='name', timestring='%Y.%m.%d', ) class TestIndexListFilterShards(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_filter_shards_raise(self): self.builder() self.assertRaises(MissingArgument, self.ilo.filter_by_shards) def test_bad_shard_count_raise_1(self): self.builder() self.assertRaises( MissingArgument, self.ilo.filter_by_shards, number_of_shards=0 ) def test_bad_shard_count_raise_2(self): self.builder() self.assertRaises( ValueError, self.ilo.filter_by_shards, number_of_shards=1, shard_filter_behavior='less_than', ) def test_bad_shard_count_raise_3(self): self.builder() self.assertRaises( ValueError, self.ilo.filter_by_shards, number_of_shards=-1, shard_filter_behavior='greater_than', ) def test_greater_than_or_equal(self): self.builder() self.ilo.filter_by_shards( number_of_shards=5, shard_filter_behavior='greater_than_or_equal' ) self.assertEqual( sorted(['index-2016.03.03', 'index-2016.03.04']), sorted(self.ilo.indices) ) def test_greater_than_or_equal_exclude(self): self.builder() self.ilo.filter_by_shards( number_of_shards=5, shard_filter_behavior='greater_than_or_equal', exclude=True, ) self.assertEqual(sorted([]), sorted(self.ilo.indices)) def test_greater_than(self): self.builder() self.ilo.filter_by_shards(number_of_shards=5) self.assertEqual(sorted([]), sorted(self.ilo.indices)) def test_greater_than_exclude(self): self.builder() self.ilo.filter_by_shards(number_of_shards=5, exclude=True) self.assertEqual( sorted(['index-2016.03.03', 'index-2016.03.04']), sorted(self.ilo.indices) ) def test_less_than_or_equal(self): self.builder() self.ilo.filter_by_shards( number_of_shards=5, shard_filter_behavior='less_than_or_equal' ) self.assertEqual( sorted(['index-2016.03.03', 'index-2016.03.04']), sorted(self.ilo.indices) ) def test_less_than_or_equal_exclude(self): self.builder() self.ilo.filter_by_shards( number_of_shards=5, shard_filter_behavior='less_than_or_equal', exclude=True ) self.assertEqual(sorted([]), sorted(self.ilo.indices)) def test_less_than(self): self.builder() self.ilo.filter_by_shards(number_of_shards=5, shard_filter_behavior='less_than') self.assertEqual(sorted([]), sorted(self.ilo.indices)) def test_less_than_exclude(self): self.builder() self.ilo.filter_by_shards( number_of_shards=5, shard_filter_behavior='less_than', exclude=True ) self.assertEqual( sorted(['index-2016.03.03', 'index-2016.03.04']), sorted(self.ilo.indices) ) def test_equal(self): self.builder() self.ilo.filter_by_shards(number_of_shards=5, shard_filter_behavior='equal') self.assertEqual( sorted(['index-2016.03.03', 'index-2016.03.04']), sorted(self.ilo.indices) ) class TestIndexListPeriodFilterName(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) self.timestring = '%Y.%m.%d' self.epoch = 1456963201 self.unit = 'days' def test_get_name_based_age_in_range(self): range_from = -1 range_to = 0 expected = ['index-2016.03.03'] self.builder() self.ilo.filter_period( unit=self.unit, range_from=range_from, range_to=range_to, source='name', timestring=self.timestring, epoch=self.epoch, ) self.assertEqual(expected, self.ilo.indices) def test_get_name_based_age_not_in_range(self): range_from = -3 range_to = -2 expected = [] self.builder() self.ilo.filter_period( unit=self.unit, range_from=range_from, range_to=range_to, source='name', timestring=self.timestring, epoch=self.epoch, ) self.assertEqual(expected, self.ilo.indices) def test_bad_arguments(self): range_from = -2 range_to = -3 self.builder() self.assertRaises( FailedExecution, self.ilo.filter_period, unit=self.unit, range_from=range_from, range_to=range_to, source='name', timestring=self.timestring, epoch=self.epoch, ) def test_missing_creation_date_raises(self): range_from = -1 range_to = 0 expected = [] self.builder() self.ilo.get_index_state() self.ilo.get_index_settings() self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date') self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date') self.ilo.filter_period( unit=self.unit, range_from=range_from, range_to=range_to, source='creation_date', epoch=self.epoch, ) self.assertEqual(expected, self.ilo.indices) def test_non_integer_range_value(self): self.builder() self.assertRaises( ConfigurationError, self.ilo.filter_period, range_from='invalid' ) class TestPeriodFilterAbsolute(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_bad_period_type(self): self.builder() self.assertRaises(ValueError, self.ilo.filter_period, period_type='invalid') def test_none_value_raises(self): self.builder() self.assertRaises( ConfigurationError, self.ilo.filter_period, period_type='absolute', date_from=None, ) def test_fail_on_bad_date(self): unit = 'months' date_from = '2016.17' date_from_format = '%Y.%m' date_to = '2017.01' date_to_format = '%Y.%m' self.builder() self.assertRaises( FailedExecution, self.ilo.filter_period, unit=unit, source='creation_date', period_type='absolute', date_from=date_from, date_to=date_to, date_from_format=date_from_format, date_to_format=date_to_format, ) class TestIndexListFilterBySize(TestCase): def builder(self, key='2'): self.client = Mock() self.client.info.return_value = get_es_ver() self.client.cat.indices.return_value = get_testvals(key, 'state') self.client.indices.get_settings.return_value = get_testvals(key, 'settings') self.client.indices.stats.return_value = get_testvals(key, 'stats') self.client.field_stats.return_value = get_testvals(key, 'fieldstats') self.client.indices.exists_alias.return_value = False self.ilo = IndexList(self.client) def test_missing_size_value(self): self.builder() self.assertRaises(MissingArgument, self.ilo.filter_by_size) def test_filter_default_result(self): self.builder() self.ilo.filter_by_size(size_threshold=0.52) self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_filter_default_result_and_exclude(self): self.builder() self.ilo.filter_by_size(size_threshold=0.52, exclude=True) self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_filter_by_threshold_behavior_less_than(self): self.builder() self.ilo.filter_by_size(size_threshold=0.52, threshold_behavior='less_than') self.assertEqual(['index-2016.03.03'], self.ilo.indices) def test_filter_by_size_behavior_total(self): self.builder() self.ilo.filter_by_size(size_threshold=1.04, size_behavior='total') self.assertEqual(['index-2016.03.04'], self.ilo.indices) def test_filter_by_size_behavior_total_and_threshold_behavior_less_than(self): self.builder() self.ilo.filter_by_size( size_threshold=1.04, size_behavior='total', threshold_behavior='less_than' ) self.assertEqual(['index-2016.03.03'], self.ilo.indices) elasticsearch-curator-8.0.21/tests/unit/test_class_snapshot_list.py000066400000000000000000000561771477314666200257420ustar00rootroot00000000000000"""test_class_snapshot_list""" from unittest import TestCase from unittest.mock import Mock import yaml from es_client.exceptions import FailedValidation from curator import SnapshotList from curator.exceptions import ConfigurationError, FailedExecution, MissingArgument, NoSnapshots # Get test variables and constants from a single source from . import testvars class TestSnapshotListClientAndInit(TestCase): def test_init_bad_client(self): client = 'not a real client' self.assertRaises(TypeError, SnapshotList, client) def test_init_no_repo_exception(self): client = Mock() self.assertRaises(MissingArgument, SnapshotList, client) def test_init_get_snapshots_exception(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get.side_effect = testvars.fake_fail client.snapshot.get_repository.return_value = {} self.assertRaises( FailedExecution, SnapshotList, client, repository=testvars.repo_name ) def test_init(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual(testvars.snapshots['snapshots'],sl.all_snapshots) self.assertEqual( ['snap_name','snapshot-2015.03.01'], sorted(sl.snapshots) ) class TestSnapshotListOtherMethods(TestCase): def test_empty_list(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual(2, len(sl.snapshots)) sl.snapshots = [] self.assertRaises(NoSnapshots, sl.empty_list_check) def test_working_list(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual(['snap_name', 'snapshot-2015.03.01'], sl.working_list()) class TestSnapshotListAgeFilterName(TestCase): def test_get_name_based_ages_match(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl._get_name_based_ages('%Y.%m.%d') self.assertEqual(1425168000, sl.snapshot_info['snapshot-2015.03.01']['age_by_name'] ) def test_get_name_based_ages_no_match(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl._get_name_based_ages('%Y.%m.%d') self.assertIsNone(sl.snapshot_info['snap_name']['age_by_name']) class TestSnapshotListStateFilter(TestCase): def test_success_inclusive(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_state(state='SUCCESS') self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) def test_success_exclusive(self): client = Mock() client.snapshot.get.return_value = testvars.inprogress client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_state(state='SUCCESS', exclude=True) self.assertEqual(['snapshot-2015.03.01'], sorted(sl.snapshots)) def test_invalid_state(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(ValueError, sl.filter_by_state, state='invalid') class TestSnapshotListRegexFilters(TestCase): def test_filter_by_regex_prefix(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='prefix', value='sna') self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='prefix', value='sna', exclude=True) self.assertEqual([], sl.snapshots) def test_filter_by_regex_middle(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='regex', value='shot') self.assertEqual( ['snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='regex', value='shot', exclude=True) self.assertEqual([], sl.snapshots) def test_filter_by_regex_prefix_exclude(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='prefix', value='snap_', exclude=True) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots) def test_filter_by_regex_timestring(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='timestring', value='%Y.%m.%d') self.assertEqual( ['snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='timestring', value='%Y.%m.%d', exclude=True) self.assertEqual([], sl.snapshots) def test_filter_by_regex_no_value(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) self.assertRaises(ValueError, sl.filter_by_regex, kind='prefix', value=None) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) sl.filter_by_regex(kind='prefix', value=0) self.assertEqual([], sl.snapshots) def test_filter_by_regex_bad_kind(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots) ) self.assertRaises( ValueError, sl.filter_by_regex, kind='invalid', value=None) class TestSnapshotListFilterByAge(TestCase): def test_filter_by_age_missing_direction(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(MissingArgument, sl.filter_by_age, unit='days', unit_count=1 ) def test_filter_by_age_bad_direction(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(ValueError, sl.filter_by_age, unit='days', unit_count=1, direction="invalid" ) def test_filter_by_age_invalid_source(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(ValueError, sl.filter_by_age, unit='days', source='invalid', unit_count=1, direction="older" ) def test_filter_by_age__name_no_timestring(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(MissingArgument, sl.filter_by_age, source='name', unit='days', unit_count=1, direction='older' ) def test_filter_by_age__name_older_than_now(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=1 ) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots) def test_filter_by_age__name_younger_than_now(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(source='name', direction='younger', timestring='%Y.%m.%d', unit='days', unit_count=1 ) self.assertEqual([], sl.snapshots) def test_filter_by_age__name_younger_than_past_date(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(source='name', direction='younger', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1422748800 ) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots) def test_filter_by_age__name_older_than_past_date(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1456963200 ) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots) def test_filter_by_age__creation_date_older_than_now(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(direction='older', unit='days', unit_count=1) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(sl.snapshots)) def test_filter_by_age__creation_date_younger_than_now(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(direction='younger', timestring='%Y.%m.%d', unit='days', unit_count=1 ) self.assertEqual([], sl.snapshots) def test_filter_by_age__creation_date_younger_than_past_date(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(direction='younger', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1422748801 ) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots) def test_filter_by_age__creation_date_older_than_past_date(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(direction='older', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1425168001 ) self.assertEqual(['snap_name'], sl.snapshots) class TestIterateFiltersSnaps(TestCase): def test_no_filters(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) slo.iterate_filters({}) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots) ) def test_no_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) config = {'filters': [{'no_filtertype':'fail'}]} self.assertRaises( FailedValidation, slo.iterate_filters, config) def test_invalid_filtertype_class(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) config = {'filters': [{'filtertype':12345.6789}]} self.assertRaises( FailedValidation, slo.iterate_filters, config) def test_invalid_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) config = yaml.load(testvars.invalid_ft, Loader=yaml.FullLoader)['actions'][1] self.assertRaises( FailedValidation, slo.iterate_filters, config ) def test_age_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) config = yaml.load(testvars.snap_age_ft, Loader=yaml.FullLoader)['actions'][1] slo.iterate_filters(config) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots)) def test_pattern_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) config = yaml.load(testvars.snap_pattern_ft, Loader=yaml.FullLoader)['actions'][1] slo.iterate_filters(config) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots)) def test_none_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) config = yaml.load(testvars.snap_none_ft, Loader=yaml.FullLoader)['actions'][1] slo.iterate_filters(config) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots)) class TestSnapshotListFilterCount(TestCase): def test_missing_count(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(MissingArgument, slo.filter_by_count) def test_without_age(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) slo.filter_by_count(count=1) self.assertEqual(['snap_name'], slo.snapshots) def test_without_age_reversed(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) slo.filter_by_count(count=1, reverse=False) self.assertEqual(['snapshot-2015.03.01'], slo.snapshots) def test_with_age(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) slo.filter_by_count( count=1, source='creation_date', use_age=True ) self.assertEqual(['snap_name'], slo.snapshots) def test_with_age_reversed(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) slo.filter_by_count( count=1, source='creation_date', use_age=True, reverse=False ) self.assertEqual(['snapshot-2015.03.01'], slo.snapshots) def test_sort_by_age(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = SnapshotList(client, repository=testvars.repo_name) slo._calculate_ages() slo.age_keyfield = 'invalid' snaps = slo.snapshots slo._sort_by_age(snaps) self.assertEqual(['snapshot-2015.03.01'], slo.snapshots) class TestSnapshotListPeriodFilter(TestCase): def test_bad_args(self): unit = 'days' range_from = -1 range_to = -2 timestring = '%Y.%m.%d' epoch = 1456963201 expected = FailedExecution client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(expected, sl.filter_period, unit=unit, range_from=range_from, range_to=range_to, source='name', timestring=timestring, epoch=epoch ) def test_in_range(self): unit = 'days' range_from = -2 range_to = 2 epoch = 1425168000 expected = ['snapshot-2015.03.01'] client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_period(source='name', range_from=range_from, epoch=epoch, range_to=range_to, timestring='%Y.%m.%d', unit=unit, ) self.assertEqual(expected, sl.snapshots) def test_not_in_range(self): unit = 'days' range_from = 2 range_to = 4 epoch = 1425168000 expected = [] client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.filter_period( source='name', range_from=range_from, epoch=epoch, range_to=range_to, timestring='%Y.%m.%d', unit=unit, ) self.assertEqual(expected, sl.snapshots) def test_no_creation_date(self): unit = 'days' range_from = -2 range_to = 2 epoch = 1456963201 expected = [] client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) sl.snapshot_info['snap_name']['start_time_in_millis'] = None sl.snapshot_info['snapshot-2015.03.01']['start_time_in_millis'] = None sl.filter_period(source='creation_date', range_from=range_from, epoch=epoch, range_to=range_to, unit=unit, ) self.assertEqual(expected, sl.snapshots) def test_invalid_period_type(self): unit = 'days' range_from = -1 range_to = -2 timestring = '%Y.%m.%d' epoch = 1456963201 expected = ValueError client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(expected, sl.filter_period, unit=unit, period_type='invalid', range_from=range_from, range_to=range_to, source='name', timestring=timestring, epoch=epoch ) def test_invalid_range_from(self): unit = 'days' range_from = -1 range_to = 'invalid' timestring = '%Y.%m.%d' epoch = 1456963201 expected = ConfigurationError client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(expected, sl.filter_period, unit=unit, period_type='relative', range_from=range_from, range_to=range_to, source='name', timestring=timestring, epoch=epoch ) def test_missing_absolute_date_values(self): unit = 'days' range_from = -1 range_to = 'invalid' timestring = '%Y.%m.%d' epoch = 1456963201 expected = ConfigurationError client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = SnapshotList(client, repository=testvars.repo_name) self.assertRaises(expected, sl.filter_period, unit=unit, period_type='absolute', range_from=range_from, range_to=range_to, source='name', timestring=timestring, epoch=epoch ) elasticsearch-curator-8.0.21/tests/unit/test_helpers_date_ops.py000066400000000000000000000632151477314666200251720ustar00rootroot00000000000000"""test_helpers_date_ops""" from datetime import datetime from unittest import TestCase from unittest.mock import Mock import pytest from elasticsearch8 import NotFoundError from elastic_transport import ApiResponseMeta from curator.exceptions import ConfigurationError from curator.helpers.date_ops import ( absolute_date_range, date_range, datetime_to_epoch, fix_epoch, get_date_regex, get_datemath, get_point_of_reference, isdatemath ) class TestGetDateRegex(TestCase): """TestGetDateRegex Test helpers.date_ops.get_date_regex functionality. """ def test_non_escaped(self): """test_non_escaped Should return a proper regex from a non-escaped Python date string """ assert '\\d{4}\\-\\d{2}\\-\\d{2}t\\d{2}' == get_date_regex('%Y-%m-%dt%H') class TestFixEpoch(TestCase): """TestFixEpoch Test helpers.date_ops.fix_epoch functionality. """ def test_fix_epoch(self): """test_fix_epoch Should return straight epoch time in seconds, removing milliseconds or more decimals """ for long_epoch, epoch in [ (1459287636, 1459287636), (14592876369, 14592876), (145928763699, 145928763), (1459287636999, 1459287636), (1459287636000000, 1459287636), (145928763600000000, 1459287636), (145928763600000001, 1459287636), (1459287636123456789, 1459287636), ]: assert epoch == fix_epoch(long_epoch) def test_fix_epoch_raise(self): """test_fix_epoch_raise Should raise a ``ValueError`` exception when an improper value is passed """ with pytest.raises(ValueError): fix_epoch(None) class TestGetPointOfReference(TestCase): """TestGetPointOfReference Test helpers.date_ops.get_point_of_reference functionality. """ def test_get_point_of_reference(self): """test_get_point_of_reference Should return a reference point n units * seconds prior to the present epoch time """ epoch = 1459288037 for unit, result in [ ('seconds', epoch-1), ('minutes', epoch-60), ('hours', epoch-3600), ('days', epoch-86400), ('weeks', epoch-(86400*7)), ('months', epoch-(86400*30)), ('years', epoch-(86400*365)), ]: # self.assertEqual(result, get_point_of_reference(unit, 1, epoch)) assert result == get_point_of_reference(unit, 1, epoch) def test_get_por_raise(self): """test_get_por_raise Should raise a ``ValueError`` exception when an improper value is passed """ self.assertRaises(ValueError, get_point_of_reference, 'invalid', 1) with pytest.raises(ValueError): get_point_of_reference('invalid', 1) class TestDateRange(TestCase): """TestDateRange Test helpers.date_ops.date_range functionality. """ EPOCH = datetime_to_epoch(datetime(2017, 4, 3, 22, 50, 17)) def test_bad_unit(self): """test_bad_unit Should raise a ``ConfigurationError`` exception when an improper unit value is passed """ with pytest.raises(ConfigurationError, match=r'"unit" must be one of'): date_range('invalid', 1, 1) def test_bad_range(self): """test_bad_range Should raise a ``ConfigurationError`` exception when an improper range value is passed """ with pytest.raises(ConfigurationError, match=r'must be greater than or equal to'): date_range('hours', 1, -1) def test_hours_single(self): """test_hours_single Should match hard-coded values when range_from = -1 and range_to = -1 and unit is hours """ unit = 'hours' range_from = -1 range_to = -1 start = datetime_to_epoch(datetime(2017, 4, 3, 21, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 3, 21, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_hours_past_range(self): """test_hours_past_range Should match hard-coded values when range_from = -3 and range_to = -1 and unit is hours """ unit = 'hours' range_from = -3 range_to = -1 start = datetime_to_epoch(datetime(2017, 4, 3, 19, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 3, 21, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_hours_future_range(self): """test_hours_future_range Should match hard-coded values when range_from = 0 and range_to = 2 and unit is hours """ unit = 'hours' range_from = 0 range_to = 2 start = datetime_to_epoch(datetime(2017, 4, 3, 22, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 4, 00, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_hours_span_range(self): """test_hours_span_range Should match hard-coded values when range_from = -1 and range_to = 2 and unit is hours """ unit = 'hours' range_from = -1 range_to = 2 start = datetime_to_epoch(datetime(2017, 4, 3, 21, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 4, 00, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_days_single(self): """test_days_single Should match hard-coded values when range_from = -1 and range_to = -1 and unit is days """ unit = 'days' range_from = -1 range_to = -1 start = datetime_to_epoch(datetime(2017, 4, 2, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 2, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_days_past_range(self): """test_days_range Should match hard-coded values when range_from = -3 and range_to = -1 and unit is days """ unit = 'days' range_from = -3 range_to = -1 start = datetime_to_epoch(datetime(2017, 3, 31, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 2, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_days_future_range(self): """test_days_future_range Should match hard-coded values when range_from = 0 and range_to = 2 and unit is days """ unit = 'days' range_from = 0 range_to = 2 start = datetime_to_epoch(datetime(2017, 4, 3, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 5, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_days_span_range(self): """test_days_span_range Should match hard-coded values when range_from = -1 and range_to = 2 and unit is days """ unit = 'days' range_from = -1 range_to = 2 start = datetime_to_epoch(datetime(2017, 4, 2, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 5, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_weeks_single(self): """test_weeks_single Should match hard-coded values when range_from = -1 and range_to = -1 and unit is weeks """ unit = 'weeks' range_from = -1 range_to = -1 start = datetime_to_epoch(datetime(2017, 3, 26, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 1, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_weeks_past_range(self): """test_weeks_past_range Should match hard-coded values when range_from = -3 and range_to = -1 and unit is weeks """ unit = 'weeks' range_from = -3 range_to = -1 start = datetime_to_epoch(datetime(2017, 3, 12, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 1, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_weeks_future_range(self): """test_weeks_future_range Should match hard-coded values when range_from = 0 and range_to = 2 and unit is weeks """ unit = 'weeks' range_from = 0 range_to = 2 start = datetime_to_epoch(datetime(2017, 4, 2, 00, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 22, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_weeks_span_range(self): """test_weeks_span_range Should match hard-coded values when range_from = -1 and range_to = 2 and unit is weeks """ unit = 'weeks' range_from = -1 range_to = 2 start = datetime_to_epoch(datetime(2017, 3, 26, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 22, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_weeks_single_iso(self): """test_weeks_single_iso Should match hard-coded values when range_from = -1 and range_to = -1, unit is weeks, and ``week_starts_on`` = ``monday`` """ unit = 'weeks' range_from = -1 range_to = -1 start = datetime_to_epoch(datetime(2017, 3, 27, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 2, 23, 59, 59)) # pylint: disable=line-too-long assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH, week_starts_on='monday') def test_weeks_past_range_iso(self): """test_weeks_past_range_iso Should match hard-coded values when range_from = -3 and range_to = -1, unit is weeks, and ``week_starts_on`` = ``monday`` """ unit = 'weeks' range_from = -3 range_to = -1 start = datetime_to_epoch(datetime(2017, 3, 13, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 2, 23, 59, 59)) # pylint: disable=line-too-long assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH, week_starts_on='monday') def test_weeks_future_range_iso(self): """test_weeks_future_range_iso Should match hard-coded values when range_from = 0 and range_to = 2, unit is weeks, and ``week_starts_on`` = ``monday`` """ unit = 'weeks' range_from = 0 range_to = 2 start = datetime_to_epoch(datetime(2017, 4, 3, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 23, 23, 59, 59)) # pylint: disable=line-too-long assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH, week_starts_on='monday') def test_weeks_span_range_iso(self): """test_weeks_span_range_iso Should match hard-coded values when range_from = -1 and range_to = 2, unit is weeks, and ``week_starts_on`` = ``monday`` """ unit = 'weeks' range_from = -1 range_to = 2 start = datetime_to_epoch(datetime(2017, 3, 27, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 4, 23, 23, 59, 59)) # pylint: disable=line-too-long assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH, week_starts_on='monday') def test_months_single(self): """test_months_single Should match hard-coded values when range_from = -1 and range_to = -1 and unit is months """ unit = 'months' range_from = -1 range_to = -1 start = datetime_to_epoch(datetime(2017, 3, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 3, 31, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_months_past_range(self): """test_months_past_range Should match hard-coded values when range_from = -4 and range_to = -1 and unit is months """ unit = 'months' range_from = -4 range_to = -1 start = datetime_to_epoch(datetime(2016, 12, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 3, 31, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_months_future_range(self): """test_months_future_range Should match hard-coded values when range_from = 7 and range_to = 10 and unit is months """ unit = 'months' range_from = 7 range_to = 10 start = datetime_to_epoch(datetime(2017, 11, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2018, 2, 28, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_months_super_future_range(self): """test_months_super_future_range Should match hard-coded values when range_from = 9 and range_to = 10 and unit is months """ unit = 'months' range_from = 9 range_to = 10 start = datetime_to_epoch(datetime(2018, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2018, 2, 28, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_months_span_range(self): """test_months_span_range Should match hard-coded values when range_from = -1 and range_to = 2 and unit is months """ unit = 'months' range_from = -1 range_to = 2 start = datetime_to_epoch(datetime(2017, 3, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 6, 30, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_years_single(self): """test_years_single Should match hard-coded values when range_from = -1 and range_to = -1 and unit is years """ unit = 'years' range_from = -1 range_to = -1 start = datetime_to_epoch(datetime(2016, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2016, 12, 31, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_years_past_range(self): """test_years_past_range Should match hard-coded values when range_from = -3 and range_to = -1 and unit is years """ unit = 'years' range_from = -3 range_to = -1 start = datetime_to_epoch(datetime(2014, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2016, 12, 31, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_years_future_range(self): """test_years_future_range Should match hard-coded values when range_from = 0 and range_to = 2 and unit is years """ unit = 'years' range_from = 0 range_to = 2 start = datetime_to_epoch(datetime(2017, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2019, 12, 31, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) def test_years_span_range(self): """test_years_span_range Should match hard-coded values when range_from = -1 and range_to = 2 and unit is years """ unit = 'years' range_from = -1 range_to = 2 start = datetime_to_epoch(datetime(2016, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2019, 12, 31, 23, 59, 59)) assert (start, end) == date_range(unit, range_from, range_to, epoch=self.EPOCH) class TestAbsoluteDateRange(TestCase): """TestAbsoluteDateRange Test helpers.date_ops.absolute_date_range functionality. """ def test_bad_unit(self): """test_bad_unit Should raise a ``ConfigurationError`` exception when an invalid value for unit is passed """ unit = 'invalid' date_from = '2017.01' date_from_format = '%Y.%m' date_to = '2017.01' date_to_format = '%Y.%m' with pytest.raises(ConfigurationError, match=r'"unit" must be one of'): absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) def test_bad_formats(self): """test_bad_formats Should raise a ``ConfigurationError`` exception when no value for ``date_from_format`` or ``date_to_format`` is passed. """ unit = 'days' with pytest.raises(ConfigurationError, match=r'Must provide "date_from_format" and "date_to_format"'): absolute_date_range(unit, 'meh', 'meh', None, 'meh') with pytest.raises(ConfigurationError, match=r'Must provide "date_from_format" and "date_to_format"'): absolute_date_range(unit, 'meh', 'meh', 'meh', None) def test_bad_dates(self): """test_bad_dates Should raise a ``ConfigurationError`` exception when date formats cannot be parsed for ``date_from_format`` and ``date_to_format`` """ unit = 'weeks' date_from_format = '%Y.%m' date_to_format = '%Y.%m' with pytest.raises(ConfigurationError, match=r'Unable to parse "date_from"'): absolute_date_range(unit, 'meh', '2017.01', date_from_format, date_to_format) with pytest.raises(ConfigurationError, match=r'Unable to parse "date_to"'): absolute_date_range(unit, '2017.01', 'meh', date_from_format, date_to_format) def test_single_month(self): """test_single_month Output should match hard-coded values """ unit = 'months' date_from = '2017.01' date_from_format = '%Y.%m' date_to = '2017.01' date_to_format = '%Y.%m' start = datetime_to_epoch(datetime(2017, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 1, 31, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_multiple_month(self): """test_multiple_month Output should match hard-coded values """ unit = 'months' date_from = '2016.11' date_from_format = '%Y.%m' date_to = '2016.12' date_to_format = '%Y.%m' start = datetime_to_epoch(datetime(2016, 11, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2016, 12, 31, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_single_year(self): """test_single_year Output should match hard-coded values """ unit = 'years' date_from = '2017' date_from_format = '%Y' date_to = '2017' date_to_format = '%Y' start = datetime_to_epoch(datetime(2017, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 12, 31, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_multiple_year(self): """test_multiple_year Output should match hard-coded values """ unit = 'years' date_from = '2016' date_from_format = '%Y' date_to = '2017' date_to_format = '%Y' start = datetime_to_epoch(datetime(2016, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 12, 31, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_single_week_uw(self): """test_single_week_UW Output should match hard-coded values """ unit = 'weeks' date_from = '2017-01' date_from_format = '%Y-%U' date_to = '2017-01' date_to_format = '%Y-%U' start = datetime_to_epoch(datetime(2017, 1, 2, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 1, 8, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_multiple_weeks_uw(self): """test_multiple_weeks_UW Output should match hard-coded values """ unit = 'weeks' date_from = '2017-01' date_from_format = '%Y-%U' date_to = '2017-04' date_to_format = '%Y-%U' start = datetime_to_epoch(datetime(2017, 1, 2, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 1, 29, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_single_week_iso(self): """test_single_week_ISO Output should match hard-coded values """ unit = 'weeks' date_from = '2014-01' date_from_format = '%G-%V' date_to = '2014-01' date_to_format = '%G-%V' start = datetime_to_epoch(datetime(2013, 12, 30, 0, 0, 0)) end = datetime_to_epoch(datetime(2014, 1, 5, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_multiple_weeks_iso(self): """test_multiple_weeks_ISO Output should match hard-coded values """ unit = 'weeks' date_from = '2014-01' date_from_format = '%G-%V' date_to = '2014-04' date_to_format = '%G-%V' start = datetime_to_epoch(datetime(2013, 12, 30, 0, 0, 0)) end = datetime_to_epoch(datetime(2014, 1, 26, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_single_day(self): """test_single_day Output should match hard-coded values """ unit = 'days' date_from = '2017.01.01' date_from_format = '%Y.%m.%d' date_to = '2017.01.01' date_to_format = '%Y.%m.%d' start = datetime_to_epoch(datetime(2017, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 1, 1, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_multiple_days(self): """test_multiple_days Output should match hard-coded values """ unit = 'days' date_from = '2016.12.31' date_from_format = '%Y.%m.%d' date_to = '2017.01.01' date_to_format = '%Y.%m.%d' start = datetime_to_epoch(datetime(2016, 12, 31, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 1, 1, 23, 59, 59)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result def test_iso8601(self): """test_ISO8601 Output should match hard-coded values """ unit = 'seconds' date_from = '2017-01-01T00:00:00' date_from_format = '%Y-%m-%dT%H:%M:%S' date_to = '2017-01-01T12:34:56' date_to_format = '%Y-%m-%dT%H:%M:%S' start = datetime_to_epoch(datetime(2017, 1, 1, 0, 0, 0)) end = datetime_to_epoch(datetime(2017, 1, 1, 12, 34, 56)) result = absolute_date_range(unit, date_from, date_to, date_from_format, date_to_format) assert (start, end) == result class TestIsDateMath(TestCase): """TestIsDateMath Test helpers.date_ops.isdatemath functionality. """ def test_positive(self): """test_positive Result should match hard-coded sample """ data = '' assert isdatemath(data) def test_negative(self): """test_negative Result should not match hard-coded sample """ data = 'not_encapsulated' assert not isdatemath(data) def test_raises(self): """test_raises Should raise ConfigurationError exception when malformed data is passed """ data = '