pax_global_header00006660000000000000000000000064145354002520014512gustar00rootroot0000000000000052 comment=25f3eef498bfd03b3733e3fc2874791ba04c5dfc concurrent-log-handler-0.9.25/000077500000000000000000000000001453540025200161635ustar00rootroot00000000000000concurrent-log-handler-0.9.25/.editorconfig000066400000000000000000000005661453540025200206470ustar00rootroot00000000000000# EditorConfig is awesome: https://EditorConfig.org # top-most EditorConfig file root = true # Unix-style newlines with a newline ending every file [*] # end_of_line = lf insert_final_newline = true # Matches multiple files with brace expansion notation # Set default charset [*.{js,py}] charset = utf-8 # 4 space indentation [*.py] indent_style = space indent_size = 4 concurrent-log-handler-0.9.25/.github/000077500000000000000000000000001453540025200175235ustar00rootroot00000000000000concurrent-log-handler-0.9.25/.github/dependabot.yml000066400000000000000000000007071453540025200223570ustar00rootroot00000000000000# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates#package-ecosystem version: 2 updates: # Enable updates for Github Actions - package-ecosystem: "github-actions" target-branch: "develop" directory: "/" schedule: # Check for updates to GitHub Actions every month interval: "monthly" labels: - "dependencies" concurrent-log-handler-0.9.25/.github/workflows/000077500000000000000000000000001453540025200215605ustar00rootroot00000000000000concurrent-log-handler-0.9.25/.github/workflows/clh_tests.yaml000066400000000000000000000035701453540025200244410ustar00rootroot00000000000000name: Concurrent Log Handler (CLH) tests on: push: branches: - master # or the name of your primary branch pull_request: branches: - master # or the name of your primary branch jobs: lint: name: Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.10 uses: actions/setup-python@v4 with: python-version: "3.10" cache: "pip" - name: Install Hatch run: | pip3 install --upgrade hatch - name: Run linting run: | hatch run lint:all test: name: Test Python ${{ matrix.python-version }} on ${{ matrix.os }} needs: - lint runs-on: ${{ matrix.os }} strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] os: [ubuntu-latest] include: - python-version: "3.11" os: windows-latest steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: pip - name: Install Hatch run: | pip3 install --upgrade hatch - name: Run tests run: | hatch run test.py${{ matrix.python-version }}:cov build: runs-on: ubuntu-latest needs: - lint steps: - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v4 with: python-version: "3.10" cache: pip - name: Install Hatch run: | pip3 install --upgrade hatch - name: Build run: | hatch build --clean - uses: actions/upload-artifact@v3 with: name: artifacts path: dist/* if-no-files-found: error retention-days: 7 concurrent-log-handler-0.9.25/.gitignore000066400000000000000000000003351453540025200201540ustar00rootroot00000000000000*.pyc build dist test_output output_tests setuptools-*.tar.gz setuptools-*.egg src/concurrent_log_handler.egg-info __pycache__ .idea env venv env_py27 env37 *.log* .__*.lock stress_test.log* .eggs .coverage coverage.json concurrent-log-handler-0.9.25/.markdownlint.json000066400000000000000000000000531453540025200216430ustar00rootroot00000000000000{ "MD013": false, "MD046": false } concurrent-log-handler-0.9.25/.run/000077500000000000000000000000001453540025200170455ustar00rootroot00000000000000concurrent-log-handler-0.9.25/.run/CLI stresstest.py.run.xml000066400000000000000000000021631453540025200236360ustar00rootroot00000000000000 concurrent-log-handler-0.9.25/.run/pytest.run.xml000066400000000000000000000017021453540025200217220ustar00rootroot00000000000000 concurrent-log-handler-0.9.25/.vscode/000077500000000000000000000000001453540025200175245ustar00rootroot00000000000000concurrent-log-handler-0.9.25/.vscode/extensions.json000066400000000000000000000005641453540025200226230ustar00rootroot00000000000000{ "recommendations": [ "charliermarsh.ruff", "ms-python.black-formatter", "streetsidesoftware.code-spell-checker", "editorconfig.editorconfig", "usernamehw.errorlens", "mhutchie.git-graph", "ms-python.python", "ms-python.vscode-pylance", "stkb.rewrap", "tamasfe.even-better-toml" ] } concurrent-log-handler-0.9.25/.vscode/launch.json000066400000000000000000000027521453540025200216770ustar00rootroot00000000000000{ // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "name": "Python: Current File", "type": "python", "request": "launch", "program": "${file}", "console": "integratedTerminal", "justMyCode": true }, { "name": "CLH PyTests", "type": "python", "request": "launch", "module": "pytest", "console": "integratedTerminal", "cwd": "${workspaceFolder}/tests", "args": [], "justMyCode": false }, { "name": "Concurrent Log Handler basic example", "type": "python", "request": "launch", "program": "${workspaceFolder}/tests/other/test.py", "console": "integratedTerminal", "cwd": "${workspaceFolder}/tests/other", "justMyCode": true }, { "name": "CLH old stress test", "type": "python", "request": "launch", "program": "${workspaceFolder}/tests/other/stresstest.py", "console": "integratedTerminal", "cwd": "${workspaceFolder}/tests/other", "args": ["--log-calls", "2000"], "justMyCode": true }, ] } concurrent-log-handler-0.9.25/.vscode/settings.json000066400000000000000000000022041453540025200222550ustar00rootroot00000000000000{ "files.exclude": { ".idea": true, ".ruff_cache": true, ".run": true, "**/.pytest_cache": true, "**/*.egg-info": true, "**/*.pyc": true, "build": true, "dist": true, "env": true, "output_tests": true, "venv": true }, "cSpell.words": [ "Alleman", "builtins", "Cabe", "concat", "CREAT", "datetime", "gunicorn", "gzipped", "Journyx", "kwarg", "kwargs", "levelname", "logfile", "logfile's", "logfiles", "maxsplit", "multiprocess", "portalocker", "POSIX", "pychecker", "pycodestyle", "pydoc", "Pyflakes", "Pylint", "pypi", "pytest", "pywin", "queuify", "stresstest", "tempname", "timeframe", "tmpname", "trfh" ], "cSpell.ignoreRegExpList": [ "/\\s(os|errno|time|pwd|grp)[.].+?\\s/gi", ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } concurrent-log-handler-0.9.25/CHANGELOG.md000066400000000000000000000123621453540025200200000ustar00rootroot00000000000000# Change Log - 0.9.25: - Improvements to project config (`pyproject.toml`) with `hatch` (PR #65), and the addition of Python typing hints (PR #69). Thanks @stumpylog. - Fixes [Issue #66](https://github.com/Preston-Landers/concurrent-log-handler/issues/66) Timed mode rollover fails if backupCount limit is hit and gzip is on. Thanks @moynihan. - Fixes [Issue #60](https://github.com/Preston-Landers/concurrent-log-handler/issues/60) Timed mode causes DeprecationWarning if you don't give the `delay` parameter. Thanks @platinops. - 0.9.24: - Fixes #58 - Eliminate `use_2to3` kwarg causing problems in setup.py. - 0.9.23: - Begin requiring Python 3.6 or higher. - Implements a `ConcurrentTimedRotatingFileHandler` class which provides both time and/or size based rotation. See the [README.md](./README.md#time-based-rotation-settings) for details. - Fix #56 - don't fail when setting `owner` on Windows, though it will have no effect. - 0.9.22: - Fix Python 2.7 compatibility (yet again) - Important note: this is the FINAL release which will support Python 2.7. Future versions will support Python 3.6+ only. - 0.9.21: - Added new optional parameter "lock_file_directory" - Creates given directory, if it does not exist. - Writes lock file into given directory, instead next to the logging file itself. - Useful when the log files reside in a cloud synced folder like Dropbox, Google Drive, OneDrive, etc. Sometimes these do not work correctly with the lock files. - Fix not replacing the last file (greatest backup number) when rotating. Thanks tzongw. - - Add support for "namer" function to customize the naming of rotated files. Thanks @dashedman. - Enhanced test suite using tox and pytest. - 0.9.20: Threaded logging queue now uses asyncio and can be used after fork (PR#32). - The classifiers have been updated to indicate generic Python 3 support without needing to specify all sub-versions. (However, Python 3.0 to 3.4 support is not claimed.) - Better performance with large values for backupCount (number of rotated files to keep). - You can set the file owner / group to 'root' (uid 0) - Test script has been made more reliable. - 0.9.19: Fix Python 2 compatibility (again), thanks @buddly27 Fix accidental detection of ' darwin' (Mac OS) as Windows in setup.py - 0.9.18: Remove ez_setup from the setup.py - 0.9.17: Contains the following fixes: - Catch exceptions when unlocking the lock. - Clarify documentation, esp. with use of multiprocessing - In Python 2, don't request/allow portalocker 2.0 which won't work. (Require portalocker< =1.7.1) NOTE: the next release will likely be a 1.0 release candidate. - 0.9.16: Fix publishing issue with incorrect code included in the wheel Affects Python 2 mainly - see Issue #21 - 0.9.15: Fix bug from last version on Python 2. (Issue #21) Thanks @condontrevor Also, on Python 2 and 3, apply unicode_error_policy (default: ignore) to convert a log message to the output stream's encoding. I.e., by default it will filter out (remove) any characters in a log message which cannot be converted to the output logfile's encoding. - 0.9.14: Fix writing LF line endings on Windows when encoding is specified. Added newline and terminator kwargs to allow customizing line ending behavior. Thanks to @vashek - 0.9.13: Fixes Crashes with ValueError: I/O operation on closed file (issue #16) Also should fix issue #13 with crashes related to Windows file locking. Big thanks to @terencehonles, @nsmcan, @wkoot, @dismine for doing the hard parts - 0.9.12: Add umask option (thanks to @blakehilliard) This adds the ability to control the permission flags when creating log files. - 0.9.11: Fix issues with gzip compression option (use buffering) - 0.9.10: Fix inadvertent lock sharing when forking Thanks to @eriktews for this fix - 0.9.9: Fix Python 2 compatibility broken in last release - 0.9.8: Bug fixes and permission features - Fix for issue #4 - AttributeError: 'NoneType' object has no attribute 'write' This error could be caused if a rollover occurred inside a logging statement that was generated from within another logging statement's format() call. - Fix for PyWin32 dependency specification (explicitly require PyWin32) - Ability to specify owner and permissions (mode) of rollover files [Unix only] - 0.9.7/0.9.6: Fix platform specifier for PyPi - 0.9.5: Add `use_gzip` option to compress rotated logs. Add an optional threaded logging queue handler based on the standard library's `logging.QueueHandler`. - 0.9.4: Fix setup.py to not include tests in distribution. - 0.9.3: Refactoring release - For publishing fork on pypi as `concurrent-log-handler` under new package name. - NOTE: PyWin32 is required on Windows but is not an explicit dependency because the PyWin32 package is not currently installable through pip. - Fix lock behavior / race condition - 0.9.2: Initial release of fork by Preston Landers based on a fork of Lowell Alleman's ConcurrentLogHandler 0.9.1 - Fixes deadlocking issue with recent versions of Python - Puts `.__` prefix in front of lock file name - Use `secrets` or `SystemRandom` if available. - Add/fix Windows support concurrent-log-handler-0.9.25/CONTRIBUTORS.md000066400000000000000000000021231453540025200204400ustar00rootroot00000000000000# Contributors The following folks were kind enough to contribute to this fork, in no particular order: [https://github.com/Preston-Landers](https://github.com/Preston-Landers) [https://github.com/stumpylog](https://github.com/stumpylog) [https://github.com/und3rc](https://github.com/und3rc) [https://github.com/wcooley](https://github.com/wcooley) [https://github.com/greenfrog82](https://github.com/greenfrog82) [https://github.com/blakehilliard](https://github.com/blakehilliard) [https://github.com/eriktews](https://github.com/eriktews) [https://github.com/ZhuYuJin](https://github.com/ZhuYuJin) [https://github.com/vashek](https://github.com/vashek) [https://github.com/terencehonles](https://github.com/terencehonles) [https://github.com/fr-ez](https://github.com/fr-ez) [https://github.com/mariusvniekerk](https://github.com/mariusvniekerk) [https://github.com/buddly27](https://github.com/buddly27) [https://github.com/aDramaQueen](https://github.com/aDramaQueen) [https://github.com/tzongw](https://github.com/tzongw) [https://github.com/dashedman](https://github.com/dashedman) concurrent-log-handler-0.9.25/LICENSE000066400000000000000000000217771453540025200172060ustar00rootroot00000000000000Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and 2. You must cause any modified files to carry prominent notices stating that You changed the files; and 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS concurrent-log-handler-0.9.25/README.md000066400000000000000000000401451453540025200174460ustar00rootroot00000000000000# concurrent-log-handler This package provides an additional log handler for Python's standard logging package (PEP 282). This handler will write log events to a log file which is rotated when the log file reaches a certain size. Multiple processes can safely write to the same log file concurrently. Rotated logs can be gzipped if desired. Both Windows and POSIX systems are supported. An optional threaded queue logging handler is provided to perform logging in the background. This is a fork of Lowell Alleman's ConcurrentLogHandler 0.9.1 which fixes a hanging/deadlocking problem. [See this](https://bugs.launchpad.net/python-concurrent-log-handler/+bug/1265150). Summary of other changes: * New: requires Python 3.6 or higher. * If you require support for Python 2.7, use version [0.9.22](https://github.com/Preston-Landers/concurrent-log-handler/releases/tag/0.9.22). * Renamed package to `concurrent_log_handler` (abbreviated CLH in this file.) * Provide `use_gzip` option to compress rotated logs * Support for Windows * Uses file locking to ensure exclusive write access Note: file locking is advisory, not a hard lock against external processes * More secure generation of random numbers for temporary filenames * Change the name of the lockfile to have .__ in front of it. * Provide a class for time-based rotation: [ConcurrentTimedRotatingFileHandler](#time-based-rotation-settings) * Provide an optional QueueListener / QueueHandler implementation for handling log events in a background thread. * Allow setting owner and mode permissions of rollover file on Unix * Depends on `portalocker` package, which (on Windows only) depends on `PyWin32` ## Links * [concurrent-log-handler on Github](https://github.com/Preston-Landers/concurrent-log-handler) * [concurrent-log-handler on the Python Package Index (PyPI)](https://pypi.org/project/concurrent-log-handler/) ## Primary use cases The main use case this is designed to support is when you have a Python application that runs in multiple processes, potentially on multiple hosts connected with a shared network drive, and you want to write all log events to a central log file and have those files rotated based on size and/or time, e.g. daily or hourly. However, this is not the only way to achieve shared logging from multiple processes. You can also centralize logging by using cloud logging services like Azure Log Monitor, Logstash, etc. Or you can implement your own remote logging server as shown here: [Logging cookbook: network](https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network) Concurrent-Log-Handler includes a QueueHandler and QueueListener implementation that can be used to perform logging in the background asynchronously, so the thread or process making the log statement doesn't have to wait for its completion. See [this section](#simple-example). Using that example code, each process still locks and writes the file separately, so there is no centralized writer. You could also write code to use QueueHandler and QueueListener to queue up log events within each process to be sent to a central server, instead of CLH's model where each process locks and writes to the log file. ### Time-based rotation The main `ConcurrentRotatingFileHandler` class supports size-based rotation only. In addition, a `ConcurrentTimedRotatingFileHandler` class is provided that supports both time-based and size-based rotation. By default, it does hourly time-based rotation and no size rotation. See [this section](#time-based-rotation-settings) for more details. ## Instructions and Usage ### Installation You can download and install the package with `pip` using the following command: pip install concurrent-log-handler This will also install the portalocker module, which on Windows in turn depends on pywin32. If installing from source, use the following command: python setup.py install ### Developer setup If you plan to modify the code, you should follow this procedure: * Clone the repository * Create a virtual environment (`venv`) and activate it. * Install the package in editable mode with the [dev] option: `pip install -e .[dev]` * Run the tests: `tox` or run `pytest` directly. Or manually run a single pass of the stress test with specific options: ```shell python tests/stresstest.py --help python tests/stresstest.py --gzip --num-processes 12 --log-calls=5000 ``` * To build a Python "wheel" for distribution, use the following: ```shell python setup.py clean --all build sdist bdist_wheel # Copy the .whl file from under the "dist" folder # or upload with twine: pip install twine twine upload dist/concurrent-log-handler-0.9.23.tar.gz dist/concurrent_log_handler-0.9.23-py3-none-any.whl ``` ### Important Requirements Concurrent Log Handler (CLH) is designed to allow multiple processes to write to the same logfile in a concurrent manner. It is important that each process involved MUST follow these requirements: * You can't serialize a handler instance and reuse it in another process. This means you cannot, for example, pass a CLH handler instance from parent process to child process using the `multiprocessing` package in spawn mode (or similar techniques that use serialized objects). Each child process must initialize its own CLH instance. * When using the `multiprocessing` module in "spawn" (non-fork) mode, each child process must create its OWN instance of the handler (`ConcurrentRotatingFileHandler`). The child target function should call code that initializes a new CLH instance. * This requirement does not apply to threads within a given process. Different threads within a process can use the same CLH instance. Thread locking is handled automatically. * This also does not apply to `fork()` based child processes such as gunicorn --preload. Child processes of a fork() call should be able to inherit the CLH object instance. * This limitation exists because the CLH object can't be serialized, passed over a network or pipe, and reconstituted at the other end. * It is important that every process or thread writing to a given logfile must all use the same settings, especially related to file rotation. Also do not attempt to mix different handler classes writing to the same file, e.g. do not also use a `RotatingFileHandler` on the same file. * Special attention may need to be paid when the log file being written to resides on a network shared drive or a cloud synced folder (Dropbox, Google Drive, etc.). Whether the multiprocess advisory lock technique (via portalocker) works in these folders may depend on the details of your configuration. Note that a `lock_file_directory` setting (kwarg) now exists (as of v0.9.21) which lets you place the lockfile at a different location from the main logfile. This might solve problems related to trying to lock files in network shares or cloud folders (Dropbox, Google Drive, etc.) However, if multiple hosts are writing to the same shared logfile, they must also have access to the same lock file. Alternatively, you may be able to set your cloud sync software to ignore all `.lock` files. * A separate handler instance is needed for each individual log file. For instance, if your app writes to two different log files you will need to set up two CLH instances per process. ### Simple Example Here is a simple direct usage example: ```python from logging import getLogger, INFO from concurrent_log_handler import ConcurrentRotatingFileHandler import os log = getLogger(__name__) # Use an absolute path to prevent file rotation trouble. logfile = os.path.abspath("mylogfile.log") # Rotate log after reaching 512K, keep 5 old copies. rotateHandler = ConcurrentRotatingFileHandler(logfile, "a", 512 * 1024, 5) log.addHandler(rotateHandler) log.setLevel(INFO) log.info("Here is a very exciting log message, just for you") ``` See also the file [src/example.py](src/example.py) for a configuration and usage example. This shows both the standard non-threaded non-async usage, and the use of the `asyncio` background logging feature. Under that option, when your program makes a logging statement, it is added to a background queue and may not be written immediately and synchronously. This queue can span multiple processes using `multiprocessing` or `concurrent.futures`, and spanning multiple hosts works due to the use of file locking on the log file. Note that with this async logging feature, currently there is no way for the caller to know when the logging statement completed (no "Promise" or "Future" object is returned). [QueueHandler](https://docs.python.org/3/library/logging.handlers.html#queuehandler) ### Configuration To use this module from a logging config file, use a handler entry like this: ```ini [handler_hand01] class = handlers.ConcurrentRotatingFileHandler level = NOTSET formatter = form01 args = ("rotating.log", "a") kwargs = {'backupCount': 5, 'maxBytes': 1048576, 'use_gzip': True} ``` That sets the files to be rotated at about 10 MB, and to keep the last 5 rotations. It also turns on gzip compression for rotated files. Please note that Python 3.7 and higher accepts keyword arguments (kwargs) in a logging config file, but earlier versions of Python only accept positional args. Note: you must have an `import concurrent_log_handler` before you call fileConfig(). For more information see Python docs on [log file formats](https://docs.python.org/3/library/logging.config.html#logging-config-fileformat) ### Limitations The size-based rotation limit (`maxBytes`) is not strict. The files may become slightly larger than `maxBytes`. How much larger depends on the size of the log message being written when the rollover occurs. By contrast, the base `RotatingLogHandler` class tries to ensure that the log file is always kept under `maxBytes` taking into account the size of the current log message being written. This limitation may be changed in the future. ### Recommended Settings For best performance, avoid setting the `backupCount` (number of rollover files to keep) too high. What counts as "too high" is situational, but a good rule of thumb might be to keep around a maximum of 20 rollover files. If necessary, increase the `maxBytes` so that each file can hold more. Too many rollover files can slow down the rollover process due to the mass file renames, and the rollover occurs while the file lock is held for the main logfile. How big to allow each file to grow (`maxBytes`) is up to your needs, but generally a value of 10 MB (1048576) to 100 MB (1048576) is reasonable. Gzip compression is turned off by default. If enabled it will reduce the storage needed for rotated files, at the cost of some minimal CPU overhead. Use of the background logging queue shown below can help offload the cost of logging to another thread. Sometimes you may need to place the lock file at a different location from the main log file. A `lock_file_directory` setting (kwarg) now exists (as of v0.9.21) which lets you place the lockfile at a different location. This can often solve problems related to trying to lock files in cloud folders (Dropbox, Google Drive, OneDrive, etc.) However, in order for this to work, each process writing to the log must have access to the same lock file location, even if they are running on different hosts. You can set the `namer` attribute of the handler to customize the naming of the rotated files, in line with the `BaseRotatingHandler` class. See the Python docs for [more details](https://docs.python.org/3.11/library/logging.handlers.html#logging.handlers.BaseRotatingHandler.namer). ### Line Endings By default, the logfile will have line endings appropriate to the platform. On Windows the line endings will be CRLF ('\r\n') and on Unix/Mac they will be LF ('\n'). It is possible to force another line ending format by using the newline and terminator arguments. The following would force Windows-style CRLF line endings on Unix: kwargs={'newline': '', 'terminator': '\r\n'} The following would force Unix-style LF line endings on Windows: kwargs={'newline': '', 'terminator': '\n'} ### Time-based rotation settings An alternative class `ConcurrentTimedRotatingFileHandler` is also provided which supports time-based rotation, defaulting to hourly. Like the main class, it uses advisory file locking to both ensure that only one process/thread is writing to the log file at a time, and to coordinate the rollover time between processes. By default, it has `maxBytes` set to 0, which means that it will not rotate based on file size, but it is possible to set `maxBytes` to a value to limit the size of each file in addition to the time-based rotation. When files are rotated based on size, they may have an additional numeric suffix like `.1` added to the filename. Note that like with the main CLH class, the file size limits are *not* strictly adhered to. All the same settings are available for this class as for the main class, including `maxBytes`, `use_gzip`, `lock_file_directory`, `newline`, and `terminator`. However, the ordering of the arguments is different, so it's recommended to use keyword arguments when using or configuring this class. The arguments shared with `TimedRotatingFileHandler` are in the same order as the base class, and the extra CLH arguments come after that, although not in the exact same order due to some overlap. For configuration, see the [configuration section](#configuration) above, but substitute in `class=handlers.ConcurrentTimedRotatingFileHandler` and other appropriate settings like `when` and `interval`. See the Python docs for `TimedRotatingFileHandler` for [more details](https://docs.python.org/3.11/library/logging.handlers.html#logging.handlers.TimedRotatingFileHandler). ### Background logging queue To use the background logging queue, you must call this code at some point in your app after it sets up logging configuration. Please read the doc string in the file `concurrent_log_handler/queue.py` for more details. This requires Python 3. See also [src/example.py](src/example.py). ```python from concurrent_log_handler.queue import setup_logging_queues # convert all configured loggers to use a background thread setup_logging_queues() ``` This module is designed to function well in a multi-threaded or multi-processes concurrent environment. However, all writers to a given log file should be using the same class and the *same settings* at the same time, otherwise unexpected behavior may result during file rotation. This may mean that if you change the logging settings at any point you may need to restart your app service so that all processes are using the same settings at the same time. ## Other Usage Details The `ConcurrentRotatingFileHandler` class is a drop-in replacement for Python's standard log handler `RotatingFileHandler`. This module uses file locking so that multiple processes can concurrently log to a single file without dropping or clobbering log events. This module provides a file rotation scheme like with `RotatingFileHandler`. Extra care is taken to ensure that logs can be safely rotated before the rotation process is started. (This module works around the file rename issue with `RotatingFileHandler` on Windows, where a rotation failure means that all subsequent log events are dropped). This module attempts to preserve log records at all cost. This means that log files will grow larger than the specified maximum (rotation) size. So if disk space is tight, you may want to stick with `RotatingFileHandler`, which will strictly adhere to the maximum file size. Important: If you have multiple instances of a script (or multiple scripts) all running at the same time and writing to the same log file, then *all* of the scripts should be using `ConcurrentRotatingFileHandler`. You should not attempt to mix and match `RotatingFileHandler` and `ConcurrentRotatingFileHandler`. The file locking is advisory only - it is respected by other Concurrent Log Handler instances, but does not protect against outside processes (or different Python logging file handlers) from writing to a log file in use. ## Changelog See [CHANGELOG.md](CHANGELOG.md) ## Contributors The original version was written by Lowell Alleman. Other contributors are listed in [CONTRIBUTORS.md](CONTRIBUTORS.md). ## License See the [LICENSE file](LICENSE) concurrent-log-handler-0.9.25/pyproject.toml000066400000000000000000000161771453540025200211130ustar00rootroot00000000000000[build-system] requires = ["hatchling"] build-backend = "hatchling.build" [project] name = "concurrent-log-handler" dynamic = ["version"] description = "RotatingFileHandler replacement with concurrency, gzip and Windows support" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.6" authors = [ { name = "Preston Landers", email = "planders@utexas.edu" }, ] keywords = [ "QueueHandler", "QueueListener", "linux", "logging", "portalocker", "rotate", "unix", "windows", ] classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: Apache Software License", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Logging", ] dependencies = [ "portalocker>=1.6.0", ] [project.urls] Homepage = "https://github.com/Preston-Landers/concurrent-log-handler" [tool.hatch.version] path = "src/concurrent_log_handler/__version__.py" [tool.hatch.build.targets.sdist] include = [ "/src", ] [tool.hatch.envs.test] dependencies = [ "coverage[toml] >= 7.2", "pytest >= 7.4", "pytest-sugar", ] [tool.hatch.envs.test.scripts] version = "python3 --version" pip-list = "pip list" test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" cov-clear = "coverage erase" cov-report = [ "- coverage combine", "coverage report", ] cov-html = "coverage html" cov-json = "coverage json" cov = [ "version", "pip-list", "cov-clear", "test-cov", "cov-report", "cov-json", "cov-html" ] [[tool.hatch.envs.test.matrix]] python = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] [tool.hatch.envs.lint] detached = true dependencies = [ "black>=23.9.1", "mypy>=1.6.0", "ruff>=0.1.0", "portalocker", ] [tool.hatch.envs.lint.scripts] typing = [ "mypy --version", "mypy --install-types --non-interactive {args:src/concurrent_log_handler}" ] style = [ "ruff {args:.}", "black --check --diff {args:.}", ] fmt = [ "black {args:.}", "ruff {args:.}", "style", ] all = [ "style", "typing", ] [tool.ruff] output-format = "grouped" target-version = "py37" fix = true ignore = [ # Never enforce `E501` (line length violations). "E501", "D415", # First line of docstring should end with a period "COM812", # missing trailing comma in Python 3.5+ "UP008", # use of super(class, self) - probably can re-apply this at some point "SIM115", # Use context handler for open() "PLR1711", # useless return stmt "EM101", # use of string literal in exception "EM102", # using f-string in exception literal "T201", # use of print "SLF001", # access to _ private members "EXE001", # executable bit set (might want to revisit the ignore) ] select = [ "E", # pycodestyle Errors - https://docs.astral.sh/ruff/rules/#error-e "W", # pycodestyle Warnings - https://docs.astral.sh/ruff/rules/#warning-w "F", # Pyflakes - https://docs.astral.sh/ruff/rules/#pyflakes-f "C90", # McCabe complexity - https://docs.astral.sh/ruff/rules/#mccabe-c90 "I", # import sorting - https://docs.astral.sh/ruff/rules/#isort-i # "D", # pydoc style - https://docs.astral.sh/ruff/rules/#pydocstyle-d # "UP", # python upgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up "YTT", # sys.version misused - https://docs.astral.sh/ruff/rules/#flake8-2020-ytt # "ANN", # annotations - https://docs.astral.sh/ruff/rules/#flake8-annotations-ann "S", # flake8 bandit - https://docs.astral.sh/ruff/rules/#flake8-bandit-s # "BLE", # blind exceptions - https://docs.astral.sh/ruff/rules/#flake8-blind-except-ble # "FBT", # boolean trap - https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt "B", # bugbear - https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "A", # flake8 builtins - https://docs.astral.sh/ruff/rules/#flake8-builtins-a "COM", # flake8 commas - https://docs.astral.sh/ruff/rules/#flake8-commas-com "C4", # flake8 comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 "DTZ", # flake8 datetime - https://docs.astral.sh/ruff/rules/#flake8-datetime-dtz "EM", # flake8 error messages - https://docs.astral.sh/ruff/rules/#flake8-errmsg-em "EXE", # flake8 executables https://docs.astral.sh/ruff/rules/#flake8-executable-exe "ISC", # implicit string concat https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc "ICN", # import conventions - https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn "G", # flake8 logging - https://docs.astral.sh/ruff/rules/#flake8-logging-format-g "INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp "PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie "T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20 "PYI", # https://docs.astral.sh/ruff/rules/#flake8-pyi-pyi "PT", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt "Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q "RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse "RET", # https://docs.astral.sh/ruff/rules/#flake8-return-ret "SLF", # https://docs.astral.sh/ruff/rules/#flake8-self-slf "SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim "TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid "TCH", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch "INT", # https://docs.astral.sh/ruff/rules/#flake8-gettext-int "ARG", # https://docs.astral.sh/ruff/rules/#flake8-unused-arguments-arg # "PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth # "ERA", # https://docs.astral.sh/ruff/rules/#eradicate-era "PD", # https://docs.astral.sh/ruff/rules/#pandas-vet-pd "PGH", # https://docs.astral.sh/ruff/rules/#pygrep-hooks-pgh # Pylint "PLC", # https://docs.astral.sh/ruff/rules/#convention-plc "PLE", # https://docs.astral.sh/ruff/rules/#error-ple "PLR", # https://docs.astral.sh/ruff/rules/#refactor-plr "PLW", # https://docs.astral.sh/ruff/rules/#warning-plw # "TRY", # https://docs.astral.sh/ruff/rules/#tryceratops-try "NPY", # https://docs.astral.sh/ruff/rules/#numpy-specific-rules-npy "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf ] [tool.pytest.ini_options] testpaths =["tests"] [tool.coverage.run] omit = [ "tests/stresstest.py" ] [tool.mypy] python_version = "3.8" platform = "linux" disallow_any_unimported = true disallow_any_explicit = true disallow_untyped_defs = true disallow_untyped_calls = true disallow_incomplete_defs = true check_untyped_defs = true strict_optional = true warn_redundant_casts = true warn_unused_ignores = true warn_no_return = true warn_return_any = true warn_unreachable = true warn_unused_configs = true concurrent-log-handler-0.9.25/src/000077500000000000000000000000001453540025200167525ustar00rootroot00000000000000concurrent-log-handler-0.9.25/src/concurrent_log_handler/000077500000000000000000000000001453540025200234725ustar00rootroot00000000000000concurrent-log-handler-0.9.25/src/concurrent_log_handler/__init__.py000066400000000000000000001110151453540025200256020ustar00rootroot00000000000000#!/usr/bin/env python # # Copyright 2013 Lowell Alleman # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy # of the License at http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """concurrent_log_handler: A smart replacement for the standard RotatingFileHandler ConcurrentRotatingFileHandler: This class is a log handler which is a drop-in replacement for the python standard log handler 'RotateFileHandler', the primary difference being that this handler will continue to write to the same file if the file cannot be rotated for some reason, whereas the RotatingFileHandler will strictly adhere to the maximum file size. Unfortunately, if you are using the RotatingFileHandler on Windows, you will find that once an attempted rotation fails, all subsequent log messages are dropped. The other major advantage of this module is that multiple processes can safely write to a single log file. To put it another way: This module's top priority is preserving your log records, whereas the standard library attempts to limit disk usage, which can potentially drop log messages. If you are trying to determine which module to use, there are number of considerations: What is most important: strict disk space usage or preservation of log messages? What OSes are you supporting? Can you afford to have processes blocked by file locks? Concurrent access is handled by using file locks, which should ensure that log messages are not dropped or clobbered. This means that a file lock is acquired and released for every log message that is written to disk. (On Windows, you may also run into a temporary situation where the log file must be opened and closed for each log message.) This can have potentially performance implications. In my testing, performance was more than adequate, but if you need a high-volume or low-latency solution, I suggest you look elsewhere. Warning: see notes in the README.md about changing rotation settings like maxBytes. If different processes are writing to the same file, they should all have the same settings at the same time, or unexpected behavior may result. This may mean that if you change the logging settings at any point you may need to restart your app service so that all processes are using the same settings at the same time. This module currently only support the 'nt' and 'posix' platforms due to the usage of the portalocker module. I do not have access to any other platforms for testing, patches are welcome. See the README file for an example usage of this module. This module supports Python 3.6 and later. (Support for older version was dropped in 0.9.23.) """ import datetime import errno import logging import os import sys import time import traceback import warnings from contextlib import contextmanager from io import TextIOWrapper from logging.handlers import BaseRotatingHandler, TimedRotatingFileHandler from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Tuple from portalocker import LOCK_EX, lock, unlock try: import grp import pwd except ImportError: pwd = grp = None # type: ignore[assignment] # Random numbers for rotation temp file names, using secrets module if available (Python 3.6). # Otherwise use `random.SystemRandom` if available, then fall back on `random.Random`. try: from secrets import randbits except ImportError: import random if hasattr(random, "SystemRandom"): # May not be present in all Python editions # Should be safe to reuse `SystemRandom` - not software state dependant randbits = random.SystemRandom().getrandbits else: def randbits(k: int) -> int: return random.Random().getrandbits(k) try: import gzip except ImportError: gzip = None # type: ignore[assignment] __all__ = [ "ConcurrentRotatingFileHandler", "ConcurrentTimedRotatingFileHandler", ] HAS_CHOWN: bool = hasattr(os, "chown") HAS_CHMOD: bool = hasattr(os, "chmod") class ConcurrentRotatingFileHandler(BaseRotatingHandler): """Handler for logging to a set of files, which switches from one file to the next when the current file reaches a certain size. Multiple processes can write to the log file concurrently, but this may mean that the file will exceed the given size. """ def __init__( # noqa: PLR0913 self, filename: str, mode: str = "a", maxBytes: int = 0, backupCount: int = 0, encoding: Optional[str] = None, debug: bool = False, delay: None = None, use_gzip: bool = False, owner: Optional[Tuple[str, str]] = None, chmod: Optional[int] = None, umask: Optional[int] = None, newline: Optional[str] = None, terminator: str = "\n", unicode_error_policy: str = "ignore", lock_file_directory: Optional[str] = None, ): """Open the specified file and use it as the stream for logging. :param filename: name of the log file to output to. :param mode: write mode: defaults to 'a' for text append :param maxBytes: rotate the file at this size in bytes :param backupCount: number of rotated files to keep before deleting. Avoid setting this very high, probably 20 or less, and prefer setting maxBytes higher. A very large number of rollover files can slow down the rollover enough to cause problems due to the mass file renaming while the main lock is held. :param encoding: text encoding for logfile :param debug: add extra debug statements to this class (for development) :param delay: DEPRECATED: value is ignored :param use_gzip: automatically gzip rotated logs if available. :param owner: 2 element sequence with (user owner, group owner) of log files. (Unix only) :param chmod: permission of log files. (Unix only) :param umask: umask settings to temporarily make when creating log files. This is an alternative to chmod. It is mainly for Unix systems but can also be used on Windows. The Windows security model is more complex and this is not the same as changing access control entries. :param newline: None (default): use CRLF on Windows, LF on Unix. Set to '' for no translation, in which case the 'terminator' argument determines the line ending. :param terminator: set to '\r\n' along with newline='' to force Windows style newlines regardless of OS platform. :param unicode_error_policy: should be one of 'ignore', 'replace', 'strict' Determines what happens when a message is written to the log that the stream encoding doesn't support. Default is to ignore, i.e., drop the unusable characters. :param lock_file_directory: name of directory for all lock files as alternative living space; this is useful for when the main log files reside in a cloud synced drive like Dropbox, OneDrive, Google Docs, etc., which may prevent the lock files from working correctly. The lock file must be accessible to all processes writing to a shared log, including across all different hosts (machines). By default, the file grows indefinitely. You can specify particular values of maxBytes and backupCount to allow the file to rollover at a predetermined size. Rollover occurs whenever the current log file is nearly maxBytes in length. If backupCount is >= 1, the system will successively create new files with the same pathname as the base file, but with extensions ".1", ".2" etc. appended to it. For example, with a backupCount of 5 and a base file name of "app.log", you would get "app.log", "app.log.1", "app.log.2", ... through to "app.log.5". The file being written to is always "app.log" - when it gets filled up, it is closed and renamed to "app.log.1", and if files "app.log.1", "app.log.2" etc. exist, then they are renamed to "app.log.2", "app.log.3" etc. respectively. If maxBytes is zero, rollover never occurs. This log handler assumes that all concurrent processes logging to a single file will are using only this class, and that the exact same parameters are provided to each instance of this class. If, for example, two different processes are using this class, but with different values for 'maxBytes' or 'backupCount', then odd behavior is expected. The same is true if this class is used by one application, but the RotatingFileHandler is used by another. """ # noinspection PyTypeChecker self.stream: Optional[TextIOWrapper] = None # type: ignore[assignment] self.stream_lock: Optional[TextIOWrapper] = None self.owner = owner self.chmod = chmod self.umask = umask self._set_uid: Optional[int] = None self._set_gid: Optional[int] = None self.maxBytes = maxBytes self.backupCount = backupCount self.newline = newline self._debug = debug self.use_gzip = bool(gzip and use_gzip) self.gzip_buffer = 8096 self.maxLockAttempts = 20 if unicode_error_policy not in ("ignore", "replace", "strict"): unicode_error_policy = "ignore" warnings.warn( "Invalid unicode_error_policy for concurrent_log_handler: " "must be ignore, replace, or strict. Defaulting to ignore.", UserWarning, stacklevel=3, ) self.unicode_error_policy = unicode_error_policy if delay not in (None, True): warnings.warn( "concurrent_log_handler parameter `delay` is now ignored and implied as True, " "please remove from your config.", DeprecationWarning, stacklevel=3, ) # Construct the handler with the given arguments in "delayed" mode # because we will handle opening the file as needed. File name # handling is done by FileHandler since Python 2.5. super(ConcurrentRotatingFileHandler, self).__init__( filename, mode, encoding=encoding, delay=True ) self.terminator = terminator or "\n" if self.owner and HAS_CHOWN and pwd and grp: self._set_uid = pwd.getpwnam(self.owner[0]).pw_uid self._set_gid = grp.getgrnam(self.owner[1]).gr_gid self.lockFilename = self.getLockFilename(lock_file_directory) self.is_locked = False # This is primarily for the benefit of the unit tests. self.num_rollovers = 0 def getLockFilename(self, lock_file_directory: Optional[str]) -> str: """ Decide the lock filename. If the logfile is file.log, then we use `.__file.lock` and not `file.log.lock`. This only removes the extension if it's `*.log`. :param lock_file_directory: name of the directory for alternative living space of lock files :return: the path to the lock file. """ lock_path, lock_name = self.baseLockFilename(self.baseFilename) if lock_file_directory: self.__create_lock_directory__(lock_file_directory) return os.path.join(lock_file_directory, lock_name) return os.path.join(lock_path, lock_name) @staticmethod def baseLockFilename(baseFilename: str) -> Tuple[str, str]: lock_file = baseFilename[:-4] if baseFilename.endswith(".log") else baseFilename lock_file += ".lock" lock_path, lock_name = os.path.split(lock_file) # hide the file on Unix and generally from file completion return lock_path, ".__" + lock_name @staticmethod def __create_lock_directory__(lock_file_directory: str) -> None: if not os.path.exists(lock_file_directory): try: os.makedirs(lock_file_directory) except OSError as err: if err.errno != errno.EEXIST: # If directory already exists, then we're done. Everything else is fishy... raise def _open_lockfile(self) -> None: if self.stream_lock and not self.stream_lock.closed: self._console_log("Lockfile already open in this process") return lock_file = self.lockFilename # self._console_log( # f"concurrent-log-handler {hash(self)} opening {lock_file}", # stack=False, # ) with self._alter_umask(): self.stream_lock = self.atomic_open(lock_file) self._do_chown_and_chmod(lock_file) def atomic_open(self, file_path: str) -> TextIOWrapper: try: # Attempt to open the file in "r+" mode file = open(file_path, "r+", encoding=self.encoding, newline=self.newline) except FileNotFoundError: # If the file doesn't exist, create it atomically and open in "r+" mode try: fd = os.open(file_path, os.O_CREAT | os.O_EXCL | os.O_RDWR) file = open(fd, "r+", encoding=self.encoding, newline=self.newline) except FileExistsError: # If the file was created between the first check and our attempt to create it, open it in "r+" mode file = open( file_path, "r+", encoding=self.encoding, newline=self.newline ) return file def _open(self, mode: None = None) -> None: # type: ignore[override] # noqa: ARG002 # Normally we don't hold the stream open. Only do_open does that # which is called from do_write(). return None def do_open(self, mode: Optional[str] = None) -> TextIOWrapper: """ Open the current base file with the (original) mode and encoding. Return the resulting stream. Note: Copied from stdlib. Added option to override 'mode' """ if mode is None: mode = self.mode with self._alter_umask(): stream = open( self.baseFilename, mode=mode, encoding=self.encoding, newline=self.newline, ) if TYPE_CHECKING: assert isinstance(stream, TextIOWrapper) self._do_chown_and_chmod(self.baseFilename) return stream @contextmanager def _alter_umask(self) -> Generator: """Temporarily alter umask to custom setting, if applicable""" if self.umask is None: yield # nothing to do else: prev_umask = os.umask(self.umask) try: yield finally: os.umask(prev_umask) def _close(self) -> None: """Close file stream. Unlike close(), we don't tear anything down, we expect the log to be re-opened after rotation.""" if self.stream: try: if not self.stream.closed: # Flushing probably isn't technically necessary, but it feels right self.stream.flush() self.stream.close() finally: # noinspection PyTypeChecker self.stream = None def _console_log(self, msg: str, stack: bool = False) -> None: if not self._debug: return import threading tid = threading.current_thread().name pid = os.getpid() stack_str = "" if stack: stack_str = ":\n" + "".join(traceback.format_stack()) asctime = time.asctime() print(f"[{tid} {pid} {asctime}] {msg}{stack_str}") def emit(self, record: logging.LogRecord) -> None: """Emit a record. Override from parent class to handle file locking for the duration of rollover and write. This also does the formatting *before* locks are obtained, in case the format itself does logging calls from within. Rollover also occurs while the lock is held. """ try: msg = self.format(record) try: self._do_lock() try: if self.shouldRollover(record): self.doRollover() except Exception as e: self._console_log( f"Unable to do rollover: {e}\n{traceback.format_exc()}" ) # Continue on anyway self.do_write(msg) finally: self._do_unlock() except (KeyboardInterrupt, SystemExit): raise except Exception: self.handleError(record) def flush(self) -> None: """Does nothing; stream is flushed on each write.""" def do_write(self, msg: str) -> None: """Handling writing an individual record; we do a fresh open every time. This assumes emit() has already locked the file.""" self.stream = self.do_open() stream = self.stream msg = msg + self.terminator try: stream.write(msg) except UnicodeError: # Try to emit in a form acceptable to the output encoding # The unicode_error_policy determines whether this is lossy. try: encoding = getattr(stream, "encoding", self.encoding or "us-ascii") msg_bin = msg.encode(encoding, self.unicode_error_policy) msg = msg_bin.decode(encoding, self.unicode_error_policy) stream.write(msg) except UnicodeError: # self._console_log(str(e)) raise stream.flush() self._close() def _do_lock(self) -> None: if self.is_locked: return # already locked... recursive? self._open_lockfile() if self.stream_lock: for _i in range(self.maxLockAttempts): try: lock(self.stream_lock, LOCK_EX) self.is_locked = True # self._console_log("Acquired lock") break except Exception: # noqa: S112 continue else: raise RuntimeError( f"Cannot acquire lock after {self.maxLockAttempts} attempts" ) else: self._console_log("No self.stream_lock to lock", stack=True) def _do_unlock(self) -> None: if self.stream_lock: if self.is_locked: try: unlock(self.stream_lock) # self._console_log("Released lock") finally: self.is_locked = False self.stream_lock.close() self.stream_lock = None else: self._console_log("No self.stream_lock to unlock", stack=True) def close(self) -> None: """Close log stream and stream_lock.""" self._console_log("In close()", stack=True) try: self._close() finally: super(ConcurrentRotatingFileHandler, self).close() def doRollover(self) -> None: # noqa: C901 """ Do a rollover, as described in __init__(). """ self._close() if self.backupCount <= 0: # Don't keep any backups, just overwrite the existing backup file # Locking doesn't much matter here; since we are overwriting it anyway self.stream = self.do_open("w") self._close() return # Determine if we can rename the log file or not. Windows refuses to # rename an open file, Unix is inode based, so it doesn't care. # Attempt to rename logfile to tempname: # There is a slight race-condition here, but it seems unavoidable tmpname = None while True: tmpname = f"{self.baseFilename}.rotate.{randbits(64):08}" if not os.path.exists(tmpname): break try: # Do a rename test to determine if we can successfully rename the log file os.rename(self.baseFilename, tmpname) if self.use_gzip: self.do_gzip(tmpname) except OSError as e: self._console_log(f"rename failed. File in use? e={e}", stack=True) return gzip_ext = ".gz" if self.use_gzip else "" def do_rename(source_fn: str, dest_fn: str) -> None: self._console_log(f"Rename {source_fn} -> {dest_fn + gzip_ext}") if os.path.exists(dest_fn): os.remove(dest_fn) if os.path.exists(dest_fn + gzip_ext): os.remove(dest_fn + gzip_ext) source_gzip = source_fn + gzip_ext if os.path.exists(source_gzip): os.rename(source_gzip, dest_fn + gzip_ext) elif os.path.exists(source_fn): os.rename(source_fn, dest_fn) # Q: Is there some way to protect this code from a KeyboardInterrupt? # This isn't necessarily a data loss issue, but it certainly does # break the rotation process during stress testing. # There is currently no mechanism in place to handle the situation # where one of these log files cannot be renamed. (Example, user # opens "logfile.3" in notepad); we could test rename each file, but # nobody's complained about this being an issue; so the additional # code complexity isn't warranted. do_renames = [] for i in range(1, self.backupCount): sfn = self.rotation_filename(f"{self.baseFilename}.{i}") dfn = self.rotation_filename(f"{self.baseFilename}.{i + 1}") if os.path.exists(sfn + gzip_ext): do_renames.append((sfn, dfn)) else: # Break looking for more rollover files as soon as we can't find one # at the expected name. break for sfn, dfn in reversed(do_renames): do_rename(sfn, dfn) dfn = self.rotation_filename(self.baseFilename + ".1") do_rename(tmpname, dfn) if self.use_gzip: logFilename = self.baseFilename + ".1.gz" self._do_chown_and_chmod(logFilename) self.num_rollovers += 1 self._console_log("Rotation completed (on size)") def shouldRollover(self, record: logging.LogRecord) -> bool: # noqa: ARG002 """ Determine if rollover should occur. For those that are keeping track. This differs from the standard library's RotatingLogHandler class. Because there is no promise to keep the file size under maxBytes we ignore the length of the current record. TODO: should we reconsider this and make it more exact? """ return self._shouldRollover() def _shouldRollover(self) -> bool: if self.maxBytes > 0: # are we rolling over? self.stream = self.do_open() try: self.stream.seek(0, 2) # due to non-posix-compliant Windows feature if self.stream.tell() >= self.maxBytes: return True finally: self._close() return False def do_gzip(self, input_filename: str) -> None: if not gzip: self._console_log("#no gzip available", stack=False) return out_filename = input_filename + ".gz" with open(input_filename, "rb") as input_fh, gzip.open( out_filename, "wb" ) as gzip_fh: while True: data = input_fh.read(self.gzip_buffer) if not data: break gzip_fh.write(data) os.remove(input_filename) self._console_log(f"#gzipped: {out_filename}", stack=False) def _do_chown_and_chmod(self, filename: str) -> None: if HAS_CHOWN and self._set_uid is not None and self._set_gid is not None: os.chown(filename, self._set_uid, self._set_gid) if HAS_CHMOD and self.chmod is not None: os.chmod(filename, self.chmod) class ConcurrentTimedRotatingFileHandler(TimedRotatingFileHandler): """A time-based rotating log handler that supports concurrent access across multiple processes or hosts (using logs on a shared network drive). You can also include size-based rotation by setting maxBytes > 0. WARNING: if you only want time-based rollover and NOT also size-based, set maxBytes=0, which is already the default. Please note that when size-based rotation is done, it still uses the naming scheme of the time-based rotation. If multiple rotations had to be done within the timeframe of the time-based rollover name, then a number like ".1" will be appended to the end of the name. Note that `errors` is ignored unless using Python 3.9 or later. """ def __init__( # type: ignore[no-untyped-def] # noqa: PLR0913 self, filename: str, when: str = "h", interval: int = 1, backupCount: int = 0, encoding: Optional[str] = None, delay: bool = False, utc: bool = False, atTime: Optional[datetime.time] = None, errors: Optional[str] = None, maxBytes: int = 0, use_gzip: bool = False, owner: Optional[Tuple[str, str]] = None, chmod: Optional[int] = None, umask: Optional[int] = None, newline: Optional[str] = None, terminator: str = "\n", unicode_error_policy: str = "ignore", lock_file_directory: Optional[str] = None, **kwargs, ): if "mode" in kwargs: del kwargs["mode"] trfh_kwargs: Dict[str, Optional[str]] = {} if sys.version_info >= (3, 9): trfh_kwargs["errors"] = errors TimedRotatingFileHandler.__init__( self, filename, when=when, interval=interval, backupCount=backupCount, encoding=encoding, delay=delay, utc=utc, atTime=atTime, **trfh_kwargs, ) self.clh = ConcurrentRotatingFileHandler( filename, mode="a", backupCount=backupCount, encoding=encoding, delay=None, maxBytes=maxBytes, use_gzip=use_gzip, owner=owner, chmod=chmod, umask=umask, newline=newline, terminator=terminator, unicode_error_policy=unicode_error_policy, lock_file_directory=lock_file_directory, **kwargs, ) self.num_rollovers = 0 self.__internal_close() self.initialize_rollover_time() def __internal_close(self) -> None: # Don't need or want to hold the main logfile handle open unless we're actively writing to it. if self.stream: self.stream.close() # noinspection PyTypeChecker self.stream = None # type: ignore[assignment] def _console_log(self, msg: str, stack: bool = False) -> None: self.clh._console_log(msg, stack=stack) def emit(self, record: logging.LogRecord) -> None: """ Emit a record. Override from parent class to handle file locking for the duration of rollover and write. This also does the formatting *before* locks are obtained, in case the format itself does logging calls from within. Rollover also occurs while the lock is held. """ try: msg = self.format(record) try: self.clh._do_lock() try: if self.shouldRollover(record): self.doRollover() except Exception as e: self._console_log( "Unable to do rollover: {}\n{}".format( e, traceback.format_exc() ) ) # time.sleep(1000) self.clh.do_write(msg) finally: self.clh._do_unlock() except (KeyboardInterrupt, SystemExit): raise except Exception: self.handleError(record) def read_rollover_time(self) -> None: # Lock must be held before calling this method lock_file = self.clh.stream_lock if not lock_file or not self.clh.is_locked: # Lock is not being held? self._console_log( "No rollover time (lock) file to read from. Lock is not held?" ) return try: lock_file.seek(0) raw_time = lock_file.read() except OSError: self.rolloverAt = 0 self._console_log(f"Couldn't read rollover time from file {lock_file!r}") return try: self.rolloverAt = int(raw_time.strip()) # self._console_log( # f"Read rollover time: {self.rolloverAt} - raw: {raw_time!r}" # ) except ValueError: self.rolloverAt = 0 self._console_log(f"Couldn't read rollover time from file: {raw_time!r}") def write_rollover_time(self) -> None: """Write the next rollover time (current value of self.rolloverAt) to the lock file.""" lock_file = self.clh.stream_lock if not lock_file or not self.clh.is_locked: self._console_log( "No rollover time (lock) file to write to. Lock is not held?" ) return lock_file.seek(0) lock_file.write(str(self.rolloverAt)) lock_file.truncate() lock_file.flush() os.fsync(lock_file.fileno()) self._console_log(f"Wrote rollover time: {self.rolloverAt}") def initialize_rollover_time(self) -> None: """Run by the __init__ to read an existing rollover time from the lockfile, and if it can't do that, compute and write a new one.""" try: self.clh._do_lock() self.read_rollover_time() self._console_log(f"Initializing; reading rollover time: {self.rolloverAt}") if self.rolloverAt != 0: return current_time = int(time.time()) new_rollover_at = self.computeRollover(current_time) while new_rollover_at <= current_time: new_rollover_at += self.interval self.rolloverAt = new_rollover_at self.write_rollover_time() self._console_log(f"Set initial rollover time: {self.rolloverAt}") finally: self.clh._do_unlock() def shouldRollover(self, record: logging.LogRecord) -> bool: """Determine if the rollover should occur.""" # Read the latest rollover time from the file self.read_rollover_time() do_rollover = False if super(ConcurrentTimedRotatingFileHandler, self).shouldRollover(record): self._console_log("Rolling over because of time") do_rollover = True elif self.clh.shouldRollover(record): self.clh._console_log("Rolling over because of size") do_rollover = True if do_rollover: return True return False def doRollover(self) -> None: # noqa: C901, PLR0912 """ do a rollover; in this case, a date/time stamp is appended to the filename when the rollover happens. However, you want the file to be named for the start of the interval, not the current time. If there is a backup count, then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. This code was adapted from the TimedRotatingFileHandler class from Python 3.11. """ self.clh._close() self.__internal_close() # get the time that this sequence started at and make it a TimeTuple currentTime = int(time.time()) dstNow = time.localtime(currentTime)[-1] t = self.rolloverAt - self.interval if self.utc: timeTuple = time.gmtime(t) else: timeTuple = time.localtime(t) dstThen = timeTuple[-1] if dstNow != dstThen: addend = 3600 if dstNow else -3600 timeTuple = time.localtime(t + addend) dfn = self.rotation_filename( self.baseFilename + "." + time.strftime(self.suffix, timeTuple) ) gzip_ext = ".gz" if self.clh.use_gzip else "" counter = 1 if os.path.exists(dfn + gzip_ext): while os.path.exists(f"{dfn}.{counter}{gzip_ext}"): ending = f".{counter - 1}{gzip_ext}" if dfn.endswith(ending): dfn = dfn[: -len(ending)] counter += 1 dfn = f"{dfn}.{counter}" # if os.path.exists(dfn): # os.remove(dfn) self.rotate(self.baseFilename, dfn) if self.clh.use_gzip: self.clh.do_gzip(dfn) if self.backupCount > 0: # File will already have gzip extension here if applicable # Thanks to @moynihan for file in self.getFilesToDelete(): os.remove(file) newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval # If DST changes and midnight or weekly rollover, adjust for this. if (self.when == "MIDNIGHT" or self.when.startswith("W")) and not self.utc: dstAtRollover = time.localtime(newRolloverAt)[-1] if dstNow != dstAtRollover: if not dstNow: # noqa: SIM108 # DST kicks in before next rollover, so we need to deduct an hour addend = -3600 else: # DST bows out before next rollover, so we need to add an hour addend = 3600 newRolloverAt += addend self.num_rollovers += 1 self.rolloverAt = newRolloverAt self.write_rollover_time() self._console_log(f"Rotation completed (on time) {dfn}") def getFilesToDelete(self) -> List[str]: """ Determine the files to delete when rolling over. Copied from Python 3.11, and only applied when the current Python seems to be Python 3.8 or lower, which is when this seemed to change. The newer version supports custom suffixes like ours, such as when hitting a size limit before the time limit. """ # If Python is 3.9 or later, then use the superclass method. if sys.version_info >= (3, 9): return super().getFilesToDelete() dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) result = [] # See bpo-44753: Don't use the extension when computing the prefix. n, e = os.path.splitext(baseName) prefix = n + "." plen = len(prefix) for fileName in fileNames: if self.namer is None: # Our files will always start with baseName if not fileName.startswith(baseName): continue # Our files could be just about anything after custom naming, but # likely candidates are of the form # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log elif ( not fileName.startswith(baseName) and fileName.endswith(e) and len(fileName) > (plen + 1) and not fileName[plen + 1].isdigit() ): continue if fileName[:plen] == prefix: suffix = fileName[plen:] # See bpo-45628: The date/time suffix could be anywhere in the # filename parts = suffix.split(".") for part in parts: if self.extMatch.match(part): result.append(os.path.join(dirName, fileName)) break if len(result) < self.backupCount: result = [] else: result.sort() result = result[: len(result) - self.backupCount] return result # Publish these classes to the "logging.handlers" module, so they can be used # from a logging config file via logging.config.fileConfig(). import logging.handlers # noqa: E402 logging.handlers.ConcurrentRotatingFileHandler = ConcurrentRotatingFileHandler # type: ignore[attr-defined] logging.handlers.ConcurrentTimedRotatingFileHandler = ConcurrentTimedRotatingFileHandler # type: ignore[attr-defined] concurrent-log-handler-0.9.25/src/concurrent_log_handler/__version__.py000066400000000000000000000000271453540025200263240ustar00rootroot00000000000000__version__ = "0.9.25" concurrent-log-handler-0.9.25/src/concurrent_log_handler/py.typed000066400000000000000000000000001453540025200251570ustar00rootroot00000000000000concurrent-log-handler-0.9.25/src/concurrent_log_handler/queue.py000066400000000000000000000143701453540025200251750ustar00rootroot00000000000000#!/usr/bin/env python # Copyright 2017 Journyx, Inc., and other contributors """ Implement a threaded queue for loggers based on the standard logging.py QueueHandler / QueueListener classes. Requires Python 3. Calls to loggers will simply place the logging request on the queue and return immediately. A background thread will handle the actual logging. This helps avoid blocking for write locks on the logfiles. Please note that this replaces the handlers of all currently configured Python loggers with a proxy (QueueHandler). Call `setup_logging_queues` to do this. That also sets up an `atexit` callback which calls stop() on the QueueListener. Source for some of these functions: https://github.com/dgilland/logconfig/blob/master/logconfig/utils.py Additional code provided by Journyx, Inc. http://www.journyx.com """ import asyncio import atexit import logging import queue import sys from logging.handlers import QueueHandler, QueueListener from typing import Dict, List, Optional, Tuple, Union __author__ = "Preston Landers " GLOBAL_LOGGER_HANDLERS: Dict[ str, Tuple[List[logging.Handler], "AsyncQueueListener"] ] = {} # create a thread with a event loop in case of creating a coroutine in self.handle class AsyncQueueListener(QueueListener): def __init__( self, queue: queue.Queue, *handlers: logging.Handler, respect_handler_level: bool = False, ): super().__init__(queue, *handlers, respect_handler_level=respect_handler_level) self.loop: Optional[asyncio.AbstractEventLoop] = None def _monitor(self) -> None: # set event loop in thread self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) super()._monitor() # type: ignore[misc] def stop(self) -> None: # stop event loop if self.loop: self.loop.stop() self.loop.close() self.enqueue_sentinel() # set timeout in case thread occurs deadlock if self._thread: self._thread.join(1) self._thread = None def setup_logging_queues() -> None: if sys.version_info.major < 3: # noqa: PLR2004 raise RuntimeError("This feature requires Python 3.") queue_listeners: List[AsyncQueueListener] = [] previous_queue_listeners = [] # Q: What about loggers created after this is called? # A: if they don't attach their own handlers they should be fine for logger_name in get_all_logger_names(include_root=True): logger = logging.getLogger(logger_name) if logger.handlers: ori_handlers: List[logging.Handler] = [] # retrieve original handlers and listeners from GLOBAL_LOGGER_HANDLERS if exist if logger_name in GLOBAL_LOGGER_HANDLERS: # get original handlers ori_handlers.extend(GLOBAL_LOGGER_HANDLERS[logger_name][0]) # reset lock in original handlers (solve deadlock) for handler in ori_handlers: handler.createLock() # recover handlers in logger logger.handlers = [] logger.handlers.extend(ori_handlers) # get previous listeners previous_queue_listeners.append(GLOBAL_LOGGER_HANDLERS[logger_name][1]) else: ori_handlers.extend(logger.handlers) log_queue: queue.Queue[str] = queue.Queue(-1) # No limit on size queue_handler = QueueHandler(log_queue) queue_listener = AsyncQueueListener(log_queue, respect_handler_level=True) queuify_logger(logger, queue_handler, queue_listener) # print("Replaced logger %s with queue listener: %s" % ( # logger, queue_listener # )) queue_listeners.append(queue_listener) # save original handlers and current listeners GLOBAL_LOGGER_HANDLERS[logger_name] = (ori_handlers, queue_listener) # stop previous listeners at first stop_queue_listeners(*previous_queue_listeners) for listener in queue_listeners: listener.start() atexit.register(stop_queue_listeners, *queue_listeners) def stop_queue_listeners(*listeners: AsyncQueueListener) -> None: for listener in listeners: try: # noqa: SIM105 listener.stop() # if sys.stderr: # sys.stderr.write("Stopped queue listener.\n") # sys.stderr.flush() except: # noqa: E722, S110 pass # Don't need this in production... # if sys.stderr: # err = "Error stopping log queue listener:\n" \ # + traceback.format_exc() + "\n" # sys.stderr.write(err) # sys.stderr.flush() def get_all_logger_names(include_root: bool = False) -> List[str]: """Return ``list`` of names of all loggers than have been accessed. Warning: this is sensitive to internal structures in the standard logging module. """ rv = list(logging.Logger.manager.loggerDict.keys()) if include_root: rv.insert(0, "") return rv def queuify_logger( logger: Union[logging.Logger, str], queue_handler: QueueHandler, queue_listener: QueueListener, ) -> None: """Replace logger's handlers with a queue handler while adding existing handlers to a queue listener. This is useful when you want to use a default logging config but then optionally add a logger's handlers to a queue during runtime. Args: logger (mixed): Logger instance or string name of logger to queue-ify handlers. queue_handler (QueueHandler): Instance of a ``QueueHandler``. queue_listener (QueueListener): Instance of a ``QueueListener``. """ if isinstance(logger, str): logger = logging.getLogger(logger) # Get handlers that aren't being listened for. handlers = [ handler for handler in logger.handlers if handler not in queue_listener.handlers ] if handlers: # The default QueueListener stores handlers as a tuple. queue_listener.handlers = tuple(list(queue_listener.handlers) + handlers) # Remove logger's handlers and replace with single queue handler. del logger.handlers[:] logger.addHandler(queue_handler) concurrent-log-handler-0.9.25/src/example.py000066400000000000000000000076031453540025200207650ustar00rootroot00000000000000import logging # noqa: INP001 import logging.config import time """ This is an example which shows how you can use ConcurrentLogHandler. If you have Two basic options are demonstrated: * ASYNC_LOGGING = False - using as a regular synchronous log handler. That means when your program logs a statement, it's processed and written to the log file as part of the original thread. * ASYNC_LOGGING = True - performs logging statements in a background thread asynchronously. This uses Python's `asyncio` package. """ def my_program(): ASYNC_LOGGING = False # Somewhere in your program, usually at startup or config time, you can # call your logging setup function. If you're in an multiprocess environment, # each separate process that wants to write to the same file should call the same # or very similar logging setup code. my_logging_setup(use_async=ASYNC_LOGGING) # Now for the meat of your program... logger = logging.getLogger("MyExample") logger.setLevel(logging.DEBUG) # optional to set this level here for idx in range(20): time.sleep(0.05) print("Loop %d; logging a message." % idx) logger.debug("%d > A debug message.", idx) if idx % 2 == 0: logger.info("%d > An info message.", idx) print("Done with example; exiting.") # Optional; you can manually stop the logging queue listeners at any point # or let it happen at process exit. if ASYNC_LOGGING: from concurrent_log_handler.queue import stop_queue_listeners stop_queue_listeners() def my_logging_setup(log_name="example.log", use_async=False): """ An example of setting up logging in Python using a JSON dictionary to configure it. You can also use an outside .conf text file; see ConcurrentLogHandler/README.md If you want to use async logging, call this after your main logging setup as shown below: concurrent_log_handler.queue.setup_logging_queues() """ # Import this to install logging.handlers.ConcurrentRotatingFileHandler import concurrent_log_handler # noqa: F401 logging_config = { "version": 1, "disable_existing_loggers": False, "formatters": { "default": {"format": "%(asctime)s %(levelname)s %(name)s %(message)s"}, "example2": { "format": "[%(asctime)s][%(levelname)s][%(filename)s:%(lineno)s]" "[%(process)d][%(message)s]", }, }, # Set up our concurrent logger handler. Need one of these per unique file. "handlers": { "my_concurrent_log": { "level": "DEBUG", "class": "concurrent_log_handler.ConcurrentRotatingFileHandler", # Example of a custom format for this log. "formatter": "example2", # 'formatter': 'default', "filename": log_name, # Optional: set an owner and group for the log file # 'owner': ['greenfrog', 'admin'], # Sets permissions to owner and group read+write "chmod": 0o0660, # Note: this is abnormally small to make it easier to demonstrate rollover. # A more reasonable value might be 10 MiB or 10485760 "maxBytes": 240, # Number of rollover files to keep "backupCount": 10, # 'use_gzip': True, } }, # Tell root logger to use our concurrent handler "root": { "handlers": ["my_concurrent_log"], "level": "DEBUG", }, } logging.config.dictConfig(logging_config) if use_async: # To enable background logging queue, call this near the end of your logging setup. from concurrent_log_handler.queue import setup_logging_queues setup_logging_queues() return if __name__ == "__main__": my_program() concurrent-log-handler-0.9.25/src/namer_example.py000066400000000000000000000023701453540025200221430ustar00rootroot00000000000000import logging # noqa: INP001 import logging.config import time from datetime import date """ This is an example which shows how you can use custom namer function with ConcurrentRotatingFileHandler """ def log_file_namer(logger_name: str) -> str: # path/name.log.N logger_name, backup_number = logger_name.rsplit(".", maxsplit=1) # path/name.log logger_name = logger_name.replace(".log", "") curr_date = date.today().strftime("%Y_%m_%d") # noqa: DTZ011 return f"{logger_name}_{curr_date}_({backup_number}).log" def my_program(): import concurrent_log_handler # Now for the meat of your program... logger = logging.getLogger("MyExample") logger.setLevel(logging.DEBUG) # optional to set this level here handler = concurrent_log_handler.ConcurrentRotatingFileHandler( "namer_test.log", "a", maxBytes=512, backupCount=2 ) handler.namer = log_file_namer logger.addHandler(handler) for idx in range(50): time.sleep(0.05) print("Loop %d; logging a message." % idx) logger.debug("%d > A debug message.", idx) if idx % 2 == 0: logger.info("%d > An info message.", idx) print("Done with example; exiting.") if __name__ == "__main__": my_program() concurrent-log-handler-0.9.25/tests/000077500000000000000000000000001453540025200173255ustar00rootroot00000000000000concurrent-log-handler-0.9.25/tests/other/000077500000000000000000000000001453540025200204465ustar00rootroot00000000000000concurrent-log-handler-0.9.25/tests/other/stresstest.py000066400000000000000000000374111453540025200232510ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8; mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vim: fileencoding=utf-8 tabstop=4 expandtab shiftwidth=4 # ruff: noqa: RET504, S311, S603 """ stresstest.py: A stress-tester for ConcurrentRotatingFileHandler This utility spawns a bunch of processes that all try to concurrently write to the same file. This is pretty much the worst-case scenario for my log handler. Once all of the processes have completed writing to the log file, the output is compared to see if any log messages have been lost. In the future, I may also add in support for testing with each process having multiple threads. """ import gzip import io import os import string import sys from optparse import OptionParser from random import choice, randint from subprocess import Popen from time import sleep # local lib; for testing from concurrent_log_handler import ConcurrentRotatingFileHandler, randbits __author__ = "Lowell Alleman" PY2 = False # No longer supporting Python 2.7 # ruff: noqa: F821, E501 # The total amount of rotated files to keep through the test run. Any data accumulated # before this is reached gets lost. It needs to be high enough so that all loop iterations # across all threads get all their data captured without losing anything otherwise the # diff at the end will fail. But if rollover file count get very high then performance # becomes slow due to the mass renaming and some threads may throw a lock acquire failure! ROTATE_COUNT = 10000 # Not all encodings will work here unless you remove some of the Unicode # chars in the test string. # ENCODING = 'cp1252' # There are some issues with the test program in utf-16 but I think the logging itself works...? # ENCODING = 'utf-16' ENCODING = "utf-8" class RotateLogStressTester: def __init__(self, sharedfile, uniquefile, name="LogStressTester"): self.sharedfile = sharedfile self.uniquefile = uniquefile self.name = name self.writeLoops = 100000 self.rotateSize = 128 * 1024 self.rotateCount = ROTATE_COUNT self.random_sleep_mode = False self.debug = True self.log = None self.use_gzip = True self.extended_unicode = True self.use_queue = False self.lock_dir = None if PY2 and ENCODING != "utf-8": # hopefully temporary... the problem is with stdout in the tester I think self.extended_unicode = False def getLogHandler(self, fn): """Override this method if you want to test a different logging handler class.""" rv = ConcurrentRotatingFileHandler( fn, "a", self.rotateSize, self.rotateCount, encoding=ENCODING, debug=self.debug, use_gzip=self.use_gzip, lock_file_directory=self.lock_dir, ) # To force LF only linefeeds on Windows: newline='', terminator='\n' # To force CRLF on Unix: newline='', terminator='\r\n' # To run the test with the standard library's RotatingFileHandler: # from logging.handlers import RotatingFileHandler # return RotatingFileHandler(fn, 'a', self.rotateSize, self.rotateCount) return rv def start(self): from logging import DEBUG, FileHandler, Formatter, getLogger self.log = getLogger(self.name) self.log.setLevel(DEBUG) formatter = Formatter( "%(asctime)s [%(process)d:%(threadName)s] %(levelname)-8s %(name)s: %(message)s" ) # Unique log handler (single file) handler = FileHandler(self.uniquefile, "w", encoding=ENCODING) handler.setLevel(DEBUG) handler.setFormatter(formatter) self.log.addHandler(handler) # If you suspect that the diff stuff isn't working, un comment the next # line. You should see this show up once per-process. # self.log.info("Here is a line that should only be in the first output.") # Setup output used for testing handler = self.getLogHandler(self.sharedfile) handler.setLevel(DEBUG) handler.setFormatter(formatter) self.log.addHandler(handler) if self.use_queue: from concurrent_log_handler.queue import setup_logging_queues setup_logging_queues() # If this ever becomes a real "Thread", then remove this line: self.run() def run(self): print("Hello, self.writeLoops: %s" % (self.writeLoops,)) c = 0 import random # Use a bunch of random quotes, numbers, and severity levels to mix it up a bit! msgs = [ "I found %d puppies", "There are %d cats in your hatz", "my favorite number is %d", "I am %d years old.", "1 + 1 = %d", "%d/0 = DivideByZero", "blah! %d thingies!", "8 15 16 23 48 %d", "the worlds largest prime number: %d", "%d happy meals!", ] if self.extended_unicode: msgs.extend( [ "\U0001d122 \U00024b00 Euro: \u20ac%d", "my favorite number is %d ①②③④⑤⑥⑦⑧!", ] ) logfuncts = [self.log.debug, self.log.info, self.log.warning, self.log.error] num_rand_bits = 64 rand_string_len = 1024 * 5 self.log.info( "c=%s Starting to write random log message. Loop=%d", c, self.writeLoops ) while c <= self.writeLoops: c += 1 self.log.debug( "c=%s Triggering logging within format of another log: %r", c, InnerLoggerExample( self.log, randbits(num_rand_bits), rand_string(rand_string_len), c ), ) msg = random.choice(msgs) logfunc = random.choice(logfuncts) logfunc("c=%s " + msg, c, randbits(num_rand_bits)) if self.random_sleep_mode and c % 1000 == 0: # Sleep from 0-5 seconds s = randint(0, 5) print("PID %d sleeping for %d seconds" % (os.getpid(), s)) sleep(s) # break self.log.info("c=%s Done writing random log messages.", c) def iter_lognames(logfile, count): """Generator for log file names based on a rotation scheme""" for i in range(count - 1, 0, -1): yield "%s.%d" % (logfile, i) yield logfile def iter_logs(iterable, missing_ok=False): """Generator to extract log entries from shared log file.""" for fn in iterable: opener = open log_path = fn log_path_gz = log_path + ".gz" if os.path.exists(log_path_gz): log_path = log_path_gz opener = gzip.open if os.path.exists(log_path): with opener(log_path, "rb") as fh: for line in fh: yield line elif not missing_ok: raise ValueError("Missing log file %s" % log_path) def combine_logs(combinedlog, iterable, mode="wb"): """write all lines (iterable) into a single log file.""" fp = io.open(combinedlog, mode) if ENCODING == "utf-16": import codecs fp.write(codecs.BOM_UTF16) for chunk in iterable: fp.write(chunk) fp.close() class InnerLoggerExample(object): def __init__(self, log, a, b, c): self.log = log self.a = a self.b = b self.c = c def __str__(self): # This should trigger a logging event within the format() handling of another event self.log.debug("c=%s Inner logging example: a=%r, b=%r", self.c, self.a, self.b) return "" % (self.a,) def __repr__(self): return str(self) allchar = string.ascii_letters + string.punctuation + string.digits def rand_string(str_len): chars = [] for i in range(str_len): c = choice(allchar) if i % 10 == 0: c = " " chars.append(c) return "".join(chars) parser = OptionParser( usage="usage: %prog", # version=__version__, description="Stress test the concurrent_log_handler module.", ) parser.add_option( "--log-calls", metavar="NUM", action="store", type="int", default=5000, help="Number of logging entries to write to each log file. Default is %d", ) parser.add_option("--random-sleep-mode", action="store_true", default=False) parser.add_option("--debug", action="store_true", default=False) parser.add_option("--use-queue", action="store_true", default=False) parser.add_option( "--lock-dir", metavar="DIR", action="store", default=None, help="Store lock files in an alternate directory.", ) def main_client(args): (options, args) = parser.parse_args(args) if len(args) != 2: # noqa: PLR2004 raise ValueError("Require 2 arguments. We have %d args" % len(args)) (shared, client) = args if os.path.isfile(client): sys.stderr.write("Already a client using output file %s\n" % client) sys.exit(1) tester = RotateLogStressTester(shared, client) tester.random_sleep_mode = options.random_sleep_mode tester.debug = options.debug tester.writeLoops = options.log_calls tester.lock_dir = options.lock_dir tester.start() print("We are done pid=%d" % os.getpid()) class TestManager: class ChildProc(object): """Very simple child container class.""" __slots__ = ["popen", "sharedfile", "clientfile"] def __init__(self, **kwargs): self.update(**kwargs) def update(self, **kwargs): for key, val in kwargs.items(): setattr(self, key, val) def __init__(self, output_path): self.output_path = output_path self.tests = [] self.client_stdout = io.open( os.path.join(output_path, "client_stdout.txt"), "a", encoding=ENCODING ) self.client_stderr = io.open( os.path.join(output_path, "client_stderr.txt"), "a", encoding=ENCODING ) def launchPopen(self, *args, **kwargs): if "stdout" not in kwargs: kwargs["stdout"] = self.client_stdout if "stderr" not in kwargs: kwargs["stderr"] = self.client_stdout proc = Popen(*args, **kwargs) cp = self.ChildProc(popen=proc) self.tests.append(cp) return cp def wait(self, check_interval=3): """Wait for all child test processes to complete.""" print("Waiting while children are out running and playing!") while True: sleep(check_interval) waiting = [] for cp in self.tests: if cp.popen.poll() is None: waiting.append(cp.popen.pid) if not waiting: break print("Waiting on %r " % waiting) print("All children have stopped.") def checkExitCodes(self): for cp in self.tests: stdout_str, stderr_str = cp.popen.communicate() exit_code = cp.popen.poll() if exit_code != 0: print(stderr_str) print(stderr_str) print("cp exit code: %s: %s" % (cp, exit_code)) return False return True def unified_diff(a, b, out=sys.stdout, out2=None): import difflib dfile = None if out2: dfile = io.open(out2, "w", encoding=ENCODING) ai = io.open(a, "r", encoding=ENCODING).readlines() bi = io.open(b, "r", encoding=ENCODING).readlines() for line in difflib.unified_diff(ai, bi, a, b): # if PY2: # line = line.encode(ENCODING) if PY2: if not isinstance(line, unicode): line = unicode(line, ENCODING) # noqa: PLW2901 line_out = line.encode(out.encoding, "ignore").decode(out.encoding) out.write(line_out) else: out.write(line) if dfile: dfile.write(line) def main_runner(args): # noqa: PLR0915 parser.add_option( "--processes", metavar="NUM", action="store", type="int", default=3, help="Number of processes to spawn. Default: %default", ) parser.add_option( "--delay", metavar="secs", action="store", type="float", default=2.5, help="Wait SECS before spawning next processes. Default: %d", ) parser.add_option( "-p", "--path", metavar="DIR", action="store", default="test_output", help="Path to a temporary directory. Default: '%d'", ) parser.add_option( "-k", "--keep", action="store_true", default=False, help="Don't automatically delete the --path directory at test start.", ) this_script = args[0] (options, args) = parser.parse_args(args) options.path = os.path.abspath(options.path) if not options.keep and os.path.exists(options.path): import shutil # Can we delete everything under the test output path but not the folder itself? shutil.rmtree(options.path) if not os.path.isdir(options.path): os.makedirs(options.path) else: existing_files = len(os.listdir(options.path)) if existing_files: sys.stderr.write( "Output directory is not empty and --keep was not given: %s files in %s.\n" % ( existing_files, options.path, ) ) sys.exit(1) manager = TestManager(options.path) shared = os.path.join(options.path, "shared.log") for client_id in range(options.processes): client = os.path.join(options.path, "client.log_client%s.log" % client_id) cmdline = [ sys.executable, this_script, "client", shared, client, "--log-calls=%d" % options.log_calls, ] if options.random_sleep_mode: cmdline.append("--random-sleep-mode") if options.debug: cmdline.append("--debug") if options.use_queue: cmdline.append("--use-queue") if options.lock_dir: cmdline.append("--lock-dir=%s" % (options.lock_dir,)) child = manager.launchPopen(cmdline) child.update(sharedfile=shared, clientfile=client) sleep(options.delay) # Wait for all of the subprocesses to exit manager.wait() # Check children exit codes if not manager.checkExitCodes(): sys.stderr.write( "One or more of the child process has failed.\n Aborting test.\n" ) sys.exit(2) client_combo = os.path.join(options.path, "client.log.combo") shared_combo = os.path.join(options.path, "shared.log.combo") # Combine all of the log files... client_files = [child.clientfile for child in manager.tests] sort_em = sorted print("Writing out combined client logs...") combine_logs(client_combo, sort_em(iter_logs(client_files))) print("done.") print("Writing out combined shared logs...") shared_log_files = iter_lognames(shared, ROTATE_COUNT) log_lines = iter_logs(shared_log_files, missing_ok=True) combine_logs(shared_combo, sort_em(log_lines)) print("done.") print( "Running internal diff: " "(If the next line is 'end of diff', then the stress test passed!)" ) diff_file = os.path.join(options.path, "diff.patch") unified_diff(client_combo, shared_combo, sys.stdout, diff_file) print(" --- end of diff ----") def decode(thing, encoding=ENCODING): if isinstance(thing, bytes): return thing.decode(encoding=encoding) return thing if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1].lower() == "client": main_client(sys.argv[2:]) else: main_runner(sys.argv) concurrent-log-handler-0.9.25/tests/other/test.py000066400000000000000000000050561453540025200220050ustar00rootroot00000000000000# See stresstest.py for a more intensive test. # noqa: INP001 # This is more like a very quick test of basic functionality. import logging.config from pathlib import Path def get_logging_config(): logconfig_dict = { "version": 1, "formatters": { "standard": { "format": "%(asctime)s - %(levelname)s - %(name)s - %(message)s" } }, "root": { "handlers": ["default"], "level": "DEBUG", }, "handlers": { "default": { "level": "DEBUG", "formatter": "standard", "class": "logging.StreamHandler", }, "gunicorn_access": { "level": "DEBUG", "encoding": "utf_8", "formatter": "standard", "class": "concurrent_log_handler.ConcurrentRotatingFileHandler", "filename": "logging/access.log", "maxBytes": 1024, "backupCount": 2, "lock_file_directory": str( Path(Path(__file__).parent, "lock/test_access") ), }, "gunicorn_error": { "level": "DEBUG", "encoding": "utf_8", "formatter": "standard", "class": "concurrent_log_handler.ConcurrentRotatingFileHandler", "filename": "logging/error.log", "maxBytes": 1024, "backupCount": 3, "lock_file_directory": str( Path(Path(__file__).parent, "lock/test_error") ), }, }, "loggers": { "gunicorn.access": { "handlers": ["gunicorn_access"], "level": "DEBUG", "propagate": False, }, "gunicorn.error": { "handlers": ["gunicorn_error"], "level": "DEBUG", "propagate": False, }, }, } return logconfig_dict # noqa: RET504 if __name__ == "__main__": # Create logging directory Path.mkdir(Path(Path(__file__).parent, "logging"), exist_ok=True) # Load logging configuration logging.config.dictConfig(get_logging_config()) # Root logger log1 = logging.getLogger(__name__) log1.debug("Here we go...") # Access logger log2 = logging.getLogger("gunicorn.access") log2.debug("There are 4 lights!!!") # Error logger log3 = logging.getLogger("gunicorn.error") log3.error("The cake is a lie!!!") concurrent-log-handler-0.9.25/tests/stresstest.py000066400000000000000000000407131453540025200221270ustar00rootroot00000000000000#!/usr/bin/env python # ruff: noqa: S311, G004 """ This is a simple stress test for concurrent_log_handler. It creates a number of processes and each process logs a number of messages to a log file. We then validate that the sum of all log files (including rotations) contains the expected number of messages and that each message is unique. It can be run directly from the CLI for a single test run with certain parameters. There is also a pytest based unit test that exercises several different sets of options. This test requires Python 3.7 due to the use of dataclasses. """ import argparse import glob import gzip import logging import multiprocessing import os import random import re import string import time from dataclasses import dataclass, field from typing import Dict, Optional from concurrent_log_handler import ( ConcurrentRotatingFileHandler, ConcurrentTimedRotatingFileHandler, ) from concurrent_log_handler.__version__ import __version__ @dataclass(frozen=True) class TestOptions: """Options to configure the stress test.""" __test__ = False # not a test case itself. # kwargs to pass to ConcurrentRotatingFileHandler log_opts: dict = field(default_factory=lambda: TestOptions.default_log_opts()) log_file: str = field(default="stress_test.log") log_dir: str = field(default="output_tests") num_processes: int = field(default=10) log_calls: int = field(default=1_000) use_asyncio: bool = field(default=False) induce_failure: bool = field(default=False) sleep_min: float = field(default=0.0001) sleep_max: float = field(default=0.01) use_timed: bool = field(default=False) "Use time-based rotation class instead of size-based." min_rollovers: int = field(default=70) """Minimum number of rollovers to expect. Useful for testing rollover behavior. Default is 70 which is appropriate for the default test settings. The actual number of rollovers will vary significantly based on the rest of the settings.""" @classmethod def default_log_opts(cls, override_values: Optional[Dict] = None) -> dict: rv = { "maxBytes": 1024 * 10, "backupCount": 2000, "encoding": "utf-8", "debug": False, "use_gzip": False, } if override_values: rv.update(override_values) return rv @classmethod def default_timed_log_opts(cls, override_values: Optional[Dict] = None) -> dict: rv = { "maxBytes": 0, "when": "S", "interval": 3, "backupCount": 2000, "encoding": "utf-8", "debug": False, "use_gzip": False, } if override_values: rv.update(override_values) return rv class SharedCounter: def __init__(self, initial_value=0): self.value = multiprocessing.Value("i", initial_value) self.lock = multiprocessing.Lock() def increment(self, n=1): with self.lock, self.value.get_lock(): self.value.value += n def get_value(self): with self.lock, self.value.get_lock(): return self.value.value class ConcurrentLogHandlerBuggyMixin: def emit(self, record): # Introduce a random chance (e.g., 5%) to skip or duplicate a log message random_choice = random.randint(1, 100) # 5% chance to skip a log message if 1 <= random_choice <= 5: # noqa: PLR2004 return # 5% chance to duplicate a log message if 6 <= random_choice <= 10: # noqa: PLR2004 super().emit(record) super().emit(record) else: super().emit(record) class ConcurrentLogHandlerBuggy( ConcurrentLogHandlerBuggyMixin, ConcurrentRotatingFileHandler ): pass class ConcurrentTimedLogHandlerBuggy( ConcurrentLogHandlerBuggyMixin, ConcurrentTimedRotatingFileHandler ): pass def worker_process(test_opts: TestOptions, process_id: int, rollover_counter): logger = logging.getLogger(f"Process-{process_id}") logger.setLevel(logging.DEBUG) log_opts = test_opts.log_opts if test_opts.use_timed: file_handler_class = ( ConcurrentTimedLogHandlerBuggy if test_opts.induce_failure else ConcurrentTimedRotatingFileHandler ) else: file_handler_class = ( ConcurrentLogHandlerBuggy if test_opts.induce_failure else ConcurrentRotatingFileHandler ) log_path = os.path.join(test_opts.log_dir, test_opts.log_file) file_handler = file_handler_class(log_path, mode="a", **log_opts) formatter = logging.Formatter("%(asctime)s - %(name)s - %(message)s") file_handler.setFormatter(formatter) logger.addHandler(file_handler) char_choices = string.ascii_letters if log_opts["encoding"] == "utf-8": # Note: this can't include a dash (-) because it's used as a delimiter in the log message char_choices = ( string.ascii_letters + " \U0001d122\U00024b00\u20a0ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ" ) for i in range(test_opts.log_calls): random_str = "".join(random.choices(char_choices, k=20)) logger.debug(f"{process_id}-{i}-{random_str}") time.sleep(random.uniform(test_opts.sleep_min, test_opts.sleep_max)) rollover_counter.increment(file_handler.num_rollovers) def validate_log_file(test_opts: TestOptions, run_time: float, expect_all=True) -> bool: process_tracker = {i: {} for i in range(test_opts.num_processes)} # Sort log files, starting with the most recent backup log_path = os.path.join(test_opts.log_dir, test_opts.log_file) all_log_files = sorted(glob.glob(f"{log_path}*"), reverse=True) encoding = test_opts.log_opts["encoding"] or "utf-8" chars_read = 0 for current_log_file in all_log_files: opener = open if test_opts.log_opts["use_gzip"]: if current_log_file.endswith(".gz"): opener = gzip.open elif current_log_file != log_path: raise AssertionError("use_gzip was set, but log file is not gzipped?") with opener(current_log_file, "rb") as file: for line_no, line in enumerate(file): line = line.decode(encoding) # noqa: PLW2901 chars_read += len(line) parts = line.strip().split(" - ") message = parts[-1] process_id, message_id, sub_message = message.split("-") process_id = int(process_id) message_id = int(message_id) msg_state = { "file": current_log_file, "line_no": line_no, "line": line, } if message_id in process_tracker[process_id]: print( f"""Error: Duplicate message from Process-{process_id}: {message_id} {process_tracker[process_id][message_id]} and {msg_state}""" ) return False process_tracker[process_id][message_id] = msg_state log_calls = test_opts.log_calls if expect_all: for process_id, message_ids in process_tracker.items(): if len(message_ids) != log_calls: print( f"Error: Missing messages from Process-{process_id}: " f"len(message_ids) {len(message_ids)} != log_calls {log_calls}" ) return False print( f"{run_time:.2f} seconds to read {chars_read} chars " f"from {len(all_log_files)} files ({chars_read / run_time:.2f} chars/sec)" ) return True def run_stress_test(test_opts: TestOptions) -> int: delete_log_files(test_opts) # Create the log directory if it doesn't exist if not os.path.exists(test_opts.log_dir): os.makedirs(test_opts.log_dir) processes = [] rollover_counter = SharedCounter() start_time = time.time() for i in range(test_opts.num_processes): p = multiprocessing.Process( target=worker_process, args=(test_opts, i, rollover_counter) ) p.start() processes.append(p) for p in processes: p.join() end_time = time.time() print( f"All processes finished. (Rollovers: " f"{rollover_counter.get_value()} - min was {test_opts.min_rollovers})" ) log_path = os.path.join(test_opts.log_dir, test_opts.log_file) all_log_files = glob.glob(f"{log_path}*") gzip_ext = "[.]gz" if test_opts.log_opts["use_gzip"] else "" # Issue #68 - check for incorrect naming of files when using TimedRotatingFileHandler # and we had to rollover more often than the normal `when` interval due to size limits # If this happens more than once per interval, the names would be like "logfile.2020-01-01.1.2.3.4.5.gz" for log_file in all_log_files: if re.search(r"\.\d+\.\d+" + gzip_ext, str(log_file)): print(f"Error: Incorrect naming of log file: {log_file}") return 1 # If backupCount is less than 10, assume we want specifically to # test backupCount. This means, in most cases, we will have deleted # some logs files and we should not expect to find all log files. # Therefore we can't do the "expect_all" check where it looks for any # missing lines. It would be nice to be able to combine these somehow. MAGIC_BACKUP_COUNT = 10 expect_all_lines = True backup_count = test_opts.log_opts.get("backupCount", 0) if backup_count > 0 and backup_count < MAGIC_BACKUP_COUNT: expect_all_lines = False # Check that backupCount was not exceeded. # The +1 is because we're counting 'backups' plus the main log file. if len(all_log_files) != backup_count + 1: print( f"Error: {len(all_log_files)} log files were created but " f"we expected {backup_count + 1}. Could indicate a failure " f"to rotate properly or to delete excessive backups (`backupCount`)." ) return 1 # Each test should trigger some minimum number of rollovers. if ( test_opts.min_rollovers and rollover_counter.get_value() < test_opts.min_rollovers ): print( f"Error: {rollover_counter.get_value()} rollovers occurred but " f"we expected at least {test_opts.min_rollovers}." ) return 1 # Check for any omissions or duplications. if validate_log_file(test_opts, end_time - start_time, expect_all=expect_all_lines): print("Stress test passed.") return 0 print("Stress test failed.") return 1 def delete_log_files(test_opts: TestOptions): log_path = os.path.join(test_opts.log_dir, test_opts.log_file) log_files_to_remove = glob.glob(f"{log_path}*") _, lock_name = ConcurrentRotatingFileHandler.baseLockFilename(test_opts.log_file) log_files_to_remove.append(os.path.join(test_opts.log_dir, lock_name)) removed_files = [] for file in log_files_to_remove: try: if os.path.exists(file): os.remove(file) removed_files.append(file) except OSError as e: print(f"Error deleting log file {file}: {e}") if removed_files: print(f"Deleted {len(removed_files)} existing log files.") def main(): """Command line driver for ConcurrentRotatingFileHandler stress test.""" parser = argparse.ArgumentParser( description=f"Concurrent Log Handler {__version__} stress test." ) default_log_opts = TestOptions.default_log_opts() parser.add_argument( "--max-bytes", type=int, default=default_log_opts["maxBytes"], help=f"Maximum log file size in bytes before rotation " f"(default: {default_log_opts['maxBytes']}). (Ignored if use-timed)", ) use_timed_default = TestOptions.__annotations__["use_timed"].default parser.add_argument( "--use-timed", type=int, default=use_timed_default, help=f"Test the timed-based rotation class. " f"(default: {use_timed_default}).", ) log_calls_default = TestOptions.__annotations__["log_calls"].default parser.add_argument( "--log-calls", type=int, default=log_calls_default, help=f"Number of log messages per process (default: {log_calls_default})", ) num_processes_default = TestOptions.__annotations__["num_processes"].default parser.add_argument( "--num-processes", type=int, default=num_processes_default, help=f"Number of worker processes (default: {num_processes_default})", ) sleep_min_default = TestOptions.__annotations__["sleep_min"].default parser.add_argument( "--sleep-min", type=float, default=sleep_min_default, help=f"Minimum random sleep time in seconds (default: {sleep_min_default})", ) sleep_max_default = TestOptions.__annotations__["sleep_max"].default parser.add_argument( "--sleep-max", type=float, default=sleep_max_default, help=f"Maximum random sleep time in seconds (default: {sleep_max_default})", ) parser.add_argument( "--asyncio", action="store_true", help="Use asyncio queue feature of Concurrent Log Handler. (TODO: Not implemented yet)", ) # If this number is too low compared to the number of log calls, the log file will be deleted # before the test is complete and fail. parser.add_argument( "--max-rotations", type=int, default=default_log_opts["maxRotations"], help=f"Maximum number of rotations before deleting oldest log file " f"(default: {default_log_opts['maxRotations']})", ) parser.add_argument( "--encoding", type=str, default=default_log_opts["encoding"], help=f"Encoding for log file (default: {default_log_opts['encoding']})", ) parser.add_argument( "--debug", type=bool, default=default_log_opts["debug"], help=f"Enable debug flag on CLH (default: {default_log_opts['debug']})", ) parser.add_argument( "--gzip", type=bool, default=default_log_opts["gzip"], help=f"Enable gzip compression on CLH (default: {default_log_opts['gzip']})", ) filename_default = TestOptions.__annotations__["log_file"].default parser.add_argument( "--filename", type=str, default=filename_default, help=f"Filename for log file (default: {filename_default})", ) log_dir_default = TestOptions.__annotations__["log_dir"].default parser.add_argument( "--log-dir", type=str, default=log_dir_default, help=f"Directory for log file output (default: {log_dir_default})", ) induce_failure_default = TestOptions.__annotations__["induce_failure"].default parser.add_argument( "--fail", type=bool, default=induce_failure_default, help=f"Induce random failures in logging to cause a test failure. " f"(default: {induce_failure_default})", ) parser.add_argument( "--when", type=str, default="s", help="Time interval for timed rotation (default: 's')", ) parser.add_argument( "--interval", type=int, default=10, help="Interval for timed rotation (default: 10)", ) args = parser.parse_args() log_opts = { "maxBytes": args.max_bytes, "backupCount": args.max_rotations, "encoding": args.encoding, "debug": args.debug, "use_gzip": args.gzip, } if args.use_timed: log_opts = { "backupCount": args.max_rotations, "encoding": args.encoding, "debug": args.debug, "use_gzip": args.gzip, "when": args.when, "interval": args.interval, } test_opts = TestOptions( log_file=args.filename, num_processes=args.num_processes, log_calls=args.log_calls, sleep_min=args.sleep_min, sleep_max=args.sleep_max, use_asyncio=args.asyncio, induce_failure=args.fail, log_opts=log_opts, ) # Implement the stress test using asyncio queue feature # TODO: future time-based rotation options if test_opts.use_asyncio: return 2 # Not implemented yet return run_stress_test(test_opts) if __name__ == "__main__": raise SystemExit(main()) concurrent-log-handler-0.9.25/tests/test_stresstest.py000066400000000000000000000134041453540025200231630ustar00rootroot00000000000000#!/usr/bin/env python # ruff: noqa: S101, PT006 """ Pytest based unit test cases to drive stresstest.py. See comments about backupCount in stresstest.py. In short, if backupCount is set here less than 10, we assume that some logs are deleted before the end of the test and therefore don't test specifically for missing lines/items. """ import pytest from stresstest import TestOptions, run_stress_test TEST_CASES = { "default test options": TestOptions(), "backupCount=3": TestOptions( log_opts=TestOptions.default_log_opts({"backupCount": 3}), ), "backupCount=3, use_gzip=True": TestOptions( log_opts=TestOptions.default_log_opts({"backupCount": 3, "use_gzip": True}), ), "num_processes=2, log_calls=6_000": TestOptions( num_processes=2, log_calls=6_000, min_rollovers=80 ), "induce_failure=True": TestOptions(induce_failure=True, min_rollovers=60), "num_processes=12, log_calls=600": TestOptions( num_processes=12, log_calls=600, min_rollovers=45 ), "num_processes=15, log_calls=1_500, maxBytes=1024 * 5": TestOptions( num_processes=15, log_calls=1_500, min_rollovers=320, log_opts=TestOptions.default_log_opts( { "maxBytes": 1024 * 5, # rotate more often } ), ), "use_gzip=True": TestOptions( log_opts=TestOptions.default_log_opts( { "use_gzip": True, } ) ), "induce_failure=True, log_calls=500, use_gzip=True": TestOptions( induce_failure=True, log_calls=500, min_rollovers=20, log_opts=TestOptions.default_log_opts( { "use_gzip": True, } ), ), "num_processes=3, log_calls=500, debug=True": TestOptions( num_processes=3, log_calls=500, min_rollovers=8, log_opts=TestOptions.default_log_opts( { "debug": True, } ), ), "use_timed=True, interval=3, log_calls=5_000, debug=True": TestOptions( use_timed=True, log_calls=5_000, min_rollovers=5, log_opts=TestOptions.default_timed_log_opts( { "interval": 3, "debug": True, } ), ), "backupCount=3, use_gzip=True, use_timed=True, interval=3, log_calls=3_000, num_processes=4": TestOptions( use_timed=True, num_processes=4, log_calls=3_000, min_rollovers=4, log_opts=TestOptions.default_timed_log_opts( { "backupCount": 3, "interval": 3, "use_gzip": True, } ), ), "backupCount=4, use_timed=True, interval=4, log_calls=3_000, num_processes=5": TestOptions( use_timed=True, num_processes=5, log_calls=3_000, min_rollovers=3, log_opts=TestOptions.default_timed_log_opts( { "backupCount": 4, "interval": 4, "use_gzip": False, } ), ), # This checks the issue in Issue #68 - in Timed mode, when we have to rollover more # often than the interval due to size limits, the naming of the files is incorrect. # The check for the incorrect names is done in `run_stress_test()`. The following case can # also check it. "use_timed=True, maxBytes=512B, interval=3, log_calls=2_000, use_gzip=True, num_processes=3": TestOptions( use_timed=True, log_calls=2_000, min_rollovers=5, num_processes=3, log_opts=TestOptions.default_timed_log_opts( { "maxBytes": 512, "interval": 3, "use_gzip": True, } ), ), "backupCount=5, use_timed=True, maxBytes=1KiB, interval=5, log_calls=1_000, use_gzip=True, debug=True": TestOptions( use_timed=True, log_calls=1_000, min_rollovers=5, log_opts=TestOptions.default_timed_log_opts( { "maxBytes": 1024, "backupCount": 5, "interval": 5, "use_gzip": True, "debug": True, } ), ), "use_timed=True, num_processes=15, interval=1, log_calls=5_000, use_gzip=True": TestOptions( use_timed=True, log_calls=5_000, num_processes=15, min_rollovers=20, log_opts=TestOptions.default_timed_log_opts( { "interval": 1, "use_gzip": True, } ), ), "use_timed=True, maxBytes=100KiB, interval=1, log_calls=6_000, debug=True": TestOptions( use_timed=True, log_calls=6_000, min_rollovers=20, log_opts=TestOptions.default_timed_log_opts( { "maxBytes": 1024 * 100, "interval": 1, "debug": True, } ), ), "use_timed=True, maxBytes=100KiB, interval=2, log_calls=5_000, use_gzip=True": TestOptions( use_timed=True, log_calls=5_000, min_rollovers=20, log_opts=TestOptions.default_timed_log_opts( { "maxBytes": 1024 * 100, "interval": 2, "use_gzip": True, "debug": True, } ), ), } use_timed_only = False test_cases = TEST_CASES if use_timed_only: test_cases = {label: case for label, case in TEST_CASES.items() if case.use_timed} @pytest.mark.parametrize("label, test_opts", test_cases.items()) def test_run_stress_test(label: str, test_opts: TestOptions): # noqa: ARG001 """Run the stress test with the given options and verify the result.""" assert run_stress_test(test_opts) == (1 if test_opts.induce_failure else 0)