pax_global_header00006660000000000000000000000064146374616210014524gustar00rootroot0000000000000052 comment=a4ef6e1ffff4f41b0f66d52a8dc93e77f6f70ac3 ecs-logging-python-2.2.0/000077500000000000000000000000001463746162100152425ustar00rootroot00000000000000ecs-logging-python-2.2.0/.ci/000077500000000000000000000000001463746162100157135ustar00rootroot00000000000000ecs-logging-python-2.2.0/.ci/scripts/000077500000000000000000000000001463746162100174025ustar00rootroot00000000000000ecs-logging-python-2.2.0/.ci/scripts/lint.sh000077500000000000000000000003461463746162100207120ustar00rootroot00000000000000#!/usr/bin/env bash set -e ## When running in a docker container in the CI then it's required to set the location ## for the tools to be installed. export PATH=${HOME}/.local/bin:${PATH} python -m pip install -U nox nox -s lint ecs-logging-python-2.2.0/.ci/scripts/test.sh000077500000000000000000000004431463746162100207210ustar00rootroot00000000000000#!/usr/bin/env bash set -e VERSION=${1:?Please specify the python version} ## When running in a docker container in the CI then it's required to set the location ## for the tools to be installed. export PATH=${HOME}/.local/bin:${PATH} python -m pip install -U nox nox -s test-"${VERSION}"ecs-logging-python-2.2.0/.ci/update-specs.yml000066400000000000000000000023631463746162100210370ustar00rootroot00000000000000--- name: update specs scms: githubConfig: kind: github spec: user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: elastic repository: ecs-logging-python token: '{{ requiredEnv "GITHUB_TOKEN" }}' username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: main commitusingapi: true actions: ecs-logging-python: kind: github/pullrequest scmid: githubConfig sourceid: sha spec: automerge: false labels: - dependencies title: 'synchronize ecs-logging spec' description: |- ### What ECS logging specs automatic sync ### Why *Changeset* * https://github.com/elastic/ecs-logging/commit/{{ source "sha" }} sources: spec.json: name: Get specs from json kind: file spec: file: https://raw.githubusercontent.com/elastic/ecs-logging/main/spec/spec.json sha: name: Get commit kind: json spec: file: 'https://api.github.com/repos/elastic/ecs-logging/commits?path=spec%2Fspec.json&page=1&per_page=1' key: ".[0].sha" targets: spec.json-update: name: 'synchronize ecs-logging spec' kind: file sourceid: spec.json scmid: githubConfig spec: file: tests/resources/spec.json ecs-logging-python-2.2.0/.flake8000066400000000000000000000001631463746162100164150ustar00rootroot00000000000000[flake8] exclude= tests/**, conftest.py, setup.py max-line-length=120 ignore=E731,W503,E203,BLK100,B301ecs-logging-python-2.2.0/.github/000077500000000000000000000000001463746162100166025ustar00rootroot00000000000000ecs-logging-python-2.2.0/.github/community-label.yml000066400000000000000000000001611463746162100224240ustar00rootroot00000000000000 # add 'community' label to all new issues and PRs created by the community community: - '.*' triage: - '.*' ecs-logging-python-2.2.0/.github/dependabot.yml000066400000000000000000000005711463746162100214350ustar00rootroot00000000000000--- version: 2 updates: # Maintain dependencies for GitHub Actions (/.github/workflows) - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" day: "sunday" time: "22:00" reviewers: - "elastic/observablt-ci" labels: - dependencies groups: github-actions: patterns: - "*" ecs-logging-python-2.2.0/.github/workflows/000077500000000000000000000000001463746162100206375ustar00rootroot00000000000000ecs-logging-python-2.2.0/.github/workflows/addToProject.yml000066400000000000000000000011221463746162100237400ustar00rootroot00000000000000 name: Auto Assign to Project(s) on: issues: types: [opened, edited, milestoned] env: MY_GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }} permissions: contents: read jobs: assign_one_project: runs-on: ubuntu-latest name: Assign milestoned to Project steps: - name: Assign issues with milestones to project uses: elastic/assign-one-project-github-action@1.2.2 if: github.event.issue && github.event.issue.milestone with: project: 'https://github.com/orgs/elastic/projects/454' project_id: '5882982' column_name: 'Planned' ecs-logging-python-2.2.0/.github/workflows/labeler.yml000066400000000000000000000017311463746162100227720ustar00rootroot00000000000000name: "Issue Labeler" on: issues: types: [opened] pull_request_target: types: [opened] # '*: write' permissions for https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#add-labels-to-an-issue permissions: contents: read issues: write pull-requests: write jobs: triage: runs-on: ubuntu-latest steps: - name: Add agent-python label uses: actions-ecosystem/action-add-labels@v1 with: labels: agent-python - id: is_elastic_member uses: elastic/apm-pipeline-library/.github/actions/is-member-elastic-org@current with: username: ${{ github.actor }} token: ${{ secrets.APM_TECH_USER_TOKEN }} - name: Add community and triage labels if: contains(steps.is_elastic_member.outputs.result, 'false') && github.actor != 'dependabot[bot]' && github.actor != 'apmmachine' uses: actions-ecosystem/action-add-labels@v1 with: labels: | community triage ecs-logging-python-2.2.0/.github/workflows/periodic.yml000066400000000000000000000007431463746162100231640ustar00rootroot00000000000000name: periodic on: # Run daily at midnight schedule: - cron: "0 0 * * *" permissions: contents: read jobs: test: runs-on: ubuntu-latest timeout-minutes: 10 strategy: matrix: python: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ] fail-fast: false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: .ci/scripts/test.sh ${{ matrix.python }} ecs-logging-python-2.2.0/.github/workflows/release.yml000066400000000000000000000024321463746162100230030ustar00rootroot00000000000000name: Release on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" branches: - main permissions: contents: read jobs: packages: permissions: attestations: write id-token: write contents: read runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: pip install build==1.2.1 - run: python -m build - name: generate build provenance uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 with: subject-path: "${{ github.workspace }}/dist/*" - name: Upload Packages uses: actions/upload-artifact@v4 with: name: packages path: | dist/*.whl dist/*tar.gz publish-pypi: needs: - packages runs-on: ubuntu-latest environment: release permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - uses: actions/download-artifact@v4 with: name: packages path: dist - name: Upload pypi.org if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 with: repository-url: https://upload.pypi.org/legacy/ ecs-logging-python-2.2.0/.github/workflows/test-docs.yml000066400000000000000000000014451463746162100232730ustar00rootroot00000000000000--- # This workflow sets the test-docs status check to success in case it's a docs only PR and test.yml is not triggered # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks name: test # The name must be the same as in ci.yml on: pull_request: paths-ignore: # This expression needs to match the paths ignored on test.yml. - '**' - '!**/*.md' - '!**/*.asciidoc' permissions: contents: read jobs: test: runs-on: ubuntu-latest timeout-minutes: 5 strategy: matrix: python: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ] fail-fast: false steps: - run: 'echo "No build required"' ecs-logging-python-2.2.0/.github/workflows/test.yml000066400000000000000000000022661463746162100223470ustar00rootroot00000000000000name: test on: push: branches: [ "main" ] paths-ignore: [ '*.md', '*.asciidoc' ] pull_request: branches: [ "main" ] paths-ignore: [ '*.md', '*.asciidoc' ] permissions: contents: read ## Concurrency is only allowed in the main branch. ## So old builds, running for old commits within the same Pull Request, are cancelled concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: pre-commit: name: Run pre-commit runs-on: ubuntu-latest steps: - uses: elastic/apm-pipeline-library/.github/actions/pre-commit@current lint: runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.10' - run: .ci/scripts/lint.sh test: runs-on: ubuntu-latest timeout-minutes: 10 strategy: matrix: python: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ] fail-fast: false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: .ci/scripts/test.sh ${{ matrix.python }} ecs-logging-python-2.2.0/.github/workflows/update-specs.yml000066400000000000000000000016101463746162100237550ustar00rootroot00000000000000--- # Send PRs to the subscribed ECS Agents if the spec files (JSON) are modified name: update-specs on: workflow_dispatch: schedule: - cron: '0 6 * * *' permissions: contents: read jobs: bump: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: elastic/oblt-actions/updatecli/run@v1 with: command: "--experimental apply --config .ci/update-specs.yml" env: GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }} - if: failure() uses: elastic/oblt-actions/slack/send@v1 with: bot-token: ${{ secrets.SLACK_BOT_TOKEN }} channel-id: "#apm-agent-python" message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on " ecs-logging-python-2.2.0/.gitignore000066400000000000000000000035161463746162100172370ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # JUnit file junit-test.xml # VSCode .vscode/ # Doc build html_docs ecs-logging-python-2.2.0/.pre-commit-config.yaml000066400000000000000000000014721463746162100215270ustar00rootroot00000000000000repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-merge-conflict - repo: https://github.com/elastic/apm-pipeline-library rev: current hooks: - id: check-bash-syntax - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910 hooks: - id: mypy args: [ --strict, --show-error-codes, --no-warn-unused-ignores, --implicit-reexport, ] - repo: https://github.com/psf/black rev: 22.12.0 hooks: - id: black language_version: python3 - repo: https://github.com/pycqa/flake8 rev: 3.9.2 hooks: - id: flake8 exclude: tests|conftest.py|setup.py ecs-logging-python-2.2.0/CHANGELOG.md000066400000000000000000000116101463746162100170520ustar00rootroot00000000000000# Changelog ## 2.2.0 (2024-06-28) - Rewrite type annotations ([#119](https://github.com/elastic/ecs-logging-python/pull/119)) - Don't de-dot `ecs.version` ([#118](https://github.com/elastic/ecs-logging-python/pull/118)) - Make it possible override the JSON serializer in `StructlogFormatter` ([#114](https://github.com/elastic/ecs-logging-python/pull/114)) - Use `fromtimestamp` instead of deprecated `utcfromtimestamp` ([#105](https://github.com/elastic/ecs-logging-python/pull/105)) - Remove unused imports and fix an undefined name ([#101](https://github.com/elastic/ecs-logging-python/pull/101)) ## 2.1.0 (2023-08-16) - Add support for `service.environment` from APM log correlation ([#96](https://github.com/elastic/ecs-logging-python/pull/96)) - Fix stack trace handling in StructLog for ECS compliance ([#97](https://github.com/elastic/ecs-logging-python/pull/97)) ## 2.0.2 (2023-05-17) - Allow flit-core 3+ ([#94](https://github.com/elastic/ecs-logging-python/pull/94)) - Remove python2 leftovers ([#94](https://github.com/elastic/ecs-logging-python/pull/94)) ## 2.0.0 (2022-05-18) - Remove python 2 support ([#78](https://github.com/elastic/ecs-logging-python/pull/78)) - Add global `extra` context fields to `StdLibFormatter` ([#65](https://github.com/elastic/ecs-logging-python/pull/65)) ## 1.1.0 (2021-10-18) - Remove python 3.5 support ([#69](https://github.com/elastic/ecs-logging-python/pull/69)) - Fix an issue where APM fields would override user-provided fields even when APM wasn't installed ([#67](https://github.com/elastic/ecs-logging-python/pull/67)) - Removed `event.dataset` field handling to match [`elastic-apm` v6.6.0](https://github.com/elastic/apm-agent-python/releases/tag/v6.6.0) ([#69](https://github.com/elastic/ecs-logging-python/pull/69)) ## 1.0.2 (2021-09-22) - Fix an signature mismatch between `StdLibFormatter` and `logging.Formatter`, which could cause issues in Django and Gunicorn ([#54](https://github.com/elastic/ecs-logging-python/pull/54)) ## 1.0.1 (2021-07-06) - Fixed an issue in `StructlogFormatter` caused by a conflict with `event` (used for the log `message`) and `event.dataset` (a field provided by the `elasticapm` integration) ([#46](https://github.com/elastic/ecs-logging-python/pull/46)) - Add default/fallback handling for json.dumps ([#47](https://github.com/elastic/ecs-logging-python/pull/47)) - Fixed an issue in `StdLibFormatter` when `exc_info=False` ([#42](https://github.com/elastic/ecs-logging-python/pull/42)) ## 1.0.0 (2021-02-08) - Remove "beta" designation ## 0.6.0 (2021-01-21) - Add validation against the ecs-logging [spec](https://github.com/elastic/ecs-logging/blob/main/spec/spec.json) ([#31](https://github.com/elastic/ecs-logging-python/pull/31)) - Add support for `service.name` from APM log correlation ([#32](https://github.com/elastic/ecs-logging-python/pull/32)) - Correctly order `@timestamp`, `log.level`, and `message` fields ([#28](https://github.com/elastic/ecs-logging-python/pull/28)) ## 0.5.0 (2020-08-27) - Updated supported ECS version to 1.6.0 ([#24](https://github.com/elastic/ecs-logging-python/pull/24)) - Added support for `LogRecord.stack_info` ([#23](https://github.com/elastic/ecs-logging-python/pull/23)) - Fixed normalizing of items in `list` that aren't of type `dict` ([#22](https://github.com/elastic/ecs-logging-python/pull/22), contributed by [`@camerondavison`](https://github.com/camerondavison)) ## 0.4 (2020-08-04) - Added automatic collection of ECS fields `trace.id`, `span.id`, and `transaction.id` for [Log Correlation](https://www.elastic.co/guide/en/apm/agent/python/master/log-correlation.html) with the Python Elastic APM agent ([#17](https://github.com/elastic/ecs-logging-python/pull/17)) ## 0.3 (2020-07-27) - Added collecting `LogRecord.exc_info` into `error.*` fields automatically for `StdlibFormatter` ([#16](https://github.com/elastic/ecs-logging-python/pull/16)) - Added collecting process and thread info from `LogRecord` into `process.*` fields automatically for `StdlibFormatter` ([#16](https://github.com/elastic/ecs-logging-python/pull/16)) - Added `exclude_fields` parameter to `StdlibFormatter` to exclude fields from being formatted to JSON ([#16](https://github.com/elastic/ecs-logging-python/pull/16)) - Added `stack_trace_limit` parameter to `StdlibFormatter` to control the number of stack trace frames being formatted in `error.stack_trace` ([#16](https://github.com/elastic/ecs-logging-python/pull/16)) Thanks to community contributor Jon Moore ([@comcast-jonm](https://github.com/comcast-jonm)) for their contributions to this release. ## 0.2 (2020-04-28) - Added support for using `log(..., extra={...})` on standard library loggers to use extended and custom fields ([#8](https://github.com/elastic/ecs-logging-python/pull/8)) ## 0.1 (2020-03-26) - Added `StdlibFormatter` for use with the standard library `logging` module - Added `StructlogFormatter` for use with the `structlog` package ecs-logging-python-2.2.0/LICENSE.txt000066400000000000000000000261351463746162100170740ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ecs-logging-python-2.2.0/NOTICE.txt000066400000000000000000000000711463746162100167620ustar00rootroot00000000000000ecs-logging-python Copyright 2020-2021 Elasticsearch B.V.ecs-logging-python-2.2.0/README.md000066400000000000000000000023301463746162100165170ustar00rootroot00000000000000# ecs-logging-python [![Build Status](https://github.com/elastic/ecs-logging-python/actions/workflows/test.yml/badge.svg)](https://github.com/elastic/ecs-logging-pythonactions/workflows/test.yml) [![PyPI](https://img.shields.io/pypi/v/ecs-logging)](https://pypi.org/project/ecs-logging) [![Versions Supported](https://img.shields.io/pypi/pyversions/ecs-logging)](https://pypi.org/project/ecs-logging) Check out the [Elastic Common Schema (ECS) reference](https://www.elastic.co/guide/en/ecs/current/index.html) for more information. The library currently implements ECS 1.6. ## Installation ```console $ python -m pip install ecs-logging ``` ## Documentation See the [ECS Logging Python reference](https://www.elastic.co/guide/en/ecs-logging/python/current/index.html) on elastic.co to get started. ## Elastic APM Log Correlation `ecs-logging-python` supports automatically collecting [ECS tracing fields](https://www.elastic.co/guide/en/ecs/current/ecs-tracing.html) from the [Elastic APM Python agent](https://github.com/elastic/apm-agent-python) in order to [correlate logs to spans, transactions and traces](https://www.elastic.co/guide/en/apm/agent/python/current/log-correlation.html) in Elastic APM. ## License Apache-2.0 ecs-logging-python-2.2.0/docs/000077500000000000000000000000001463746162100161725ustar00rootroot00000000000000ecs-logging-python-2.2.0/docs/index.asciidoc000066400000000000000000000007361463746162100210070ustar00rootroot00000000000000:ecs-repo-dir: {ecs-logging-root}/docs/ include::{docs-root}/shared/versions/stack/current.asciidoc[] include::{docs-root}/shared/attributes.asciidoc[] ifdef::env-github[] NOTE: For the best reading experience, please view this documentation at https://www.elastic.co/guide/en/ecs-logging/python/current/index.html[elastic.co] endif::[] = ECS Logging Python Reference ifndef::env-github[] include::./intro.asciidoc[Introduction] include::./setup.asciidoc[Set up] endif::[] ecs-logging-python-2.2.0/docs/intro.asciidoc000066400000000000000000000010711463746162100210240ustar00rootroot00000000000000[[intro]] == Introduction ECS loggers are formatter/encoder plugins for your favorite logging libraries. They make it easy to format your logs into ECS-compatible JSON. TIP: Want to learn more about ECS, ECS logging, and other available language plugins? See the {ecs-logging-ref}/intro.html[ECS logging guide]. Ready to jump into `ecs-logging-python`? <>. If you'd like to try out a tutorial using Python ECS logging, see {cloud}/ec-getting-started-search-use-cases-python-logs.html[Ingest logs from a Python application using Filebeat]. ecs-logging-python-2.2.0/docs/setup.asciidoc000066400000000000000000000107361463746162100210410ustar00rootroot00000000000000[[installation]] == Installation [source,cmd] ---- $ python -m pip install ecs-logging ---- [float] [[gettingstarted]] == Getting Started `ecs-logging-python` has formatters for the standard library https://docs.python.org/3/library/logging.html[`logging`] module and the https://www.structlog.org/en/stable/[`structlog`] package. [float] [[logging]] === Standard Library `logging` Module [source,python] ---- import logging import ecs_logging # Get the Logger logger = logging.getLogger("app") logger.setLevel(logging.DEBUG) # Add an ECS formatter to the Handler handler = logging.StreamHandler() handler.setFormatter(ecs_logging.StdlibFormatter()) logger.addHandler(handler) # Emit a log! logger.debug("Example message!", extra={"http.request.method": "get"}) ---- [source,json] ---- { "@timestamp": "2020-03-20T18:11:37.895Z", "log.level": "debug", "message": "Example message!", "ecs": { "version": "1.6.0" }, "http": { "request": { "method": "get" } }, "log": { "logger": "app", "origin": { "file": { "line": 14, "name": "test.py" }, "function": "func" }, "original": "Example message!" } } ---- [float] ==== Excluding Fields You can exclude fields from being collected by using the `exclude_fields` option in the `StdlibFormatter` constructor: [source,python] ---- from ecs_logging import StdlibFormatter formatter = StdlibFormatter( exclude_fields=[ # You can specify individual fields to ignore: "log.original", # or you can also use prefixes to ignore # whole categories of fields: "process", "log.origin", ] ) ---- [float] ==== Limiting Stack Traces The `StdlibLogger` automatically gathers `exc_info` into ECS `error.*` fields. If you'd like to control the number of stack frames that are included in `error.stack_trace` you can use the `stack_trace_limit` parameter (by default all frames are collected): [source,python] ---- from ecs_logging import StdlibFormatter formatter = StdlibFormatter( # Only collects 3 stack frames stack_trace_limit=3, ) formatter = StdlibFormatter( # Disable stack trace collection stack_trace_limit=0, ) ---- [float] [[structlog]] === Structlog Example Note that the structlog processor should be the last processor in the list, as it handles the conversion to JSON as well as the ECS field enrichment. [source,python] ---- import structlog import ecs_logging # Configure Structlog structlog.configure( processors=[ecs_logging.StructlogFormatter()], wrapper_class=structlog.BoundLogger, context_class=dict, logger_factory=structlog.PrintLoggerFactory(), ) # Get the Logger logger = structlog.get_logger("app") # Add additional context logger = logger.bind(**{ "http": { "version": "2", "request": { "method": "get", "bytes": 1337, }, }, "url": { "domain": "example.com", "path": "/", "port": 443, "scheme": "https", "registered_domain": "example.com", "top_level_domain": "com", "original": "https://example.com", } }) # Emit a log! logger.debug("Example message!") ---- [source,json] ---- { "@timestamp": "2020-03-26T13:08:11.728Z", "ecs": { "version": "1.6.0" }, "http": { "request": { "bytes": 1337, "method": "get" }, "version": "2" }, "log": { "level": "debug" }, "message": "Example message!", "url": { "domain": "example.com", "original": "https://example.com", "path": "/", "port": 443, "registered_domain": "example.com", "scheme": "https", "top_level_domain": "com" } } ---- [float] [[correlation]] == Elastic APM Log Correlation `ecs-logging-python` supports automatically collecting {ecs-ref}/ecs-tracing.html[ECS tracing fields] from the https://github.com/elastic/apm-agent-python[Elastic APM Python agent] in order to {apm-py-ref}/logs.html[correlate logs to spans, transactions and traces] in Elastic APM. You can also quickly turn on ECS-formatted logs in your python app by setting {apm-py-ref}/configuration.html#config-log_ecs_reformatting[`LOG_ECS_REFORMATTING=override`] in the Elastic APM Python agent. [float] [[filebeat]] == Install Filebeat The best way to collect the logs once they are ECS-formatted is with https://www.elastic.co/beats/filebeat[Filebeat]: include::{ecs-repo-dir}/setup.asciidoc[tag=configure-filebeat] ecs-logging-python-2.2.0/ecs_logging/000077500000000000000000000000001463746162100175225ustar00rootroot00000000000000ecs-logging-python-2.2.0/ecs_logging/__init__.py000066400000000000000000000020411463746162100216300ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """Logging formatters for ECS (Elastic Common Schema) in Python""" from ._meta import ECS_VERSION from ._stdlib import StdlibFormatter from ._structlog import StructlogFormatter __version__ = "2.2.0" __all__ = [ "ECS_VERSION", "StdlibFormatter", "StructlogFormatter", ] ecs-logging-python-2.2.0/ecs_logging/_meta.py000066400000000000000000000014331463746162100211620ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ECS_VERSION = "1.6.0" ecs-logging-python-2.2.0/ecs_logging/_stdlib.py000066400000000000000000000265621463746162100215270ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import collections.abc import logging import sys import time from functools import lru_cache from traceback import format_tb from ._meta import ECS_VERSION from ._utils import ( de_dot, flatten_dict, json_dumps, merge_dicts, ) from typing import Any, Callable, Dict, Optional, Sequence, Union try: from typing import Literal # type: ignore except ImportError: from typing_extensions import Literal # type: ignore # Load the attributes of a LogRecord so if some are # added in the future we won't mistake them for 'extra=...' try: _LOGRECORD_DIR = set(dir(logging.LogRecord("", 0, "", 0, "", (), None))) except Exception: # LogRecord signature changed? _LOGRECORD_DIR = set() class StdlibFormatter(logging.Formatter): """ECS Formatter for the standard library ``logging`` module""" _LOGRECORD_DICT = { "name", "msg", "args", "asctime", "levelname", "levelno", "pathname", "filename", "module", "exc_info", "exc_text", "stack_info", "lineno", "funcName", "created", "msecs", "relativeCreated", "thread", "threadName", "processName", "process", "message", } | _LOGRECORD_DIR converter = time.gmtime def __init__( self, fmt: Optional[str] = None, datefmt: Optional[str] = None, style: Union[Literal["%"], Literal["{"], Literal["$"]] = "%", validate: Optional[bool] = None, stack_trace_limit: Optional[int] = None, extra: Optional[Dict[str, Any]] = None, exclude_fields: Sequence[str] = (), ) -> None: """Initialize the ECS formatter. :param int stack_trace_limit: Specifies the maximum number of frames to include for stack traces. Defaults to ``None`` which includes all available frames. Setting this to zero will suppress stack traces. This setting doesn't affect ``LogRecord.stack_info`` because this attribute is typically already pre-formatted. :param Optional[Dict[str, Any]] extra: Specifies the collection of meta-data fields to add to all records. :param Sequence[str] exclude_fields: Specifies any fields that should be suppressed from the resulting fields, expressed with dot notation:: exclude_keys=["error.stack_trace"] You can also use field prefixes to exclude whole groups of fields:: exclude_keys=["error"] """ _kwargs = {} if validate is not None: # validate was introduced in py3.8 so we need to only provide it if the user provided it _kwargs["validate"] = validate super().__init__( # type: ignore[call-arg] fmt=fmt, datefmt=datefmt, style=style, **_kwargs # type: ignore[arg-type] ) if stack_trace_limit is not None: if not isinstance(stack_trace_limit, int): raise TypeError( "'stack_trace_limit' must be None, or a non-negative integer" ) elif stack_trace_limit < 0: raise ValueError( "'stack_trace_limit' must be None, or a non-negative integer" ) if ( not isinstance(exclude_fields, collections.abc.Sequence) or isinstance(exclude_fields, str) or any(not isinstance(item, str) for item in exclude_fields) ): raise TypeError("'exclude_fields' must be a sequence of strings") self._extra = extra self._exclude_fields = frozenset(exclude_fields) self._stack_trace_limit = stack_trace_limit def _record_error_type(self, record: logging.LogRecord) -> Optional[str]: exc_info = record.exc_info if not exc_info: # exc_info is either an iterable or bool. If it doesn't # evaluate to True, then no error type is used. return None if isinstance(exc_info, bool): # if it is a bool, then look at sys.exc_info exc_info = sys.exc_info() if isinstance(exc_info, (list, tuple)) and exc_info[0] is not None: return exc_info[0].__name__ return None def _record_error_message(self, record: logging.LogRecord) -> Optional[str]: exc_info = record.exc_info if not exc_info: # exc_info is either an iterable or bool. If it doesn't # evaluate to True, then no error message is used. return None if isinstance(exc_info, bool): # if it is a bool, then look at sys.exc_info exc_info = sys.exc_info() if isinstance(exc_info, (list, tuple)) and exc_info[1]: return str(exc_info[1]) return None def format(self, record: logging.LogRecord) -> str: result = self.format_to_ecs(record) return json_dumps(result) def format_to_ecs(self, record: logging.LogRecord) -> Dict[str, Any]: """Function that can be overridden to add additional fields to (or remove fields from) the JSON before being dumped into a string. .. code-block: python class MyFormatter(StdlibFormatter): def format_to_ecs(self, record): result = super().format_to_ecs(record) del result["log"]["original"] # remove unwanted field(s) result["my_field"] = "my_value" # add custom field return result """ extractors: Dict[str, Callable[[logging.LogRecord], Any]] = { "@timestamp": self._record_timestamp, "ecs.version": lambda _: ECS_VERSION, "log.level": lambda r: (r.levelname.lower() if r.levelname else None), "log.origin.function": self._record_attribute("funcName"), "log.origin.file.line": self._record_attribute("lineno"), "log.origin.file.name": self._record_attribute("filename"), "log.original": lambda r: r.getMessage(), "log.logger": self._record_attribute("name"), "process.pid": self._record_attribute("process"), "process.name": self._record_attribute("processName"), "process.thread.id": self._record_attribute("thread"), "process.thread.name": self._record_attribute("threadName"), "error.type": self._record_error_type, "error.message": self._record_error_message, "error.stack_trace": self._record_error_stack_trace, } result: Dict[str, Any] = {} for field in set(extractors.keys()).difference(self._exclude_fields): if self._is_field_excluded(field): continue value = extractors[field](record) if value is not None: # special case ecs.version that should not be de-dotted if field == "ecs.version": field_dict = {field: value} else: field_dict = de_dot(field, value) merge_dicts(field_dict, result) available = record.__dict__ # This is cleverness because 'message' is NOT a member # key of ``record.__dict__`` the ``getMessage()`` method # is effectively ``msg % args`` (actual keys) By manually # adding 'message' to ``available``, it simplifies the code available["message"] = record.getMessage() # Pull all extras and flatten them to be sent into '_is_field_excluded' # since they can be defined as 'extras={"http": {"method": "GET"}}' extra_keys = set(available).difference(self._LOGRECORD_DICT) extras = flatten_dict({key: available[key] for key in extra_keys}) # Merge in any global extra's if self._extra is not None: for field, value in self._extra.items(): merge_dicts(de_dot(field, value), extras) # Pop all Elastic APM extras and add them # to standard tracing ECS fields. extras.setdefault("span.id", extras.pop("elasticapm_span_id", None)) extras.setdefault( "transaction.id", extras.pop("elasticapm_transaction_id", None) ) extras.setdefault("trace.id", extras.pop("elasticapm_trace_id", None)) extras.setdefault("service.name", extras.pop("elasticapm_service_name", None)) extras.setdefault( "service.environment", extras.pop("elasticapm_service_environment", None) ) # Merge in any keys that were set within 'extra={...}' for field, value in extras.items(): if field.startswith("elasticapm_labels."): continue # Unconditionally remove, we don't need this info. if value is None or self._is_field_excluded(field): continue merge_dicts(de_dot(field, value), result) # The following is mostly for the ecs format. You can't have 2x # 'message' keys in _WANTED_ATTRS, so we set the value to # 'log.original' in ecs, and this code block guarantees it # still appears as 'message' too. if not self._is_field_excluded("message"): result.setdefault("message", available["message"]) return result @lru_cache() def _is_field_excluded(self, field: str) -> bool: field_path = [] for path in field.split("."): field_path.append(path) if ".".join(field_path) in self._exclude_fields: return True return False def _record_timestamp(self, record: logging.LogRecord) -> str: return "%s.%03dZ" % ( self.formatTime(record, datefmt="%Y-%m-%dT%H:%M:%S"), record.msecs, ) def _record_attribute( self, attribute: str ) -> Callable[[logging.LogRecord], Optional[Any]]: return lambda r: getattr(r, attribute, None) def _record_error_stack_trace(self, record: logging.LogRecord) -> Optional[str]: # Using stack_info=True will add 'error.stack_trace' even # if the type is not 'error', exc_info=True only gathers # when there's an active exception. if ( record.exc_info and record.exc_info[2] is not None and (self._stack_trace_limit is None or self._stack_trace_limit > 0) ): return ( "".join(format_tb(record.exc_info[2], limit=self._stack_trace_limit)) or None ) # LogRecord only has 'stack_info' if it's passed via .log(..., stack_info=True) stack_info = getattr(record, "stack_info", None) if stack_info: return str(stack_info) return None ecs-logging-python-2.2.0/ecs_logging/_structlog.py000066400000000000000000000043651463746162100222710ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import time import datetime from typing import Any, Dict from ._meta import ECS_VERSION from ._utils import json_dumps, normalize_dict class StructlogFormatter: """ECS formatter for the ``structlog`` module""" def __call__(self, _: Any, name: str, event_dict: Dict[str, Any]) -> str: # Handle event -> message now so that stuff like `event.dataset` doesn't # cause problems down the line event_dict["message"] = str(event_dict.pop("event")) event_dict = normalize_dict(event_dict) event_dict.setdefault("log", {}).setdefault("level", name.lower()) event_dict = self.format_to_ecs(event_dict) return self._json_dumps(event_dict) def format_to_ecs(self, event_dict: Dict[str, Any]) -> Dict[str, Any]: if "@timestamp" not in event_dict: event_dict["@timestamp"] = ( datetime.datetime.fromtimestamp( time.time(), tz=datetime.timezone.utc ).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" ) if "exception" in event_dict: stack_trace = event_dict.pop("exception") if "error" in event_dict: event_dict["error"]["stack_trace"] = stack_trace else: event_dict["error"] = {"stack_trace": stack_trace} event_dict.setdefault("ecs.version", ECS_VERSION) return event_dict def _json_dumps(self, value: Dict[str, Any]) -> str: return json_dumps(value=value) ecs-logging-python-2.2.0/ecs_logging/_utils.py000066400000000000000000000124751463746162100214040ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import collections.abc import json import functools from typing import Any, Dict, Mapping __all__ = [ "normalize_dict", "de_dot", "merge_dicts", "json_dumps", ] def flatten_dict(value: Mapping[str, Any]) -> Dict[str, Any]: """Adds dots to all nested fields in dictionaries. Raises an error if there are entries which are represented with different forms of nesting. (ie {"a": {"b": 1}, "a.b": 2}) """ top_level = {} for key, val in value.items(): if not isinstance(val, collections.abc.Mapping): if key in top_level: raise ValueError(f"Duplicate entry for '{key}' with different nesting") top_level[key] = val else: val = flatten_dict(val) for vkey, vval in val.items(): vkey = f"{key}.{vkey}" if vkey in top_level: raise ValueError( f"Duplicate entry for '{vkey}' with different nesting" ) top_level[vkey] = vval return top_level def normalize_dict(value: Dict[str, Any]) -> Dict[str, Any]: """Expands all dotted names to nested dictionaries""" if not isinstance(value, dict): return value keys = list(value.keys()) for key in keys: if "." in key: merge_dicts(de_dot(key, value.pop(key)), value) for key, val in value.items(): if isinstance(val, dict): normalize_dict(val) elif isinstance(val, list): val[:] = [normalize_dict(x) for x in val] return value def de_dot(dot_string: str, msg: Any) -> Dict[str, Any]: """Turn value and dotted string key into a nested dictionary""" arr = dot_string.split(".") ret = {arr[-1]: msg} for i in range(len(arr) - 2, -1, -1): ret = {arr[i]: ret} return ret def merge_dicts(from_: Dict[Any, Any], into: Dict[Any, Any]) -> Dict[Any, Any]: """Merge deeply nested dictionary structures. When called has side-effects within 'destination'. """ for key, value in from_.items(): into.setdefault(key, {}) if isinstance(value, dict) and isinstance(into[key], dict): merge_dicts(value, into[key]) elif into[key] != {}: raise TypeError( "Type mismatch at key `{}`: merging dicts would replace value `{}` with `{}`. This is likely due to " "dotted keys in the event dict being turned into nested dictionaries, causing a conflict.".format( key, into[key], value ) ) else: into[key] = value return into def json_dumps(value: Dict[str, Any]) -> str: # Ensure that the first three fields are '@timestamp', # 'log.level', and 'message' per ECS spec ordered_fields = [] try: ordered_fields.append(("@timestamp", value.pop("@timestamp"))) except KeyError: pass # log.level can either be nested or not nested so we have to try both try: ordered_fields.append(("log.level", value["log"].pop("level"))) if not value["log"]: # Remove the 'log' dictionary if it's now empty value.pop("log", None) except KeyError: try: ordered_fields.append(("log.level", value.pop("log.level"))) except KeyError: pass try: ordered_fields.append(("message", value.pop("message"))) except KeyError: pass json_dumps = functools.partial( json.dumps, sort_keys=True, separators=(",", ":"), default=_json_dumps_fallback ) # Because we want to use 'sorted_keys=True' we manually build # the first three keys and then build the rest with json.dumps() if ordered_fields: # Need to call json.dumps() on values just in # case the given values aren't strings (even though # they should be according to the spec) ordered_json = ",".join(f'"{k}":{json_dumps(v)}' for k, v in ordered_fields) if value: return "{{{},{}".format( ordered_json, json_dumps(value)[1:], ) else: return "{%s}" % ordered_json # If there are no fields with ordering requirements we # pass everything into json.dumps() else: return json_dumps(value) def _json_dumps_fallback(value: Any) -> Any: """ Fallback handler for json.dumps to handle objects json doesn't know how to serialize. """ try: # This is what structlog's json fallback does return value.__structlog__() except AttributeError: return repr(value) ecs-logging-python-2.2.0/ecs_logging/py.typed000066400000000000000000000000001463746162100212070ustar00rootroot00000000000000ecs-logging-python-2.2.0/mypy.ini000066400000000000000000000001441463746162100167400ustar00rootroot00000000000000[mypy] exclude = "/tests/" [mypy-tests.*] ignore_errors = true [mypy-noxfile] ignore_errors = trueecs-logging-python-2.2.0/noxfile.py000066400000000000000000000035751463746162100172720ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import nox SOURCE_FILES = ("noxfile.py", "tests/", "ecs_logging/") def tests_impl(session): session.install(".[develop]") # Install `elastic-apm` from master branch session.install( "elastic-apm @ https://github.com/elastic/apm-agent-python/archive/master.zip" ) session.run( "pytest", "--junitxml=junit-test.xml", "--cov=ecs_logging", *(session.posargs or ("tests/",)), env={"PYTHONWARNINGS": "always::DeprecationWarning"}, ) @nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]) def test(session): tests_impl(session) @nox.session() def blacken(session): session.install("black") session.run("black", "--target-version=py36", *SOURCE_FILES) lint(session) @nox.session def lint(session): session.install("flake8", "black", "mypy") session.run("black", "--check", "--target-version=py36", *SOURCE_FILES) session.run("flake8", "--ignore=E501,W503", *SOURCE_FILES) session.run( "mypy", "--strict", "--show-error-codes", "--no-warn-unused-ignores", "ecs_logging/", ) ecs-logging-python-2.2.0/pyproject.toml000066400000000000000000000025741463746162100201660ustar00rootroot00000000000000[build-system] requires = ["flit_core >=2,<4"] build-backend = "flit_core.buildapi" [tool.flit.metadata] dist-name = "ecs-logging" module = "ecs_logging" description-file = "README.md" author = "Seth Michael Larson" author-email = "seth.larson@elastic.co" home-page = "https://github.com/elastic/ecs-logging-python" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: System :: Logging", "License :: OSI Approved :: Apache Software License" ] requires = [] requires-python = ">=3.6" [tool.flit.metadata.requires-extra] develop = [ "pytest", "pytest-cov", "mock", "structlog", "elastic-apm", ] [tool.flit.metadata.urls] "Source" = "https://github.com/elastic/ecs-logging-python" "Download" = "https://github.com/elastic/ecs-logging-python/releases" "Documentation" = "https://github.com/elastic/ecs-logging-python" "Issue Tracker" = "https://github.com/elastic/ecs-logging-python/issues" "Changelog" = "https://github.com/elastic/ecs-logging-python/blob/main/CHANGELOG.md" ecs-logging-python-2.2.0/pytest.ini000066400000000000000000000001631463746162100172730ustar00rootroot00000000000000[pytest] junit_logging = system-out junit_log_passing_tests = True junit_duration_report = call junit_family=xunit1ecs-logging-python-2.2.0/tests/000077500000000000000000000000001463746162100164045ustar00rootroot00000000000000ecs-logging-python-2.2.0/tests/__init__.py000066400000000000000000000014051463746162100205150ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ecs-logging-python-2.2.0/tests/conftest.py000066400000000000000000000062121463746162100206040ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import collections import datetime import json import logging import os import elasticapm import pytest class ValidationError(Exception): pass @pytest.fixture def spec_validator(): with open(os.path.join(os.path.dirname(__file__), "resources", "spec.json")) as fh: spec = json.load(fh) def validator(data_json): """ Throws a ValidationError if anything doesn't match the spec. Returns the original json (pass-through) """ fields = spec["fields"] data = json.loads(data_json, object_pairs_hook=collections.OrderedDict) for k, v in fields.items(): if v.get("required"): found = False if k in data: found = True elif "." in k: # Dotted keys could be nested, like ecs.version subkeys = k.split(".") subval = data for subkey in subkeys: subval = subval.get(subkey, {}) if subval: found = True if not found: raise ValidationError(f"Missing required key {k}") if k in data: if v["type"] == "string" and not isinstance(data[k], str): raise ValidationError( "Value {} for key {} should be string, is {}".format( data[k], k, type(data[k]) ) ) if v["type"] == "datetime": try: datetime.datetime.strptime(data[k], "%Y-%m-%dT%H:%M:%S.%fZ") except ValueError: raise ValidationError( "Value {} for key {} doesn't parse as an ISO datetime".format( data[k], k ) ) if v.get("index") and list(data.keys())[v.get("index")] != k: raise ValidationError(f"Key {k} is not at index {v.get('index')}") return data_json return validator @pytest.fixture def apm(): record_factory = logging.getLogRecordFactory() apm = elasticapm.Client( {"SERVICE_NAME": "apm-service", "ENVIRONMENT": "dev", "DISABLE_SEND": True} ) yield apm apm.close() logging.setLogRecordFactory(record_factory) ecs-logging-python-2.2.0/tests/resources/000077500000000000000000000000001463746162100204165ustar00rootroot00000000000000ecs-logging-python-2.2.0/tests/resources/spec.json000066400000000000000000000173151463746162100222520ustar00rootroot00000000000000{ "version": 1.0, "url": "https://www.elastic.co/guide/en/ecs/current/index.html", "ecs": { "version": "1.x" }, "fields": { "@timestamp": { "type": "datetime", "required": true, "index": 0, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html", "comment": [ "Field order, as specified by 'index', is RECOMMENDED.", "ECS loggers must implement field order unless the logging framework makes that impossible." ] }, "log.level": { "type": "string", "required": true, "index": 1, "top_level_field": true, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", "comment": [ "This field SHOULD NOT be a nested object field but at the top level with a dot in the property name.", "This is to make the JSON logs more human-readable.", "Loggers MAY indent the log level so that the `message` field always starts at the exact same offset,", "no matter the number of characters the log level has.", "For example: `'DEBUG'` (5 chars) will not be indented, whereas ` 'WARN'` (4 chars) will be indented by one space character." ] }, "message": { "type": "string", "required": false, "index": 2, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html", "comment": [ "A message field is typically included in all log records, but some logging libraries allow records with no message.", "That's typically the case for libraries that allow for structured logging." ] }, "ecs.version": { "type": "string", "required": true, "top_level_field": true, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-ecs.html", "comment": [ "This field SHOULD NOT be a nested object field but at the top level with a dot in the property name.", "This is to make the JSON logs more human-readable." ] }, "labels": { "type": "object", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html", "sanitization": { "key": { "replacements": [".", "*", "\\"], "substitute": "_" } } }, "trace.id": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-tracing.html", "comment": "When APM agents add this field to the context, ecs loggers should pick it up and add it to the log event." }, "transaction.id": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-tracing.html", "comment": "When APM agents add this field to the context, ecs loggers should pick it up and add it to the log event." }, "service.name": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-service.html", "comment": [ "Configurable by users.", "When an APM agent is active, it should auto-configure this field if not already set." ] }, "service.node.name": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-service.html", "comment": [ "Configurable by users.", "When an APM agent is active and `service_node_name` is manually configured, the agent should auto-configure this field if not already set." ] }, "service.version": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-service.html#field-service-version", "comment": [ "Configurable by users.", "When an APM agent is active, it should auto-configure it if not already set." ] }, "event.dataset": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-event.html", "default": "${service.name} OR ${service.name}.${appender.name}", "comment": [ "Configurable by users.", "If the user manually configures the service name,", "the logging library should set `event.dataset=${service.name}` if not explicitly configured otherwise.", "", "When agents auto-configure the app to use an ECS logger,", "they should set `event.dataset=${service.name}.${appender.name}` if the appender name is available in the logging library.", "Otherwise, agents should also set `event.dataset=${service.name}`", "", "The field helps to filter for different log streams from the same pod, for example and is required for log anomaly detection." ] }, "service.environment": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-service.html#field-service-environment", "comment": [ "Configurable by users.", "When an APM agent is active, it should auto-configure it if not already set." ] }, "process.thread.name": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-process.html" }, "log.logger": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html" }, "log.origin.file.line": { "type": "integer", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", "comment": "Should be opt-in as it requires the logging library to capture a stack trace for each log event." }, "log.origin.file.name": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", "comment": "Should be opt-in as it requires the logging library to capture a stack trace for each log event." }, "log.origin.function": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html", "comment": "Should be opt-in as it requires the logging library to capture a stack trace for each log event." }, "error.type": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-error.html", "comment": "The exception type or class, such as `java.lang.IllegalArgumentException`." }, "error.message": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-error.html", "comment": "The message of the exception." }, "error.stack_trace": { "type": "string", "required": false, "url": "https://www.elastic.co/guide/en/ecs/current/ecs-error.html", "comment": "The stack trace of the exception as plain text." } } } ecs-logging-python-2.2.0/tests/test_apm.py000066400000000000000000000141101463746162100205670ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import json import logging from io import StringIO import elasticapm import structlog from elasticapm.handlers.logging import LoggingFilter from elasticapm.handlers.structlog import structlog_processor import ecs_logging def test_elasticapm_structlog_log_correlation_ecs_fields(spec_validator, apm): stream = StringIO() logger = structlog.PrintLogger(stream) logger = structlog.wrap_logger( logger, processors=[structlog_processor, ecs_logging.StructlogFormatter()] ) log = logger.new() apm.begin_transaction("test-transaction") try: with elasticapm.capture_span("test-span"): span_id = elasticapm.get_span_id() trace_id = elasticapm.get_trace_id() transaction_id = elasticapm.get_transaction_id() log.info("test message") finally: apm.end_transaction("test-transaction") ecs = json.loads(spec_validator(stream.getvalue().rstrip())) ecs.pop("@timestamp") assert ecs == { "ecs.version": "1.6.0", "log.level": "info", "message": "test message", "span": {"id": span_id}, "trace": {"id": trace_id}, "transaction": {"id": transaction_id}, "service": {"name": "apm-service", "environment": "dev"}, } def test_elastic_apm_stdlib_no_filter_log_correlation_ecs_fields(apm): stream = StringIO() logger = logging.getLogger("apm-logger") handler = logging.StreamHandler(stream) handler.setFormatter( ecs_logging.StdlibFormatter( exclude_fields=["@timestamp", "process", "log.origin.file.line"] ) ) logger.addHandler(handler) logger.setLevel(logging.DEBUG) apm.begin_transaction("test-transaction") try: with elasticapm.capture_span("test-span"): span_id = elasticapm.get_span_id() trace_id = elasticapm.get_trace_id() transaction_id = elasticapm.get_transaction_id() logger.info("test message") finally: apm.end_transaction("test-transaction") ecs = json.loads(stream.getvalue().rstrip()) assert ecs == { "ecs.version": "1.6.0", "log.level": "info", "log": { "logger": "apm-logger", "origin": { "file": {"name": "test_apm.py"}, "function": "test_elastic_apm_stdlib_no_filter_log_correlation_ecs_fields", }, "original": "test message", }, "message": "test message", "span": {"id": span_id}, "trace": {"id": trace_id}, "transaction": {"id": transaction_id}, "service": {"name": "apm-service", "environment": "dev"}, } def test_elastic_apm_stdlib_with_filter_log_correlation_ecs_fields(apm): stream = StringIO() logger = logging.getLogger("apm-logger") handler = logging.StreamHandler(stream) handler.setFormatter( ecs_logging.StdlibFormatter( exclude_fields=["@timestamp", "process", "log.origin.file.line"] ) ) handler.addFilter(LoggingFilter()) logger.addHandler(handler) logger.setLevel(logging.DEBUG) apm.begin_transaction("test-transaction") try: with elasticapm.capture_span("test-span"): span_id = elasticapm.get_span_id() trace_id = elasticapm.get_trace_id() transaction_id = elasticapm.get_transaction_id() logger.info("test message") finally: apm.end_transaction("test-transaction") ecs = json.loads(stream.getvalue().rstrip()) assert ecs == { "ecs.version": "1.6.0", "log.level": "info", "log": { "logger": "apm-logger", "origin": { "file": {"name": "test_apm.py"}, "function": "test_elastic_apm_stdlib_with_filter_log_correlation_ecs_fields", }, "original": "test message", }, "message": "test message", "span": {"id": span_id}, "trace": {"id": trace_id}, "transaction": {"id": transaction_id}, "service": {"name": "apm-service", "environment": "dev"}, } def test_elastic_apm_stdlib_exclude_fields(apm): stream = StringIO() logger = logging.getLogger("apm-logger") handler = logging.StreamHandler(stream) handler.setFormatter( ecs_logging.StdlibFormatter( exclude_fields=[ "@timestamp", "process", "log.origin.file.line", "span", "transaction.id", ] ) ) logger.addHandler(handler) logger.setLevel(logging.DEBUG) apm.begin_transaction("test-transaction") try: with elasticapm.capture_span("test-span"): trace_id = elasticapm.get_trace_id() logger.info("test message") finally: apm.end_transaction("test-transaction") ecs = json.loads(stream.getvalue().rstrip()) assert ecs == { "ecs.version": "1.6.0", "log.level": "info", "log": { "logger": "apm-logger", "origin": { "file": {"name": "test_apm.py"}, "function": "test_elastic_apm_stdlib_exclude_fields", }, "original": "test message", }, "message": "test message", "trace": {"id": trace_id}, "service": {"name": "apm-service", "environment": "dev"}, } ecs-logging-python-2.2.0/tests/test_meta.py000066400000000000000000000016211463746162100207430ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import re from ecs_logging import ECS_VERSION def test_ecs_version_format(): assert re.match(r"[0-9](?:[.0-9]*[0-9])?", ECS_VERSION) ecs-logging-python-2.2.0/tests/test_stdlib_formatter.py000066400000000000000000000274371463746162100233760ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import logging import logging.config from unittest import mock import pytest import json import time import random import ecs_logging from io import StringIO @pytest.fixture(scope="function") def logger(): return logging.getLogger(f"test-logger-{time.time():f}-{random.random():f}") def make_record(): record = logging.LogRecord( name="logger-name", level=logging.DEBUG, pathname="/path/file.py", lineno=10, msg="%d: %s", args=(1, "hello"), func="test_function", exc_info=None, ) record.created = 1584713566 record.msecs = 123 return record def test_record_formatted(spec_validator): formatter = ecs_logging.StdlibFormatter(exclude_fields=["process"]) assert spec_validator(formatter.format(make_record())) == ( '{"@timestamp":"2020-03-20T14:12:46.123Z","log.level":"debug","message":"1: hello","ecs.version":"1.6.0",' '"log":{"logger":"logger-name","origin":{"file":{"line":10,"name":"file.py"},"function":"test_function"},' '"original":"1: hello"}}' ) def test_extra_global_is_merged(spec_validator): formatter = ecs_logging.StdlibFormatter( exclude_fields=["process"], extra={"environment": "dev"} ) assert spec_validator(formatter.format(make_record())) == ( '{"@timestamp":"2020-03-20T14:12:46.123Z","log.level":"debug","message":"1: hello","ecs.version":"1.6.0",' '"environment":"dev",' '"log":{"logger":"logger-name","origin":{"file":{"line":10,"name":"file.py"},"function":"test_function"},' '"original":"1: hello"}}' ) def test_can_be_overridden(spec_validator): class CustomFormatter(ecs_logging.StdlibFormatter): def format_to_ecs(self, record): ecs_dict = super().format_to_ecs(record) ecs_dict["custom"] = "field" return ecs_dict formatter = CustomFormatter(exclude_fields=["process"]) assert spec_validator(formatter.format(make_record())) == ( '{"@timestamp":"2020-03-20T14:12:46.123Z","log.level":"debug","message":"1: hello",' '"custom":"field","ecs.version":"1.6.0","log":{"logger":"logger-name","origin":' '{"file":{"line":10,"name":"file.py"},"function":"test_function"},"original":"1: hello"}}' ) def test_can_be_set_on_handler(): stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter(ecs_logging.StdlibFormatter(exclude_fields=["process"])) handler.handle(make_record()) assert stream.getvalue() == ( '{"@timestamp":"2020-03-20T14:12:46.123Z","log.level":"debug","message":"1: hello",' '"ecs.version":"1.6.0","log":{"logger":"logger-name","origin":{"file":{"line":10,' '"name":"file.py"},"function":"test_function"},"original":"1: hello"}}\n' ) @mock.patch("time.time") def test_extra_is_merged(time, logger): time.return_value = 1584720997.187709 stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter( ecs_logging.StdlibFormatter(exclude_fields=["process", "tls.client"]) ) logger.addHandler(handler) logger.setLevel(logging.INFO) logger.info( "hey world", extra={ "tls": { "cipher": "AES", "client": {"hash": {"md5": "0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC"}}, }, "tls.established": True, "tls.client.certificate": "cert", }, ) ecs = json.loads(stream.getvalue().rstrip()) assert isinstance(ecs["log"]["origin"]["file"].pop("line"), int) assert ecs == { "@timestamp": "2020-03-20T16:16:37.187Z", "ecs.version": "1.6.0", "log.level": "info", "log": { "logger": logger.name, "origin": { "file": {"name": "test_stdlib_formatter.py"}, "function": "test_extra_is_merged", }, "original": "hey world", }, "message": "hey world", "tls": {"cipher": "AES", "established": True}, } @pytest.mark.parametrize("kwargs", [{}, {"stack_trace_limit": None}]) def test_stack_trace_limit_default(kwargs, logger): def f(): g() def g(): h() def h(): raise ValueError("error!") stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter(ecs_logging.StdlibFormatter(**kwargs)) logger.addHandler(handler) logger.setLevel(logging.DEBUG) try: f() except ValueError: logger.info("there was an error", exc_info=True) ecs = json.loads(stream.getvalue().rstrip()) error_stack_trace = ecs["error"].pop("stack_trace") assert all(x in error_stack_trace for x in ("f()", "g()", "h()")) @pytest.mark.parametrize("stack_trace_limit", [0, False]) def test_stack_trace_limit_disabled(stack_trace_limit, logger): stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter( ecs_logging.StdlibFormatter(stack_trace_limit=stack_trace_limit) ) logger.addHandler(handler) logger.setLevel(logging.DEBUG) try: raise ValueError("error!") except ValueError: logger.info("there was an error", exc_info=True) ecs = json.loads(stream.getvalue().rstrip()) assert ecs["error"] == {"message": "error!", "type": "ValueError"} assert ecs["log.level"] == "info" assert ecs["message"] == "there was an error" assert ecs["log"]["original"] == "there was an error" def test_exc_info_false_does_not_raise(logger): stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter(ecs_logging.StdlibFormatter()) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.info("there was %serror", "no ", exc_info=False) ecs = json.loads(stream.getvalue().rstrip()) assert ecs["log.level"] == "info" assert ecs["message"] == "there was no error" assert "error" not in ecs def test_stack_trace_limit_traceback(logger): def f(): g() def g(): h() def h(): raise ValueError("error!") stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter(ecs_logging.StdlibFormatter(stack_trace_limit=2)) logger.addHandler(handler) logger.setLevel(logging.DEBUG) try: f() except ValueError: logger.info("there was an error", exc_info=True) ecs = json.loads(stream.getvalue().rstrip()) error_stack_trace = ecs["error"].pop("stack_trace") assert all(x in error_stack_trace for x in ("f()", "g()")) assert "h()" not in error_stack_trace assert ecs["error"] == { "message": "error!", "type": "ValueError", } assert ecs["log.level"] == "info" assert ecs["message"] == "there was an error" assert ecs["log"]["original"] == "there was an error" def test_stack_trace_limit_types_and_values(): with pytest.raises(TypeError) as e: ecs_logging.StdlibFormatter(stack_trace_limit="a") assert str(e.value) == "'stack_trace_limit' must be None, or a non-negative integer" with pytest.raises(ValueError) as e: ecs_logging.StdlibFormatter(stack_trace_limit=-1) assert str(e.value) == "'stack_trace_limit' must be None, or a non-negative integer" @pytest.mark.parametrize( "exclude_fields", [ "process", "log", "log.level", "message", ["log.origin", "log.origin.file", "log.origin.file.line"], ], ) def test_exclude_fields(exclude_fields): if isinstance(exclude_fields, str): exclude_fields = [exclude_fields] formatter = ecs_logging.StdlibFormatter(exclude_fields=exclude_fields) ecs = formatter.format_to_ecs(make_record()) for entry in exclude_fields: field_path = entry.split(".") try: obj = ecs for path in field_path[:-1]: obj = obj[path] except KeyError: continue assert field_path[-1] not in obj @pytest.mark.parametrize( "exclude_fields", [ "ecs.version", ], ) def test_exclude_fields_not_dedotted(exclude_fields): formatter = ecs_logging.StdlibFormatter(exclude_fields=[exclude_fields]) ecs = formatter.format_to_ecs(make_record()) for entry in exclude_fields: assert entry not in ecs def test_exclude_fields_empty_json_object(): """Assert that if all JSON objects attributes are excluded then the object doesn't appear.""" formatter = ecs_logging.StdlibFormatter( exclude_fields=["process.pid", "process.name", "process.thread"] ) ecs = formatter.format_to_ecs(make_record()) assert "process" not in ecs formatter = ecs_logging.StdlibFormatter(exclude_fields=["ecs.version"]) ecs = formatter.format_to_ecs(make_record()) assert "ecs" not in ecs def test_exclude_fields_type_and_values(): with pytest.raises(TypeError) as e: ecs_logging.StdlibFormatter(exclude_fields="a") assert str(e.value) == "'exclude_fields' must be a sequence of strings" with pytest.raises(TypeError) as e: ecs_logging.StdlibFormatter(exclude_fields={"a"}) assert str(e.value) == "'exclude_fields' must be a sequence of strings" with pytest.raises(TypeError) as e: ecs_logging.StdlibFormatter(exclude_fields=[1]) assert str(e.value) == "'exclude_fields' must be a sequence of strings" def test_stack_info(logger): stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter(ecs_logging.StdlibFormatter()) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.info("stack info!", stack_info=True) ecs = json.loads(stream.getvalue().rstrip()) assert list(ecs["error"].keys()) == ["stack_trace"] error_stack_trace = ecs["error"].pop("stack_trace") assert "test_stack_info" in error_stack_trace and __file__ in error_stack_trace @pytest.mark.parametrize("exclude_fields", [["error"], ["error.stack_trace"]]) def test_stack_info_excluded(logger, exclude_fields): stream = StringIO() handler = logging.StreamHandler(stream) handler.setFormatter(ecs_logging.StdlibFormatter(exclude_fields=exclude_fields)) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.info("stack info!", stack_info=True) ecs = json.loads(stream.getvalue().rstrip()) assert "error" not in ecs def test_stdlibformatter_signature(): logging.config.dictConfig( { "version": 1, "formatters": {"my_formatter": {"class": "ecs_logging.StdlibFormatter"}}, } ) def test_apm_data_conflicts(spec_validator): record = make_record() record.service = {"version": "1.0.0", "name": "myapp", "environment": "dev"} formatter = ecs_logging.StdlibFormatter(exclude_fields=["process"]) assert spec_validator(formatter.format(record)) == ( '{"@timestamp":"2020-03-20T14:12:46.123Z","log.level":"debug","message":"1: hello","ecs.version":"1.6.0",' '"log":{"logger":"logger-name","origin":{"file":{"line":10,"name":"file.py"},"function":"test_function"},' '"original":"1: hello"},"service":{"environment":"dev","name":"myapp","version":"1.0.0"}}' ) ecs-logging-python-2.2.0/tests/test_structlog_formatter.py000066400000000000000000000064421463746162100241340ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import json from io import StringIO from unittest import mock import pytest import structlog import ecs_logging class NotSerializable: def __repr__(self): return "" @pytest.fixture def event_dict(): return { "event": "test message", "log.logger": "logger-name", "foo": "bar", "baz": NotSerializable(), } @pytest.fixture def event_dict_with_exception(): return { "event": "test message", "log.logger": "logger-name", "foo": "bar", "exception": "", } def test_conflicting_event_dict(event_dict): formatter = ecs_logging.StructlogFormatter() event_dict["foo.bar"] = "baz" with pytest.raises(TypeError): formatter(None, "debug", event_dict) @mock.patch("time.time") def test_event_dict_formatted(time, spec_validator, event_dict): time.return_value = 1584720997.187709 formatter = ecs_logging.StructlogFormatter() assert spec_validator(formatter(None, "debug", event_dict)) == ( '{"@timestamp":"2020-03-20T16:16:37.187Z","log.level":"debug",' '"message":"test message",' '"baz":"",' '"ecs.version":"1.6.0",' '"foo":"bar",' '"log":{"logger":"logger-name"}}' ) @mock.patch("time.time") def test_can_be_set_as_processor(time, spec_validator): time.return_value = 1584720997.187709 stream = StringIO() structlog.configure( processors=[ecs_logging.StructlogFormatter()], wrapper_class=structlog.BoundLogger, context_class=dict, logger_factory=structlog.PrintLoggerFactory(stream), ) logger = structlog.get_logger("logger-name") logger.debug("test message", custom="key", **{"dot.ted": 1}) assert spec_validator(stream.getvalue()) == ( '{"@timestamp":"2020-03-20T16:16:37.187Z","log.level":"debug",' '"message":"test message","custom":"key","dot":{"ted":1},' '"ecs.version":"1.6.0"}\n' ) def test_exception_log_is_ecs_compliant_when_used_with_format_exc_info( event_dict_with_exception, ): formatter = ecs_logging.StructlogFormatter() formatted_event_dict = json.loads( formatter(None, "debug", event_dict_with_exception) ) assert ( "exception" not in formatted_event_dict ), "The key 'exception' at the root of a log is not ECS-compliant" assert "error" in formatted_event_dict assert "stack_trace" in formatted_event_dict["error"] assert "" in formatted_event_dict["error"]["stack_trace"] ecs-logging-python-2.2.0/tests/test_utils.py000066400000000000000000000064761463746162100211720ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import pytest from ecs_logging._utils import flatten_dict, de_dot, normalize_dict, json_dumps def test_flatten_dict(): assert flatten_dict( {"a": {"b": 1}, "a.c": {"d.e": {"f": 1}, "d.e.g": [{"f.c": 2}]}} ) == {"a.b": 1, "a.c.d.e.f": 1, "a.c.d.e.g": [{"f.c": 2}]} with pytest.raises(ValueError) as e: flatten_dict({"a": {"b": 1}, "a.b": 2}) assert str(e.value) == "Duplicate entry for 'a.b' with different nesting" with pytest.raises(ValueError) as e: flatten_dict({"a": {"b": {"c": 1}}, "a.b": {"c": 2}, "a.b.c": 1}) assert str(e.value) == "Duplicate entry for 'a.b.c' with different nesting" def test_de_dot(): assert de_dot("x.y.z", {"a": {"b": 1}}) == {"x": {"y": {"z": {"a": {"b": 1}}}}} def test_normalize_dict(): assert normalize_dict( {"a": {"b": 1}, "a.c": {"d.e": {"f": 1}, "d.e.g": [{"f.c": 2}]}} ) == {"a": {"b": 1, "c": {"d": {"e": {"f": 1, "g": [{"f": {"c": 2}}]}}}}} def test_normalize_dict_with_array(): assert normalize_dict({"a": ["1", "2"]}) == {"a": ["1", "2"]} @pytest.mark.parametrize( ["value", "expected"], [ ({}, "{}"), ({"log": {"level": "info"}}, '{"log.level":"info"}'), ({"log.level": "info"}, '{"log.level":"info"}'), ( {"log": {"level": "info", "message": "hello"}}, '{"log.level":"info","log":{"message":"hello"}}', ), ({"@timestamp": "2021-01-01..."}, '{"@timestamp":"2021-01-01..."}'), ({"message": "hello"}, '{"message":"hello"}'), ({"message": 1}, '{"message":1}'), ({"message": ["hello"]}, '{"message":["hello"]}'), ({"message": {"key": "val"}}, '{"message":{"key":"val"}}'), ({"custom": "value"}, '{"custom":"value"}'), ({"log.level": "info"}, '{"log.level":"info"}'), ( {"log": {"message": "hello"}, "message": "hello"}, '{"message":"hello","log":{"message":"hello"}}', ), ( { "log": {"message": "hello", "level": "info"}, "message": "hello", "@timestamp": "2021-01-01...", }, '{"@timestamp":"2021-01-01...","log.level":"info","message":"hello","log":{"message":"hello"}}', ), ( { "log": {"level": "info"}, "message": "hello", "@timestamp": "2021-01-01...", }, '{"@timestamp":"2021-01-01...","log.level":"info","message":"hello"}', ), ], ) def test_json_dumps(value, expected): assert json_dumps(value) == expected ecs-logging-python-2.2.0/utils/000077500000000000000000000000001463746162100164025ustar00rootroot00000000000000ecs-logging-python-2.2.0/utils/check-license-headers.sh000077500000000000000000000016221463746162100230500ustar00rootroot00000000000000#!/usr/bin/env bash # Check that source code files in this repo have the appropriate license # header. if [ "$TRACE" != "" ]; then export PS4='${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' set -o xtrace fi set -o errexit set -o pipefail TOP=$(cd "$(dirname "$0")/.." >/dev/null && pwd) NLINES=$(wc -l utils/license-header.txt | awk '{print $1}') function check_license_header { local f f=$1 if ! diff utils/license-header.txt <(head -$NLINES "$f") >/dev/null; then echo "check-license-headers: error: '$f' does not have required license header, see 'diff -u utils/license-header.txt <(head -$NLINES $f)'" return 1 else return 0 fi } cd "$TOP" nErrors=0 for f in $(git ls-files | grep '\.py$'); do if ! check_license_header $f; then nErrors=$((nErrors+1)) fi done if [[ $nErrors -eq 0 ]]; then exit 0 else exit 1 fi ecs-logging-python-2.2.0/utils/license-header.txt000066400000000000000000000014051463746162100220130ustar00rootroot00000000000000# Licensed to Elasticsearch B.V. under one or more contributor # license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright # ownership. Elasticsearch B.V. licenses this file to you under # the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License.