pax_global_header00006660000000000000000000000064150070054070014510gustar00rootroot0000000000000052 comment=36701bed11497ec5fee22239f0a8c2e4a8358e78 ijson-3.4.0/000077500000000000000000000000001500700540700126365ustar00rootroot00000000000000ijson-3.4.0/.coveragerc000066400000000000000000000022031500700540700147540ustar00rootroot00000000000000# # Coverage configuration file # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2019 # Copyright by UWA (in the framework of the ICRAR) # All rights reserved # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, # MA 02111-1307 USA # # We currently exclude the deploy directory because it's not # part of the core software [run] source = src [report] exclude_lines = raise ImportError\('no backends available'\) if stdout.isatty(): ijson-3.4.0/.github/000077500000000000000000000000001500700540700141765ustar00rootroot00000000000000ijson-3.4.0/.github/ISSUE_TEMPLATE/000077500000000000000000000000001500700540700163615ustar00rootroot00000000000000ijson-3.4.0/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000014061500700540700210540ustar00rootroot00000000000000--- name: Bug report about: Create a report to help us improve title: '' labels: bug assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **How to reproduce*** Provide both code and a small JSON document that triggers the problem. ```python JSON = b""" [1, 2, 3, 4] """ for o in ijson.items(JSON, 'item'): raise ValueError('this failed') ``` **Expected behavior** A clear and concise description of what you expected to happen. **Execution information:** - Python version [e.g. 3.7] - ijson version [e.g. 3.1.2] - ijson backend (if applies) [e.g. yajl2_c] - ijson installation method [e.g. package manager, pip, git sources, conda] - OS [e.g. linux] **Additional context** Add any other context about the problem here. ijson-3.4.0/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000012101500700540700221000ustar00rootroot00000000000000--- name: Feature request about: Suggest an idea for this project title: '' labels: feature assignees: '' --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. It would be great if it was possible for ijson to [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered, and why/how they failed. **Additional context** Add any other context or screenshots about the feature request here. ijson-3.4.0/.github/ISSUE_TEMPLATE/usage-question.md000066400000000000000000000007451500700540700216620ustar00rootroot00000000000000--- name: Usage question about: Ask how to achieve a particular task title: '' labels: question assignees: '' --- **Description** A clear and concise description of what your question is. Ex. How can I use ijson to [...] **Detailed description** Add more details on what exactly you need to achieve, including example JSON documents and code. **Why is this not clear from the documentation** Is the question not clearly answerable from the documentation? If not, why do you think? ijson-3.4.0/.github/dependabot.yml000066400000000000000000000001661500700540700170310ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" ijson-3.4.0/.github/tools/000077500000000000000000000000001500700540700153365ustar00rootroot00000000000000ijson-3.4.0/.github/tools/install_yajl.sh000066400000000000000000000004751500700540700203650ustar00rootroot00000000000000#!/bin/sh install_unix() { mkdir cextern/yajl/build; cd cextern/yajl/build cmake .. -DCMAKE_C_FLAGS="-O3" make all -j 4 make install } OS="`uname -s`" echo "Building libyajl on $OS" case "$OS" in MSYS_*|MINGW_*|Darwin) echo "No extra step required in Windows or macOS" ;; Linux) install_unix ;; esac ijson-3.4.0/.github/workflows/000077500000000000000000000000001500700540700162335ustar00rootroot00000000000000ijson-3.4.0/.github/workflows/deploy-to-pypi.yml000066400000000000000000000114161500700540700216540ustar00rootroot00000000000000name: Build distributions and upload to PyPI # Build on every branch push, tag push, and pull request change: on: push: pull_request: schedule: # 00:00 UTC every Saturday, don't bother anyone - cron: '0 0 * * 6' jobs: calculate_wheels_to_build: name: Calculate OS/archs to build wheels on runs-on: ubuntu-latest env: BUILD_TYPE: ${{ (startsWith(github.event.ref, 'refs/tags/v') || startsWith(github.event.ref, 'refs/heads/fullci_') || github.event.schedule == '0 0 * * 6') && 'FULL' || 'BASE' }} ARCHS_LINUX_BASE: '["x86_64"]' ARCHS_LINUX_FULL: '["x86_64", "i686", "aarch64"]' ARCHS_MACOS_BASE: '["x86_64"]' ARCHS_MACOS_FULL: '["x86_64", "arm64", "universal2"]' ARCHS_WINDOWS_BASE: '["AMD64"]' ARCHS_WINDOWS_FULL: '["AMD64", "x86"]' outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Calculate strategy matrix shell: python id: set-matrix run: | import json import os combinations = ( ("ubuntu-22.04", "ARCHS_LINUX"), ("macos-14", "ARCHS_MACOS"), ("windows-2019", "ARCHS_WINDOWS"), ) build_type = os.getenv("BUILD_TYPE") includes = [ {"os": os_name, "arch": arch} for os_name, archs_envvar in combinations for arch in json.loads(os.getenv(f'{archs_envvar}_{build_type}')) ] matrix = {"include": includes} with open(os.getenv("GITHUB_OUTPUT"), "at") as github_output: github_output.write(f'matrix={json.dumps(matrix)}') print(f"Calculated matrix strategy:\n{json.dumps(matrix, indent=2)}") build_wheels: name: Build wheels for ${{ matrix.os }} / ${{matrix.arch}} needs: calculate_wheels_to_build runs-on: ${{ matrix.os }} strategy: matrix: ${{ fromJson(needs.calculate_wheels_to_build.outputs.matrix) }} env: CIBW_ARCHS: ${{ matrix.arch }} steps: - uses: actions/checkout@v4 with: submodules: true - uses: docker/setup-qemu-action@v3 if: ${{ matrix.arch == 'aarch64' && matrix.os == 'ubuntu-22.04' }} name: Set up QEMU - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 with: output-dir: wheelhouse env: CIBW_BEFORE_ALL: "bash -c 'cd \"{project}\"; sh .github/tools/install_yajl.sh'" CIBW_BUILD_VERBOSITY: 1 CIBW_ENVIRONMENT_MACOS: "IJSON_EMBED_YAJL=1" CIBW_ENVIRONMENT_WINDOWS: "IJSON_EMBED_YAJL=1" CIBW_BEFORE_TEST: pip install -r test-requirements.txt CIBW_TEST_COMMAND: "bash -c 'cd \"{project}\"; pytest -vv'" # Our C extension made PyPy < 7.3.13 crash (a bug on their end) # it doesn't make sense to build wheels for platforms not supported # not supported by that version CIBW_SKIP: pp{39}* CIBW_ENABLE: pypy cpython-freethreading - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.os }}-${{ matrix.arch }} path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.13' - name: Install build run: pip install build - name: Build sdist run: python -m build -s - uses: actions/upload-artifact@v4 with: name: source-dist path: dist/*.tar.gz test_sdist: name: Check source distribution is usable needs: [build_sdist] runs-on: ubuntu-latest steps: - uses: actions/setup-python@v5 name: Install Python with: python-version: "3.13" - uses: actions/download-artifact@v4 with: name: source-dist - name: Extract source distribution run: tar xvf ijson*.tar.gz && rm ijson*.tar.gz - name: Install source distribution run: pip install ./ijson* - name: Install test dependencies run: pip install pytest cffi - name: Run source distribution tests run: cd ijson* && pytest merge_artifacts: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest steps: - name: Merge Artifacts uses: actions/upload-artifact/merge@v4 with: name: all-artifacts upload_pypi: needs: [merge_artifacts] runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags/v') steps: - uses: actions/download-artifact@v4 with: name: all-artifacts path: dist - uses: pypa/gh-action-pypi-publish@master with: user: __token__ password: ${{ secrets.pypi_password }} ijson-3.4.0/.github/workflows/fast-built-and-test.yml000066400000000000000000000024351500700540700225510ustar00rootroot00000000000000name: Build and test ijson # Build on every branch push, tag push, and pull request change: on: push: pull_request: schedule: # 00:00 UTC every Saturday, don't bother anyone - cron: '0 0 * * 6' jobs: fast_tests: name: Build ijson and run unit tests (${{ matrix.os }}, ${{ matrix.python_version }}) runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, windows-2019, macos-14] python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"] include: - os: ubuntu-latest python_version: "pypy3.10" steps: - uses: actions/checkout@v4 with: submodules: true - uses: actions/setup-python@v5 name: Install Python with: python-version: ${{ matrix.python_version }} - name: Install Yajl if: ${{ matrix.os == 'ubuntu-latest' }} run: sudo apt install libyajl-dev - name: Build ijson env: IJSON_EMBED_YAJL: ${{ matrix.os != 'ubuntu-latest' && '1' || '0' }} run: pip install . - name: Install test dependencies run: pip install -r test-requirements.txt - name: Run unittests run: pytest -vv - name: Run doctests run: pytest --doctest-modules --doctest-ignore-import-errors src ijson-3.4.0/.github/workflows/memleak-tests.yml000066400000000000000000000021501500700540700215270ustar00rootroot00000000000000name: Run memory leak tests # Build on every branch push, tag push, and pull request change: on: push: pull_request: schedule: # 00:00 UTC every Saturday, don't bother anyone - cron: '0 0 * * 6' jobs: memleak_tests: name: Build memory leak tests (${{ matrix.os }}, ${{ matrix.python_version }}) runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-14] python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 with: submodules: true - uses: actions/setup-python@v5 name: Install Python with: python-version: ${{ matrix.python_version }} - name: Install Yajl if: ${{ matrix.os == 'ubuntu-latest' }} run: sudo apt install libyajl-dev - name: Build ijson env: IJSON_EMBED_YAJL: ${{ matrix.os != 'ubuntu-latest' && '1' || '0' }} run: pip install . - name: Install test dependencies run: pip install -r test-requirements.txt - name: Run memory leak tests run: pytest --memleaks-only -v ijson-3.4.0/.gitignore000066400000000000000000000002131500700540700146220ustar00rootroot00000000000000*.pyc .tox ijson.egg-info *.so build *.whl *.egg .coverage dist .project .pydevproject .settings .cproject .autotools yajl-patched-sources ijson-3.4.0/.gitmodules000066400000000000000000000001261500700540700150120ustar00rootroot00000000000000[submodule "cextern/yajl"] path = cextern/yajl url = https://github.com/rtobar/yajl ijson-3.4.0/CHANGELOG.md000066400000000000000000000346741500700540700144650ustar00rootroot00000000000000# Changelog ## Development ## [3.4.0] * Added support for PEP 489 multi-phase initialisation and per-module state for our C extension, allowing us to support sub-interpreters with per-interpreter GIL. * Advertise support for free-threading python mode. * Removed support for Python < 3.9. * Enhanced generators so they yield all possible results to users before errors are raised (#123). * Added `ijson.ALL_BACKENDS` constant listing all supported backends (which might or not be available at runtime). * Added a `capabilities` constant to each backend describing which capabilities it supports. * Exposing backend's name under ``.backend_name``, and default backend's name under ``ijson.backend_name``. This is similar to the already existing ``name`` constant, only slightly better named to hopefully avoid confusion. * Restructured source code so all code lives under `src/`, and the `ijson.backends._yajl2` extension under `src/ijson/backends/ext/_yajl2`. This allows C backend tests to actually run on cibuildwheel. * Improved performance of `parse` routine in C backend by ~4%. * Fixed several potential stability issues in C backend around correct error handling. * Fixed corner-case wrong behaviour of `yajl2_c` backend, which didn't work correctly with user-provided event names. * Pointing to our own fork of yajl (for when we build it ourselves) that contains fixes for all known CVEs (#126). * Removed leftover compatibility bits in the C backend. * Fixed potential issue with `yajl` and `yajl2` backends where crashes could occur at interpreter shutdown. * Removed tox. * Moved static project metadata to `pyproject.toml`. ## [3.3.0] * Removed support for Python 2.7 and 3.4, 3.5+ is still supported. * Distribute the existing `benchmark.py` script as ``ijson.benchmark``. The module is an improved version of the script, supporting #iterations for a given function invocation, multiple input files, and more. ## [3.2.3] * Fixed several issues in the ``yajl2_c`` backend and its async generators that were only made apparent when running it with PyPy and/or a CPython debug build (#101). As part of that, an issue was found and fixed in PyPy itself affecting all versions up to 7.3.12, so users will need to wait until the next version is released to be able to use async generators (https://foss.heptapod.net/pypy/pypy/-/issues/3956). * Adapted ``yajl2_c`` async generators to work against PyPy shortcomings (https://foss.heptapod.net/pypy/pypy/-/issues/3965). ## [3.2.2] * Fixed compilation and ``async`` support of the ``yajl2_c`` backend in pyhthon 3.12 (#98). * Check ``IJSON_BUILD_YAJL2C`` environment variable when building ijson to force/skip building the ``yajl2_c`` backend (#102). ## [3.2.1] * Added support for Python 3.12. * Fixed a memory leak in the ``yajl2_c`` backend triggered only when the underlying ``yajl`` functions reported a failure (#97). ## [3.2.0.post0] * Fixed minor README rendering issues that prevented upload of 3.2.0 distributions to PyPI. ## [3.2.0] * New ``ijson.dump`` command-line utility for simple inspection of the ijson iteration process. This tool should be useful for new users who are usually confused with how to use the library, and the prefix in particular. * Fixed bug in ``yajl2_c`` backend introduced in 3.1.2 where random crashes could occur due to an unsafe reference decrement when constructing the parse/items/kvitems generators (#66). * Mark Python 3.10 and 3.11 as explicitly supported. ## [3.1.4] * Fixed bug in ``yajl2_c`` backend introduced in 3.1.0 where ``ijson.items`` didn't work correctly against member names containing ``.`` (#41). * Python backend raises errors on incomplete JSON content that previously wasn't recognised as such, aligning itself with the rest of the backends (#42). ## [3.1.3] * Python backed correctly raises errors when JSON numbers with leading zeros are found in the stream (#40). * Likewise, JSON numbers with fractions where the decimal point is not surrounded by at least one digit on both sides also produce an error now on the python backend. * Fixed detection of file objects with generator-based ``read`` coroutines (i.e., a ``read`` generator decorated with ``@types.coroutine``) for the purpose of automatically routing user calls done through the main entry points. For example, when using ``aiofiles`` objects users could invoke ``async for item in ijson.parse_async(f)`` but not ``async for item in ijson.parse(f)``, while the latter has been possible since 3.1 for native coroutines. ## [3.1.2.post0] * Moved binary wheel generation from GitHub Actions to Travis. This gained us binary ARM wheels, which are becoming increasingly popular (#35) ## [3.1.2] * Fixed minor memory leaks in the initialization methods of the generators of the ``yajl2_c`` backend. All generators (i.e., ``basic_parse``, ``parse``, ``kvitems`` and ``items``) in both their sync and async versions, were affected. ## [3.1.1] * Fixed two problems in the ``yajl2_c`` backend related to `asyncio` support, which prevented some objects like those from ``aiofiles`` from working properly (#32). * Ironing out and documenting some corner cases related to the use of ``use_float=True`` and its side-effect on integer number parsing. ## [3.1.post0] * Removed ``test`` package from binary distributions. ## [3.1] * A new ``use_float`` option has been added to all backends to control whether ``float`` values should be returned for non-integer numbers instead of ``Decimal`` objects. Using this option trades loss of precision (which most applications probably don't care) for performance (which most application do care about). Historically ijson has returned ``Decimal`` objects, and therefore the option defaults to ``False`` for backwards compatibility, but in later releases this default could change to ``True``. * Improved the performance of the ``items`` and ``kvitems`` methods of the ``yajl2_c`` backend (by internally avoiding unnecessary string concatenations). Local tests show a performance improvement of up to ~15%, but mileage might vary depending on your use case and system. * The "raw" functions ``basic_parse``, ``parse``, ``items`` and ``kvitems`` can now be used with different types of inputs. In particular they accept not only file-like objects, but also asynchronous file-like objects, behaving like their ``*_async`` counterparts. They also accept ``bytes`` and ``str`` objects directly (and ``unicode`` objects in python 2.7). Finally, they also accept iterables, in which case they behave like the ``ijson.common.*`` functions, allowing users to tap into the event pipeline. * ``ijson.common`` routines ``parse``, ``items`` and ``kvitems`` are marked as deprecated. Users should use the ``ijson.*`` routines instead, which now accept event iterables. * New ``ijson.get_backend`` function for users to import a backend programmatically (without having to manually use importlib). * New ``IJSON_BACKEND`` environment variable can be used to choose the default backend to be exposed by ijson. * Unicode decoding errors are now reported more clearly to users. In the past there was a mix of empty messages and error types. Now the error type is always the same and there should always be an error messages indicating the offending byte sequence. * ``ijson.common.number`` is marked as deprecated, and will be removed on some later release. ## [3.0.4] * Fixed errors triggered by JSON documents where the top-level value is an object containing an empty-named member (e.g., ``{"": 1}``). Although such documents are valid JSON, they broke basic assumptions made by the ``kvitems`` and ``items`` functions (and all their variants) in all backends, producing different types of unexpected failures, including segmentation faults, raising unexpected exceptions, and producing wrong results. ## [3.0.3] * Fixed segmentation fault in ``yajl2_c`` backend's ``parse`` caused by the previous fix introduced in 3.0.2 (#29). ## [3.0.2] * Fixed memory leak in ``yajl2_c`` backend's ``parse`` functionality (#28). ## [3.0.1] * Adding back the ``parse``, ``kvitems`` and ``items`` functions under the ``ijson.common`` module (#27). These functions take an events iterable instead of a file and are backend-independent (which is not great for performance). They were accidentally removed in the redesign of ijson 3.0, which is why they are coming back. In the future they will slowly transition into being backend-specific rather than independent. ## [3.0] * Exposing backend's name under ``.backend``, and default backend's name under ``ijson.backend``. * Exposing ``ijson.sendable_list`` to users in case it comes in handy. ## [3.0rc3] * Implemented all asynchronous iterables (i.e., ``*_async`` functions) in C for the ``yajl2_c`` backend for increased performance. * Adding Windows builds via AppVeyor, generating binary wheels for Python 3.5+. ## [3.0rc2] * Fixed known problem with 3.0rc1, namely checking that asynchronous files are opened in the correct mode (i.e., binary). * Improved the protocol for user-facing coroutines, where instead of having to send a final, empty bytes string to finish the parsing process users can simply call ``.close()`` on the coroutine. * Greatly increased testing of user-facing coroutines, which in turn uncovered problems that were fixed. * Adding ability to benchmark coroutines with ``benchmark.py``. * Including C code in coverage measurements, and increased overall code coverage up to 99%. ## [3.0rc1] * Full re-design of ijson: instead of working with generators on a "pull" model, it now uses coroutines on a "push" model. The current set of generators (``basic_parse``, ``parse``, ``kvitems`` and ``items``) are implemented on top of these coroutines, and are fully backward compatible. Some text comparing the old a new designs can be found [here](notes/design_notes.rst). * Initial support for ``asyncio`` in python 3.5+ in the for of ``async for``-enabled asynchronous iterators. These are named ``*_async``, and take a file-like object whose ``read()`` method can be ``awaited`` on. * Exposure of underlying infrastructure implementing the push model. These are named ``*_coro``, and take a coroutine-like object (i.e., implementing a ``send`` method) instead of file-like objects. In this scheme, users are in charge of sending chunks of data into the coroutines using ``coro.send(chunk)``. * C backend performance improved by avoiding memory copies when possible when reading data off a file (i.e., using ``readinto`` when possible) and by avoiding tuple packing/unpacking in certain situations. * C extension broken down into separate source files for easier understanding and maintenance. ## [2.6.1] * Fixed a deprecation warning in the C backend present in python 3.8 when parsing Decimal values. ## [2.6.0] * New `kvitems` method in all backends. Like `items`, it takes a prefix, and iterates over the key/value pairs of matching objects (instead of iterating over objects themselves, like in `items`). This is useful for iterating over big objects that would otherwise consume too much memory. * When using python 2, all backends now return `map_key` values as `unicode` objects, not `str` (until now only the Python backend did so). This is what the `json` built-in module does, and allows for correctly handling non-ascii key names. Comparison between `unicode` and `str` objects is possible, so most client code should be unaffected. * Improving error handling in yajl2 backend (ctypes-based) so exceptions caught in callbacks interrupt the parsing process. * Including more files in source distributions (#14). * Adjusting python backend to avoid reading off the input stream too eagerly (#15). ## [2.5.1] * Fixing backwards compatibility, allowing string readers in all backends (#12, #13). ## [2.5] * Default backend changed (#5). Instead of using the python backend, now the fastest available backend is selected by default. * Added support for new `map_type` option (#7). * Fixed bug in `multiple_values` support in C backend (#8). * Added support for ``multiple_values`` flag in python backend (#9). * Forwarding `**kwargs` from `ijson.items` to `ijson.parse` and `ijson.basic_parse` (#10). * Fixing support for yajl versions < 1.0.12. * Improving `common.number` implementation. * Documenting how events and the prefix work (#4). ## [2.4] - New `ijson.backends.yajl2_c` backend written in C and based on the yajl2 library. It performs ~10x faster than cffi backend. - Adding more builds to Travis matrix. - Preventing memory leaks in `ijson.items` - Parse numbers consistent with stdlib json - Correct JSON string parsing in python backend - Publishing package version in __init__.py - Various small fixes in cffi backend [2.4]: https://github.com/ICRAR/ijson/releases/tag/2.4 [2.5]: https://github.com/ICRAR/ijson/releases/tag/v2.5 [2.5.1]: https://github.com/ICRAR/ijson/releases/tag/v2.5.1 [2.6.0]: https://github.com/ICRAR/ijson/releases/tag/v2.6.0 [2.6.1]: https://github.com/ICRAR/ijson/releases/tag/v2.6.1 [3.0rc1]: https://github.com/ICRAR/ijson/releases/tag/v3.0rc1 [3.0rc2]: https://github.com/ICRAR/ijson/releases/tag/v3.0rc2 [3.0rc3]: https://github.com/ICRAR/ijson/releases/tag/v3.0rc3 [3.0]: https://github.com/ICRAR/ijson/releases/tag/v3.0 [3.0.1]: https://github.com/ICRAR/ijson/releases/tag/v3.0.1 [3.0.2]: https://github.com/ICRAR/ijson/releases/tag/v3.0.2 [3.0.3]: https://github.com/ICRAR/ijson/releases/tag/v3.0.3 [3.0.4]: https://github.com/ICRAR/ijson/releases/tag/v3.0.4 [3.1]: https://github.com/ICRAR/ijson/releases/tag/v3.1 [3.1.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.1.post0 [3.1.1]: https://github.com/ICRAR/ijson/releases/tag/v3.1.1 [3.1.2]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2 [3.1.2.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2.post0 [3.1.3]: https://github.com/ICRAR/ijson/releases/tag/v3.1.3 [3.1.4]: https://github.com/ICRAR/ijson/releases/tag/v3.1.4 [3.2.0]: https://github.com/ICRAR/ijson/releases/tag/v3.2.0 [3.2.0.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.2.0.post0 [3.2.1]: https://github.com/ICRAR/ijson/releases/tag/v3.2.1 [3.2.2]: https://github.com/ICRAR/ijson/releases/tag/v3.2.2 [3.2.3]: https://github.com/ICRAR/ijson/releases/tag/v3.2.3 [3.3.0]: https://github.com/ICRAR/ijson/releases/tag/v3.3.0 [3.4.0]: https://github.com/ICRAR/ijson/releases/tag/v3.4.0 ijson-3.4.0/LICENSE.txt000066400000000000000000000043311500700540700144620ustar00rootroot00000000000000ijson ===== Copyright (c) 2010, Ivan Sagalaev All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name "ijson" nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. yajl ==== Copyright (c) 2007-2014, Lloyd Hilaiel Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ijson-3.4.0/MANIFEST.in000066400000000000000000000002271500700540700143750ustar00rootroot00000000000000include LICENSE.txt include README.rst include CHANGELOG.md graft src/ijson/backends/ext/ graft tests graft yajl-patched-sources/ global-exclude *.pyc ijson-3.4.0/README.rst000066400000000000000000000503701500700540700143320ustar00rootroot00000000000000.. image:: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml/badge.svg :target: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml .. image:: https://github.com/ICRAR/ijson/actions/workflows/fast-built-and-test.yml/badge.svg :target: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml .. image:: https://coveralls.io/repos/github/ICRAR/ijson/badge.svg?branch=master :target: https://coveralls.io/github/ICRAR/ijson?branch=master .. image:: https://badge.fury.io/py/ijson.svg :target: https://badge.fury.io/py/ijson .. image:: https://img.shields.io/pypi/pyversions/ijson.svg :target: https://pypi.python.org/pypi/ijson .. image:: https://img.shields.io/pypi/dd/ijson.svg :target: https://pypi.python.org/pypi/ijson .. image:: https://img.shields.io/pypi/dw/ijson.svg :target: https://pypi.python.org/pypi/ijson .. image:: https://img.shields.io/pypi/dm/ijson.svg :target: https://pypi.python.org/pypi/ijson ===== ijson ===== Ijson is an iterative JSON parser with standard Python iterator interfaces. .. contents:: :local: Installation ============ Ijson is hosted in PyPI, so you should be able to install it via ``pip``:: pip install ijson Binary wheels are provided for major platforms and python versions. These are built and published automatically using `cibuildwheel `_ via GitHub Actions. Usage ===== All usage example will be using a JSON document describing geographical objects: .. code-block:: json { "earth": { "europe": [ {"name": "Paris", "type": "city", "info": { ... }}, {"name": "Thames", "type": "river", "info": { ... }}, // ... ], "america": [ {"name": "Texas", "type": "state", "info": { ... }}, // ... ] } } High-level interfaces --------------------- Most common usage is having ijson yield native Python objects out of a JSON stream located under a prefix. This is done using the ``items`` function. Here's how to process all European cities: .. code-block:: python import ijson f = urlopen('http://.../') objects = ijson.items(f, 'earth.europe.item') cities = (o for o in objects if o['type'] == 'city') for city in cities: do_something_with(city) For how to build a prefix see the prefix_ section below. Other times it might be useful to iterate over object members rather than objects themselves (e.g., when objects are too big). In that case one can use the ``kvitems`` function instead: .. code-block:: python import ijson f = urlopen('http://.../') european_places = ijson.kvitems(f, 'earth.europe.item') names = (v for k, v in european_places if k == 'name') for name in names: do_something_with(name) Lower-level interfaces ---------------------- Sometimes when dealing with a particularly large JSON payload it may worth to not even construct individual Python objects and react on individual events immediately producing some result. This is achieved using the ``parse`` function: .. code-block:: python import ijson parser = ijson.parse(urlopen('http://.../')) stream.write('') for prefix, event, value in parser: if (prefix, event) == ('earth', 'map_key'): stream.write('<%s>' % value) continent = value elif prefix.endswith('.name'): stream.write('' % value) elif (prefix, event) == ('earth.%s' % continent, 'end_map'): stream.write('' % continent) stream.write('') Even more bare-bones is the ability to react on individual events without even calculating a prefix using the ``basic_parse`` function: .. code-block:: python import ijson events = ijson.basic_parse(urlopen('http://.../')) num_names = sum(1 for event, value in events if event == 'map_key' and value == 'name') .. _command_line: Command line ------------ A command line utility is included with ijson to help visualise the output of each of the routines above. It reads JSON from the standard input, and it prints the results of the parsing method chosen by the user to the standard output. The tool is available by running the ``ijson.dump`` module. For example:: $> echo '{"A": 0, "B": [1, 2, 3, 4]}' | python -m ijson.dump -m parse #: path, name, value -------------------- 0: , start_map, None 1: , map_key, A 2: A, number, 0 3: , map_key, B 4: B, start_array, None 5: B.item, number, 1 6: B.item, number, 2 7: B.item, number, 3 8: B.item, number, 4 9: B, end_array, None 10: , end_map, None Using ``-h/--help`` will show all available options. .. _benchmarking: Benchmarking ------------ A command line utility is included with ijson to help benchmarking the different methods offered by the package. It offers some built-in example inputs that try to mimic different scenarios, but more importantly it also supports user-provided inputs. You can also specify which backends to time, number of iterations, and more. The tool is available by running the ``ijson.benchmark`` module. For example:: $> python -m ijson.benchmark my/json/file.json -m items -p values.item Using ``-h/--help`` will show all available options. ``bytes``/``str`` support ------------------------- Although not usually how they are meant to be run, all the functions above also accept ``bytes`` and ``str`` objects directly as inputs. These are then internally wrapped into a file object, and further processed. This is useful for testing and prototyping, but probably not extremely useful in real-life scenarios. ``asyncio`` support ------------------- All of the methods above work also on file-like asynchronous objects, so they can be iterated asynchronously. In other words, something like this: .. code-block:: python import asyncio import ijson async def run(): f = await async_urlopen('http://..../') async for object in ijson.items(f, 'earth.europe.item'): if object['type'] == 'city': do_something_with(city) asyncio.run(run()) An explicit set of ``*_async`` functions also exists offering the same functionality, except they will fail if anything other than a file-like asynchronous object is given to them. (so the example above can also be written using ``ijson.items_async``). In fact in ijson version 3.0 this was the only way to access the ``asyncio`` support. Intercepting events ------------------- The four routines shown above internally chain against each other: tuples generated by ``basic_parse`` are the input for ``parse``, whose results are the input to ``kvitems`` and ``items``. Normally users don't see this interaction, as they only care about the final output of the function they invoked, but there are occasions when tapping into this invocation chain this could be handy. This is supported by passing the output of one function (i.e., an iterable of events, usually a generator) as the input of another, opening the door for user event filtering or injection. For instance if one wants to skip some content before full item parsing: .. code-block:: python import io import ijson parse_events = ijson.parse(io.BytesIO(b'["skip", {"a": 1}, {"b": 2}, {"c": 3}]')) while True: prefix, event, value = next(parse_events) if value == "skip": break for obj in ijson.items(parse_events, 'item'): print(obj) Note that this interception only makes sense for the ``basic_parse -> parse``, ``parse -> items`` and ``parse -> kvitems`` interactions. Note also that event interception is currently not supported by the ``async`` functions. Push interfaces --------------- All examples above use a file-like object as the data input (both the normal case, and for ``asyncio`` support), and hence are "pull" interfaces, with the library reading data as necessary. If for whatever reason it's not possible to use such method, you can still **push** data through yet a different interface: `coroutines `_ (via generators, not ``asyncio`` coroutines). Coroutines effectively allow users to send data to them at any point in time, with a final *target* coroutine-like object receiving the results. In the following example the user is doing the reading instead of letting the library do it: .. code-block:: python import ijson @ijson.coroutine def print_cities(): while True: obj = (yield) if obj['type'] != 'city': continue print(obj) coro = ijson.items_coro(print_cities(), 'earth.europe.item') f = urlopen('http://.../') for chunk in iter(functools.partial(f.read, buf_size)): coro.send(chunk) coro.close() All four ijson iterators have a ``*_coro`` counterpart that work by pushing data into them. Instead of receiving a file-like object and option buffer size as arguments, they receive a single ``target`` argument, which should be a coroutine-like object (anything implementing a ``send`` method) through which results will be published. An alternative to providing a coroutine is to use ``ijson.sendable_list`` to accumulate results, providing the list is cleared after each parsing iteration, like this: .. code-block:: python import ijson events = ijson.sendable_list() coro = ijson.items_coro(events, 'earth.europe.item') f = urlopen('http://.../') for chunk in iter(functools.partial(f.read, buf_size)): coro.send(chunk) process_accumulated_events(events) del events[:] coro.close() process_accumulated_events(events) .. _options: Options ======= Additional options are supported by **all** ijson functions to give users more fine-grained control over certain operations: - The ``use_float`` option (defaults to ``False``) controls how non-integer values are returned to the user. If set to ``True`` users receive ``float()`` values; otherwise ``Decimal`` values are constructed. Note that building ``float`` values is usually faster, but on the other hand there might be loss of precision (which most applications will not care about) and will raise an exception when overflow occurs (e.g., if ``1e400`` is encountered). This option also has the side-effect that integer numbers bigger than ``2^64`` (but *sometimes* ``2^32``, see capabilities_) will also raise an overflow error, due to similar reasons. Future versions of ijson might change the default value of this option to ``True``. - The ``multiple_values`` option (defaults to ``False``) controls whether multiple top-level values are supported. JSON content should contain a single top-level value (see `the JSON Grammar `_). However there are plenty of JSON files out in the wild that contain multiple top-level values, often separated by newlines. By default ijson will fail to process these with a ``parse error: trailing garbage`` error unless ``multiple_values=True`` is specified. - Similarly the ``allow_comments`` option (defaults to ``False``) controls whether C-style comments (e.g., ``/* a comment */``), which are not supported by the JSON standard, are allowed in the content or not. - For functions taking a file-like object, an additional ``buf_size`` option (defaults to ``65536`` or 64KB) specifies the amount of bytes the library should attempt to read each time. - The ``items`` and ``kvitems`` functions, and all their variants, have an optional ``map_type`` argument (defaults to ``dict``) used to construct objects from the JSON stream. This should be a dict-like type supporting item assignment. Events ====== When using the lower-level ``ijson.parse`` function, three-element tuples are generated containing a prefix, an event name, and a value. Events will be one of the following: - ``start_map`` and ``end_map`` indicate the beginning and end of a JSON object, respectively. They carry a ``None`` as their value. - ``start_array`` and ``end_array`` indicate the beginning and end of a JSON array, respectively. They also carry a ``None`` as their value. - ``map_key`` indicates the name of a field in a JSON object. Its associated value is the name itself. - ``null``, ``boolean``, ``integer``, ``double``, ``number`` and ``string`` all indicate actual content, which is stored in the associated value. .. _prefix: Prefix ====== A prefix represents the context within a JSON document where an event originates at. It works as follows: - It starts as an empty string. - A ```` part is appended when the parser starts parsing the contents of a JSON object member called ``name``, and removed once the content finishes. - A literal ``item`` part is appended when the parser is parsing elements of a JSON array, and removed when the array ends. - Parts are separated by ``.``. When using the ``ijson.items`` function, the prefix works as the selection for which objects should be automatically built and returned by ijson. .. _backends: Backends ======== Ijson provides several implementations of the actual parsing in the form of backends located in ijson/backends: - ``yajl2_c``: a C extension using `YAJL `__ 2.x. This is the fastest, but *might* require a compiler and the YAJL development files to be present when installing this package. Binary wheel distributions exist for major platforms/architectures to spare users from having to compile the package. - ``yajl2_cffi``: wrapper around `YAJL `__ 2.x using CFFI. - ``yajl2``: wrapper around YAJL 2.x using ctypes, for when you can't use CFFI for some reason. - ``yajl``: deprecated YAJL 1.x + ctypes wrapper, for even older systems. - ``python``: pure Python parser, good to use with PyPy This list of backend names is available under the ``ijson.ALL_BACKENDS`` constant. You can import a specific backend and use it in the same way as the top level library: .. code-block:: python import ijson.backends.yajl2_cffi as ijson for item in ijson.items(...): # ... Importing the top level library as ``import ijson`` uses the first available backend in the same order of the list above, and its name is recorded under ``ijson.backend``. If the ``IJSON_BACKEND`` environment variable is set its value takes precedence and is used to select the default backend. You can also use the ``ijson.get_backend`` function to get a specific backend based on a name: .. code-block:: python backend = ijson.get_backend('yajl2_c') for item in backend.items(...): # ... .. _capabilities: Capabilities ------------ Apart from their performance, all backends are designed to support the same capabilities. There are however some small known differences, all of which can be queried by inspecting the ``capabilities`` module constant. It contains the following members: * ``c_comments``: C-style comments are supported. * ``multiple_values``: multiple top-level JSON values are supported. * ``detects_invalid_leading_zeros``: numbers with leading zeroes are reported as invalid (as they should, as pert the JSON standard), raising a ``ValueError``. * ``detects_incomplete_json_tokens``: detects incomplete JSON tokens at the end of an incomplete document (e.g., ``{"a": fals``), raising an ``IncompleteJSONError``. * ``int64``: when using ``use_float=True``, values greater than or equal to ``2^32`` are correctly returned. These capabilities are supported by all backends, with the following exceptions: * The ``yajl`` backend doesn't support ``multiple_values``, ``detects_invalid_leading_zeros`` and ``detects_incomplete_json_tokens``. It also doesn't support ``int64`` in platforms with a 32-bit C ``long`` type. * The ``python`` backend doesn't support ``c_comments``. Performance tips ================ In more-or-less decreasing order, these are the most common actions you can take to ensure you get most of the performance out of ijson: - Make sure you use the fastest backend available. See backends_ for details. - If you know your JSON data contains only numbers that are "well behaved" consider turning on the ``use_float`` option. See options_ for details. - Make sure you feed ijson with binary data instead of text data. See faq_ #1 for details. - Play with the ``buf_size`` option, as depending on your data source and your system a value different from the default might show better performance. See options_ for details. The benchmarking_ tool should help with trying some of these options and observing their effect on your input files. .. _faq: FAQ === #. **Q**: Does ijson work with ``bytes`` or ``str`` values? **A**: In short: both are accepted as input, outputs are only ``str``. All ijson functions expecting a file-like object should ideally be given one that is opened in binary mode (i.e., its ``read`` function returns ``bytes`` objects, not ``str``). However if a text-mode file object is given then the library will automatically encode the strings into UTF-8 bytes. A warning is currently issued (but not visible by default) alerting users about this automatic conversion. On the other hand ijson always returns text data (JSON string values, object member names, event names, etc) as ``str`` objects. This mimics the behavior of the system ``json`` module. #. **Q**: How are numbers dealt with? **A**: ijson returns ``int`` values for integers and ``decimal.Decimal`` values for floating-point numbers. This is mostly because of historical reasons. Since 3.1 a new ``use_float`` option (defaults to ``False``) is available to return ``float`` values instead. See the options_ section for details. #. **Q**: I'm getting an ``UnicodeDecodeError``, or an ``IncompleteJSONError`` with no message **A**: This error is caused by byte sequences that are not valid in UTF-8. In other words, the data given to ijson is not *really* UTF-8 encoded, or at least not properly. Depending on where the data comes from you have different options: * If you have control over the source of the data, fix it. * If you have a way to intercept the data flow, do so and pass it through a "byte corrector". For instance, if you have a shell pipeline feeding data through ``stdin`` into your process you can add something like ``... | iconv -f utf8 -t utf8 -c | ...`` in between to correct invalid byte sequences. * If you are working purely in python, you can create a UTF-8 decoder using codecs' `incrementaldecoder `_ to leniently decode your bytes into strings, and feed those strings (using a file-like class) into ijson (see our `string_reader_async internal class `_ for some inspiration). In the future ijson might offer something out of the box to deal with invalid UTF-8 byte sequences. #. **Q**: I'm getting ``parse error: trailing garbage`` or ``Additional data found`` errors **A**: This error signals that the input contains more data than the top-level JSON value it's meant to contain. This is *usually* caused by JSON data sources containing multiple values, and is *usually* solved by passing the ``multiple_values=True`` to the ijson function in use. See the options_ section for details. Acknowledgements ================ ijson was originally developed and actively maintained until 2016 by `Ivan Sagalaev `_. In 2019 he `handed over `_ the maintenance of the project and the PyPI ownership. Python parser in ijson is relatively simple thanks to `Douglas Crockford `_ who invented a strict, easy to parse syntax. The `YAJL `__ library by `Lloyd Hilaiel `_ is the most popular and efficient way to parse JSON in an iterative fashion. When building the library ourselves, we use `our own fork `__ that contains fixes for all known CVEs. Ijson was inspired by `yajl-py `_ wrapper by `Hatem Nassrat `_. Though ijson borrows almost nothing from the actual yajl-py code it was used as an example of integration with yajl using ctypes. ijson-3.4.0/cextern/000077500000000000000000000000001500700540700143065ustar00rootroot00000000000000ijson-3.4.0/cextern/yajl/000077500000000000000000000000001500700540700152455ustar00rootroot00000000000000ijson-3.4.0/notes/000077500000000000000000000000001500700540700137665ustar00rootroot00000000000000ijson-3.4.0/notes/design_notes.rst000066400000000000000000000431721500700540700172100ustar00rootroot00000000000000ijson design notes ################## Version 3.0 will come with a complete re-design of the underlying mechanism used to move data through the different library layers. This was done to address some limitations of the current design, and allow more advanced use cases out of the box. This document explains the design in ijson 2.x, and then it shows the new design coming with ijson 3.x, and how it offers both backward compatibility (with no performance cost) and new features out of the box. 2.x design (pull) ================= ijson is a library for iterating over big (or small) JSON documents. The main goal of the library is to avoid loading the entire document into memory; instead the document is iterated over in small chunks, results are given back to the user as they are found, and hence memory consumption stays low throughout the whole parsing process. This is achieved using python generators, which naturally lend themselves to this kind of problems: just write a ``for`` loop and ``yield`` values out of it, and the caller can iterate over them in their own ``for`` loop. ijson offers four modes of iterating over a JSON document, (not long ago they were three, as ``kvitems`` was recently added), each returning different types of results: * ``basic_parse`` is the most basic one, and returns ``(event, value)`` tuples each time an element is found in the JSON stream (e.g., the beginning of an array, a object member name, etc). * ``parse`` returns ``(prefix, event, value)`` tuples containing not only the information returned by ``basic_parse``, but also the *prefix* or *location* in the document hierarchy under which the event occurred. * ``items`` returns fully-built python objects constructed from the contents found at a given ``prefix``. * Finally, ``kvitems`` is similar to ``items``, but instead of returning full objects it returns ``(name, value)`` tuples consisting on its *members*, which is useful when individual objects in the JSON document are too big. As a side note, in ijson there are also different *backends*, all of which offer the same four iteration modes. Most of them share most of the code, but most interestingly the ``yajl2_c`` backend doesn't, re-implementing all the logic in C. These four iteration modes are all implemented as python generators. They also have the nice property that they are built on top of one another. For example, ``parse`` basically works like this: .. code-block:: python def parse(...): for event, value in basic_parse(...): prefix = calculate_prefix(event, value) yield (prefix, event, value) All in all, the different generators are combined like this to form a *pipeline*:: +---------+ ---| kvitems | +-------------+ +-------+ | +---------+ | basic_parse |-----| parse |----+ +-------------+ +-------+ | +---------+ |---| items | +---------+ Now, what the diagram above doesn't show is *who* calls *who*. Python generators yielding values work in a "pull" fashion: the caller drives the execution by calling ``next(generator)``, the generator runs until the next ``yield`` statement is executed and the yielded value is returned to the caller (or until the generator exists, in which case ``StopIteration`` is raised). So, adding the direction of how calls are made (including internal and external call points), the diagram above looks like this:: +---------+ next() ---| kvitems |<------- +-------------+ next() +-------+ next() | +---------+ | basic_parse |<--------| parse |<--------+ +-------------+ +-------+ | +---------+ next() ^ ^ |---| items |<------- | next() | next() +---------+ The only missing bit in this diagram now is: where does data reading happen? Because this is a pull model, the only possible solution is to do the reading at the lowest level possible: ``basic_parse`` reads a chunk out of the JSON stream when it is invoked for new tuples (and none are held in memory), calculates new tuples with the parsing results, yields them, and eventually when there are no more contents on the stream it exits, provoking the whole pipeline to exit In other words, ``basic_parse`` exhausts the JSON stream, but users drive the process by iterating over the generator of choice until it returns. All in all, the library works like this:: +---------+ next() ---| kvitems |<------- +------+ read() +-------------+ next() +-------+ next() | +---------+ | file |<--------| basic_parse |<--------| parse |<--------+ +------+ +-------------+ +-------+ | +---------+ next() ^ ^ |---| items |<------- | next() | next() +---------+ Limitations =========== This is all well and good if you live in a blocking I/O world. All you need to do is pass down your blocking, file-like object when you call ``items``, ``kvitems``, ``parse`` or ``basic_parse``, and ijson will do the reading for you as you iterate over the results. However, this is also a main fundamental limitation as it prevents users from easily using ijson in other scenarios. One such scenario comes up when using ``asyncio``. In an ``asyncio`` world users don't perform blocking reads anymore, and are instead handed over chunks of data to operate on. The only way to operate with ijson in these situations is emulating a blocking, file-like object, which is error-prone and not a great solution overall. Another example in which ijson doesn't work well would be the following: consider a multi-stage, blocking socket-based conversation between ``A`` and ``B``. ``A`` first receives some JSON content from ``B``, and after parsing it it needs to reply to ``B``, which will send more content through the same channel of communication, and so on. In such case, one would need to wrap the socket into a new object that emulates exhausting of the stream when required, even though the underlying socket is not exhausted yet. In both cases what we want to do is let users *push* data into ijson rather than having ijson *pull* data. This is basis for the new design. 3.x design (push) ================= In ijson 3.x we completely redesigned the parsing pipeline to work in a push model rather than a pull model, where chunks of data are pushed by the user into the pipeline. This effectively decouples I/O from the core parsing logic. In other words, at the low level instead of working like this:: +---------+ next() ---| kvitems |<------- +------+ read() +-------------+ next() +-------+ next() | +---------+ | file |<--------| basic_parse |<--------| parse |<--------+ +------+ +-------------+ +-------+ | +---------+ next() ^ ^ |---| items |<------- | next() | next() +---------+ the library now works like this:: +---------+ +-->| kvitems | +-------------+ +-------+ | +---------+ chunk ---->| basic_parse |---->| parse |----+ +-------------+ +-------+ | +---------+ |-->| items | +---------+ Here, ``chunk`` is a piece of data held by the user, who sends it into ``basic_parse``, who upon calculating a new tuple sends it to ``parse`` and so on, depending on what iteration mode you request. Now the user is in full control of feeding data into the pipeline and reacting to its results, without the library being in charge anymore. This is implemented using... python generators! Generators are usually used to yield values, but since python 2.5 they can also to *receive* values back from their callers. This turns then effectively into "coroutines", very much like those found in python 3.5+ ``asyncio`` coroutines using the ``async`` and ``await`` syntax. On that note: why didn't we use *those*? There are at least two drawbacks to using them directly: * Using ``asyncio`` coroutines would require users of the library to work within an ``asyncio`` event loop when interacting with ijson routines. As of now, that represents 0% of the current user base of ijson, which so far hasn't offered support for ``asyncio`` out of the box. Using generator coroutines doesn't impose any execution context, giving users the freedom to use them wherever they want. * Python 2.7 support would be removed. While python 2.7 is no longer maintained by the Python Software Foundation, there might still be programs out there using ijson with python 2.7, and we don't want to break them (yet). Using generator coroutines we maintain python 2.7 support in the library core. For the rest of the text, "coroutine" then means "plain, generator-based coroutine" and not "python 3 ``asyncio``-based coroutine", unless explicitly mentioned. How do these new coroutines look like? Firstly, they have new names to avoid clashing with those of the current set of generators, and hence they all end up with a ``_basecoro`` suffix (more on this later). Apart from this they look fairly similar to the original generators. For example, let's see ``basic_parse_basecoro`` and ``parse_basecoro`` in action (this is not actual code, just a skeleton): .. code-block:: python def basic_parse_basecoro(target, ...): while True: chunk = (yield) event, value = do_some_parsing(chunk) target.send((event, value)) def parse_basecoro(target, ...): while True: event, value = (yield) prefix = calculate_prefix(event, value) target.send((prefix, event, value)) The key components are the ``(yield)`` statements, which allow coroutines to receive data, and the ``target.send`` calls, which is how one sends data into a coroutine. Moreover, we can chain them again forming a pipeline, with data being pushed from one side, and the appropriate events making it out on the other. With these changes the core pipeline now looks like this (again including internal and external calls):: +------------------+ send() +-->| kvitems_basecoro |-------> send() +----------------------+ send() +----------------+ send() | +------------------+ chunk ------->| basic_parse_basecoro |------->| parse_basecoro |--------+ +----------------------+ +----------------+ | +------------------+ send() | | |-->| items_basecoro |-------> +--> send() +--> send() +------------------+ Backwards-compatibility ----------------------- Implementing the original generators on top of this coroutine-based pipeline can be easily done, thus retaining backwards-compatibility for all users. This is basically how ``parse`` works now: .. code-block:: python def sendable_list(list): send = list.append def parse(f, ...): results = sendable_list() coro = parse_basecoro(results) coro = basic_parse_basecoro(parse) while True: chunk = f.read() coro.send(chunk) for result in results: yield result del results[:] if not chunk: break Or, in other words:: parse +------------------------------------------------------------------------------+ +------+ read() | +----------------------+ send() +----------------+ send() +------+ | next() | file |<--------| chunk --->| basic_parse_basecoro |------->| parse_basecoro |------->| list | |<------- +------+ | +----------------------+ +----------------+ +------+ | +------------------------------------------------------------------------------+ The other generators work similarly, with the corresponding coroutine pipeline constructed inside each generator. Support for asyncio ------------------- Using this new framework, adding support for ``asyncio`` generators (i.e., that can be used in an ``async for`` loop) is also trivial. Now when running under python 3.5+ sll generators have an ``async`` counterpart ending with an ``_async`` suffix, which are roughly implemented like this: .. code-block:: python def async_iterable(object): def __init__(self, f): self.f = f self.events = sendable_deque() async def __anext__(self): data = await self.f.read() try: self.coro.send(data) except StopIteration: raise StopAsyncIteration return self.events.pop() def basic_parse_async(f, ...): iterable = async_iterable(f) iterable.coro = basic_parse_basecoro(iterable.events) return iterable Or, in other words:: parse_async +-----------------------------------------------------------------------------------+ +------+ await read() | send() +----------------------+ send() +----------------+ send() +-------+ | __anext__() | file |<--------------| chunk ------->| basic_parse_basecoro |------->| parse_basecoro |------->| deque | |<------------ +------+ | +----------------------+ +----------------+ +-------+ | +-----------------------------------------------------------------------------------+ Again, the other async generators work similarly, with the full corresponding coroutine pipeline constructed inside the async generator. User-facing coroutines ---------------------- Finally, it would also make sense to offer users access to the underlying coroutines, with users pushing data into them, and registering a target to receive the results. The ``*_basecoro`` coroutines are designed to work each on their own though, and users would have to create them inidividually and then chain them together manually, which can be error prone. Instead we also offer "pre-chained" coroutines for each of the iterators, which receive a chunk of data on one side, and send out the relevant event to the user-provided coroutine. These are called ``*_coro`` (which is why the *core* ones are called ``*_basecoro`` instead). They are roughly implemented like this: .. code-block:: python def parse_coro(target, ...): return basic_parse_basecoro(parse_basecoro(target), ...) Or, in other words:: parse_coro +-----------------------------------------------------+ send() | +----------------------+ send() +----------------+ | send() chunk --------|->| basic_parse_basecoro |------->| parse_basecoro |-|-------> | +----------------------+ +----------------+ | +-----------------------------------------------------+ The other user-facing coroutines are constructed similarly. Performance =========== This is the best part: performance is still on-par with the previous implementation, and has even been improved as part of this exercise. The plot below shows a comparison of processing speeds of each generator over three different backends (python, yajl2 and yajl2_c) for different synthetic test cases. The other backends should have similar results. For each generator/backend/test combination, the old implementation (pure generator) is compared to the new (coroutine-based generator). Values are processing speeds in [MB/s], so higher is better. All these measurements have been made using the ``benchmark.py`` tool found in the git repository of ijson, so you can give it a try as well. .. image:: performance_comparison.png These measurements were run on an Intel(R) Core(TM) i7-5600U CPU using python 3.7 (full results: `old.csv <./old.csv>`_, `new.csv <./new.csv>`_). It can be seen that results vary depending on the backend, but overall speeds are comparable to the original implementation. Special attention was given to the C backend, as it was, and remains, the fastest of all, usually by a factor of 10x. During the porting to the new push model, we added some modifications to its inner working to avoid unnecessary data copies and tuple packing/unpacking where possible, leading to a noticeable improvement on performance (~25% as the median). Again, your mileage might vary depending on the document you are parsing, but overall these are very good results. No proper performance comparison has been made yet on the ``asyncio`` generators offered by ijson, but early results suggest there is still work to be done to fully catch up with the generators' speed. On the one hand, implementing them as ``async`` generators (which would require python 3.6+) instead of classes with ``__aiter__`` and ``__anext__`` apparently would give a boost in speed. Other strategies could also be investigated for storing the temporary results rather than keeping them in a deque. Finally, the C backend could see its own implementation of the ``async`` iterables, which will probably not be too hard. ijson-3.4.0/notes/new.csv000066400000000000000000000463771500700540700153150ustar00rootroot00000000000000#mbytes,method,test_case,backend,time,mb_per_sec 0.954, basic_parse, long_list, python, 1.331, 0.716 0.954, basic_parse, long_list, yajl2, 0.978, 0.975 0.954, basic_parse, long_list, yajl2_c, 0.077, 12.324 10.278, basic_parse, big_int_object, python, 2.780, 3.697 10.278, basic_parse, big_int_object, yajl2, 1.845, 5.571 10.278, basic_parse, big_int_object, yajl2_c, 0.150, 68.385 11.232, basic_parse, big_decimal_object, python, 2.969, 3.783 11.232, basic_parse, big_decimal_object, yajl2, 1.851, 6.069 11.232, basic_parse, big_decimal_object, yajl2_c, 0.243, 46.182 9.431, basic_parse, big_null_object, python, 2.537, 3.718 9.431, basic_parse, big_null_object, yajl2, 0.955, 9.873 9.431, basic_parse, big_null_object, yajl2_c, 0.105, 89.630 9.669, basic_parse, big_bool_object, python, 2.454, 3.940 9.669, basic_parse, big_bool_object, yajl2, 1.063, 9.094 9.669, basic_parse, big_bool_object, yajl2_c, 0.117, 82.632 14.093, basic_parse, big_str_object, python, 2.830, 4.980 14.093, basic_parse, big_str_object, yajl2, 1.484, 9.498 14.093, basic_parse, big_str_object, yajl2_c, 0.124, 113.655 40.425, basic_parse, big_longstr_object, python, 2.972, 13.600 40.425, basic_parse, big_longstr_object, yajl2, 1.487, 27.195 40.425, basic_parse, big_longstr_object, yajl2_c, 0.128, 315.975 96.321, basic_parse, object_with_10_keys, python, 28.759, 3.349 96.321, basic_parse, object_with_10_keys, yajl2, 14.782, 6.516 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.344, 71.645 1.907, basic_parse, empty_lists, python, 1.389, 1.373 1.907, basic_parse, empty_lists, yajl2, 0.404, 4.716 1.907, basic_parse, empty_lists, yajl2_c, 0.092, 20.730 1.907, basic_parse, empty_objects, python, 1.401, 1.362 1.907, basic_parse, empty_objects, yajl2, 0.398, 4.791 1.907, basic_parse, empty_objects, yajl2_c, 0.090, 21.287 0.954, basic_parse, long_list, python, 1.277, 0.747 0.954, basic_parse, long_list, yajl2, 0.911, 1.047 0.954, basic_parse, long_list, yajl2_c, 0.077, 12.316 10.278, basic_parse, big_int_object, python, 2.704, 3.801 10.278, basic_parse, big_int_object, yajl2, 2.350, 4.374 10.278, basic_parse, big_int_object, yajl2_c, 0.147, 70.118 11.232, basic_parse, big_decimal_object, python, 2.881, 3.899 11.232, basic_parse, big_decimal_object, yajl2, 1.777, 6.319 11.232, basic_parse, big_decimal_object, yajl2_c, 0.253, 44.366 9.431, basic_parse, big_null_object, python, 2.496, 3.779 9.431, basic_parse, big_null_object, yajl2, 0.906, 10.409 9.431, basic_parse, big_null_object, yajl2_c, 0.106, 88.968 9.669, basic_parse, big_bool_object, python, 2.423, 3.990 9.669, basic_parse, big_bool_object, yajl2, 1.000, 9.671 9.669, basic_parse, big_bool_object, yajl2_c, 0.119, 81.382 14.093, basic_parse, big_str_object, python, 2.721, 5.179 14.093, basic_parse, big_str_object, yajl2, 1.416, 9.951 14.093, basic_parse, big_str_object, yajl2_c, 0.124, 113.303 40.425, basic_parse, big_longstr_object, python, 2.871, 14.078 40.425, basic_parse, big_longstr_object, yajl2, 1.449, 27.899 40.425, basic_parse, big_longstr_object, yajl2_c, 0.131, 308.726 96.321, basic_parse, object_with_10_keys, python, 27.844, 3.459 96.321, basic_parse, object_with_10_keys, yajl2, 14.668, 6.567 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.356, 71.024 1.907, basic_parse, empty_lists, python, 1.401, 1.362 1.907, basic_parse, empty_lists, yajl2, 0.403, 4.727 1.907, basic_parse, empty_lists, yajl2_c, 0.097, 19.683 1.907, basic_parse, empty_objects, python, 1.412, 1.351 1.907, basic_parse, empty_objects, yajl2, 0.398, 4.787 1.907, basic_parse, empty_objects, yajl2_c, 0.089, 21.457 0.954, basic_parse, long_list, python, 1.277, 0.747 0.954, basic_parse, long_list, yajl2, 0.937, 1.017 0.954, basic_parse, long_list, yajl2_c, 0.079, 12.061 10.278, basic_parse, big_int_object, python, 2.651, 3.877 10.278, basic_parse, big_int_object, yajl2, 2.334, 4.404 10.278, basic_parse, big_int_object, yajl2_c, 0.147, 70.047 11.232, basic_parse, big_decimal_object, python, 2.849, 3.942 11.232, basic_parse, big_decimal_object, yajl2, 1.848, 6.079 11.232, basic_parse, big_decimal_object, yajl2_c, 0.260, 43.123 9.431, basic_parse, big_null_object, python, 2.433, 3.876 9.431, basic_parse, big_null_object, yajl2, 0.921, 10.238 9.431, basic_parse, big_null_object, yajl2_c, 0.106, 88.759 9.669, basic_parse, big_bool_object, python, 2.358, 4.101 9.669, basic_parse, big_bool_object, yajl2, 1.015, 9.525 9.669, basic_parse, big_bool_object, yajl2_c, 0.118, 82.069 14.093, basic_parse, big_str_object, python, 2.645, 5.329 14.093, basic_parse, big_str_object, yajl2, 1.427, 9.879 14.093, basic_parse, big_str_object, yajl2_c, 0.124, 113.494 40.425, basic_parse, big_longstr_object, python, 2.805, 14.414 40.425, basic_parse, big_longstr_object, yajl2, 1.467, 27.557 40.425, basic_parse, big_longstr_object, yajl2_c, 0.128, 316.739 96.321, basic_parse, object_with_10_keys, python, 27.124, 3.551 96.321, basic_parse, object_with_10_keys, yajl2, 14.835, 6.493 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.368, 70.414 1.907, basic_parse, empty_lists, python, 1.370, 1.392 1.907, basic_parse, empty_lists, yajl2, 0.401, 4.754 1.907, basic_parse, empty_lists, yajl2_c, 0.097, 19.602 1.907, basic_parse, empty_objects, python, 1.380, 1.382 1.907, basic_parse, empty_objects, yajl2, 0.398, 4.791 1.907, basic_parse, empty_objects, yajl2_c, 0.101, 18.952 0.954, parse, long_list, python, 1.415, 0.674 0.954, parse, long_list, yajl2, 1.126, 0.847 0.954, parse, long_list, yajl2_c, 0.087, 10.925 10.278, parse, big_int_object, python, 2.947, 3.488 10.278, parse, big_int_object, yajl2, 2.784, 3.692 10.278, parse, big_int_object, yajl2_c, 0.176, 58.336 11.232, parse, big_decimal_object, python, 3.142, 3.575 11.232, parse, big_decimal_object, yajl2, 2.318, 4.845 11.232, parse, big_decimal_object, yajl2_c, 0.278, 40.389 9.431, parse, big_null_object, python, 2.722, 3.464 9.431, parse, big_null_object, yajl2, 1.248, 7.557 9.431, parse, big_null_object, yajl2_c, 0.135, 69.694 9.669, parse, big_bool_object, python, 2.663, 3.631 9.669, parse, big_bool_object, yajl2, 1.349, 7.167 9.669, parse, big_bool_object, yajl2_c, 0.145, 66.678 14.093, parse, big_str_object, python, 2.932, 4.806 14.093, parse, big_str_object, yajl2, 1.824, 7.725 14.093, parse, big_str_object, yajl2_c, 0.156, 90.305 40.425, parse, big_longstr_object, python, 3.110, 12.999 40.425, parse, big_longstr_object, yajl2, 1.856, 21.779 40.425, parse, big_longstr_object, yajl2_c, 0.151, 267.206 96.321, parse, object_with_10_keys, python, 30.939, 3.113 96.321, parse, object_with_10_keys, yajl2, 19.392, 4.967 96.321, parse, object_with_10_keys, yajl2_c, 2.057, 46.835 1.907, parse, empty_lists, python, 1.766, 1.080 1.907, parse, empty_lists, yajl2, 0.721, 2.645 1.907, parse, empty_lists, yajl2_c, 0.148, 12.848 1.907, parse, empty_objects, python, 1.749, 1.091 1.907, parse, empty_objects, yajl2, 0.653, 2.922 1.907, parse, empty_objects, yajl2_c, 0.109, 17.559 0.954, parse, long_list, python, 1.426, 0.669 0.954, parse, long_list, yajl2, 1.088, 0.876 0.954, parse, long_list, yajl2_c, 0.089, 10.671 10.278, parse, big_int_object, python, 2.968, 3.463 10.278, parse, big_int_object, yajl2, 2.112, 4.866 10.278, parse, big_int_object, yajl2_c, 0.201, 51.140 11.232, parse, big_decimal_object, python, 3.220, 3.488 11.232, parse, big_decimal_object, yajl2, 2.209, 5.084 11.232, parse, big_decimal_object, yajl2_c, 0.280, 40.121 9.431, parse, big_null_object, python, 2.749, 3.430 9.431, parse, big_null_object, yajl2, 1.203, 7.840 9.431, parse, big_null_object, yajl2_c, 0.135, 69.700 9.669, parse, big_bool_object, python, 2.675, 3.615 9.669, parse, big_bool_object, yajl2, 1.288, 7.509 9.669, parse, big_bool_object, yajl2_c, 0.146, 66.184 14.093, parse, big_str_object, python, 2.925, 4.819 14.093, parse, big_str_object, yajl2, 1.704, 8.269 14.093, parse, big_str_object, yajl2_c, 0.156, 90.259 40.425, parse, big_longstr_object, python, 3.087, 13.096 40.425, parse, big_longstr_object, yajl2, 1.756, 23.021 40.425, parse, big_longstr_object, yajl2_c, 0.152, 265.778 96.321, parse, object_with_10_keys, python, 31.228, 3.084 96.321, parse, object_with_10_keys, yajl2, 18.520, 5.201 96.321, parse, object_with_10_keys, yajl2_c, 2.069, 46.549 1.907, parse, empty_lists, python, 1.783, 1.070 1.907, parse, empty_lists, yajl2, 0.701, 2.720 1.907, parse, empty_lists, yajl2_c, 0.145, 13.119 1.907, parse, empty_objects, python, 1.743, 1.094 1.907, parse, empty_objects, yajl2, 0.651, 2.932 1.907, parse, empty_objects, yajl2_c, 0.111, 17.210 0.954, parse, long_list, python, 1.403, 0.680 0.954, parse, long_list, yajl2, 1.082, 0.882 0.954, parse, long_list, yajl2_c, 0.086, 11.133 10.278, parse, big_int_object, python, 2.939, 3.498 10.278, parse, big_int_object, yajl2, 2.667, 3.854 10.278, parse, big_int_object, yajl2_c, 0.176, 58.464 11.232, parse, big_decimal_object, python, 3.108, 3.614 11.232, parse, big_decimal_object, yajl2, 2.196, 5.116 11.232, parse, big_decimal_object, yajl2_c, 0.279, 40.213 9.431, parse, big_null_object, python, 2.702, 3.490 9.431, parse, big_null_object, yajl2, 1.209, 7.800 9.431, parse, big_null_object, yajl2_c, 0.134, 70.563 9.669, parse, big_bool_object, python, 2.653, 3.645 9.669, parse, big_bool_object, yajl2, 1.281, 7.547 9.669, parse, big_bool_object, yajl2_c, 0.146, 66.400 14.093, parse, big_str_object, python, 2.925, 4.818 14.093, parse, big_str_object, yajl2, 1.745, 8.076 14.093, parse, big_str_object, yajl2_c, 0.154, 91.303 40.425, parse, big_longstr_object, python, 3.102, 13.033 40.425, parse, big_longstr_object, yajl2, 1.775, 22.771 40.425, parse, big_longstr_object, yajl2_c, 0.151, 267.658 96.321, parse, object_with_10_keys, python, 30.655, 3.142 96.321, parse, object_with_10_keys, yajl2, 18.543, 5.195 96.321, parse, object_with_10_keys, yajl2_c, 2.074, 46.438 1.907, parse, empty_lists, python, 1.765, 1.081 1.907, parse, empty_lists, yajl2, 0.720, 2.651 1.907, parse, empty_lists, yajl2_c, 0.144, 13.270 1.907, parse, empty_objects, python, 1.735, 1.099 1.907, parse, empty_objects, yajl2, 0.658, 2.897 1.907, parse, empty_objects, yajl2_c, 0.108, 17.686 0.954, kvitems, long_list, python, 1.384, 0.689 0.954, kvitems, long_list, yajl2, 1.063, 0.897 0.954, kvitems, long_list, yajl2_c, 0.046, 20.516 10.278, kvitems, big_int_object, python, 3.861, 2.662 10.278, kvitems, big_int_object, yajl2, 2.991, 3.437 10.278, kvitems, big_int_object, yajl2_c, 0.160, 64.334 11.232, kvitems, big_decimal_object, python, 4.047, 2.775 11.232, kvitems, big_decimal_object, yajl2, 3.069, 3.659 11.232, kvitems, big_decimal_object, yajl2_c, 0.258, 43.619 9.431, kvitems, big_null_object, python, 3.513, 2.684 9.431, kvitems, big_null_object, yajl2, 2.059, 4.580 9.431, kvitems, big_null_object, yajl2_c, 0.117, 80.738 9.669, kvitems, big_bool_object, python, 3.498, 2.764 9.669, kvitems, big_bool_object, yajl2, 2.166, 4.464 9.669, kvitems, big_bool_object, yajl2_c, 0.127, 75.897 14.093, kvitems, big_str_object, python, 3.789, 3.720 14.093, kvitems, big_str_object, yajl2, 2.597, 5.427 14.093, kvitems, big_str_object, yajl2_c, 0.124, 113.333 40.425, kvitems, big_longstr_object, python, 3.994, 10.120 40.425, kvitems, big_longstr_object, yajl2, 2.633, 15.353 40.425, kvitems, big_longstr_object, yajl2_c, 0.153, 264.141 96.321, kvitems, object_with_10_keys, python, 31.773, 3.032 96.321, kvitems, object_with_10_keys, yajl2, 19.105, 5.042 96.321, kvitems, object_with_10_keys, yajl2_c, 1.398, 68.878 1.907, kvitems, empty_lists, python, 1.731, 1.102 1.907, kvitems, empty_lists, yajl2, 0.685, 2.786 1.907, kvitems, empty_lists, yajl2_c, 0.081, 23.503 1.907, kvitems, empty_objects, python, 1.719, 1.110 1.907, kvitems, empty_objects, yajl2, 0.638, 2.989 1.907, kvitems, empty_objects, yajl2_c, 0.049, 38.989 0.954, kvitems, long_list, python, 1.399, 0.682 0.954, kvitems, long_list, yajl2, 1.021, 0.934 0.954, kvitems, long_list, yajl2_c, 0.046, 20.669 10.278, kvitems, big_int_object, python, 4.266, 2.410 10.278, kvitems, big_int_object, yajl2, 3.191, 3.221 10.278, kvitems, big_int_object, yajl2_c, 0.158, 65.209 11.232, kvitems, big_decimal_object, python, 4.133, 2.718 11.232, kvitems, big_decimal_object, yajl2, 3.006, 3.737 11.232, kvitems, big_decimal_object, yajl2_c, 0.259, 43.353 9.431, kvitems, big_null_object, python, 3.535, 2.668 9.431, kvitems, big_null_object, yajl2, 2.040, 4.624 9.431, kvitems, big_null_object, yajl2_c, 0.116, 81.083 9.669, kvitems, big_bool_object, python, 3.480, 2.779 9.669, kvitems, big_bool_object, yajl2, 2.140, 4.519 9.669, kvitems, big_bool_object, yajl2_c, 0.127, 75.977 14.093, kvitems, big_str_object, python, 3.750, 3.759 14.093, kvitems, big_str_object, yajl2, 2.550, 5.526 14.093, kvitems, big_str_object, yajl2_c, 0.127, 110.920 40.425, kvitems, big_longstr_object, python, 3.984, 10.148 40.425, kvitems, big_longstr_object, yajl2, 2.617, 15.448 40.425, kvitems, big_longstr_object, yajl2_c, 0.154, 262.579 96.321, kvitems, object_with_10_keys, python, 31.667, 3.042 96.321, kvitems, object_with_10_keys, yajl2, 18.280, 5.269 96.321, kvitems, object_with_10_keys, yajl2_c, 1.386, 69.508 1.907, kvitems, empty_lists, python, 1.737, 1.098 1.907, kvitems, empty_lists, yajl2, 0.675, 2.827 1.907, kvitems, empty_lists, yajl2_c, 0.081, 23.418 1.907, kvitems, empty_objects, python, 1.754, 1.087 1.907, kvitems, empty_objects, yajl2, 0.638, 2.989 1.907, kvitems, empty_objects, yajl2_c, 0.049, 38.582 0.954, kvitems, long_list, python, 1.418, 0.672 0.954, kvitems, long_list, yajl2, 1.062, 0.898 0.954, kvitems, long_list, yajl2_c, 0.046, 20.625 10.278, kvitems, big_int_object, python, 4.531, 2.268 10.278, kvitems, big_int_object, yajl2, 3.079, 3.338 10.278, kvitems, big_int_object, yajl2_c, 0.158, 65.042 11.232, kvitems, big_decimal_object, python, 4.252, 2.642 11.232, kvitems, big_decimal_object, yajl2, 3.176, 3.536 11.232, kvitems, big_decimal_object, yajl2_c, 0.262, 42.900 9.431, kvitems, big_null_object, python, 3.606, 2.615 9.431, kvitems, big_null_object, yajl2, 2.091, 4.509 9.431, kvitems, big_null_object, yajl2_c, 0.117, 80.600 9.669, kvitems, big_bool_object, python, 3.538, 2.733 9.669, kvitems, big_bool_object, yajl2, 2.200, 4.395 9.669, kvitems, big_bool_object, yajl2_c, 0.128, 75.585 14.093, kvitems, big_str_object, python, 3.824, 3.686 14.093, kvitems, big_str_object, yajl2, 2.595, 5.430 14.093, kvitems, big_str_object, yajl2_c, 0.124, 113.757 40.425, kvitems, big_longstr_object, python, 4.026, 10.040 40.425, kvitems, big_longstr_object, yajl2, 2.645, 15.285 40.425, kvitems, big_longstr_object, yajl2_c, 0.153, 264.368 96.321, kvitems, object_with_10_keys, python, 32.490, 2.965 96.321, kvitems, object_with_10_keys, yajl2, 18.816, 5.119 96.321, kvitems, object_with_10_keys, yajl2_c, 1.370, 70.325 1.907, kvitems, empty_lists, python, 1.748, 1.091 1.907, kvitems, empty_lists, yajl2, 0.687, 2.774 1.907, kvitems, empty_lists, yajl2_c, 0.082, 23.264 1.907, kvitems, empty_objects, python, 1.750, 1.090 1.907, kvitems, empty_objects, yajl2, 0.646, 2.952 1.907, kvitems, empty_objects, yajl2_c, 0.049, 39.013 0.954, items, long_list, python, 1.642, 0.581 0.954, items, long_list, yajl2, 1.384, 0.689 0.954, items, long_list, yajl2_c, 0.060, 15.794 10.278, items, big_int_object, python, 3.677, 2.795 10.278, items, big_int_object, yajl2, 2.877, 3.573 10.278, items, big_int_object, yajl2_c, 0.261, 39.381 11.232, items, big_decimal_object, python, 3.849, 2.918 11.232, items, big_decimal_object, yajl2, 3.032, 3.705 11.232, items, big_decimal_object, yajl2_c, 0.398, 28.247 9.431, items, big_null_object, python, 3.317, 2.843 9.431, items, big_null_object, yajl2, 1.885, 5.003 9.431, items, big_null_object, yajl2_c, 0.202, 46.577 9.669, items, big_bool_object, python, 3.250, 2.975 9.669, items, big_bool_object, yajl2, 2.007, 4.818 9.669, items, big_bool_object, yajl2_c, 0.206, 47.051 14.093, items, big_str_object, python, 3.606, 3.909 14.093, items, big_str_object, yajl2, 2.551, 5.525 14.093, items, big_str_object, yajl2_c, 0.238, 59.234 40.425, items, big_longstr_object, python, 3.868, 10.452 40.425, items, big_longstr_object, yajl2, 2.616, 15.453 40.425, items, big_longstr_object, yajl2_c, 0.284, 142.489 96.321, items, object_with_10_keys, python, 38.865, 2.478 96.321, items, object_with_10_keys, yajl2, 27.483, 3.505 96.321, items, object_with_10_keys, yajl2_c, 2.568, 37.505 1.907, items, empty_lists, python, 2.423, 0.787 1.907, items, empty_lists, yajl2, 1.377, 1.385 1.907, items, empty_lists, yajl2_c, 0.284, 6.714 1.907, items, empty_objects, python, 2.393, 0.797 1.907, items, empty_objects, yajl2, 1.286, 1.484 1.907, items, empty_objects, yajl2_c, 0.157, 12.131 0.954, items, long_list, python, 1.602, 0.595 0.954, items, long_list, yajl2, 1.253, 0.761 0.954, items, long_list, yajl2_c, 0.052, 18.172 10.278, items, big_int_object, python, 4.124, 2.492 10.278, items, big_int_object, yajl2, 2.694, 3.816 10.278, items, big_int_object, yajl2_c, 0.240, 42.825 11.232, items, big_decimal_object, python, 3.861, 2.910 11.232, items, big_decimal_object, yajl2, 2.863, 3.924 11.232, items, big_decimal_object, yajl2_c, 0.355, 31.656 9.431, items, big_null_object, python, 3.309, 2.850 9.431, items, big_null_object, yajl2, 1.744, 5.409 9.431, items, big_null_object, yajl2_c, 0.186, 50.587 9.669, items, big_bool_object, python, 3.253, 2.972 9.669, items, big_bool_object, yajl2, 1.892, 5.111 9.669, items, big_bool_object, yajl2_c, 0.188, 51.423 14.093, items, big_str_object, python, 3.587, 3.928 14.093, items, big_str_object, yajl2, 2.300, 6.126 14.093, items, big_str_object, yajl2_c, 0.226, 62.340 40.425, items, big_longstr_object, python, 3.820, 10.583 40.425, items, big_longstr_object, yajl2, 2.401, 16.837 40.425, items, big_longstr_object, yajl2_c, 0.269, 150.113 96.321, items, object_with_10_keys, python, 37.733, 2.553 96.321, items, object_with_10_keys, yajl2, 24.460, 3.938 96.321, items, object_with_10_keys, yajl2_c, 2.269, 42.453 1.907, items, empty_lists, python, 2.379, 0.802 1.907, items, empty_lists, yajl2, 1.360, 1.403 1.907, items, empty_lists, yajl2_c, 0.276, 6.917 1.907, items, empty_objects, python, 2.373, 0.804 1.907, items, empty_objects, yajl2, 1.282, 1.487 1.907, items, empty_objects, yajl2_c, 0.150, 12.728 0.954, items, long_list, python, 1.600, 0.596 0.954, items, long_list, yajl2, 1.268, 0.752 0.954, items, long_list, yajl2_c, 0.053, 17.996 10.278, items, big_int_object, python, 4.162, 2.470 10.278, items, big_int_object, yajl2, 2.753, 3.733 10.278, items, big_int_object, yajl2_c, 0.241, 42.670 11.232, items, big_decimal_object, python, 3.875, 2.898 11.232, items, big_decimal_object, yajl2, 2.907, 3.864 11.232, items, big_decimal_object, yajl2_c, 0.360, 31.228 9.431, items, big_null_object, python, 3.317, 2.843 9.431, items, big_null_object, yajl2, 1.841, 5.122 9.431, items, big_null_object, yajl2_c, 0.189, 49.869 9.669, items, big_bool_object, python, 3.283, 2.946 9.669, items, big_bool_object, yajl2, 1.934, 5.000 9.669, items, big_bool_object, yajl2_c, 0.191, 50.754 14.093, items, big_str_object, python, 3.554, 3.966 14.093, items, big_str_object, yajl2, 2.397, 5.880 14.093, items, big_str_object, yajl2_c, 0.225, 62.642 40.425, items, big_longstr_object, python, 3.811, 10.608 40.425, items, big_longstr_object, yajl2, 2.489, 16.242 40.425, items, big_longstr_object, yajl2_c, 0.271, 148.938 96.321, items, object_with_10_keys, python, 37.847, 2.545 96.321, items, object_with_10_keys, yajl2, 25.621, 3.759 96.321, items, object_with_10_keys, yajl2_c, 2.290, 42.064 1.907, items, empty_lists, python, 2.414, 0.790 1.907, items, empty_lists, yajl2, 1.388, 1.374 1.907, items, empty_lists, yajl2_c, 0.273, 6.982 1.907, items, empty_objects, python, 2.412, 0.791 1.907, items, empty_objects, yajl2, 1.306, 1.461 1.907, items, empty_objects, yajl2_c, 0.150, 12.743 ijson-3.4.0/notes/old.csv000066400000000000000000000463561500700540700152770ustar00rootroot00000000000000#mbytes,method,test_case,backend,time,mb_per_sec 0.954, basic_parse, long_list, python, 1.216, 0.784 0.954, basic_parse, long_list, yajl2, 0.934, 1.022 0.954, basic_parse, long_list, yajl2_c, 0.077, 12.331 10.278, basic_parse, big_int_object, python, 2.602, 3.950 10.278, basic_parse, big_int_object, yajl2, 2.387, 4.307 10.278, basic_parse, big_int_object, yajl2_c, 0.152, 67.801 11.232, basic_parse, big_decimal_object, python, 2.700, 4.161 11.232, basic_parse, big_decimal_object, yajl2, 1.841, 6.100 11.232, basic_parse, big_decimal_object, yajl2_c, 0.253, 44.418 9.431, basic_parse, big_null_object, python, 2.335, 4.038 9.431, basic_parse, big_null_object, yajl2, 0.929, 10.149 9.431, basic_parse, big_null_object, yajl2_c, 0.113, 83.529 9.669, basic_parse, big_bool_object, python, 2.317, 4.174 9.669, basic_parse, big_bool_object, yajl2, 1.020, 9.483 9.669, basic_parse, big_bool_object, yajl2_c, 0.122, 79.066 14.093, basic_parse, big_str_object, python, 2.607, 5.405 14.093, basic_parse, big_str_object, yajl2, 1.438, 9.798 14.093, basic_parse, big_str_object, yajl2_c, 0.130, 108.284 40.425, basic_parse, big_longstr_object, python, 2.730, 14.808 40.425, basic_parse, big_longstr_object, yajl2, 1.473, 27.440 40.425, basic_parse, big_longstr_object, yajl2_c, 0.134, 301.499 96.321, basic_parse, object_with_10_keys, python, 28.131, 3.424 96.321, basic_parse, object_with_10_keys, yajl2, 15.014, 6.415 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.434, 67.175 1.907, basic_parse, empty_lists, python, 1.486, 1.284 1.907, basic_parse, empty_lists, yajl2, 0.424, 4.498 1.907, basic_parse, empty_lists, yajl2_c, 0.095, 20.125 1.907, basic_parse, empty_objects, python, 1.489, 1.281 1.907, basic_parse, empty_objects, yajl2, 0.422, 4.519 1.907, basic_parse, empty_objects, yajl2_c, 0.096, 19.852 0.954, basic_parse, long_list, python, 1.218, 0.783 0.954, basic_parse, long_list, yajl2, 0.977, 0.976 0.954, basic_parse, long_list, yajl2_c, 0.078, 12.163 10.278, basic_parse, big_int_object, python, 2.621, 3.921 10.278, basic_parse, big_int_object, yajl2, 2.442, 4.208 10.278, basic_parse, big_int_object, yajl2_c, 0.176, 58.551 11.232, basic_parse, big_decimal_object, python, 2.839, 3.957 11.232, basic_parse, big_decimal_object, yajl2, 1.925, 5.834 11.232, basic_parse, big_decimal_object, yajl2_c, 0.255, 44.048 9.431, basic_parse, big_null_object, python, 2.386, 3.952 9.431, basic_parse, big_null_object, yajl2, 0.955, 9.879 9.431, basic_parse, big_null_object, yajl2_c, 0.114, 83.068 9.669, basic_parse, big_bool_object, python, 2.339, 4.134 9.669, basic_parse, big_bool_object, yajl2, 1.071, 9.025 9.669, basic_parse, big_bool_object, yajl2_c, 0.122, 79.234 14.093, basic_parse, big_str_object, python, 2.672, 5.274 14.093, basic_parse, big_str_object, yajl2, 1.506, 9.356 14.093, basic_parse, big_str_object, yajl2_c, 0.132, 107.023 40.425, basic_parse, big_longstr_object, python, 2.800, 14.435 40.425, basic_parse, big_longstr_object, yajl2, 1.529, 26.432 40.425, basic_parse, big_longstr_object, yajl2_c, 0.135, 298.699 96.321, basic_parse, object_with_10_keys, python, 29.161, 3.303 96.321, basic_parse, object_with_10_keys, yajl2, 15.523, 6.205 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.428, 67.442 1.907, basic_parse, empty_lists, python, 1.490, 1.280 1.907, basic_parse, empty_lists, yajl2, 0.415, 4.599 1.907, basic_parse, empty_lists, yajl2_c, 0.095, 20.104 1.907, basic_parse, empty_objects, python, 1.469, 1.298 1.907, basic_parse, empty_objects, yajl2, 0.413, 4.618 1.907, basic_parse, empty_objects, yajl2_c, 0.095, 19.981 0.954, basic_parse, long_list, python, 1.222, 0.780 0.954, basic_parse, long_list, yajl2, 0.986, 0.967 0.954, basic_parse, long_list, yajl2_c, 0.078, 12.274 10.278, basic_parse, big_int_object, python, 2.587, 3.973 10.278, basic_parse, big_int_object, yajl2, 1.793, 5.734 10.278, basic_parse, big_int_object, yajl2_c, 0.156, 65.943 11.232, basic_parse, big_decimal_object, python, 2.726, 4.120 11.232, basic_parse, big_decimal_object, yajl2, 1.878, 5.979 11.232, basic_parse, big_decimal_object, yajl2_c, 0.256, 43.955 9.431, basic_parse, big_null_object, python, 2.333, 4.042 9.431, basic_parse, big_null_object, yajl2, 0.947, 9.963 9.431, basic_parse, big_null_object, yajl2_c, 0.122, 77.363 9.669, basic_parse, big_bool_object, python, 2.300, 4.204 9.669, basic_parse, big_bool_object, yajl2, 1.062, 9.105 9.669, basic_parse, big_bool_object, yajl2_c, 0.126, 76.706 14.093, basic_parse, big_str_object, python, 2.612, 5.395 14.093, basic_parse, big_str_object, yajl2, 1.479, 9.531 14.093, basic_parse, big_str_object, yajl2_c, 0.134, 104.899 40.425, basic_parse, big_longstr_object, python, 2.705, 14.947 40.425, basic_parse, big_longstr_object, yajl2, 1.506, 26.836 40.425, basic_parse, big_longstr_object, yajl2_c, 0.141, 286.686 96.321, basic_parse, object_with_10_keys, python, 28.038, 3.435 96.321, basic_parse, object_with_10_keys, yajl2, 15.295, 6.298 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.461, 65.907 1.907, basic_parse, empty_lists, python, 1.480, 1.288 1.907, basic_parse, empty_lists, yajl2, 0.423, 4.513 1.907, basic_parse, empty_lists, yajl2_c, 0.097, 19.611 1.907, basic_parse, empty_objects, python, 1.493, 1.278 1.907, basic_parse, empty_objects, yajl2, 0.419, 4.554 1.907, basic_parse, empty_objects, yajl2_c, 0.095, 19.979 0.954, parse, long_list, python, 1.342, 0.711 0.954, parse, long_list, yajl2, 1.066, 0.895 0.954, parse, long_list, yajl2_c, 0.098, 9.752 10.278, parse, big_int_object, python, 2.881, 3.568 10.278, parse, big_int_object, yajl2, 2.015, 5.102 10.278, parse, big_int_object, yajl2_c, 0.204, 50.394 11.232, parse, big_decimal_object, python, 3.086, 3.640 11.232, parse, big_decimal_object, yajl2, 2.081, 5.398 11.232, parse, big_decimal_object, yajl2_c, 0.305, 36.789 9.431, parse, big_null_object, python, 2.661, 3.544 9.431, parse, big_null_object, yajl2, 1.193, 7.908 9.431, parse, big_null_object, yajl2_c, 0.161, 58.604 9.669, parse, big_bool_object, python, 2.675, 3.615 9.669, parse, big_bool_object, yajl2, 1.300, 7.439 9.669, parse, big_bool_object, yajl2_c, 0.168, 57.546 14.093, parse, big_str_object, python, 2.996, 4.704 14.093, parse, big_str_object, yajl2, 1.746, 8.072 14.093, parse, big_str_object, yajl2_c, 0.171, 82.554 40.425, parse, big_longstr_object, python, 3.043, 13.282 40.425, parse, big_longstr_object, yajl2, 1.759, 22.980 40.425, parse, big_longstr_object, yajl2_c, 0.173, 233.269 96.321, parse, object_with_10_keys, python, 31.998, 3.010 96.321, parse, object_with_10_keys, yajl2, 18.427, 5.227 96.321, parse, object_with_10_keys, yajl2_c, 2.275, 42.335 1.907, parse, empty_lists, python, 1.838, 1.038 1.907, parse, empty_lists, yajl2, 0.683, 2.791 1.907, parse, empty_lists, yajl2_c, 0.171, 11.179 1.907, parse, empty_objects, python, 1.706, 1.118 1.907, parse, empty_objects, yajl2, 0.656, 2.907 1.907, parse, empty_objects, yajl2_c, 0.148, 12.912 0.954, parse, long_list, python, 1.349, 0.707 0.954, parse, long_list, yajl2, 1.048, 0.910 0.954, parse, long_list, yajl2_c, 0.098, 9.744 10.278, parse, big_int_object, python, 3.138, 3.275 10.278, parse, big_int_object, yajl2, 2.256, 4.557 10.278, parse, big_int_object, yajl2_c, 0.205, 50.119 11.232, parse, big_decimal_object, python, 3.061, 3.670 11.232, parse, big_decimal_object, yajl2, 2.055, 5.466 11.232, parse, big_decimal_object, yajl2_c, 0.307, 36.615 9.431, parse, big_null_object, python, 2.612, 3.611 9.431, parse, big_null_object, yajl2, 1.150, 8.203 9.431, parse, big_null_object, yajl2_c, 0.161, 58.438 9.669, parse, big_bool_object, python, 2.585, 3.741 9.669, parse, big_bool_object, yajl2, 1.251, 7.728 9.669, parse, big_bool_object, yajl2_c, 0.169, 57.167 14.093, parse, big_str_object, python, 2.886, 4.884 14.093, parse, big_str_object, yajl2, 1.661, 8.484 14.093, parse, big_str_object, yajl2_c, 0.174, 80.967 40.425, parse, big_longstr_object, python, 3.001, 13.471 40.425, parse, big_longstr_object, yajl2, 1.695, 23.846 40.425, parse, big_longstr_object, yajl2_c, 0.174, 232.151 96.321, parse, object_with_10_keys, python, 31.260, 3.081 96.321, parse, object_with_10_keys, yajl2, 17.703, 5.441 96.321, parse, object_with_10_keys, yajl2_c, 2.272, 42.390 1.907, parse, empty_lists, python, 1.819, 1.048 1.907, parse, empty_lists, yajl2, 0.667, 2.859 1.907, parse, empty_lists, yajl2_c, 0.174, 10.986 1.907, parse, empty_objects, python, 1.735, 1.099 1.907, parse, empty_objects, yajl2, 0.622, 3.068 1.907, parse, empty_objects, yajl2_c, 0.148, 12.921 0.954, parse, long_list, python, 1.317, 0.724 0.954, parse, long_list, yajl2, 1.027, 0.928 0.954, parse, long_list, yajl2_c, 0.097, 9.833 10.278, parse, big_int_object, python, 2.849, 3.607 10.278, parse, big_int_object, yajl2, 2.522, 4.076 10.278, parse, big_int_object, yajl2_c, 0.205, 50.106 11.232, parse, big_decimal_object, python, 3.051, 3.681 11.232, parse, big_decimal_object, yajl2, 2.030, 5.532 11.232, parse, big_decimal_object, yajl2_c, 0.307, 36.630 9.431, parse, big_null_object, python, 2.595, 3.634 9.431, parse, big_null_object, yajl2, 1.132, 8.329 9.431, parse, big_null_object, yajl2_c, 0.162, 58.292 9.669, parse, big_bool_object, python, 2.565, 3.769 9.669, parse, big_bool_object, yajl2, 1.237, 7.820 9.669, parse, big_bool_object, yajl2_c, 0.169, 57.261 14.093, parse, big_str_object, python, 2.881, 4.892 14.093, parse, big_str_object, yajl2, 1.653, 8.524 14.093, parse, big_str_object, yajl2_c, 0.172, 82.162 40.425, parse, big_longstr_object, python, 3.001, 13.469 40.425, parse, big_longstr_object, yajl2, 1.709, 23.650 40.425, parse, big_longstr_object, yajl2_c, 0.175, 230.897 96.321, parse, object_with_10_keys, python, 31.050, 3.102 96.321, parse, object_with_10_keys, yajl2, 17.577, 5.480 96.321, parse, object_with_10_keys, yajl2_c, 2.275, 42.335 1.907, parse, empty_lists, python, 1.821, 1.048 1.907, parse, empty_lists, yajl2, 0.658, 2.901 1.907, parse, empty_lists, yajl2_c, 0.177, 10.770 1.907, parse, empty_objects, python, 1.723, 1.107 1.907, parse, empty_objects, yajl2, 0.619, 3.082 1.907, parse, empty_objects, yajl2_c, 0.148, 12.881 0.954, kvitems, long_list, python, 1.398, 0.682 0.954, kvitems, long_list, yajl2, 1.088, 0.877 0.954, kvitems, long_list, yajl2_c, 0.104, 9.190 10.278, kvitems, big_int_object, python, 3.898, 2.637 10.278, kvitems, big_int_object, yajl2, 2.631, 3.907 10.278, kvitems, big_int_object, yajl2_c, 0.227, 45.254 11.232, kvitems, big_decimal_object, python, 4.216, 2.664 11.232, kvitems, big_decimal_object, yajl2, 2.731, 4.113 11.232, kvitems, big_decimal_object, yajl2_c, 0.334, 33.661 9.431, kvitems, big_null_object, python, 3.582, 2.633 9.431, kvitems, big_null_object, yajl2, 1.822, 5.177 9.431, kvitems, big_null_object, yajl2_c, 0.190, 49.754 9.669, kvitems, big_bool_object, python, 3.499, 2.763 9.669, kvitems, big_bool_object, yajl2, 1.916, 5.047 9.669, kvitems, big_bool_object, yajl2_c, 0.196, 49.237 14.093, kvitems, big_str_object, python, 3.835, 3.675 14.093, kvitems, big_str_object, yajl2, 2.327, 6.056 14.093, kvitems, big_str_object, yajl2_c, 0.202, 69.738 40.425, kvitems, big_longstr_object, python, 3.975, 10.171 40.425, kvitems, big_longstr_object, yajl2, 2.350, 17.204 40.425, kvitems, big_longstr_object, yajl2_c, 0.206, 195.991 96.321, kvitems, object_with_10_keys, python, 32.431, 2.970 96.321, kvitems, object_with_10_keys, yajl2, 18.863, 5.106 96.321, kvitems, object_with_10_keys, yajl2_c, 2.443, 39.433 1.907, kvitems, empty_lists, python, 1.927, 0.990 1.907, kvitems, empty_lists, yajl2, 0.750, 2.543 1.907, kvitems, empty_lists, yajl2_c, 0.185, 10.287 1.907, kvitems, empty_objects, python, 1.833, 1.040 1.907, kvitems, empty_objects, yajl2, 0.712, 2.679 1.907, kvitems, empty_objects, yajl2_c, 0.157, 12.178 0.954, kvitems, long_list, python, 1.397, 0.683 0.954, kvitems, long_list, yajl2, 1.073, 0.889 0.954, kvitems, long_list, yajl2_c, 0.103, 9.304 10.278, kvitems, big_int_object, python, 4.293, 2.394 10.278, kvitems, big_int_object, yajl2, 2.792, 3.681 10.278, kvitems, big_int_object, yajl2_c, 0.228, 45.143 11.232, kvitems, big_decimal_object, python, 4.091, 2.745 11.232, kvitems, big_decimal_object, yajl2, 2.740, 4.099 11.232, kvitems, big_decimal_object, yajl2_c, 0.336, 33.430 9.431, kvitems, big_null_object, python, 3.554, 2.653 9.431, kvitems, big_null_object, yajl2, 1.822, 5.176 9.431, kvitems, big_null_object, yajl2_c, 0.191, 49.258 9.669, kvitems, big_bool_object, python, 3.459, 2.795 9.669, kvitems, big_bool_object, yajl2, 1.937, 4.991 9.669, kvitems, big_bool_object, yajl2_c, 0.197, 49.116 14.093, kvitems, big_str_object, python, 3.752, 3.757 14.093, kvitems, big_str_object, yajl2, 2.314, 6.091 14.093, kvitems, big_str_object, yajl2_c, 0.204, 69.009 40.425, kvitems, big_longstr_object, python, 4.018, 10.061 40.425, kvitems, big_longstr_object, yajl2, 2.336, 17.302 40.425, kvitems, big_longstr_object, yajl2_c, 0.210, 192.685 96.321, kvitems, object_with_10_keys, python, 32.961, 2.922 96.321, kvitems, object_with_10_keys, yajl2, 18.683, 5.156 96.321, kvitems, object_with_10_keys, yajl2_c, 2.442, 39.445 1.907, kvitems, empty_lists, python, 1.890, 1.009 1.907, kvitems, empty_lists, yajl2, 0.737, 2.587 1.907, kvitems, empty_lists, yajl2_c, 0.187, 10.191 1.907, kvitems, empty_objects, python, 1.826, 1.044 1.907, kvitems, empty_objects, yajl2, 0.697, 2.736 1.907, kvitems, empty_objects, yajl2_c, 0.157, 12.150 0.954, kvitems, long_list, python, 1.369, 0.697 0.954, kvitems, long_list, yajl2, 1.054, 0.905 0.954, kvitems, long_list, yajl2_c, 0.106, 9.034 10.278, kvitems, big_int_object, python, 4.198, 2.448 10.278, kvitems, big_int_object, yajl2, 2.797, 3.674 10.278, kvitems, big_int_object, yajl2_c, 0.227, 45.268 11.232, kvitems, big_decimal_object, python, 4.246, 2.646 11.232, kvitems, big_decimal_object, yajl2, 2.680, 4.191 11.232, kvitems, big_decimal_object, yajl2_c, 0.337, 33.335 9.431, kvitems, big_null_object, python, 3.509, 2.687 9.431, kvitems, big_null_object, yajl2, 1.781, 5.296 9.431, kvitems, big_null_object, yajl2_c, 0.192, 49.222 9.669, kvitems, big_bool_object, python, 3.464, 2.792 9.669, kvitems, big_bool_object, yajl2, 1.892, 5.110 9.669, kvitems, big_bool_object, yajl2_c, 0.200, 48.251 14.093, kvitems, big_str_object, python, 3.766, 3.742 14.093, kvitems, big_str_object, yajl2, 2.285, 6.168 14.093, kvitems, big_str_object, yajl2_c, 0.202, 69.864 40.425, kvitems, big_longstr_object, python, 3.902, 10.360 40.425, kvitems, big_longstr_object, yajl2, 2.318, 17.440 40.425, kvitems, big_longstr_object, yajl2_c, 0.208, 194.403 96.321, kvitems, object_with_10_keys, python, 32.344, 2.978 96.321, kvitems, object_with_10_keys, yajl2, 18.428, 5.227 96.321, kvitems, object_with_10_keys, yajl2_c, 2.363, 40.760 1.907, kvitems, empty_lists, python, 1.902, 1.003 1.907, kvitems, empty_lists, yajl2, 0.739, 2.579 1.907, kvitems, empty_lists, yajl2_c, 0.184, 10.370 1.907, kvitems, empty_objects, python, 1.839, 1.037 1.907, kvitems, empty_objects, yajl2, 0.703, 2.713 1.907, kvitems, empty_objects, yajl2_c, 0.155, 12.329 0.954, items, long_list, python, 1.616, 0.590 0.954, items, long_list, yajl2, 1.254, 0.760 0.954, items, long_list, yajl2_c, 0.111, 8.630 10.278, items, big_int_object, python, 4.178, 2.460 10.278, items, big_int_object, yajl2, 2.498, 4.114 10.278, items, big_int_object, yajl2_c, 0.334, 30.778 11.232, items, big_decimal_object, python, 3.783, 2.969 11.232, items, big_decimal_object, yajl2, 2.641, 4.253 11.232, items, big_decimal_object, yajl2_c, 0.465, 24.156 9.431, items, big_null_object, python, 3.237, 2.913 9.431, items, big_null_object, yajl2, 1.692, 5.574 9.431, items, big_null_object, yajl2_c, 0.283, 33.379 9.669, items, big_bool_object, python, 3.187, 3.034 9.669, items, big_bool_object, yajl2, 1.802, 5.367 9.669, items, big_bool_object, yajl2_c, 0.289, 33.475 14.093, items, big_str_object, python, 3.552, 3.968 14.093, items, big_str_object, yajl2, 2.236, 6.302 14.093, items, big_str_object, yajl2_c, 0.324, 43.561 40.425, items, big_longstr_object, python, 3.798, 10.643 40.425, items, big_longstr_object, yajl2, 2.299, 17.587 40.425, items, big_longstr_object, yajl2_c, 0.346, 117.005 96.321, items, object_with_10_keys, python, 39.468, 2.441 96.321, items, object_with_10_keys, yajl2, 23.463, 4.105 96.321, items, object_with_10_keys, yajl2_c, 3.583, 26.884 1.907, items, empty_lists, python, 2.450, 0.779 1.907, items, empty_lists, yajl2, 1.315, 1.450 1.907, items, empty_lists, yajl2_c, 0.370, 5.161 1.907, items, empty_objects, python, 2.459, 0.776 1.907, items, empty_objects, yajl2, 1.214, 1.572 1.907, items, empty_objects, yajl2_c, 0.269, 7.094 0.954, items, long_list, python, 1.579, 0.604 0.954, items, long_list, yajl2, 1.250, 0.763 0.954, items, long_list, yajl2_c, 0.110, 8.688 10.278, items, big_int_object, python, 3.582, 2.870 10.278, items, big_int_object, yajl2, 2.521, 4.078 10.278, items, big_int_object, yajl2_c, 0.337, 30.476 11.232, items, big_decimal_object, python, 3.832, 2.931 11.232, items, big_decimal_object, yajl2, 2.664, 4.217 11.232, items, big_decimal_object, yajl2_c, 0.465, 24.136 9.431, items, big_null_object, python, 3.237, 2.913 9.431, items, big_null_object, yajl2, 1.722, 5.477 9.431, items, big_null_object, yajl2_c, 0.282, 33.388 9.669, items, big_bool_object, python, 3.175, 3.046 9.669, items, big_bool_object, yajl2, 1.791, 5.398 9.669, items, big_bool_object, yajl2_c, 0.293, 33.018 14.093, items, big_str_object, python, 3.511, 4.014 14.093, items, big_str_object, yajl2, 2.228, 6.325 14.093, items, big_str_object, yajl2_c, 0.323, 43.683 40.425, items, big_longstr_object, python, 3.748, 10.787 40.425, items, big_longstr_object, yajl2, 2.300, 17.575 40.425, items, big_longstr_object, yajl2_c, 0.345, 117.070 96.321, items, object_with_10_keys, python, 38.391, 2.509 96.321, items, object_with_10_keys, yajl2, 23.647, 4.073 96.321, items, object_with_10_keys, yajl2_c, 3.586, 26.858 1.907, items, empty_lists, python, 2.451, 0.778 1.907, items, empty_lists, yajl2, 1.320, 1.445 1.907, items, empty_lists, yajl2_c, 0.366, 5.211 1.907, items, empty_objects, python, 2.465, 0.774 1.907, items, empty_objects, yajl2, 1.219, 1.564 1.907, items, empty_objects, yajl2_c, 0.263, 7.250 0.954, items, long_list, python, 1.582, 0.603 0.954, items, long_list, yajl2, 1.281, 0.744 0.954, items, long_list, yajl2_c, 0.111, 8.627 10.278, items, big_int_object, python, 4.098, 2.508 10.278, items, big_int_object, yajl2, 2.567, 4.005 10.278, items, big_int_object, yajl2_c, 0.336, 30.614 11.232, items, big_decimal_object, python, 3.790, 2.964 11.232, items, big_decimal_object, yajl2, 2.652, 4.236 11.232, items, big_decimal_object, yajl2_c, 0.461, 24.361 9.431, items, big_null_object, python, 3.270, 2.884 9.431, items, big_null_object, yajl2, 1.704, 5.534 9.431, items, big_null_object, yajl2_c, 0.285, 33.079 9.669, items, big_bool_object, python, 3.215, 3.008 9.669, items, big_bool_object, yajl2, 1.827, 5.293 9.669, items, big_bool_object, yajl2_c, 0.296, 32.684 14.093, items, big_str_object, python, 3.509, 4.016 14.093, items, big_str_object, yajl2, 2.270, 6.208 14.093, items, big_str_object, yajl2_c, 0.317, 44.439 40.425, items, big_longstr_object, python, 3.779, 10.696 40.425, items, big_longstr_object, yajl2, 2.311, 17.495 40.425, items, big_longstr_object, yajl2_c, 0.333, 121.372 96.321, items, object_with_10_keys, python, 38.635, 2.493 96.321, items, object_with_10_keys, yajl2, 23.612, 4.079 96.321, items, object_with_10_keys, yajl2_c, 3.573, 26.956 1.907, items, empty_lists, python, 2.468, 0.773 1.907, items, empty_lists, yajl2, 1.335, 1.429 1.907, items, empty_lists, yajl2_c, 0.367, 5.193 1.907, items, empty_objects, python, 2.505, 0.762 1.907, items, empty_objects, yajl2, 1.215, 1.569 1.907, items, empty_objects, yajl2_c, 0.265, 7.201 ijson-3.4.0/notes/performance_comparison.png000066400000000000000000002167231500700540700212420ustar00rootroot00000000000000PNG  IHDR!FsBIT|d pHYsaa?i8tEXtSoftwarematplotlib version3.1.3, http://matplotlib.org/1D IDATx}xLw$b IܵAJ=Y-&FKP[BEUժn EGVݫP(i4jFՒ$d~t4M$3y>k99}rg:g>`2L]"DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DR0bլYeʙx :uʼ.88Xv Sp._%$$ػ@%?+66V:t=ܣ:u(88X;wwi;C!2ܦ˗/+66pۆ _>7n_W5jHsь3t%S˖-aƅCU5{TVrttinݺӪSys=VZi̙9rdY (Æ SxxF>.0@ÇիWb S?Pnq'2J-&&FA?x vښ0a$I]vU`` oڴBBBt)խ[W+ 5`լYSuՋ/B>z/ѨM_dg04n8mذA-ZhT͵uRzw{n _Xիe0N#F/ggggŋ-Oy$FGgΜѥKJeQ2 V\pzjEFFj„ ڿx͜9ӆ!DP%<S\\飅 g}=jSCUݺuxbI>РA ڵk_W׮]Mdc=7xC{մiSM?~0%sÁZ]o¡{{t;oWI.vڥpM0A.]Ҏ;tQ+nq'2*!22by͛7]ׇ~h{Pk֬рTF>s=g#_yyrtt?o^dҖ-[,_˖-fO@銈P~~֭[g^f]z|ܖ?ԩ$o_|Y?\\\,.nJ׵ 6mRVVy >ǎ{jƌ=z,X믵o߾څð;pb ڵKׇ~HM:Uvc=vuT 7Xnذ?5ӧ_Jvܩ 6춏l7{Gy~jժeѯYfիWq~O@zԾ}{)-VZN:QF_~E&L\\\Tn]HEq T[N~~~ww2?cթS|Jh!2*_BBB+WJm* Y?7ҚopcڳgΜ9'O*))b*'xBK,s=O>D۷o7?=z6mڤxupsɓ'մiSUL!2*X>qℊԠAIO=֭[_U6lА!C,BҸ_~}={3~s;?PV|IIү]viԩճgOwuɓ'kٲez74dȐ8 m!!2*aѢEo$)44Լnذa_?Y999w!%O>*,,om7ސ``?uQhhV\UVwުS`o޼yz5m4M0XEpذaC?~\W\ZvUBZZ{1[Zrz)nZ-ZڵkլY3ib...zf5iDjѢZhquOݺuK/SN)00P۷oƍ5qD HDD,I={yt颹sʕ+{}v8ׯה)SԸqc5k<5={wOpS_8˫0֤I,L&"(U5kh̙:uUqi޼yEDDhʔ)Vk4i /QO?T3gԚ5kl25h@ /P~{QQQQܯ^ZǏעEd2ԫW/mٲD;|ߦ^{nBd(caDDVX(}zG;wjر߿j!DP%ԭ[Wk׮eի`0駟a{PP:Tl}||㋭QLLź5kj?MkSNt;@pppPjԯ_?9;;[{OmBIxQBBE۪yf+Zz>cծ][?z; nd2iҥڵիgrva]pA%ǹsd0YJ[Ep9shΜ9OBdU^nn>S޽[Gƍ]8NgV֭յk;GFF֭[w}WAAA懳'.".\zJ6mZŋkʕjժ *ǎɓաC-Yt xᰠ@M宎w`6&@%6b wkԧOƍUM ֭M,[L#F D('~W%''ߴOkp.P~`=TxEEE:{jժ%`rL&.]$???98TkŌ z ()Bd{j޼yJNNֹs~z 0>b-_bmݺqYZPϺ]]0.@q p<.dQrssgyF a޽{kٲeexGǨU>J^,T76VE z ()BdP޴hOq'innn|)?e\zUy\@0 .!!A^^^jڴƌ/ڻ$ÝȰ޽{kРA ɓ'5m4*11Q7&??_l[ T9Ȱpz!lR 6TBB}nX[Ti(W~թSG'N"GGG+**ʼ|mRxdիWUXXhR*GGGUV9TZ wq@eǸpgPVQ9sF/^>FQFцU(+:w._lR*WWWz.JB0.Jqee*''G'N0/)%%ETll㣓'Ojʔ)jԨBBBX5Jߴk}mT ʃ"Q~~~^:Wod2@.\PZZ7n,J,Y6pP0&T9 wqeeСC֭y4Çŋwi̔zٳgs1PHruuw9O?@. JB0.Jqe(S2L^rqqѶmtyԩSzmWʌϸ;W^߳ŋe˖rsses{^^"##UvmլYSaaaȰӧշo_K'OիWm}*쨼~ƕgg( ܉ Gtg kƍd2i߿[5o\&Mk]ƍӠAW_I շo_h:w"""W_}g@¥ JYBB 233 CU>}qjҤ^yլYSIIIҥK5|u]m۶ղe˴~%%%Io߮T\RZRhhfϞEg%STl܉ (wnC|P6 vZ*((Hɺrza^zJLLTNzb3FZnmS`g!2L9rDAAAS͚5~z=JIIQՋitIRzzz\[F󕟟o^.b: G_@9^^^^rvv?ZzUŋmX-,1&X״iS3f2=f\\/2=*#BdJ`ʔ)㏵|r}7jԨBBB/{5JƍSJJu9sءj@Y`LzjԨڶm87ߔ !I222_k&::ZYYY?\g7Ǹʈ,CZx*IZdvءK}|M[SL$5iD֭[m^;t1&ܙ"m۶rrrҮ]&I:~N> IRPP^y?^^^^;vM>cFƲ?[s{Fwqw"pN<+WsuNNNС;VcԱcGu@jԾ}{ժUK^^^0`?n'88Xs٩b9뢣w^:uJGQttO]FRTTvޭd9RAAAԩ$W^z5l0>|X۶mIH b\@eŝf٣Ho^W^մiԫW/F~G֬Yˮ(pΟ?;wNjٲmۦ={Jx 988(,,L ;cQ6mҘ1c5jh BdPÆ Uzu}W_$ʕ+:x&NXftuIII6duż]@y`ҥKoEiѢEVԯ__7o.0.b: P54fMU/EEE8q:w-Z?SZrvޭh}:tE`Ƙ*#d2ٻndgg]YYYrssw9[?q &)--Mrvqwf]EL3fl٢}/У>'NaÆ󕟟o^Ζx[ BUøPre\(++Jqe9͍7N6m޽{o K2?Zl4e4ˤN!2!ɤkJHHP@@-III$uy D6իWkƍU%Irqqɓ'zjGkw}I&K.jٲŋ%I-[#Fzڹs,X\+,,"|gU.P~""DXE `U5{@116>^m3|o.`!2w(88X?L"OOO(&&ܞ?O[Խ{w>|X%GGG:tHTTT$OOOuɼʕ+osz (˗F:pΝYfiǎ\ϟז-[6mG/"wwwjJ #G`0oUNN$iϞ=ڵN P 1.!DMeeee@ӲeKjܸ"""Ԯ];ڵK_kת]vjܸ^uyxxhݺu~3ڗ„S͚5Ӿ}R z lQ̡CԻwovڳg$?o e˖˾:>ծ][5k4tIIR׮]o>jϞ= 6Q<{N8`;c\@eS|ٿw{WConS+UŲ`PQQrrr{ n.]ҥKow^k@qƶ8 @)a\\q !2,L6M͚5SRR.]d"KRnݴ|r;U_6mjժA7ᡖ-[ߖxyyy'ԦMiT" 1.b: X8xF)(P{Uzz*G Ҁ}v:uJK/d~²&Mg}kjϞ=:{ dǪjc: XHIIѱc4dm۶W^Q~~z)M0*[Z,KҥtR^Zݻw$-[L͚5SRR:udJNdX2e֬Yc^NKK&I{gTB&%T8=IꞞd]rE=z0yT^=%&&ڥFU>l @egܝ=CY DÇ6/XBou >>*((PffV9::ZQQQl] BTqnqdPaz뭷h4jÇc* ___yyyʕ+.BprrwL&?^ׯWBB,۶m+'''ڵKaaaǏ >FFc1.ܹ>.0.9BdX3g.\>@Zn"##o{{ռy󔜜si0`d2_֒%KΝ;kjܸqё/:UDddV^7VZy"www5JQQQƏ u#D5kjժUVΜ9#WW_nn3hРAΝ j Ќ3T9;;<ŋ%I-[#FHx 988(,,L ;cJ"Wtt^|brww}*44m&I ,տIҊ+ 6(<yyyTڵUfM۷\]]ɓ'ի<>>ڿΝ;999W_P"Ku999:qy9--M)))Tz4qD͙3G7V@@f̘!??? 0T`[[nXեKeeeiҥZzw.鷛Z5k$uI۷oWjjv)ooojJg_Ĩz85@:tZn֭[KԺuk͜9S4e?^>ڷomݺU,@)ʒ$yzzJuPz(IJLLC=d0egg￷aTm܉boQVV, .]z[ dn04k,͚5jP~iĉܹZh!IJOOWa[>گH~~٥vTUȰO?[n:u<<<%OOOeffPuQ͚5]& $22RGվ}Xqqq-P0,L IRPP9ϛرCnnnzox\(777;Lg /_V $Innn2 `H}{T"22RWƍUV-rqqF(yzzMǏWPP:u$Iի|A 6LsUzzOHF{U w"BztIRjt*))ܞ*ggg{Xx,___k͚5>o?*,,L]t>s6m$GGGiСw"BݵqF˒#F(..N(""U(L&-8;;kѢEZh>͛K4paaԩ:xe45m4={V֭z)͟?eBdXWիg^vvv￯ߎUBdXe2tIRݺue0\[z(&55Unnn}O}jرK{ ݫ~OA6lh1b ūwv 1~gҤIҞ={o?K.T!"U``y 4}ze˖F!2,|={vY5fؾ0@Л1QE[a: X_0`$$$KM6՘1ctE{U!2,̜9S .TJJJoVow"L޽b ڵK_g6ζxJYBRRնm[PF$I?բE %&&*11Ѽ`ЛoiHxx=Zl *!!A> SllJ*~m+}WGё#G,"߯:uĉVChEEEz J!2,ٻΜ9/j(hêjaNdܕl=3] QJJy4ɓSNi׮]߿5j;WU!2_ٳ.P:tH[V֭%IQQQjݺfΜ)GGG}wzԤI5Jm۶՗_~ɝ`GLgl&88X&jmlX vp'2*BdUV" {"㮙L&{Tw(_^˗/ruuUjVQQ-K`C܉ ??Xmܹ^xVaa֭L^^^={`+ȰCiԩڸq֯_/IrssO?W_}U~~~v"___-_\&I.\$խ[WΕ5BdXe0e2!r7k, KrppЬYn`Ќ3lP{#Dbbbd0EիWWLL-!D{***RaaWn^իU*{_~`І ,M&fΜ)___GT-@"D6@-Zs… QBBBgJ0n裏>}~. PA*44m&I ,տIҊ+ 6(<<ܖ;aaԨQjѢy@:uҟ'EFFUVoX!JKKSzzza^;*11ю@ƝȰ{n :Լzj=zTVR``˻XuM6?Pj$oooW~~y9;;l D~,!2,A 6]v2d$iњ7o^yڹsyZ5krj0󛶟z**b#qdg&;"5j(33StU%$$hZj)+jժǧ 85PVn(rvvˮP P"B6mduM~.]~OƏ>L ӛo5kaW^QHHڵk'ɤC׫sΥz̎;*>>^M6չsGyDGUZgPqĉ崴4Sĉ5g5nX1c,Bܵ\gѠAϝ;W .͟!!!JMMoWP~i;wN;vЕ+W4rH=ZzO(sL<#Dv~ᡮ]2335vXu!44-[cǎ_5jTl*C[nkÇW||L\=?[ÄiPBCC-_d2i>}/IZbaرcںu>>ڵky]vv8 IRPP233l_H;vo(777;ȰpIO$I*((ۃ.?wޱ[}*(%%EoKIIӧe04qD͙3G~9if͚w=zk}W7ngSaNdXpwwիW%InnnruuWZn C[n娨(I)S(77W>233k֭rvv6oj*7N> \aE:|ySNZx飢"W&MXa4MOFUCppL&vYfi֬YVxzzjeQpboўe:aaСzw/ѨXCՓP"ȑ#5rHrΝ>zU!D%(W w7tQm޼YN$5h@3@"PPȰ?d2 I***Rtt~i^+(^%6@F _b ;VǏWÆ e0t -\P/,X`RQbٻ(|dFJR7eT$ҴiСC41, QDnN>=|=zΝ;K߯9rDk֬1"T|ep94md29ڵk֯_~Iٳ'KZiH͞="2qlQԇOeb߯(tM;vJJJ 4f"\ĉ;w޽[[&خV$PD>#%%}Q||:vw}W'O^pgk׮:p1plWTTCQkJr7Fen |!&"2Y:xƍrLpp?cF h/>'ԦMtajȑ Ԙ1c ,f"q13FNRvtk˖-j׮ѡ@EիWg("`_SL<5.QDDEdK.QDDR|Hp MH}Poᐦr,gp"2%)a(" _h'E\ޟ.&{)\\b&2&?ԍ999ԩBBB/!'.E^X֬Y ͙3G;vP\\uICx9p)ƣ K/iʔ)4ize˖e˖z7 e "2 WUUb%%%9())IEEEF6rR|/փ~(22idd{x&./h}EEk~7 92: 1}}_B F_1\]h7pu9A"/}! 1}}_@ Ƒ<+14B F_b0 d w% ?Sns̱Kh4Z=ѣ5VWvF]I#/h4XLdm۶ TiiRY,gff*##ñ][[ӧOM62L|nי3get(jsD^ pƞ`0\տ(55U҅_6>88XN¼+ IDAT4f !/R9/8YYYYFj֬YС5k,ܹS˗/u]gtx/"'.E^xDOxGkٲZ۷/{"/`<u 0: p"2%("\ p"2%("\ p"2%("\ p"2%("\ p"2%("\ p"2%("\ p)_VǏW֭e2 euEEE) iD^ / V?~\:t0: )GՍ7ht / TS 6nZ҅_GƪP"y"/ZQDFw+i<i_%/ TS 6,~p"2%X`UWWFЬY3xyʑ4+G^PDnjP0Y,^ y"/\<"2?p/E_q0""B-[!v]?N<)Ij߾׮ "!/I27_8U䅫C^'QD@ئMi4Zh!I:y"""A^6pm ^05Zlip$ş'kG^ ׎O 0_Iz3~]=~f?3xEdK,s h$6n(T[sssŨF '.E^@cŋ>>??ܫ\ordggkڵڻwZhAbccci&~_kٲe풒M6M}:M0A ,Y03q6oެ#F(**J&I֭sUWW駟V޽ժU+EEEi:~xʒdrjݺuƀoڦM-[hÆ СCuYqSLщ'm… >\UUU*,,[o\͞=۷@FWٳSNNe}?vءYfiǎZvۧ￿?DV6Mo~ڶm񹹹V˖-5rH:uʋ/0ys駟֘1csU~~.]C;Й3g\-h:t-KTuu׬Y3vmڳge٣x} y+ݻwkNNd[cǎo_ky Wvze۵tzǦ衇R>}?Peeezw]rG;zo+//O}nz^,8p@dXTZZ4b|^("í.9 6; .aaaڵ㡱. ujM7|7o/±Z۶mS=.߽{wmݺiߖ-[<' vק~ٹs$}.۵kN<ssF]?_ E^bMdgM6W}J|X9r5f\^FҸqt-裏t_6vz7+(..NfΜi@O 'mҥ*//Wbbڷohk֬$5o\|nݺ??4j(u#00Pyyy TBB~_i7oQ "/v 8lܸQwue'L,_Oϔ(Iԩ&N,Iѣyf:uJڵ߮曯8 fU5w8OUh`sСCQHH4*Ϡh('H䅦p g'  yڑ ,g+p%8| %("րԺukEDD(55Uss9M64j(15lڴIiii0`Ο?gyFCշ~VZIx }zd6|P_|Oi%^ P'ksssb 2DZ|VZ[b u][l4)//$KU]]$ǘnݺ)::ZEEEuf©܇"20DmmOW^$ժ͛+,,ilddVkΖlv:xZd5knݺ)""B@֔Bvv ֭[+""BڷoӘs)--Mmڴu]QFiLII-[*""BO=Ο?[iynۍ͛ /X'N￯TGnל9soLҥK\'''G/Vꫯn***d6U^^k?x^}PoG?AVΝ;C)&&F!!!FӨw"?Ƣ µռ0l0=Z 3hoժU+IҴi(77WfY _|!IQ߾}eX /ĉ?~L{ /4|Vϑ4nD;{┓Sg… xb-[L[nUVsι<5k9shǎSrrN<8qz쩸8檤DŒr-_\/n_+VPaal"Ijʕ۷RRR4|䨪I+ hȑv˚9sxGo?uֹ}^zIV7o0Z1/_ p?ppu=zTSSsUH^a6C0zݻwkp?ht233U^^hG5:$.+//O}nF~Ţ*9/--bq)--b_]QD[\|]۶mxUHRppBCCƫ~f㮞vO?ULLSլY38۷O%%%JHH$%$$h׮]N/ްaBCCգG:8_ dt111X,*((P߾}%IںuMV1͛7WUPPTI~(==k0FǏ]vj޼L&a4ݮ*} P͍ ܆p|=/iժUZ~ZnXl6E2͚] ,P.]Yf)**Q {G#Gt3224az뭺/ٳ4iw(&&F'NǍQiٲµռtRIRbb+VhĉE) @FfSrr^{5@iڴiJHHPV4a͛7[ kyEd\۷뮻rlgddH&L\͘1CgϞԩSUVVo] qsAGyDfϞ-ժ}*??O͛7WttΟ?i, ~p|9/DŽ(''G999.tQ~;CH/4nq}4L7o^>|پt0ɤf͚YfF o`^;%("\ p"2%("ټyF(L&[Ωĉ2LNmذaE ("/:{┓r̰at Gӟ?dtHIIQJJJceX!Dnl٢l=ڿ$駟~Ҏ;TYYipt8۸q"""iӦԩSlpjǪjzgxb=zTCW^18JiذazUPP?ڴiRRRTSSlfGС#Gُ͚5KyyyZt' C=!F_{Vjjm6mܸ1*//w0Ed??IӦMԩS~YÀ27tڶmPp~ɓݻ@O^s1:uJ۷7:hӡCݻe_|Ν;{1"@SWYY4СCڹssjԨQX,:xf̘Ν;+99ic&{G믫ȱd2Ix ?~Q۷_~ׯ$)##CٳZvɓ'_94]Dc>l٢!C{2Lz'ti;vLwx 4!N/z>ȋ3X͕+V覛nRndԧOL&~_iݺuogi-6oެ#F(**J&I֭s8qL&S6lӘӧOkر UXX&OJoE&nO?79sԩe&IiiiuͽllHH]gϞU\\rrr\6lN8hӟǎoF6lP^^6oެSz:t3Ǟ}Y>tbbbtA=3JIIQQQeZtLPPeZ] 3XPPnEرc:uڷo/„2O?Twy>/~ؽޫ;vhÆ #<5d=+4Jh׮]<館tr!ܹS ܹs5j(Y,}~b(..N˗/7:L>"{Wo:u$ioVgΜѩS /<#FH5k&I$f=cz fΟ?/I U˖-uQG֭eZ m޼Y#FPTTL&֭[o5{lo^-ZPRRoP"_ի+tR}w:z^uuMٳg:.\ŋkٲeںuZjd;wˑ. 2:xί~+-[L6M;w-_ Д(%%>ݮ_~Y3g< Izui ("I&iҤIo_WjСDCj*))ɱl6+>>^EEE ݻw?Ç%I:uRJJ dONddzll튊 MEd?f_Z-ݮ K`*33ScǎyG ܹs/cO?~mM6M{ѹsdٴgV\3f&$"I*--u_ZZKffѣ~lʕ7n,YX)00PرcrJ@#ŢǾ mݺU . VhhSEd?V]]4hΟ?kfeed29nݺ{{ｧnݺ)$$D{և~֘R;wΝ;%]xΝ;URR"ɤӧkڵKǏWTTRSS p6oެ#F(**J&I֭s={ڷo-Z())Iwsi;V ɓUYYגGСC~ݞ={ĉ矻[XX1chTjn `۷_~ׯ$)##Cٳ%I3f?SjT~~BBB p Ξ=8ٿpB-^X˖-֭[ժU+%''ܹs1cǎ7| 6(//O7oԩSu b=?6|=zΝ;K߯9rDk֬ӧ E wK+6lz)G6lВ%Klٲ_$&&n7L7o͛Ũ:v^~e͜9S<$Vdd֭[ѣGkϞ=׶mt뭷J^}Uw}z{Ǻw.Iڵk֯_w|=.;]wRHH:)##i_rre_u?:$ժ$>٬xi***RXX,IIII ֭[5rH#BIfϞ-k+77W:qΝ;CwV֭/oZ/22RV5l6l6c}7.>Yj*""©?((H|^(",_үGر}]M<-ܹsr. /փGk׮:p@ENJKK]S933SvQ .>YbɓNϟӧ1UYYjܸqu'$$@ӧOw۰a\388XnN!r7F`Y,o߾.,;uVM6M҅ eee*..V%I~jkk|^("í|I1B;v5gj̘1nPvv$鷿NիWkhmx\CXB$gTVV:}#СCڹsӧkҥbbb4k,EEE)55U҅6LSLѲeT]]t=ZQQQFMEdձc4f:uJڵ߮-[]vsAiժU9syuE֭S^D4 /}vu]파 I҄ 3fٳ:ut+??_!!!cy{Q@@Fŋ{^h("íV^]oƍ/C顇PDX~9JLLnwo24o<͛7pZ9x%f"&e pEHb&2D9f3.1` !K@ p"2%XMBkK=?K Ed@ KP@cD^(d7uEdhh'h El \&\b&2a*c&2YYY2LN[nFM3Oٳ>vP>O~*bxPY~EEE馛nرcURRbtHФ1nkj޽jѢ ?uyLnn&M/88XΝt\ĉ;w޽[[&خVKH .,͝;i_ll+I:w?CWfSrr^{5EFF5%G\ ڴi4`?^<oVZry\hh6LcRRRݧOǫcǎzw5y:ξh<[ '|{OfYz_j7I wUDD5dǙL&ֿ\&,,L]vՁ\TFFcB:tFx7p~yy/_UV$XBݻwז-[4p@o @EU^~TǎU[[[nE=z>RԸq\ Vpp ~HHhZIIIݺuStt("EXS[[ӧkիqz7~z\R4h;Vxͦ O>6mڤÇP#GT``ƌcth~~~.]C;Й3gdZռys9)|^ IKKݻ;.!!A A{z5Ƴ%cǎi̘1:uڵko][lQv ߢEk:' /ԇ"2<"==]yyyڼynƫ:Yfׯ/Yի%t-{WUUU*++s\ZZZTQD[v=zqF\9jjjk.w}u%.] j֬ 4j(IҾ}TRRMƟap4ZJׯW֭kfƏnAْyiܹ /ȑ#z OjĈر?9s87͚s]93W{x|EdK.QDDEdK.QDDEdK.QDDEdK.QDDEdK.QD襁VgΜQNd2 e7(,,L>>[1y@^kX9sFFԩS[y漀ơSN| j*<< /\Ƣ-i|)ͷz\˛O.QDDk" WSSjhڶm+___fE^ y#/PDFs=B={V۶mSbbnkɒ%ڰa5|p[Nz20jbURRrCiQeXxV8yq hѬ4p@=c4iuVK/LEDDhѢE_|!???"FsԻX9MS\B:%v].\PYY$)44k('HoD^1x W'yƐМ("Y%$$(!!>ݮ5kh…8q$iӦM 5ydw jjj_;wlt8-F%Ieee V5yq Z+B\("0ǏWIIƎh,"l6l6ǶjmX&.*W4СE:##Cn `o {ٳu޽[7n͘1CgϞuUV9jjj4~x]tIWff222xbw_4 o hXbQvv $5k0RuNn;9dee9mgdd(88X9rc\N"y_hϞ= ѠA3'ҥKծ]fc0ͪREEE***tazEEE:yL&͛˗>~XaaaJLL48rMCo޼Y]t߮T]pї8=G%..NVU{Ѽ (IRJJ-X@s̙35tPUVV*++K~~~F l;w#F.gdd[nС^;w΍9a7ooݱ믿?Pz_RRR烸fj:5p'Z#hV׵ IWl2ŋڳgzml-,X@o233'nS\\Ο?|M>]***ѣ|r4rBfϞ>L[nu?sLi2e6mڤm۶JOOW@@QDUUUiݺuz甐kÆ j߾^{ƿ⋊ׂ Իwo͝;WqqqDhj䄆%''kǎu-;6::Zt1IW]׃%55Uvԩz ZQDZÇwk۶N9rGqQ|ULLL h~vm6}h5W*{sa9޽[߿0wj.Vm̞=[[lџ'uɱq@@ڷobmٲEw}:wO?TOr쫮կ_?;;p@ h~֭[ *44|MIRvg7N}s%%%wqW;v쐯bbbCֲeˌ,y3A;vԬY4|[nZj.\ӧСCNΝÇk՚8qvڥ,4%rkvpݻt];wlYZ1FXr4uTq:vvڥoÆ ӆ /jzpB4rZF&{C?j* @<4xzO=XV4a40ŋuqEDDpZ;>yZr$X9 SP?ރVbi@ڐLd60ad2}vGyD&ɩ-@bMdFUUU8p{1M41ڸqcl6+5kΝ;gtH՘ /+׮]n@μ@N !x%ܠX͝;W ,PPP,.]///׏cuU뮻t!IREE|}}UPP IUPP x믿p^ kQDjjjh"EDD}ٳyzoweffcǎתUl2޽[t}L uwh̘1:4h H>,ɤQee$i޽5jQhZ6ag}V֭ӯk9rD>VZ_~H-YDz?!C(;;[?2dzիW+00Po+3~)COsK!, hmx ~M8QǏ$Co>c#"##CCCUVVCR;wvX4j(k޽{5n8Y,(22RǎSll. kPD~W__W[Ҿ}Զm[mɤZUVV*441sZ#Go'|\Wbʕ+5p@W^ @!/E^@kC{ꩧdZշo_F+VД)SodV]@M6ѣGc_j۶`ڱc@+B^\5a?ڼyl٢O>DZz233G㉤<رcD:q߯_',KW9ۼy `PP7|/Њxs^HOOСCթS'+11QGusE͞=[;wM7ݤ$:9yƏ:(88X˗y)d9/e ͟?_O=&Ohԩz'^TUTT8کS1f2sN9R>zɓ'7j(8e{>@ya޽={8ݻwZƍSUUcO 2:$ '=wyGF` '.\Д)S( Df:|a_VvvV^QDp'?SO=]cǎwjͱw=Cܹڷo`=8YxVX}{2d[F?ÇkzԵkW}W曛GN֯_kqDg}Vڸqc_DD[ p\=s*,,ٳgm6%&&:v,Y 6\Ç׺uԫW/rpr%?mdI!CSppam6VS UUU8p֮][gUK/iWǎ/9Rw &(,,L&I۷owGd2Z||Әkʔ)W``OJw^Ed{G_zοoYfvҬY4w\eff9>==]x˗{ｮnk͚5Zp&NHmڴIgΜ| p(I:{NSLݻwkǎ̙3;toXN,Y_?4}tuM׍ jsjȐ!կ~%Ig}kڴi׍OMMUJJcjRHV*))رc>>ϯhd2Ln=uQ}_ս{whCZxM4I*..O?W%%% vzM6mďEd8YtO?~_W_իXƹz;siiBCCKKK5h h]hTϞ=1c4@zWEEE.]QCj۶mz7tgњ5k4eʔf=/DDDb(;;۱j*??_111Fp[oU]tѱc$]qi˗uy(KW~\wj໡(>>^:tPΝw^I?M8Q999M~{Gŋu͘10^eeTTT$􊊊tIL&͛7O˗/ןg>|X?”hpvi;wq7JLLUXXVF W '׈#W_顇RmmK.{ԧ@QQQ$(**J/$-X@s̙35tPUVV*++K~~~F h~8u8qBٚ8qn6Ix͘1C>#%''k 3: 'O?oo[ѣG+33Ӡ-]llv~ɤe˖iٲen  4zhuMuO?UffqgqZ~JNN֘1c㣤$KnEd89xe6UYYy]z R?ڵci˖-Mhm:-a릛nrcDDN zξ*mܸQFrsTBNTPPޓ$:tHo5x`}ZhQpDhܹSf?,I.IٳvܩH#CFqKGUQQ+֪gϞS9׍+//ջwow@m۶)--Md2+W^sl``6m"24sLs=;l2%$$81Lرzr*44T_~ 68*"25x^nѮ]ԦMu] d2);; 0Ed/gU[[خdj5Ed/S6FNv{3- h=qJ>F}O?>#CοrJL&͛7ϐpFNFw9ru_:x[}A+t4"2*++֭[uwjݺu6lz쩧~ZEEErJM2E6l7,>0ad2}v~ݮŋ+44T۷رcW_99L"j骬teQDFڷoOokz4x`۷9{l?^cǎmcp* 8Pk׮ժUz饗~zcǎŋcL?\w֎;3gzWǎh„ /f|W[n'|ͰllmڤPgnך5kpBM8Qi&h/ 0Ҁ+EdQkϞ=6m<?bUפ*%%űmZ%Edջwo;v,ƨPDWk1 QYYbM:PkQD܄B64&Lݻ̙3Zd|}}܈f[ BנxӧOйsԵkW1BP׮]_"2[n5: xgH8ܥ^"2+Rnŏ{Z x.("p:t:u`%&&ѣF@ w^͞=[C˗Okܸq/ԱcGUbMf`ҥJKKsקO}嗒/?n*ͦ87QHH("pYYYN VaaFiPTgv63'лﾫ? Prr&M>ȈPZq**QTgffsl[ViFz״eu]7_~:p PZ R[[yiG wsW_}0z뭚2eN<)I*,,TuuƎ۷o_uMyyy.gdZn 2{l}gڷo1JIIql[V 45P>}tY??gDڵS``kBBBTRR׭ ڱcrssu-g6e6搐wddս{wPuL:ia~ - k,gv%''k۶matH ޽{رcX,t˝ƔֹUfYN |7DfϞ-[O:u5- ѳJWTqqNm۶VRR$ѣ:ybbb a.>auIbccoܸQ</4au$ IDAT]gΜђ%Kx@>}RRR$͙3G1116lѡWC?HhVa8nt N>x@ΝS׮]5b8p@]v$ QRRl6>%nonZo֮]k׺)"0KSQDZ ctEY` x:f"\ p, O;Ed[- Ed"p x%f"ᥘ p,xf"\b&2* 5Þp %(`=xkתGStt>cCycQDGx7%KO>2C9p-ƣ kƌzGտ_^:t~;C9p-ƣ ]tI;vcƎ<#9p-p?TSS!!!/od$ `5c[F`=! 1}~O7w W? n4'Hz~O{B Fb7ye-1{B Fb0Cs]}Nϟo;d$FN:宏&u9n'/h4ڷiF]Zj^q uE*--u_ZZ*rT8kkkuyuY&Of7(,,PFsD^ p`0\v4x`egg+11Qҕ/zJNNnllvX%0:Fќ !\%wҥKעE.٬EHn&9p-c&2<߯Z/VII h k0nqsQDDEdK.QDDEdK.QDDEdK.QDDEdK.QDDEdK.QDDEdK.QD襁VgΜQNd2 e7(,,L>>[1y@^kX9sFFԩS[y漀ơSN| j*<< /F\=s*,,ٳgm6%&&:yeff:&..NYYYkΜ9zw㣤$⋺馛u^@cQDFw4u p=#BUU{L&MsL||6n6NSLٳg{nUWWG̙3e˖oy/q("\BBc6eX;r䈲tA 2D/ի16999 V>}4k,;wї@GYƎ+E:##Cn `rZT,gf{N:{mۦDGnג%KakZnze`wz+ǻ|μ@N 1fUUUjڵuZJ/֯_|uQqqqx#PhV Z|v֬Y jĉԦMtm߾݀h۳l;w#F.gdd[nС^;w΍9p-Z#0URRc:(::ZyyyF [`~meffO>mݦ8?>}UTTѣGkD h hX)))$8 qff9Vk.TUUiݺuPBB$iÆ ڽ{^{5 :i/x-X@Իwo߿_YYYnд kZ1-Nzz-<<xbUWWk}m۶՝wީ#G\7ȑ#vq9p-Z+0b$:/--u%55UvԩffaY,egg;YVlSwٳڵk>ȱZT߯_?;;p@ xKo 'E^@kŚhV:vc***RPPuyiի"""h")11~;vԬY4|٪UtM>]r?w\ >\Wĉk.8Vy3Ѭ (IRJJxbIWX:g͜9SCUeegdР+W*))ISNwܡcǎi׮]曯;l0mذA/_ .4 j@s 'E^@kdv ժUTTԻ{,hh`/㊈T{g"AKPN ކx -cDNhu G^@s`&2%("Մ &ɤ۷;?#2LN->>ޠhEdFUUU8p֮]rL||Ξ=ho#6FGBBc6eX!D%''Gӧf͚sx5f"I&)""Bz駕<&ضZ Ed1&O={*''Gcƌ5JKKsWuXx[oU]tѱc\IMMUEE:uʍ@LdN>s)44,ƨPDxn>_{*++f?~\EEE RPPҔ$Ţb-X@v ܙ [0Ed6=zc;%%E4m4[N~233U^^07N< 3@ ܹs`bhҥr?V׮]ﯻK$UTTWyxU񻳐tXY#(,:Ȧ #QADeQ0(((*( @"js?K=IT&\W]}$ݜ\+WN[ Zj~cǎ2ϟ(\R?륗^RJl gD.NG.$b 99Yڴi|I%%%飏>$]wuJOO|-[Eܹ~n5kLk׮$ڵK.K۶mSvv$iݺuСo  ӑ (iXD/hҤƏ:uhРAjٲVZ 6/ŋղeKթSGO=dIyg&(\vJ5h@6l/I|3n\@I5 4irJOO׎;<:_ѺuԵkW%$$hڵjҤ߯;[rp:r% >].rssʕ+{8]\\$/o[j4iOiӦRԩoPL4,"PZh#G(,,L&MSxxׯ+{=>%8DuEmڴQ޽g};,Ky9{W/˕+ 7E! tW,":ǣqƩf͚҅^G}Tۭr|r]~2d֭~PJuAZf;v,08 ӑ 8_ +ulҤIzgF/Ԑ!C뮻:ʒVffbcc1U&dc7ĉ:xj֬HW4w2A" ב gב pDu}zꥫZ_]_|p9 X׶m[ZJ~$iǎڰaza3 |Aeee~ ?NNrrr*t8ֽzWki֭JNNSO=Bqƺ馛4j(M<ǎL'1gJ2qs$1gpuǏWH-tEDD5 w 3P $":rN`S?WFi۶mzg4tPۭpPhh┞.I*]\.1FǏWzzj%(6ŸG.(ȅ?\XDu>ƍ;CRzGla }as;(Iȅ\PR  ' bbb4}tM>v+rrʪXN3gjڴiխ[78q»W_飏>{ｧkLd@ѣzQsM>]?z%IZ`*Ue˖믵b m޼Y-[w?JO=T`Ǚ:x9.]xnZj7J6nܨ8$uE!!!ڴi{ q&29"ITxJ9rD+Vy>,,LʕSxgeeW-D'OnժU=~ IJKKOKK>tO:_~ŻOaƎLZ|XDU͚5UVyDzi&iFԦMeddh˖-}V^\jժȯX o&2eggkU\9U^]s{1թSG5kԸqTJ[ԠAu]zfϞ'OjĈ߿Tb ( (v_~:u}ʕkD{O`> M6iݒ?C[>vZz7onM~c͚5۷kݻwW_ݻ'Z? GQbbeԲeK 0@ 6ԭުM6k_ Ȑ$:uJk׮Unݼ(33V{k"G-4guI~7TR%'WnԲeKctK/>tRkbEdhٲݫ>LqqqС q>cJ6իWqqqլY3KwׯWϞ=UJ\.-[ycyU\YQQQҥg["2pI}ڶmnZ`+nҍ7ި+**J7֗_~Yuv;vLM6լY }'̙35{lmڴI֭N8N|dddh̘1zWx#%Sl5WkN:u|x۷Oe˖-УGѣ1>}~aK`UTI˖-S* 7|jժn5Ljժi޼yޱ5k:^X<#GK.1ۭVZiƍE."(''8++^ ~.M6o5}]uM]w֭[ .@wqnVȑ#J*WT\a&O':3 ˗WڵZ /:uhʕu]w)99srrرcRSSm% 1|p-ZH~-ZhҤIj޼[oUg.tɓ'v{jժWs$Iiii>iii X P||7N999jٲn&UZU۷oլ\6l3֠A[?vX=8++d(j֬ZJ͚5wߴinv@b>:իWkھ}{\.yc~jԨQ(<۷\r^=cSj֬qƩJ*ݻŮ C֭[5vXjJnFR۶m5i$]/K/饗^r6Ku8%uرc>|222Ծ}{XBZ"2|lذA<_r%hҥ;vTfMM>][ر1E>r$?v8#!!Aʕ{ݿo,v,{Νlۭ 'NPxxj׮믿^ժUShh>.KF!b>ƌs=W>,"Ed8xF[@XDF:+==]SժUx)]:J c4zhլYSѣJg}rEd:uf̘1c裏>1V߾}[oY? sѠA4i$5k֬M4 c>RSSնm"VVV;`d IDAT`QbE-[Tzu?v&o߾={;$}ᇚ?:[3cĉ\5kAriʔ)j߾z&M衇&?a>n>s:t"##n:eddhOTtim0 DEE?lq&2|\qZjUϯYFW\q;`vZ|zz֭[ǎ,P*+&&ƏMpL ( %''+99Ӝ9s 엑;wꪫg{,rO?駟~$oѣGnMsu'xB.Ksc5;Dnv~풤5kjƌk͛/I&~ p,"IR%{Ӽ_\5p@͙3G=X~mA.11Q.KJ*}|6X;uWK.,"E +r)<#EFFߙ T? Yj߾>sKMO999֭y]|_(aaageq T -:󑑑5kf͚姎@aXDe߾}R"##զMMϟ+j˖--u@Bl7̼w˕+g{GڵE]T>999>W{@a;ݻaÆ" c"`D.^R!!!&--ѣGMHHH=U~2?S2˗7w63gӤIx̛oiׯx}cILL4~1Ƙ?ř+WaÆ+/==C;=خuLd?{=خr\ a[L!KRNNJ*.xvv>ST?ncud1Fx|rUXѱ+jvءr9V7P㊉$}۷BBBԺukח#GZj{O_vDj/=>!f.)p2A 9}<" A~`{l? :3gΔ$\.͝;Weʔ>x~zկ_߱G7nJ.SӦMj֬c%yr\r\ܹs(x<:xwX8ouxriĉ՗epCǣlvm9ڵkkٲeӧV\QFIx})gjժiŊz$Xx<ֶ}<?`sv&H3A?l?`c"IEdiӦI ٳg+44\Rٳg;V۶mvyTRjڴƌX}Iݻ$i֭OA~fctW護yTRQTX}I>}1:t&Ns6EiƱ0#nШQԹsgyח}nP:u?GԮ]ڶwv.1v&H `g\L$1\3LQ%\NolٲV2D3f̰arr﯈+U^Џ˺uԮ];3,)ȑ#:|6m~rݪWO<3f(55U7|дi[nl~9!\ cLρd7Ȅ}o߾'(0>es:^{f֬Y_xӣGc%\b/^\`2^zԩcV^]`|ڵnݺ7Ƙƍ|`4ix}s`ngwܹs1Ɯ:uʴmָ\.m֬Yx+3tPݧ_3`;L ؟۹@& \ !!!|![֯_x=~zoڴI:u*0ޱcGmڴghѢx͵g매f͚kԨKҾ}԰aoksVVw_vg^ϊK/VӦM%I׿k޽5j8^СCJNNѣGձcGկ__SLё#G<3A 3A 3A"2!P D. y 7CIEpeee9^?''GN*0~IחVZ>u*V;w*11g|ǎ*_%v+ݞu7\ohΝj۶=矕 )c]w֭ԧOGiiiZp5n8u]Æ SϞ=?C \ \ RdB` \  @~7n7xE }]zz饗 Ϟ=[_|%k׮;v233cz衇tW:^뮻fy<y<^Zw}x}Iի8p;~{ｺkok֬YUV%KhmÆ JIIq]u--cbbbn4|p-\UTI{ъ+Ǐ zi߾ڴiڵKօ^k`b;$`;$`;2^;rA"ȅ<\Ew5aaafРAff榛n2aaafҥ߰a4]v0a0aLddY~14j2nttřzꙔ믿޸\.nMhh2dq1ddd֭[0hMXXԩ_o{{x30a9vƏon_^9q1Ƙ_~ٴn/=9rL:4lDFF>cLvvMK0 ۙ`cL02!`@.C.C.C.\c{!פI}vEEEI&?~:t۷oדO>;vx;VuK})﮺O Pxxzo7n5jw>g}_[sb )SF۷$͚5KsQÆ 5k,m:uT})<M@ o. dPr\ "2ZFF,YNcƌQruVUTI\p8yUbE}^5jx}I?_|Q}6nܨ5jhڴiU?~\#GTrrwkժ#G .Ѓ>h}I?ٳg+--øq㔘aÆ9Z)SFwVbb&Lݻwkɒ%ںu*766V[nUڵ}߯-[*##Ñ3g3gq߻ˑ s EFFv&Hsf&H3A?sLE&E. @.s ;vxSvmf8`1馛vZetbJ*?ydӯ_?c?o*T`{1a޼ycǎ׿뮻_l>l2ӬY3cĉMZ… MTTEVlٲ櫯2Ӯ];/c9x𠉊r1ƚ[/M2eh=sQ[͚5!1IIIJ*&44{s:^3`;L ؟۹@& \ !!L.ưչsgs}c)S@駟5j8^u駟.PӦM .p14h{c{صk)_Wn6nX}LLL1 /4qkx}sгgOӭ[7d͏?h1fʕN:7Ƙo3N:ut/=A v3`;L ؟۹@& \ !!@a⣀e7o֋/X` .Gwڥ^{xŊuQKy7)h޼y;v?*VX`رc~ءC |4Jɓ'o{{9qZd^xG3?u4e]~W.2I'|,^/=ض`Kܹn6xӦMw^ۙ ۙ ?&2L  """U`oU||taլYg|۶m~Z͚5}_b4hx-[ȑ#%"hܹgi@qjذ>sdɒB_47sPzu{ƧMx| 6Ν;sy8=h 1]ѣygyƱ>J \ cLρ\ Ȅ@A.  \@`AkQRR|MIyax@s~hr\է~1ch#j IDATРAח;ԉ'd_|_]'Oܹs?i$C{ѩS4c ٳG}֭[x}IzG4x`:tHz7hX*n6 ++K?I~NR&MZmvN @J \ \  dB` \ iddd.]8jUf_ncnf\. 7!!!oޓ.\hj׮m\.q\ .ons% ;w1Ƭ_tǛ(Ӯ];rJ4c1.˄efM6-X|'~\S_wٲev'x”.]L:r-TR?,z(v&`; 3rL \ ȅsG.scl 6hΝV-ԥKOIIݻ͛N:~.z[(Y֭[v),,g/t~z-tM8pjϞ=U{9-_\˗/ws۷VZ?%%%iǎ#<]{-v&H dBp \  \C.r!Pұ GJ.wW5j( 4H111ڱcjժm۶G~y͹8?1F ӑ gF& pLdNG.*G3g3gq2eʨQFjժU=z}QEGGBZ[-ZЪUTlY5oC=jժKeêXXrN:zթSb/Iʕӷ~ *lٲ4SLQ&M9عs9[\|7 nedd8^?L:Uw_qǣoQJ>ۙ ۙ ?&R. dM h TM*22wQzzFSKm۶yz hƌZ|$K޽{UVo,6^{5k֜etk޽R_=$M>=,_\C і-[9h֬\.A%S,5$!!AWbb .MSNUr4l0Q{nl3A 3AL 9\ rA"$r!А . >CSBk7olbcc7&&ꧥ/Z}cIII1|äILÆ nbbb'|b.\h̙3ù*S9p@/řŋc9yӧiР9|p_c; 3`;)ٹ@& r\0\p@. ppMd ~wK?>P^oFT^7̔vҥK_K6l^z^_~ e˖~(sp&W_}Νʕ+6hҤI dB rܑ J>DFzuwkU6m$I,YiӦ;t#Խ{wedd(99YՓw!C(66V+Vp~ q@*Uk[ڵkB ޱ~3 \ cL"Ȅ\ $r |,"#hUZU>F3>k,M4Ir~||fΜ9r=h}Iʕ+վ}{O>Dݻwױc/R5߽{.>S}Ʒl٢.)'۞sQ/ +66Hz}m9Q>}zk˥Hծ][7p?aȐ!y. g;$`;$DۙC@&"r\ȯO.r\@8,##Cݻw/0޵kW=?ydҺu)KR Z}IuUZZZD]%Zj;`J*9v{_^NNfϞO?Tc,[Lqqq%I[nUFFv7xCSLѪUԮ]bW^}jٲ(>3A 3AL 9L @.H-ej= 0O>dS1b5jT{q7Ƙ_|t玮6]v5gvfffw{MFŋMjjIMM5/676#۲e̥^j6oۼyiݺYt#5m΅?|&_})]c_0~x<1cFaƎkrssMv\X9d3`# cL0&\2!,r\0\3r%@P9sYYYzꩧԮ];}駺{{ѣG{|)͟?_իWW֭%I6mRJJ g}KR}>o>z꒤EDDN:ںuk oNӧ?x<^_ʖ-ñct)ݍ_~YYjj>Nx{n5mԑOUn]oVm۶ѣGk.]veps抗LLmgd Ȅ3#@.H_E. (ʴi|-[V{ў={cqqqzWyam69p$B Pbwގ}sf%iV;v옢ϺC=r9éS4qD͜9Sْm6rH?^ uEa~{-p޽摑^ /ۙ ۙ ?&v. \ $rῑ 3 )SF_Z)rVRRϝ'L޽{^pKz!]r%͛7kҤI4c ͝;W׆ (]isL8wd  Ȁ ~<~GIywa˖-%I5*pa'edd_?tP[ǣe˖p5(44/me4|-_\:t Hy7Xhz3|r 0@xOSZZ$RJ9rx*%%E!!!J0ۙ  ۙ w. Lr\ r7CpKNN6]t0qf~xĉMll 1!!!v$ ;)--tɸ\.SlYSlYrW\ayfS\9s>}>}U˛-[8^cgԩcJ.m7on7onJ.mիgx@cIOO7O?iܸ 3W_}y뭷ɓ'oS`|Ϟ=B o71 D,Y63`;L ؟|rL8wdr\r!`"2O?mJ.m~;wyw}tgq>h?ovavaf͚eC=x}cM˖-}B2-[4w~7ɓf.s1tޱGݻ0m̙&""¸\.oƍg;X''NxN8ah&LXͻ^t3`;L ؟3ȄsG&8\ !B.&rNcA+11$''?ILLt~ʕ;S`|ٲeJ*7ƘX_ߴiqݎ׏4_uDEE9^cJ.mvY`|&::0s3eӠASti3p@zj`ӨQ#sWk(򷘘SBӹsgӹsgSBkSurs7ʕ+P>?ddd{{ cL01v&c3Ȅrrtپ`Çնmm۶Ç/~ׯ_~u~llRRR Cjjbbb/I gggTR׷=o͛+WaÆ;t7*..λO۶mՠAboׯjժk曕qƩr~ܹs3oW^ 6o3ۙ ۙ ?&2?ȄA.  x^liԨy ?梋.r^jFY`|ĈUV7Ƙk\~СCޱt#G4UV5-2)))&%%żjժv1tMQF?7&77lܸ\tEf׷=fᅞe%beʔ1۶m{'|Ҕ.]/_OOOyE"55c]jprr2aooO/Qtڕ,V.ӧF_l'l'U1Yd{Nj *䋬^`"xa=y g-Г5k`ʔ)pqq?3}W_Epp0ׯ|2RRR`4(6ld;NߘB xNj *䋨}aÆhݺ5 -- & ;vХ,書p7o3g<<d{b Ȁ^`",]0 S}RSD& & ~~~W???dffbΜ9hӦ ڴis"33ZҥqyyyHMMEjj*rssqRCYʕ+1tPs[~N"W<|PwתU w-~=F!жmr?ߑ@d:PcL(" * {A #Td0:qΝ o+@,==Wz~IlllpEZ_~(((o<^L4.X@`%ANNN8t6mjў;֭[R* ۻҿ;,, Gǎ?#Fv᫯L0Lxw_ŋrJ-Z+N䌉*9`/Tv#%셗^`/kDf ''ED4M@ܦMhG>_?kO?_ţA9rHcn3YaÆ!66+{=ￏ ԬY;?  C`` W[>z(V\),ڑ@-/v gLT {A {B ȌUQ|eYH/" A4 5j< u,YYYYpwwGƍ+X|*$W8Aq㑘_깹 C;-dM6!++ NI&Bs;w3gμs Qa,P 83&;({A>E` {yDfnݺIoܸ" ٕ>}h'yu_e"{?~>>!S@ /@z" j!{ʆX#<'2cܺu 櫉ZBxxng@Ǐ-'<{2dL&,8p`oV*{=zTh;MӰk׮J3QQQpqq]JD'Nx9u`'^xu`/Idj9|0[[[ÇҕP+gϞocǎY{VtW?~+WТQFBsq!ԭ[ע֭[h۶-Ξ=+$woT+9s`prrҡ*5qttѣG,bvd93&N8^x1*{ Df]I&XryO?ٳg"4ނ 元dff"<<,Z`0ҥKV||2<<vBa'^PPu`/09PF L6 ۷~ڵ ...0 0 ҥ ̙qUxU|5j7tj;vx$$KIDAT;w.B}0b$&&bРAر֬Y ={'wwwkaN{AuDd{{ U{a30V+رTUx={i׮]DDEԩSdG44 }h4RfӥOOOzKGGGڻwp?>kTF ۷/mذr uޝ ۓK6l@f͚ԺukZxq㬬,Y.LW^]FF%'N ?&@$;^( {AE hx: 걳Lԩk^rry%Yl{$%%a׿S"11QZmׯץ?dC… ={6ׯ/eaÇwž}СC2qwwGddΕ˭[p2W (ͪP ^@"r ;u`/E" ^x{QHv #B̜9ǽ{>sEΝ{ş'c֬YBGD 4;vnzz:`0^BkqvvF^^<<<}v̜9E 0CѢE A-C9/բpuu ɓ'#<<g϶dYv *9?&v ; ^ *^`A4 #777ڴiS7|GGGڻwhF/_6/׬KyƎK77x֭Kw%" <_d;vPN.R7Y .dҥ""9^P DDd{NP{Ad| *Ȍrƍ2hBA4h9%Aze OOOa޼y\xcƌѭBdee`ppl $$гgRi&~LL g`ř@AARRRtC6}Ç-E9^P 1@>vs j^`/^`/ȆωX- 4˩:tm6`ٲehܸЬJ߾}777٥T:{ｇsΡp($AT޽ϻuVy;w 63OOODEE!00PX 2ټyիWÇ^C͚5-ׯYT/ԠyNP Ba/{z^`TO"3Vݻѷo_4jAAA#//[nE׮]_zFJJ J@VW>8d2I&Qi͚5_|77Rsծ][h~Eg,_X~=4h^^^ҥK畤Go+[ ]G?P+T/d{8^`';A? {Q΂Zu놌 ,]O 8cƌC… ={6ׯz*1`hVUǣt邍7 ;hѤI!߯:6la#GѣG۷ocغu~ C!v^{ڝ;Q^{Q`fѣGeQ!L䟔DDwܡ;wPRRі-[h޽ԪU+ "zA۶m>hӦ ^TFjj*կ_^D\\jՊF#Fjժ\R|٬^K?z?LxAxD$ U D v {^U;͛ǩS-[QN-ZÇb t֫W/L&5 'N… .L<.]*s~)a*p̙2]6nݺK 9,XE'N\DEERLZ߽{ÇGXXʬ N |/X^`'T | a/0 ,6bHooM5"GGGڽ{;vPN(99]Fo߶x&;V===L&dkk[EhVe0 ˋRW&__J+ ZvmkRݺuuA6ѕ+WJ=z%Td}vQ;d{8zNPBa/{^`TDfcbȐ!5ZPP1c`ر8vϮ;5L߮];L: Wg L6 :tl2a5ȑ#1~xZ _~1e̘1C}K i4 zBϵ\PPXE @ U`/{A>yȌՒכ &MBBB Χ~*䱽x 646mܻw}YgѸq m׷o_MX-!OB <@pp0jժ)S ""bذał ,WXP]jŀGE>}`oooh4 NDG|/T' v**{ {Q2 c%tSN5ElܸsŁ$U7o.]4L&4i^^^Bk(,,Dbb"222͛7G޽a0ZooJ^?~,ܻw-[8@MDD_?"77aaasϕVΝ;qZ|j*!  UFuAFF\\\ M^7tz턢T*QUȓ/5`/Xx^P}a/0*s"3Vrrr BȮƏ+VSN^zd2aԨQ8q.\p!_|о}{+>` {kFqX%/0F j {.{Q>X]8HB&;;qY@ӦMq5!˖-W_}aÆ >P|,^ea*2|BN(Yl/0 2a/Xx^Pd^`/0'%ÇsAK]1^j^O>3KҮ];L: ;^iӦCLJȕǏ-[#@FD 4;vʫaaaѣ#D9# v;^`/F~ {(1 S.]۸qcڷo_gӧyd4LJ|||h4R-̙3DDRBBiӦQTTlfϞM7oެ*H{]TFAM6%MӨaÆJ+W ٥1%"^Pa<*^"5A&vBU^ b/^`ԁc jɄSNˢٳhٲ%+5E"11͛w0 ³Ǐ5kִ|kؼys횦d2I&*вeK[N U . %%wݻ777?^vip ^= NL a'T ba/P#??ް+uɍ7$U~)~9r֭O`0j! G z v;y{J?Ȅ^`ԁ` 5"Ǐx'|?RÇnݺ>ziy_=(zaÆeVXQj}aԨQ8qJTdo 77#F@KUR|e`@z0qD 8͚5]D>, *G2 7vBUҰ '|a^œ9s0zh899 {ԩSEӦMQV-!9/h(sB̜9ǽ{<;>4m&L~(<;22;w.:w ػw/?!??f^Cv0uT$$$瓺z*M:233!}-ZÇe!֭[ucǢ+@dG v;A ϐ=!d^`/0 A chF4Mx  7nɿvӜMDDÇI& ,Ɍ3W^ӧӦMhӦM4}t3fwssM6j߸q# '":}45oޜF#FjѢ9s[JHH/T`ǎԩS'JNNk׮۷-^@aa!O4|z뭷ٙlll( &L <@{AHd;H~ ; 5# {Q>X-IIIHIIIts%%%QPPmٲKZp!Æ >}P^^ۛ oN-[YZvmkRݺuתU|UӧOd_DAAm۶-ZD-۷SAA.ٲ@JaV*5D5jԠvѤIhtMeY@$ *G*xA ^Pc}0p@3r ɄniC@6l؀aÆ!44GGoٳ^PjWf*NxAHd;PdN8 jL F#1Vʙ3gпM6Yfظq#޽aÆUzѫW/mvB~p ܸq탏OgiصkTh85%8q"4x{{ȑ#~K..Qr@xji cgmĉETTW_}YYYh߻w/1" {<;A>F% r`/0a/0:IdƪHjWe# bccrJ :֯_?#"" u`ȑ?~=77.0"] [project] name = "ijson" license = "BSD-3-Clause AND ISC" description = "Iterative JSON parser with standard Python iterator interfaces" readme = "README.rst" authors = [ { name = "Rodrigo Tobar", email = "rtobar@icrar.org" }, { name = "Ivan Sagalaev", email = "maniac@softwaremaniacs.org" }, ] classifiers = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Libraries :: Python Modules', ] requires-python = ">=3.9" dynamic = ["version"] [project.urls] "Homepage" = "https://github.com/ICRAR/ijson" [tool.setuptools.dynamic] version = { attr = "ijson.version.__version__" } [tool.setuptools.exclude-package-data] "ijson.backends.ext._yajl2" = ["*.c", "*.h"] ijson-3.4.0/pytest.ini000066400000000000000000000001451500700540700146670ustar00rootroot00000000000000[pytest] markers = pull_only: Only test against adaptors for pulling functions (*_gen and *_async) ijson-3.4.0/setup.py000066400000000000000000000067661500700540700143670ustar00rootroot00000000000000import glob import os import shutil import tempfile from setuptools import setup, Extension from setuptools._distutils import ccompiler from setuptools._distutils import sysconfig setupArgs = {} # Check if the yajl library + headers are present # We don't use compiler.has_function because it leaves a lot of files behind # without properly cleaning up def yajl_present(): compiler = ccompiler.new_compiler(verbose=1) sysconfig.customize_compiler(compiler) # CC, CFLAGS, LDFLAGS, etc yajl_version_test_file = tempfile.NamedTemporaryFile(suffix=".c", prefix="yajl_version", delete=False) try: yajl_version_test_file.write(b''' #include int main(int args, char **argv) { #if YAJL_MAJOR != 2 fail to compile #else yajl_version(); #endif return 0; } ''') yajl_version_test_file.close() try: objs = compiler.compile([yajl_version_test_file.name]) compiler.link_shared_lib(objs, 'a', libraries=["yajl"]) return True finally: os.remove(compiler.library_filename('a', lib_type='shared')) for obj in objs: os.remove(obj) except: return False finally: os.remove(yajl_version_test_file.name) ORIGINAL_SOURCES = os.path.join("cextern", "yajl") PATCHED_SOURCES = "yajl-patched-sources" def patch_yajl_sources() -> None: """Make yajl sources ready for direct compilation against them""" # cp cextern/yajl -R $yajl_sources_copy # mkdir $yajl_sources_copy/yajl # cp $yajl_sources_copy/src/api/*.h $yajl_sources_copy/yajl if os.path.isdir(PATCHED_SOURCES): if os.path.isdir(ORIGINAL_SOURCES): print("Folder for yajl sources already exists, removing it") shutil.rmtree(PATCHED_SOURCES) else: # Wheels are build in an isolated environment # where the original sources aren't available print("Yajl Sources are already in the correct place. Skip copy") return print(f"Copy yajl sources to '{PATCHED_SOURCES}'") shutil.copytree( ORIGINAL_SOURCES, PATCHED_SOURCES, ignore=shutil.ignore_patterns("example", "test"), ) headers_original = os.path.join(PATCHED_SOURCES, "src", "api") headers_copy = os.path.join(PATCHED_SOURCES, "yajl") shutil.copytree(headers_original, headers_copy) extra_sources = [] extra_include_dirs = [] libs = ['yajl'] embed_yajl = os.environ.get('IJSON_EMBED_YAJL', None) == '1' if not embed_yajl: have_yajl = yajl_present() else: patch_yajl_sources() extra_sources = sorted(glob.glob(os.path.join(PATCHED_SOURCES, 'src', '*.c'))) extra_sources.remove(os.path.join(PATCHED_SOURCES, 'src', 'yajl_version.c')) extra_include_dirs = [PATCHED_SOURCES, os.path.join(PATCHED_SOURCES, 'src')] libs = [] build_yajl_default = '1' if embed_yajl or have_yajl else '0' build_yajl = os.environ.get('IJSON_BUILD_YAJL2C', build_yajl_default) == '1' if build_yajl: yajl_ext = Extension('ijson.backends._yajl2', language='c', sources=sorted(glob.glob('src/ijson/backends/ext/_yajl2/*.c')) + extra_sources, include_dirs=['src/ijson/backends/ext/_yajl2'] + extra_include_dirs, libraries=libs, depends=glob.glob('src/ijson/backends/ext/_yajl2/*.h')) setupArgs['ext_modules'] = [yajl_ext] setup(**setupArgs) ijson-3.4.0/src/000077500000000000000000000000001500700540700134255ustar00rootroot00000000000000ijson-3.4.0/src/ijson/000077500000000000000000000000001500700540700145475ustar00rootroot00000000000000ijson-3.4.0/src/ijson/__init__.py000066400000000000000000000034161500700540700166640ustar00rootroot00000000000000''' Iterative JSON parser. Main API: - ``ijson.parse``: iterator returning parsing events with the object tree context, see ``ijson.common.parse`` for docs. - ``ijson.items``: iterator returning Python objects found under a specified prefix, see ``ijson.common.items`` for docs. Top-level ``ijson`` module exposes method from the pure Python backend. There's also two other backends using the C library yajl in ``ijson.backends`` that have the same API and are faster under CPython. ''' from ijson.common import JSONError, IncompleteJSONError, ObjectBuilder from ijson.utils import coroutine, sendable_list from .version import __version__ ALL_BACKENDS = ('yajl2_c', 'yajl2_cffi', 'yajl2', 'yajl', 'python') """ All supported backends, in descending order or speed. Not all might be available at runtime. """ def get_backend(backend): """Import the backend named ``backend``""" import importlib return importlib.import_module('ijson.backends.' + backend) def _default_backend(): import os if 'IJSON_BACKEND' in os.environ: return get_backend(os.environ['IJSON_BACKEND']) for backend in ALL_BACKENDS: try: return get_backend(backend) except ImportError: continue raise ImportError('no backends available') backend = _default_backend() del _default_backend basic_parse = backend.basic_parse basic_parse_coro = backend.basic_parse_coro parse = backend.parse parse_coro = backend.parse_coro items = backend.items items_coro = backend.items_coro kvitems = backend.kvitems kvitems_coro = backend.kvitems_coro basic_parse_async = backend.basic_parse_async parse_async = backend.parse_async items_async = backend.items_async kvitems_async = backend.kvitems_async backend_name = backend.backend_name backend = backend.backendijson-3.4.0/src/ijson/backends/000077500000000000000000000000001500700540700163215ustar00rootroot00000000000000ijson-3.4.0/src/ijson/backends/__init__.py000066400000000000000000000033461500700540700204400ustar00rootroot00000000000000import os import warnings class YAJLImportError(ImportError): pass def require_version(version, required): ''' Asserts that the major component of 'version' is equal to 'required'. Raises YAJLImportError otherwise. ''' major, rest = divmod(version, 10000) minor, micro = divmod(rest, 100) if major != required: raise YAJLImportError('YAJL version %s.x required, found %s.%s.%s' % (required, major, minor, micro)) def get_yajl_version(yajl): try: return yajl.yajl_version() except AttributeError: warnings.warn('Cannot determine yajl version, assuming <1.0.12') return 10000 def find_yajl_ctypes(required): ''' Finds and loads yajl shared object of the required major version (1, 2, ...) using ctypes. ''' # Importing ``ctypes`` should be in scope of this function to prevent failure # of `backends`` package load in a runtime where ``ctypes`` is not available. # Example of such environment is Google App Engine (GAE). from ctypes import util, cdll so_name = os.getenv('YAJL_DLL') or util.find_library('yajl') if so_name is None: raise YAJLImportError('YAJL shared object not found.') try: yajl = cdll.LoadLibrary(so_name) except OSError: raise YAJLImportError('Unable to load YAJL.') require_version(get_yajl_version(yajl), required) return yajl def find_yajl_cffi(ffi, required): ''' Finds and loads yajl shared object of the required major version (1, 2, ...) using cffi. ''' try: yajl = ffi.dlopen(os.getenv('YAJL_DLL') or 'yajl') except OSError: raise YAJLImportError('Unable to load YAJL.') require_version(get_yajl_version(yajl), required) return yajl ijson-3.4.0/src/ijson/backends/_yajl2_ctypes_common.py000066400000000000000000000053331500700540700230160ustar00rootroot00000000000000''' Common ctypes routines for yajl library handling ''' from ctypes import Structure, c_uint, c_char, c_ubyte, c_int, c_long, c_longlong, c_double,\ c_void_p, c_char_p, CFUNCTYPE, POINTER, string_at, cast from ijson import common, backends C_EMPTY = CFUNCTYPE(c_int, c_void_p) C_INT = CFUNCTYPE(c_int, c_void_p, c_int) C_LONG = CFUNCTYPE(c_int, c_void_p, c_long) C_LONGLONG = CFUNCTYPE(c_int, c_void_p, c_longlong) C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double) C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint) _all_cfunct_types = (C_EMPTY, C_INT, C_LONG, C_LONGLONG, C_DOUBLE, C_STR) def _get_callback_data(yajl_version): return [ # Mapping of JSON parser events to callback C types and value converters. # Used to define the Callbacks structure and actual callback functions # inside the parse function. ('null', 'null', C_EMPTY, lambda: None), ('boolean', 'boolean', C_INT, lambda v: bool(v)), ('integer', 'number', C_LONG if yajl_version == 1 else C_LONGLONG, lambda v: int(v)), ('double', 'number', C_DOUBLE, lambda v: v), ('number', 'number', C_STR, lambda v, l: common.integer_or_decimal(string_at(v, l).decode("utf-8"))), ('string', 'string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')), ('start_map', 'start_map', C_EMPTY, lambda: None), ('map_key', 'map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')), ('end_map', 'end_map', C_EMPTY, lambda: None), ('start_array', 'start_array', C_EMPTY, lambda: None), ('end_array', 'end_array', C_EMPTY, lambda: None), ] YAJL_OK = 0 YAJL_CANCELLED = 1 YAJL_INSUFFICIENT_DATA = 2 YAJL_ERROR = 3 def get_yajl(version): yajl = backends.find_yajl_ctypes(version) yajl.yajl_alloc.restype = POINTER(c_char) yajl.yajl_get_error.restype = POINTER(c_char) return yajl def _make_callback(send, use_float, field, event, func_type, func): if use_float and field == 'number': return func_type() def c_callback(_context, *args): send((event, func(*args))) return 1 return func_type(c_callback) def make_callbaks(send, use_float, yajl_version): callback_data = _get_callback_data(yajl_version) class Callbacks(Structure): _fields_ = [(name, type) for name, _, type, _ in callback_data] return Callbacks(*[_make_callback(send, use_float, *data) for data in callback_data]), _all_cfunct_types def yajl_get_error(yajl, handle, buffer): perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value try: error = error.decode('utf-8') except UnicodeDecodeError: pass yajl.yajl_free_error(handle, perror) return error ijson-3.4.0/src/ijson/backends/ext/000077500000000000000000000000001500700540700171215ustar00rootroot00000000000000ijson-3.4.0/src/ijson/backends/ext/_yajl2/000077500000000000000000000000001500700540700203015ustar00rootroot00000000000000ijson-3.4.0/src/ijson/backends/ext/_yajl2/async_reading_generator.c000066400000000000000000000144131500700540700253240ustar00rootroot00000000000000/* * asynchronous reading generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "common.h" static int async_reading_generator_init(async_reading_generator *self, PyObject *args, PyObject *kwargs) { self->coro = NULL; self->file = NULL; self->read_func = NULL; self->buf_size = NULL; self->awaitable = NULL; self->events = NULL; self->index = 0; self->file_exhausted = 0; M1_Z(PyArg_ParseTuple(args, "OO", &self->file, &self->buf_size)); Py_INCREF(self->file); Py_INCREF(self->buf_size); if (!PyNumber_Check(self->buf_size)) { PyErr_SetString(PyExc_TypeError, "buf_size argument is not a number"); return -1; } M1_N(self->events = PyList_New(0)); return 0; } int async_reading_generator_add_coro(async_reading_generator *self, pipeline_node *coro_pipeline) { M1_N(self->coro = chain(self->events, coro_pipeline)); assert(("async_reading_generator works only with basic_parse_basecoro", BasicParseBasecoro_Check(self->coro))); return 0; } static void async_reading_generator_dealloc(async_reading_generator *self) { Py_XDECREF(self->events); Py_XDECREF(self->awaitable); Py_XDECREF(self->buf_size); Py_XDECREF(self->read_func); Py_XDECREF(self->file); Py_XDECREF(self->coro); Py_TYPE(self)->tp_free((PyObject*)self); } static void raise_stopiteration(PyObject *value) { #if defined(PYPY_VERSION) // PyPy doesn't seem to support normalised exceptions for coroutines, // see https://foss.heptapod.net/pypy/pypy/-/issues/3965 PyObject *ex_value = PyObject_CallFunctionObjArgs(PyExc_StopIteration, value, NULL); PyErr_SetObject(PyExc_StopIteration, ex_value); Py_DECREF(ex_value); #else PyObject *stop_iteration_args = PyTuple_New(1); PyTuple_SET_ITEM(stop_iteration_args, 0, value); PyErr_SetObject(PyExc_StopIteration, stop_iteration_args); Py_DECREF(stop_iteration_args); #endif } static PyObject *maybe_pop_event(async_reading_generator *self) { PyObject *events = self->events; Py_ssize_t nevents = PyList_Size(events); if (nevents == 0) { return NULL; } PyObject *event = PyList_GET_ITEM(events, self->index++); Py_INCREF(event); if (self->index == nevents) { if (PySequence_DelSlice(events, 0, self->index) == -1) { Py_RETURN_NONE; } self->index = 0; } raise_stopiteration(event); return event; } static int is_gen_coroutine(PyObject *o) { if (PyGen_CheckExact(o)) { PyCodeObject *code = (PyCodeObject *)PyObject_GetAttrString(o, "gi_code"); return code->co_flags & CO_ITERABLE_COROUTINE; } return 0; } static PyObject* value_from_stopiteration() { PyObject *ptype, *pvalue, *ptraceback, *return_value; PyErr_Fetch(&ptype, &pvalue, &ptraceback); if (PyErr_GivenExceptionMatches(pvalue, PyExc_StopIteration)) { return_value = PyObject_GetAttrString(pvalue, "value"); Py_XDECREF(pvalue); } else { return_value = pvalue; } Py_XDECREF(ptype); Py_XDECREF(ptraceback); return return_value; } static PyObject *async_reading_generator_next(PyObject *self) { async_reading_generator *gen = (async_reading_generator *)self; // values are returned via StopIteration exception values if (maybe_pop_event(gen)) { return NULL; } // No events available and nothing else to read, we are done if (gen->file_exhausted) { PyErr_SetNone(PyExc_StopAsyncIteration); return NULL; } // prepare corresponding awaitable if (gen->awaitable == NULL) { if (gen->read_func == NULL) { PyObject *utils35, *get_read_func, *get_read_coro; N_N(utils35 = PyImport_ImportModule("ijson.utils35")); N_N(get_read_func = PyObject_GetAttrString(utils35, "_get_read")); N_N(get_read_coro = PyObject_CallFunctionObjArgs(get_read_func, gen->file, NULL)); N_N(gen->awaitable = PyObject_CallMethod(get_read_coro, "__await__", NULL)); assert(PyIter_Check(gen->awaitable)); Py_DECREF(get_read_coro); Py_DECREF(get_read_func); Py_DECREF(utils35); Py_CLEAR(gen->file); } else { PyObject *read_coro; N_N(read_coro = PyObject_CallFunctionObjArgs(gen->read_func, gen->buf_size, NULL)); // this can be a "normal" awaitable (has an __await__ method) // or a function decorated with types.coroutine (a generator) if (is_gen_coroutine(read_coro)) { gen->awaitable = read_coro; Py_INCREF(gen->awaitable); } else { N_N(gen->awaitable = PyObject_CallMethod(read_coro, "__await__", NULL)); } assert(PyIter_Check(gen->awaitable)); Py_DECREF(read_coro); } } // Propagate values/errors that are not StopIteration PyObject *value = Py_TYPE(gen->awaitable)->tp_iternext(gen->awaitable); if (value) { return value; } else if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { return NULL; } Py_CLEAR(gen->awaitable); // We await on two things: getting the correct read function (only once), // and reading from it (many times, self->read_func is set) if (gen->read_func == NULL) { gen->read_func = value_from_stopiteration(); Py_RETURN_NONE; } // Finished awaiting on read() result, parse it PyObject *buffer = value_from_stopiteration(); Py_buffer view; N_M1(PyObject_GetBuffer(buffer, &view, PyBUF_SIMPLE)); gen->file_exhausted = (view.len == 0); BasicParseBasecoro *basic_parse_basecoro = (BasicParseBasecoro *)gen->coro; PyObject *res = ijson_yajl_parse(basic_parse_basecoro, view.buf, view.len); N_N(res); Py_DECREF(res); PyBuffer_Release(&view); Py_DECREF(buffer); // values are returned via StopIteration exception values if (maybe_pop_event(gen)) { return NULL; } // Keep trying Py_RETURN_NONE; } static PyAsyncMethods async_reading_generator_methods = { .am_await = ijson_return_self, }; PyTypeObject AsyncReadingGeneratorType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(async_reading_generator), .tp_name = "_yajl2.async_reading_generator", .tp_doc = "The awaitable yielded by the asynchronous iterables", .tp_init = (initproc)async_reading_generator_init, .tp_dealloc = (destructor)async_reading_generator_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &async_reading_generator_methods, .tp_iter = ijson_return_self, .tp_iternext = async_reading_generator_next, };ijson-3.4.0/src/ijson/backends/ext/_yajl2/async_reading_generator.h000066400000000000000000000021431500700540700253260ustar00rootroot00000000000000/* * asynchronous reading generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ASYNC_READING_GENERATOR_H #define ASYNC_READING_GENERATOR_H #define PY_SSIZE_T_CLEAN #include #include "coro_utils.h" /** * async_reading_generator definition. It keeps the state of the current reading * process, and the list holding the events generated by the yajl parsing. */ typedef struct { PyObject_HEAD PyObject *coro; PyObject *file; PyObject *read_func; PyObject *buf_size; PyObject *awaitable; PyObject *events; Py_ssize_t index; int file_exhausted; } async_reading_generator; /* * Creates and connects the requested coroutine pipeline to this async generator. * Returns -1 on failure, 0 on success. */ int async_reading_generator_add_coro(async_reading_generator *self, pipeline_node *coro_pipeline); extern PyTypeObject AsyncReadingGeneratorType; #endif // ASYNC_READING_GENERATOR_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/basic_parse.c000066400000000000000000000025461500700540700227270ustar00rootroot00000000000000/* * basic_parse generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "basic_parse.h" #include "basic_parse_basecoro.h" #include "common.h" /* * __init__, destructor, __iter__ and __next__ */ static int basicparsegen_init(BasicParseGen *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, args, coro_pipeline)); return 0; } static void basicparsegen_dealloc(BasicParseGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* basicparsegen_iternext(PyObject *self) { BasicParseGen *gen = (BasicParseGen *)self; return reading_generator_next(&gen->reading_gen); } PyTypeObject BasicParseGen_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(BasicParseGen), .tp_name = "_yajl2.basic_parse", .tp_doc = "Generator of (evt,value)", .tp_init = (initproc)basicparsegen_init, .tp_dealloc = (destructor)basicparsegen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = basicparsegen_iternext }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/basic_parse.h000066400000000000000000000012141500700540700227230ustar00rootroot00000000000000/* * basic_parse generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BASIC_PARSE_H #define BASIC_PARSE_H #define PY_SSIZE_T_CLEAN #include #include "reading_generator.h" /** * basic_parse generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } BasicParseGen; /** * basic_parse generator object type */ extern PyTypeObject BasicParseGen_Type; #endif // BASIC_PARSE_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/basic_parse_async.c000066400000000000000000000035651500700540700241260ustar00rootroot00000000000000/* * basic_parse_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "common.h" #include "coro_utils.h" /** * basic_parse_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } BasicParseAsync; /* * __init__, destructor and __anext__ */ static int basicparseasync_init(BasicParseAsync *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, args)); return async_reading_generator_add_coro(self->reading_generator, coro_pipeline); } static void basicparseasync_dealloc(BasicParseAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *basicparseasync_anext(PyObject *self) { BasicParseAsync *gen = (BasicParseAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods basicparseasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = basicparseasync_anext }; PyTypeObject BasicParseAsync_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(BasicParseAsync), .tp_name = "_yajl2.basic_parse_async", .tp_doc = "Asynchronous iterable yielding (evt,value) tuples", .tp_init = (initproc)basicparseasync_init, .tp_dealloc = (destructor)basicparseasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &basicparseasync_methods };ijson-3.4.0/src/ijson/backends/ext/_yajl2/basic_parse_async.h000066400000000000000000000010221500700540700241150ustar00rootroot00000000000000/* * basic_parse_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BASIC_PARSE_ASYNC_H #define BASIC_PARSE_ASYNC_H #define PY_SSIZE_T_CLEAN #include /** * basic_parse_async asynchronous iterable object type */ extern PyTypeObject BasicParseAsync_Type; #endif // BASIC_PARSE_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/basic_parse_basecoro.c000066400000000000000000000176551500700540700246130ustar00rootroot00000000000000/* * basic_parse_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include #include "basic_parse_basecoro.h" #include "common.h" #include "parse_basecoro.h" static inline enames_t get_enames(void *ctx) { return ((yajl_parse_context *)ctx)->module_state->enames; } /* * The YAJL callbacks, they add (evt,value) to a list */ static inline int add_event_and_value(void *ctx, PyObject *evt_name, PyObject *val) { PyObject *target_send = ((yajl_parse_context *)ctx)->target_send; if (ParseBasecoro_Check(target_send)) { Z_N(parse_basecoro_send_impl(target_send, evt_name, val)); Py_DECREF(val); return 1; } PyObject *tuple; Z_N(tuple = PyTuple_New(2)); Py_INCREF(evt_name); // this is an element of our module state's enames member PyTuple_SET_ITEM(tuple, 0, evt_name); PyTuple_SET_ITEM(tuple, 1, val); CORO_SEND(target_send, tuple); Py_DECREF(tuple); return 1; } static int null(void * ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, get_enames(ctx).null_ename, Py_None); } static int boolean(void * ctx, int val) { PyObject *bval = val == 0 ? Py_False : Py_True; Py_INCREF(bval); return add_event_and_value(ctx, get_enames(ctx).boolean_ename, bval); } static int yajl_integer(void *ctx, long long val) { PyObject *ival; Z_N(ival = PyLong_FromLongLong(val)) return add_event_and_value(ctx, get_enames(ctx).number_ename, ival); } static int yajl_double(void *ctx, double val) { PyObject *dval; Z_N(dval = PyFloat_FromDouble(val)) return add_event_and_value(ctx, get_enames(ctx).number_ename, dval); } static int number(void * ctx, const char *numberVal, size_t numberLen) { // If original string has a dot or an "e/E" we return a Decimal // just like in the common module int is_decimal = 0; const char *iter = numberVal; size_t i; for(i=0; i!=numberLen; i++) { char c = *iter++; if( c == '.' || c == 'e' || c == 'E' ) { is_decimal = 1; break; } } PyObject *val; if( !is_decimal ) { char *nval = (char *)malloc(numberLen + 1); memcpy(nval, numberVal, numberLen); nval[numberLen] = 0; char *endptr; val = PyLong_FromString(nval, &endptr, 10); free(nval); assert(("string provided by yajl is not an integer", val != NULL && endptr != nval)); } else { Z_N(val = PyObject_CallFunction(((yajl_parse_context *)ctx)->module_state->Decimal, "s#", numberVal, numberLen)); } return add_event_and_value(ctx, get_enames(ctx).number_ename, val); } static int string_cb(void * ctx, const unsigned char *stringVal, size_t stringLen) { PyObject *val; Z_N(val = PyUnicode_FromStringAndSize((const char *)stringVal, stringLen)) return add_event_and_value(ctx, get_enames(ctx).string_ename, val); } static int start_map(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, get_enames(ctx).start_map_ename, Py_None); } static int map_key(void *ctx, const unsigned char *key, size_t stringLen) { PyObject *val; Z_N(val = PyUnicode_FromStringAndSize((const char *)key, stringLen)) return add_event_and_value(ctx, get_enames(ctx).map_key_ename, val); } static int end_map(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, get_enames(ctx).end_map_ename, Py_None); } static int start_array(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, get_enames(ctx).start_array_ename, Py_None); } static int end_array(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, get_enames(ctx).end_array_ename, Py_None); } static yajl_callbacks decimal_callbacks = { null, boolean, NULL, NULL, number, string_cb, start_map, map_key, end_map, start_array, end_array }; static yajl_callbacks float_callbacks = { null, boolean, yajl_integer, yajl_double, NULL, string_cb, start_map, map_key, end_map, start_array, end_array }; PyObject* ijson_yajl_parse(BasicParseBasecoro *coro, char *buffer, size_t length) { yajl_handle handle = coro->h; yajl_status status; if (length == 0) { status = yajl_complete_parse(handle); } else { status = yajl_parse(handle, (unsigned char *)buffer, length); } if (status != yajl_status_ok) { // An actual problem with the JSON data (otherwise a user error) if (status != yajl_status_client_canceled) { unsigned char *perror = yajl_get_error(handle, 1, (unsigned char *)buffer, length); PyObject *error_obj = PyUnicode_FromString((char *)perror); // error about invalid UTF8 byte sequences can't be converted to string // automatically, so we show the bytes instead if (!error_obj) { PyErr_Clear(); error_obj = PyBytes_FromString((char *)perror); PyErr_Clear(); } PyErr_SetObject(coro->ctx.module_state->IncompleteJSONError, error_obj); if (error_obj) { Py_DECREF(error_obj); } yajl_free_error(handle, perror); } return NULL; } Py_RETURN_NONE; } /* * __init__, destructor, __iter__ and __next__ */ static int basic_parse_basecoro_init(BasicParseBasecoro *self, PyObject *args, PyObject *kwargs) { PyObject *allow_comments = Py_False; PyObject *multiple_values = Py_False; PyObject *use_float = Py_False; PyObject *target_send = NULL; self->h = NULL; char *kwlist[] = {"target_send", "allow_comments", "multiple_values", "use_float", NULL}; if( !PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOO", kwlist, &target_send, &allow_comments, &multiple_values, &use_float) ) { return -1; } self->ctx.target_send = target_send; Py_INCREF(self->ctx.target_send); M1_N(self->ctx.module_state = get_state_from_imported_module()); /* * Prepare yajl handle and configure it * The context given to yajl is the coroutine's target, so the callbacks * directly send values to the target */ yajl_callbacks *callbacks; if (PyObject_IsTrue(use_float)) { callbacks = &float_callbacks; } else { callbacks = &decimal_callbacks; } self->h = yajl_alloc(callbacks, NULL, (void *)&self->ctx); if (!self->h) { PyErr_SetString(PyExc_RuntimeError, "Cannot allocate yajl handler"); return -1; } if (PyObject_IsTrue(allow_comments)) { yajl_config(self->h, yajl_allow_comments, 1); } if (PyObject_IsTrue(multiple_values)) { yajl_config(self->h, yajl_allow_multiple_values, 1); } return 0; } static void basic_parse_basecoro_dealloc(BasicParseBasecoro *self) { if (self->h) { yajl_free(self->h); } Py_XDECREF(self->ctx.target_send); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* basic_parse_basecoro_send(PyObject *self, PyObject *arg) { /* Preempt our execution, which might be very long */ // N_M1(PyErr_CheckSignals()); Py_buffer bufview; N_M1(PyObject_GetBuffer(arg, &bufview, PyBUF_SIMPLE)); BasicParseBasecoro *gen = (BasicParseBasecoro *)self; PyObject *ret = ijson_yajl_parse(gen, bufview.buf, bufview.len); if (ret != NULL && bufview.len == 0) { // This was the last one, let's end now PyErr_SetNone(PyExc_StopIteration); ret = NULL; } PyBuffer_Release(&bufview); return ret; } static PyObject* basic_parse_basecoro_close(PyObject *self, PyObject *args) { BasicParseBasecoro *gen = (BasicParseBasecoro *)self; N_N(ijson_yajl_parse(gen, NULL, 0)); Py_RETURN_NONE; } static PyMethodDef basic_parse_basecoro_methods[] = { {"send", basic_parse_basecoro_send, METH_O, "coroutine's send method"}, {"close", basic_parse_basecoro_close, METH_NOARGS, "coroutine's close method"}, {NULL, NULL, 0, NULL} }; PyTypeObject BasicParseBasecoro_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(BasicParseBasecoro), .tp_name = "_yajl2.basic_parse_basecoro", .tp_doc = "Coroutine dispatching (evt,value) pairs", .tp_init = (initproc)basic_parse_basecoro_init, .tp_dealloc = (destructor)basic_parse_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = basic_parse_basecoro_methods }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/basic_parse_basecoro.h000066400000000000000000000023141500700540700246020ustar00rootroot00000000000000/* * basic_parse_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BASIC_PARSE_BASECORO_H #define BASIC_PARSE_BASECORO_H #define PY_SSIZE_T_CLEAN #include #include #include #include "module_state.h" typedef struct _yajl_parse_context { yajl2_state *module_state; PyObject *target_send; } yajl_parse_context; /** * basic_parse_basecoro coroutine object structure */ typedef struct { PyObject_HEAD yajl_handle h; yajl_parse_context ctx; } BasicParseBasecoro; /** * basic_parse_basecoro coroutine object type */ extern PyTypeObject BasicParseBasecoro_Type; /** * Utility function to check if an object is an basic_parse_basecoro coroutine or not */ #define BasicParseBasecoro_Check(o) (Py_TYPE(o) == &BasicParseBasecoro_Type) /** * yajl parsing routine wrapper that turns yajl errors into exceptions */ PyObject* ijson_yajl_parse(BasicParseBasecoro *coro, char *buffer, size_t length); #endif // BASIC_PARSE_BASECORO_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/builder.h000066400000000000000000000110441500700540700221000ustar00rootroot00000000000000/* * builder_t type and associated methods * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2019 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BUILDER_H #define BUILDER_H #include #include "common.h" #include "event_names.h" #define PY_SSIZE_T_CLEAN #include /** * builder_t structure. * * This is the parallel of the ObjectBuilder class from the ijson.common module, * only a bit more complicated since it's all C */ typedef struct _builder { PyObject *value; int active; PyObject *key; PyObject *value_stack; PyObject *map_type; } builder_t; /** * Initializes an empty builder which can be safely destroyed. * * @param builder the builder to empty-initialize */ static inline void builder_create(builder_t *builder) { builder->value = NULL; builder->map_type = NULL; builder->value_stack = NULL; } /** * Initialilzes a builder capable of assembling Python objects out of prefixed * events. * * @param builder the builder to initialize * @param map_type The mapping type to use for constructing objects out of * (key, value) pairs. If None then dict is used. */ static inline int builder_init(builder_t *builder, PyObject *map_type) { M1_N(builder->value_stack = PyList_New(0)); if (map_type != Py_None) { builder->map_type = map_type; Py_INCREF(map_type); } return 0; } /** * Destroys a builder and all its associated contents * @param builder The builder to destroy */ static inline void builder_destroy(builder_t *builder) { Py_XDECREF(builder->value_stack); Py_XDECREF(builder->map_type); Py_XDECREF(builder->value); } /** * Returns whether the builder is currently active or not. * @param builder A builder * @return whether the builder is active (1) or not (0) */ static inline int builder_isactive(builder_t *builder) { return builder->active; } /** * Returns a new reference to the current value constructed by the builder. * * @param builder The builder * @return The value as currently constructed by this builder */ static inline PyObject *builder_value(builder_t *builder) { Py_INCREF(builder->value); return builder->value; } /** * Resets a builder to a pristine state so it can be used to build a new value. * @param builder The builder to reset * @return 0 if successful, -1 in case of an error */ static inline int builder_reset(builder_t *builder) { builder->active = 0; Py_CLEAR(builder->value); Py_CLEAR(builder->key); Py_ssize_t nvals = PyList_GET_SIZE(builder->value_stack); M1_M1(PyList_SetSlice(builder->value_stack, 0, nvals, NULL)); return 0; } static inline int _builder_add(builder_t *builder, PyObject *value) { Py_ssize_t nvals = PyList_GET_SIZE(builder->value_stack); if (nvals == 0) { Py_INCREF(value); builder->value = value; } else { PyObject *last = PyList_GET_ITEM(builder->value_stack, nvals - 1); assert(("stack element not list or dict-like", PyList_Check(last) || PyMapping_Check(last))); if (PyList_Check(last)) { M1_M1(PyList_Append(last, value)); } else { // it's a dict-like object M1_M1(PyObject_SetItem(last, builder->key, value)); } } return 0; } /** * Feed an (event, value) pair to the builder for further constructing the * underlying value * @param builder A builder * @param ename The event name * @param value The value associated to this event * @return 0 if successful, -1 in case of an error */ static inline int builder_event(builder_t *builder, enames_t enames, PyObject *ename, PyObject *value) { builder->active = 1; if (ename == enames.map_key_ename) { Py_XDECREF(builder->key); builder->key = value; Py_INCREF(builder->key); } else if (ename == enames.start_map_ename) { PyObject *mappable; if (builder->map_type) { mappable = PyObject_CallFunctionObjArgs(builder->map_type, NULL); } else { mappable = PyDict_New(); } M1_N(mappable); M1_M1(_builder_add(builder, mappable)); M1_M1(PyList_Append(builder->value_stack, mappable)); Py_DECREF(mappable); } else if (ename == enames.start_array_ename) { PyObject *list; M1_N(list = PyList_New(0)); M1_M1(_builder_add(builder, list)); M1_M1(PyList_Append(builder->value_stack, list)); Py_DECREF(list); } else if (ename == enames.end_array_ename || ename == enames.end_map_ename) { // pop Py_ssize_t nvals = PyList_GET_SIZE(builder->value_stack); M1_M1(PyList_SetSlice(builder->value_stack, nvals-1, nvals, NULL)); } else { M1_M1(_builder_add(builder, value)); } return 0; } #endif /* BUILDER_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/common.h000066400000000000000000000036751500700540700217550ustar00rootroot00000000000000/* * Common definitions for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2019 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef COMMON_H #define COMMON_H #define PY_SSIZE_T_CLEAN #include /* * Error-handling macros to help reducing clutter in the code. * N: NULL, M1: -1, Z: zero, NZ: not-zero, LZ: less-than-zero * */ #define RETURN_X_IF_COND(statement, X, cond) \ do { \ if ((statement) cond) { \ return X; \ } \ } while(0); #define M1_M1(stmt) RETURN_X_IF_COND(stmt, -1, == -1) #define M1_N(stmt) RETURN_X_IF_COND(stmt, -1, == NULL) #define M1_NZ(stmt) RETURN_X_IF_COND(stmt, -1, != 0) #define M1_Z(stmt) RETURN_X_IF_COND(stmt, -1, == 0) #define N_M1(stmt) RETURN_X_IF_COND(stmt, NULL, == -1) #define N_N(stmt) RETURN_X_IF_COND(stmt, NULL, == NULL) #define N_Z(stmt) RETURN_X_IF_COND(stmt, NULL, == 0) #define N_NZ(stmt) RETURN_X_IF_COND(stmt, NULL, != 0) #define Z_M1(stmt) RETURN_X_IF_COND(stmt, 0, == -1) #define Z_N(stmt) RETURN_X_IF_COND(stmt, 0, == NULL) #define Z_NZ(stmt) RETURN_X_IF_COND(stmt, 0, != 0) #define X_LZ(stmt, X) RETURN_X_IF_COND(stmt, X, < 0) #define X_N(stmt, X) RETURN_X_IF_COND(stmt, X, == NULL) #define CORO_SEND(target_send, event) \ { \ if (PyList_Check(target_send)) { \ Z_M1(PyList_Append(target_send, event)); \ } \ else { \ Z_N( PyObject_CallFunctionObjArgs(target_send, event, NULL) ); \ } \ } /* Common function used by __iter__ method in coroutines/generators */ PyObject* ijson_return_self(PyObject *self); /* Common function used by empty methods in coroutines/generators */ PyObject* ijson_return_none(PyObject *self); /* Unpacks an iterable into multiple values, returns 0 on success, -1 on error */ int ijson_unpack(PyObject *o, Py_ssize_t expected, ...); #endif /* COMMON_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/coro_utils.c000066400000000000000000000023301500700540700226250ustar00rootroot00000000000000/* * Coroutine utilities implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "coro_utils.h" static PyObject * create_coro(PyObject *target, pipeline_node node) { PyObject *coro_args; if (node.args) { int nargs = PyTuple_Size(node.args); N_N(coro_args = PyTuple_New(nargs + 1)); Py_INCREF(target); PyTuple_SET_ITEM(coro_args, 0, target); int i; for (i = 0; i != nargs; i++) { PyTuple_SET_ITEM(coro_args, i + 1, PySequence_GetItem(node.args, i)); } } else { N_N(coro_args = PyTuple_Pack(1, target)); } PyObject *coro = PyObject_Call((PyObject *)node.type, coro_args, node.kwargs); Py_DECREF(coro_args); return coro; } PyObject *chain(PyObject *sink, pipeline_node *coro_pipeline) { PyObject *coro = NULL; PyObject *prev = sink; int element = 0; Py_INCREF(prev); while (1) { pipeline_node node = coro_pipeline[element++]; if (node.type == NULL) { break; } coro = create_coro(prev, node); Py_DECREF(prev); N_N(coro); prev = coro; } return coro; }ijson-3.4.0/src/ijson/backends/ext/_yajl2/coro_utils.h000066400000000000000000000017331500700540700226400ustar00rootroot00000000000000/* * Coroutine utilities for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef CORO_UTILS_H #define CORO_UTILS_H #define PY_SSIZE_T_CLEAN #include /** * A tuple defining how to create an object */ typedef struct _pipeline_node { PyTypeObject *type; PyObject *args; PyObject *kwargs; } pipeline_node; /** * Creates coroutines as described in coro_pipeline, with a final sink * * @param sink the final coroutine-like object that will receive the result of * the pipeline * @param coro_pipeline the description of all elements to create in a coroutine * pipeline * @return The head of the coroutine pipeline (i.e., the coroutine where users * will send elements to) */ PyObject *chain(PyObject *sink, pipeline_node *coro_pipeline); #endif // CORO_UTILS_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/event_names.h000066400000000000000000000042751500700540700227660ustar00rootroot00000000000000/* * Event name singletons for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2024 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef EVENT_NAMES_H #define EVENT_NAMES_H #define PY_SSIZE_T_CLEAN #include #define N_EVENTS 11 /* * A structure (and variable) holding utf-8 strings with the event names * This way we avoid calculating them every time, and we can compare them * via direct equality comparison instead of via strcmp. * In other words, this is our own idea of string interning. */ typedef struct _event_names { PyObject *null_ename; PyObject *boolean_ename; PyObject *integer_ename; PyObject *double_ename; PyObject *number_ename; PyObject *string_ename; PyObject *start_map_ename; PyObject *map_key_ename; PyObject *end_map_ename; PyObject *start_array_ename; PyObject *end_array_ename; Py_hash_t hashes[N_EVENTS]; /* same order as before */ } enames_t; #define FOR_EACH_EVENT(f) \ f(0, null_ename, "null"); \ f(1, boolean_ename, "boolean"); \ f(2, integer_ename, "integer"); \ f(3, double_ename, "double"); \ f(4, number_ename, "number"); \ f(5, string_ename, "string"); \ f(6, start_map_ename, "start_map"); \ f(7, map_key_ename, "map_key"); \ f(8, end_map_ename, "end_map"); \ f(9, start_array_ename, "start_array"); \ f(10, end_array_ename, "end_array"); /* Returns the module-internal event name unicode object for the given */ static inline PyObject *get_builtin_ename(enames_t *enames, PyObject *event) { #define SWAP(x, y) { Py_INCREF(y); Py_DECREF(x); return y; } /* Compare by pointer equality first, then by hash */ #define MATCH(i, member, _value) if (enames->member == event) SWAP(event, enames->member) FOR_EACH_EVENT(MATCH) #undef MATCH Py_hash_t hash = PyObject_Hash(event); Py_hash_t *hashes = enames->hashes; #define MATCH(i, member, _value) if (hashes[i] == hash) SWAP(event, enames->member) FOR_EACH_EVENT(MATCH) #undef MATCH return event; } #endif /* EVENT_NAMES_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/items.c000066400000000000000000000031571500700540700215740ustar00rootroot00000000000000/* * items generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "items.h" #include "items_basecoro.h" #include "parse_basecoro.h" #include "basic_parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int itemsgen_init(ItemsGen *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *items_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&ItemsBasecoro_Type, items_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; int res = reading_generator_init(&self->reading_gen, reading_args, coro_pipeline); Py_DECREF(items_args); Py_DECREF(reading_args); return res; } static void itemsgen_dealloc(ItemsGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* itemsgen_iternext(PyObject *self) { ItemsGen *gen = (ItemsGen *)self; return reading_generator_next(&gen->reading_gen); } /* * items generator object type */ PyTypeObject ItemsGen_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(ItemsGen), .tp_name = "_yajl2.items", .tp_doc = "Generates items", .tp_init = (initproc)itemsgen_init, .tp_dealloc = (destructor)itemsgen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = itemsgen_iternext }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/items.h000066400000000000000000000010561500700540700215750ustar00rootroot00000000000000/* * items generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ITEMS_H #define ITEMS_H #include "reading_generator.h" /** * items generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } ItemsGen; /** * items generator object type */ extern PyTypeObject ItemsGen_Type; #endif /* ITEMS_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/items_async.c000066400000000000000000000041361500700540700227670ustar00rootroot00000000000000/* * items_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "parse_basecoro.h" #include "items_basecoro.h" #include "common.h" #include "coro_utils.h" /** * items_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } ItemsAsync; /* * __init__, destructor and __anext__ */ static int itemsasync_init(ItemsAsync *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *items_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&ItemsBasecoro_Type, items_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); int ret = async_reading_generator_add_coro(self->reading_generator, coro_pipeline); Py_DECREF(items_args); Py_DECREF(reading_args); return ret; } static void itemsasync_dealloc(ItemsAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *itemsasync_anext(PyObject *self) { ItemsAsync *gen = (ItemsAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods itemsasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = itemsasync_anext }; PyTypeObject ItemsAsync_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(ItemsAsync), .tp_name = "_yajl2._items_async", .tp_doc = "Asynchronous iterable yielding fully-built items", .tp_init = (initproc)itemsasync_init, .tp_dealloc = (destructor)itemsasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &itemsasync_methods };ijson-3.4.0/src/ijson/backends/ext/_yajl2/items_async.h000066400000000000000000000007651500700540700230000ustar00rootroot00000000000000/* * items_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ITEMS_ASYNC_H #define ITEMS_ASYNC_H #define PY_SSIZE_T_CLEAN #include /** * items_async asynchronous iterable object type */ extern PyTypeObject ItemsAsync_Type; #endif // ITEMS_ASYNC_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/items_basecoro.c000066400000000000000000000062361500700540700234520ustar00rootroot00000000000000/* * items_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "items_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int items_basecoro_init(ItemsBasecoro *self, PyObject *args, PyObject *kwargs) { self->target_send = NULL; self->prefix = NULL; self->object_depth = 0; M1_N(self->module_state = get_state_from_imported_module()); builder_create(&self->builder); PyObject *map_type; M1_Z(PyArg_ParseTuple(args, "OOO", &(self->target_send), &(self->prefix), &map_type)); Py_INCREF(self->target_send); Py_INCREF(self->prefix); M1_M1(builder_init(&self->builder, map_type)); return 0; } static void items_basecoro_dealloc(ItemsBasecoro *self) { Py_XDECREF(self->prefix); Py_XDECREF(self->target_send); builder_destroy(&self->builder); Py_TYPE(self)->tp_free((PyObject*)self); } PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value) { ItemsBasecoro *coro = (ItemsBasecoro *)self; enames_t enames = coro->module_state->enames; if (builder_isactive(&coro->builder)) { coro->object_depth += (event == enames.start_map_ename || event == enames.start_array_ename); coro->object_depth -= (event == enames.end_map_ename || event == enames.end_array_ename); if (coro->object_depth > 0) { N_M1( builder_event(&coro->builder, enames, event, value) ); } else { PyObject *retval = builder_value(&coro->builder); CORO_SEND(coro->target_send, retval); Py_DECREF(retval); N_M1(builder_reset(&coro->builder)); } } else { int cmp = PyObject_RichCompareBool(path, coro->prefix, Py_EQ); N_M1(cmp); if (cmp) { if (event == enames.start_map_ename || event == enames.start_array_ename) { coro->object_depth = 1; N_M1(builder_event(&coro->builder, enames, event, value)); } else { CORO_SEND(coro->target_send, value); } } } Py_RETURN_NONE; } static PyObject* items_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *path = NULL; PyObject *event = NULL; PyObject *value = NULL; PyObject *result = NULL; if(!ijson_unpack(tuple, 3, &path, &event, &value)) { event = get_builtin_ename(&((ItemsBasecoro *)self)->module_state->enames, event); result = items_basecoro_send_impl(self, path, event, value); } Py_XDECREF(value); Py_XDECREF(event); Py_XDECREF(path); return result; } static PyMethodDef items_basecoro_methods[] = { {"send", items_basecoro_send, METH_O, "coroutine's send method"}, {NULL, NULL, 0, NULL} }; /* * items generator object type */ PyTypeObject ItemsBasecoro_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(ItemsBasecoro), .tp_name = "_yajl2.items_basecoro", .tp_doc = "Coroutine dispatching fully-built objects for the given prefix", .tp_init = (initproc)items_basecoro_init, .tp_dealloc = (destructor)items_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = items_basecoro_methods }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/items_basecoro.h000066400000000000000000000024011500700540700234450ustar00rootroot00000000000000/* * items_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ITEMS_BASECORO_H #define ITEMS_BASECORO_H #include "builder.h" #include "module_state.h" /** * items_basecoro coroutine object structure */ typedef struct { PyObject_HEAD builder_t builder; PyObject *target_send; PyObject *prefix; int object_depth; yajl2_state *module_state; } ItemsBasecoro; /** * items_basecoro coroutine object type */ extern PyTypeObject ItemsBasecoro_Type; /** * Utility function to check if an object is an items_basecoro coroutine or not */ #define ItemsBasecoro_Check(o) (Py_TYPE(o) == &ItemsBasecoro_Type) /** * The implementation of the items_basecoro.send() method accepting an unpacked * event * @param self An items_basecoro coroutine * @param path The path of this event * @param event The event name * @param value The value of this event * @return None, or NULL in case of an error */ PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value); #endif // ITEMS_BASECORO_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/kvitems.c000066400000000000000000000032431500700540700221310ustar00rootroot00000000000000/* * kvitems generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "basic_parse_basecoro.h" #include "common.h" #include "kvitems.h" #include "kvitems_basecoro.h" #include "parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int kvitemsgen_init(KVItemsGen *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *kvitems_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&KVItemsBasecoro_Type, kvitems_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; int res = reading_generator_init(&self->reading_gen, reading_args, coro_pipeline); Py_DECREF(kvitems_args); Py_DECREF(reading_args); return res; } static void kvitemsgen_dealloc(KVItemsGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* kvitemsgen_iternext(PyObject *self) { KVItemsGen *gen = (KVItemsGen *)self; return reading_generator_next(&gen->reading_gen); } /* * kvitems generator object type */ PyTypeObject KVItemsGen_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(KVItemsGen), .tp_name = "_yajl2.kvitems", .tp_doc = "Generates key/value pairs", .tp_init = (initproc)kvitemsgen_init, .tp_dealloc = (destructor)kvitemsgen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = kvitemsgen_iternext }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/kvitems.h000066400000000000000000000010751500700540700221370ustar00rootroot00000000000000/* * kvitems generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef KVITEMS_H #define KVITEMS_H #include "reading_generator.h" /** * kvitems generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } KVItemsGen; /** * kvitems generator object type */ extern PyTypeObject KVItemsGen_Type; #endif /* KVITEMS_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/kvitems_async.c000066400000000000000000000042121500700540700233230ustar00rootroot00000000000000/* * kvitems_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "parse_basecoro.h" #include "kvitems_basecoro.h" #include "common.h" #include "coro_utils.h" /** * kvitems_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } KVItemsAsync; /* * __init__, destructor and __anext__ */ static int kvitemsasync_init(KVItemsAsync *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *kvitems_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&KVItemsBasecoro_Type, kvitems_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); int ret = async_reading_generator_add_coro(self->reading_generator, coro_pipeline); Py_DECREF(kvitems_args); Py_DECREF(reading_args); return ret; } static void kvitemsasync_dealloc(KVItemsAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *kvitemsasync_anext(PyObject *self) { KVItemsAsync *gen = (KVItemsAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods kvitemsasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = kvitemsasync_anext }; PyTypeObject KVItemsAsync_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(KVItemsAsync), .tp_name = "_yajl2._kvitems_async", .tp_doc = "Asynchronous iterable yielding key/value pairs", .tp_init = (initproc)kvitemsasync_init, .tp_dealloc = (destructor)kvitemsasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &kvitemsasync_methods };ijson-3.4.0/src/ijson/backends/ext/_yajl2/kvitems_async.h000066400000000000000000000010011500700540700233210ustar00rootroot00000000000000/* * kvitems_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef KVITEMS_ASYNC_H #define KVITEMS_ASYNC_H #define PY_SSIZE_T_CLEAN #include /** * kvitems_async asynchronous iterable object type */ extern PyTypeObject KVItemsAsync_Type; #endif // KVITEMS_ASYNC_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/kvitems_basecoro.c000066400000000000000000000072651500700540700240160ustar00rootroot00000000000000/* * kvitems_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "kvitems_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int kvitems_basecoro_init(KVItemsBasecoro *self, PyObject *args, PyObject *kwargs) { self->target_send = NULL; self->prefix = NULL; self->key = NULL; M1_N(self->module_state = get_state_from_imported_module()); builder_create(&self->builder); PyObject *map_type; M1_Z(PyArg_ParseTuple(args, "OOO", &(self->target_send), &(self->prefix), &map_type)); Py_INCREF(self->target_send); Py_INCREF(self->prefix); M1_M1(builder_init(&self->builder, map_type)); return 0; } static void kvitems_basecoro_dealloc(KVItemsBasecoro *self) { Py_XDECREF(self->prefix); Py_XDECREF(self->key); Py_XDECREF(self->target_send); builder_destroy(&self->builder); Py_TYPE(self)->tp_free((PyObject*)self); } static int kvitems_basecoro_start_new_member(KVItemsBasecoro *coro, PyObject *key) { coro->object_depth = 0; Py_XDECREF(coro->key); coro->key = key; Py_INCREF(coro->key); M1_M1(builder_reset(&coro->builder)); coro->builder.active = 1; return 0; } PyObject* kvitems_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value) { KVItemsBasecoro *coro = (KVItemsBasecoro *)self; enames_t enames = coro->module_state->enames; PyObject *retval = NULL; PyObject *retkey = NULL; if (builder_isactive(&coro->builder)) { coro->object_depth += (event == enames.start_map_ename); coro->object_depth -= (event == enames.end_map_ename); if ((event != enames.map_key_ename || coro->object_depth != 0) && (event != enames.end_map_ename || coro->object_depth != -1)) { N_M1(builder_event(&coro->builder, enames, event, value)); } else { retval = builder_value(&coro->builder); retkey = coro->key; Py_INCREF(retkey); if (event == enames.map_key_ename) { N_M1(kvitems_basecoro_start_new_member(coro, value)); } else { Py_CLEAR(coro->key); coro->builder.active = 0; } } } else { int cmp = PyObject_RichCompareBool(path, coro->prefix, Py_EQ); N_M1(cmp); if (cmp == 1 && event == enames.map_key_ename) { N_M1(kvitems_basecoro_start_new_member(coro, value)); } } if (retval) { PyObject *tuple; N_N(tuple = PyTuple_Pack(2, retkey, retval)); Py_XDECREF(retkey); Py_XDECREF(retval); CORO_SEND(coro->target_send, tuple); Py_DECREF(tuple); } Py_RETURN_NONE; } static PyObject* kvitems_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *path = NULL; PyObject *event = NULL; PyObject *value = NULL; PyObject *result = NULL; if(!ijson_unpack(tuple, 3, &path, &event, &value)) { event = get_builtin_ename(&((KVItemsBasecoro *)self)->module_state->enames, event); result = kvitems_basecoro_send_impl(self, path, event, value); } Py_XDECREF(value); Py_XDECREF(event); Py_XDECREF(path); return result; } static PyMethodDef kvitems_basecoro_methods[] = { {"send", kvitems_basecoro_send, METH_O, "coroutine's send method"}, {NULL, NULL, 0, NULL} }; /* * kvitems generator object type */ PyTypeObject KVItemsBasecoro_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(KVItemsBasecoro), .tp_name = "_yajl2.kvitems_basecoro", .tp_doc = "Coroutine dispatching (key, value) tuples", .tp_init = (initproc)kvitems_basecoro_init, .tp_dealloc = (destructor)kvitems_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = kvitems_basecoro_methods }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/kvitems_basecoro.h000066400000000000000000000025371500700540700240200ustar00rootroot00000000000000/* * kvitems_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef KVITEMS_BASECORO_H #define KVITEMS_BASECORO_H #define PY_SSIZE_T_CLEAN #include #include "builder.h" #include "module_state.h" /** * kvitems_basecoro coroutine object structure */ typedef struct { PyObject_HEAD builder_t builder; PyObject *target_send; PyObject *prefix; PyObject *key; int object_depth; yajl2_state *module_state; } KVItemsBasecoro; /** * kvitems_basecoro coroutine object type */ extern PyTypeObject KVItemsBasecoro_Type; /** * Utility function to check if an object is a kvitems_basecoro coroutine or not */ #define KVItemsBasecoro_Check(o) (Py_TYPE(o) == &KVItemsBasecoro_Type) /** * The implementation of the kvitems_basecoro.send() method accepting an unpacked * event * @param self A kvitems_basecoro coroutine * @param path The path of this event * @param event The event name * @param value The value of this event * @return None, or NULL in case of an error */ PyObject* kvitems_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value); #endif /* KVITEMS_BASECORO_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/module.c000066400000000000000000000133041500700540700217330ustar00rootroot00000000000000/* * _yajl2 backend for ijson * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2016 * Copyright by UWA (in the framework of the ICRAR) */ #include #include #include "common.h" #include "async_reading_generator.h" #include "basic_parse.h" #include "basic_parse_async.h" #include "basic_parse_basecoro.h" #include "parse.h" #include "parse_async.h" #include "parse_basecoro.h" #include "items.h" #include "items_async.h" #include "items_basecoro.h" #include "kvitems.h" #include "kvitems_async.h" #include "kvitems_basecoro.h" #define MODULE_NAME "_yajl2" static PyMethodDef yajl2_methods[] = { {NULL, NULL, 0, NULL} /* Sentinel */ }; static int _yajl2_mod_exec(PyObject *m); static void _yajl2_mod_free(void *m); static PyModuleDef_Slot yajl2_slots[] = { {Py_mod_exec, _yajl2_mod_exec}, #if PY_VERSION_HEX >= 0x030C0000 {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, #endif #ifdef Py_GIL_DISABLED {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL}, }; PyObject* ijson_return_self(PyObject *self) { Py_INCREF(self); return self; } PyObject* ijson_return_none(PyObject *self) { Py_RETURN_NONE; } int ijson_unpack(PyObject *o, Py_ssize_t expected, ...) { va_list args; va_start(args, expected); PyObject *iter = PyObject_GetIter(o); if (!iter) { PyErr_Format(PyExc_TypeError, "cannot unpack non-iterable %s object", Py_TYPE(o)->tp_name); return -1; } Py_ssize_t count = 0; for (PyObject *o; (o = PyIter_Next(iter)); count++) { if (count >= expected) { continue; } PyObject **target = va_arg(args, PyObject **); *target = o; } Py_DECREF(iter); if (PyErr_Occurred()) { return -1; } if (count > expected) { PyErr_Format(PyExc_ValueError, "too many values to unpack (excepted %d, got %zd)", expected, count); return -1; } else if (count < expected) { PyErr_Format(PyExc_ValueError, "not enough values to unpack (excepted %d, got %zd)", expected, count); return -1; } return 0; } /* Module initialization */ /* Support for Python 2/3 */ static struct PyModuleDef moduledef = { .m_base = PyModuleDef_HEAD_INIT, .m_name = MODULE_NAME, .m_doc = "wrapper for yajl2 methods", .m_size = sizeof(yajl2_state), .m_methods = yajl2_methods, .m_slots = yajl2_slots, .m_free = _yajl2_mod_free, }; static yajl2_state *get_state(PyObject *module) { yajl2_state *module_state = PyModule_GetState(module); if (!module_state) { PyErr_SetString(PyExc_RuntimeError, "No module state :("); } return module_state; } yajl2_state *get_state_from_imported_module() { #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM <= 0x07031100 // Until 7.3.17 PyPy didn't correctly export PyImport_ImportModuleLevel // see https://github.com/pypy/pypy/issues/5013 PyObject *module = PyImport_ImportModule("ijson.backends." MODULE_NAME); #else PyObject *module = PyImport_ImportModuleLevel( MODULE_NAME, PyEval_GetGlobals(), Py_None, NULL, 1 ); #endif N_N(module); yajl2_state *module_state = get_state(module); Py_DECREF(module); return module_state; } PyMODINIT_FUNC PyInit__yajl2(void) { return PyModuleDef_Init(&moduledef); } static int _yajl2_mod_exec(PyObject *m) { #define ADD_TYPE(name, type) \ { \ type.tp_new = PyType_GenericNew; \ M1_M1(PyType_Ready(&type)); \ Py_INCREF(&type); \ PyModule_AddObject(m, name, (PyObject *)&type); \ } ADD_TYPE("basic_parse_basecoro", BasicParseBasecoro_Type); ADD_TYPE("basic_parse", BasicParseGen_Type); ADD_TYPE("parse_basecoro", ParseBasecoro_Type); ADD_TYPE("parse", ParseGen_Type); ADD_TYPE("kvitems_basecoro", KVItemsBasecoro_Type); ADD_TYPE("kvitems", KVItemsGen_Type); ADD_TYPE("items_basecoro", ItemsBasecoro_Type); ADD_TYPE("items", ItemsGen_Type); ADD_TYPE("_async_reading_iterator", AsyncReadingGeneratorType); ADD_TYPE("basic_parse_async", BasicParseAsync_Type); ADD_TYPE("parse_async", ParseAsync_Type); ADD_TYPE("kvitems_async", KVItemsAsync_Type); ADD_TYPE("items_async", ItemsAsync_Type); yajl2_state *state; M1_N(state = get_state(m)); M1_N(state->dot = PyUnicode_FromString(".")); M1_N(state->item = PyUnicode_FromString("item")); M1_N(state->dotitem = PyUnicode_FromString(".item")); #define INIT_ENAME(i, member, value) \ { \ M1_N(state->enames.member = PyUnicode_FromString(value)); \ state->enames.hashes[i] = PyObject_Hash(state->enames.member); \ } FOR_EACH_EVENT(INIT_ENAME); #undef INIT_ENAME // Import globally-used names PyObject *ijson_common = PyImport_ImportModule("ijson.common"); M1_N(ijson_common); state->JSONError = PyObject_GetAttrString(ijson_common, "JSONError"); state->IncompleteJSONError = PyObject_GetAttrString(ijson_common, "IncompleteJSONError"); Py_DECREF(ijson_common); M1_N(state->JSONError); M1_N(state->IncompleteJSONError); PyObject *decimal_module = PyImport_ImportModule("decimal"); M1_N(decimal_module); state->Decimal = PyObject_GetAttrString(decimal_module, "Decimal"); Py_DECREF(decimal_module); M1_N(state->Decimal); return 0; } void _yajl2_mod_free(void *self) { yajl2_state *state = PyModule_GetState((PyObject *)self); if (!state) { // module could have been initialised but not executed. // We saw this in 3.8 when importing with importlib.module_from_spec // and *not* executing the resulting module with spec.loader.exec_module. return; } Py_XDECREF(state->Decimal); Py_XDECREF(state->IncompleteJSONError); Py_XDECREF(state->JSONError); Py_XDECREF(state->dotitem); Py_XDECREF(state->item); Py_XDECREF(state->dot); #define DECREF(_i, member, _value) Py_XDECREF(state->enames.member) FOR_EACH_EVENT(DECREF) #undef DECREF }ijson-3.4.0/src/ijson/backends/ext/_yajl2/module_state.h000066400000000000000000000016701500700540700231430ustar00rootroot00000000000000/* * Module state definitions for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2024 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef MODULE_STATE_H #define MODULE_STATE_H #define PY_SSIZE_T_CLEAN #include #include "event_names.h" /* * Structure holding the full yajl2_c module state, * thus avoiding static global variables that would end up being shared * across sub-interpreters. */ typedef struct _yajl2_state { enames_t enames; PyObject *dot; PyObject *item; PyObject *dotitem; PyObject *JSONError; PyObject *IncompleteJSONError; PyObject *Decimal; } yajl2_state; /** * Gets the (typed) state from this module, internally import it. * Usable only after the module has been imported. */ yajl2_state *get_state_from_imported_module(); #endif /* MODULE_STATE_H */ijson-3.4.0/src/ijson/backends/ext/_yajl2/parse.c000066400000000000000000000025321500700540700215610ustar00rootroot00000000000000/* * parse generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "basic_parse_basecoro.h" #include "common.h" #include "parse.h" #include "parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int parsegen_init(ParseGen *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, args, coro_pipeline)); return 0; } static void parsegen_dealloc(ParseGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* parsegen_iternext(PyObject *self) { ParseGen *gen = (ParseGen *)self; return reading_generator_next(&gen->reading_gen); } PyTypeObject ParseGen_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(ParseGen), .tp_name = "_yajl2.parse", .tp_doc = "Generates (path,evt,value)", .tp_init = (initproc)parsegen_init, .tp_dealloc = (destructor)parsegen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = parsegen_iternext }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/parse.h000066400000000000000000000010601500700540700215610ustar00rootroot00000000000000/* * parse generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef PARSE_H #define PARSE_H #include "reading_generator.h" /** * parse generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } ParseGen; /** * parse generator object type */ extern PyTypeObject ParseGen_Type; #endif // PARSE_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/parse_async.c000066400000000000000000000035371500700540700227640ustar00rootroot00000000000000/* * parse_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "parse_basecoro.h" #include "common.h" #include "coro_utils.h" /** * parse_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } ParseAsync; /* * __init__, destructor and __anext__ */ static int parseasync_init(ParseAsync *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, args)); return async_reading_generator_add_coro(self->reading_generator, coro_pipeline); } static void parseasync_dealloc(ParseAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *parseasync_anext(PyObject *self) { ParseAsync *gen = (ParseAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods parseasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = parseasync_anext }; PyTypeObject ParseAsync_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(ParseAsync), .tp_name = "_yajl2._parse_async", .tp_doc = "Asynchronous iterable yielding (path,evt,value) tuples", .tp_init = (initproc)parseasync_init, .tp_dealloc = (destructor)parseasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &parseasync_methods };ijson-3.4.0/src/ijson/backends/ext/_yajl2/parse_async.h000066400000000000000000000007651500700540700227710ustar00rootroot00000000000000/* * parse_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef PARSE_ASYNC_H #define PARSE_ASYNC_H #define PY_SSIZE_T_CLEAN #include /** * parse_async asynchronous iterable object type */ extern PyTypeObject ParseAsync_Type; #endif // PARSE_ASYNC_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/parse_basecoro.c000066400000000000000000000104021500700540700234310ustar00rootroot00000000000000/* * parse_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "items_basecoro.h" #include "kvitems_basecoro.h" #include "parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int parse_basecoro_init(ParseBasecoro *self, PyObject *args, PyObject *kwargs) { M1_Z(PyArg_ParseTuple(args, "O", &self->target_send)); Py_INCREF(self->target_send); M1_N(self->path = PyList_New(0)); M1_N(self->module_state = get_state_from_imported_module()); PyObject *empty; M1_N(empty = PyUnicode_FromString("")); int res = PyList_Append(self->path, empty); Py_DECREF(empty); M1_M1(res); return 0; } static void parse_basecoro_dealloc(ParseBasecoro *self) { Py_XDECREF(self->path); Py_XDECREF(self->target_send); Py_TYPE(self)->tp_free((PyObject*)self); } PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *value) { ParseBasecoro *gen = (ParseBasecoro *)self; Py_ssize_t npaths = PyList_GET_SIZE(gen->path); enames_t enames = gen->module_state->enames; PyObject *dot = gen->module_state->dot; PyObject *dotitem = gen->module_state->dotitem; PyObject *item = gen->module_state->item; // Calculate current prefix PyObject *prefix; if (event == enames.end_array_ename || event == enames.end_map_ename) { // pop N_M1(PyList_SetSlice(gen->path, npaths - 1, npaths, NULL)); npaths--; prefix = PyList_GET_ITEM(gen->path, npaths - 1); } else if (event == enames.map_key_ename) { // last_path = path_stack[-2] // to_append = '.' + value if len(path_stack) > 1 else value // new_path = path_stack[-2] + to_append PyObject *last_path = PyList_GET_ITEM(gen->path, npaths - 2); PyObject *last_path_dot = NULL; if (npaths > 2) { last_path_dot = PyUnicode_Concat(last_path, dot); N_N(last_path_dot); last_path = last_path_dot; } PyObject *new_path = PyUnicode_Concat(last_path, value); N_N(new_path); Py_XDECREF(last_path_dot); PyList_SetItem(gen->path, npaths - 1, new_path); prefix = PyList_GET_ITEM(gen->path, npaths - 2); } else { prefix = PyList_GET_ITEM(gen->path, npaths - 1); } // If entering a map/array, append name to path if (event == enames.start_array_ename) { // to_append = '.item' if path_stack[-1] else 'item' // path_stack.append(path_stack[-1] + to_append) PyObject *last_path = PyList_GET_ITEM(gen->path, npaths - 1); if (PyUnicode_GET_LENGTH(last_path) > 0) { PyObject *new_path = PyUnicode_Concat(last_path, dotitem); N_N(new_path); N_M1(PyList_Append(gen->path, new_path)); Py_DECREF(new_path); } else { N_M1(PyList_Append(gen->path, item)); } } else if (event == enames.start_map_ename) { Py_INCREF(Py_None); N_M1(PyList_Append(gen->path, Py_None)); } if (KVItemsBasecoro_Check(gen->target_send)) { kvitems_basecoro_send_impl(gen->target_send, prefix, event, value); } else if (ItemsBasecoro_Check(gen->target_send)) { items_basecoro_send_impl(gen->target_send, prefix, event, value); } else { PyObject *res; N_N(res = PyTuple_Pack(3, prefix, event, value)); CORO_SEND(gen->target_send, res); Py_DECREF(res); } Py_RETURN_NONE; } static PyObject* parse_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *event = NULL; PyObject *value = NULL; PyObject *result = NULL; if(!ijson_unpack(tuple, 2, &event, &value)) { event = get_builtin_ename(&((ParseBasecoro *)self)->module_state->enames, event); result = parse_basecoro_send_impl(self, event, value); } Py_XDECREF(value); Py_XDECREF(event); return result; } static PyMethodDef parse_basecoro_methods[] = { {"send", parse_basecoro_send, METH_O, "coroutine's send method"}, {NULL, NULL, 0, NULL} }; PyTypeObject ParseBasecoro_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_basicsize = sizeof(ParseBasecoro), .tp_name = "_yajl2.parse_basecoro", .tp_doc = "Coroutine dispatching (path,evt,value) tuples", .tp_init = (initproc)parse_basecoro_init, .tp_dealloc = (destructor)parse_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = parse_basecoro_methods }; ijson-3.4.0/src/ijson/backends/ext/_yajl2/parse_basecoro.h000066400000000000000000000023311500700540700234400ustar00rootroot00000000000000/* * parse_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef PARSE_BASECORO_H #define PARSE_BASECORO_H #define PY_SSIZE_T_CLEAN #include #include "module_state.h" /** * parse_basecoro coroutine object structure */ typedef struct { PyObject_HEAD PyObject *target_send; PyObject *path; yajl2_state *module_state; } ParseBasecoro; /** * parse_basecoro coroutine object type */ extern PyTypeObject ParseBasecoro_Type; /** * Utility function to check if an object is a parse_basecoro coroutine or not */ #define ParseBasecoro_Check(o) (Py_TYPE(o) == &ParseBasecoro_Type) /** * The implementation of the parse_basecoro.send() method accepting an unpacked * event * @param self A parse_basecoro coroutine * @param path The path of this event * @param event The event name * @param value The value of this event * @return None, or NULL in case of an error */ PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *value); #endif // PARSE_BASECORO_Hijson-3.4.0/src/ijson/backends/ext/_yajl2/reading_generator.c000066400000000000000000000106301500700540700241240ustar00rootroot00000000000000/* * reading_generator_t object and methods implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include #include "basic_parse_basecoro.h" #include "common.h" #include "reading_generator.h" int reading_generator_init(reading_generator_t *self, PyObject *args, pipeline_node *coro_pipeline) { PyObject *file; Py_ssize_t buf_size = 64 * 1024; M1_Z(PyArg_ParseTuple(args, "On", &file, &buf_size)); // Handle both "read" and "readinto" functions. // The latter allocates a bytearray, which is how we distinguish between // the two cases later if (PyObject_HasAttrString(file, "readinto")) { M1_N(self->read_func = PyObject_GetAttrString(file, "readinto")); PyObject *pbuf_size = Py_BuildValue("n", buf_size); self->buffer = PyObject_CallFunctionObjArgs((PyObject *)&PyByteArray_Type, pbuf_size, NULL); M1_N(self->buffer); Py_DECREF(pbuf_size); } else { M1_N(self->read_func = PyObject_GetAttrString(file, "read")); self->buf_size = PyLong_FromSsize_t(buf_size); self->buffer = NULL; } M1_N(self->events = PyList_New(0)); self->pos = 0; M1_N(self->coro = chain(self->events, coro_pipeline)); assert(("reading_generator works only with basic_parse_basecoro", BasicParseBasecoro_Check(self->coro))); return 0; } void reading_generator_dealloc(reading_generator_t *self) { #if PY_VERSION_HEX >= 0x030C0000 Py_XDECREF(self->exc); #else Py_XDECREF(self->exc.type); Py_XDECREF(self->exc.value); Py_XDECREF(self->exc.traceback); #endif Py_XDECREF(self->read_func); Py_XDECREF(self->events); Py_XDECREF(self->buffer); Py_XDECREF(self->buf_size); Py_XDECREF(self->coro); } static void reading_generator_catch_exception(reading_generator_t *self) { #if PY_VERSION_HEX >= 0x030C0000 self->exc = PyErr_GetRaisedException(); #else PyErr_Fetch(&self->exc.type, &self->exc.value, &self->exc.traceback); #endif } static int reading_generator_exception_caught(reading_generator_t *self) { #if PY_VERSION_HEX >= 0x030C0000 return self->exc != NULL; #else return self->exc.type != NULL; #endif } static void reading_generator_set_exception(reading_generator_t *self) { #if PY_VERSION_HEX >= 0x030C0000 assert(self->exc); PyErr_SetRaisedException(self->exc); self->exc = NULL; #else assert(self->exc.type); PyErr_Restore(self->exc.type, self->exc.value, self->exc.traceback); self->exc.type = NULL; self->exc.value = NULL; self->exc.traceback = NULL; #endif } PyObject *reading_generator_next(reading_generator_t *self) { PyObject *events = self->events; Py_ssize_t nevents = PyList_Size(events); BasicParseBasecoro *basic_parse_basecoro = (BasicParseBasecoro *)self->coro; while (nevents == 0 && !reading_generator_exception_caught(self)) { /* Read data and pass it down to the co-routine */ Py_buffer view; Py_ssize_t length; if (self->buffer == NULL) { // read_func is "read" PyObject *pbuffer = PyObject_CallFunctionObjArgs(self->read_func, self->buf_size, NULL); N_N(pbuffer); N_M1(PyObject_GetBuffer(pbuffer, &view, PyBUF_SIMPLE)); length = view.len; PyObject *send_res = ijson_yajl_parse(basic_parse_basecoro, view.buf, view.len); Py_DECREF(pbuffer); PyBuffer_Release(&view); if (!send_res) { reading_generator_catch_exception(self); } } else { // read_func is "readinto" PyObject *plength = PyObject_CallFunctionObjArgs(self->read_func, self->buffer, NULL); N_N(plength); length = PyLong_AsLong(plength); N_M1(length); Py_DECREF(plength); N_M1(PyObject_GetBuffer(self->buffer, &view, PyBUF_SIMPLE)); PyObject *send_res = ijson_yajl_parse(basic_parse_basecoro, view.buf, length); PyBuffer_Release(&view); if (!send_res) { reading_generator_catch_exception(self); } } nevents = PyList_Size(events); if (length == 0) { break; } } // events are now probably available if (nevents > 0) { PyObject *val = PyList_GetItem(events, self->pos++); Py_INCREF(val); /* empty the list if fully iterated over */ if (self->pos == nevents) { self->pos = 0; N_M1(PySequence_DelSlice(events, 0, nevents)); } return val; } if (reading_generator_exception_caught(self)) { reading_generator_set_exception(self); return NULL; } // no events, let's end the show PyErr_SetNone(PyExc_StopIteration); return NULL; }ijson-3.4.0/src/ijson/backends/ext/_yajl2/reading_generator.h000066400000000000000000000040151500700540700241310ustar00rootroot00000000000000/* * reading_generator_t type and methods for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2019 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef READING_GENERATOR_H #define READING_GENERATOR_H #define PY_SSIZE_T_CLEAN #include #include "coro_utils.h" /** * reading_generator_t type definition */ typedef struct _reading_generator { PyObject *coro; PyObject *read_func; PyObject *buf_size; PyObject *buffer; PyObject *events; #if PY_VERSION_HEX >= 0x030C0000 PyObject *exc; #else struct _exc { PyObject *type; PyObject *value; PyObject *traceback; } exc; #endif Py_ssize_t pos; } reading_generator_t; /** * Initialises a reading_generator_t object from the given arguments, which * should contain a file-like object and a buffer size (optional). * * @param self A reading_generator_t object * @param args A tuple containing a file-like object and a buffer size (optional) * @param coro_pipeline A description of the coroutine pipeline to create internally * in this reading generator, where data will be pushed to, and which will send * events to the events list * @return 0 if successful, -1 in case of an error */ int reading_generator_init(reading_generator_t *self, PyObject *args, pipeline_node *coro_pipeline); /** * Deallocates all resources associated to the given reading_generator_t object. * @param self A reading_generator_t object */ void reading_generator_dealloc(reading_generator_t *self); /** * Advances the reading_generator_t object by reading data off the underlying * file-like object, feeding it into its coro, with results ending * up in self->events, from which they are returned. * @param self A reading_generator_t object * @return The next event generated from this iterative process */ PyObject *reading_generator_next(reading_generator_t *self); #endif // READING_GENERATOR_Hijson-3.4.0/src/ijson/backends/python.py000066400000000000000000000232461500700540700202230ustar00rootroot00000000000000''' Pure-python parsing backend. ''' from json.decoder import scanstring import re from ijson import common, utils import codecs LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S') UNARY_LEXEMES = set('[]{},') EOF = -1, None class UnexpectedSymbol(common.JSONError): def __init__(self, symbol, pos): super(UnexpectedSymbol, self).__init__( 'Unexpected symbol %r at %d' % (symbol, pos) ) @utils.coroutine def utf8_encoder(target): decoder = codecs.getincrementaldecoder('utf-8')() decode = decoder.decode send = target.send while True: try: final = False bdata = (yield) except GeneratorExit: final = True bdata = b'' try: sdata = decode(bdata, final) except UnicodeDecodeError as e: try: target.close() except: pass raise common.IncompleteJSONError(e) if sdata: send(sdata) elif not bdata: target.close() break @utils.coroutine def Lexer(target): """ Parses lexemes out of the incoming content, and sends them to parse_value. A special EOF result is sent when the data source has been exhausted to give parse_value the possibility of raising custom exceptions due to missing content. """ try: data = (yield) except GeneratorExit: data = '' buf = data pos = 0 discarded = 0 send = target.send while True: match = LEXEME_RE.search(buf, pos) if match: lexeme = match.group() if lexeme == '"': pos = match.start() start = pos + 1 while True: try: end = buf.index('"', start) escpos = end - 1 while buf[escpos] == '\\': escpos -= 1 if (end - escpos) % 2 == 0: start = end + 1 else: break except ValueError: try: data = (yield) except GeneratorExit: data = '' if not data: raise common.IncompleteJSONError('Incomplete string lexeme') buf += data send((discarded + pos, buf[pos:end + 1])) pos = end + 1 else: while lexeme not in UNARY_LEXEMES and match.end() == len(buf): try: data = (yield) except GeneratorExit: data = '' if not data: break buf += data match = LEXEME_RE.search(buf, pos) lexeme = match.group() send((discarded + match.start(), lexeme)) pos = match.end() else: # Don't ask data from an already exhausted source if data: try: data = (yield) except GeneratorExit: data = '' if not data: # Normally should raise StopIteration, but can raise # IncompleteJSONError too, which is the point of sending EOF try: target.send(EOF) except StopIteration: pass break discarded += len(buf) buf = data pos = 0 # Parsing states _PARSE_VALUE = 0 _PARSE_ARRAY_ELEMENT_END = 1 _PARSE_OBJECT_KEY = 2 _PARSE_OBJECT_END = 3 # infinity singleton for overflow checks inf = float("inf") @utils.coroutine def parse_value(target, multivalue, use_float): """ Parses results coming out of the Lexer into ijson events, which are sent to `target`. A stack keeps track of the type of object being parsed at the time (a value, and object or array -- the last two being values themselves). A special EOF result coming from the Lexer indicates that no more content is expected. This is used to check for incomplete content and raise the appropriate exception, which wouldn't be possible if the Lexer simply closed this co-routine (either explicitly via .close(), or implicitly by itself finishing and decreasing the only reference to the co-routine) since that causes a GeneratorExit exception that cannot be replaced with a custom one. """ state_stack = [_PARSE_VALUE] pop = state_stack.pop push = state_stack.append send = target.send prev_pos, prev_symbol = None, None to_number = common.integer_or_float if use_float else common.integer_or_decimal while True: if prev_pos is None: pos, symbol = (yield) if (pos, symbol) == EOF: if state_stack: raise common.IncompleteJSONError('Incomplete JSON content') break else: pos, symbol = prev_pos, prev_symbol prev_pos, prev_symbol = None, None try: state = state_stack[-1] except IndexError: if multivalue: state = _PARSE_VALUE push(state) else: raise common.JSONError('Additional data found') assert state_stack if state == _PARSE_VALUE: # Simple, common cases if symbol == 'null': send(('null', None)) pop() elif symbol == 'true': send(('boolean', True)) pop() elif symbol == 'false': send(('boolean', False)) pop() elif symbol[0] == '"': send(('string', parse_string(symbol))) pop() # Array start elif symbol == '[': send(('start_array', None)) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol == ']': send(('end_array', None)) pop() else: prev_pos, prev_symbol = pos, symbol push(_PARSE_ARRAY_ELEMENT_END) push(_PARSE_VALUE) # Object start elif symbol == '{': send(('start_map', None)) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol == '}': send(('end_map', None)) pop() else: prev_pos, prev_symbol = pos, symbol push(_PARSE_OBJECT_KEY) # A number else: # JSON numbers can't contain leading zeros if ((len(symbol) > 1 and symbol[0] == '0' and symbol[1] not in ('e', 'E', '.')) or (len(symbol) > 2 and symbol[0:2] == '-0' and symbol[2] not in ('e', 'E', '.'))): raise common.JSONError('Invalid JSON number: %s' % (symbol,)) # Fractions need a leading digit and must be followed by a digit if symbol[0] == '.' or symbol[-1] == '.': raise common.JSONError('Invalid JSON number: %s' % (symbol,)) try: number = to_number(symbol) if number == inf: raise common.JSONError("float overflow: %s" % (symbol,)) except: if 'true'.startswith(symbol) or 'false'.startswith(symbol) or 'null'.startswith(symbol): raise common.IncompleteJSONError('Incomplete JSON content') raise UnexpectedSymbol(symbol, pos) else: send(('number', number)) pop() elif state == _PARSE_OBJECT_KEY: if symbol[0] != '"': raise UnexpectedSymbol(symbol, pos) send(('map_key', parse_string(symbol))) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol != ':': raise UnexpectedSymbol(symbol, pos) state_stack[-1] = _PARSE_OBJECT_END push(_PARSE_VALUE) elif state == _PARSE_OBJECT_END: if symbol == ',': state_stack[-1] = _PARSE_OBJECT_KEY elif symbol != '}': raise UnexpectedSymbol(symbol, pos) else: send(('end_map', None)) pop() pop() elif state == _PARSE_ARRAY_ELEMENT_END: if symbol == ',': state_stack[-1] = _PARSE_ARRAY_ELEMENT_END push(_PARSE_VALUE) elif symbol != ']': raise UnexpectedSymbol(symbol, pos) else: send(('end_array', None)) pop() pop() def parse_string(symbol): return scanstring(symbol, 1)[0] def basic_parse_basecoro(target, multiple_values=False, allow_comments=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - file: a readable file-like object with JSON input ''' if allow_comments: raise ValueError("Comments are not supported by the python backend") return utf8_encoder(Lexer(parse_value(target, multiple_values, use_float))) common.enrich_backend(globals(), c_comments=False) ijson-3.4.0/src/ijson/backends/yajl.py000066400000000000000000000040001500700540700176240ustar00rootroot00000000000000''' Wrapper for YAJL C library version 1.x. ''' import ctypes from ctypes import Structure, c_uint, byref from ijson import common, utils from ijson.backends import _yajl2_ctypes_common yajl = _yajl2_ctypes_common.get_yajl(1) class Config(Structure): _fields_ = [ ("allowComments", c_uint), ("checkUTF8", c_uint) ] @utils.coroutine def basic_parse_basecoro(target, allow_comments=False, multiple_values=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - check_utf8: if True, parser will cause an error if input is invalid utf-8 - buf_size: a size of an input buffer ''' if multiple_values: raise ValueError("yajl backend doesn't support multiple_values") callbacks, _keepalive = _yajl2_ctypes_common.make_callbaks(target.send, use_float, 1) config = Config(allow_comments, True) handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_parse_complete(handle) if result == _yajl2_ctypes_common.YAJL_ERROR: error = _yajl2_ctypes_common.yajl_get_error(yajl, handle, buffer) raise common.JSONError(error) elif not buffer: if result == _yajl2_ctypes_common.YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError('Incomplete JSON data') break finally: yajl.yajl_free(handle) common.enrich_backend( globals(), multiple_values=False, invalid_leading_zeros_detection=False, incomplete_json_tokens_detection=False, int64=ctypes.sizeof(ctypes.c_long) == 0, ) ijson-3.4.0/src/ijson/backends/yajl2.py000066400000000000000000000034001500700540700177110ustar00rootroot00000000000000''' Wrapper for YAJL C library version 2.x. ''' from ctypes import byref from ijson import common, utils from ijson.backends import _yajl2_ctypes_common yajl = _yajl2_ctypes_common.get_yajl(2) # constants defined in yajl_parse.h YAJL_ALLOW_COMMENTS = 1 YAJL_MULTIPLE_VALUES = 8 @utils.coroutine def basic_parse_basecoro(target, allow_comments=False, multiple_values=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - buf_size: a size of an input buffer - multiple_values: allows the parser to parse multiple JSON objects ''' callbacks, _keepalive = _yajl2_ctypes_common.make_callbaks(target.send, use_float, 2) handle = yajl.yajl_alloc(byref(callbacks), None, None) if allow_comments: yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, 1) if multiple_values: yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, 1) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_complete_parse(handle) if result != _yajl2_ctypes_common.YAJL_OK: error = _yajl2_ctypes_common.yajl_get_error(yajl, handle, buffer) exception = common.IncompleteJSONError if result == _yajl2_ctypes_common.YAJL_INSUFFICIENT_DATA else common.JSONError raise exception(error) if not buffer: break finally: yajl.yajl_free(handle) common.enrich_backend(globals()) ijson-3.4.0/src/ijson/backends/yajl2_c.py000066400000000000000000000043501500700540700202200ustar00rootroot00000000000000# # Contributed by Rodrigo Tobar # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2016 # Copyright by UWA (in the framework of the ICRAR) # ''' Wrapper for _yajl2 C extension module ''' from ijson import common, compat, utils from . import _yajl2 _get_buf_size = lambda kwargs: kwargs.pop('buf_size', 64 * 1024) @utils.coroutine def basic_parse_basecoro(target, **kwargs): return _yajl2.basic_parse_basecoro(target.send, **kwargs) def basic_parse_gen(file, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.basic_parse(f, buf_size, **kwargs) def basic_parse_async(file, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.basic_parse_async(file, buf_size, **kwargs) @utils.coroutine def parse_basecoro(target, **kwargs): return _yajl2.parse_basecoro(target.send, **kwargs) def parse_gen(file, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.parse(f, buf_size, **kwargs) def parse_async(file, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.parse_async(file, buf_size, **kwargs) @utils.coroutine def kvitems_basecoro(target, prefix, map_type=None, **kwargs): return _yajl2.kvitems_basecoro(target.send, prefix, map_type, **kwargs) def kvitems_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.kvitems(f, buf_size, prefix, map_type, **kwargs) def kvitems_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.kvitems_async(file, buf_size, prefix, map_type, **kwargs) @utils.coroutine def items_basecoro(target, prefix, map_type=None, **kwargs): return _yajl2.items_basecoro(target.send, prefix, map_type, **kwargs) def items_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.items(f, buf_size, prefix, map_type, **kwargs) def items_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.items_async(file, buf_size, prefix, map_type, **kwargs) common.enrich_backend(globals()) ijson-3.4.0/src/ijson/backends/yajl2_cffi.py000066400000000000000000000143231500700540700207060ustar00rootroot00000000000000''' CFFI-Wrapper for YAJL C library version 2.x. ''' from cffi import FFI import functools from ijson import common, backends, utils ffi = FFI() ffi.cdef(""" typedef void * (*yajl_malloc_func)(void *ctx, size_t sz); typedef void (*yajl_free_func)(void *ctx, void * ptr); typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz); typedef struct { yajl_malloc_func malloc; yajl_realloc_func realloc; yajl_free_func free; void * ctx; } yajl_alloc_funcs; typedef struct yajl_handle_t * yajl_handle; typedef enum { yajl_status_ok, yajl_status_client_canceled, yajl_status_error } yajl_status; typedef enum { yajl_allow_comments = 0x01, yajl_dont_validate_strings = 0x02, yajl_allow_trailing_garbage = 0x04, yajl_allow_multiple_values = 0x08, yajl_allow_partial_values = 0x10 } yajl_option; typedef struct { int (* yajl_null)(void * ctx); int (* yajl_boolean)(void * ctx, int boolVal); int (* yajl_integer)(void * ctx, long long integerVal); int (* yajl_double)(void * ctx, double doubleVal); int (* yajl_number)(void * ctx, const char * numberVal, size_t numberLen); int (* yajl_string)(void * ctx, const unsigned char * stringVal, size_t stringLen); int (* yajl_start_map)(void * ctx); int (* yajl_map_key)(void * ctx, const unsigned char * key, size_t stringLen); int (* yajl_end_map)(void * ctx); int (* yajl_start_array)(void * ctx); int (* yajl_end_array)(void * ctx); } yajl_callbacks; int yajl_version(void); yajl_handle yajl_alloc(const yajl_callbacks *callbacks, yajl_alloc_funcs *afs, void *ctx); int yajl_config(yajl_handle h, yajl_option opt, ...); yajl_status yajl_parse(yajl_handle hand, const unsigned char *jsonText, size_t jsonTextLength); yajl_status yajl_complete_parse(yajl_handle hand); unsigned char* yajl_get_error(yajl_handle hand, int verbose, const unsigned char *jsonText, size_t jsonTextLength); void yajl_free_error(yajl_handle hand, unsigned char * str); void yajl_free(yajl_handle handle); """) yajl = backends.find_yajl_cffi(ffi, 2) YAJL_OK = 0 YAJL_CANCELLED = 1 YAJL_INSUFFICIENT_DATA = 2 YAJL_ERROR = 3 # constants defined in yajl_parse.h YAJL_ALLOW_COMMENTS = 1 YAJL_MULTIPLE_VALUES = 8 def append_event_to_ctx(event): def wrapper(func): @functools.wraps(func) def wrapped(ctx, *args, **kwargs): value = func(*args, **kwargs) send = ffi.from_handle(ctx) send((event, value)) return 1 return wrapped return wrapper @ffi.callback('int(void *ctx)') @append_event_to_ctx('null') def null(): return None @ffi.callback('int(void *ctx, int val)') @append_event_to_ctx('boolean') def boolean(val): return bool(val) @ffi.callback('int(void *ctx, long long integerVal)') @append_event_to_ctx('number') def integer(val): return int(val) @ffi.callback('int(void * ctx, double doubleVal)') @append_event_to_ctx('number') def double(val): return val @ffi.callback('int(void *ctx, const char *numberVal, size_t numberLen)') @append_event_to_ctx('number') def number(val, length): return common.integer_or_decimal(ffi.string(val, maxlen=length).decode("utf-8")) @ffi.callback('int(void *ctx, const unsigned char *stringVal, size_t stringLen)') @append_event_to_ctx('string') def string(val, length): return ffi.string(val, maxlen=length).decode('utf-8') @ffi.callback('int(void *ctx)') @append_event_to_ctx('start_map') def start_map(): return None @ffi.callback('int(void *ctx, const unsigned char *key, size_t stringLen)') @append_event_to_ctx('map_key') def map_key(key, length): return ffi.string(key, maxlen=length).decode('utf-8') @ffi.callback('int(void *ctx)') @append_event_to_ctx('end_map') def end_map(): return None @ffi.callback('int(void *ctx)') @append_event_to_ctx('start_array') def start_array(): return None @ffi.callback('int(void *ctx)') @append_event_to_ctx('end_array') def end_array(): return None _decimal_callback_data = ( null, boolean, ffi.NULL, ffi.NULL, number, string, start_map, map_key, end_map, start_array, end_array ) _float_callback_data = ( null, boolean, integer, double, ffi.NULL, string, start_map, map_key, end_map, start_array, end_array ) def yajl_init(scope, send, allow_comments=False, multiple_values=False, use_float=False): scope.ctx = ffi.new_handle(send) if use_float: scope.callbacks = ffi.new('yajl_callbacks*', _float_callback_data) else: scope.callbacks = ffi.new('yajl_callbacks*', _decimal_callback_data) handle = yajl.yajl_alloc(scope.callbacks, ffi.NULL, scope.ctx) if allow_comments: yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, ffi.cast('int', 1)) if multiple_values: yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, ffi.cast('int', 1)) return handle def yajl_parse(handle, buffer): if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_complete_parse(handle) if result != YAJL_OK: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = ffi.string(perror) try: error = error.decode('utf8') except UnicodeDecodeError: pass yajl.yajl_free_error(handle, perror) exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError raise exception(error) class Container: pass @utils.coroutine def basic_parse_basecoro(target, **config): ''' Coroutine dispatching unprefixed events. Parameters: - allow_comments: tells parser to allow comments in JSON input - multiple_values: allows the parser to parse multiple JSON objects ''' # the scope objects makes sure the C objects allocated in _yajl.init # are kept alive until this function is done scope = Container() handle = yajl_init(scope, target.send, **config) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' yajl_parse(handle, buffer) if not buffer: break finally: yajl.yajl_free(handle) common.enrich_backend(globals()) ijson-3.4.0/src/ijson/benchmark.py000066400000000000000000000217121500700540700170560ustar00rootroot00000000000000# # Contributed by Rodrigo Tobar # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2019 # Copyright by UWA (in the framework of the ICRAR) # ''' Benchmarking utility for ijson ''' import argparse import collections import contextlib import io import os import sys import time import ijson _benchmarks = collections.OrderedDict() def benchmark(f): _benchmarks[f.__name__] = f return f @benchmark def long_list(n): return b'[' + b','.join([b'1' for _ in range(n)]) + b']' @benchmark def big_int_object(n): return b'{' + b',\n'.join([b'"key_%d": %d' % (i, i) for i in range(n)]) + b'}' @benchmark def big_decimal_object(n): return b'{' + b',\n'.join([b'"key_%d": %d.0' % (i, i) for i in range(n)]) + b'}' @benchmark def big_null_object(n): return b'{' + b',\n'.join([b'"key_%d": null' % (i,) for i in range(n)]) + b'}' @benchmark def big_bool_object(n): return b'{' + b',\n'.join([ b'"key_%d": %s' % (i, b"true" if i % 2 == 0 else b"false") for i in range(n)]) + b'}' @benchmark def big_str_object(n): return b'{' + b',\n'.join([b'"key_%d": "value_%d"' % (i, i) for i in range(n)]) + b'}' @benchmark def big_longstr_object(n): str_template = b"value that is very long and should cause a bit less of JSON parsing" return b'{' + b',\n'.join([b'"key_%d": "%s"' % (i, str_template) for i in range(n)]) + b'}' @benchmark def object_with_10_keys(n): template = b'{' + b',\n'.join([b'"key_%d": "value_%d"' % (i, i) for i in range(10)]) + b'}' return b'[' + b',\n'.join( template for _ in range(n)) + b']' @benchmark def empty_lists(n): return b'[' + b', '.join(b'[]' for _ in range(n)) + b']' @benchmark def empty_objects(n): return b'[' + b', '.join(b'{}' for _ in range(n)) + b']' def parse_benchmarks(s): return [_benchmarks[name] for name in s.split(',')] def load_backends(): backends = collections.OrderedDict() for backend_name in ijson.ALL_BACKENDS: try: backends[backend_name] = ijson.get_backend(backend_name) except ImportError: continue return backends _backends = load_backends() def parse_backends(s): backends = collections.OrderedDict() for name in s.split(','): backends[name] = _backends[name] return backends def _stdout_tty_write_flush(message): stdout = sys.stdout if stdout.isatty(): stdout.write(message) stdout.flush() class progress_message: def __init__(self, message): self.message = message def __enter__(self): _stdout_tty_write_flush(self.message) return self def __exit__(self, *args): _stdout_tty_write_flush('\r\033[K') class AsyncReader: def __init__(self, data): self.data = io.BytesIO(data) async def read(self, n=-1): return self.data.read(n) def close(self): self.data.close() async def _run_async(method, reader, *method_args, **method_kwargs): async for _ in method(reader, *method_args, **method_kwargs): pass def median(values): sorted_values = sorted(values) n = len(sorted_values) if n % 2 == 0: return (sorted_values[n // 2 - 1] + sorted_values[n // 2]) / 2. return sorted_values[n // 2] def stats(values): total = sum(values) return min(values), total / float(len(values)), median(values), max(values) def run_benchmarks(args, benchmark_func=None, fname=None): assert bool(benchmark_func) != bool(fname) if benchmark_func: bname = benchmark_func.__name__ with progress_message('Generating data for benchmark %s...' % (bname,)): data = benchmark_func(args.size) size = len(data) else: bname = fname size = os.stat(fname).st_size # Prepare reader def get_reader(): if not benchmark_func: return open(fname, 'rb') elif args.run_async: return AsyncReader(data) return io.BytesIO(data) for backend_name, backend in args.backends.items(): # Get correct method and prepare its arguments method = args.method if args.run_async: method += '_async' elif args.run_coro: method += '_coro' method = getattr(backend, method) method_args = () if args.method in ('items', 'kvitems'): method_args = args.prefix, method_kwargs = { 'multiple_values': args.multiple_values, 'use_float': args.use_float } if not args.run_coro: method_kwargs['buf_size'] = args.bufsize # Prepare function that will run the benchmark if args.run_async: import asyncio loop = asyncio.new_event_loop() def run(reader): try: loop.run_until_complete(_run_async(method, reader, *method_args, **method_kwargs)) finally: loop.close() elif args.run_coro: def run(reader): from ijson.utils import sendable_list events = sendable_list() coro = method(events, *method_args, **method_kwargs) for chunk in iter(lambda: reader.read(args.bufsize), b''): coro.send(chunk) del events[:] coro.close() else: def run(reader): for _ in method(reader, *method_args, **method_kwargs): pass # Go, go, go! durations = [] now = time.perf_counter for iteration in range(args.iterations): with contextlib.closing(get_reader()) as reader: start = now() run(reader) durations.append(now() - start) megabytes = size / 1024. / 1024. results = ( (megabytes, args.method, bname, backend_name) + stats(durations) + stats([megabytes / duration for duration in durations]) ) print("%.3f, %s, %s, %s, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f, %.3f" % results) def main(): DEFAULT_N = 100000 DEFAULT_ITERATIONS = 1 DEFAULT_BUFSIZE = 64 * 1024 ALL_BENCHMARKS = ','.join(_benchmarks) ALL_BACKENDS = ','.join(_backends) parser = argparse.ArgumentParser() parser.add_argument('-s', '--size', type=int, help='Size of JSON content; actual size in bytes might differ, defaults to %d' % DEFAULT_N, default=DEFAULT_N) parser.add_argument('-I', '--iterations', type=int, help='How many times each method should be tested, defaults to %d' % DEFAULT_ITERATIONS, default=DEFAULT_ITERATIONS) parser.add_argument('-S', '--bufsize', type=int, help='Buffer size used during parsing; defaults to %d' % DEFAULT_BUFSIZE, default=DEFAULT_BUFSIZE) parser.add_argument('-b', '--benchmarks', type=parse_benchmarks, help='Comma-separated list of benchmarks to include, defaults to %s' % ALL_BENCHMARKS, default=ALL_BENCHMARKS) parser.add_argument('-B', '--backends', type=parse_backends, help='Comma-separated list of backends to include, defaults to %s' % ALL_BACKENDS, default=ALL_BACKENDS) parser.add_argument('-l', '--list', action='store_true', help='List available benchmarks and backends') parser.add_argument('inputs', nargs='*', help='File to use for benchmarks rather than built-in benchmarking functions') parser.add_argument('-M', '--multiple-values', action='store_true', default=False, help='Content has multiple JSON values, useful when used with -i') parser.add_argument('-f', '--use-float', action='store_true', default=False, help='Parse non-integer numbers as float instead of Decimal') parser.add_argument('-m', '--method', choices=['basic_parse', 'parse', 'kvitems', 'items'], help='The method to benchmark', default='basic_parse') parser.add_argument('-c', '--coro', action='store_true', default=False, dest='run_coro', help='Benchmark coroutine methods') parser.add_argument('-a', '--async', action='store_true', default=False, dest='run_async', help='Benchmark asyncio-enabled methods') parser.add_argument('-p', '--prefix', help='Prefix (used with -M items|kvitems)', default='') args = parser.parse_args() if args.list: msg = 'Backends:\n' msg += '\n'.join(' - %s' % name for name in _backends) msg += '\nBenchmarks:\n' msg += '\n'.join(' - %s' % name for name in _benchmarks) print(msg) return print("#mbytes, method, test_case, backend, time_min, time_avg, time_median, time_max, mb_per_sec_min, mb_per_sec_avg, mb_per_sec_median, mb_per_sec_max") if args.inputs: for filename in args.inputs: run_benchmarks(args, fname=filename) else: for benchmark in args.benchmarks: run_benchmarks(args, benchmark) if __name__ == '__main__': main() ijson-3.4.0/src/ijson/common.py000066400000000000000000000372561500700540700164260ustar00rootroot00000000000000''' Backend independent higher level interfaces, common exceptions. ''' import decimal import inspect import io import warnings from ijson import compat, utils, utils35 class JSONError(Exception): ''' Base exception for all parsing errors. ''' pass class IncompleteJSONError(JSONError): ''' Raised when the parser can't read expected data from a stream. ''' pass @utils.coroutine def parse_basecoro(target): ''' A coroutine dispatching parsing events with the information about their location with the JSON object tree. Events are tuples ``(prefix, type, value)``. Available types and values are: ('null', None) ('boolean', ) ('number', ) ('string', ) ('map_key', ) ('start_map', None) ('end_map', None) ('start_array', None) ('end_array', None) Prefixes represent the path to the nested elements from the root of the JSON document. For example, given this document:: { "array": [1, 2], "map": { "key": "value" } } the parser would yield events: ('', 'start_map', None) ('', 'map_key', 'array') ('array', 'start_array', None) ('array.item', 'number', 1) ('array.item', 'number', 2) ('array', 'end_array', None) ('', 'map_key', 'map') ('map', 'start_map', None) ('map', 'map_key', 'key') ('map.key', 'string', u'value') ('map', 'end_map', None) ('', 'end_map', None) ''' path = [] while True: event, value = yield if event == 'map_key': prefix = '.'.join(path[:-1]) path[-1] = value elif event == 'start_map': prefix = '.'.join(path) path.append(None) elif event == 'end_map': path.pop() prefix = '.'.join(path) elif event == 'start_array': prefix = '.'.join(path) path.append('item') elif event == 'end_array': path.pop() prefix = '.'.join(path) else: # any scalar value prefix = '.'.join(path) target.send((prefix, event, value)) class ObjectBuilder: ''' Incrementally builds an object from JSON parser events. Events are passed into the `event` function that accepts two parameters: event type and value. The object being built is available at any time from the `value` attribute. Example:: >>> from io import BytesIO >>> from ijson import basic_parse >>> from ijson.common import ObjectBuilder >>> builder = ObjectBuilder() >>> f = BytesIO(b'{"key": "value"}') >>> for event, value in basic_parse(f): ... builder.event(event, value) >>> builder.value == {'key': 'value'} True ''' def __init__(self, map_type=None): def initial_set(value): self.value = value self.containers = [initial_set] self.map_type = map_type or dict def event(self, event, value): if event == 'map_key': self.key = value elif event == 'start_map': mappable = self.map_type() self.containers[-1](mappable) def setter(value): mappable[self.key] = value self.containers.append(setter) elif event == 'start_array': array = [] self.containers[-1](array) self.containers.append(array.append) elif event == 'end_array' or event == 'end_map': self.containers.pop() else: self.containers[-1](value) @utils.coroutine def items_basecoro(target, prefix, map_type=None): ''' An couroutine dispatching native Python objects constructed from the events under a given prefix. ''' while True: current, event, value = (yield) if current == prefix: if event in ('start_map', 'start_array'): object_depth = 1 builder = ObjectBuilder(map_type=map_type) while object_depth: builder.event(event, value) current, event, value = (yield) if event in ('start_map', 'start_array'): object_depth += 1 elif event in ('end_map', 'end_array'): object_depth -= 1 del builder.containers[:] target.send(builder.value) else: target.send(value) @utils.coroutine def kvitems_basecoro(target, prefix, map_type=None): ''' An coroutine dispatching (key, value) pairs constructed from the events under a given prefix. The prefix should point to JSON objects ''' builder = None while True: path, event, value = (yield) while path == prefix and event == 'map_key': object_depth = 0 key = value builder = ObjectBuilder(map_type=map_type) path, event, value = (yield) if event == 'start_map': object_depth += 1 while ( (event != 'map_key' or object_depth != 0) and (event != 'end_map' or object_depth != -1)): builder.event(event, value) path, event, value = (yield) if event == 'start_map': object_depth += 1 elif event == 'end_map': object_depth -= 1 del builder.containers[:] target.send((key, builder.value)) def integer_or_decimal(str_value): ''' Converts string with a numeric value into an int or a Decimal. Used in different backends for consistent number representation. ''' if not ('.' in str_value or 'e' in str_value or 'E' in str_value): return int(str_value) return decimal.Decimal(str_value) def integer_or_float(str_value): ''' Converts string with a numeric value into an int or a float. Used in different backends for consistent number representation. ''' if not ('.' in str_value or 'e' in str_value or 'E' in str_value): return int(str_value) return float(str_value) def number(str_value): warnings.warn("number() function will be removed in a later release", DeprecationWarning) return integer_or_decimal(str_value) def file_source(f, buf_size=64*1024): '''A generator that yields data from a file-like object''' f = compat.bytes_reader(f) while True: data = f.read(buf_size) yield data if not data: break def _basic_parse_pipeline(backend, config): return ( (backend['basic_parse_basecoro'], [], config), ) def _parse_pipeline(backend, config): return ( (backend['parse_basecoro'], [], {}), (backend['basic_parse_basecoro'], [], config) ) def _items_pipeline(backend, prefix, map_type, config): return ( (backend['items_basecoro'], (prefix,), {'map_type': map_type}), (backend['parse_basecoro'], [], {}), (backend['basic_parse_basecoro'], [], config) ) def _kvitems_pipeline(backend, prefix, map_type, config): return ( (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}), (backend['parse_basecoro'], [], {}), (backend['basic_parse_basecoro'], [], config) ) def _make_basic_parse_coro(backend): def basic_parse_coro(target, **config): return utils.chain( target, *_basic_parse_pipeline(backend, config) ) return basic_parse_coro def _make_parse_coro(backend): def parse_coro(target, **config): return utils.chain( target, *_parse_pipeline(backend, config) ) return parse_coro def _make_items_coro(backend): def items_coro(target, prefix, map_type=None, **config): return utils.chain( target, *_items_pipeline(backend, prefix, map_type, config) ) return items_coro def _make_kvitems_coro(backend): def kvitems_coro(target, prefix, map_type=None, **config): return utils.chain( target, *_kvitems_pipeline(backend, prefix, map_type, config) ) return kvitems_coro def is_awaitablefunction(func): """True if `func` is an awaitable function""" return ( inspect.iscoroutinefunction(func) or ( inspect.isgeneratorfunction(func) and (func.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE) ) ) def is_async_file(f): """True if `f` has an asynchronous `read` method""" return ( hasattr(f, 'read') and is_awaitablefunction(f.read) ) def is_file(x): """True if x has a `read` method""" return hasattr(x, 'read') def is_iterable(x): """True if x can be iterated over""" return hasattr(x, '__iter__') def _get_source(source): if isinstance(source, bytes): return io.BytesIO(source) elif isinstance(source, str): return io.StringIO(source) return source def _make_basic_parse_gen(backend): def basic_parse_gen(file_obj, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_basic_parse_pipeline(backend, config) ) return basic_parse_gen def _make_parse_gen(backend): def parse_gen(file_obj, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_parse_pipeline(backend, config) ) return parse_gen def _make_items_gen(backend): def items_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_items_pipeline(backend, prefix, map_type, config) ) return items_gen def _make_kvitems_gen(backend): def kvitems_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_kvitems_pipeline(backend, prefix, map_type, config) ) return kvitems_gen def _make_basic_parse(backend): def basic_parse(source, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['basic_parse_async']( source, buf_size=buf_size, **config ) elif is_file(source): return backend['basic_parse_gen']( source, buf_size=buf_size, **config ) raise ValueError("Unknown source type: %r" % type(source)) return basic_parse def _make_parse(backend): def parse(source, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['parse_async']( source, buf_size=buf_size, **config ) elif is_file(source): return backend['parse_gen']( source, buf_size=buf_size, **config ) elif is_iterable(source): return utils.coros2gen(source, (backend['parse_basecoro'], (), {}) ) raise ValueError("Unknown source type: %r" % type(source)) return parse def _make_items(backend): def items(source, prefix, map_type=None, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['items_async']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_file(source): return backend['items_gen']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_iterable(source): return utils.coros2gen(source, (backend['items_basecoro'], (prefix,), {'map_type': map_type}) ) raise ValueError("Unknown source type: %r" % type(source)) return items def _make_kvitems(backend): def kvitems(source, prefix, map_type=None, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['kvitems_async']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_file(source): return backend['kvitems_gen']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_iterable(source): return utils.coros2gen(source, (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}) ) raise ValueError("Unknown source type: %r" % type(source)) return kvitems _common_functions_warn = ''' Don't use the ijson.common.* functions; instead go directly with the ijson.* ones. See the documentation for more information. ''' def parse(events): """Like ijson.parse, but takes events generated via ijson.basic_parse instead of a file""" warnings.warn(_common_functions_warn, DeprecationWarning) return utils.coros2gen(events, (parse_basecoro, (), {}) ) def kvitems(events, prefix, map_type=None): """Like ijson.kvitems, but takes events generated via ijson.parse instead of a file""" warnings.warn(_common_functions_warn, DeprecationWarning) return utils.coros2gen(events, (kvitems_basecoro, (prefix,), {'map_type': map_type}) ) def items(events, prefix, map_type=None): """Like ijson.items, but takes events generated via ijson.parse instead of a file""" warnings.warn(_common_functions_warn, DeprecationWarning) return utils.coros2gen(events, (items_basecoro, (prefix,), {'map_type': map_type}) ) class BackendCapabilities: ''' Capabilities supported by a backend. ''' __slots__ = { 'c_comments': 'C-ctyle comments (non-standard in JSON)', 'multiple_values': 'Multiple top-level values (non-standard in JSON)', 'invalid_leading_zeros_detection': 'Detection of leading zeros in numbers, marking them as invalid', 'incomplete_json_tokens_detection': 'Documents with incomplete JSON tokens', 'int64': '64 bit integers supported when running with ``use_float=True``', } def __init__(self): self.c_comments = True self.multiple_values = True self.invalid_leading_zeros_detection = True self.incomplete_json_tokens_detection = True self.int64 = True def enrich_backend(backend, **capabilities_overrides): ''' Provides a backend with any missing coroutines/generators/async-iterables it might be missing by using the generic ones written in python. ''' # Backends unset some of these capabilities = BackendCapabilities() for name, value in capabilities_overrides.items(): setattr(capabilities, name, value) backend['capabilities'] = capabilities backend['backend'] = backend['__name__'].split('.')[-1] backend['backend_name'] = backend['backend'] for name in ('basic_parse', 'parse', 'items', 'kvitems'): basecoro_name = name + '_basecoro' if basecoro_name not in backend: backend[basecoro_name] = globals()[basecoro_name] coro_name = name + '_coro' if coro_name not in backend: factory = globals()['_make_' + coro_name] backend[coro_name] = factory(backend) gen_name = name + '_gen' if gen_name not in backend: factory = globals()['_make_' + gen_name] backend[gen_name] = factory(backend) async_name = name + '_async' if async_name not in backend: factory = getattr(utils35, '_make_' + async_name) backend[async_name] = factory(backend) factory = globals()['_make_' + name] backend[name] = factory(backend)ijson-3.4.0/src/ijson/compat.py000066400000000000000000000023051500700540700164040ustar00rootroot00000000000000''' Python2/Python3 compatibility utilities. ''' import sys import warnings class utf8reader: """Takes a utf8-encoded string reader and reads bytes out of it""" def __init__(self, str_reader): self.str_reader = str_reader def read(self, n): return self.str_reader.read(n).encode('utf-8') _str_vs_bytes_warning = ''' ijson works by reading bytes, but a string reader has been given instead. This probably, but not necessarily, means a file-like object has been opened in text mode ('t') rather than binary mode ('b'). An automatic conversion is being performed on the fly to continue, but on the other hand this creates unnecessary encoding/decoding operations that decrease the efficiency of the system. In the future this automatic conversion will be removed, and users will receive errors instead of this warning. To avoid this problem make sure file-like objects are opened in binary mode instead of text mode. ''' def _warn_and_return(o): warnings.warn(_str_vs_bytes_warning, DeprecationWarning) return o def bytes_reader(f): """Returns a file-like object that reads bytes""" if type(f.read(0)) == bytes: return f return _warn_and_return(utf8reader(f))ijson-3.4.0/src/ijson/dump.py000066400000000000000000000031561500700540700160730ustar00rootroot00000000000000'''Dumping command-line utility''' import argparse import sys import ijson HEADERS = { 'basic_parse': 'name, value', 'parse': 'path, name, value', 'kvitems': 'key, value', 'items': 'value', } def to_string(o): if isinstance(o, bytes): return o.decode("utf-8") return str(o) def dump(): parser = argparse.ArgumentParser(description='Dump ijson events') parser.add_argument('-m', '--method', choices=['basic_parse', 'parse', 'kvitems', 'items'], help='The method to use for dumping', default='basic_parse') parser.add_argument('-p', '--prefix', help='Prefix (used with -M items|kvitems)', default='') parser.add_argument('-M', '--multiple-values', help='Allow multiple values', action='store_true') args = parser.parse_args() method = getattr(ijson, args.method) method_args = () method_kwargs = {} if args.method in ('items', 'kvitems'): method_args = args.prefix, if args.multiple_values: method_kwargs['multiple_values'] = True header = '#: ' + HEADERS[args.method] print(header) print('-' * len(header)) # Use the raw bytes stream in stdin if possible stdin = sys.stdin if hasattr(stdin, 'buffer'): stdin = stdin.buffer enumerated_results = enumerate(method(stdin, *method_args, **method_kwargs)) if args.method == 'items': for i, result in enumerated_results: print('%i: %s' % (i, result)) else: for i, result in enumerated_results: print('%i: %s' % (i, ', '.join(to_string(bit) for bit in result))) if __name__ == '__main__': dump()ijson-3.4.0/src/ijson/utils.py000066400000000000000000000041311500700540700162600ustar00rootroot00000000000000# -*- coding:utf-8 -*- from functools import wraps def coroutine(func): ''' Wraps a generator which is intended to be used as a pure coroutine by .send()ing it values. The only thing that the wrapper does is calling .next() for the first time which is required by Python generator protocol. ''' @wraps(func) def wrapper(*args, **kwargs): g = func(*args, **kwargs) next(g) return g return wrapper def chain(sink, *coro_pipeline): ''' Chains together a sink and a number of coroutines to form a coroutine pipeline. The pipeline works by calling send() on the coroutine created with the information in `coro_pipeline[-1]`, which sends its results to the coroutine created from `coro_pipeline[-2]`, and so on, until the final result is sent to `sink`. ''' f = sink for coro_func, coro_args, coro_kwargs in coro_pipeline: f = coro_func(f, *coro_args, **coro_kwargs) return f class sendable_list(list): ''' A list that mimics a coroutine receiving values. Coroutine are sent values via their send() method. This class defines such a method so that values sent into it are appended into the list, which can be inspected later. As such, this type can be used as an "accumulating sink" in a pipeline consisting on many coroutines. ''' send = list.append def coros2gen(source, *coro_pipeline): ''' A utility function that returns a generator yielding values dispatched by a coroutine pipeline after *it* has received values coming from `source`. ''' events = sendable_list() f = chain(events, *coro_pipeline) try: for value in source: try: f.send(value) except Exception as ex: for event in events: yield event if isinstance(ex, StopIteration): return raise for event in events: yield event del events[:] except GeneratorExit: try: f.close() except: passijson-3.4.0/src/ijson/utils35.py000066400000000000000000000054671500700540700164450ustar00rootroot00000000000000''' Python3.5+ specific utilities ''' import collections from ijson import utils, common, compat class utf8reader_async(compat.utf8reader): """ Takes a utf8-encoded string asynchronous reader and asynchronously reads bytes out of it """ async def read(self, n): data = await self.str_reader.read(n) return data.encode('utf-8') async def _get_read(f): """Returns an awaitable read function that reads the requested type""" if type(await f.read(0)) == bytes: return f.read return compat._warn_and_return(utf8reader_async(f).read) class sendable_deque(collections.deque): '''Like utils.sendable_list, but for deque objects''' send = collections.deque.append class async_iterable: ''' A utility class that implements an async iterator returning values dispatched by a coroutine pipeline after *it* has received values coming from an async file-like object. ''' def __init__(self, f, buf_size, *coro_pipeline): self.events = sendable_deque() self.coro = utils.chain(self.events, *coro_pipeline) self.coro_finished = False self.f = f self.buf_size = buf_size self.read = None def __aiter__(self): return self async def __anext__(self): if not self.read: self.read = await _get_read(self.f) if self.events: return self.events.popleft() if self.coro_finished: raise StopAsyncIteration while True: data = await self.read(self.buf_size) try: self.coro.send(data) if self.events: return self.events.popleft() except StopIteration: self.coro_finished = True if self.events: return self.events.popleft() raise StopAsyncIteration def _make_basic_parse_async(backend): def basic_parse_async(f, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._basic_parse_pipeline(backend, config) ) return basic_parse_async def _make_parse_async(backend): def parse_async(f, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._parse_pipeline(backend, config) ) return parse_async def _make_items_async(backend): def items_async(f, prefix, map_type=None, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._items_pipeline(backend, prefix, map_type, config) ) return items_async def _make_kvitems_async(backend): def kvitems_async(f, prefix, map_type=None, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._kvitems_pipeline(backend, prefix, map_type, config) ) return kvitems_asyncijson-3.4.0/src/ijson/version.py000066400000000000000000000000261500700540700166040ustar00rootroot00000000000000__version__ = '3.4.0' ijson-3.4.0/test-requirements.txt000066400000000000000000000002201500700540700170710ustar00rootroot00000000000000pytest # Keep cffi away from 3.13+ until it gainst free-threaded wheels # otherwise the CI pipeline gets too complex cffi;python_version<"3.13" ijson-3.4.0/tests/000077500000000000000000000000001500700540700140005ustar00rootroot00000000000000ijson-3.4.0/tests/__init__.py000066400000000000000000000000001500700540700160770ustar00rootroot00000000000000ijson-3.4.0/tests/conftest.py000066400000000000000000000075101500700540700162020ustar00rootroot00000000000000import enum import pathlib import ijson import pytest def _get_available_backends(): backends = [] for backend in ijson.ALL_BACKENDS: try: backends.append(ijson.get_backend(backend)) except ImportError: pass return backends _available_backends = _get_available_backends() class InputType(enum.Enum): ASYNC_FILE = enum.auto() ASYNC_TYPES_COROUTINES_FILE = enum.auto() FILE = enum.auto() SENDABLE = enum.auto() class BackendAdaptor: """ Ties a backend together with an input type to provide easy access to calling the backend's methods and retrieving all results in a single call. """ def __init__(self, backend, input_type, suffix, get_all): self.backend = backend self._input_type = input_type self._suffix = suffix self._get_all = get_all @property def pytest_parameter_id(self): return f"{self.backend.backend_name}-{self._input_type.name.lower()}" def __getattr__(self, name): routine = getattr(self.backend, name + self._suffix) def get_all_for_name(*args, **kwargs): return self._get_all(routine, *args, **kwargs) return get_all_for_name from .support.async_ import get_all as get_all_async from .support.async_types_coroutines import get_all as get_all_async_types_coroutines from .support.coroutines import get_all as get_all_coro from .support.generators import get_all as get_all_gen _pull_backend_adaptors = [ backend_adaptor for backend in _available_backends for backend_adaptor in [ BackendAdaptor(backend, InputType.ASYNC_FILE, "_async", get_all_async), BackendAdaptor(backend, InputType.ASYNC_TYPES_COROUTINES_FILE, "_async", get_all_async_types_coroutines), BackendAdaptor(backend, InputType.FILE, "_gen", get_all_gen), ] ] _push_backend_adaptors = [ backend_adaptor for backend in _available_backends for backend_adaptor in [ BackendAdaptor(backend, InputType.SENDABLE, "_coro", get_all_coro), ] ] _all_backend_adaptors = _pull_backend_adaptors + _push_backend_adaptors BACKEND_PARAM_NAME = "backend" ADAPTOR_PARAM_NAME = "adaptor" def pytest_generate_tests(metafunc): requires_backend = BACKEND_PARAM_NAME in metafunc.fixturenames requires_adaptor = ADAPTOR_PARAM_NAME in metafunc.fixturenames assert not (requires_backend and requires_adaptor) names = [] # if both are required we need to match backend and adaptors correctly if requires_backend: names = BACKEND_PARAM_NAME values = _available_backends ids = [backend.backend_name for backend in _available_backends] elif requires_adaptor: pull_only = bool(list(metafunc.definition.iter_markers('pull_only'))) adaptors = _pull_backend_adaptors if pull_only else _all_backend_adaptors names = ADAPTOR_PARAM_NAME values = adaptors ids = [adaptor.pytest_parameter_id for adaptor in adaptors] if names: metafunc.parametrize(names, values, ids=ids) def pytest_addoption(parser): group = parser.getgroup("Memory leak tests") group.addoption("--memleaks", action="store_true", help="include memory leak tests") group.addoption("--memleaks-only", action="store_true", help="run ONLY memory leak tests") def pytest_collection_modifyitems(config, items): if config.option.memleaks_only: skip_mark = pytest.mark.skip(reason="running only memleak tests") for item in items: if pathlib.Path(item.fspath).name != "test_memleaks.py": item.add_marker(skip_mark) elif not config.option.memleaks: skip_mark = pytest.mark.skip(reason="run with --memleaks or --memleaks-only option") for item in items: if pathlib.Path(item.fspath).name == "test_memleaks.py": item.add_marker(skip_mark) ijson-3.4.0/tests/support/000077500000000000000000000000001500700540700155145ustar00rootroot00000000000000ijson-3.4.0/tests/support/__init__.py000066400000000000000000000000001500700540700176130ustar00rootroot00000000000000ijson-3.4.0/tests/support/_async_common.py000066400000000000000000000007371500700540700207210ustar00rootroot00000000000000# -*- coding:utf-8 -*- import asyncio import contextlib def _aiorun(f): with contextlib.closing(asyncio.new_event_loop()) as loop: loop.run_until_complete(f) def _get_all(reader): def get_all(routine, json_content, *args, **kwargs): events = [] async def run(): async for event in routine(reader(json_content), *args, **kwargs): events.append(event) _aiorun(run()) return events return get_all ijson-3.4.0/tests/support/async_.py000066400000000000000000000005631500700540700173460ustar00rootroot00000000000000import asyncio import io from ._async_common import _get_all class AsyncReader: def __init__(self, data): if type(data) == bytes: self.data = io.BytesIO(data) else: self.data = io.StringIO(data) async def read(self, n=-1): await asyncio.sleep(0) return self.data.read(n) get_all = _get_all(AsyncReader)ijson-3.4.0/tests/support/async_types_coroutines.py000066400000000000000000000007111500700540700227000ustar00rootroot00000000000000import io import types from ._async_common import _get_all class AsyncReaderTypesCoroutine: def __init__(self, data): if type(data) == bytes: self.data = io.BytesIO(data) else: self.data = io.StringIO(data) async def _read(self, n=-1): return self.data.read(n) @types.coroutine def read(self, n=-1): return (yield from self._read(n)) get_all = _get_all(AsyncReaderTypesCoroutine) ijson-3.4.0/tests/support/coroutines.py000066400000000000000000000005001500700540700202530ustar00rootroot00000000000000from ijson import utils def bytesiter(x): for b in x: yield bytes([b]) def get_all(routine, json_content, *args, **kwargs): events = utils.sendable_list() coro = routine(events, *args, **kwargs) for datum in bytesiter(json_content): coro.send(datum) coro.close() return eventsijson-3.4.0/tests/support/generators.py000066400000000000000000000003601500700540700202360ustar00rootroot00000000000000import io def _reader(json): if type(json) == bytes: return io.BytesIO(json) return io.StringIO(json) def get_all(routine, json_content, *args, **kwargs): return list(routine(_reader(json_content), *args, **kwargs)) ijson-3.4.0/tests/test_base.py000066400000000000000000000200011500700540700163140ustar00rootroot00000000000000# -*- coding:utf-8 -*- import collections from decimal import Decimal JSON = b''' { "docs": [ { "null": null, "boolean": false, "true": true, "integer": 0, "double": 0.5, "exponent": 1.0e+2, "long": 10000000000, "string": "\\u0441\\u0442\\u0440\\u043e\\u043a\\u0430 - \xd1\x82\xd0\xb5\xd1\x81\xd1\x82", "\xc3\xb1and\xc3\xba": null }, { "meta": [[1], {}] }, { "meta": {"key": "value"} }, { "meta": null }, { "meta": [] } ] } ''' JSON_OBJECT = { "docs": [ { "null": None, "boolean": False, "true": True, "integer": 0, "double": Decimal("0.5"), "exponent": 1e+2, "long": 10000000000, "string": "строка - тест", "ñandú": None }, { "meta": [[1], {}] }, { "meta": { "key": "value" } }, { "meta": None }, { "meta": [] } ] } JSON_PARSE_EVENTS = [ ('', 'start_map', None), ('', 'map_key', 'docs'), ('docs', 'start_array', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'null'), ('docs.item.null', 'null', None), ('docs.item', 'map_key', 'boolean'), ('docs.item.boolean', 'boolean', False), ('docs.item', 'map_key', 'true'), ('docs.item.true', 'boolean', True), ('docs.item', 'map_key', 'integer'), ('docs.item.integer', 'number', 0), ('docs.item', 'map_key', 'double'), ('docs.item.double', 'number', Decimal('0.5')), ('docs.item', 'map_key', 'exponent'), ('docs.item.exponent', 'number', Decimal('1.0E+2')), ('docs.item', 'map_key', 'long'), ('docs.item.long', 'number', 10000000000), ('docs.item', 'map_key', 'string'), ('docs.item.string', 'string', 'строка - тест'), ('docs.item', 'map_key', 'ñandú'), ('docs.item.ñandú', 'null', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'start_array', None), ('docs.item.meta.item', 'start_array', None), ('docs.item.meta.item.item', 'number', 1), ('docs.item.meta.item', 'end_array', None), ('docs.item.meta.item', 'start_map', None), ('docs.item.meta.item', 'end_map', None), ('docs.item.meta', 'end_array', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'start_map', None), ('docs.item.meta', 'map_key', 'key'), ('docs.item.meta.key', 'string', 'value'), ('docs.item.meta', 'end_map', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'null', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'start_array', None), ('docs.item.meta', 'end_array', None), ('docs.item', 'end_map', None), ('docs', 'end_array', None), ('', 'end_map', None) ] JSON_KVITEMS = [ ("null", None), ("boolean", False), ("true", True), ("integer", 0), ("double", Decimal("0.5")), ("exponent", 1e+2), ("long", 10000000000), ("string", "строка - тест"), ("ñandú", None), ("meta", [[1], {}]), ("meta", {"key": "value"}), ("meta", None), ("meta", []) ] JSON_KVITEMS_META = [ ('key', 'value') ] JSON_EVENTS = [ ('start_map', None), ('map_key', 'docs'), ('start_array', None), ('start_map', None), ('map_key', 'null'), ('null', None), ('map_key', 'boolean'), ('boolean', False), ('map_key', 'true'), ('boolean', True), ('map_key', 'integer'), ('number', 0), ('map_key', 'double'), ('number', Decimal('0.5')), ('map_key', 'exponent'), ('number', 100), ('map_key', 'long'), ('number', 10000000000), ('map_key', 'string'), ('string', 'строка - тест'), ('map_key', 'ñandú'), ('null', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('start_array', None), ('start_array', None), ('number', 1), ('end_array', None), ('start_map', None), ('end_map', None), ('end_array', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('start_map', None), ('map_key', 'key'), ('string', 'value'), ('end_map', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('null', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('start_array', None), ('end_array', None), ('end_map', None), ('end_array', None), ('end_map', None), ] # Like JSON, but with an additional top-level array structure ARRAY_JSON = b'[' + JSON + b']' ARRAY_JSON_EVENTS = ( [('start_array', None)] + JSON_EVENTS + [('end_array', None)] ) ARRAY_JSON_PARSE_EVENTS = ( [('', 'start_array', None)] + [('.'.join(filter(None, ('item', p))), t, e) for p, t, e in JSON_PARSE_EVENTS] + [('', 'end_array', None)] ) ARRAY_JSON_OBJECT = [JSON_OBJECT] SCALAR_JSON = b'0' INVALID_JSONS = [ b'["key", "value",]', # trailing comma b'["key" "value"]', # no comma b'{"key": "value",}', # trailing comma b'{"key": "value" "key"}', # no comma b'{"key" "value"}', # no colon b'invalid', # unknown lexeme b'[1, 2] dangling junk', # dangling junk b'}', # no corresponding opening token b']', # no corresponding opening token b'"\xa8"' # invalid UTF-8 byte sequence ] INVALID_JSON_WITH_DANGLING_JUNK = INVALID_JSONS[6] INCOMPLETE_JSONS = [ b'', b'"test', b'[', b'[1', b'[1,', b'{', b'{"key"', b'{"key":', b'{"key": "value"', b'{"key": "value",', ] INCOMPLETE_JSON_TOKENS = [ b'n', b'nu', b'nul', b't', b'tr', b'tru', b'f', b'fa', b'fal', b'fals', b'[f', b'[fa', b'[fal', b'[fals', b'[t', b'[tr', b'[tru', b'[n', b'[nu', b'[nul', b'{"key": t', b'{"key": tr', b'{"key": tru', b'{"key": f', b'{"key": fa', b'{"key": fal', b'{"key": fals', b'{"key": n', b'{"key": nu', b'{"key": nul', ] STRINGS_JSON = br''' { "str1": "", "str2": "\"", "str3": "\\", "str4": "\\\\", "special\t": "\b\f\n\r\t" } ''' SURROGATE_PAIRS_JSON = br'"\uD83D\uDCA9"' PARTIAL_ARRAY_JSONS = [ (b'[1,', 1), (b'[1, 2 ', 1, 2), (b'[1, "abc"', 1, 'abc'), (b'[{"abc": [0, 1]}', {'abc': [0, 1]}), (b'[{"abc": [0, 1]},', {'abc': [0, 1]}), ] items_test_case = collections.namedtuple('items_test_case', 'json, prefix, kvitems, items') EMPTY_MEMBER_TEST_CASES = { 'simple': items_test_case( b'{"a": {"": {"b": 1, "c": 2}}}', 'a.', [("b", 1), ("c", 2)], [{"b": 1, "c": 2}] ), 'embedded': items_test_case( b'{"a": {"": {"": {"b": 1, "c": 2}}}}', 'a..', [("b", 1), ("c", 2)], [{"b": 1, "c": 2}] ), 'top_level': items_test_case( b'{"": 1, "a": 2}', '', [("", 1), ("a", 2)], [{"": 1, "a": 2}] ), 'top_level_embedded': items_test_case( b'{"": {"": 1}, "a": 2}', '', [("", {"": 1}), ("a", 2)], [{"": {"": 1}, "a": 2}] ) }ijson-3.4.0/tests/test_basic_parse.py000066400000000000000000000130641500700540700176700ustar00rootroot00000000000000"""Tests for the ijson.basic_parse method""" import itertools import threading from decimal import Decimal import pytest from ijson import common from .test_base import ARRAY_JSON, ARRAY_JSON_EVENTS, INCOMPLETE_JSONS, INCOMPLETE_JSON_TOKENS, INVALID_JSONS, JSON, JSON_EVENTS, SCALAR_JSON, SURROGATE_PAIRS_JSON, STRINGS_JSON def _raises_json_error(adaptor, json, **kwargs): with pytest.raises(common.JSONError): adaptor.basic_parse(json, **kwargs) def _raises_incomplete_json_error(adaptor, json): with pytest.raises(common.IncompleteJSONError): adaptor.basic_parse(json) def test_basic_parse(adaptor): assert JSON_EVENTS == adaptor.basic_parse(JSON) def test_basic_parse_threaded(adaptor): thread = threading.Thread(target=test_basic_parse, args=(adaptor,)) thread.start() thread.join() def test_basic_parse_array(adaptor): assert ARRAY_JSON_EVENTS == adaptor.basic_parse(ARRAY_JSON) def test_basic_parse_array_threaded(adaptor): thread = threading.Thread(target=test_basic_parse_array, args=(adaptor,)) thread.start() thread.join() def test_scalar(adaptor): assert [('number', 0)] == adaptor.basic_parse(SCALAR_JSON) def test_strings(adaptor): events = adaptor.basic_parse(STRINGS_JSON) strings = [value for event, value in events if event == 'string'] assert ['', '"', '\\', '\\\\', '\b\f\n\r\t'] == strings assert ('map_key', 'special\t') in events def test_surrogate_pairs(adaptor): event = adaptor.basic_parse(SURROGATE_PAIRS_JSON)[0] parsed_string = event[1] assert '💩' == parsed_string def _get_numbers(adaptor, json, use_float): events = adaptor.basic_parse(json, use_float=use_float) return [value for event, value in events if event == 'number'] @pytest.mark.parametrize( "json, expected_float_type, expected_numbers, use_float", ( (b'[1, 1.0, 1E2]', Decimal, [1, Decimal("1.0"), Decimal("1e2")], False), (b'[1, 1.0, 1E2]', float, [1, 1., 100.], True), (b'1e400', Decimal, [Decimal('1e400')], False), (b'1e-400', Decimal, [Decimal('1e-400')], False), (b'1e-400', float, [0], True), ) ) def test_numbers(adaptor, json, expected_float_type, expected_numbers, use_float): """Check that numbers are correctly parsed""" numbers = _get_numbers(adaptor, json, use_float=use_float) float_types = set(type(number) for number in numbers) float_types -= {int} assert 1 == len(float_types) assert expected_float_type == next(iter(float_types)) assert expected_numbers == numbers def test_32bit_ints(adaptor): """Test for 64-bit integers support when using use_float=true""" past32bits = 2 ** 32 + 1 past32bits_as_json = ('%d' % past32bits).encode('utf8') if adaptor.backend.capabilities.int64: parsed_number = _get_numbers(adaptor, past32bits_as_json, use_float=True)[0] assert past32bits == parsed_number else: _raises_json_error(adaptor, past32bits_as_json, use_float=True) def test_max_double(adaptor): """Check that numbers bigger than MAX_DOUBLE (usually ~1e308) cannot be represented""" _raises_json_error(adaptor, b'1e400', use_float=True) @pytest.mark.parametrize( "json", [ sign + prefix + suffix for sign, prefix, suffix in itertools.product( (b'', b'-'), (b'00', b'01', b'001'), (b'', b'.0', b'e0', b'E0') ) ] ) def test_invalid_leading_zeros(adaptor, json): """Check leading zeros are invalid""" if not adaptor.backend.capabilities.invalid_leading_zeros_detection: return _raises_json_error(adaptor, json) @pytest.mark.parametrize("json", (b'1e', b'0.1e', b'0E')) def test_incomplete_exponents(adaptor, json): """incomplete exponents are invalid JSON""" _raises_json_error(adaptor, json) @pytest.mark.parametrize("json", (b'1.', b'.1')) def test_incomplete_fractions(adaptor, json): """incomplete fractions are invalid JSON""" _raises_json_error(adaptor, json) def test_incomplete(adaptor): for json in INCOMPLETE_JSONS: _raises_incomplete_json_error(adaptor, json) def test_incomplete_tokens(adaptor): if not adaptor.backend.capabilities.incomplete_json_tokens_detection: return for json in INCOMPLETE_JSON_TOKENS: _raises_incomplete_json_error(adaptor, json) def test_invalid(adaptor): for json in INVALID_JSONS: if not adaptor.backend.capabilities.incomplete_json_tokens_detection and json == INVALID_JSON_WITH_DANGLING_JUNK: continue _raises_json_error(adaptor, json) def test_comments(adaptor): json = b'{"a": 2 /* a comment */}' if adaptor.backend.capabilities.c_comments: events = adaptor.basic_parse(json, allow_comments=True) assert events is not None else: with pytest.raises(ValueError): adaptor.basic_parse(json, allow_comments=True) def test_multiple_values_raises_if_not_supported(adaptor): """Test that setting multiple_values raises if not supported""" if not adaptor.backend.capabilities.multiple_values: with pytest.raises(ValueError): adaptor.basic_parse("", multiple_values=True) def test_multiple_values(adaptor): """Test that multiple_values are supported""" multiple_json = JSON + JSON + JSON with pytest.raises(common.JSONError): adaptor.basic_parse(multiple_json) with pytest.raises(common.JSONError): adaptor.basic_parse(multiple_json, multiple_values=False) result = adaptor.basic_parse(multiple_json, multiple_values=True) assert JSON_EVENTS + JSON_EVENTS + JSON_EVENTS == result ijson-3.4.0/tests/test_benchmark.py000066400000000000000000000033301500700540700173420ustar00rootroot00000000000000import os import subprocess import sys import tempfile import pytest from tests.test_base import JSON def _do_test_benchmark(method="basic_parse", multiple_values=True, extra_args=None): # Use python backend to ensure multiple_values works if multiple_values: env = dict(os.environ) env['IJSON_BACKEND'] = 'python' # Ensure printing works on the subprocess in Windows # by using utf-8 on its stdout if sys.platform == 'win32': env = dict(os.environ) env['PYTHONIOENCODING'] = 'utf-8' cmd = [sys.executable, '-m', 'ijson.benchmark', '-m', method, '-p', '', '-s', '1'] if multiple_values: cmd.append('-M') if extra_args: cmd += extra_args proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) out, err = proc.communicate() status = proc.wait() assert 0 == status, "out:\n%s\nerr:%s" % (out.decode('utf-8'), err.decode('utf-8')) @pytest.mark.parametrize("input_type", ("gen", "coro", "async")) @pytest.mark.parametrize("method", ("basic_parse", "parse", "kvitems", "items")) def test_benchmark(method, input_type): extra_args = {"gen": [], "coro": ["-c"], "async": ["-a"]} _do_test_benchmark(method, extra_args=extra_args[input_type]) def test_list(): _do_test_benchmark(extra_args=['-l']) @pytest.mark.parametrize("iterations", (1, 2, 3)) def test_more_iterations(iterations): _do_test_benchmark(extra_args=['-I', str(iterations)]) def test_input_files(): fd, fname = tempfile.mkstemp() os.write(fd, JSON) os.close(fd) try: _do_test_benchmark(extra_args=[fname]) _do_test_benchmark(extra_args=[fname, fname]) finally: os.unlink(fname) ijson-3.4.0/tests/test_dump.py000066400000000000000000000020311500700540700163520ustar00rootroot00000000000000import os import subprocess import sys import pytest from tests.test_base import JSON @pytest.mark.parametrize("multiple_values", (True, False)) @pytest.mark.parametrize("method", ("basic_parse", "parse", "kvitems", "items")) def test_dump(method, multiple_values): # Use python backend to ensure multiple_values works env = dict(os.environ) env['IJSON_BACKEND'] = 'python' # Ensure printing works on the subprocess in Windows # by using utf-8 on its stdout if sys.platform == 'win32': env = dict(os.environ) env['PYTHONIOENCODING'] = 'utf-8' cmd = [sys.executable, '-m', 'ijson.dump', '-m', method, '-p', ''] if multiple_values: cmd.append('-M') proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) input_data = JSON if multiple_values: input_data += JSON out, err = proc.communicate(input_data) status = proc.wait() assert 0 == status, "out:\n%s\nerr:%s" % (out.decode('utf-8'), err.decode('utf-8'))ijson-3.4.0/tests/test_generators.py000066400000000000000000000101761500700540700175670ustar00rootroot00000000000000import io import pytest from ijson import common from .test_base import JSON, JSON_EVENTS, PARTIAL_ARRAY_JSONS, INVALID_JSONS class SingleReadFile: '''A bytes file that can be read only once''' def __init__(self, raw_value): self.raw_value = raw_value def read(self, size=-1): if size == 0: return bytes() val = self.raw_value if not val: raise AssertionError('read twice') self.raw_value = bytes() return val def test_utf8_split(backend): buf_size = JSON.index(b'\xd1') + 1 try: list(backend.basic_parse_gen(io.BytesIO(JSON), buf_size=buf_size)) except UnicodeDecodeError: pytest.fail('UnicodeDecodeError raised') def test_lazy(backend): # shouldn't fail since iterator is not exhausted basic_parse = backend.basic_parse_gen(io.BytesIO(INVALID_JSONS[0])) assert basic_parse is not None def test_extra_content_still_generates_results(backend): """Extra content raises an error, but still generates all events""" with pytest.raises(common.JSONError): events = [] for event in backend.basic_parse(JSON + b"#"): events.append(event) assert events == JSON_EVENTS def test_boundary_lexeme(backend): buf_size = JSON.index(b'false') + 1 events = list(backend.basic_parse(JSON, buf_size=buf_size)) assert events == JSON_EVENTS def test_boundary_whitespace(backend): buf_size = JSON.index(b' ') + 1 events = list(backend.basic_parse_gen(io.BytesIO(JSON), buf_size=buf_size)) assert events == JSON_EVENTS def test_item_building_greediness(backend): _test_item_iteration_validity(backend, io.BytesIO) def test_lazy_file_reading(backend): _test_item_iteration_validity(backend, SingleReadFile) def _test_item_iteration_validity(backend, file_type): for json in PARTIAL_ARRAY_JSONS: json, expected_items = json[0], json[1:] iterable = backend.items_gen(file_type(json), 'item') for expected_item in expected_items: assert expected_item == next(iterable) COMMON_DATA = b''' { "skip": "skip_value", "c": {"d": "e", "f": "g"}, "list": [{"o1": 1}, {"o2": 2}] }''' COMMON_PARSE = [ ('', 'start_map', None), ('', 'map_key', 'skip'), ('skip', 'string', 'skip_value'), ('', 'map_key', 'c'), ('c', 'start_map', None), ('c', 'map_key', 'd'), ('c.d', 'string', 'e'), ('c', 'map_key', 'f'), ('c.f', 'string', 'g'), ('c', 'end_map', None), ('', 'map_key', 'list'), ('list', 'start_array', None), ('list.item', 'start_map', None), ('list.item', 'map_key', 'o1'), ('list.item.o1', 'number', 1), ('list.item', 'end_map', None), ('list.item', 'start_map', None), ('list.item', 'map_key', 'o2'), ('list.item.o2', 'number', 2), ('list.item', 'end_map', None), ('list', 'end_array', None), ('', 'end_map', None), ] def _skip_parse_events(events): skip_value = None for prefix, _, value in events: if prefix == 'skip': skip_value = value break assert skip_value == 'skip_value' def _test_common_routine(backend, routine, *args, **kwargs): base_routine_name = kwargs.pop('base_routine_name', 'parse') base_routine = getattr(backend, base_routine_name) events = base_routine(io.BytesIO(COMMON_DATA)) if base_routine_name == 'parse': _skip_parse_events(events) # Rest of events can still be used return list(routine(events, *args)) def test_common_parse(backend): with pytest.warns() as warns: results = _test_common_routine( backend, common.parse, base_routine_name='basic_parse' ) assert COMMON_PARSE == results assert 1 == len(warns) def test_common_kvitems(backend): with pytest.warns() as warns: results = _test_common_routine(backend, common.kvitems, 'c') assert [("d", "e"), ("f", "g")] == results assert 1 == len(warns) def test_common_items(backend): with pytest.warns() as warns: results = _test_common_routine(backend, common.items, 'list.item') assert [{"o1": 1}, {"o2": 2}] == results assert 1 == len(warns)ijson-3.4.0/tests/test_items.py000066400000000000000000000064521500700540700165410ustar00rootroot00000000000000"""Tests for the ijson.items method""" import collections import pytest from .test_base import ARRAY_JSON, ARRAY_JSON_OBJECT, EMPTY_MEMBER_TEST_CASES, JSON, JSON_OBJECT from ijson import JSONError def test_items(adaptor): assert [JSON_OBJECT] == adaptor.items(JSON, '') def test_items_array(adaptor): assert [ARRAY_JSON_OBJECT] == adaptor.items(ARRAY_JSON, '') def test_items_twodictlevels(adaptor): json = b'{"meta":{"view":{"columns":[{"id": -1}, {"id": -2}]}}}' ids = adaptor.items(json, 'meta.view.columns.item.id') assert 2 == len(ids) assert [-2,-1], sorted(ids) @pytest.mark.parametrize( "json, prefix, expected_items", ( (b'{"0.1": 0}', '0.1', [0]), (b'{"0.1": [{"a.b": 0}]}', '0.1.item.a.b', [0]), (b'{"0.1": 0, "0": {"1": 1}}', '0.1', [0, 1]), (b'{"abc.def": 0}', 'abc.def', [0]), (b'{"abc.def": 0}', 'abc', []), (b'{"abc.def": 0}', 'def', []), ) ) def test_items_with_dotted_name(adaptor, json, prefix, expected_items): assert expected_items == adaptor.items(json, prefix) def test_map_type(adaptor): obj = adaptor.items(JSON, '')[0] assert isinstance(obj, dict) obj = adaptor.items(JSON, '', map_type=collections.OrderedDict)[0] assert isinstance(obj, collections.OrderedDict) @pytest.mark.parametrize("test_case", [ pytest.param(value, id=name) for name, value in EMPTY_MEMBER_TEST_CASES.items() ]) def test_items_empty_member(adaptor, test_case): assert test_case.items == adaptor.items(test_case.json, test_case.prefix) def test_multiple_values_raises_if_not_supported(adaptor): """Test that setting multiple_values raises if not supported""" if not adaptor.backend.capabilities.multiple_values: with pytest.raises(ValueError): adaptor.items("", "", multiple_values=True) def test_multiple_values(adaptor): """Test that multiple_values are supported""" multiple_json = JSON + JSON + JSON with pytest.raises(JSONError): adaptor.items(multiple_json, "") with pytest.raises(JSONError): adaptor.items(multiple_json, "", multiple_values=False) result = adaptor.items(multiple_json, "", multiple_values=True) assert [JSON_OBJECT, JSON_OBJECT, JSON_OBJECT] == result def test_coro_needs_input_with_three_elements(backend): int_element_parse_events = list(backend.parse(b'0')) # all good assert [0] == list(backend.items(int_element_parse_events, '')) # one more element in event with pytest.raises(ValueError, match="too many values"): next(backend.items((event + ('extra dummy',) for event in int_element_parse_events), '')) # one less with pytest.raises(ValueError, match="not enough values"): next(backend.items((event[:-1] for event in int_element_parse_events), '')) # not an iterable with pytest.raises(TypeError, match="cannot unpack"): next(backend.items([None], '')) def test_user_events(backend): """ User-provided events work correct -- yajl2_c used to fail with event names that weren't generated by itself. """ embedded_empty_list_events = [ ('', 'start_array', None), ('item', 'start_array', None), ('item', 'end_array', None), ('', 'end_array', None), ] assert [[]] == list(backend.items(embedded_empty_list_events, 'item')) ijson-3.4.0/tests/test_kvitems.py000066400000000000000000000045621500700540700171020ustar00rootroot00000000000000"""Tests for the ijson.kvitems method""" from .test_base import ARRAY_JSON, EMPTY_MEMBER_TEST_CASES, JSON, JSON_KVITEMS, JSON_KVITEMS_META, JSON_OBJECT import pytest def test_kvitems(adaptor): assert JSON_KVITEMS == adaptor.kvitems(JSON, 'docs.item') def test_kvitems_toplevel(adaptor): kvitems = adaptor.kvitems(JSON, '') assert 1 == len(kvitems) key, value = kvitems[0] assert 'docs' == key assert JSON_OBJECT['docs'] == value def test_kvitems_empty(adaptor): assert [] == adaptor.kvitems(JSON, 'docs') def test_kvitems_twodictlevels(adaptor): json = b'{"meta":{"view":{"columns":[{"id": -1}, {"id": -2}]}}}' view = adaptor.kvitems(json, 'meta.view') assert 1 == len(view) key, value = view[0] assert 'columns' == key assert [{'id': -1}, {'id': -2}] == value def test_kvitems_different_underlying_types(adaptor): assert JSON_KVITEMS_META == adaptor.kvitems(JSON, 'docs.item.meta') def test_kvitems_array(adaptor): assert JSON_KVITEMS == adaptor.kvitems(ARRAY_JSON, 'item.docs.item') @pytest.mark.parametrize("test_case", [ pytest.param(value, id=name) for name, value in EMPTY_MEMBER_TEST_CASES.items() ]) def test_kvitems_empty_member(adaptor, test_case): assert test_case.kvitems == adaptor.kvitems(test_case.json, test_case.prefix) def test_coro_needs_input_with_three_elements(backend): int_element_parse_events = list(backend.parse(b'{"a": 0}')) # all good assert [('a', 0)] == list(backend.kvitems(int_element_parse_events, '')) # one more element in event with pytest.raises(ValueError, match="too many values"): next(backend.kvitems((event + ('extra dummy',) for event in int_element_parse_events), '')) # one less with pytest.raises(ValueError, match="not enough values"): next(backend.kvitems((event[:-1] for event in int_element_parse_events), '')) # not an iterable with pytest.raises(TypeError, match="cannot unpack"): next(backend.kvitems([None], '')) def test_user_events(backend): """ User-provided events work correct -- yajl2_c used to fail with event names that weren't generated by itself. """ int_element_parse_events = [ ('', 'start_map', None), ('', 'map_key', 'a'), ('a', 'number', 0), ('', 'end_map', None), ] assert [('a', 0)] == list(backend.kvitems(int_element_parse_events, '')) ijson-3.4.0/tests/test_memleaks.py000066400000000000000000000062521500700540700172140ustar00rootroot00000000000000import ijson import io import pytest def _memusage(): """memory usage in MB. getrusage defaults to KB on Linux.""" import resource return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss def _exhaust(it): sum(1 for _ in it) def _repeat(n, f, *args, exhaust=False, **kwargs): for _ in range(n): obj = f(*args, **kwargs) if exhaust: _exhaust(obj) @pytest.fixture(autouse=True) def _memory_leak_detection(): THRESHOLD = 0.1 memusage_start = _memusage() yield memusage_end = _memusage() assert (memusage_end - memusage_start) / memusage_start <= THRESHOLD @pytest.mark.parametrize("exhaust", (True, False)) @pytest.mark.parametrize("n", (100000,)) @pytest.mark.parametrize("function", ( pytest.param(construction, id=name) for name, construction in { "basic_parse": lambda backend: backend.basic_parse_gen(io.BytesIO(b'[1, 2, 3, 4, 5]')), "parse": lambda backend: backend.parse_gen(io.BytesIO(b'[1, 2, 3, 4, 5]')), "items": lambda backend: backend.items_gen(io.BytesIO(b'[1, 2, 3, 4, 5]'), 'item'), "kvitems": lambda backend: backend.kvitems_gen(io.BytesIO(b'{"a": 0, "b": 1, "c": 2}'), ''), }.items() ) ) def test_generator(backend, function, n, exhaust): """Tests that running parse doesn't leak""" _repeat(n, function, backend, exhaust=exhaust) @pytest.mark.parametrize("n", (100000,)) @pytest.mark.parametrize("invalid_construction", ( pytest.param(construction, id=name) for name, construction in { "basic_parse": lambda backend: next(backend.basic_parse_gen(io.BytesIO(b'[1, 2, 3, 4, 5]'), not_a_kwarg=0)), "parse": lambda backend: next(backend.parse_gen(io.BytesIO(b'[1, 2, 3, 4, 5]'), not_a_kwarg=0)), "items": lambda backend: next(backend.items_gen(io.BytesIO(b'[1, 2, 3, 4, 5]'), 'item', not_a_kwarg=0)), "kvitems": lambda backend: next(backend.kvitems_gen(io.BytesIO(b'{"a": 0, "b": 1, "c": 2}'), '', not_a_kwarg=0)), }.items() ) ) def test_erroneous_generator_construction(backend, invalid_construction, n): """Tests that running parse doesn't leak""" def erroneous_generator_construction(): with pytest.raises(TypeError): invalid_construction(backend) _repeat(n, erroneous_generator_construction) @pytest.mark.parametrize("n", (100000,)) @pytest.mark.parametrize("invalid_construction", ( pytest.param(construction, id=name) for name, construction in { "basic_parse": lambda backend: backend.basic_parse_async(b'[1, 2, 3, 4, 5]', not_a_kwarg=0), "parse": lambda backend: backend.parse_async(b'[1, 2, 3, 4, 5]', not_a_kwarg=0), "items": lambda backend: backend.items_async(b'[1, 2, 3, 4, 5]', 'item', not_a_kwarg=0), "kvitems": lambda backend: backend.kvitems_async(b'{"a": 0, "b": 1, "c": 2}', '', not_a_kwarg=0), }.items() ) ) def test_erroneous_async_construction(backend, n ,invalid_construction): def erroneous_async_construction(): with pytest.raises(TypeError): invalid_construction(backend) _repeat(n, erroneous_async_construction) ijson-3.4.0/tests/test_misc.py000066400000000000000000000067071500700540700163560ustar00rootroot00000000000000import importlib.util import io import pytest from ijson import common from tests.test_base import JSON, JSON_EVENTS, JSON_PARSE_EVENTS, JSON_OBJECT,\ JSON_KVITEMS class TestMisc: """Miscellaneous unit tests""" def test_common_number_is_deprecated(self): with pytest.deprecated_call(): common.number("1") def test_yajl2_c_loadable(self): spec = importlib.util.find_spec("ijson.backends._yajl2") if spec is None: pytest.skip("yajl2_c is not built") importlib.util.module_from_spec(spec) class TestMainEntryPoints: """Tests that main API entry points work against different types of inputs automatically""" def _assert_invalid_type(self, routine, *args, **kwargs): # Functions are not valid inputs with pytest.raises(ValueError): routine(lambda _: JSON, *args, **kwargs) def _assert_bytes(self, expected_results, routine, *args, **kwargs): results = list(routine(JSON, *args, **kwargs)) assert expected_results == results def _assert_str(self, expected_results, routine, *args, **kwargs): with pytest.deprecated_call(): results = list(routine(JSON.decode("utf-8"), *args, **kwargs)) def _assert_file(self, expected_results, routine, *args, **kwargs): results = list(routine(io.BytesIO(JSON), *args, **kwargs)) assert expected_results == results def _assert_async_file(self, expected_results, routine, *args, **kwargs): from .support.async_ import get_all results = get_all(routine, JSON, *args, **kwargs) expected_results == results def _assert_async_types_coroutine(self, expected_results, routine, *args, **kwargs): from .support.async_types_coroutines import get_all results = get_all(routine, JSON, *args, **kwargs) assert expected_results == results def _assert_events(self, expected_results, previous_routine, routine, *args, **kwargs): events = previous_routine(io.BytesIO(JSON)) # Using a different generator to make the point that we can chain # user-provided code def event_yielder(): for evt in events: yield evt results = list(routine(event_yielder(), *args, **kwargs)) assert expected_results == results def _assert_entry_point(self, expected_results, previous_routine, routine, *args, **kwargs): self._assert_invalid_type(routine, *args, **kwargs) self._assert_bytes(expected_results, routine, *args, **kwargs) self._assert_str(expected_results, routine, *args, **kwargs) self._assert_file(expected_results, routine, *args, **kwargs) self._assert_async_file(expected_results, routine, *args, **kwargs) self._assert_async_types_coroutine(expected_results, routine, *args, **kwargs) if previous_routine: self._assert_events(expected_results, previous_routine, routine, *args, **kwargs) def test_rich_basic_parse(self, backend): self._assert_entry_point(JSON_EVENTS, None, backend.basic_parse) def test_rich_parse(self, backend): self._assert_entry_point(JSON_PARSE_EVENTS, backend.basic_parse, backend.parse) def test_rich_items(self, backend): self._assert_entry_point([JSON_OBJECT], backend.parse, backend.items, '') def test_rich_kvitems(self, backend): self._assert_entry_point(JSON_KVITEMS, backend.parse, backend.kvitems, 'docs.item')ijson-3.4.0/tests/test_parse.py000066400000000000000000000030001500700540700165140ustar00rootroot00000000000000"""Tests for the ijson.parse method""" import pytest from .test_base import ARRAY_JSON, ARRAY_JSON_PARSE_EVENTS, JSON, JSON_PARSE_EVENTS def test_parse(adaptor): assert JSON_PARSE_EVENTS == adaptor.parse(JSON) def test_parse_array(adaptor): assert ARRAY_JSON_PARSE_EVENTS == adaptor.parse(ARRAY_JSON) def test_coro_needs_input_with_two_elements(backend): int_element_basic_parse_events = list(backend.basic_parse(b'0', use_float=True)) # all good assert [('', 'number', 0)] == list(backend.parse(int_element_basic_parse_events)) # one more element in event with pytest.raises(ValueError, match="too many values"): next(backend.parse(event + ('extra dummy',) for event in int_element_basic_parse_events)) # one less with pytest.raises(ValueError, match="not enough values"): next(backend.parse(event[:-1] for event in int_element_basic_parse_events)) # not an iterable with pytest.raises(TypeError, match="cannot unpack"): next(backend.parse([None])) def test_user_events(backend): """ User-provided events work correct -- yajl2_c used to fail with event names that weren't generated by itself. """ int_array_basic_parse_events = [ ('start_array', None), ('number', 0), ('end_array', None), ] expected_parse_events = [ ('', 'start_array', None), ('item', 'number', 0), ('', 'end_array', None), ] assert expected_parse_events == list(backend.parse(int_array_basic_parse_events)) ijson-3.4.0/tests/test_pulling.py000066400000000000000000000007161500700540700170670ustar00rootroot00000000000000import pytest from .test_base import JSON, JSON_EVENTS @pytest.mark.pull_only def test_string_stream(adaptor): with pytest.deprecated_call(): events = adaptor.basic_parse(JSON.decode('utf-8')) assert JSON_EVENTS == events @pytest.mark.pull_only @pytest.mark.parametrize("buf_size", (2 ** exp for exp in range(0, 13, 2))) def test_different_buf_sizes(adaptor, buf_size): assert JSON_EVENTS == adaptor.basic_parse(JSON, buf_size=buf_size) ijson-3.4.0/tests/test_subinterpreter.py000066400000000000000000000025301500700540700204660ustar00rootroot00000000000000import importlib.util import os import pickle import queue import sys import threading import time import pytest try: from test.support import interpreters from test.support.interpreters import queues except ImportError: pytest.skip("No sub-interpreter support", allow_module_level=True) @pytest.fixture(name="interpreter") def interpreter_fixture(): interpreter = interpreters.create() yield interpreter interpreter.close() def test_ijson_can_be_loaded(interpreter): interpreter.exec('import ijson') def test_ijson_yajl2_backend_can_be_loaded(interpreter): spec = importlib.util.find_spec("ijson.backends._yajl2") if spec is None: pytest.skip("yajl2_c is not built") interpreter.exec('import ijson') interpreter.exec('ijson.get_backend("yajl2_c")') def test_ijson_can_run(interpreter): queue = queues.create() VALUE = 43 interpreter.prepare_main(queue_id=queue.id, value_in=VALUE) def ijson_in_subinterpreter(): from test.support.interpreters.queues import Queue import ijson value_out = next(ijson.items(str(value_in).encode('ascii'), prefix='')) queue = Queue(queue_id) queue.put(value_out) thread = interpreter.call_in_thread(ijson_in_subinterpreter) value_out = queue.get(timeout=5) thread.join() assert VALUE == value_out