pax_global_header00006660000000000000000000000064144566630270014527gustar00rootroot0000000000000052 comment=3a94e5899ea88a406a94ddc217be687e9206be9c ijson-3.2.3/000077500000000000000000000000001445666302700126565ustar00rootroot00000000000000ijson-3.2.3/.appveyor.yml000066400000000000000000000017661445666302700153360ustar00rootroot00000000000000# AppVeyor configuration for ijson # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2020 # Copyright by UWA (in the framework of the ICRAR) # All rights reserved image: Visual Studio 2019 environment: matrix: # For Python versions available on Appveyor, see # http://www.appveyor.com/docs/installed-software#python - PYTHON: "C:\\Python35-x64" APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON: "C:\\Python36-x64" - PYTHON: "C:\\Python37-x64" - PYTHON: "C:\\Python38-x64" - PYTHON: "C:\\Python39-x64" - PYTHON: "C:\\Python310-x64" - PYTHON: "C:\\Python311-x64" install: - "%PYTHON%\\python.exe -m pip install pytest" - cmd: git submodule update --init --recursive build: off # Build, then run tests/benchmark with hw/sw implementations test_script: - "set IJSON_EMBED_YAJL=1" - "%PYTHON%\\python.exe setup.py develop" - "%PYTHON%\\python.exe -mpytest -v" artifacts: - path: dist\* ijson-3.2.3/.coveragerc000066400000000000000000000021571445666302700150040ustar00rootroot00000000000000# # Coverage configuration file # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2019 # Copyright by UWA (in the framework of the ICRAR) # All rights reserved # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, # MA 02111-1307 USA # # We currently exclude the deploy directory because it's not # part of the core software [run] source = ijson [report] exclude_lines = raise ImportError\('no backends available'\) ijson-3.2.3/.github/000077500000000000000000000000001445666302700142165ustar00rootroot00000000000000ijson-3.2.3/.github/ISSUE_TEMPLATE/000077500000000000000000000000001445666302700164015ustar00rootroot00000000000000ijson-3.2.3/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000014061445666302700210740ustar00rootroot00000000000000--- name: Bug report about: Create a report to help us improve title: '' labels: bug assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **How to reproduce*** Provide both code and a small JSON document that triggers the problem. ```python JSON = b""" [1, 2, 3, 4] """ for o in ijson.items(JSON, 'item'): raise ValueError('this failed') ``` **Expected behavior** A clear and concise description of what you expected to happen. **Execution information:** - Python version [e.g. 3.7] - ijson version [e.g. 3.1.2] - ijson backend (if applies) [e.g. yajl2_c] - ijson installation method [e.g. package manager, pip, git sources, conda] - OS [e.g. linux] **Additional context** Add any other context about the problem here. ijson-3.2.3/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000012101445666302700221200ustar00rootroot00000000000000--- name: Feature request about: Suggest an idea for this project title: '' labels: feature assignees: '' --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. It would be great if it was possible for ijson to [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered, and why/how they failed. **Additional context** Add any other context or screenshots about the feature request here. ijson-3.2.3/.github/ISSUE_TEMPLATE/usage-question.md000066400000000000000000000007451445666302700217020ustar00rootroot00000000000000--- name: Usage question about: Ask how to achieve a particular task title: '' labels: question assignees: '' --- **Description** A clear and concise description of what your question is. Ex. How can I use ijson to [...] **Detailed description** Add more details on what exactly you need to achieve, including example JSON documents and code. **Why is this not clear from the documentation** Is the question not clearly answerable from the documentation? If not, why do you think? ijson-3.2.3/.github/tools/000077500000000000000000000000001445666302700153565ustar00rootroot00000000000000ijson-3.2.3/.github/tools/install_yajl.sh000066400000000000000000000004751445666302700204050ustar00rootroot00000000000000#!/bin/sh install_unix() { mkdir cextern/yajl/build; cd cextern/yajl/build cmake .. -DCMAKE_C_FLAGS="-O3" make all -j 4 make install } OS="`uname -s`" echo "Building libyajl on $OS" case "$OS" in MSYS_*|MINGW_*|Darwin) echo "No extra step required in Windows or macOS" ;; Linux) install_unix ;; esac ijson-3.2.3/.github/workflows/000077500000000000000000000000001445666302700162535ustar00rootroot00000000000000ijson-3.2.3/.github/workflows/deploy-to-pypi.yml000066400000000000000000000043141445666302700216730ustar00rootroot00000000000000name: Build, test, and upload to PyPI # Build on every branch push, tag push, and pull request change: on: [push, pull_request] jobs: build_wheels: name: Build wheels on ${{matrix.arch}} for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-20.04, windows-2019, macos-12] arch: [auto] include: - os: ubuntu-20.04 arch: aarch64 steps: - uses: actions/checkout@v3 with: submodules: true - uses: actions/setup-python@v4 name: Install Python with: python-version: '3.7' - uses: docker/setup-qemu-action@v1 if: ${{ matrix.arch == 'aarch64' }} name: Set up QEMU - name: Install cibuildwheel run: | python -m pip install cibuildwheel - name: Build wheels run: | python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS_LINUX: ${{matrix.arch}} CIBW_ARCHS_MACOS: "x86_64 arm64 universal2" CIBW_BEFORE_ALL: "bash -c 'cd \"{project}\"; sh .github/tools/install_yajl.sh'" CIBW_BUILD_VERBOSITY: 1 CIBW_ENVIRONMENT_MACOS: "IJSON_EMBED_YAJL=1" CIBW_ENVIRONMENT_WINDOWS: "IJSON_EMBED_YAJL=1" CIBW_TEST_COMMAND: "bash -c 'cd \"{project}\"; pytest -vv'" CIBW_TEST_REQUIRES: pytest cffi - uses: actions/upload-artifact@v3 with: path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 name: Install Python with: python-version: '3.7' - name: Build sdist run: python setup.py sdist - uses: actions/upload-artifact@v3 with: path: dist/*.tar.gz upload_pypi: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags/v') steps: - uses: actions/download-artifact@v3 with: name: artifact path: dist - uses: pypa/gh-action-pypi-publish@master with: user: __token__ password: ${{ secrets.pypi_password }} ijson-3.2.3/.gitignore000066400000000000000000000001661445666302700146510ustar00rootroot00000000000000*.pyc .tox ijson.egg-info *.so build *.whl *.egg .coverage dist .project .pydevproject .settings .cproject .autotools ijson-3.2.3/.gitmodules000066400000000000000000000001251445666302700150310ustar00rootroot00000000000000[submodule "cextern/yajl"] path = cextern/yajl url = https://github.com/lloyd/yajl ijson-3.2.3/CHANGELOG.md000066400000000000000000000306251445666302700144750ustar00rootroot00000000000000# Changelog ## [3.2.3] * Fixed several issues in the ``yajl2_c`` backend and its async generators that were only made apparent when running it with PyPy and/or a CPython debug build (#101). As part of that, an issue was found and fixed in PyPy itself affecting all versions up to 7.3.12, so users will need to wait until the next version is released to be able to use async generators (https://foss.heptapod.net/pypy/pypy/-/issues/3956). * Adapted ``yajl2_c`` async generators to work against PyPy shortcomings (https://foss.heptapod.net/pypy/pypy/-/issues/3965). ## [3.2.2] * Fixed compilation and ``async`` support of the ``yajl2_c`` backend in pyhthon 3.12 (#98). * Check ``IJSON_BUILD_YAJL2C`` environment variable when building ijson to force/skip building the ``yajl2_c`` backend (#102). ## [3.2.1] * Added support for Python 3.12. * Fixed a memory leak in the ``yajl2_c`` backend triggered only when the underlying ``yajl`` functions reported a failure (#97). ## [3.2.0.post0] * Fixed minor README rendering issues that prevented upload of 3.2.0 distributions to PyPI. ## [3.2.0] * New ``ijson.dump`` command-line utility for simple inspection of the ijson iteration process. This tool should be useful for new users who are usually confused with how to use the library, and the prefix in particular. * Fixed bug in ``yajl2_c`` backend introduced in 3.1.2 where random crashes could occur due to an unsafe reference decrement when constructing the parse/items/kvitems generators (#66). * Mark Python 3.10 and 3.11 as explicitly supported. ## [3.1.4] * Fixed bug in ``yajl2_c`` backend introduced in 3.1.0 where ``ijson.items`` didn't work correctly against member names containing ``.`` (#41). * Python backend raises errors on incomplete JSON content that previously wasn't recognised as such, aligning itself with the rest of the backends (#42). ## [3.1.3] * Python backed correctly raises errors when JSON numbers with leading zeros are found in the stream (#40). * Likewise, JSON numbers with fractions where the decimal point is not surrounded by at least one digit on both sides also produce an error now on the python backend. * Fixed detection of file objects with generator-based ``read`` coroutines (i.e., a ``read`` generator decorated with ``@types.coroutine``) for the purpose of automatically routing user calls done through the main entry points. For example, when using ``aiofiles`` objects users could invoke ``async for item in ijson.parse_async(f)`` but not ``async for item in ijson.parse(f)``, while the latter has been possible since 3.1 for native coroutines. ## [3.1.2.post0] * Moved binary wheel generation from GitHub Actions to Travis. This gained us binary ARM wheels, which are becoming increasingly popular (#35) ## [3.1.2] * Fixed minor memory leaks in the initialization methods of the generators of the ``yajl2_c`` backend. All generators (i.e., ``basic_parse``, ``parse``, ``kvitems`` and ``items``) in both their sync and async versions, were affected. ## [3.1.1] * Fixed two problems in the ``yajl2_c`` backend related to `asyncio` support, which prevented some objects like those from ``aiofiles`` from working properly (#32). * Ironing out and documenting some corner cases related to the use of ``use_float=True`` and its side-effect on integer number parsing. ## [3.1.post0] * Removed ``test`` package from binary distributions. ## [3.1] * A new ``use_float`` option has been added to all backends to control whether ``float`` values should be returned for non-integer numbers instead of ``Decimal`` objects. Using this option trades loss of precision (which most applications probably don't care) for performance (which most application do care about). Historically ijson has returned ``Decimal`` objects, and therefore the option defaults to ``False`` for backwards compatibility, but in later releases this default could change to ``True``. * Improved the performance of the ``items`` and ``kvitems`` methods of the ``yajl2_c`` backend (by internally avoiding unnecessary string concatenations). Local tests show a performance improvement of up to ~15%, but mileage might vary depending on your use case and system. * The "raw" functions ``basic_parse``, ``parse``, ``items`` and ``kvitems`` can now be used with different types of inputs. In particular they accept not only file-like objects, but also asynchronous file-like objects, behaving like their ``*_async`` counterparts. They also accept ``bytes`` and ``str`` objects directly (and ``unicode`` objects in python 2.7). Finally, they also accept iterables, in which case they behave like the ``ijson.common.*`` functions, allowing users to tap into the event pipeline. * ``ijson.common`` routines ``parse``, ``items`` and ``kvitems`` are marked as deprecated. Users should use the ``ijson.*`` routines instead, which now accept event iterables. * New ``ijson.get_backend`` function for users to import a backend programmatically (without having to manually use importlib). * New ``IJSON_BACKEND`` environment variable can be used to choose the default backend to be exposed by ijson. * Unicode decoding errors are now reported more clearly to users. In the past there was a mix of empty messages and error types. Now the error type is always the same and there should always be an error messages indicating the offending byte sequence. * ``ijson.common.number`` is marked as deprecated, and will be removed on some later release. ## [3.0.4] * Fixed errors triggered by JSON documents where the top-level value is an object containing an empty-named member (e.g., ``{"": 1}``). Although such documents are valid JSON, they broke basic assumptions made by the ``kvitems`` and ``items`` functions (and all their variants) in all backends, producing different types of unexpected failures, including segmentation faults, raising unexpected exceptions, and producing wrong results. ## [3.0.3] * Fixed segmentation fault in ``yajl2_c`` backend's ``parse`` caused by the previous fix introduced in 3.0.2 (#29). ## [3.0.2] * Fixed memory leak in ``yajl2_c`` backend's ``parse`` functionality (#28). ## [3.0.1] * Adding back the ``parse``, ``kvitems`` and ``items`` functions under the ``ijson.common`` module (#27). These functions take an events iterable instead of a file and are backend-independent (which is not great for performance). They were accidentally removed in the redesign of ijson 3.0, which is why they are coming back. In the future they will slowly transition into being backend-specific rather than independent. ## [3.0] * Exposing backend's name under ``.backend``, and default backend's name under ``ijson.backend``. * Exposing ``ijson.sendable_list`` to users in case it comes in handy. ## [3.0rc3] * Implemented all asynchronous iterables (i.e., ``*_async`` functions) in C for the ``yajl2_c`` backend for increased performance. * Adding Windows builds via AppVeyor, generating binary wheels for Python 3.5+. ## [3.0rc2] * Fixed known problem with 3.0rc1, namely checking that asynchronous files are opened in the correct mode (i.e., binary). * Improved the protocol for user-facing coroutines, where instead of having to send a final, empty bytes string to finish the parsing process users can simply call ``.close()`` on the coroutine. * Greatly increased testing of user-facing coroutines, which in turn uncovered problems that were fixed. * Adding ability to benchmark coroutines with ``benchmark.py``. * Including C code in coverage measurements, and increased overall code coverage up to 99%. ## [3.0rc1] * Full re-design of ijson: instead of working with generators on a "pull" model, it now uses coroutines on a "push" model. The current set of generators (``basic_parse``, ``parse``, ``kvitems`` and ``items``) are implemented on top of these coroutines, and are fully backward compatible. Some text comparing the old a new designs can be found [here](notes/design_notes.rst). * Initial support for ``asyncio`` in python 3.5+ in the for of ``async for``-enabled asynchronous iterators. These are named ``*_async``, and take a file-like object whose ``read()`` method can be ``awaited`` on. * Exposure of underlying infrastructure implementing the push model. These are named ``*_coro``, and take a coroutine-like object (i.e., implementing a ``send`` method) instead of file-like objects. In this scheme, users are in charge of sending chunks of data into the coroutines using ``coro.send(chunk)``. * C backend performance improved by avoiding memory copies when possible when reading data off a file (i.e., using ``readinto`` when possible) and by avoiding tuple packing/unpacking in certain situations. * C extension broken down into separate source files for easier understanding and maintenance. ## [2.6.1] * Fixed a deprecation warning in the C backend present in python 3.8 when parsing Decimal values. ## [2.6.0] * New `kvitems` method in all backends. Like `items`, it takes a prefix, and iterates over the key/value pairs of matching objects (instead of iterating over objects themselves, like in `items`). This is useful for iterating over big objects that would otherwise consume too much memory. * When using python 2, all backends now return `map_key` values as `unicode` objects, not `str` (until now only the Python backend did so). This is what the `json` built-in module does, and allows for correctly handling non-ascii key names. Comparison between `unicode` and `str` objects is possible, so most client code should be unaffected. * Improving error handling in yajl2 backend (ctypes-based) so exceptions caught in callbacks interrupt the parsing process. * Including more files in source distributions (#14). * Adjusting python backend to avoid reading off the input stream too eagerly (#15). ## [2.5.1] * Fixing backwards compatibility, allowing string readers in all backends (#12, #13). ## [2.5] * Default backend changed (#5). Instead of using the python backend, now the fastest available backend is selected by default. * Added support for new `map_type` option (#7). * Fixed bug in `multiple_values` support in C backend (#8). * Added support for ``multiple_values`` flag in python backend (#9). * Forwarding `**kwargs` from `ijson.items` to `ijson.parse` and `ijson.basic_parse` (#10). * Fixing support for yajl versions < 1.0.12. * Improving `common.number` implementation. * Documenting how events and the prefix work (#4). ## [2.4] - New `ijson.backends.yajl2_c` backend written in C and based on the yajl2 library. It performs ~10x faster than cffi backend. - Adding more builds to Travis matrix. - Preventing memory leaks in `ijson.items` - Parse numbers consistent with stdlib json - Correct JSON string parsing in python backend - Publishing package version in __init__.py - Various small fixes in cffi backend [2.4]: https://github.com/ICRAR/ijson/releases/tag/2.4 [2.5]: https://github.com/ICRAR/ijson/releases/tag/v2.5 [2.5.1]: https://github.com/ICRAR/ijson/releases/tag/v2.5.1 [2.6.0]: https://github.com/ICRAR/ijson/releases/tag/v2.6.0 [2.6.1]: https://github.com/ICRAR/ijson/releases/tag/v2.6.1 [3.0rc1]: https://github.com/ICRAR/ijson/releases/tag/v3.0rc1 [3.0rc2]: https://github.com/ICRAR/ijson/releases/tag/v3.0rc2 [3.0rc3]: https://github.com/ICRAR/ijson/releases/tag/v3.0rc3 [3.0]: https://github.com/ICRAR/ijson/releases/tag/v3.0 [3.0.1]: https://github.com/ICRAR/ijson/releases/tag/v3.0.1 [3.0.2]: https://github.com/ICRAR/ijson/releases/tag/v3.0.2 [3.0.3]: https://github.com/ICRAR/ijson/releases/tag/v3.0.3 [3.0.4]: https://github.com/ICRAR/ijson/releases/tag/v3.0.4 [3.1]: https://github.com/ICRAR/ijson/releases/tag/v3.1 [3.1.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.1.post0 [3.1.1]: https://github.com/ICRAR/ijson/releases/tag/v3.1.1 [3.1.2]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2 [3.1.2.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2.post0 [3.1.3]: https://github.com/ICRAR/ijson/releases/tag/v3.1.3 [3.1.4]: https://github.com/ICRAR/ijson/releases/tag/v3.1.4 [3.2.0]: https://github.com/ICRAR/ijson/releases/tag/v3.2.0 [3.2.0.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.2.0.post0 [3.2.1]: https://github.com/ICRAR/ijson/releases/tag/v3.2.1 [3.2.2]: https://github.com/ICRAR/ijson/releases/tag/v3.2.2 [3.2.3]: https://github.com/ICRAR/ijson/releases/tag/v3.2.3 ijson-3.2.3/LICENSE.txt000066400000000000000000000043311445666302700145020ustar00rootroot00000000000000ijson ===== Copyright (c) 2010, Ivan Sagalaev All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name "ijson" nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. yajl ==== Copyright (c) 2007-2014, Lloyd Hilaiel Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ijson-3.2.3/MANIFEST.in000066400000000000000000000001641445666302700144150ustar00rootroot00000000000000include LICENSE.txt include README.rst include CHANGELOG.md graft test include tox.ini graft ijson/backends/yajl2_c ijson-3.2.3/README.rst000066400000000000000000000463441445666302700143600ustar00rootroot00000000000000.. image:: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml/badge.svg :target: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml .. image:: https://ci.appveyor.com/api/projects/status/32wiho6ojw3eakp8/branch/master?svg=true :target: https://ci.appveyor.com/project/rtobar/ijson/branch/master .. image:: https://coveralls.io/repos/github/ICRAR/ijson/badge.svg?branch=master :target: https://coveralls.io/github/ICRAR/ijson?branch=master .. image:: https://badge.fury.io/py/ijson.svg :target: https://badge.fury.io/py/ijson .. image:: https://img.shields.io/pypi/pyversions/ijson.svg :target: https://pypi.python.org/pypi/ijson .. image:: https://img.shields.io/pypi/dd/ijson.svg :target: https://pypi.python.org/pypi/ijson .. image:: https://img.shields.io/pypi/dw/ijson.svg :target: https://pypi.python.org/pypi/ijson .. image:: https://img.shields.io/pypi/dm/ijson.svg :target: https://pypi.python.org/pypi/ijson ===== ijson ===== Ijson is an iterative JSON parser with standard Python iterator interfaces. .. contents:: :local: Installation ============ Ijson is hosted in PyPI, so you should be able to install it via ``pip``:: pip install ijson Binary wheels are provided for major platforms and python versions. These are built and published automatically using `cibuildwheel `_ via Travis CI. Usage ===== All usage example will be using a JSON document describing geographical objects: .. code-block:: json { "earth": { "europe": [ {"name": "Paris", "type": "city", "info": { ... }}, {"name": "Thames", "type": "river", "info": { ... }}, // ... ], "america": [ {"name": "Texas", "type": "state", "info": { ... }}, // ... ] } } High-level interfaces --------------------- Most common usage is having ijson yield native Python objects out of a JSON stream located under a prefix. This is done using the ``items`` function. Here's how to process all European cities: .. code-block:: python import ijson f = urlopen('http://.../') objects = ijson.items(f, 'earth.europe.item') cities = (o for o in objects if o['type'] == 'city') for city in cities: do_something_with(city) For how to build a prefix see the prefix_ section below. Other times it might be useful to iterate over object members rather than objects themselves (e.g., when objects are too big). In that case one can use the ``kvitems`` function instead: .. code-block:: python import ijson f = urlopen('http://.../') european_places = ijson.kvitems(f, 'earth.europe.item') names = (v for k, v in european_places if k == 'name') for name in names: do_something_with(name) Lower-level interfaces ---------------------- Sometimes when dealing with a particularly large JSON payload it may worth to not even construct individual Python objects and react on individual events immediately producing some result. This is achieved using the ``parse`` function: .. code-block:: python import ijson parser = ijson.parse(urlopen('http://.../')) stream.write('') for prefix, event, value in parser: if (prefix, event) == ('earth', 'map_key'): stream.write('<%s>' % value) continent = value elif prefix.endswith('.name'): stream.write('' % value) elif (prefix, event) == ('earth.%s' % continent, 'end_map'): stream.write('' % continent) stream.write('') Even more bare-bones is the ability to react on individual events without even calculating a prefix using the ``basic_parse`` function: .. code-block:: python import ijson events = ijson.basic_parse(urlopen('http://.../')) num_names = sum(1 for event, value in events if event == 'map_key' and value == 'name') .. _command_line: Command line ------------ A command line utility is included with ijson to help visualise the output of each of the routines above. It reads JSON from the standard input, and it prints the results of the parsing method chosen by the user to the standard output. The tool is available by running the ``ijson.dump`` module. For example:: $> echo '{"A": 0, "B": [1, 2, 3, 4]}' | python -m ijson.dump -m parse #: path, name, value -------------------- 0: , start_map, None 1: , map_key, A 2: A, number, 0 3: , map_key, B 4: B, start_array, None 5: B.item, number, 1 6: B.item, number, 2 7: B.item, number, 3 8: B.item, number, 4 9: B, end_array, None 10: , end_map, None Using ``-h/--help`` will show all available options. ``bytes``/``str`` support ------------------------- Although not usually how they are meant to be run, all the functions above also accept ``bytes`` and ``str`` objects (and ``unicode`` in python 2.7) directly as inputs. These are then internally wrapped into a file object, and further processed. This is useful for testing and prototyping, but probably not extremely useful in real-life scenarios. ``asyncio`` support ------------------- In python 3.5+ all of the methods above work also on file-like asynchronous objects, so they can be iterated asynchronously. In other words, something like this: .. code-block:: python import asyncio import ijson async def run(): f = await async_urlopen('http://..../') async for object in ijson.items(f, 'earth.europe.item'): if object['type'] == 'city': do_something_with(city) asyncio.run(run()) An explicit set of ``*_async`` functions also exists offering the same functionality, except they will fail if anything other than a file-like asynchronous object is given to them. (so the example above can also be written using ``ijson.items_async``). In fact in ijson version 3.0 this was the only way to access the ``asyncio`` support. Intercepting events ------------------- The four routines shown above internally chain against each other: tuples generated by ``basic_parse`` are the input for ``parse``, whose results are the input to ``kvitems`` and ``items``. Normally users don't see this interaction, as they only care about the final output of the function they invoked, but there are occasions when tapping into this invocation chain this could be handy. This is supported by passing the output of one function (i.e., an iterable of events, usually a generator) as the input of another, opening the door for user event filtering or injection. For instance if one wants to skip some content before full item parsing: .. code-block:: python import io import ijson parse_events = ijson.parse(io.BytesIO(b'["skip", {"a": 1}, {"b": 2}, {"c": 3}]')) while True: prefix, event, value = next(parse_events) if value == "skip": break for obj in ijson.items(parse_events, 'item'): print(obj) Note that this interception only makes sense for the ``basic_parse -> parse``, ``parse -> items`` and ``parse -> kvitems`` interactions. Note also that event interception is currently not supported by the ``async`` functions. Push interfaces --------------- All examples above use a file-like object as the data input (both the normal case, and for ``asyncio`` support), and hence are "pull" interfaces, with the library reading data as necessary. If for whatever reason it's not possible to use such method, you can still **push** data through yet a different interface: `coroutines `_ (via generators, not ``asyncio`` coroutines). Coroutines effectively allow users to send data to them at any point in time, with a final *target* coroutine-like object receiving the results. In the following example the user is doing the reading instead of letting the library do it: .. code-block:: python import ijson @ijson.coroutine def print_cities(): while True: obj = (yield) if obj['type'] != 'city': continue print(obj) coro = ijson.items_coro(print_cities(), 'earth.europe.item') f = urlopen('http://.../') for chunk in iter(functools.partial(f.read, buf_size)): coro.send(chunk) coro.close() All four ijson iterators have a ``*_coro`` counterpart that work by pushing data into them. Instead of receiving a file-like object and option buffer size as arguments, they receive a single ``target`` argument, which should be a coroutine-like object (anything implementing a ``send`` method) through which results will be published. An alternative to providing a coroutine is to use ``ijson.sendable_list`` to accumulate results, providing the list is cleared after each parsing iteration, like this: .. code-block:: python import ijson events = ijson.sendable_list() coro = ijson.items_coro(events, 'earth.europe.item') f = urlopen('http://.../') for chunk in iter(functools.partial(f.read, buf_size)): coro.send(chunk) process_accumulated_events(events) del events[:] coro.close() process_accumulated_events(events) .. _options: Options ======= Additional options are supported by **all** ijson functions to give users more fine-grained control over certain operations: - The ``use_float`` option (defaults to ``False``) controls how non-integer values are returned to the user. If set to ``True`` users receive ``float()`` values; otherwise ``Decimal`` values are constructed. Note that building ``float`` values is usually faster, but on the other hand there might be loss of precision (which most applications will not care about) and will raise an exception when overflow occurs (e.g., if ``1e400`` is encountered). This option also has the side-effect that integer numbers bigger than ``2^64`` (but *sometimes* ``2^32``, see backends_) will also raise an overflow error, due to similar reasons. Future versions of ijson might change the default value of this option to ``True``. - The ``multiple_values`` option (defaults to ``False``) controls whether multiple top-level values are supported. JSON content should contain a single top-level value (see `the JSON Grammar `_). However there are plenty of JSON files out in the wild that contain multiple top-level values, often separated by newlines. By default ijson will fail to process these with a ``parse error: trailing garbage`` error unless ``multiple_values=True`` is specified. - Similarly the ``allow_comments`` option (defaults to ``False``) controls whether C-style comments (e.g., ``/* a comment */``), which are not supported by the JSON standard, are allowed in the content or not. - For functions taking a file-like object, an additional ``buf_size`` option (defaults to ``65536`` or 64KB) specifies the amount of bytes the library should attempt to read each time. - The ``items`` and ``kvitems`` functions, and all their variants, have an optional ``map_type`` argument (defaults to ``dict``) used to construct objects from the JSON stream. This should be a dict-like type supporting item assignment. Events ====== When using the lower-level ``ijson.parse`` function, three-element tuples are generated containing a prefix, an event name, and a value. Events will be one of the following: - ``start_map`` and ``end_map`` indicate the beginning and end of a JSON object, respectively. They carry a ``None`` as their value. - ``start_array`` and ``end_array`` indicate the beginning and end of a JSON array, respectively. They also carry a ``None`` as their value. - ``map_key`` indicates the name of a field in a JSON object. Its associated value is the name itself. - ``null``, ``boolean``, ``integer``, ``double``, ``number`` and ``string`` all indicate actual content, which is stored in the associated value. .. _prefix: Prefix ====== A prefix represents the context within a JSON document where an event originates at. It works as follows: - It starts as an empty string. - A ```` part is appended when the parser starts parsing the contents of a JSON object member called ``name``, and removed once the content finishes. - A literal ``item`` part is appended when the parser is parsing elements of a JSON array, and removed when the array ends. - Parts are separated by ``.``. When using the ``ijson.items`` function, the prefix works as the selection for which objects should be automatically built and returned by ijson. .. _backends: Backends ======== Ijson provides several implementations of the actual parsing in the form of backends located in ijson/backends: - ``yajl2_c``: a C extension using `YAJL `__ 2.x. This is the fastest, but *might* require a compiler and the YAJL development files to be present when installing this package. Binary wheel distributions exist for major platforms/architectures to spare users from having to compile the package. - ``yajl2_cffi``: wrapper around `YAJL `__ 2.x using CFFI. - ``yajl2``: wrapper around YAJL 2.x using ctypes, for when you can't use CFFI for some reason. - ``yajl``: deprecated YAJL 1.x + ctypes wrapper, for even older systems. - ``python``: pure Python parser, good to use with PyPy You can import a specific backend and use it in the same way as the top level library: .. code-block:: python import ijson.backends.yajl2_cffi as ijson for item in ijson.items(...): # ... Importing the top level library as ``import ijson`` uses the first available backend in the same order of the list above, and its name is recorded under ``ijson.backend``. If the ``IJSON_BACKEND`` environment variable is set its value takes precedence and is used to select the default backend. You can also use the ``ijson.get_backend`` function to get a specific backend based on a name: .. code-block:: python backend = ijson.get_backend('yajl2_c') for item in backend.items(...): # ... Performance tips ================ In more-or-less decreasing order, these are the most common actions you can take to ensure you get most of the performance out of ijson: - Make sure you use the fastest backend available. See backends_ for details. - If you know your JSON data contains only numbers that are "well behaved" consider turning on the ``use_float`` option. See options_ for details. - Make sure you feed ijson with binary data instead of text data. See faq_ #1 for details. - Play with the ``buf_size`` option, as depending on your data source and your system a value different from the default might show better performance. See options_ for details. .. _faq: FAQ === #. **Q**: Does ijson work with ``bytes`` or ``str`` values? **A**: In short: both are accepted as input, outputs are only ``str``. All ijson functions expecting a file-like object should ideally be given one that is opened in binary mode (i.e., its ``read`` function returns ``bytes`` objects, not ``str``). However if a text-mode file object is given then the library will automatically encode the strings into UTF-8 bytes. A warning is currently issued (but not visible by default) alerting users about this automatic conversion. On the other hand ijson always returns text data (JSON string values, object member names, event names, etc) as ``str`` objects in python 3, and ``unicode`` objects in python 2.7. This mimics the behavior of the system ``json`` module. #. **Q**: How are numbers dealt with? **A**: ijson returns ``int`` values for integers and ``decimal.Decimal`` values for floating-point numbers. This is mostly because of historical reasons. Since 3.1 a new ``use_float`` option (defaults to ``False``) is available to return ``float`` values instead. See the options_ section for details. #. **Q**: I'm getting an ``UnicodeDecodeError``, or an ``IncompleteJSONError`` with no message **A**: This error is caused by byte sequences that are not valid in UTF-8. In other words, the data given to ijson is not *really* UTF-8 encoded, or at least not properly. Depending on where the data comes from you have different options: * If you have control over the source of the data, fix it. * If you have a way to intercept the data flow, do so and pass it through a "byte corrector". For instance, if you have a shell pipeline feeding data through ``stdin`` into your process you can add something like ``... | iconv -f utf8 -t utf8 -c | ...`` in between to correct invalid byte sequences. * If you are working purely in python, you can create a UTF-8 decoder using codecs' `incrementaldecoder `_ to leniently decode your bytes into strings, and feed those strings (using a file-like class) into ijson (see our `string_reader_async internal class `_ for some inspiration). In the future ijson might offer something out of the box to deal with invalid UTF-8 byte sequences. #. **Q**: I'm getting ``parse error: trailing garbage`` or ``Additional data found`` errors **A**: This error signals that the input contains more data than the top-level JSON value it's meant to contain. This is *usually* caused by JSON data sources containing multiple values, and is *usually* solved by passing the ``multiple_values=True`` to the ijson function in use. See the options_ section for details. #. **Q**: Are there any differences between the backends? **A**: Apart from their performance, all backends are designed to support the same capabilities. There are however some small known differences: * The ``yajl`` backend doesn't support ``multiple_values=True``. It also doesn't complain about additional data found after the end of the top-level JSON object. When using ``use_float=True`` it also doesn't properly support values greater than 2^32 in 32-bit platforms or Windows. Numbers with leading zeros are not reported as invalid (although they are invalid JSON numbers). Incomplete JSON tokens at the end of an incomplete document (e.g., ``{"a": fals``) are not reported as ``IncompleteJSONError``. * The ``python`` backend doesn't support ``allow_comments=True`` It also internally works with ``str`` objects, not ``bytes``, but this is an internal detail that users shouldn't need to worry about, and might change in the future. Acknowledgements ================ ijson was originally developed and actively maintained until 2016 by `Ivan Sagalaev `_. In 2019 he `handed over `_ the maintenance of the project and the PyPI ownership. Python parser in ijson is relatively simple thanks to `Douglas Crockford `_ who invented a strict, easy to parse syntax. The `YAJL `__ library by `Lloyd Hilaiel `_ is the most popular and efficient way to parse JSON in an iterative fashion. Ijson was inspired by `yajl-py `_ wrapper by `Hatem Nassrat `_. Though ijson borrows almost nothing from the actual yajl-py code it was used as an example of integration with yajl using ctypes. ijson-3.2.3/benchmark.py000066400000000000000000000206511445666302700151660ustar00rootroot00000000000000# # Contributed by Rodrigo Tobar # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2019 # Copyright by UWA (in the framework of the ICRAR) # ''' Benchmarking utility for ijson ''' import argparse import collections import io import os import sys import time import ijson from ijson import compat _benchmarks = collections.OrderedDict() def benchmark(f): _benchmarks[f.__name__] = f return f @benchmark def long_list(n): return b'[' + b','.join([b'1' for _ in range(n)]) + b']' @benchmark def big_int_object(n): return b'{' + b',\n'.join([b'"key_%d": %d' % (i, i) for i in range(n)]) + b'}' @benchmark def big_decimal_object(n): return b'{' + b',\n'.join([b'"key_%d": %d.0' % (i, i) for i in range(n)]) + b'}' @benchmark def big_null_object(n): return b'{' + b',\n'.join([b'"key_%d": null' % (i,) for i in range(n)]) + b'}' @benchmark def big_bool_object(n): return b'{' + b',\n'.join([ b'"key_%d": %s' % (i, b"true" if i % 2 == 0 else b"false") for i in range(n)]) + b'}' @benchmark def big_str_object(n): return b'{' + b',\n'.join([b'"key_%d": "value_%d"' % (i, i) for i in range(n)]) + b'}' @benchmark def big_longstr_object(n): str_template = b"value that is very long and should cause a bit less of JSON parsing" return b'{' + b',\n'.join([b'"key_%d": "%s"' % (i, str_template) for i in range(n)]) + b'}' @benchmark def object_with_10_keys(n): template = b'{' + b',\n'.join([b'"key_%d": "value_%d"' % (i, i) for i in range(10)]) + b'}' return b'[' + b',\n'.join( template for _ in range(n)) + b']' @benchmark def empty_lists(n): return b'[' + b', '.join(b'[]' for _ in range(n)) + b']' @benchmark def empty_objects(n): return b'[' + b', '.join(b'{}' for _ in range(n)) + b']' def parse_benchmarks(s): return [_benchmarks[name] for name in s.split(',')] BACKEND_NAMES = 'python', 'yajl', 'yajl2', 'yajl2_cffi', 'yajl2_c' def load_backends(): backends = collections.OrderedDict() for backend_name in BACKEND_NAMES: try: backends[backend_name] = ijson.get_backend(backend_name) except ImportError: continue return backends _backends = load_backends() def parse_backends(s): backends = collections.OrderedDict() for name in s.split(','): backends[name] = _backends[name] return backends def _stdout_tty_write_flush(message): stdout = sys.stdout if stdout.isatty(): stdout.write(message) stdout.flush() class progress_message(object): def __init__(self, message): self.message = message def __enter__(self): _stdout_tty_write_flush(self.message) return self def __exit__(self, *args): _stdout_tty_write_flush('\r\033[K') if compat.IS_PY35: exec(''' class AsyncReader(object): def __init__(self, data): self.data = io.BytesIO(data) async def read(self, n=-1): return self.data.read(n) def close(self): self.data.close() async def _run_async(method, reader, *method_args, **method_kwargs): async for _ in method(reader, *method_args, **method_kwargs): pass ''') def run_benchmarks(args, benchmark_func=None, fname=None): if bool(benchmark_func) == bool(fname): raise ValueError("Either benchmark_func or fname must be given") if benchmark_func: bname = benchmark_func.__name__ with progress_message('Generating data for benchmark %s...' % (bname,)): data = benchmark_func(args.size) size = len(data) else: bname = fname size = os.stat(args.input).st_size for backend_name, backend in args.backends.items(): # Get correct method and prepare its arguments method = args.method if args.run_async: method += '_async' elif args.run_coro: method += '_coro' method = getattr(backend, method) method_args = () if args.method in ('items', 'kvitems'): method_args = args.prefix, method_kwargs = { 'multiple_values': args.multiple_values, 'use_float': args.use_float } if not args.run_coro: method_kwargs['buf_size'] = args.bufsize # Prepare reader reader = None if not benchmark_func: reader = open(fname, 'rb') else: reader = AsyncReader(data) if args.run_async else io.BytesIO(data) # Prepare function that will run the benchmark if args.run_async: import asyncio loop = asyncio.new_event_loop() def run(): try: loop.run_until_complete(_run_async(method, reader, *method_args, **method_kwargs)) finally: loop.close() elif args.run_coro: def run(): from ijson.utils import sendable_list events = sendable_list() coro = method(events, *method_args, **method_kwargs) if reader: chunks = iter(lambda: reader.read(args.bufsize), b'') else: chunks = (data[pos:pos + args.bufsize] for pos in range(0, len(data), args.bufsize)) for chunk in chunks: coro.send(chunk) del events[:] coro.close() else: def run(): for _ in method(reader, *method_args, **method_kwargs): pass # Go, go, go! start = time.time() run() duration = time.time() - start megabytes = size / 1024. / 1024. print("%.3f, %s, %s, %s, %.3f, %.3f" % (megabytes, args.method, bname, backend_name, duration, megabytes / duration)) reader.close() def main(): DEFAULT_N = 100000 DEFAULT_BUFSIZE = 64 * 1024 ALL_BENCHMARKS = ','.join(_benchmarks) ALL_BACKENDS = ','.join(_backends) parser = argparse.ArgumentParser() parser.add_argument('-s', '--size', type=int, help='Size of JSON content; actual size in bytes might differ, defaults to %d' % DEFAULT_N, default=DEFAULT_N) parser.add_argument('-S', '--bufsize', type=int, help='Buffer size used during parsing; defaults to %d' % DEFAULT_BUFSIZE, default=DEFAULT_BUFSIZE) parser.add_argument('-b', '--benchmarks', type=parse_benchmarks, help='Comma-separated list of benchmarks to include, defaults to %s' % ALL_BENCHMARKS, default=ALL_BENCHMARKS) parser.add_argument('-B', '--backends', type=parse_backends, help='Comma-separated list of backends to include, defaults to %s' % ALL_BACKENDS, default=ALL_BACKENDS) parser.add_argument('-l', '--list', action='store_true', help='List available benchmarks and backends') parser.add_argument('-i', '--input', help='File to use for benchmarks rather than built-in benchmarking functions') parser.add_argument('-m', '--multiple-values', action='store_true', default=False, help='Content has multiple JSON values, useful when used with -i') parser.add_argument('-f', '--use-float', action='store_true', default=False, help='Parse non-integer numbers as float instead of Decimal') parser.add_argument('-M', '--method', choices=['basic_parse', 'parse', 'kvitems', 'items'], help='The method to benchmark', default='basic_parse') parser.add_argument('-c', '--coro', action='store_true', default=False, dest='run_coro', help='Benchmark coroutine methods') if compat.IS_PY35: parser.add_argument('-a', '--async', action='store_true', default=False, dest='run_async', help='Benchmark asyncio-enabled methods') parser.add_argument('-p', '--prefix', help='Prefix (used with -M items|kvitems)', default='') args = parser.parse_args() if args.list: msg = 'Backends:\n' msg += '\n'.join(' - %s' % name for name in _backends) msg += '\nBenchmarks:\n' msg += '\n'.join(' - %s' % name for name in _benchmarks) print(msg) return print("#mbytes,method,test_case,backend,time,mb_per_sec") if args.input: run_benchmarks(args, fname=args.input) else: for benchmark in args.benchmarks: run_benchmarks(args, benchmark) if __name__ == '__main__': main() ijson-3.2.3/cextern/000077500000000000000000000000001445666302700143265ustar00rootroot00000000000000ijson-3.2.3/cextern/yajl/000077500000000000000000000000001445666302700152655ustar00rootroot00000000000000ijson-3.2.3/ijson/000077500000000000000000000000001445666302700140005ustar00rootroot00000000000000ijson-3.2.3/ijson/__init__.py000066400000000000000000000032221445666302700161100ustar00rootroot00000000000000''' Iterative JSON parser. Main API: - ``ijson.parse``: iterator returning parsing events with the object tree context, see ``ijson.common.parse`` for docs. - ``ijson.items``: iterator returning Python objects found under a specified prefix, see ``ijson.common.items`` for docs. Top-level ``ijson`` module exposes method from the pure Python backend. There's also two other backends using the C library yajl in ``ijson.backends`` that have the same API and are faster under CPython. ''' from ijson.common import JSONError, IncompleteJSONError, ObjectBuilder, compat from ijson.utils import coroutine, sendable_list from .version import __version__ def get_backend(backend): """Import the backend named ``backend``""" import importlib return importlib.import_module('ijson.backends.' + backend) def _default_backend(): import os if 'IJSON_BACKEND' in os.environ: return get_backend(os.environ['IJSON_BACKEND']) for backend in ('yajl2_c', 'yajl2_cffi', 'yajl2', 'yajl', 'python'): try: return get_backend(backend) except ImportError: continue raise ImportError('no backends available') backend = _default_backend() del _default_backend basic_parse = backend.basic_parse basic_parse_coro = backend.basic_parse_coro parse = backend.parse parse_coro = backend.parse_coro items = backend.items items_coro = backend.items_coro kvitems = backend.kvitems kvitems_coro = backend.kvitems_coro if compat.IS_PY35: basic_parse_async = backend.basic_parse_async parse_async = backend.parse_async items_async = backend.items_async kvitems_async = backend.kvitems_async backend = backend.backendijson-3.2.3/ijson/backends/000077500000000000000000000000001445666302700155525ustar00rootroot00000000000000ijson-3.2.3/ijson/backends/__init__.py000066400000000000000000000033461445666302700176710ustar00rootroot00000000000000import os import warnings class YAJLImportError(ImportError): pass def require_version(version, required): ''' Asserts that the major component of 'version' is equal to 'required'. Raises YAJLImportError otherwise. ''' major, rest = divmod(version, 10000) minor, micro = divmod(rest, 100) if major != required: raise YAJLImportError('YAJL version %s.x required, found %s.%s.%s' % (required, major, minor, micro)) def get_yajl_version(yajl): try: return yajl.yajl_version() except AttributeError: warnings.warn('Cannot determine yajl version, assuming <1.0.12') return 10000 def find_yajl_ctypes(required): ''' Finds and loads yajl shared object of the required major version (1, 2, ...) using ctypes. ''' # Importing ``ctypes`` should be in scope of this function to prevent failure # of `backends`` package load in a runtime where ``ctypes`` is not available. # Example of such environment is Google App Engine (GAE). from ctypes import util, cdll so_name = os.getenv('YAJL_DLL') or util.find_library('yajl') if so_name is None: raise YAJLImportError('YAJL shared object not found.') try: yajl = cdll.LoadLibrary(so_name) except OSError: raise YAJLImportError('Unable to load YAJL.') require_version(get_yajl_version(yajl), required) return yajl def find_yajl_cffi(ffi, required): ''' Finds and loads yajl shared object of the required major version (1, 2, ...) using cffi. ''' try: yajl = ffi.dlopen(os.getenv('YAJL_DLL') or 'yajl') except OSError: raise YAJLImportError('Unable to load YAJL.') require_version(get_yajl_version(yajl), required) return yajl ijson-3.2.3/ijson/backends/_yajl2_ctypes_common.py000066400000000000000000000052061445666302700222460ustar00rootroot00000000000000''' Common ctypes routines for yajl library handling ''' from ctypes import Structure, c_uint, c_char, c_ubyte, c_int, c_long, c_longlong, c_double,\ c_void_p, c_char_p, CFUNCTYPE, POINTER, string_at, cast from ijson import common, backends from ijson.compat import b2s C_EMPTY = CFUNCTYPE(c_int, c_void_p) C_INT = CFUNCTYPE(c_int, c_void_p, c_int) C_LONG = CFUNCTYPE(c_int, c_void_p, c_long) C_LONGLONG = CFUNCTYPE(c_int, c_void_p, c_longlong) C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double) C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint) def _get_callback_data(yajl_version): return [ # Mapping of JSON parser events to callback C types and value converters. # Used to define the Callbacks structure and actual callback functions # inside the parse function. ('null', 'null', C_EMPTY, lambda: None), ('boolean', 'boolean', C_INT, lambda v: bool(v)), ('integer', 'number', C_LONG if yajl_version == 1 else C_LONGLONG, lambda v: int(v)), ('double', 'number', C_DOUBLE, lambda v: v), ('number', 'number', C_STR, lambda v, l: common.integer_or_decimal(b2s(string_at(v, l)))), ('string', 'string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')), ('start_map', 'start_map', C_EMPTY, lambda: None), ('map_key', 'map_key', C_STR, lambda v, l: string_at(v, l).decode('utf-8')), ('end_map', 'end_map', C_EMPTY, lambda: None), ('start_array', 'start_array', C_EMPTY, lambda: None), ('end_array', 'end_array', C_EMPTY, lambda: None), ] YAJL_OK = 0 YAJL_CANCELLED = 1 YAJL_INSUFFICIENT_DATA = 2 YAJL_ERROR = 3 def get_yajl(version): yajl = backends.find_yajl_ctypes(version) yajl.yajl_alloc.restype = POINTER(c_char) yajl.yajl_get_error.restype = POINTER(c_char) return yajl def _callback(send, use_float, field, event, func_type, func): if use_float and field == 'number': return func_type() def c_callback(_context, *args): send((event, func(*args))) return 1 return func_type(c_callback) def make_callbaks(send, use_float, yajl_version): callback_data = _get_callback_data(yajl_version) class Callbacks(Structure): _fields_ = [(name, type) for name, _, type, _ in callback_data] return Callbacks(*[_callback(send, use_float, *data) for data in callback_data]) def yajl_get_error(yajl, handle, buffer): perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = cast(perror, c_char_p).value try: error = error.decode('utf-8') except UnicodeDecodeError: pass yajl.yajl_free_error(handle, perror) return error ijson-3.2.3/ijson/backends/python.py000066400000000000000000000232241445666302700174500ustar00rootroot00000000000000''' Pure-python parsing backend. ''' from json.decoder import scanstring import re from ijson import common, utils import codecs LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S') UNARY_LEXEMES = set('[]{},') EOF = -1, None class UnexpectedSymbol(common.JSONError): def __init__(self, symbol, pos): super(UnexpectedSymbol, self).__init__( 'Unexpected symbol %r at %d' % (symbol, pos) ) @utils.coroutine def utf8_encoder(target): decoder = codecs.getincrementaldecoder('utf-8')() decode = decoder.decode send = target.send while True: try: final = False bdata = (yield) except GeneratorExit: final = True bdata = b'' try: sdata = decode(bdata, final) except UnicodeDecodeError as e: try: target.close() except: pass raise common.IncompleteJSONError(e) if sdata: send(sdata) elif not bdata: target.close() break @utils.coroutine def Lexer(target): """ Parses lexemes out of the incoming content, and sends them to parse_value. A special EOF result is sent when the data source has been exhausted to give parse_value the possibility of raising custom exceptions due to missing content. """ try: data = (yield) except GeneratorExit: data = '' buf = data pos = 0 discarded = 0 send = target.send while True: match = LEXEME_RE.search(buf, pos) if match: lexeme = match.group() if lexeme == '"': pos = match.start() start = pos + 1 while True: try: end = buf.index('"', start) escpos = end - 1 while buf[escpos] == '\\': escpos -= 1 if (end - escpos) % 2 == 0: start = end + 1 else: break except ValueError: try: data = (yield) except GeneratorExit: data = '' if not data: raise common.IncompleteJSONError('Incomplete string lexeme') buf += data send((discarded + pos, buf[pos:end + 1])) pos = end + 1 else: while lexeme not in UNARY_LEXEMES and match.end() == len(buf): try: data = (yield) except GeneratorExit: data = '' if not data: break buf += data match = LEXEME_RE.search(buf, pos) lexeme = match.group() send((discarded + match.start(), lexeme)) pos = match.end() else: # Don't ask data from an already exhausted source if data: try: data = (yield) except GeneratorExit: data = '' if not data: # Normally should raise StopIteration, but can raise # IncompleteJSONError too, which is the point of sending EOF try: target.send(EOF) except StopIteration: pass break discarded += len(buf) buf = data pos = 0 # Parsing states _PARSE_VALUE = 0 _PARSE_ARRAY_ELEMENT_END = 1 _PARSE_OBJECT_KEY = 2 _PARSE_OBJECT_END = 3 # infinity singleton for overflow checks inf = float("inf") @utils.coroutine def parse_value(target, multivalue, use_float): """ Parses results coming out of the Lexer into ijson events, which are sent to `target`. A stack keeps track of the type of object being parsed at the time (a value, and object or array -- the last two being values themselves). A special EOF result coming from the Lexer indicates that no more content is expected. This is used to check for incomplete content and raise the appropriate exception, which wouldn't be possible if the Lexer simply closed this co-routine (either explicitly via .close(), or implicitly by itself finishing and decreasing the only reference to the co-routine) since that causes a GeneratorExit exception that cannot be replaced with a custom one. """ state_stack = [_PARSE_VALUE] pop = state_stack.pop push = state_stack.append send = target.send prev_pos, prev_symbol = None, None to_number = common.integer_or_float if use_float else common.integer_or_decimal while True: if prev_pos is None: pos, symbol = (yield) if (pos, symbol) == EOF: if state_stack: raise common.IncompleteJSONError('Incomplete JSON content') break else: pos, symbol = prev_pos, prev_symbol prev_pos, prev_symbol = None, None try: state = state_stack[-1] except IndexError: if multivalue: state = _PARSE_VALUE push(state) else: raise common.JSONError('Additional data found') assert state_stack if state == _PARSE_VALUE: # Simple, common cases if symbol == 'null': send(('null', None)) pop() elif symbol == 'true': send(('boolean', True)) pop() elif symbol == 'false': send(('boolean', False)) pop() elif symbol[0] == '"': send(('string', parse_string(symbol))) pop() # Array start elif symbol == '[': send(('start_array', None)) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol == ']': send(('end_array', None)) pop() else: prev_pos, prev_symbol = pos, symbol push(_PARSE_ARRAY_ELEMENT_END) push(_PARSE_VALUE) # Object start elif symbol == '{': send(('start_map', None)) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol == '}': send(('end_map', None)) pop() else: prev_pos, prev_symbol = pos, symbol push(_PARSE_OBJECT_KEY) # A number else: # JSON numbers can't contain leading zeros if ((len(symbol) > 1 and symbol[0] == '0' and symbol[1] not in ('e', 'E', '.')) or (len(symbol) > 2 and symbol[0:2] == '-0' and symbol[2] not in ('e', 'E', '.'))): raise common.JSONError('Invalid JSON number: %s' % (symbol,)) # Fractions need a leading digit and must be followed by a digit if symbol[0] == '.' or symbol[-1] == '.': raise common.JSONError('Invalid JSON number: %s' % (symbol,)) try: number = to_number(symbol) if number == inf: raise common.JSONError("float overflow: %s" % (symbol,)) except: if 'true'.startswith(symbol) or 'false'.startswith(symbol) or 'null'.startswith(symbol): raise common.IncompleteJSONError('Incomplete JSON content') raise UnexpectedSymbol(symbol, pos) else: send(('number', number)) pop() elif state == _PARSE_OBJECT_KEY: if symbol[0] != '"': raise UnexpectedSymbol(symbol, pos) send(('map_key', parse_string(symbol))) pos, symbol = (yield) if (pos, symbol) == EOF: raise common.IncompleteJSONError('Incomplete JSON content') if symbol != ':': raise UnexpectedSymbol(symbol, pos) state_stack[-1] = _PARSE_OBJECT_END push(_PARSE_VALUE) elif state == _PARSE_OBJECT_END: if symbol == ',': state_stack[-1] = _PARSE_OBJECT_KEY elif symbol != '}': raise UnexpectedSymbol(symbol, pos) else: send(('end_map', None)) pop() pop() elif state == _PARSE_ARRAY_ELEMENT_END: if symbol == ',': state_stack[-1] = _PARSE_ARRAY_ELEMENT_END push(_PARSE_VALUE) elif symbol != ']': raise UnexpectedSymbol(symbol, pos) else: send(('end_array', None)) pop() pop() def parse_string(symbol): return scanstring(symbol, 1)[0] def basic_parse_basecoro(target, multiple_values=False, allow_comments=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - file: a readable file-like object with JSON input ''' if allow_comments: raise ValueError("Comments are not supported by the python backend") return utf8_encoder(Lexer(parse_value(target, multiple_values, use_float))) common.enrich_backend(globals()) ijson-3.2.3/ijson/backends/yajl.py000066400000000000000000000035001445666302700170610ustar00rootroot00000000000000''' Wrapper for YAJL C library version 1.x. ''' from ctypes import Structure, c_uint, byref from ijson import common, utils from ijson.backends import _yajl2_ctypes_common yajl = _yajl2_ctypes_common.get_yajl(1) class Config(Structure): _fields_ = [ ("allowComments", c_uint), ("checkUTF8", c_uint) ] @utils.coroutine def basic_parse_basecoro(target, allow_comments=False, multiple_values=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - check_utf8: if True, parser will cause an error if input is invalid utf-8 - buf_size: a size of an input buffer ''' if multiple_values: raise ValueError("yajl backend doesn't support multiple_values") callbacks = _yajl2_ctypes_common.make_callbaks(target.send, use_float, 1) config = Config(allow_comments, True) handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_parse_complete(handle) if result == _yajl2_ctypes_common.YAJL_ERROR: error = _yajl2_ctypes_common.yajl_get_error(yajl, handle, buffer) raise common.JSONError(error) elif not buffer: if result == _yajl2_ctypes_common.YAJL_INSUFFICIENT_DATA: raise common.IncompleteJSONError('Incomplete JSON data') break finally: yajl.yajl_free(handle) common.enrich_backend(globals()) ijson-3.2.3/ijson/backends/yajl2.py000066400000000000000000000033641445666302700171530ustar00rootroot00000000000000''' Wrapper for YAJL C library version 2.x. ''' from ctypes import byref from ijson import common, utils from ijson.backends import _yajl2_ctypes_common yajl = _yajl2_ctypes_common.get_yajl(2) # constants defined in yajl_parse.h YAJL_ALLOW_COMMENTS = 1 YAJL_MULTIPLE_VALUES = 8 @utils.coroutine def basic_parse_basecoro(target, allow_comments=False, multiple_values=False, use_float=False): ''' Iterator yielding unprefixed events. Parameters: - f: a readable file-like object with JSON input - allow_comments: tells parser to allow comments in JSON input - buf_size: a size of an input buffer - multiple_values: allows the parser to parse multiple JSON objects ''' callbacks = _yajl2_ctypes_common.make_callbaks(target.send, use_float, 2) handle = yajl.yajl_alloc(byref(callbacks), None, None) if allow_comments: yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, 1) if multiple_values: yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, 1) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_complete_parse(handle) if result != _yajl2_ctypes_common.YAJL_OK: error = _yajl2_ctypes_common.yajl_get_error(yajl, handle, buffer) exception = common.IncompleteJSONError if result == _yajl2_ctypes_common.YAJL_INSUFFICIENT_DATA else common.JSONError raise exception(error) if not buffer: break finally: yajl.yajl_free(handle) common.enrich_backend(globals()) ijson-3.2.3/ijson/backends/yajl2_c.py000066400000000000000000000043501445666302700174510ustar00rootroot00000000000000# # Contributed by Rodrigo Tobar # # ICRAR - International Centre for Radio Astronomy Research # (c) UWA - The University of Western Australia, 2016 # Copyright by UWA (in the framework of the ICRAR) # ''' Wrapper for _yajl2 C extension module ''' from ijson import common, compat, utils from . import _yajl2 _get_buf_size = lambda kwargs: kwargs.pop('buf_size', 64 * 1024) @utils.coroutine def basic_parse_basecoro(target, **kwargs): return _yajl2.basic_parse_basecoro(target.send, **kwargs) def basic_parse_gen(file, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.basic_parse(f, buf_size, **kwargs) def basic_parse_async(file, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.basic_parse_async(file, buf_size, **kwargs) @utils.coroutine def parse_basecoro(target, **kwargs): return _yajl2.parse_basecoro(target.send, **kwargs) def parse_gen(file, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.parse(f, buf_size, **kwargs) def parse_async(file, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.parse_async(file, buf_size, **kwargs) @utils.coroutine def kvitems_basecoro(target, prefix, map_type=None, **kwargs): return _yajl2.kvitems_basecoro(target.send, prefix, map_type, **kwargs) def kvitems_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.kvitems(f, buf_size, prefix, map_type, **kwargs) def kvitems_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.kvitems_async(file, buf_size, prefix, map_type, **kwargs) @utils.coroutine def items_basecoro(target, prefix, map_type=None, **kwargs): return _yajl2.items_basecoro(target.send, prefix, map_type, **kwargs) def items_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) return _yajl2.items(f, buf_size, prefix, map_type, **kwargs) def items_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.items_async(file, buf_size, prefix, map_type, **kwargs) common.enrich_backend(globals()) ijson-3.2.3/ijson/backends/yajl2_c/000077500000000000000000000000001445666302700170755ustar00rootroot00000000000000ijson-3.2.3/ijson/backends/yajl2_c/async_reading_generator.c000066400000000000000000000145041445666302700241210ustar00rootroot00000000000000/* * asynchronous reading generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "common.h" #if PY_VERSION_HEX >= 0x03050000 static int async_reading_generator_init(async_reading_generator *self, PyObject *args, PyObject *kwargs) { self->coro = NULL; self->file = NULL; self->read_func = NULL; self->buf_size = NULL; self->awaitable = NULL; self->events = NULL; self->index = 0; self->file_exhausted = 0; M1_Z(PyArg_ParseTuple(args, "OO", &self->file, &self->buf_size)); M1_Z(PyNumber_Check(self->buf_size)); Py_INCREF(self->file); Py_INCREF(self->buf_size); M1_N(self->events = PyList_New(0)); return 0; } void async_reading_generator_add_coro(async_reading_generator *self, pipeline_node *coro_pipeline) { self->coro = chain(self->events, coro_pipeline); assert(("async_reading_generator works only with basic_parse_basecoro", BasicParseBasecoro_Check(self->coro))); } static void async_reading_generator_dealloc(async_reading_generator *self) { Py_XDECREF(self->events); Py_XDECREF(self->awaitable); Py_XDECREF(self->buf_size); Py_XDECREF(self->read_func); Py_XDECREF(self->file); Py_XDECREF(self->coro); Py_TYPE(self)->tp_free((PyObject*)self); } static void raise_stopiteration(PyObject *value) { #if defined(PYPY_VERSION) // PyPy doesn't seem to support normalised exceptions for coroutines, // see https://foss.heptapod.net/pypy/pypy/-/issues/3965 PyObject *ex_value = PyObject_CallFunctionObjArgs(PyExc_StopIteration, value, NULL); PyErr_SetObject(PyExc_StopIteration, ex_value); Py_DECREF(ex_value); #else PyObject *stop_iteration_args = PyTuple_New(1); PyTuple_SET_ITEM(stop_iteration_args, 0, value); PyErr_SetObject(PyExc_StopIteration, stop_iteration_args); Py_DECREF(stop_iteration_args); #endif } static PyObject *maybe_pop_event(async_reading_generator *self) { PyObject *events = self->events; Py_ssize_t nevents = PyList_Size(events); if (nevents == 0) { return NULL; } PyObject *event = PyList_GET_ITEM(events, self->index++); Py_INCREF(event); if (self->index == nevents) { if (PySequence_DelSlice(events, 0, self->index) == -1) { Py_RETURN_NONE; } self->index = 0; } raise_stopiteration(event); return event; } static int is_gen_coroutine(PyObject *o) { if (PyGen_CheckExact(o)) { PyCodeObject *code = (PyCodeObject *)PyObject_GetAttrString(o, "gi_code"); return code->co_flags & CO_ITERABLE_COROUTINE; } return 0; } static PyObject* value_from_stopiteration() { PyObject *ptype, *pvalue, *ptraceback, *return_value; PyErr_Fetch(&ptype, &pvalue, &ptraceback); if (PyErr_GivenExceptionMatches(pvalue, PyExc_StopIteration)) { return_value = PyObject_GetAttrString(pvalue, "value"); Py_XDECREF(pvalue); } else { return_value = pvalue; } Py_XDECREF(ptype); Py_XDECREF(ptraceback); return return_value; } static PyObject *async_reading_generator_next(PyObject *self) { async_reading_generator *gen = (async_reading_generator *)self; // values are returned via StopIteration exception values if (maybe_pop_event(gen)) { return NULL; } // No events available and nothing else to read, we are done if (gen->file_exhausted) { PyErr_SetNone(PyExc_StopAsyncIteration); return NULL; } // prepare corresponding awaitable if (gen->awaitable == NULL) { if (gen->read_func == NULL) { PyObject *utils35, *get_read_func, *get_read_coro; N_N(utils35 = PyImport_ImportModule("ijson.utils35")); N_N(get_read_func = PyObject_GetAttrString(utils35, "_get_read")); N_N(get_read_coro = PyObject_CallFunctionObjArgs(get_read_func, gen->file, NULL)); N_N(gen->awaitable = PyObject_CallMethod(get_read_coro, "__await__", NULL)); assert(PyIter_Check(gen->awaitable)); Py_DECREF(get_read_coro); Py_DECREF(get_read_func); Py_DECREF(utils35); Py_CLEAR(gen->file); } else { PyObject *read_coro; N_N(read_coro = PyObject_CallFunctionObjArgs(gen->read_func, gen->buf_size, NULL)); // this can be a "normal" awaitable (has an __await__ method) // or a function decorated with types.coroutine (a generator) if (is_gen_coroutine(read_coro)) { gen->awaitable = read_coro; Py_INCREF(gen->awaitable); } else { N_N(gen->awaitable = PyObject_CallMethod(read_coro, "__await__", NULL)); } assert(PyIter_Check(gen->awaitable)); Py_DECREF(read_coro); } } // Propagate values/errors that are not StopIteration PyObject *value = Py_TYPE(gen->awaitable)->tp_iternext(gen->awaitable); if (value) { return value; } else if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { return NULL; } Py_CLEAR(gen->awaitable); // We await on two things: getting the correct read function (only once), // and reading from it (many times, self->read_func is set) if (gen->read_func == NULL) { gen->read_func = value_from_stopiteration(); Py_RETURN_NONE; } // Finished awaiting on read() result, parse it PyObject *buffer = value_from_stopiteration(); Py_buffer view; N_M1(PyObject_GetBuffer(buffer, &view, PyBUF_SIMPLE)); gen->file_exhausted = (view.len == 0); BasicParseBasecoro *basic_parse_basecoro = (BasicParseBasecoro *)gen->coro; PyObject *res = ijson_yajl_parse(basic_parse_basecoro->h, view.buf, view.len); N_N(res); Py_DECREF(res); PyBuffer_Release(&view); Py_DECREF(buffer); // values are returned via StopIteration exception values if (maybe_pop_event(gen)) { return NULL; } // Keep trying Py_RETURN_NONE; } static PyAsyncMethods async_reading_generator_methods = { .am_await = ijson_return_self, }; PyTypeObject AsyncReadingGeneratorType = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(async_reading_generator), .tp_name = "_yajl2.async_reading_generator", .tp_doc = "The awaitable yielded by the asynchronous iterables", .tp_init = (initproc)async_reading_generator_init, .tp_dealloc = (destructor)async_reading_generator_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_as_async = &async_reading_generator_methods, .tp_iter = ijson_return_self, .tp_iternext = async_reading_generator_next, }; #endif // PY_VERSION_HEX >= 0x03050000ijson-3.2.3/ijson/backends/yajl2_c/async_reading_generator.h000066400000000000000000000020531445666302700241220ustar00rootroot00000000000000/* * asynchronous reading generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ASYNC_READING_GENERATOR_H #define ASYNC_READING_GENERATOR_H #define PY_SSIZE_T_CLEAN #include #if PY_VERSION_HEX >= 0x03050000 #include "coro_utils.h" /** * async_reading_generator definition. It keeps the state of the current reading * process, and the list holding the events generated by the yajl parsing. */ typedef struct { PyObject_HEAD PyObject *coro; PyObject *file; PyObject *read_func; PyObject *buf_size; PyObject *awaitable; PyObject *events; Py_ssize_t index; int file_exhausted; } async_reading_generator; void async_reading_generator_add_coro(async_reading_generator *self, pipeline_node *coro_pipeline); extern PyTypeObject AsyncReadingGeneratorType; #endif // PY_VERSION_HEX >= 0x03050000 #endif // ASYNC_READING_GENERATOR_Hijson-3.2.3/ijson/backends/yajl2_c/basic_parse.c000066400000000000000000000026761445666302700215270ustar00rootroot00000000000000/* * basic_parse generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "basic_parse.h" #include "basic_parse_basecoro.h" #include "common.h" /* * __init__, destructor, __iter__ and __next__ */ static int basicparsegen_init(BasicParseGen *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, args, coro_pipeline)); return 0; } static void basicparsegen_dealloc(BasicParseGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* basicparsegen_iternext(PyObject *self) { BasicParseGen *gen = (BasicParseGen *)self; return reading_generator_next(&gen->reading_gen); } PyTypeObject BasicParseGen_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(BasicParseGen), .tp_name = "_yajl2.basic_parse", .tp_doc = "Generator of (evt,value)", .tp_init = (initproc)basicparsegen_init, .tp_dealloc = (destructor)basicparsegen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = basicparsegen_iternext }; ijson-3.2.3/ijson/backends/yajl2_c/basic_parse.h000066400000000000000000000012141445666302700215170ustar00rootroot00000000000000/* * basic_parse generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BASIC_PARSE_H #define BASIC_PARSE_H #define PY_SSIZE_T_CLEAN #include #include "reading_generator.h" /** * basic_parse generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } BasicParseGen; /** * basic_parse generator object type */ extern PyTypeObject BasicParseGen_Type; #endif // BASIC_PARSE_Hijson-3.2.3/ijson/backends/yajl2_c/basic_parse_async.c000066400000000000000000000040031445666302700227060ustar00rootroot00000000000000/* * basic_parse_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "common.h" #include "coro_utils.h" #if PY_VERSION_HEX >= 0x03050000 /** * basic_parse_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } BasicParseAsync; /* * __init__, destructor and __anext__ */ static int basicparseasync_init(BasicParseAsync *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, args)); async_reading_generator_add_coro(self->reading_generator, coro_pipeline); return 0; } static void basicparseasync_dealloc(BasicParseAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *basicparseasync_anext(PyObject *self) { BasicParseAsync *gen = (BasicParseAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods basicparseasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = basicparseasync_anext }; PyTypeObject BasicParseAsync_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(BasicParseAsync), .tp_name = "_yajl2.basic_parse_async", .tp_doc = "Asynchronous iterable yielding (evt,value) tuples", .tp_init = (initproc)basicparseasync_init, .tp_dealloc = (destructor)basicparseasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &basicparseasync_methods }; #endif // PY_VERSION_HEX >= 0x03050000ijson-3.2.3/ijson/backends/yajl2_c/basic_parse_async.h000066400000000000000000000011321445666302700227130ustar00rootroot00000000000000/* * basic_parse_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BASIC_PARSE_ASYNC_H #define BASIC_PARSE_ASYNC_H #define PY_SSIZE_T_CLEAN #include #if PY_VERSION_HEX >= 0x03050000 /** * basic_parse_async asynchronous iterable object type */ extern PyTypeObject BasicParseAsync_Type; #endif // PY_VERSION_HEX >= 0x03050000 #endif // BASIC_PARSE_Hijson-3.2.3/ijson/backends/yajl2_c/basic_parse_basecoro.c000066400000000000000000000174171445666302700234030ustar00rootroot00000000000000/* * basic_parse_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include #include "basic_parse_basecoro.h" #include "common.h" #include "parse_basecoro.h" /* * The YAJL callbacks, they add (evt,value) to a list */ static inline int add_event_and_value(void *ctx, PyObject *evt_name, PyObject *val) { PyObject *target_send = (PyObject *)ctx; if (ParseBasecoro_Check(target_send)) { Z_N(parse_basecoro_send_impl(target_send, evt_name, val)); Py_DECREF(val); return 1; } PyObject *tuple; Z_N(tuple = PyTuple_New(2)); Py_INCREF(evt_name); // this is an element of our static enames var PyTuple_SET_ITEM(tuple, 0, evt_name); PyTuple_SET_ITEM(tuple, 1, val); CORO_SEND(target_send, tuple); Py_DECREF(tuple); return 1; } static int null(void * ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, enames.null_ename, Py_None); } static int boolean(void * ctx, int val) { PyObject *bval = val == 0 ? Py_False : Py_True; Py_INCREF(bval); return add_event_and_value(ctx, enames.boolean_ename, bval); } static int yajl_integer(void *ctx, long long val) { PyObject *ival; #if PY_MAJOR_VERSION < 3 if (val <= 0xFFFFFFFF) { Z_N(ival = PyInt_FromLong((long)val)); } else #endif { Z_N(ival = PyLong_FromLongLong(val)) } return add_event_and_value(ctx, enames.number_ename, ival); } static int yajl_double(void *ctx, double val) { PyObject *dval; Z_N(dval = PyFloat_FromDouble(val)) return add_event_and_value(ctx, enames.number_ename, dval); } static int number(void * ctx, const char *numberVal, size_t numberLen) { // If original string has a dot or an "e/E" we return a Decimal // just like in the common module int is_decimal = 0; const char *iter = numberVal; size_t i; for(i=0; i!=numberLen; i++) { char c = *iter++; if( c == '.' || c == 'e' || c == 'E' ) { is_decimal = 1; break; } } PyObject *val; if( !is_decimal ) { char *nval = (char *)malloc(numberLen + 1); memcpy(nval, numberVal, numberLen); nval[numberLen] = 0; char *endptr; #if PY_MAJOR_VERSION >= 3 val = PyLong_FromString(nval, &endptr, 10); #else // returns either PyLong or PyInt val = PyInt_FromString(nval, &endptr, 10); #endif free(nval); assert(("string provided by yajl is not an integer", val != NULL && endptr != nval)); } else { Z_N(val = PyObject_CallFunction(Decimal, "s#", numberVal, numberLen)); } return add_event_and_value(ctx, enames.number_ename, val); } static int string_cb(void * ctx, const unsigned char *stringVal, size_t stringLen) { PyObject *val; Z_N(val = PyUnicode_FromStringAndSize((char *)stringVal, stringLen)) return add_event_and_value(ctx, enames.string_ename, val); } static int start_map(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, enames.start_map_ename, Py_None); } static int map_key(void *ctx, const unsigned char *key, size_t stringLen) { PyObject *val; Z_N(val = STRING_FROM_UTF8(key, stringLen)) return add_event_and_value(ctx, enames.map_key_ename, val); } static int end_map(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, enames.end_map_ename, Py_None); } static int start_array(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, enames.start_array_ename, Py_None); } static int end_array(void *ctx) { Py_INCREF(Py_None); return add_event_and_value(ctx, enames.end_array_ename, Py_None); } static yajl_callbacks decimal_callbacks = { null, boolean, NULL, NULL, number, string_cb, start_map, map_key, end_map, start_array, end_array }; static yajl_callbacks float_callbacks = { null, boolean, yajl_integer, yajl_double, NULL, string_cb, start_map, map_key, end_map, start_array, end_array }; PyObject* ijson_yajl_parse(yajl_handle handle, char *buffer, size_t length) { yajl_status status; if (length == 0) { status = yajl_complete_parse(handle); } else { status = yajl_parse(handle, (unsigned char *)buffer, length); } if (status != yajl_status_ok) { // An actual problem with the JSON data (otherwise a user error) if (status != yajl_status_client_canceled) { unsigned char *perror = yajl_get_error(handle, 1, (unsigned char *)buffer, length); PyObject *error_obj = PyUnicode_FromString((char *)perror); // error about invalid UTF8 byte sequences can't be converted to string // automatically, so we show the bytes instead if (!error_obj) { PyErr_Clear(); #if PY_MAJOR_VERSION >= 3 error_obj = PyBytes_FromString((char *)perror); #else error_obj = PyString_FromString((char *)perror); #endif PyErr_Clear(); } PyErr_SetObject(IncompleteJSONError, error_obj); if (error_obj) { Py_DECREF(error_obj); } yajl_free_error(handle, perror); } return NULL; } Py_RETURN_NONE; } /* * __init__, destructor, __iter__ and __next__ */ static int basic_parse_basecoro_init(BasicParseBasecoro *self, PyObject *args, PyObject *kwargs) { PyObject *allow_comments = Py_False; PyObject *multiple_values = Py_False; PyObject *use_float = Py_False; self->h = NULL; self->target_send = NULL; char *kwlist[] = {"target_send", "allow_comments", "multiple_values", "use_float", NULL}; if( !PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOO", kwlist, &self->target_send, &allow_comments, &multiple_values, &use_float) ) { return -1; } Py_INCREF(self->target_send); /* * Prepare yajl handle and configure it * The context given to yajl is the coroutine's target, so the callbacks * directly send values to the target */ yajl_callbacks *callbacks; if (PyObject_IsTrue(use_float)) { callbacks = &float_callbacks; } else { callbacks = &decimal_callbacks; } M1_N(self->h = yajl_alloc(callbacks, NULL, (void *)self->target_send)); if (PyObject_IsTrue(allow_comments)) { yajl_config(self->h, yajl_allow_comments, 1); } if (PyObject_IsTrue(multiple_values)) { yajl_config(self->h, yajl_allow_multiple_values, 1); } return 0; } static void basic_parse_basecoro_dealloc(BasicParseBasecoro *self) { if (self->h) { yajl_free(self->h); } Py_XDECREF(self->target_send); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* basic_parse_basecoro_send(PyObject *self, PyObject *arg) { /* Preempt our execution, which might be very long */ // N_M1(PyErr_CheckSignals()); Py_buffer bufview; N_M1(PyObject_GetBuffer(arg, &bufview, PyBUF_SIMPLE)); BasicParseBasecoro *gen = (BasicParseBasecoro *)self; PyObject *ret = ijson_yajl_parse(gen->h, bufview.buf, bufview.len); if (ret != NULL && bufview.len == 0) { // This was the last one, let's end now PyErr_SetNone(PyExc_StopIteration); ret = NULL; } PyBuffer_Release(&bufview); return ret; } static PyObject* basic_parse_basecoro_close(PyObject *self, PyObject *args) { BasicParseBasecoro *gen = (BasicParseBasecoro *)self; N_N(ijson_yajl_parse(gen->h, NULL, 0)); Py_RETURN_NONE; } static PyMethodDef basic_parse_basecoro_methods[] = { {"send", basic_parse_basecoro_send, METH_O, "coroutine's send method"}, {"close", basic_parse_basecoro_close, METH_NOARGS, "coroutine's close method"}, {NULL, NULL, 0, NULL} }; PyTypeObject BasicParseBasecoro_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(BasicParseBasecoro), .tp_name = "_yajl2.basic_parse_basecoro", .tp_doc = "Coroutine dispatching (evt,value) pairs", .tp_init = (initproc)basic_parse_basecoro_init, .tp_dealloc = (destructor)basic_parse_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = basic_parse_basecoro_methods }; ijson-3.2.3/ijson/backends/yajl2_c/basic_parse_basecoro.h000066400000000000000000000020721445666302700233770ustar00rootroot00000000000000/* * basic_parse_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BASIC_PARSE_BASECORO_H #define BASIC_PARSE_BASECORO_H #define PY_SSIZE_T_CLEAN #include #include #include /** * basic_parse_basecoro coroutine object structure */ typedef struct { PyObject_HEAD yajl_handle h; PyObject *target_send; } BasicParseBasecoro; /** * basic_parse_basecoro coroutine object type */ extern PyTypeObject BasicParseBasecoro_Type; /** * Utility function to check if an object is an basic_parse_basecoro coroutine or not */ #define BasicParseBasecoro_Check(o) (Py_TYPE(o) == &BasicParseBasecoro_Type) /** * yajl parsing routine wrapper that turns yajl errors into exceptions */ PyObject* ijson_yajl_parse(yajl_handle handle, char *buffer, size_t length); #endif // BASIC_PARSE_BASECORO_Hijson-3.2.3/ijson/backends/yajl2_c/builder.h000066400000000000000000000107701445666302700207010ustar00rootroot00000000000000/* * builder_t type and associated methods * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2019 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef BUILDER_H #define BUILDER_H #include #include "common.h" #define PY_SSIZE_T_CLEAN #include /** * builder_t structure. * * This is the parallel of the ObjectBuilder class from the ijson.common module, * only a bit more complicated since it's all C */ typedef struct _builder { PyObject *value; int active; PyObject *key; PyObject *value_stack; PyObject *map_type; } builder_t; /** * Initializes an empty builder which can be safely destroyed. * * @param builder the builder to empty-initialize */ static inline void builder_create(builder_t *builder) { builder->value = NULL; builder->map_type = NULL; builder->value_stack = NULL; } /** * Initialilzes a builder capable of assembling Python objects out of prefixed * events. * * @param builder the builder to initialize * @param map_type The mapping type to use for constructing objects out of * (key, value) pairs. If None then dict is used. */ static inline int builder_init(builder_t *builder, PyObject *map_type) { M1_N(builder->value_stack = PyList_New(0)); if (map_type != Py_None) { builder->map_type = map_type; Py_INCREF(map_type); } return 0; } /** * Destroys a builder and all its associated contents * @param builder The builder to destroy */ static inline void builder_destroy(builder_t *builder) { Py_DECREF(builder->value_stack); Py_XDECREF(builder->map_type); Py_XDECREF(builder->value); } /** * Returns whether the builder is currently active or not. * @param builder A builder * @return whether the builder is active (1) or not (0) */ static inline int builder_isactive(builder_t *builder) { return builder->active; } /** * Returns a new reference to the current value constructed by the builder. * * @param builder The builder * @return The value as currently constructed by this builder */ static inline PyObject *builder_value(builder_t *builder) { Py_INCREF(builder->value); return builder->value; } /** * Resets a builder to a pristine state so it can be used to build a new value. * @param builder The builder to reset * @return 0 if successful, -1 in case of an error */ static inline int builder_reset(builder_t *builder) { builder->active = 0; Py_CLEAR(builder->value); Py_CLEAR(builder->key); Py_ssize_t nvals = PyList_Size(builder->value_stack); M1_M1(PyList_SetSlice(builder->value_stack, 0, nvals, NULL)); return 0; } static inline int _builder_add(builder_t *builder, PyObject *value) { Py_ssize_t nvals = PyList_Size(builder->value_stack); if (nvals == 0) { Py_INCREF(value); builder->value = value; } else { PyObject *last; M1_N(last = PyList_GetItem(builder->value_stack, nvals-1)); assert(("stack element not list or dict-like", PyList_Check(last) || PyMapping_Check(last))); if (PyList_Check(last)) { M1_M1(PyList_Append(last, value)); } else { // it's a dict-like object M1_M1(PyObject_SetItem(last, builder->key, value)); } } return 0; } /** * Feed an (event, value) pair to the builder for further constructing the * underlying value * @param builder A builder * @param ename The event name * @param value The value associated to this event * @return 0 if successful, -1 in case of an error */ static inline int builder_event(builder_t *builder, PyObject *ename, PyObject *value) { builder->active = 1; if (ename == enames.map_key_ename) { Py_XDECREF(builder->key); builder->key = value; Py_INCREF(builder->key); } else if (ename == enames.start_map_ename) { PyObject *mappable; if (builder->map_type) { mappable = PyObject_CallFunctionObjArgs(builder->map_type, NULL); } else { mappable = PyDict_New(); } M1_N(mappable); M1_M1(_builder_add(builder, mappable)); M1_M1(PyList_Append(builder->value_stack, mappable)); Py_DECREF(mappable); } else if (ename == enames.start_array_ename) { PyObject *list; M1_N(list = PyList_New(0)); M1_M1(_builder_add(builder, list)); M1_M1(PyList_Append(builder->value_stack, list)); Py_DECREF(list); } else if (ename == enames.end_array_ename || ename == enames.end_map_ename) { // pop Py_ssize_t nvals = PyList_Size(builder->value_stack); M1_M1(PyList_SetSlice(builder->value_stack, nvals-1, nvals, NULL)); } else { M1_M1(_builder_add(builder, value)); } return 0; } #endif /* BUILDER_H */ijson-3.2.3/ijson/backends/yajl2_c/common.h000066400000000000000000000052041445666302700205370ustar00rootroot00000000000000/* * Common definitions for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2019 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef COMMON_H #define COMMON_H #define PY_SSIZE_T_CLEAN #include #define STRING_FROM_UTF8(val, len) PyUnicode_FromStringAndSize((const char *)val, len) #if PY_MAJOR_VERSION >= 3 #define Py_TPFLAGS_HAVE_ITER 0 #endif /* * Error-handling macros to help reducing clutter in the code. * N: NULL, M1: -1, Z: zero, NZ: not-zero, LZ: less-than-zero * */ #define RETURN_X_IF_COND(statement, X, cond) \ do { \ if ((statement) cond) { \ return X; \ } \ } while(0); #define M1_M1(stmt) RETURN_X_IF_COND(stmt, -1, == -1) #define M1_N(stmt) RETURN_X_IF_COND(stmt, -1, == NULL) #define M1_NZ(stmt) RETURN_X_IF_COND(stmt, -1, != 0) #define M1_Z(stmt) RETURN_X_IF_COND(stmt, -1, == 0) #define N_M1(stmt) RETURN_X_IF_COND(stmt, NULL, == -1) #define N_N(stmt) RETURN_X_IF_COND(stmt, NULL, == NULL) #define N_Z(stmt) RETURN_X_IF_COND(stmt, NULL, == 0) #define N_NZ(stmt) RETURN_X_IF_COND(stmt, NULL, != 0) #define Z_M1(stmt) RETURN_X_IF_COND(stmt, 0, == -1) #define Z_N(stmt) RETURN_X_IF_COND(stmt, 0, == NULL) #define Z_NZ(stmt) RETURN_X_IF_COND(stmt, 0, != 0) #define X_LZ(stmt, X) RETURN_X_IF_COND(stmt, X, < 0) #define X_N(stmt, X) RETURN_X_IF_COND(stmt, X, == NULL) /* * A structure (and variable) holding utf-8 strings with the event names * This way we avoid calculating them every time, and we can compare them * via direct equality comparison instead of via strcmp. */ typedef struct _event_names { PyObject *null_ename; PyObject *boolean_ename; PyObject *integer_ename; PyObject *double_ename; PyObject *number_ename; PyObject *string_ename; PyObject *start_map_ename; PyObject *map_key_ename; PyObject *end_map_ename; PyObject *start_array_ename; PyObject *end_array_ename; } enames_t; extern enames_t enames; extern PyObject *dot, *item, *dotitem; extern PyObject *JSONError; extern PyObject *IncompleteJSONError; extern PyObject *Decimal; #define CORO_SEND(target_send, event) \ { \ if (PyList_Check(target_send)) { \ Z_M1(PyList_Append(target_send, event)); \ } \ else { \ Z_N( PyObject_CallFunctionObjArgs(target_send, event, NULL) ); \ } \ } /* Common function used by __iter__ method in coroutines/generators */ PyObject* ijson_return_self(PyObject *self); /* Common function used by empty methods in coroutines/generators */ PyObject* ijson_return_none(PyObject *self); #endif /* COMMON_H */ijson-3.2.3/ijson/backends/yajl2_c/coro_utils.c000066400000000000000000000020601445666302700214210ustar00rootroot00000000000000/* * Coroutine utilities implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "coro_utils.h" PyObject *chain(PyObject *sink, pipeline_node *coro_pipeline) { PyObject *coro = sink; Py_INCREF(coro); int element = 0; while (1) { pipeline_node node = coro_pipeline[element++]; if (node.type == NULL) { break; } PyObject *coro_args; if (node.args) { int nargs = PyTuple_Size(node.args); N_N(coro_args = PyTuple_New(nargs + 1)); Py_INCREF(coro); PyTuple_SET_ITEM(coro_args, 0, coro); int i; for (i = 0; i != nargs; i++) { PyTuple_SET_ITEM(coro_args, i + 1, PySequence_GetItem(node.args, i)); } } else { N_N(coro_args = PyTuple_Pack(1, coro)); } Py_DECREF(coro); N_N(coro = PyObject_Call((PyObject *)node.type, coro_args, node.kwargs)); Py_DECREF(coro_args); } return coro; }ijson-3.2.3/ijson/backends/yajl2_c/coro_utils.h000066400000000000000000000017331445666302700214340ustar00rootroot00000000000000/* * Coroutine utilities for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef CORO_UTILS_H #define CORO_UTILS_H #define PY_SSIZE_T_CLEAN #include /** * A tuple defining how to create an object */ typedef struct _pipeline_node { PyTypeObject *type; PyObject *args; PyObject *kwargs; } pipeline_node; /** * Creates coroutines as described in coro_pipeline, with a final sink * * @param sink the final coroutine-like object that will receive the result of * the pipeline * @param coro_pipeline the description of all elements to create in a coroutine * pipeline * @return The head of the coroutine pipeline (i.e., the coroutine where users * will send elements to) */ PyObject *chain(PyObject *sink, pipeline_node *coro_pipeline); #endif // CORO_UTILS_Hijson-3.2.3/ijson/backends/yajl2_c/items.c000066400000000000000000000033021445666302700203600ustar00rootroot00000000000000/* * items generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "items.h" #include "items_basecoro.h" #include "parse_basecoro.h" #include "basic_parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int itemsgen_init(ItemsGen *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *items_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&ItemsBasecoro_Type, items_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, reading_args, coro_pipeline)); Py_DECREF(items_args); Py_DECREF(reading_args); return 0; } static void itemsgen_dealloc(ItemsGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* itemsgen_iternext(PyObject *self) { ItemsGen *gen = (ItemsGen *)self; return reading_generator_next(&gen->reading_gen); } /* * items generator object type */ PyTypeObject ItemsGen_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(ItemsGen), .tp_name = "_yajl2.items", .tp_doc = "Generates items", .tp_init = (initproc)itemsgen_init, .tp_dealloc = (destructor)itemsgen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = itemsgen_iternext }; ijson-3.2.3/ijson/backends/yajl2_c/items.h000066400000000000000000000010561445666302700203710ustar00rootroot00000000000000/* * items generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ITEMS_H #define ITEMS_H #include "reading_generator.h" /** * items generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } ItemsGen; /** * items generator object type */ extern PyTypeObject ItemsGen_Type; #endif /* ITEMS_H */ijson-3.2.3/ijson/backends/yajl2_c/items_async.c000066400000000000000000000043341445666302700215630ustar00rootroot00000000000000/* * items_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "parse_basecoro.h" #include "items_basecoro.h" #include "common.h" #include "coro_utils.h" #if PY_VERSION_HEX >= 0x03050000 /** * items_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } ItemsAsync; /* * __init__, destructor and __anext__ */ static int itemsasync_init(ItemsAsync *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *items_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&ItemsBasecoro_Type, items_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); async_reading_generator_add_coro(self->reading_generator, coro_pipeline); Py_DECREF(items_args); Py_DECREF(reading_args); return 0; } static void itemsasync_dealloc(ItemsAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *itemsasync_anext(PyObject *self) { ItemsAsync *gen = (ItemsAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods itemsasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = itemsasync_anext }; PyTypeObject ItemsAsync_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(ItemsAsync), .tp_name = "_yajl2._items_async", .tp_doc = "Asynchronous iterable yielding fully-built items", .tp_init = (initproc)itemsasync_init, .tp_dealloc = (destructor)itemsasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &itemsasync_methods }; #endif // PY_VERSION_HEX >= 0x03050000ijson-3.2.3/ijson/backends/yajl2_c/items_async.h000066400000000000000000000010751445666302700215670ustar00rootroot00000000000000/* * items_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ITEMS_ASYNC_H #define ITEMS_ASYNC_H #define PY_SSIZE_T_CLEAN #include #if PY_VERSION_HEX >= 0x03050000 /** * items_async asynchronous iterable object type */ extern PyTypeObject ItemsAsync_Type; #endif // PY_VERSION_HEX >= 0x03050000 #endif // ITEMS_ASYNC_Hijson-3.2.3/ijson/backends/yajl2_c/items_basecoro.c000066400000000000000000000057041445666302700222450ustar00rootroot00000000000000/* * items_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "items_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int items_basecoro_init(ItemsBasecoro *self, PyObject *args, PyObject *kwargs) { self->target_send = NULL; self->prefix = NULL; self->object_depth = 0; builder_create(&self->builder); PyObject *map_type; M1_Z(PyArg_ParseTuple(args, "OOO", &(self->target_send), &(self->prefix), &map_type)); Py_INCREF(self->target_send); Py_INCREF(self->prefix); M1_M1(builder_init(&self->builder, map_type)); return 0; } static void items_basecoro_dealloc(ItemsBasecoro *self) { Py_XDECREF(self->prefix); Py_XDECREF(self->target_send); builder_destroy(&self->builder); Py_TYPE(self)->tp_free((PyObject*)self); } PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value) { ItemsBasecoro *coro = (ItemsBasecoro *)self; if (builder_isactive(&coro->builder)) { coro->object_depth += (event == enames.start_map_ename || event == enames.start_array_ename); coro->object_depth -= (event == enames.end_map_ename || event == enames.end_array_ename); if (coro->object_depth > 0) { N_M1( builder_event(&coro->builder, event, value) ); } else { PyObject *retval = builder_value(&coro->builder); CORO_SEND(coro->target_send, retval); Py_DECREF(retval); N_M1(builder_reset(&coro->builder)); } } else { int cmp = PyObject_RichCompareBool(path, coro->prefix, Py_EQ); N_M1(cmp); if (cmp) { if (event == enames.start_map_ename || event == enames.start_array_ename) { coro->object_depth = 1; N_M1(builder_event(&coro->builder, event, value)); } else { CORO_SEND(coro->target_send, value); } } } Py_RETURN_NONE; } static PyObject* items_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *path = PyTuple_GetItem(tuple, 0); PyObject *event = PyTuple_GetItem(tuple, 1); PyObject *value = PyTuple_GetItem(tuple, 2); return items_basecoro_send_impl(self, path, event, value); } static PyMethodDef items_basecoro_methods[] = { {"send", items_basecoro_send, METH_O, "coroutine's send method"}, {NULL, NULL, 0, NULL} }; /* * items generator object type */ PyTypeObject ItemsBasecoro_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(ItemsBasecoro), .tp_name = "_yajl2.items_basecoro", .tp_doc = "Coroutine dispatching fully-built objects for the given prefix", .tp_init = (initproc)items_basecoro_init, .tp_dealloc = (destructor)items_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = items_basecoro_methods }; ijson-3.2.3/ijson/backends/yajl2_c/items_basecoro.h000066400000000000000000000023101445666302700222400ustar00rootroot00000000000000/* * items_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef ITEMS_BASECORO_H #define ITEMS_BASECORO_H #include "builder.h" /** * items_basecoro coroutine object structure */ typedef struct { PyObject_HEAD builder_t builder; PyObject *target_send; PyObject *prefix; int object_depth; } ItemsBasecoro; /** * items_basecoro coroutine object type */ extern PyTypeObject ItemsBasecoro_Type; /** * Utility function to check if an object is an items_basecoro coroutine or not */ #define ItemsBasecoro_Check(o) (Py_TYPE(o) == &ItemsBasecoro_Type) /** * The implementation of the items_basecoro.send() method accepting an unpacked * event * @param self An items_basecoro coroutine * @param path The path of this event * @param event The event name * @param value The value of this event * @return None, or NULL in case of an error */ PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value); #endif // ITEMS_BASECORO_Hijson-3.2.3/ijson/backends/yajl2_c/kvitems.c000066400000000000000000000033661445666302700207330ustar00rootroot00000000000000/* * kvitems generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "basic_parse_basecoro.h" #include "common.h" #include "kvitems.h" #include "kvitems_basecoro.h" #include "parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int kvitemsgen_init(KVItemsGen *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *kvitems_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&KVItemsBasecoro_Type, kvitems_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, reading_args, coro_pipeline)); Py_DECREF(kvitems_args); Py_DECREF(reading_args); return 0; } static void kvitemsgen_dealloc(KVItemsGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* kvitemsgen_iternext(PyObject *self) { KVItemsGen *gen = (KVItemsGen *)self; return reading_generator_next(&gen->reading_gen); } /* * kvitems generator object type */ PyTypeObject KVItemsGen_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(KVItemsGen), .tp_name = "_yajl2.kvitems", .tp_doc = "Generates key/value pairs", .tp_init = (initproc)kvitemsgen_init, .tp_dealloc = (destructor)kvitemsgen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = kvitemsgen_iternext }; ijson-3.2.3/ijson/backends/yajl2_c/kvitems.h000066400000000000000000000010751445666302700207330ustar00rootroot00000000000000/* * kvitems generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef KVITEMS_H #define KVITEMS_H #include "reading_generator.h" /** * kvitems generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } KVItemsGen; /** * kvitems generator object type */ extern PyTypeObject KVItemsGen_Type; #endif /* KVITEMS_H */ijson-3.2.3/ijson/backends/yajl2_c/kvitems_async.c000066400000000000000000000044101445666302700221170ustar00rootroot00000000000000/* * kvitems_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "parse_basecoro.h" #include "kvitems_basecoro.h" #include "common.h" #include "coro_utils.h" #if PY_VERSION_HEX >= 0x03050000 /** * kvitems_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } KVItemsAsync; /* * __init__, destructor and __anext__ */ static int kvitemsasync_init(KVItemsAsync *self, PyObject *args, PyObject *kwargs) { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *kvitems_args = PySequence_GetSlice(args, 2, 4); pipeline_node coro_pipeline[] = { {&KVItemsBasecoro_Type, kvitems_args, NULL}, {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); async_reading_generator_add_coro(self->reading_generator, coro_pipeline); Py_DECREF(kvitems_args); Py_DECREF(reading_args); return 0; } static void kvitemsasync_dealloc(KVItemsAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *kvitemsasync_anext(PyObject *self) { KVItemsAsync *gen = (KVItemsAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods kvitemsasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = kvitemsasync_anext }; PyTypeObject KVItemsAsync_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(KVItemsAsync), .tp_name = "_yajl2._kvitems_async", .tp_doc = "Asynchronous iterable yielding key/value pairs", .tp_init = (initproc)kvitemsasync_init, .tp_dealloc = (destructor)kvitemsasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &kvitemsasync_methods }; #endif // PY_VERSION_HEX >= 0x03050000ijson-3.2.3/ijson/backends/yajl2_c/kvitems_async.h000066400000000000000000000011111445666302700221170ustar00rootroot00000000000000/* * kvitems_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef KVITEMS_ASYNC_H #define KVITEMS_ASYNC_H #define PY_SSIZE_T_CLEAN #include #if PY_VERSION_HEX >= 0x03050000 /** * kvitems_async asynchronous iterable object type */ extern PyTypeObject KVItemsAsync_Type; #endif // PY_VERSION_HEX >= 0x03050000 #endif // KVITEMS_ASYNC_Hijson-3.2.3/ijson/backends/yajl2_c/kvitems_basecoro.c000066400000000000000000000067231445666302700226100ustar00rootroot00000000000000/* * kvitems_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "kvitems_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int kvitems_basecoro_init(KVItemsBasecoro *self, PyObject *args, PyObject *kwargs) { self->target_send = NULL; self->prefix = NULL; self->key = NULL; builder_create(&self->builder); PyObject *map_type; M1_Z(PyArg_ParseTuple(args, "OOO", &(self->target_send), &(self->prefix), &map_type)); Py_INCREF(self->target_send); Py_INCREF(self->prefix); M1_M1(builder_init(&self->builder, map_type)); return 0; } static void kvitems_basecoro_dealloc(KVItemsBasecoro *self) { Py_XDECREF(self->prefix); Py_XDECREF(self->key); Py_XDECREF(self->target_send); builder_destroy(&self->builder); Py_TYPE(self)->tp_free((PyObject*)self); } static int kvitems_basecoro_start_new_member(KVItemsBasecoro *coro, PyObject *key) { coro->object_depth = 0; Py_XDECREF(coro->key); coro->key = key; Py_INCREF(coro->key); M1_M1(builder_reset(&coro->builder)); coro->builder.active = 1; return 0; } PyObject* kvitems_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value) { KVItemsBasecoro *coro = (KVItemsBasecoro *)self; PyObject *retval = NULL; PyObject *retkey = NULL; if (builder_isactive(&coro->builder)) { coro->object_depth += (event == enames.start_map_ename); coro->object_depth -= (event == enames.end_map_ename); if ((event != enames.map_key_ename || coro->object_depth != 0) && (event != enames.end_map_ename || coro->object_depth != -1)) { N_M1(builder_event(&coro->builder, event, value)); } else { retval = builder_value(&coro->builder); retkey = coro->key; Py_INCREF(retkey); if (event == enames.map_key_ename) { N_M1(kvitems_basecoro_start_new_member(coro, value)); } else { Py_CLEAR(coro->key); coro->builder.active = 0; } } } else { int cmp = PyObject_RichCompareBool(path, coro->prefix, Py_EQ); N_M1(cmp); if (cmp == 1 && event == enames.map_key_ename) { N_M1(kvitems_basecoro_start_new_member(coro, value)); } } if (retval) { PyObject *tuple = PyTuple_Pack(2, retkey, retval); Py_XDECREF(retkey); Py_XDECREF(retval); CORO_SEND(coro->target_send, tuple); Py_DECREF(tuple); } Py_RETURN_NONE; } static PyObject* kvitems_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *path = PyTuple_GetItem(tuple, 0); PyObject *event = PyTuple_GetItem(tuple, 1); PyObject *value = PyTuple_GetItem(tuple, 2); return kvitems_basecoro_send_impl(self, path, event, value); } static PyMethodDef kvitems_basecoro_methods[] = { {"send", kvitems_basecoro_send, METH_O, "coroutine's send method"}, {NULL, NULL, 0, NULL} }; /* * kvitems generator object type */ PyTypeObject KVItemsBasecoro_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(KVItemsBasecoro), .tp_name = "_yajl2.kvitems_basecoro", .tp_doc = "Coroutine dispatching (key, value) tuples", .tp_init = (initproc)kvitems_basecoro_init, .tp_dealloc = (destructor)kvitems_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = kvitems_basecoro_methods }; ijson-3.2.3/ijson/backends/yajl2_c/kvitems_basecoro.h000066400000000000000000000024461445666302700226130ustar00rootroot00000000000000/* * kvitems_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef KVITEMS_BASECORO_H #define KVITEMS_BASECORO_H #define PY_SSIZE_T_CLEAN #include #include "builder.h" /** * kvitems_basecoro coroutine object structure */ typedef struct { PyObject_HEAD builder_t builder; PyObject *target_send; PyObject *prefix; PyObject *key; int object_depth; } KVItemsBasecoro; /** * kvitems_basecoro coroutine object type */ extern PyTypeObject KVItemsBasecoro_Type; /** * Utility function to check if an object is a kvitems_basecoro coroutine or not */ #define KVItemsBasecoro_Check(o) (Py_TYPE(o) == &KVItemsBasecoro_Type) /** * The implementation of the kvitems_basecoro.send() method accepting an unpacked * event * @param self A kvitems_basecoro coroutine * @param path The path of this event * @param event The event name * @param value The value of this event * @return None, or NULL in case of an error */ PyObject* kvitems_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value); #endif /* KVITEMS_BASECORO_H */ijson-3.2.3/ijson/backends/yajl2_c/module.c000066400000000000000000000067171445666302700205410ustar00rootroot00000000000000/* * _yajl2 backend for ijson * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2016 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "async_reading_generator.h" #include "basic_parse.h" #include "basic_parse_async.h" #include "basic_parse_basecoro.h" #include "parse.h" #include "parse_async.h" #include "parse_basecoro.h" #include "items.h" #include "items_async.h" #include "items_basecoro.h" #include "kvitems.h" #include "kvitems_async.h" #include "kvitems_basecoro.h" enames_t enames; PyObject *dot, *item, *dotitem; PyObject *JSONError; PyObject *IncompleteJSONError; PyObject *Decimal; static PyMethodDef yajl2_methods[] = { {NULL, NULL, 0, NULL} /* Sentinel */ }; PyObject* ijson_return_self(PyObject *self) { Py_INCREF(self); return self; } PyObject* ijson_return_none(PyObject *self) { Py_RETURN_NONE; } /* Module initialization */ /* Support for Python 2/3 */ #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, "_yajl2", "wrapper for yajl2 methods", -1, yajl2_methods}; #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) #define MOD_DEF(m, name, doc, methods) \ m = PyModule_Create(&moduledef); #define MOD_VAL(v) v #else #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) #define MOD_DEF(m, name, doc, methods) \ m = Py_InitModule3(name, methods, doc); #define MOD_VAL(v) #endif #define ADD_TYPE(name, type) \ { \ type.tp_new = PyType_GenericNew; \ X_LZ(PyType_Ready(&type), MOD_VAL(NULL)); \ Py_INCREF(&type); \ PyModule_AddObject(m, name, (PyObject *)&type); \ } MOD_INIT(_yajl2) { PyObject *m; MOD_DEF(m, "_yajl2", "wrapper for yajl2 methods", yajl2_methods); X_N(m, MOD_VAL(NULL)); ADD_TYPE("basic_parse_basecoro", BasicParseBasecoro_Type); ADD_TYPE("basic_parse", BasicParseGen_Type); ADD_TYPE("parse_basecoro", ParseBasecoro_Type); ADD_TYPE("parse", ParseGen_Type); ADD_TYPE("kvitems_basecoro", KVItemsBasecoro_Type); ADD_TYPE("kvitems", KVItemsGen_Type); ADD_TYPE("items_basecoro", ItemsBasecoro_Type); ADD_TYPE("items", ItemsGen_Type); #if PY_VERSION_HEX >= 0x03050000 ADD_TYPE("_async_reading_iterator", AsyncReadingGeneratorType); ADD_TYPE("basic_parse_async", BasicParseAsync_Type); ADD_TYPE("parse_async", ParseAsync_Type); ADD_TYPE("kvitems_async", KVItemsAsync_Type); ADD_TYPE("items_async", ItemsAsync_Type); #endif // PY_VERSION_HEX >= 0x03050000 dot = STRING_FROM_UTF8(".", 1); item = STRING_FROM_UTF8("item", 4); dotitem = STRING_FROM_UTF8(".item", 5); #define INIT_ENAME(x) enames.x##_ename = STRING_FROM_UTF8(#x, strlen(#x)) INIT_ENAME(null); INIT_ENAME(boolean); INIT_ENAME(integer); INIT_ENAME(double); INIT_ENAME(number); INIT_ENAME(string); INIT_ENAME(start_map); INIT_ENAME(map_key); INIT_ENAME(end_map); INIT_ENAME(start_array); INIT_ENAME(end_array); // Import globally-used names PyObject *ijson_common = PyImport_ImportModule("ijson.common"); PyObject *decimal_module = PyImport_ImportModule("decimal"); X_N(ijson_common, MOD_VAL(NULL)); X_N(decimal_module, MOD_VAL(NULL)); JSONError = PyObject_GetAttrString(ijson_common, "JSONError"); IncompleteJSONError = PyObject_GetAttrString(ijson_common, "IncompleteJSONError"); Decimal = PyObject_GetAttrString(decimal_module, "Decimal"); X_N(JSONError, MOD_VAL(NULL)); X_N(IncompleteJSONError, MOD_VAL(NULL)); X_N(Decimal, MOD_VAL(NULL)); return MOD_VAL(m); }ijson-3.2.3/ijson/backends/yajl2_c/parse.c000066400000000000000000000026621445666302700203610ustar00rootroot00000000000000/* * parse generator implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "basic_parse_basecoro.h" #include "common.h" #include "parse.h" #include "parse_basecoro.h" /* * __init__, destructor, __iter__ and __next__ */ static int parsegen_init(ParseGen *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, args, coro_pipeline)); return 0; } static void parsegen_dealloc(ParseGen *self) { reading_generator_dealloc(&self->reading_gen); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject* parsegen_iternext(PyObject *self) { ParseGen *gen = (ParseGen *)self; return reading_generator_next(&gen->reading_gen); } PyTypeObject ParseGen_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(ParseGen), .tp_name = "_yajl2.parse", .tp_doc = "Generates (path,evt,value)", .tp_init = (initproc)parsegen_init, .tp_dealloc = (destructor)parsegen_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = parsegen_iternext }; ijson-3.2.3/ijson/backends/yajl2_c/parse.h000066400000000000000000000010601445666302700203550ustar00rootroot00000000000000/* * parse generator for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef PARSE_H #define PARSE_H #include "reading_generator.h" /** * parse generator object structure */ typedef struct { PyObject_HEAD reading_generator_t reading_gen; } ParseGen; /** * parse generator object type */ extern PyTypeObject ParseGen_Type; #endif // PARSE_Hijson-3.2.3/ijson/backends/yajl2_c/parse_async.c000066400000000000000000000037551445666302700215620ustar00rootroot00000000000000/* * parse_async asynchronous iterable implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "async_reading_generator.h" #include "basic_parse_basecoro.h" #include "parse_basecoro.h" #include "common.h" #include "coro_utils.h" #if PY_VERSION_HEX >= 0x03050000 /** * parse_async asynchronous iterable object structure */ typedef struct { PyObject_HEAD async_reading_generator *reading_generator; } ParseAsync; /* * __init__, destructor and __anext__ */ static int parseasync_init(ParseAsync *self, PyObject *args, PyObject *kwargs) { pipeline_node coro_pipeline[] = { {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, args)); async_reading_generator_add_coro(self->reading_generator, coro_pipeline); return 0; } static void parseasync_dealloc(ParseAsync *self) { Py_XDECREF(self->reading_generator); Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject *parseasync_anext(PyObject *self) { ParseAsync *gen = (ParseAsync *)self; Py_INCREF(gen->reading_generator); return (PyObject *)gen->reading_generator; } static PyAsyncMethods parseasync_methods = { .am_await = ijson_return_self, .am_aiter = ijson_return_self, .am_anext = parseasync_anext }; PyTypeObject ParseAsync_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(ParseAsync), .tp_name = "_yajl2._parse_async", .tp_doc = "Asynchronous iterable yielding (path,evt,value) tuples", .tp_init = (initproc)parseasync_init, .tp_dealloc = (destructor)parseasync_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_as_async = &parseasync_methods }; #endif // PY_VERSION_HEX >= 0x03050000ijson-3.2.3/ijson/backends/yajl2_c/parse_async.h000066400000000000000000000010751445666302700215600ustar00rootroot00000000000000/* * parse_async asynchronous iterable for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef PARSE_ASYNC_H #define PARSE_ASYNC_H #define PY_SSIZE_T_CLEAN #include #if PY_VERSION_HEX >= 0x03050000 /** * parse_async asynchronous iterable object type */ extern PyTypeObject ParseAsync_Type; #endif // PY_VERSION_HEX >= 0x03050000 #endif // PARSE_ASYNC_Hijson-3.2.3/ijson/backends/yajl2_c/parse_basecoro.c000066400000000000000000000103051445666302700222270ustar00rootroot00000000000000/* * parse_basecoro coroutine implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include "common.h" #include "items_basecoro.h" #include "kvitems_basecoro.h" #include "parse_basecoro.h" #if PY_MAJOR_VERSION >= 3 #define ijson_unicode_length PyUnicode_GET_LENGTH #else #define ijson_unicode_length PyUnicode_GET_SIZE #endif /* * __init__, destructor, __iter__ and __next__ */ static int parse_basecoro_init(ParseBasecoro *self, PyObject *args, PyObject *kwargs) { M1_Z(PyArg_ParseTuple(args, "O", &self->target_send)); Py_INCREF(self->target_send); M1_N(self->path = PyList_New(0)); PyObject *empty; M1_N(empty = STRING_FROM_UTF8("", 0)); int res = PyList_Append(self->path, empty); Py_DECREF(empty); M1_M1(res); return 0; } static void parse_basecoro_dealloc(ParseBasecoro *self) { Py_XDECREF(self->path); Py_XDECREF(self->target_send); Py_TYPE(self)->tp_free((PyObject*)self); } #define CONCAT(tgt, first, second) \ do { \ tgt = PyUnicode_Concat(first, second); \ Py_DECREF(first); \ N_N(tgt); \ } while(0); PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *value) { ParseBasecoro *gen = (ParseBasecoro *)self; Py_ssize_t npaths = PyList_Size(gen->path); // Calculate current prefix PyObject *prefix; if (event == enames.end_array_ename || event == enames.end_map_ename) { // pop N_M1(PyList_SetSlice(gen->path, npaths - 1, npaths, NULL)); npaths--; prefix = PySequence_GetItem(gen->path, npaths - 1); } else if (event == enames.map_key_ename) { // last_path = path_stack[-2] // to_append = '.' + value if len(path_stack) > 1 else value // new_path = path_stack[-2] + to_append PyObject *last_path; N_N(last_path = PySequence_GetItem(gen->path, npaths - 2)); if (npaths > 2) { PyObject *last_path_dot; CONCAT(last_path_dot, last_path, dot); last_path = last_path_dot; } PyObject *new_path; CONCAT(new_path, last_path, value); PyList_SetItem(gen->path, npaths - 1, new_path); prefix = PySequence_GetItem(gen->path, npaths - 2); } else { prefix = PySequence_GetItem(gen->path, npaths - 1); } N_N(prefix); // If entering a map/array, append name to path if (event == enames.start_array_ename) { // to_append = '.item' if path_stack[-1] else 'item' // path_stack.append(path_stack[-1] + to_append) PyObject *last_path; N_N(last_path = PySequence_GetItem(gen->path, npaths - 1)); if (ijson_unicode_length(last_path) > 0) { PyObject *new_path; CONCAT(new_path, last_path, dotitem); N_M1(PyList_Append(gen->path, new_path)); Py_DECREF(new_path); } else { N_M1(PyList_Append(gen->path, item)); Py_DECREF(last_path); } } else if (event == enames.start_map_ename) { Py_INCREF(Py_None); N_M1(PyList_Append(gen->path, Py_None)); } if (KVItemsBasecoro_Check(gen->target_send)) { kvitems_basecoro_send_impl(gen->target_send, prefix, event, value); } else if (ItemsBasecoro_Check(gen->target_send)) { items_basecoro_send_impl(gen->target_send, prefix, event, value); } else { PyObject *res = PyTuple_Pack(3, prefix, event, value); CORO_SEND(gen->target_send, res); Py_DECREF(res); } Py_DECREF(prefix); Py_RETURN_NONE; } static PyObject* parse_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *event = PyTuple_GetItem(tuple, 0); PyObject *value = PyTuple_GetItem(tuple, 1); return parse_basecoro_send_impl(self, event, value); } static PyMethodDef parse_basecoro_methods[] = { {"send", parse_basecoro_send, METH_O, "coroutine's send method"}, {NULL, NULL, 0, NULL} }; PyTypeObject ParseBasecoro_Type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) #endif .tp_basicsize = sizeof(ParseBasecoro), .tp_name = "_yajl2.parse_basecoro", .tp_doc = "Coroutine dispatching (path,evt,value) tuples", .tp_init = (initproc)parse_basecoro_init, .tp_dealloc = (destructor)parse_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, .tp_iter = ijson_return_self, .tp_iternext = ijson_return_none, .tp_methods = parse_basecoro_methods }; ijson-3.2.3/ijson/backends/yajl2_c/parse_basecoro.h000066400000000000000000000022371445666302700222410ustar00rootroot00000000000000/* * parse_basecoro coroutine for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef PARSE_BASECORO_H #define PARSE_BASECORO_H #define PY_SSIZE_T_CLEAN #include /** * parse_basecoro coroutine object structure */ typedef struct { PyObject_HEAD PyObject *target_send; PyObject *path; } ParseBasecoro; /** * parse_basecoro coroutine object type */ extern PyTypeObject ParseBasecoro_Type; /** * Utility function to check if an object is a parse_basecoro coroutine or not */ #define ParseBasecoro_Check(o) (Py_TYPE(o) == &ParseBasecoro_Type) /** * The implementation of the parse_basecoro.send() method accepting an unpacked * event * @param self A parse_basecoro coroutine * @param path The path of this event * @param event The event name * @param value The value of this event * @return None, or NULL in case of an error */ PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *value); #endif // PARSE_BASECORO_Hijson-3.2.3/ijson/backends/yajl2_c/reading_generator.c000066400000000000000000000063671445666302700227340ustar00rootroot00000000000000/* * reading_generator_t object and methods implementation for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2020 * Copyright by UWA (in the framework of the ICRAR) */ #include #include "basic_parse_basecoro.h" #include "common.h" #include "reading_generator.h" int reading_generator_init(reading_generator_t *self, PyObject *args, pipeline_node *coro_pipeline) { PyObject *file; Py_ssize_t buf_size = 64 * 1024; M1_Z(PyArg_ParseTuple(args, "On", &file, &buf_size)); // Handle both "read" and "readinto" functions. // The latter allocates a bytearray, which is how we distinguish between // the two cases later if (PyObject_HasAttrString(file, "readinto")) { M1_N(self->read_func = PyObject_GetAttrString(file, "readinto")); PyObject *pbuf_size = Py_BuildValue("n", buf_size); self->buffer = PyObject_CallFunctionObjArgs((PyObject *)&PyByteArray_Type, pbuf_size, NULL); M1_N(self->buffer); Py_DECREF(pbuf_size); } else { M1_N(self->read_func = PyObject_GetAttrString(file, "read")); self->buf_size = PyLong_FromSsize_t(buf_size); self->buffer = NULL; } M1_N(self->events = PyList_New(0)); self->pos = 0; self->finished = 0; M1_N(self->coro = chain(self->events, coro_pipeline)); assert(("reading_generator works only with basic_parse_basecoro", BasicParseBasecoro_Check(self->coro))); return 0; } void reading_generator_dealloc(reading_generator_t *self) { Py_XDECREF(self->read_func); Py_XDECREF(self->events); Py_XDECREF(self->buffer); Py_XDECREF(self->buf_size); Py_XDECREF(self->coro); } PyObject *reading_generator_next(reading_generator_t *self) { PyObject *events = self->events; Py_ssize_t nevents = PyList_Size(events); BasicParseBasecoro *basic_parse_basecoro = (BasicParseBasecoro *)self->coro; while (nevents == 0) { /* Read data and pass it down to the co-routine */ Py_buffer view; Py_ssize_t length; if (self->buffer == NULL) { // read_func is "read" PyObject *pbuffer = PyObject_CallFunctionObjArgs(self->read_func, self->buf_size, NULL); N_N(pbuffer); N_M1(PyObject_GetBuffer(pbuffer, &view, PyBUF_SIMPLE)); length = view.len; PyObject *send_res = ijson_yajl_parse(basic_parse_basecoro->h, view.buf, view.len); Py_DECREF(pbuffer); PyBuffer_Release(&view); N_N(send_res); } else { // read_func is "readinto" PyObject *plength = PyObject_CallFunctionObjArgs(self->read_func, self->buffer, NULL); N_N(plength); length = PyLong_AsLong(plength); N_M1(length); Py_DECREF(plength); N_M1(PyObject_GetBuffer(self->buffer, &view, PyBUF_SIMPLE)); PyObject *send_res = ijson_yajl_parse(basic_parse_basecoro->h, view.buf, length); PyBuffer_Release(&view); N_N(send_res); } nevents = PyList_Size(events); if (length == 0) { break; } } // events are now probably available if (nevents > 0) { PyObject *val = PyList_GetItem(events, self->pos++); Py_INCREF(val); /* empty the list if fully iterated over */ if (self->pos == nevents) { self->pos = 0; N_M1(PySequence_DelSlice(events, 0, nevents)); } return val; } // no events, let's end the show PyErr_SetNone(PyExc_StopIteration); return NULL; }ijson-3.2.3/ijson/backends/yajl2_c/reading_generator.h000066400000000000000000000035641445666302700227350ustar00rootroot00000000000000/* * reading_generator_t type and methods for ijson's C backend * * Contributed by Rodrigo Tobar * * ICRAR - International Centre for Radio Astronomy Research * (c) UWA - The University of Western Australia, 2019 * Copyright by UWA (in the framework of the ICRAR) */ #ifndef READING_GENERATOR_H #define READING_GENERATOR_H #define PY_SSIZE_T_CLEAN #include #include "coro_utils.h" /** * reading_generator_t type definition */ typedef struct _reading_generator { PyObject *coro; PyObject *read_func; PyObject *buf_size; PyObject *buffer; PyObject *events; Py_ssize_t pos; int finished; } reading_generator_t; /** * Initialises a reading_generator_t object from the given arguments, which * should contain a file-like object and a buffer size (optional). * * @param self A reading_generator_t object * @param args A tuple containing a file-like object and a buffer size (optional) * @param coro_pipeline A description of the coroutine pipeline to create internally * in this reading generator, where data will be pushed to, and which will send * events to the events list * @return 0 if successful, -1 in case of an error */ int reading_generator_init(reading_generator_t *self, PyObject *args, pipeline_node *coro_pipeline); /** * Deallocates all resources associated to the given reading_generator_t object. * @param self A reading_generator_t object */ void reading_generator_dealloc(reading_generator_t *self); /** * Advances the reading_generator_t object by reading data off the underlying * file-like object, feeding it into its coro, with results ending * up in self->events, from which they are returned. * @param self A reading_generator_t object * @return The next event generated from this iterative process */ PyObject *reading_generator_next(reading_generator_t *self); #endif // READING_GENERATOR_Hijson-3.2.3/ijson/backends/yajl2_cffi.py000066400000000000000000000143551445666302700201440ustar00rootroot00000000000000''' CFFI-Wrapper for YAJL C library version 2.x. ''' from cffi import FFI import functools from ijson import common, backends, utils from ijson.compat import b2s ffi = FFI() ffi.cdef(""" typedef void * (*yajl_malloc_func)(void *ctx, size_t sz); typedef void (*yajl_free_func)(void *ctx, void * ptr); typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz); typedef struct { yajl_malloc_func malloc; yajl_realloc_func realloc; yajl_free_func free; void * ctx; } yajl_alloc_funcs; typedef struct yajl_handle_t * yajl_handle; typedef enum { yajl_status_ok, yajl_status_client_canceled, yajl_status_error } yajl_status; typedef enum { yajl_allow_comments = 0x01, yajl_dont_validate_strings = 0x02, yajl_allow_trailing_garbage = 0x04, yajl_allow_multiple_values = 0x08, yajl_allow_partial_values = 0x10 } yajl_option; typedef struct { int (* yajl_null)(void * ctx); int (* yajl_boolean)(void * ctx, int boolVal); int (* yajl_integer)(void * ctx, long long integerVal); int (* yajl_double)(void * ctx, double doubleVal); int (* yajl_number)(void * ctx, const char * numberVal, size_t numberLen); int (* yajl_string)(void * ctx, const unsigned char * stringVal, size_t stringLen); int (* yajl_start_map)(void * ctx); int (* yajl_map_key)(void * ctx, const unsigned char * key, size_t stringLen); int (* yajl_end_map)(void * ctx); int (* yajl_start_array)(void * ctx); int (* yajl_end_array)(void * ctx); } yajl_callbacks; int yajl_version(void); yajl_handle yajl_alloc(const yajl_callbacks *callbacks, yajl_alloc_funcs *afs, void *ctx); int yajl_config(yajl_handle h, yajl_option opt, ...); yajl_status yajl_parse(yajl_handle hand, const unsigned char *jsonText, size_t jsonTextLength); yajl_status yajl_complete_parse(yajl_handle hand); unsigned char* yajl_get_error(yajl_handle hand, int verbose, const unsigned char *jsonText, size_t jsonTextLength); void yajl_free_error(yajl_handle hand, unsigned char * str); void yajl_free(yajl_handle handle); """) yajl = backends.find_yajl_cffi(ffi, 2) YAJL_OK = 0 YAJL_CANCELLED = 1 YAJL_INSUFFICIENT_DATA = 2 YAJL_ERROR = 3 # constants defined in yajl_parse.h YAJL_ALLOW_COMMENTS = 1 YAJL_MULTIPLE_VALUES = 8 def append_event_to_ctx(event): def wrapper(func): @functools.wraps(func) def wrapped(ctx, *args, **kwargs): value = func(*args, **kwargs) send = ffi.from_handle(ctx) send((event, value)) return 1 return wrapped return wrapper @ffi.callback('int(void *ctx)') @append_event_to_ctx('null') def null(): return None @ffi.callback('int(void *ctx, int val)') @append_event_to_ctx('boolean') def boolean(val): return bool(val) @ffi.callback('int(void *ctx, long long integerVal)') @append_event_to_ctx('number') def integer(val): return int(val) @ffi.callback('int(void * ctx, double doubleVal)') @append_event_to_ctx('number') def double(val): return val @ffi.callback('int(void *ctx, const char *numberVal, size_t numberLen)') @append_event_to_ctx('number') def number(val, length): return common.integer_or_decimal(b2s(ffi.string(val, maxlen=length))) @ffi.callback('int(void *ctx, const unsigned char *stringVal, size_t stringLen)') @append_event_to_ctx('string') def string(val, length): return ffi.string(val, maxlen=length).decode('utf-8') @ffi.callback('int(void *ctx)') @append_event_to_ctx('start_map') def start_map(): return None @ffi.callback('int(void *ctx, const unsigned char *key, size_t stringLen)') @append_event_to_ctx('map_key') def map_key(key, length): return ffi.string(key, maxlen=length).decode('utf-8') @ffi.callback('int(void *ctx)') @append_event_to_ctx('end_map') def end_map(): return None @ffi.callback('int(void *ctx)') @append_event_to_ctx('start_array') def start_array(): return None @ffi.callback('int(void *ctx)') @append_event_to_ctx('end_array') def end_array(): return None _decimal_callback_data = ( null, boolean, ffi.NULL, ffi.NULL, number, string, start_map, map_key, end_map, start_array, end_array ) _float_callback_data = ( null, boolean, integer, double, ffi.NULL, string, start_map, map_key, end_map, start_array, end_array ) def yajl_init(scope, send, allow_comments=False, multiple_values=False, use_float=False): scope.ctx = ffi.new_handle(send) if use_float: scope.callbacks = ffi.new('yajl_callbacks*', _float_callback_data) else: scope.callbacks = ffi.new('yajl_callbacks*', _decimal_callback_data) handle = yajl.yajl_alloc(scope.callbacks, ffi.NULL, scope.ctx) if allow_comments: yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, ffi.cast('int', 1)) if multiple_values: yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, ffi.cast('int', 1)) return handle def yajl_parse(handle, buffer): if buffer: result = yajl.yajl_parse(handle, buffer, len(buffer)) else: result = yajl.yajl_complete_parse(handle) if result != YAJL_OK: perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer)) error = ffi.string(perror) try: error = error.decode('utf8') except UnicodeDecodeError: pass yajl.yajl_free_error(handle, perror) exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError raise exception(error) class Container(object): pass @utils.coroutine def basic_parse_basecoro(target, **config): ''' Coroutine dispatching unprefixed events. Parameters: - allow_comments: tells parser to allow comments in JSON input - multiple_values: allows the parser to parse multiple JSON objects ''' # the scope objects makes sure the C objects allocated in _yajl.init # are kept alive until this function is done scope = Container() handle = yajl_init(scope, target.send, **config) try: while True: try: buffer = (yield) except GeneratorExit: buffer = b'' yajl_parse(handle, buffer) if not buffer: break finally: yajl.yajl_free(handle) common.enrich_backend(globals()) ijson-3.2.3/ijson/common.py000066400000000000000000000354111445666302700156460ustar00rootroot00000000000000''' Backend independent higher level interfaces, common exceptions. ''' import decimal import inspect import warnings from ijson import compat, utils class JSONError(Exception): ''' Base exception for all parsing errors. ''' pass class IncompleteJSONError(JSONError): ''' Raised when the parser can't read expected data from a stream. ''' pass @utils.coroutine def parse_basecoro(target): ''' A coroutine dispatching parsing events with the information about their location with the JSON object tree. Events are tuples ``(prefix, type, value)``. Available types and values are: ('null', None) ('boolean', ) ('number', ) ('string', ) ('map_key', ) ('start_map', None) ('end_map', None) ('start_array', None) ('end_array', None) Prefixes represent the path to the nested elements from the root of the JSON document. For example, given this document:: { "array": [1, 2], "map": { "key": "value" } } the parser would yield events: ('', 'start_map', None) ('', 'map_key', 'array') ('array', 'start_array', None) ('array.item', 'number', 1) ('array.item', 'number', 2) ('array', 'end_array', None) ('', 'map_key', 'map') ('map', 'start_map', None) ('map', 'map_key', 'key') ('map.key', 'string', u'value') ('map', 'end_map', None) ('', 'end_map', None) ''' path = [] while True: event, value = yield if event == 'map_key': prefix = '.'.join(path[:-1]) path[-1] = value elif event == 'start_map': prefix = '.'.join(path) path.append(None) elif event == 'end_map': path.pop() prefix = '.'.join(path) elif event == 'start_array': prefix = '.'.join(path) path.append('item') elif event == 'end_array': path.pop() prefix = '.'.join(path) else: # any scalar value prefix = '.'.join(path) target.send((prefix, event, value)) class ObjectBuilder(object): ''' Incrementally builds an object from JSON parser events. Events are passed into the `event` function that accepts two parameters: event type and value. The object being built is available at any time from the `value` attribute. Example:: >>> from ijson import basic_parse >>> from ijson.common import ObjectBuilder >>> from ijson.compat import BytesIO >>> builder = ObjectBuilder() >>> f = BytesIO(b'{"key": "value"}') >>> for event, value in basic_parse(f): ... builder.event(event, value) >>> builder.value == {'key': 'value'} True ''' def __init__(self, map_type=None): def initial_set(value): self.value = value self.containers = [initial_set] self.map_type = map_type or dict def event(self, event, value): if event == 'map_key': self.key = value elif event == 'start_map': mappable = self.map_type() self.containers[-1](mappable) def setter(value): mappable[self.key] = value self.containers.append(setter) elif event == 'start_array': array = [] self.containers[-1](array) self.containers.append(array.append) elif event == 'end_array' or event == 'end_map': self.containers.pop() else: self.containers[-1](value) @utils.coroutine def items_basecoro(target, prefix, map_type=None): ''' An couroutine dispatching native Python objects constructed from the events under a given prefix. ''' while True: current, event, value = (yield) if current == prefix: if event in ('start_map', 'start_array'): object_depth = 1 builder = ObjectBuilder(map_type=map_type) while object_depth: builder.event(event, value) current, event, value = (yield) if event in ('start_map', 'start_array'): object_depth += 1 elif event in ('end_map', 'end_array'): object_depth -= 1 del builder.containers[:] target.send(builder.value) else: target.send(value) @utils.coroutine def kvitems_basecoro(target, prefix, map_type=None): ''' An coroutine dispatching (key, value) pairs constructed from the events under a given prefix. The prefix should point to JSON objects ''' builder = None while True: path, event, value = (yield) while path == prefix and event == 'map_key': object_depth = 0 key = value builder = ObjectBuilder(map_type=map_type) path, event, value = (yield) if event == 'start_map': object_depth += 1 while ( (event != 'map_key' or object_depth != 0) and (event != 'end_map' or object_depth != -1)): builder.event(event, value) path, event, value = (yield) if event == 'start_map': object_depth += 1 elif event == 'end_map': object_depth -= 1 del builder.containers[:] target.send((key, builder.value)) def integer_or_decimal(str_value): ''' Converts string with a numeric value into an int or a Decimal. Used in different backends for consistent number representation. ''' if not ('.' in str_value or 'e' in str_value or 'E' in str_value): return int(str_value) return decimal.Decimal(str_value) def integer_or_float(str_value): ''' Converts string with a numeric value into an int or a float. Used in different backends for consistent number representation. ''' if not ('.' in str_value or 'e' in str_value or 'E' in str_value): return int(str_value) return float(str_value) def number(str_value): warnings.warn("number() function will be removed in a later release", DeprecationWarning) return integer_or_decimal(str_value) def file_source(f, buf_size=64*1024): '''A generator that yields data from a file-like object''' f = compat.bytes_reader(f) while True: data = f.read(buf_size) yield data if not data: break def _basic_parse_pipeline(backend, config): return ( (backend['basic_parse_basecoro'], [], config), ) def _parse_pipeline(backend, config): return ( (backend['parse_basecoro'], [], {}), (backend['basic_parse_basecoro'], [], config) ) def _items_pipeline(backend, prefix, map_type, config): return ( (backend['items_basecoro'], (prefix,), {'map_type': map_type}), (backend['parse_basecoro'], [], {}), (backend['basic_parse_basecoro'], [], config) ) def _kvitems_pipeline(backend, prefix, map_type, config): return ( (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}), (backend['parse_basecoro'], [], {}), (backend['basic_parse_basecoro'], [], config) ) def _make_basic_parse_coro(backend): def basic_parse_coro(target, **config): return utils.chain( target, *_basic_parse_pipeline(backend, config) ) return basic_parse_coro def _make_parse_coro(backend): def parse_coro(target, **config): return utils.chain( target, *_parse_pipeline(backend, config) ) return parse_coro def _make_items_coro(backend): def items_coro(target, prefix, map_type=None, **config): return utils.chain( target, *_items_pipeline(backend, prefix, map_type, config) ) return items_coro def _make_kvitems_coro(backend): def kvitems_coro(target, prefix, map_type=None, **config): return utils.chain( target, *_kvitems_pipeline(backend, prefix, map_type, config) ) return kvitems_coro def is_awaitablefunction(func): """True if `func` is an awaitable function""" return ( inspect.iscoroutinefunction(func) or ( inspect.isgeneratorfunction(func) and (func.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE) ) ) def is_async_file(f): """True if `f` has an asynchronous `read` method""" return ( compat.IS_PY35 and hasattr(f, 'read') and is_awaitablefunction(f.read) ) def is_file(x): """True if x has a `read` method""" return hasattr(x, 'read') def is_iterable(x): """True if x can be iterated over""" return hasattr(x, '__iter__') def _get_source(source): if isinstance(source, compat.bytetype): return compat.BytesIO(source) elif isinstance(source, compat.texttype): return compat.StringIO(source) return source def _make_basic_parse_gen(backend): def basic_parse_gen(file_obj, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_basic_parse_pipeline(backend, config) ) return basic_parse_gen def _make_parse_gen(backend): def parse_gen(file_obj, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_parse_pipeline(backend, config) ) return parse_gen def _make_items_gen(backend): def items_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_items_pipeline(backend, prefix, map_type, config) ) return items_gen def _make_kvitems_gen(backend): def kvitems_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): return utils.coros2gen( file_source(file_obj, buf_size=buf_size), *_kvitems_pipeline(backend, prefix, map_type, config) ) return kvitems_gen def _make_basic_parse(backend): def basic_parse(source, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['basic_parse_async']( source, buf_size=buf_size, **config ) elif is_file(source): return backend['basic_parse_gen']( source, buf_size=buf_size, **config ) raise ValueError("Unknown source type: %r" % type(source)) return basic_parse def _make_parse(backend): def parse(source, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['parse_async']( source, buf_size=buf_size, **config ) elif is_file(source): return backend['parse_gen']( source, buf_size=buf_size, **config ) elif is_iterable(source): return utils.coros2gen(source, (parse_basecoro, (), {}) ) raise ValueError("Unknown source type: %r" % type(source)) return parse def _make_items(backend): def items(source, prefix, map_type=None, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['items_async']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_file(source): return backend['items_gen']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_iterable(source): return utils.coros2gen(source, (backend['items_basecoro'], (prefix,), {'map_type': map_type}) ) raise ValueError("Unknown source type: %r" % type(source)) return items def _make_kvitems(backend): def kvitems(source, prefix, map_type=None, buf_size=64*1024, **config): source = _get_source(source) if is_async_file(source): return backend['kvitems_async']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_file(source): return backend['kvitems_gen']( source, prefix, map_type=map_type, buf_size=buf_size, **config ) elif is_iterable(source): return utils.coros2gen(source, (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}) ) raise ValueError("Unknown source type: %r" % type(source)) return kvitems _common_functions_warn = ''' Don't use the ijson.common.* functions; instead go directly with the ijson.* ones. See the documentation for more information. ''' def parse(events): """Like ijson.parse, but takes events generated via ijson.basic_parse instead of a file""" warnings.warn(_common_functions_warn, DeprecationWarning) return utils.coros2gen(events, (parse_basecoro, (), {}) ) def kvitems(events, prefix, map_type=None): """Like ijson.kvitems, but takes events generated via ijson.parse instead of a file""" warnings.warn(_common_functions_warn, DeprecationWarning) return utils.coros2gen(events, (kvitems_basecoro, (prefix,), {'map_type': map_type}) ) def items(events, prefix, map_type=None): """Like ijson.items, but takes events generated via ijson.parse instead of a file""" warnings.warn(_common_functions_warn, DeprecationWarning) return utils.coros2gen(events, (items_basecoro, (prefix,), {'map_type': map_type}) ) def enrich_backend(backend): ''' Provides a backend with any missing coroutines/generators/async-iterables it might be missing by using the generic ones written in python. ''' backend['backend'] = backend['__name__'].split('.')[-1] for name in ('basic_parse', 'parse', 'items', 'kvitems'): basecoro_name = name + '_basecoro' if basecoro_name not in backend: backend[basecoro_name] = globals()[basecoro_name] coro_name = name + '_coro' if coro_name not in backend: factory = globals()['_make_' + coro_name] backend[coro_name] = factory(backend) gen_name = name + '_gen' if gen_name not in backend: factory = globals()['_make_' + gen_name] backend[gen_name] = factory(backend) if compat.IS_PY35: from . import utils35 async_name = name + '_async' if async_name not in backend: factory = getattr(utils35, '_make_' + async_name) backend[async_name] = factory(backend) factory = globals()['_make_' + name] backend[name] = factory(backend)ijson-3.2.3/ijson/compat.py000066400000000000000000000030331445666302700156340ustar00rootroot00000000000000''' Python2/Python3 compatibility utilities. ''' import sys import warnings IS_PY2 = sys.version_info[0] < 3 IS_PY35 = sys.version_info[0:2] >= (3, 5) if IS_PY2: b2s = lambda s: s bytetype = str texttype = unicode from StringIO import StringIO BytesIO = StringIO else: b2s = lambda b: b.decode('utf-8') bytetype = bytes texttype = str from io import BytesIO, StringIO class utf8reader(object): """Takes a utf8-encoded string reader and reads bytes out of it""" def __init__(self, str_reader): self.str_reader = str_reader def read(self, n): return self.str_reader.read(n).encode('utf-8') _str_vs_bytes_warning = ''' ijson works by reading bytes, but a string reader has been given instead. This probably, but not necessarily, means a file-like object has been opened in text mode ('t') rather than binary mode ('b'). An automatic conversion is being performed on the fly to continue, but on the other hand this creates unnecessary encoding/decoding operations that decrease the efficiency of the system. In the future this automatic conversion will be removed, and users will receive errors instead of this warning. To avoid this problem make sure file-like objects are opened in binary mode instead of text mode. ''' def _warn_and_return(o): warnings.warn(_str_vs_bytes_warning, DeprecationWarning) return o def bytes_reader(f): """Returns a file-like object that reads bytes""" if type(f.read(0)) == bytetype: return f return _warn_and_return(utf8reader(f))ijson-3.2.3/ijson/dump.py000066400000000000000000000033371445666302700153250ustar00rootroot00000000000000'''Dumping command-line utility''' import argparse import sys import ijson from . import compat HEADERS = { 'basic_parse': 'name, value', 'parse': 'path, name, value', 'kvitems': 'key, value', 'items': 'value', } def to_string(o): if isinstance(o, compat.texttype) and compat.IS_PY2: o = o.encode('utf8') if isinstance(o, compat.bytetype): return compat.b2s(o) return str(o) def dump(): parser = argparse.ArgumentParser(description='Dump ijson events') parser.add_argument('-m', '--method', choices=['basic_parse', 'parse', 'kvitems', 'items'], help='The method to use for dumping', default='basic_parse') parser.add_argument('-p', '--prefix', help='Prefix (used with -M items|kvitems)', default='') parser.add_argument('-M', '--multiple-values', help='Allow multiple values', action='store_true') args = parser.parse_args() method = getattr(ijson, args.method) method_args = () method_kwargs = {} if args.method in ('items', 'kvitems'): method_args = args.prefix, if args.multiple_values: method_kwargs['multiple_values'] = True header = '#: ' + HEADERS[args.method] print(header) print('-' * len(header)) # Use the raw bytes stream in stdin if possible stdin = sys.stdin if hasattr(stdin, 'buffer'): stdin = stdin.buffer enumerated_results = enumerate(method(stdin, *method_args, **method_kwargs)) if args.method == 'items': for i, result in enumerated_results: print('%i: %s' % (i, result)) else: for i, result in enumerated_results: print('%i: %s' % (i, ', '.join(to_string(bit) for bit in result))) if __name__ == '__main__': dump()ijson-3.2.3/ijson/utils.py000066400000000000000000000040131445666302700155100ustar00rootroot00000000000000# -*- coding:utf-8 -*- from functools import wraps def coroutine(func): ''' Wraps a generator which is intended to be used as a pure coroutine by .send()ing it values. The only thing that the wrapper does is calling .next() for the first time which is required by Python generator protocol. ''' @wraps(func) def wrapper(*args, **kwargs): g = func(*args, **kwargs) next(g) return g return wrapper def chain(sink, *coro_pipeline): ''' Chains together a sink and a number of coroutines to form a coroutine pipeline. The pipeline works by calling send() on the coroutine created with the information in `coro_pipeline[-1]`, which sends its results to the coroutine created from `coro_pipeline[-2]`, and so on, until the final result is sent to `sink`. ''' f = sink for coro_func, coro_args, coro_kwargs in coro_pipeline: f = coro_func(f, *coro_args, **coro_kwargs) return f class sendable_list(list): ''' A list that mimics a coroutine receiving values. Coroutine are sent values via their send() method. This class defines such a method so that values sent into it are appended into the list, which can be inspected later. As such, this type can be used as an "accumulating sink" in a pipeline consisting on many coroutines. ''' send = list.append def coros2gen(source, *coro_pipeline): ''' A utility function that returns a generator yielding values dispatched by a coroutine pipeline after *it* has received values coming from `source`. ''' events = sendable_list() f = chain(events, *coro_pipeline) try: for value in source: try: f.send(value) except StopIteration: for event in events: yield event return for event in events: yield event del events[:] except GeneratorExit: try: f.close() except: passijson-3.2.3/ijson/utils35.py000066400000000000000000000055111445666302700156640ustar00rootroot00000000000000''' Python3.5+ specific utilities ''' import collections from ijson import utils, common, compat class utf8reader_async(compat.utf8reader): """ Takes a utf8-encoded string asynchronous reader and asynchronously reads bytes out of it """ async def read(self, n): data = await self.str_reader.read(n) return data.encode('utf-8') async def _get_read(f): """Returns an awaitable read function that reads the requested type""" if type(await f.read(0)) == compat.bytetype: return f.read return compat._warn_and_return(utf8reader_async(f).read) class sendable_deque(collections.deque): '''Like utils.sendable_list, but for deque objects''' send = collections.deque.append class async_iterable(object): ''' A utility class that implements an async iterator returning values dispatched by a coroutine pipeline after *it* has received values coming from an async file-like object. ''' def __init__(self, f, buf_size, *coro_pipeline): self.events = sendable_deque() self.coro = utils.chain(self.events, *coro_pipeline) self.coro_finished = False self.f = f self.buf_size = buf_size self.read = None def __aiter__(self): return self async def __anext__(self): if not self.read: self.read = await _get_read(self.f) if self.events: return self.events.popleft() if self.coro_finished: raise StopAsyncIteration while True: data = await self.read(self.buf_size) try: self.coro.send(data) if self.events: return self.events.popleft() except StopIteration: self.coro_finished = True if self.events: return self.events.popleft() raise StopAsyncIteration def _make_basic_parse_async(backend): def basic_parse_async(f, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._basic_parse_pipeline(backend, config) ) return basic_parse_async def _make_parse_async(backend): def parse_async(f, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._parse_pipeline(backend, config) ) return parse_async def _make_items_async(backend): def items_async(f, prefix, map_type=None, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._items_pipeline(backend, prefix, map_type, config) ) return items_async def _make_kvitems_async(backend): def kvitems_async(f, prefix, map_type=None, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._kvitems_pipeline(backend, prefix, map_type, config) ) return kvitems_asyncijson-3.2.3/ijson/version.py000066400000000000000000000000261445666302700160350ustar00rootroot00000000000000__version__ = '3.2.3' ijson-3.2.3/notes/000077500000000000000000000000001445666302700140065ustar00rootroot00000000000000ijson-3.2.3/notes/design_notes.rst000066400000000000000000000431721445666302700172300ustar00rootroot00000000000000ijson design notes ################## Version 3.0 will come with a complete re-design of the underlying mechanism used to move data through the different library layers. This was done to address some limitations of the current design, and allow more advanced use cases out of the box. This document explains the design in ijson 2.x, and then it shows the new design coming with ijson 3.x, and how it offers both backward compatibility (with no performance cost) and new features out of the box. 2.x design (pull) ================= ijson is a library for iterating over big (or small) JSON documents. The main goal of the library is to avoid loading the entire document into memory; instead the document is iterated over in small chunks, results are given back to the user as they are found, and hence memory consumption stays low throughout the whole parsing process. This is achieved using python generators, which naturally lend themselves to this kind of problems: just write a ``for`` loop and ``yield`` values out of it, and the caller can iterate over them in their own ``for`` loop. ijson offers four modes of iterating over a JSON document, (not long ago they were three, as ``kvitems`` was recently added), each returning different types of results: * ``basic_parse`` is the most basic one, and returns ``(event, value)`` tuples each time an element is found in the JSON stream (e.g., the beginning of an array, a object member name, etc). * ``parse`` returns ``(prefix, event, value)`` tuples containing not only the information returned by ``basic_parse``, but also the *prefix* or *location* in the document hierarchy under which the event occurred. * ``items`` returns fully-built python objects constructed from the contents found at a given ``prefix``. * Finally, ``kvitems`` is similar to ``items``, but instead of returning full objects it returns ``(name, value)`` tuples consisting on its *members*, which is useful when individual objects in the JSON document are too big. As a side note, in ijson there are also different *backends*, all of which offer the same four iteration modes. Most of them share most of the code, but most interestingly the ``yajl2_c`` backend doesn't, re-implementing all the logic in C. These four iteration modes are all implemented as python generators. They also have the nice property that they are built on top of one another. For example, ``parse`` basically works like this: .. code-block:: python def parse(...): for event, value in basic_parse(...): prefix = calculate_prefix(event, value) yield (prefix, event, value) All in all, the different generators are combined like this to form a *pipeline*:: +---------+ ---| kvitems | +-------------+ +-------+ | +---------+ | basic_parse |-----| parse |----+ +-------------+ +-------+ | +---------+ |---| items | +---------+ Now, what the diagram above doesn't show is *who* calls *who*. Python generators yielding values work in a "pull" fashion: the caller drives the execution by calling ``next(generator)``, the generator runs until the next ``yield`` statement is executed and the yielded value is returned to the caller (or until the generator exists, in which case ``StopIteration`` is raised). So, adding the direction of how calls are made (including internal and external call points), the diagram above looks like this:: +---------+ next() ---| kvitems |<------- +-------------+ next() +-------+ next() | +---------+ | basic_parse |<--------| parse |<--------+ +-------------+ +-------+ | +---------+ next() ^ ^ |---| items |<------- | next() | next() +---------+ The only missing bit in this diagram now is: where does data reading happen? Because this is a pull model, the only possible solution is to do the reading at the lowest level possible: ``basic_parse`` reads a chunk out of the JSON stream when it is invoked for new tuples (and none are held in memory), calculates new tuples with the parsing results, yields them, and eventually when there are no more contents on the stream it exits, provoking the whole pipeline to exit In other words, ``basic_parse`` exhausts the JSON stream, but users drive the process by iterating over the generator of choice until it returns. All in all, the library works like this:: +---------+ next() ---| kvitems |<------- +------+ read() +-------------+ next() +-------+ next() | +---------+ | file |<--------| basic_parse |<--------| parse |<--------+ +------+ +-------------+ +-------+ | +---------+ next() ^ ^ |---| items |<------- | next() | next() +---------+ Limitations =========== This is all well and good if you live in a blocking I/O world. All you need to do is pass down your blocking, file-like object when you call ``items``, ``kvitems``, ``parse`` or ``basic_parse``, and ijson will do the reading for you as you iterate over the results. However, this is also a main fundamental limitation as it prevents users from easily using ijson in other scenarios. One such scenario comes up when using ``asyncio``. In an ``asyncio`` world users don't perform blocking reads anymore, and are instead handed over chunks of data to operate on. The only way to operate with ijson in these situations is emulating a blocking, file-like object, which is error-prone and not a great solution overall. Another example in which ijson doesn't work well would be the following: consider a multi-stage, blocking socket-based conversation between ``A`` and ``B``. ``A`` first receives some JSON content from ``B``, and after parsing it it needs to reply to ``B``, which will send more content through the same channel of communication, and so on. In such case, one would need to wrap the socket into a new object that emulates exhausting of the stream when required, even though the underlying socket is not exhausted yet. In both cases what we want to do is let users *push* data into ijson rather than having ijson *pull* data. This is basis for the new design. 3.x design (push) ================= In ijson 3.x we completely redesigned the parsing pipeline to work in a push model rather than a pull model, where chunks of data are pushed by the user into the pipeline. This effectively decouples I/O from the core parsing logic. In other words, at the low level instead of working like this:: +---------+ next() ---| kvitems |<------- +------+ read() +-------------+ next() +-------+ next() | +---------+ | file |<--------| basic_parse |<--------| parse |<--------+ +------+ +-------------+ +-------+ | +---------+ next() ^ ^ |---| items |<------- | next() | next() +---------+ the library now works like this:: +---------+ +-->| kvitems | +-------------+ +-------+ | +---------+ chunk ---->| basic_parse |---->| parse |----+ +-------------+ +-------+ | +---------+ |-->| items | +---------+ Here, ``chunk`` is a piece of data held by the user, who sends it into ``basic_parse``, who upon calculating a new tuple sends it to ``parse`` and so on, depending on what iteration mode you request. Now the user is in full control of feeding data into the pipeline and reacting to its results, without the library being in charge anymore. This is implemented using... python generators! Generators are usually used to yield values, but since python 2.5 they can also to *receive* values back from their callers. This turns then effectively into "coroutines", very much like those found in python 3.5+ ``asyncio`` coroutines using the ``async`` and ``await`` syntax. On that note: why didn't we use *those*? There are at least two drawbacks to using them directly: * Using ``asyncio`` coroutines would require users of the library to work within an ``asyncio`` event loop when interacting with ijson routines. As of now, that represents 0% of the current user base of ijson, which so far hasn't offered support for ``asyncio`` out of the box. Using generator coroutines doesn't impose any execution context, giving users the freedom to use them wherever they want. * Python 2.7 support would be removed. While python 2.7 is no longer maintained by the Python Software Foundation, there might still be programs out there using ijson with python 2.7, and we don't want to break them (yet). Using generator coroutines we maintain python 2.7 support in the library core. For the rest of the text, "coroutine" then means "plain, generator-based coroutine" and not "python 3 ``asyncio``-based coroutine", unless explicitly mentioned. How do these new coroutines look like? Firstly, they have new names to avoid clashing with those of the current set of generators, and hence they all end up with a ``_basecoro`` suffix (more on this later). Apart from this they look fairly similar to the original generators. For example, let's see ``basic_parse_basecoro`` and ``parse_basecoro`` in action (this is not actual code, just a skeleton): .. code-block:: python def basic_parse_basecoro(target, ...): while True: chunk = (yield) event, value = do_some_parsing(chunk) target.send((event, value)) def parse_basecoro(target, ...): while True: event, value = (yield) prefix = calculate_prefix(event, value) target.send((prefix, event, value)) The key components are the ``(yield)`` statements, which allow coroutines to receive data, and the ``target.send`` calls, which is how one sends data into a coroutine. Moreover, we can chain them again forming a pipeline, with data being pushed from one side, and the appropriate events making it out on the other. With these changes the core pipeline now looks like this (again including internal and external calls):: +------------------+ send() +-->| kvitems_basecoro |-------> send() +----------------------+ send() +----------------+ send() | +------------------+ chunk ------->| basic_parse_basecoro |------->| parse_basecoro |--------+ +----------------------+ +----------------+ | +------------------+ send() | | |-->| items_basecoro |-------> +--> send() +--> send() +------------------+ Backwards-compatibility ----------------------- Implementing the original generators on top of this coroutine-based pipeline can be easily done, thus retaining backwards-compatibility for all users. This is basically how ``parse`` works now: .. code-block:: python def sendable_list(list): send = list.append def parse(f, ...): results = sendable_list() coro = parse_basecoro(results) coro = basic_parse_basecoro(parse) while True: chunk = f.read() coro.send(chunk) for result in results: yield result del results[:] if not chunk: break Or, in other words:: parse +------------------------------------------------------------------------------+ +------+ read() | +----------------------+ send() +----------------+ send() +------+ | next() | file |<--------| chunk --->| basic_parse_basecoro |------->| parse_basecoro |------->| list | |<------- +------+ | +----------------------+ +----------------+ +------+ | +------------------------------------------------------------------------------+ The other generators work similarly, with the corresponding coroutine pipeline constructed inside each generator. Support for asyncio ------------------- Using this new framework, adding support for ``asyncio`` generators (i.e., that can be used in an ``async for`` loop) is also trivial. Now when running under python 3.5+ sll generators have an ``async`` counterpart ending with an ``_async`` suffix, which are roughly implemented like this: .. code-block:: python def async_iterable(object): def __init__(self, f): self.f = f self.events = sendable_deque() async def __anext__(self): data = await self.f.read() try: self.coro.send(data) except StopIteration: raise StopAsyncIteration return self.events.pop() def basic_parse_async(f, ...): iterable = async_iterable(f) iterable.coro = basic_parse_basecoro(iterable.events) return iterable Or, in other words:: parse_async +-----------------------------------------------------------------------------------+ +------+ await read() | send() +----------------------+ send() +----------------+ send() +-------+ | __anext__() | file |<--------------| chunk ------->| basic_parse_basecoro |------->| parse_basecoro |------->| deque | |<------------ +------+ | +----------------------+ +----------------+ +-------+ | +-----------------------------------------------------------------------------------+ Again, the other async generators work similarly, with the full corresponding coroutine pipeline constructed inside the async generator. User-facing coroutines ---------------------- Finally, it would also make sense to offer users access to the underlying coroutines, with users pushing data into them, and registering a target to receive the results. The ``*_basecoro`` coroutines are designed to work each on their own though, and users would have to create them inidividually and then chain them together manually, which can be error prone. Instead we also offer "pre-chained" coroutines for each of the iterators, which receive a chunk of data on one side, and send out the relevant event to the user-provided coroutine. These are called ``*_coro`` (which is why the *core* ones are called ``*_basecoro`` instead). They are roughly implemented like this: .. code-block:: python def parse_coro(target, ...): return basic_parse_basecoro(parse_basecoro(target), ...) Or, in other words:: parse_coro +-----------------------------------------------------+ send() | +----------------------+ send() +----------------+ | send() chunk --------|->| basic_parse_basecoro |------->| parse_basecoro |-|-------> | +----------------------+ +----------------+ | +-----------------------------------------------------+ The other user-facing coroutines are constructed similarly. Performance =========== This is the best part: performance is still on-par with the previous implementation, and has even been improved as part of this exercise. The plot below shows a comparison of processing speeds of each generator over three different backends (python, yajl2 and yajl2_c) for different synthetic test cases. The other backends should have similar results. For each generator/backend/test combination, the old implementation (pure generator) is compared to the new (coroutine-based generator). Values are processing speeds in [MB/s], so higher is better. All these measurements have been made using the ``benchmark.py`` tool found in the git repository of ijson, so you can give it a try as well. .. image:: performance_comparison.png These measurements were run on an Intel(R) Core(TM) i7-5600U CPU using python 3.7 (full results: `old.csv <./old.csv>`_, `new.csv <./new.csv>`_). It can be seen that results vary depending on the backend, but overall speeds are comparable to the original implementation. Special attention was given to the C backend, as it was, and remains, the fastest of all, usually by a factor of 10x. During the porting to the new push model, we added some modifications to its inner working to avoid unnecessary data copies and tuple packing/unpacking where possible, leading to a noticeable improvement on performance (~25% as the median). Again, your mileage might vary depending on the document you are parsing, but overall these are very good results. No proper performance comparison has been made yet on the ``asyncio`` generators offered by ijson, but early results suggest there is still work to be done to fully catch up with the generators' speed. On the one hand, implementing them as ``async`` generators (which would require python 3.6+) instead of classes with ``__aiter__`` and ``__anext__`` apparently would give a boost in speed. Other strategies could also be investigated for storing the temporary results rather than keeping them in a deque. Finally, the C backend could see its own implementation of the ``async`` iterables, which will probably not be too hard. ijson-3.2.3/notes/new.csv000066400000000000000000000463771445666302700153350ustar00rootroot00000000000000#mbytes,method,test_case,backend,time,mb_per_sec 0.954, basic_parse, long_list, python, 1.331, 0.716 0.954, basic_parse, long_list, yajl2, 0.978, 0.975 0.954, basic_parse, long_list, yajl2_c, 0.077, 12.324 10.278, basic_parse, big_int_object, python, 2.780, 3.697 10.278, basic_parse, big_int_object, yajl2, 1.845, 5.571 10.278, basic_parse, big_int_object, yajl2_c, 0.150, 68.385 11.232, basic_parse, big_decimal_object, python, 2.969, 3.783 11.232, basic_parse, big_decimal_object, yajl2, 1.851, 6.069 11.232, basic_parse, big_decimal_object, yajl2_c, 0.243, 46.182 9.431, basic_parse, big_null_object, python, 2.537, 3.718 9.431, basic_parse, big_null_object, yajl2, 0.955, 9.873 9.431, basic_parse, big_null_object, yajl2_c, 0.105, 89.630 9.669, basic_parse, big_bool_object, python, 2.454, 3.940 9.669, basic_parse, big_bool_object, yajl2, 1.063, 9.094 9.669, basic_parse, big_bool_object, yajl2_c, 0.117, 82.632 14.093, basic_parse, big_str_object, python, 2.830, 4.980 14.093, basic_parse, big_str_object, yajl2, 1.484, 9.498 14.093, basic_parse, big_str_object, yajl2_c, 0.124, 113.655 40.425, basic_parse, big_longstr_object, python, 2.972, 13.600 40.425, basic_parse, big_longstr_object, yajl2, 1.487, 27.195 40.425, basic_parse, big_longstr_object, yajl2_c, 0.128, 315.975 96.321, basic_parse, object_with_10_keys, python, 28.759, 3.349 96.321, basic_parse, object_with_10_keys, yajl2, 14.782, 6.516 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.344, 71.645 1.907, basic_parse, empty_lists, python, 1.389, 1.373 1.907, basic_parse, empty_lists, yajl2, 0.404, 4.716 1.907, basic_parse, empty_lists, yajl2_c, 0.092, 20.730 1.907, basic_parse, empty_objects, python, 1.401, 1.362 1.907, basic_parse, empty_objects, yajl2, 0.398, 4.791 1.907, basic_parse, empty_objects, yajl2_c, 0.090, 21.287 0.954, basic_parse, long_list, python, 1.277, 0.747 0.954, basic_parse, long_list, yajl2, 0.911, 1.047 0.954, basic_parse, long_list, yajl2_c, 0.077, 12.316 10.278, basic_parse, big_int_object, python, 2.704, 3.801 10.278, basic_parse, big_int_object, yajl2, 2.350, 4.374 10.278, basic_parse, big_int_object, yajl2_c, 0.147, 70.118 11.232, basic_parse, big_decimal_object, python, 2.881, 3.899 11.232, basic_parse, big_decimal_object, yajl2, 1.777, 6.319 11.232, basic_parse, big_decimal_object, yajl2_c, 0.253, 44.366 9.431, basic_parse, big_null_object, python, 2.496, 3.779 9.431, basic_parse, big_null_object, yajl2, 0.906, 10.409 9.431, basic_parse, big_null_object, yajl2_c, 0.106, 88.968 9.669, basic_parse, big_bool_object, python, 2.423, 3.990 9.669, basic_parse, big_bool_object, yajl2, 1.000, 9.671 9.669, basic_parse, big_bool_object, yajl2_c, 0.119, 81.382 14.093, basic_parse, big_str_object, python, 2.721, 5.179 14.093, basic_parse, big_str_object, yajl2, 1.416, 9.951 14.093, basic_parse, big_str_object, yajl2_c, 0.124, 113.303 40.425, basic_parse, big_longstr_object, python, 2.871, 14.078 40.425, basic_parse, big_longstr_object, yajl2, 1.449, 27.899 40.425, basic_parse, big_longstr_object, yajl2_c, 0.131, 308.726 96.321, basic_parse, object_with_10_keys, python, 27.844, 3.459 96.321, basic_parse, object_with_10_keys, yajl2, 14.668, 6.567 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.356, 71.024 1.907, basic_parse, empty_lists, python, 1.401, 1.362 1.907, basic_parse, empty_lists, yajl2, 0.403, 4.727 1.907, basic_parse, empty_lists, yajl2_c, 0.097, 19.683 1.907, basic_parse, empty_objects, python, 1.412, 1.351 1.907, basic_parse, empty_objects, yajl2, 0.398, 4.787 1.907, basic_parse, empty_objects, yajl2_c, 0.089, 21.457 0.954, basic_parse, long_list, python, 1.277, 0.747 0.954, basic_parse, long_list, yajl2, 0.937, 1.017 0.954, basic_parse, long_list, yajl2_c, 0.079, 12.061 10.278, basic_parse, big_int_object, python, 2.651, 3.877 10.278, basic_parse, big_int_object, yajl2, 2.334, 4.404 10.278, basic_parse, big_int_object, yajl2_c, 0.147, 70.047 11.232, basic_parse, big_decimal_object, python, 2.849, 3.942 11.232, basic_parse, big_decimal_object, yajl2, 1.848, 6.079 11.232, basic_parse, big_decimal_object, yajl2_c, 0.260, 43.123 9.431, basic_parse, big_null_object, python, 2.433, 3.876 9.431, basic_parse, big_null_object, yajl2, 0.921, 10.238 9.431, basic_parse, big_null_object, yajl2_c, 0.106, 88.759 9.669, basic_parse, big_bool_object, python, 2.358, 4.101 9.669, basic_parse, big_bool_object, yajl2, 1.015, 9.525 9.669, basic_parse, big_bool_object, yajl2_c, 0.118, 82.069 14.093, basic_parse, big_str_object, python, 2.645, 5.329 14.093, basic_parse, big_str_object, yajl2, 1.427, 9.879 14.093, basic_parse, big_str_object, yajl2_c, 0.124, 113.494 40.425, basic_parse, big_longstr_object, python, 2.805, 14.414 40.425, basic_parse, big_longstr_object, yajl2, 1.467, 27.557 40.425, basic_parse, big_longstr_object, yajl2_c, 0.128, 316.739 96.321, basic_parse, object_with_10_keys, python, 27.124, 3.551 96.321, basic_parse, object_with_10_keys, yajl2, 14.835, 6.493 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.368, 70.414 1.907, basic_parse, empty_lists, python, 1.370, 1.392 1.907, basic_parse, empty_lists, yajl2, 0.401, 4.754 1.907, basic_parse, empty_lists, yajl2_c, 0.097, 19.602 1.907, basic_parse, empty_objects, python, 1.380, 1.382 1.907, basic_parse, empty_objects, yajl2, 0.398, 4.791 1.907, basic_parse, empty_objects, yajl2_c, 0.101, 18.952 0.954, parse, long_list, python, 1.415, 0.674 0.954, parse, long_list, yajl2, 1.126, 0.847 0.954, parse, long_list, yajl2_c, 0.087, 10.925 10.278, parse, big_int_object, python, 2.947, 3.488 10.278, parse, big_int_object, yajl2, 2.784, 3.692 10.278, parse, big_int_object, yajl2_c, 0.176, 58.336 11.232, parse, big_decimal_object, python, 3.142, 3.575 11.232, parse, big_decimal_object, yajl2, 2.318, 4.845 11.232, parse, big_decimal_object, yajl2_c, 0.278, 40.389 9.431, parse, big_null_object, python, 2.722, 3.464 9.431, parse, big_null_object, yajl2, 1.248, 7.557 9.431, parse, big_null_object, yajl2_c, 0.135, 69.694 9.669, parse, big_bool_object, python, 2.663, 3.631 9.669, parse, big_bool_object, yajl2, 1.349, 7.167 9.669, parse, big_bool_object, yajl2_c, 0.145, 66.678 14.093, parse, big_str_object, python, 2.932, 4.806 14.093, parse, big_str_object, yajl2, 1.824, 7.725 14.093, parse, big_str_object, yajl2_c, 0.156, 90.305 40.425, parse, big_longstr_object, python, 3.110, 12.999 40.425, parse, big_longstr_object, yajl2, 1.856, 21.779 40.425, parse, big_longstr_object, yajl2_c, 0.151, 267.206 96.321, parse, object_with_10_keys, python, 30.939, 3.113 96.321, parse, object_with_10_keys, yajl2, 19.392, 4.967 96.321, parse, object_with_10_keys, yajl2_c, 2.057, 46.835 1.907, parse, empty_lists, python, 1.766, 1.080 1.907, parse, empty_lists, yajl2, 0.721, 2.645 1.907, parse, empty_lists, yajl2_c, 0.148, 12.848 1.907, parse, empty_objects, python, 1.749, 1.091 1.907, parse, empty_objects, yajl2, 0.653, 2.922 1.907, parse, empty_objects, yajl2_c, 0.109, 17.559 0.954, parse, long_list, python, 1.426, 0.669 0.954, parse, long_list, yajl2, 1.088, 0.876 0.954, parse, long_list, yajl2_c, 0.089, 10.671 10.278, parse, big_int_object, python, 2.968, 3.463 10.278, parse, big_int_object, yajl2, 2.112, 4.866 10.278, parse, big_int_object, yajl2_c, 0.201, 51.140 11.232, parse, big_decimal_object, python, 3.220, 3.488 11.232, parse, big_decimal_object, yajl2, 2.209, 5.084 11.232, parse, big_decimal_object, yajl2_c, 0.280, 40.121 9.431, parse, big_null_object, python, 2.749, 3.430 9.431, parse, big_null_object, yajl2, 1.203, 7.840 9.431, parse, big_null_object, yajl2_c, 0.135, 69.700 9.669, parse, big_bool_object, python, 2.675, 3.615 9.669, parse, big_bool_object, yajl2, 1.288, 7.509 9.669, parse, big_bool_object, yajl2_c, 0.146, 66.184 14.093, parse, big_str_object, python, 2.925, 4.819 14.093, parse, big_str_object, yajl2, 1.704, 8.269 14.093, parse, big_str_object, yajl2_c, 0.156, 90.259 40.425, parse, big_longstr_object, python, 3.087, 13.096 40.425, parse, big_longstr_object, yajl2, 1.756, 23.021 40.425, parse, big_longstr_object, yajl2_c, 0.152, 265.778 96.321, parse, object_with_10_keys, python, 31.228, 3.084 96.321, parse, object_with_10_keys, yajl2, 18.520, 5.201 96.321, parse, object_with_10_keys, yajl2_c, 2.069, 46.549 1.907, parse, empty_lists, python, 1.783, 1.070 1.907, parse, empty_lists, yajl2, 0.701, 2.720 1.907, parse, empty_lists, yajl2_c, 0.145, 13.119 1.907, parse, empty_objects, python, 1.743, 1.094 1.907, parse, empty_objects, yajl2, 0.651, 2.932 1.907, parse, empty_objects, yajl2_c, 0.111, 17.210 0.954, parse, long_list, python, 1.403, 0.680 0.954, parse, long_list, yajl2, 1.082, 0.882 0.954, parse, long_list, yajl2_c, 0.086, 11.133 10.278, parse, big_int_object, python, 2.939, 3.498 10.278, parse, big_int_object, yajl2, 2.667, 3.854 10.278, parse, big_int_object, yajl2_c, 0.176, 58.464 11.232, parse, big_decimal_object, python, 3.108, 3.614 11.232, parse, big_decimal_object, yajl2, 2.196, 5.116 11.232, parse, big_decimal_object, yajl2_c, 0.279, 40.213 9.431, parse, big_null_object, python, 2.702, 3.490 9.431, parse, big_null_object, yajl2, 1.209, 7.800 9.431, parse, big_null_object, yajl2_c, 0.134, 70.563 9.669, parse, big_bool_object, python, 2.653, 3.645 9.669, parse, big_bool_object, yajl2, 1.281, 7.547 9.669, parse, big_bool_object, yajl2_c, 0.146, 66.400 14.093, parse, big_str_object, python, 2.925, 4.818 14.093, parse, big_str_object, yajl2, 1.745, 8.076 14.093, parse, big_str_object, yajl2_c, 0.154, 91.303 40.425, parse, big_longstr_object, python, 3.102, 13.033 40.425, parse, big_longstr_object, yajl2, 1.775, 22.771 40.425, parse, big_longstr_object, yajl2_c, 0.151, 267.658 96.321, parse, object_with_10_keys, python, 30.655, 3.142 96.321, parse, object_with_10_keys, yajl2, 18.543, 5.195 96.321, parse, object_with_10_keys, yajl2_c, 2.074, 46.438 1.907, parse, empty_lists, python, 1.765, 1.081 1.907, parse, empty_lists, yajl2, 0.720, 2.651 1.907, parse, empty_lists, yajl2_c, 0.144, 13.270 1.907, parse, empty_objects, python, 1.735, 1.099 1.907, parse, empty_objects, yajl2, 0.658, 2.897 1.907, parse, empty_objects, yajl2_c, 0.108, 17.686 0.954, kvitems, long_list, python, 1.384, 0.689 0.954, kvitems, long_list, yajl2, 1.063, 0.897 0.954, kvitems, long_list, yajl2_c, 0.046, 20.516 10.278, kvitems, big_int_object, python, 3.861, 2.662 10.278, kvitems, big_int_object, yajl2, 2.991, 3.437 10.278, kvitems, big_int_object, yajl2_c, 0.160, 64.334 11.232, kvitems, big_decimal_object, python, 4.047, 2.775 11.232, kvitems, big_decimal_object, yajl2, 3.069, 3.659 11.232, kvitems, big_decimal_object, yajl2_c, 0.258, 43.619 9.431, kvitems, big_null_object, python, 3.513, 2.684 9.431, kvitems, big_null_object, yajl2, 2.059, 4.580 9.431, kvitems, big_null_object, yajl2_c, 0.117, 80.738 9.669, kvitems, big_bool_object, python, 3.498, 2.764 9.669, kvitems, big_bool_object, yajl2, 2.166, 4.464 9.669, kvitems, big_bool_object, yajl2_c, 0.127, 75.897 14.093, kvitems, big_str_object, python, 3.789, 3.720 14.093, kvitems, big_str_object, yajl2, 2.597, 5.427 14.093, kvitems, big_str_object, yajl2_c, 0.124, 113.333 40.425, kvitems, big_longstr_object, python, 3.994, 10.120 40.425, kvitems, big_longstr_object, yajl2, 2.633, 15.353 40.425, kvitems, big_longstr_object, yajl2_c, 0.153, 264.141 96.321, kvitems, object_with_10_keys, python, 31.773, 3.032 96.321, kvitems, object_with_10_keys, yajl2, 19.105, 5.042 96.321, kvitems, object_with_10_keys, yajl2_c, 1.398, 68.878 1.907, kvitems, empty_lists, python, 1.731, 1.102 1.907, kvitems, empty_lists, yajl2, 0.685, 2.786 1.907, kvitems, empty_lists, yajl2_c, 0.081, 23.503 1.907, kvitems, empty_objects, python, 1.719, 1.110 1.907, kvitems, empty_objects, yajl2, 0.638, 2.989 1.907, kvitems, empty_objects, yajl2_c, 0.049, 38.989 0.954, kvitems, long_list, python, 1.399, 0.682 0.954, kvitems, long_list, yajl2, 1.021, 0.934 0.954, kvitems, long_list, yajl2_c, 0.046, 20.669 10.278, kvitems, big_int_object, python, 4.266, 2.410 10.278, kvitems, big_int_object, yajl2, 3.191, 3.221 10.278, kvitems, big_int_object, yajl2_c, 0.158, 65.209 11.232, kvitems, big_decimal_object, python, 4.133, 2.718 11.232, kvitems, big_decimal_object, yajl2, 3.006, 3.737 11.232, kvitems, big_decimal_object, yajl2_c, 0.259, 43.353 9.431, kvitems, big_null_object, python, 3.535, 2.668 9.431, kvitems, big_null_object, yajl2, 2.040, 4.624 9.431, kvitems, big_null_object, yajl2_c, 0.116, 81.083 9.669, kvitems, big_bool_object, python, 3.480, 2.779 9.669, kvitems, big_bool_object, yajl2, 2.140, 4.519 9.669, kvitems, big_bool_object, yajl2_c, 0.127, 75.977 14.093, kvitems, big_str_object, python, 3.750, 3.759 14.093, kvitems, big_str_object, yajl2, 2.550, 5.526 14.093, kvitems, big_str_object, yajl2_c, 0.127, 110.920 40.425, kvitems, big_longstr_object, python, 3.984, 10.148 40.425, kvitems, big_longstr_object, yajl2, 2.617, 15.448 40.425, kvitems, big_longstr_object, yajl2_c, 0.154, 262.579 96.321, kvitems, object_with_10_keys, python, 31.667, 3.042 96.321, kvitems, object_with_10_keys, yajl2, 18.280, 5.269 96.321, kvitems, object_with_10_keys, yajl2_c, 1.386, 69.508 1.907, kvitems, empty_lists, python, 1.737, 1.098 1.907, kvitems, empty_lists, yajl2, 0.675, 2.827 1.907, kvitems, empty_lists, yajl2_c, 0.081, 23.418 1.907, kvitems, empty_objects, python, 1.754, 1.087 1.907, kvitems, empty_objects, yajl2, 0.638, 2.989 1.907, kvitems, empty_objects, yajl2_c, 0.049, 38.582 0.954, kvitems, long_list, python, 1.418, 0.672 0.954, kvitems, long_list, yajl2, 1.062, 0.898 0.954, kvitems, long_list, yajl2_c, 0.046, 20.625 10.278, kvitems, big_int_object, python, 4.531, 2.268 10.278, kvitems, big_int_object, yajl2, 3.079, 3.338 10.278, kvitems, big_int_object, yajl2_c, 0.158, 65.042 11.232, kvitems, big_decimal_object, python, 4.252, 2.642 11.232, kvitems, big_decimal_object, yajl2, 3.176, 3.536 11.232, kvitems, big_decimal_object, yajl2_c, 0.262, 42.900 9.431, kvitems, big_null_object, python, 3.606, 2.615 9.431, kvitems, big_null_object, yajl2, 2.091, 4.509 9.431, kvitems, big_null_object, yajl2_c, 0.117, 80.600 9.669, kvitems, big_bool_object, python, 3.538, 2.733 9.669, kvitems, big_bool_object, yajl2, 2.200, 4.395 9.669, kvitems, big_bool_object, yajl2_c, 0.128, 75.585 14.093, kvitems, big_str_object, python, 3.824, 3.686 14.093, kvitems, big_str_object, yajl2, 2.595, 5.430 14.093, kvitems, big_str_object, yajl2_c, 0.124, 113.757 40.425, kvitems, big_longstr_object, python, 4.026, 10.040 40.425, kvitems, big_longstr_object, yajl2, 2.645, 15.285 40.425, kvitems, big_longstr_object, yajl2_c, 0.153, 264.368 96.321, kvitems, object_with_10_keys, python, 32.490, 2.965 96.321, kvitems, object_with_10_keys, yajl2, 18.816, 5.119 96.321, kvitems, object_with_10_keys, yajl2_c, 1.370, 70.325 1.907, kvitems, empty_lists, python, 1.748, 1.091 1.907, kvitems, empty_lists, yajl2, 0.687, 2.774 1.907, kvitems, empty_lists, yajl2_c, 0.082, 23.264 1.907, kvitems, empty_objects, python, 1.750, 1.090 1.907, kvitems, empty_objects, yajl2, 0.646, 2.952 1.907, kvitems, empty_objects, yajl2_c, 0.049, 39.013 0.954, items, long_list, python, 1.642, 0.581 0.954, items, long_list, yajl2, 1.384, 0.689 0.954, items, long_list, yajl2_c, 0.060, 15.794 10.278, items, big_int_object, python, 3.677, 2.795 10.278, items, big_int_object, yajl2, 2.877, 3.573 10.278, items, big_int_object, yajl2_c, 0.261, 39.381 11.232, items, big_decimal_object, python, 3.849, 2.918 11.232, items, big_decimal_object, yajl2, 3.032, 3.705 11.232, items, big_decimal_object, yajl2_c, 0.398, 28.247 9.431, items, big_null_object, python, 3.317, 2.843 9.431, items, big_null_object, yajl2, 1.885, 5.003 9.431, items, big_null_object, yajl2_c, 0.202, 46.577 9.669, items, big_bool_object, python, 3.250, 2.975 9.669, items, big_bool_object, yajl2, 2.007, 4.818 9.669, items, big_bool_object, yajl2_c, 0.206, 47.051 14.093, items, big_str_object, python, 3.606, 3.909 14.093, items, big_str_object, yajl2, 2.551, 5.525 14.093, items, big_str_object, yajl2_c, 0.238, 59.234 40.425, items, big_longstr_object, python, 3.868, 10.452 40.425, items, big_longstr_object, yajl2, 2.616, 15.453 40.425, items, big_longstr_object, yajl2_c, 0.284, 142.489 96.321, items, object_with_10_keys, python, 38.865, 2.478 96.321, items, object_with_10_keys, yajl2, 27.483, 3.505 96.321, items, object_with_10_keys, yajl2_c, 2.568, 37.505 1.907, items, empty_lists, python, 2.423, 0.787 1.907, items, empty_lists, yajl2, 1.377, 1.385 1.907, items, empty_lists, yajl2_c, 0.284, 6.714 1.907, items, empty_objects, python, 2.393, 0.797 1.907, items, empty_objects, yajl2, 1.286, 1.484 1.907, items, empty_objects, yajl2_c, 0.157, 12.131 0.954, items, long_list, python, 1.602, 0.595 0.954, items, long_list, yajl2, 1.253, 0.761 0.954, items, long_list, yajl2_c, 0.052, 18.172 10.278, items, big_int_object, python, 4.124, 2.492 10.278, items, big_int_object, yajl2, 2.694, 3.816 10.278, items, big_int_object, yajl2_c, 0.240, 42.825 11.232, items, big_decimal_object, python, 3.861, 2.910 11.232, items, big_decimal_object, yajl2, 2.863, 3.924 11.232, items, big_decimal_object, yajl2_c, 0.355, 31.656 9.431, items, big_null_object, python, 3.309, 2.850 9.431, items, big_null_object, yajl2, 1.744, 5.409 9.431, items, big_null_object, yajl2_c, 0.186, 50.587 9.669, items, big_bool_object, python, 3.253, 2.972 9.669, items, big_bool_object, yajl2, 1.892, 5.111 9.669, items, big_bool_object, yajl2_c, 0.188, 51.423 14.093, items, big_str_object, python, 3.587, 3.928 14.093, items, big_str_object, yajl2, 2.300, 6.126 14.093, items, big_str_object, yajl2_c, 0.226, 62.340 40.425, items, big_longstr_object, python, 3.820, 10.583 40.425, items, big_longstr_object, yajl2, 2.401, 16.837 40.425, items, big_longstr_object, yajl2_c, 0.269, 150.113 96.321, items, object_with_10_keys, python, 37.733, 2.553 96.321, items, object_with_10_keys, yajl2, 24.460, 3.938 96.321, items, object_with_10_keys, yajl2_c, 2.269, 42.453 1.907, items, empty_lists, python, 2.379, 0.802 1.907, items, empty_lists, yajl2, 1.360, 1.403 1.907, items, empty_lists, yajl2_c, 0.276, 6.917 1.907, items, empty_objects, python, 2.373, 0.804 1.907, items, empty_objects, yajl2, 1.282, 1.487 1.907, items, empty_objects, yajl2_c, 0.150, 12.728 0.954, items, long_list, python, 1.600, 0.596 0.954, items, long_list, yajl2, 1.268, 0.752 0.954, items, long_list, yajl2_c, 0.053, 17.996 10.278, items, big_int_object, python, 4.162, 2.470 10.278, items, big_int_object, yajl2, 2.753, 3.733 10.278, items, big_int_object, yajl2_c, 0.241, 42.670 11.232, items, big_decimal_object, python, 3.875, 2.898 11.232, items, big_decimal_object, yajl2, 2.907, 3.864 11.232, items, big_decimal_object, yajl2_c, 0.360, 31.228 9.431, items, big_null_object, python, 3.317, 2.843 9.431, items, big_null_object, yajl2, 1.841, 5.122 9.431, items, big_null_object, yajl2_c, 0.189, 49.869 9.669, items, big_bool_object, python, 3.283, 2.946 9.669, items, big_bool_object, yajl2, 1.934, 5.000 9.669, items, big_bool_object, yajl2_c, 0.191, 50.754 14.093, items, big_str_object, python, 3.554, 3.966 14.093, items, big_str_object, yajl2, 2.397, 5.880 14.093, items, big_str_object, yajl2_c, 0.225, 62.642 40.425, items, big_longstr_object, python, 3.811, 10.608 40.425, items, big_longstr_object, yajl2, 2.489, 16.242 40.425, items, big_longstr_object, yajl2_c, 0.271, 148.938 96.321, items, object_with_10_keys, python, 37.847, 2.545 96.321, items, object_with_10_keys, yajl2, 25.621, 3.759 96.321, items, object_with_10_keys, yajl2_c, 2.290, 42.064 1.907, items, empty_lists, python, 2.414, 0.790 1.907, items, empty_lists, yajl2, 1.388, 1.374 1.907, items, empty_lists, yajl2_c, 0.273, 6.982 1.907, items, empty_objects, python, 2.412, 0.791 1.907, items, empty_objects, yajl2, 1.306, 1.461 1.907, items, empty_objects, yajl2_c, 0.150, 12.743 ijson-3.2.3/notes/old.csv000066400000000000000000000463561445666302700153170ustar00rootroot00000000000000#mbytes,method,test_case,backend,time,mb_per_sec 0.954, basic_parse, long_list, python, 1.216, 0.784 0.954, basic_parse, long_list, yajl2, 0.934, 1.022 0.954, basic_parse, long_list, yajl2_c, 0.077, 12.331 10.278, basic_parse, big_int_object, python, 2.602, 3.950 10.278, basic_parse, big_int_object, yajl2, 2.387, 4.307 10.278, basic_parse, big_int_object, yajl2_c, 0.152, 67.801 11.232, basic_parse, big_decimal_object, python, 2.700, 4.161 11.232, basic_parse, big_decimal_object, yajl2, 1.841, 6.100 11.232, basic_parse, big_decimal_object, yajl2_c, 0.253, 44.418 9.431, basic_parse, big_null_object, python, 2.335, 4.038 9.431, basic_parse, big_null_object, yajl2, 0.929, 10.149 9.431, basic_parse, big_null_object, yajl2_c, 0.113, 83.529 9.669, basic_parse, big_bool_object, python, 2.317, 4.174 9.669, basic_parse, big_bool_object, yajl2, 1.020, 9.483 9.669, basic_parse, big_bool_object, yajl2_c, 0.122, 79.066 14.093, basic_parse, big_str_object, python, 2.607, 5.405 14.093, basic_parse, big_str_object, yajl2, 1.438, 9.798 14.093, basic_parse, big_str_object, yajl2_c, 0.130, 108.284 40.425, basic_parse, big_longstr_object, python, 2.730, 14.808 40.425, basic_parse, big_longstr_object, yajl2, 1.473, 27.440 40.425, basic_parse, big_longstr_object, yajl2_c, 0.134, 301.499 96.321, basic_parse, object_with_10_keys, python, 28.131, 3.424 96.321, basic_parse, object_with_10_keys, yajl2, 15.014, 6.415 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.434, 67.175 1.907, basic_parse, empty_lists, python, 1.486, 1.284 1.907, basic_parse, empty_lists, yajl2, 0.424, 4.498 1.907, basic_parse, empty_lists, yajl2_c, 0.095, 20.125 1.907, basic_parse, empty_objects, python, 1.489, 1.281 1.907, basic_parse, empty_objects, yajl2, 0.422, 4.519 1.907, basic_parse, empty_objects, yajl2_c, 0.096, 19.852 0.954, basic_parse, long_list, python, 1.218, 0.783 0.954, basic_parse, long_list, yajl2, 0.977, 0.976 0.954, basic_parse, long_list, yajl2_c, 0.078, 12.163 10.278, basic_parse, big_int_object, python, 2.621, 3.921 10.278, basic_parse, big_int_object, yajl2, 2.442, 4.208 10.278, basic_parse, big_int_object, yajl2_c, 0.176, 58.551 11.232, basic_parse, big_decimal_object, python, 2.839, 3.957 11.232, basic_parse, big_decimal_object, yajl2, 1.925, 5.834 11.232, basic_parse, big_decimal_object, yajl2_c, 0.255, 44.048 9.431, basic_parse, big_null_object, python, 2.386, 3.952 9.431, basic_parse, big_null_object, yajl2, 0.955, 9.879 9.431, basic_parse, big_null_object, yajl2_c, 0.114, 83.068 9.669, basic_parse, big_bool_object, python, 2.339, 4.134 9.669, basic_parse, big_bool_object, yajl2, 1.071, 9.025 9.669, basic_parse, big_bool_object, yajl2_c, 0.122, 79.234 14.093, basic_parse, big_str_object, python, 2.672, 5.274 14.093, basic_parse, big_str_object, yajl2, 1.506, 9.356 14.093, basic_parse, big_str_object, yajl2_c, 0.132, 107.023 40.425, basic_parse, big_longstr_object, python, 2.800, 14.435 40.425, basic_parse, big_longstr_object, yajl2, 1.529, 26.432 40.425, basic_parse, big_longstr_object, yajl2_c, 0.135, 298.699 96.321, basic_parse, object_with_10_keys, python, 29.161, 3.303 96.321, basic_parse, object_with_10_keys, yajl2, 15.523, 6.205 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.428, 67.442 1.907, basic_parse, empty_lists, python, 1.490, 1.280 1.907, basic_parse, empty_lists, yajl2, 0.415, 4.599 1.907, basic_parse, empty_lists, yajl2_c, 0.095, 20.104 1.907, basic_parse, empty_objects, python, 1.469, 1.298 1.907, basic_parse, empty_objects, yajl2, 0.413, 4.618 1.907, basic_parse, empty_objects, yajl2_c, 0.095, 19.981 0.954, basic_parse, long_list, python, 1.222, 0.780 0.954, basic_parse, long_list, yajl2, 0.986, 0.967 0.954, basic_parse, long_list, yajl2_c, 0.078, 12.274 10.278, basic_parse, big_int_object, python, 2.587, 3.973 10.278, basic_parse, big_int_object, yajl2, 1.793, 5.734 10.278, basic_parse, big_int_object, yajl2_c, 0.156, 65.943 11.232, basic_parse, big_decimal_object, python, 2.726, 4.120 11.232, basic_parse, big_decimal_object, yajl2, 1.878, 5.979 11.232, basic_parse, big_decimal_object, yajl2_c, 0.256, 43.955 9.431, basic_parse, big_null_object, python, 2.333, 4.042 9.431, basic_parse, big_null_object, yajl2, 0.947, 9.963 9.431, basic_parse, big_null_object, yajl2_c, 0.122, 77.363 9.669, basic_parse, big_bool_object, python, 2.300, 4.204 9.669, basic_parse, big_bool_object, yajl2, 1.062, 9.105 9.669, basic_parse, big_bool_object, yajl2_c, 0.126, 76.706 14.093, basic_parse, big_str_object, python, 2.612, 5.395 14.093, basic_parse, big_str_object, yajl2, 1.479, 9.531 14.093, basic_parse, big_str_object, yajl2_c, 0.134, 104.899 40.425, basic_parse, big_longstr_object, python, 2.705, 14.947 40.425, basic_parse, big_longstr_object, yajl2, 1.506, 26.836 40.425, basic_parse, big_longstr_object, yajl2_c, 0.141, 286.686 96.321, basic_parse, object_with_10_keys, python, 28.038, 3.435 96.321, basic_parse, object_with_10_keys, yajl2, 15.295, 6.298 96.321, basic_parse, object_with_10_keys, yajl2_c, 1.461, 65.907 1.907, basic_parse, empty_lists, python, 1.480, 1.288 1.907, basic_parse, empty_lists, yajl2, 0.423, 4.513 1.907, basic_parse, empty_lists, yajl2_c, 0.097, 19.611 1.907, basic_parse, empty_objects, python, 1.493, 1.278 1.907, basic_parse, empty_objects, yajl2, 0.419, 4.554 1.907, basic_parse, empty_objects, yajl2_c, 0.095, 19.979 0.954, parse, long_list, python, 1.342, 0.711 0.954, parse, long_list, yajl2, 1.066, 0.895 0.954, parse, long_list, yajl2_c, 0.098, 9.752 10.278, parse, big_int_object, python, 2.881, 3.568 10.278, parse, big_int_object, yajl2, 2.015, 5.102 10.278, parse, big_int_object, yajl2_c, 0.204, 50.394 11.232, parse, big_decimal_object, python, 3.086, 3.640 11.232, parse, big_decimal_object, yajl2, 2.081, 5.398 11.232, parse, big_decimal_object, yajl2_c, 0.305, 36.789 9.431, parse, big_null_object, python, 2.661, 3.544 9.431, parse, big_null_object, yajl2, 1.193, 7.908 9.431, parse, big_null_object, yajl2_c, 0.161, 58.604 9.669, parse, big_bool_object, python, 2.675, 3.615 9.669, parse, big_bool_object, yajl2, 1.300, 7.439 9.669, parse, big_bool_object, yajl2_c, 0.168, 57.546 14.093, parse, big_str_object, python, 2.996, 4.704 14.093, parse, big_str_object, yajl2, 1.746, 8.072 14.093, parse, big_str_object, yajl2_c, 0.171, 82.554 40.425, parse, big_longstr_object, python, 3.043, 13.282 40.425, parse, big_longstr_object, yajl2, 1.759, 22.980 40.425, parse, big_longstr_object, yajl2_c, 0.173, 233.269 96.321, parse, object_with_10_keys, python, 31.998, 3.010 96.321, parse, object_with_10_keys, yajl2, 18.427, 5.227 96.321, parse, object_with_10_keys, yajl2_c, 2.275, 42.335 1.907, parse, empty_lists, python, 1.838, 1.038 1.907, parse, empty_lists, yajl2, 0.683, 2.791 1.907, parse, empty_lists, yajl2_c, 0.171, 11.179 1.907, parse, empty_objects, python, 1.706, 1.118 1.907, parse, empty_objects, yajl2, 0.656, 2.907 1.907, parse, empty_objects, yajl2_c, 0.148, 12.912 0.954, parse, long_list, python, 1.349, 0.707 0.954, parse, long_list, yajl2, 1.048, 0.910 0.954, parse, long_list, yajl2_c, 0.098, 9.744 10.278, parse, big_int_object, python, 3.138, 3.275 10.278, parse, big_int_object, yajl2, 2.256, 4.557 10.278, parse, big_int_object, yajl2_c, 0.205, 50.119 11.232, parse, big_decimal_object, python, 3.061, 3.670 11.232, parse, big_decimal_object, yajl2, 2.055, 5.466 11.232, parse, big_decimal_object, yajl2_c, 0.307, 36.615 9.431, parse, big_null_object, python, 2.612, 3.611 9.431, parse, big_null_object, yajl2, 1.150, 8.203 9.431, parse, big_null_object, yajl2_c, 0.161, 58.438 9.669, parse, big_bool_object, python, 2.585, 3.741 9.669, parse, big_bool_object, yajl2, 1.251, 7.728 9.669, parse, big_bool_object, yajl2_c, 0.169, 57.167 14.093, parse, big_str_object, python, 2.886, 4.884 14.093, parse, big_str_object, yajl2, 1.661, 8.484 14.093, parse, big_str_object, yajl2_c, 0.174, 80.967 40.425, parse, big_longstr_object, python, 3.001, 13.471 40.425, parse, big_longstr_object, yajl2, 1.695, 23.846 40.425, parse, big_longstr_object, yajl2_c, 0.174, 232.151 96.321, parse, object_with_10_keys, python, 31.260, 3.081 96.321, parse, object_with_10_keys, yajl2, 17.703, 5.441 96.321, parse, object_with_10_keys, yajl2_c, 2.272, 42.390 1.907, parse, empty_lists, python, 1.819, 1.048 1.907, parse, empty_lists, yajl2, 0.667, 2.859 1.907, parse, empty_lists, yajl2_c, 0.174, 10.986 1.907, parse, empty_objects, python, 1.735, 1.099 1.907, parse, empty_objects, yajl2, 0.622, 3.068 1.907, parse, empty_objects, yajl2_c, 0.148, 12.921 0.954, parse, long_list, python, 1.317, 0.724 0.954, parse, long_list, yajl2, 1.027, 0.928 0.954, parse, long_list, yajl2_c, 0.097, 9.833 10.278, parse, big_int_object, python, 2.849, 3.607 10.278, parse, big_int_object, yajl2, 2.522, 4.076 10.278, parse, big_int_object, yajl2_c, 0.205, 50.106 11.232, parse, big_decimal_object, python, 3.051, 3.681 11.232, parse, big_decimal_object, yajl2, 2.030, 5.532 11.232, parse, big_decimal_object, yajl2_c, 0.307, 36.630 9.431, parse, big_null_object, python, 2.595, 3.634 9.431, parse, big_null_object, yajl2, 1.132, 8.329 9.431, parse, big_null_object, yajl2_c, 0.162, 58.292 9.669, parse, big_bool_object, python, 2.565, 3.769 9.669, parse, big_bool_object, yajl2, 1.237, 7.820 9.669, parse, big_bool_object, yajl2_c, 0.169, 57.261 14.093, parse, big_str_object, python, 2.881, 4.892 14.093, parse, big_str_object, yajl2, 1.653, 8.524 14.093, parse, big_str_object, yajl2_c, 0.172, 82.162 40.425, parse, big_longstr_object, python, 3.001, 13.469 40.425, parse, big_longstr_object, yajl2, 1.709, 23.650 40.425, parse, big_longstr_object, yajl2_c, 0.175, 230.897 96.321, parse, object_with_10_keys, python, 31.050, 3.102 96.321, parse, object_with_10_keys, yajl2, 17.577, 5.480 96.321, parse, object_with_10_keys, yajl2_c, 2.275, 42.335 1.907, parse, empty_lists, python, 1.821, 1.048 1.907, parse, empty_lists, yajl2, 0.658, 2.901 1.907, parse, empty_lists, yajl2_c, 0.177, 10.770 1.907, parse, empty_objects, python, 1.723, 1.107 1.907, parse, empty_objects, yajl2, 0.619, 3.082 1.907, parse, empty_objects, yajl2_c, 0.148, 12.881 0.954, kvitems, long_list, python, 1.398, 0.682 0.954, kvitems, long_list, yajl2, 1.088, 0.877 0.954, kvitems, long_list, yajl2_c, 0.104, 9.190 10.278, kvitems, big_int_object, python, 3.898, 2.637 10.278, kvitems, big_int_object, yajl2, 2.631, 3.907 10.278, kvitems, big_int_object, yajl2_c, 0.227, 45.254 11.232, kvitems, big_decimal_object, python, 4.216, 2.664 11.232, kvitems, big_decimal_object, yajl2, 2.731, 4.113 11.232, kvitems, big_decimal_object, yajl2_c, 0.334, 33.661 9.431, kvitems, big_null_object, python, 3.582, 2.633 9.431, kvitems, big_null_object, yajl2, 1.822, 5.177 9.431, kvitems, big_null_object, yajl2_c, 0.190, 49.754 9.669, kvitems, big_bool_object, python, 3.499, 2.763 9.669, kvitems, big_bool_object, yajl2, 1.916, 5.047 9.669, kvitems, big_bool_object, yajl2_c, 0.196, 49.237 14.093, kvitems, big_str_object, python, 3.835, 3.675 14.093, kvitems, big_str_object, yajl2, 2.327, 6.056 14.093, kvitems, big_str_object, yajl2_c, 0.202, 69.738 40.425, kvitems, big_longstr_object, python, 3.975, 10.171 40.425, kvitems, big_longstr_object, yajl2, 2.350, 17.204 40.425, kvitems, big_longstr_object, yajl2_c, 0.206, 195.991 96.321, kvitems, object_with_10_keys, python, 32.431, 2.970 96.321, kvitems, object_with_10_keys, yajl2, 18.863, 5.106 96.321, kvitems, object_with_10_keys, yajl2_c, 2.443, 39.433 1.907, kvitems, empty_lists, python, 1.927, 0.990 1.907, kvitems, empty_lists, yajl2, 0.750, 2.543 1.907, kvitems, empty_lists, yajl2_c, 0.185, 10.287 1.907, kvitems, empty_objects, python, 1.833, 1.040 1.907, kvitems, empty_objects, yajl2, 0.712, 2.679 1.907, kvitems, empty_objects, yajl2_c, 0.157, 12.178 0.954, kvitems, long_list, python, 1.397, 0.683 0.954, kvitems, long_list, yajl2, 1.073, 0.889 0.954, kvitems, long_list, yajl2_c, 0.103, 9.304 10.278, kvitems, big_int_object, python, 4.293, 2.394 10.278, kvitems, big_int_object, yajl2, 2.792, 3.681 10.278, kvitems, big_int_object, yajl2_c, 0.228, 45.143 11.232, kvitems, big_decimal_object, python, 4.091, 2.745 11.232, kvitems, big_decimal_object, yajl2, 2.740, 4.099 11.232, kvitems, big_decimal_object, yajl2_c, 0.336, 33.430 9.431, kvitems, big_null_object, python, 3.554, 2.653 9.431, kvitems, big_null_object, yajl2, 1.822, 5.176 9.431, kvitems, big_null_object, yajl2_c, 0.191, 49.258 9.669, kvitems, big_bool_object, python, 3.459, 2.795 9.669, kvitems, big_bool_object, yajl2, 1.937, 4.991 9.669, kvitems, big_bool_object, yajl2_c, 0.197, 49.116 14.093, kvitems, big_str_object, python, 3.752, 3.757 14.093, kvitems, big_str_object, yajl2, 2.314, 6.091 14.093, kvitems, big_str_object, yajl2_c, 0.204, 69.009 40.425, kvitems, big_longstr_object, python, 4.018, 10.061 40.425, kvitems, big_longstr_object, yajl2, 2.336, 17.302 40.425, kvitems, big_longstr_object, yajl2_c, 0.210, 192.685 96.321, kvitems, object_with_10_keys, python, 32.961, 2.922 96.321, kvitems, object_with_10_keys, yajl2, 18.683, 5.156 96.321, kvitems, object_with_10_keys, yajl2_c, 2.442, 39.445 1.907, kvitems, empty_lists, python, 1.890, 1.009 1.907, kvitems, empty_lists, yajl2, 0.737, 2.587 1.907, kvitems, empty_lists, yajl2_c, 0.187, 10.191 1.907, kvitems, empty_objects, python, 1.826, 1.044 1.907, kvitems, empty_objects, yajl2, 0.697, 2.736 1.907, kvitems, empty_objects, yajl2_c, 0.157, 12.150 0.954, kvitems, long_list, python, 1.369, 0.697 0.954, kvitems, long_list, yajl2, 1.054, 0.905 0.954, kvitems, long_list, yajl2_c, 0.106, 9.034 10.278, kvitems, big_int_object, python, 4.198, 2.448 10.278, kvitems, big_int_object, yajl2, 2.797, 3.674 10.278, kvitems, big_int_object, yajl2_c, 0.227, 45.268 11.232, kvitems, big_decimal_object, python, 4.246, 2.646 11.232, kvitems, big_decimal_object, yajl2, 2.680, 4.191 11.232, kvitems, big_decimal_object, yajl2_c, 0.337, 33.335 9.431, kvitems, big_null_object, python, 3.509, 2.687 9.431, kvitems, big_null_object, yajl2, 1.781, 5.296 9.431, kvitems, big_null_object, yajl2_c, 0.192, 49.222 9.669, kvitems, big_bool_object, python, 3.464, 2.792 9.669, kvitems, big_bool_object, yajl2, 1.892, 5.110 9.669, kvitems, big_bool_object, yajl2_c, 0.200, 48.251 14.093, kvitems, big_str_object, python, 3.766, 3.742 14.093, kvitems, big_str_object, yajl2, 2.285, 6.168 14.093, kvitems, big_str_object, yajl2_c, 0.202, 69.864 40.425, kvitems, big_longstr_object, python, 3.902, 10.360 40.425, kvitems, big_longstr_object, yajl2, 2.318, 17.440 40.425, kvitems, big_longstr_object, yajl2_c, 0.208, 194.403 96.321, kvitems, object_with_10_keys, python, 32.344, 2.978 96.321, kvitems, object_with_10_keys, yajl2, 18.428, 5.227 96.321, kvitems, object_with_10_keys, yajl2_c, 2.363, 40.760 1.907, kvitems, empty_lists, python, 1.902, 1.003 1.907, kvitems, empty_lists, yajl2, 0.739, 2.579 1.907, kvitems, empty_lists, yajl2_c, 0.184, 10.370 1.907, kvitems, empty_objects, python, 1.839, 1.037 1.907, kvitems, empty_objects, yajl2, 0.703, 2.713 1.907, kvitems, empty_objects, yajl2_c, 0.155, 12.329 0.954, items, long_list, python, 1.616, 0.590 0.954, items, long_list, yajl2, 1.254, 0.760 0.954, items, long_list, yajl2_c, 0.111, 8.630 10.278, items, big_int_object, python, 4.178, 2.460 10.278, items, big_int_object, yajl2, 2.498, 4.114 10.278, items, big_int_object, yajl2_c, 0.334, 30.778 11.232, items, big_decimal_object, python, 3.783, 2.969 11.232, items, big_decimal_object, yajl2, 2.641, 4.253 11.232, items, big_decimal_object, yajl2_c, 0.465, 24.156 9.431, items, big_null_object, python, 3.237, 2.913 9.431, items, big_null_object, yajl2, 1.692, 5.574 9.431, items, big_null_object, yajl2_c, 0.283, 33.379 9.669, items, big_bool_object, python, 3.187, 3.034 9.669, items, big_bool_object, yajl2, 1.802, 5.367 9.669, items, big_bool_object, yajl2_c, 0.289, 33.475 14.093, items, big_str_object, python, 3.552, 3.968 14.093, items, big_str_object, yajl2, 2.236, 6.302 14.093, items, big_str_object, yajl2_c, 0.324, 43.561 40.425, items, big_longstr_object, python, 3.798, 10.643 40.425, items, big_longstr_object, yajl2, 2.299, 17.587 40.425, items, big_longstr_object, yajl2_c, 0.346, 117.005 96.321, items, object_with_10_keys, python, 39.468, 2.441 96.321, items, object_with_10_keys, yajl2, 23.463, 4.105 96.321, items, object_with_10_keys, yajl2_c, 3.583, 26.884 1.907, items, empty_lists, python, 2.450, 0.779 1.907, items, empty_lists, yajl2, 1.315, 1.450 1.907, items, empty_lists, yajl2_c, 0.370, 5.161 1.907, items, empty_objects, python, 2.459, 0.776 1.907, items, empty_objects, yajl2, 1.214, 1.572 1.907, items, empty_objects, yajl2_c, 0.269, 7.094 0.954, items, long_list, python, 1.579, 0.604 0.954, items, long_list, yajl2, 1.250, 0.763 0.954, items, long_list, yajl2_c, 0.110, 8.688 10.278, items, big_int_object, python, 3.582, 2.870 10.278, items, big_int_object, yajl2, 2.521, 4.078 10.278, items, big_int_object, yajl2_c, 0.337, 30.476 11.232, items, big_decimal_object, python, 3.832, 2.931 11.232, items, big_decimal_object, yajl2, 2.664, 4.217 11.232, items, big_decimal_object, yajl2_c, 0.465, 24.136 9.431, items, big_null_object, python, 3.237, 2.913 9.431, items, big_null_object, yajl2, 1.722, 5.477 9.431, items, big_null_object, yajl2_c, 0.282, 33.388 9.669, items, big_bool_object, python, 3.175, 3.046 9.669, items, big_bool_object, yajl2, 1.791, 5.398 9.669, items, big_bool_object, yajl2_c, 0.293, 33.018 14.093, items, big_str_object, python, 3.511, 4.014 14.093, items, big_str_object, yajl2, 2.228, 6.325 14.093, items, big_str_object, yajl2_c, 0.323, 43.683 40.425, items, big_longstr_object, python, 3.748, 10.787 40.425, items, big_longstr_object, yajl2, 2.300, 17.575 40.425, items, big_longstr_object, yajl2_c, 0.345, 117.070 96.321, items, object_with_10_keys, python, 38.391, 2.509 96.321, items, object_with_10_keys, yajl2, 23.647, 4.073 96.321, items, object_with_10_keys, yajl2_c, 3.586, 26.858 1.907, items, empty_lists, python, 2.451, 0.778 1.907, items, empty_lists, yajl2, 1.320, 1.445 1.907, items, empty_lists, yajl2_c, 0.366, 5.211 1.907, items, empty_objects, python, 2.465, 0.774 1.907, items, empty_objects, yajl2, 1.219, 1.564 1.907, items, empty_objects, yajl2_c, 0.263, 7.250 0.954, items, long_list, python, 1.582, 0.603 0.954, items, long_list, yajl2, 1.281, 0.744 0.954, items, long_list, yajl2_c, 0.111, 8.627 10.278, items, big_int_object, python, 4.098, 2.508 10.278, items, big_int_object, yajl2, 2.567, 4.005 10.278, items, big_int_object, yajl2_c, 0.336, 30.614 11.232, items, big_decimal_object, python, 3.790, 2.964 11.232, items, big_decimal_object, yajl2, 2.652, 4.236 11.232, items, big_decimal_object, yajl2_c, 0.461, 24.361 9.431, items, big_null_object, python, 3.270, 2.884 9.431, items, big_null_object, yajl2, 1.704, 5.534 9.431, items, big_null_object, yajl2_c, 0.285, 33.079 9.669, items, big_bool_object, python, 3.215, 3.008 9.669, items, big_bool_object, yajl2, 1.827, 5.293 9.669, items, big_bool_object, yajl2_c, 0.296, 32.684 14.093, items, big_str_object, python, 3.509, 4.016 14.093, items, big_str_object, yajl2, 2.270, 6.208 14.093, items, big_str_object, yajl2_c, 0.317, 44.439 40.425, items, big_longstr_object, python, 3.779, 10.696 40.425, items, big_longstr_object, yajl2, 2.311, 17.495 40.425, items, big_longstr_object, yajl2_c, 0.333, 121.372 96.321, items, object_with_10_keys, python, 38.635, 2.493 96.321, items, object_with_10_keys, yajl2, 23.612, 4.079 96.321, items, object_with_10_keys, yajl2_c, 3.573, 26.956 1.907, items, empty_lists, python, 2.468, 0.773 1.907, items, empty_lists, yajl2, 1.335, 1.429 1.907, items, empty_lists, yajl2_c, 0.367, 5.193 1.907, items, empty_objects, python, 2.505, 0.762 1.907, items, empty_objects, yajl2, 1.215, 1.569 1.907, items, empty_objects, yajl2_c, 0.265, 7.201 ijson-3.2.3/notes/performance_comparison.png000066400000000000000000002167231445666302700212620ustar00rootroot00000000000000PNG  IHDR!FsBIT|d pHYsaa?i8tEXtSoftwarematplotlib version3.1.3, http://matplotlib.org/1D IDATx}xLw$b IܵAJ=Y-&FKP[BEUժn EGVݫP(i4jFՒ$d~t4M$3y>k99}rg:g>`2L]"DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DXE `!2*BdUV""DR0bլYeʙx :uʼ.88Xv Sp._%$$ػ@%?+66V:t=ܣ:u(88X;wwi;C!2ܦ˗/+66pۆ _>7n_W5jHsь3t%S˖-aƅCU5{TVrttinݺӪSys=VZi̙9rdY (Æ SxxF>.0@ÇիWb S?Pnq'2J-&&FA?x vښ0a$I]vU`` oڴBBBt)խ[W+ 5`լYSuՋ/B>z/ѨM_dg04n8mذA-ZhT͵uRzw{n _Xիe0N#F/ggggŋ-Oy$FGgΜѥKJeQ2 V\pzjEFFj„ ڿx͜9ӆ!DP%<S\\飅 g}=jSCUݺuxbI>РA ڵk_W׮]Mdc=7xC{մiSM?~0%sÁZ]o¡{{t;oWI.vڥpM0A.]Ҏ;tQ+nq'2*!22by͛7]ׇ~h{Pk֬рTF>s=g#_yyrtt?o^dҖ-[,_˖-fO@銈P~~֭[g^f]z|ܖ?ԩ$o_|Y?\\\,.nJ׵ 6mRVVy >ǎ{jƌ=z,X믵o߾څð;pb ڵKׇ~HM:Uvc=vuT 7Xnذ?5ӧ_Jvܩ 6춏l7{Gy~jժeѯYfիWq~O@zԾ}{)-VZN:QF_~E&L\\\Tn]HEq T[N~~~ww2?cթS|Jh!2*_BBB+WJm* Y?7ҚopcڳgΜ9'O*))b*'xBK,s=O>D۷o7?=z6mڤxupsɓ'մiSUL!2*X>qℊԠAIO=֭[_U6lА!C,BҸ_~}={3~s;?PV|IIү]viԩճgOwuɓ'kٲez74dȐ8 m!!2*aѢEo$)44Լnذa_?Y999w!%O>*,,om7ސ``?uQhhV\UVwުS`o޼yz5m4M0XEpذaC?~\W\ZvUBZZ{1[Zrz)nZ-ZڵkլY3ib...zf5iDjѢZhquOݺuK/SN)00P۷oƍ5qD HDD,I={yt颹sʕ+{}v8ׯה)SԸqc5k<5={wOpS_8˫0֤I,L&"(U5kh̙:uUqi޼yEDDhʔ)Vk4i /QO?T3gԚ5kl25h@ /P~{QQQQܯ^ZǏעEd2ԫW/mٲD;|ߦ^{nBd(caDDVX(}zG;wjر߿j!DP%ԭ[Wk׮eի`0駟a{PP:Tl}||㋭QLLź5kj?MkSNt;@pppPjԯ_?9;;[{OmBIxQBBE۪yf+Zz>cծ][?z; nd2iҥڵիgrva]pA%ǹsd0YJ[Ep9shΜ9OBdU^nn>S޽[Gƍ]8NgV֭յk;GFF֭[w}WAAA懳'.".\zJ6mZŋkʕjժ *ǎɓաC-Yt xᰠ@M宎w`6&@%6b wkԧOƍUM ֭M,[L#F D('~W%''ߴOkp.P~`=TxEEE:{jժ%`rL&.]$???98TkŌ z ()Bd{j޼yJNNֹs~z 0>b-_bmݺqYZPϺ]]0.@q p<.dQrssgyF a޽{kٲeexGǨU>J^,T76VE z ()BdP޴hOq'innn|)?e\zUy\@0 .!!A^^^jڴƌ/ڻ$ÝȰ޽{kРA ɓ'5m4*11Q7&??_l[ T9Ȱpz!lR 6TBB}nX[Ti(W~թSG'N"GGG+**ʼ|mRxdիWUXXhR*GGGUV9TZ wq@eǸpgPVQ9sF/^>FQFцU(+:w._lR*WWWz.JB0.Jqee*''G'N0/)%%ETll㣓'Ojʔ)jԨBBBX5Jߴk}mT ʃ"Q~~~^:Wod2@.\PZZ7n,J,Y6pP0&T9 wqeeСC֭y4Çŋwi̔zٳgs1PHruuw9O?@. JB0.Jqe(S2L^rqqѶmtyԩSzmWʌϸ;W^߳ŋe˖rsses{^^"##UvmլYSaaaȰӧշo_K'OիWm}*쨼~ƕgg( ܉ Gtg kƍd2i߿[5o\&Mk]ƍӠAW_I շo_h:w"""W_}g@¥ JYBB 233 CU>}qjҤ^yլYSIIIҥK5|u]m۶ղe˴~%%%Io߮T\RZRhhfϞEg%STl܉ (wnC|P6 vZ*((Hɺrza^zJLLTNzb3FZnmS`g!2L9rDAAAS͚5~z=JIIQՋitIRzzz\[F󕟟o^.b: G_@9^^^^rvv?ZzUŋmX-,1&X״iS3f2=f\\/2=*#BdJ`ʔ)㏵|r}7jԨBBB/{5JƍSJJu9sءj@Y`LzjԨڶm87ߔ !I222_k&::ZYYY?\g7Ǹʈ,CZx*IZdvءK}|M[SL$5iD֭[m^;t1&ܙ"m۶rrrҮ]&I:~N> IRPP^y?^^^^;vM>cFƲ?[s{Fwqw"pN<+WsuNNNС;VcԱcGu@jԾ}{ժUK^^^0`?n'88Xs٩b9뢣w^:uJGQttO]FRTTvޭd9RAAAԩ$W^z5l0>|X۶mIH b\@eŝf٣Ho^W^մiԫW/F~G֬Yˮ(pΟ?;wNjٲmۦ={Jx 988(,,L ;cQ6mҘ1c5jh BdPÆ Uzu}W_$ʕ+:x&NXftuIII6duż]@y`ҥKoEiѢEVԯ__7o.0.b: P54fMU/EEE8q:w-Z?SZrvޭh}:tE`Ƙ*#d2ٻndgg]YYYrssw9[?q &)--Mrvqwf]EL3fl٢}/У>'NaÆ󕟟o^Ζx[ BUøPre\(++Jqe9͍7N6m޽{o K2?Zl4e4ˤN!2!ɤkJHHP@@-III$uy D6իWkƍU%Irqqɓ'zjGkw}I&K.jٲŋ%I-[#Fzڹs,X\+,,"|gU.P~""DXE `U5{@116>^m3|o.`!2w(88X?L"OOO(&&ܞ?O[Խ{w>|X%GGG:tHTTT$OOOuɼʕ+osz (˗F:pΝYfiǎ\ϟז-[6mG/"wwwjJ #G`0oUNN$iϞ=ڵN P 1.!DMeeee@ӲeKjܸ"""Ԯ];ڵK_kת]vjܸ^uyxxhݺu~3ڗ„S͚5Ӿ}R z lQ̡CԻwovڳg$?o e˖˾:>ծ][5k4tIIR׮]o>jϞ= 6Q<{N8`;c\@eS|ٿw{WConS+UŲ`PQQrrr{ n.]ҥKow^k@qƶ8 @)a\\q !2,L6M͚5SRR.]d"KRnݴ|r;U_6mjժA7ᡖ-[ߖxyyy'ԦMiT" 1.b: X8xF)(P{Uzz*G Ҁ}v:uJK/d~²&Mg}kjϞ=:{ dǪjc: XHIIѱc4dm۶W^Q~~z)M0*[Z,KҥtR^Zݻw$-[L͚5SRR:udJNdX2e֬Yc^NKK&I{gTB&%T8=IꞞd]rE=z0yT^=%&&ڥFU>l @egܝ=CY DÇ6/XBou >>*((PffV9::ZQQQl] BTqnqdPaz뭷h4jÇc* ___yyyʕ+.BprrwL&?^ׯWBB,۶m+'''ڵKaaaǏ >FFc1.ܹ>.0.9BdX3g.\>@Zn"##o{{ռy󔜜si0`d2_֒%KΝ;kjܸqё/:UDddV^7VZy"www5JQQQƏ u#D5kjժUVΜ9#WW_nn3hРAΝ j Ќ3T9;;<ŋ%I-[#FHx 988(,,L ;cJ"Wtt^|brww}*44m&I ,տIҊ+ 6(<yyyTڵUfM۷\]]ɓ'ի<>>ڿΝ;999W_P"Ku999:qy9--M)))Tz4qD͙3G7V@@f̘!??? 0T`[[nXեKeeeiҥZzw.鷛Z5k$uI۷oWjjv)ooojJg_Ĩz85@:tZn֭[KԺuk͜9S4e?^>ڷomݺU,@)ʒ$yzzJuPz(IJLLC=d0egg￷aTm܉boQVV, .]z[ dn04k,͚5jP~iĉܹZh!IJOOWa[>گH~~٥vTUȰO?[n:u<<<%OOOeffPuQ͚5]& $22RGվ}Xqqq-P0,L IRPP9ϛرCnnnzox\(777;Lg /_V $Innn2 `H}{T"22RWƍUV-rqqF(yzzMǏWPP:u$Iի|A 6LsUzzOHF{U w"BztIRjt*))ܞ*ggg{Xx,___k͚5>o?*,,L]t>s6m$GGGiСw"BݵqF˒#F(..N(""U(L&-8;;kѢEZh>͛K4paaԩ:xe45m4={V֭z)͟?eBdXWիg^vvv￯ߎUBdXe2tIRݺue0\[z(&55Unnn}O}jرK{ ݫ~OA6lh1b ūwv 1~gҤIҞ={o?K.T!"U``y 4}ze˖F!2,|={vY5fؾ0@Л1QE[a: X_0`$$$KM6՘1ctE{U!2,̜9S .TJJJoVow"L޽b ڵK_g6ζxJYBRRնm[PF$I?բE %&&*11Ѽ`ЛoiHxx=Zl *!!A> SllJ*~m+}WGё#G,"߯:uĉVChEEEz J!2,ٻΜ9/j(hêjaNdܕl=3] QJJy4ɓSNi׮]߿5j;WU!2_ٳ.P:tH[V֭%IQQQjݺfΜ)GGG}wzԤI5Jm۶՗_~ɝ`GLgl&88X&jmlX vp'2*BdUV" {"㮙L&{Tw(_^˗/ruuUjVQQ-K`C܉ ??Xmܹ^xVaa֭L^^^={`+ȰCiԩڸq֯_/IrssO?W_}U~~~v"___-_\&I.\$խ[WΕ5BdXe0e2!r7k, KrppЬYn`Ќ3lP{#Dbbbd0EիWWLL-!D{***RaaWn^իU*{_~`І ,M&fΜ)___GT-@"D6@-Zs… QBBBgJ0n裏>}~. PA*44m&I ,տIҊ+ 6(<<ܖ;aaԨQjѢy@:uҟ'EFFUVoX!JKKSzzza^;*11ю@ƝȰ{n :Լzj=zTVR``˻XuM6?Pj$oooW~~y9;;l D~,!2,A 6]v2d$iњ7o^yڹsyZ5krj0󛶟z**b#qdg&;"5j(33StU%$$hZj)+jժǧ 85PVn(rvvˮP P"B6mduM~.]~OƏ>L ӛo5kaW^QHHڵk'ɤC׫sΥz̎;*>>^M6չsGyDGUZgPqĉ崴4Sĉ5g5nX1c,Bܵ\gѠAϝ;W .͟!!!JMMoWP~i;wN;vЕ+W4rH=ZzO(sL<#Dv~ᡮ]2335vXu!44-[cǎ_5jTl*C[nkÇW||L\=?[ÄiPBCC-_d2i>}/IZbaرcںu>>ڵky]vv8 IRPP233l_H;vo(777;ȰpIO$I*((ۃ.?wޱ[}*(%%EoKIIӧe04qD͙3G~9if͚w=zk}W7ngSaNdXpwwիW%InnnruuWZn C[n娨(I)S(77W>233k֭rvv6oj*7N> \aE:|ySNZx飢"W&MXa4MOFUCppL&vYfi֬YVxzzjeQpboўe:aaСzw/ѨXCՓP"ȑ#5rHrΝ>zU!D%(W w7tQm޼YN$5h@3@"PPȰ?d2 I***Rtt~i^+(^%6@F _b ;VǏWÆ e0t -\P/,X`RQbٻ(|dFJR7eT$ҴiСC41, QDnN>=|=zΝ;K߯9rDk֬1"T|ep94md29ڵk֯_~Iٳ'KZiH͞="2qlQԇOeb߯(tM;vJJJ 4f"\ĉ;w޽[[&خV$PD>#%%}Q||:vw}W'O^pgk׮:p1plWTTCQkJr7Fen |!&"2Y:xƍrLpp?cF h/>'ԦMtajȑ Ԙ1c ,f"q13FNRvtk˖-j׮ѡ@EիWg("`_SL<5.QDDEdK.QDDR|Hp MH}Poᐦr,gp"2%)a(" _h'E\ޟ.&{)\\b&2&?ԍ999ԩBBB/!'.E^X֬Y ͙3G;vP\\uICx9p)ƣ K/iʔ)4ize˖e˖z7 e "2 WUUb%%%9())IEEEF6rR|/փ~(22idd{x&./h}EEk~7 92: 1}}_B F_1\]h7pu9A"/}! 1}}_@ Ƒ<+14B F_b0 d w% ?Sns̱Kh4Z=ѣ5VWvF]I#/h4XLdm۶ TiiRY,gff*##ñ][[ӧOM62L|nי3get(jsD^ pƞ`0\տ(55U҅_6>88XN¼+ IDAT4f !/R9/8YYYYFj֬YС5k,ܹS˗/u]gtx/"'.E^xDOxGkٲZ۷/{"/`<u 0: p"2%("\ p"2%("\ p"2%("\ p"2%("\ p"2%("\ p"2%("\ p)_VǏW֭e2 euEEE) iD^ / V?~\:t0: )GՍ7ht / TS 6nZ҅_GƪP"y"/ZQDFw+i<i_%/ TS 6,~p"2%X`UWWFЬY3xyʑ4+G^PDnjP0Y,^ y"/\<"2?p/E_q0""B-[!v]?N<)Ij߾׮ "!/I27_8U䅫C^'QD@ئMi4Zh!I:y"""A^6pm ^05Zlip$ş'kG^ ׎O 0_Iz3~]=~f?3xEdK,s h$6n(T[sssŨF '.E^@cŋ>>??ܫ\ordggkڵڻwZhAbccci&~_kٲe풒M6M}:M0A ,Y03q6oެ#F(**J&I֭sUWW駟V޽ժU+EEEi:~xʒdrjݺuƀoڦM-[hÆ СCuYqSLщ'm… >\UUU*,,[o\͞=۷@FWٳSNNe}?vءYfiǎZvۧ￿?DV6Mo~ڶm񹹹V˖-5rH:uʋ/0ys駟֘1csU~~.]C;Й3g\-h:t-KTuu׬Y3vmڳge٣x} y+ݻwkNNd[cǎo_ky Wvze۵tzǦ衇R>}?Peeezw]rG;zo+//O}nz^,8p@dXTZZ4b|^("í.9 6; .aaaڵ㡱. ujM7|7o/±Z۶mS=.߽{wmݺiߖ-[<' vק~ٹs$}.۵kN<ssF]?_ E^bMdgM6W}J|X9r5f\^FҸqt-裏t_6vz7+(..NfΜi@O 'mҥ*//Wbbڷohk֬$5o\|nݺ??4j(u#00Pyyy TBB~_i7oQ "/v 8lܸQwue'L,_Oϔ(Iԩ&N,Iѣyf:uJڵ߮曯8 fU5w8OUh`sСCQHH4*Ϡh('H䅦p g'  yڑ ,g+p%8| %("րԺukEDD(55Uss9M64j(15lڴIiii0`Ο?gyFCշ~VZIx }zd6|P_|Oi%^ P'ksssb 2DZ|VZ[b u][l4)//$KU]]$ǘnݺ)::ZEEEuf©܇"20DmmOW^$ժ͛+,,ilddVkΖlv:xZd5knݺ)""B@֔Bvv ֭[+""BڷoӘs)--Mmڴu]QFiLII-[*""BO=Ο?[iynۍ͛ /X'N￯TGnל9soLҥK\'''G/Vꫯn***d6U^^k?x^}PoG?AVΝ;C)&&F!!!FӨw"?Ƣ µռ0l0=Z 3hoժU+IҴi(77WfY _|!IQ߾}eX /ĉ?~L{ /4|Vϑ4nD;{┓Sg… xb-[L[nUVsι<5k9shǎSrrN<8qz쩸8檤DŒr-_\/n_+VPaal"Ijʕ۷RRR4|䨪I+ hȑv˚9sxGo?uֹ}^zIV7o0Z1/_ p?ppu=zTSSsUH^a6C0zݻwkp?ht233U^^hG5:$.+//O}nF~Ţ*9/--bq)--b_]QD[\|]۶mxUHRppBCCƫ~f㮞vO?ULLSլY38۷O%%%JHH$%$$h׮]N/ްaBCCգG:8_ dt111X,*((P߾}%IںuMV1͛7WUPPTI~(==k0FǏ]vj޼L&a4ݮ*} P͍ ܆p|=/iժUZ~ZnXl6E2͚] ,P.]Yf)**Q {G#Gt3224az뭺/ٳ4iw(&&F'NǍQiٲµռtRIRbb+VhĉE) @FfSrr^{5@iڴiJHHPV4a͛7[ kyEd\۷뮻rlgddH&L\͘1CgϞԩSUVVo] qsAGyDfϞ-ժ}*??O͛7WttΟ?i, ~p|9/DŽ(''G999.tQ~;CH/4nq}4L7o^>|پt0ɤf͚YfF o`^;%("\ p"2%("ټyF(L&[Ωĉ2LNmذaE ("/:{┓r̰at Gӟ?dtHIIQJJJceX!Dnl٢l=ڿ$駟~Ҏ;TYYipt8۸q"""iӦԩSlpjǪjzgxb=zTCW^18JiذazUPP?ڴiRRRTSSlfGС#Gُ͚5KyyyZt' C=!F_{Vjjm6mܸ1*//w0Ed??IӦMԩS~YÀ27tڶmPp~ɓݻ@O^s1:uJ۷7:hӡCݻe_|Ν;{1"@SWYY4СCڹssjԨQX,:xf̘Ν;+99ic&{G믫ȱd2Ix ?~Q۷_~ׯ$)##CٳZvɓ'_94]Dc>l٢!C{2Lz'ti;vLwx 4!N/z>ȋ3X͕+V覛nRndԧOL&~_iݺuogi-6oެ#F(**J&I֭s8qL&S6lӘӧOkر UXX&OJoE&nO?79sԩe&IiiiuͽllHH]gϞU\\rrr\6lN8hӟǎoF6lP^^6oެSz:t3Ǟ}Y>tbbbtA=3JIIQQQeZtLPPeZ] 3XPPnEرc:uڷo/„2O?Twy>/~ؽޫ;vhÆ #<5d=+4Jh׮]<館tr!ܹS ܹs5j(Y,}~b(..N˗/7:L>"{Wo:u$ioVgΜѩS /<#FH5k&I$f=cz fΟ?/I U˖-uQG֭eZ m޼Y#FPTTL&֭[o5{lo^-ZPRRoP"_ի+tR}w:z^uuMٳg:.\ŋkٲeںuZjd;wˑ. 2:xί~+-[L6M;w-_ Д(%%>ݮ_~Y3g< Izui ("I&iҤIo_WjСDCj*))ɱl6+>>^EEE ݻw?Ç%I:uRJJ dONddzll튊 MEd?f_Z-ݮ K`*33ScǎyG ܹs/cO?~mM6M{ѹsdٴgV\3f&$"I*--u_ZZKffѣ~lʕ7n,YX)00PرcrJ@#ŢǾ mݺU . VhhSEd?V]]4hΟ?kfeed29nݺ{{ｧnݺ)$$D{և~֘R;wΝ;%]xΝ;URR"ɤӧkڵKǏWTTRSS p6oެ#F(**J&I֭s={ڷo-Z())Iwsi;V ɓUYYגGСC~ݞ={ĉ矻[XX1chTjn `۷_~ׯ$)##Cٳ%I3f?SjT~~BBB p Ξ=8ٿpB-^X˖-֭[ժU+%''ܹs1cǎ7| 6(//O7oԩSu b=?6|=zΝ;K߯9rDk֬ӧ E wK+6lz)G6lВ%Klٲ_$&&n7L7o͛Ũ:v^~e͜9S<$Vdd֭[ѣGkϞ=׶mt뭷J^}Uw}z{Ǻw.Iڵk֯_w|=.;]wRHH:)##i_rre_u?:$ժ$>٬xi***RXX,IIII ֭[5rH#BIfϞ-k+77W:qΝ;CwV֭/oZ/22RV5l6l6c}7.>Yj*""©?((H|^(",_үGر}]M<-ܹsr. /փGk׮:p@ENJKK]S933SvQ .>YbɓNϟӧ1UYYjܸqu'$$@ӧOw۰a\388XnN!r7F`Y,o߾.,;uVM6M҅ eee*..V%I~jkk|^("í|I1B;v5gj̘1nPvv$鷿NիWkhmx\CXB$gTVV:}#СCڹsӧkҥbbb4k,EEE)55U҅6LSLѲeT]]t=ZQQQFMEdձc4f:uJڵ߮-[]vsAiժU9syuE֭S^D4 /}vu]파 I҄ 3fٳ:ut+??_!!!cy{Q@@Fŋ{^h("íV^]oƍ/C顇PDX~9JLLnwo24o<͛7pZ9x%f"&e pEHb&2D9f3.1` !K@ p"2%XMBkK=?K Ed@ KP@cD^(d7uEdhh'h El \&\b&2a*c&2YYY2LN[nFM3Oٳ>vP>O~*bxPY~EEE馛nرcURRbtHФ1nkj޽jѢ ?uyLnn&M/88XΝt\ĉ;w޽[[&خVKH .,͝;i_ll+I:w?CWfSrr^{5EFF5%G\ ڴi4`?^<oVZry\hh6LcRRRݧOǫcǎzw5y:ξh<[ '|{OfYz_j7I wUDD5dǙL&ֿ\&,,L]vՁ\TFFcB:tFx7p~yy/_UV$XBݻwז-[4p@o @EU^~TǎU[[[nE=z>RԸq\ Vpp ~HHhZIIIݺuStt("EXS[[ӧkիqz7~z\R4h;Vxͦ O>6mڤÇP#GT``ƌcth~~~.]C;Й3gdZռys9)|^ IKKݻ;.!!A A{z5Ƴ%cǎi̘1:uڵko][lQv ߢEk:' /ԇ"2<"==]yyyڼynƫ:Yfׯ/Yի%t-{WUUU*++s\ZZZTQD[v=zqF\9jjjk.w}u%.] j֬ 4j(IҾ}TRRMƟap4ZJׯW֭kfƏnAْyiܹ /ȑ#z OjĈر?9s87͚s]93W{x|EdK.QDDEdK.QDDEdK.QDDEdK.QDDEdK.QD襁VgΜQNd2 e7(,,L>>[1y@^kX9sFFԩS[y漀ơSN| j*<< /\Ƣ-i|)ͷz\˛O.QDDk" WSSjhڶm+___fE^ y#/PDFs=B={V۶mSbbnkɒ%ڰa5|p[Nz20jbURRrCiQeXxV8yq hѬ4p@=c4iuVK/LEDDhѢE_|!???"FsԻX9MS\B:%v].\PYY$)44k('HoD^1x W'yƐМ("Y%$$(!!>ݮ5kh…8q$iӦM 5ydw jjj_;wlt8-F%Ieee V5yq Z+B\("0ǏWIIƎh,"l6l6ǶjmX&.*W4СE:##Cn `o {ٳu޽[7n͘1CgϞuUV9jjj4~x]tIWff222xbw_4 o hXbQvv $5k0RuNn;9dee9mgdd(88X9rc\N"y_hϞ= ѠA3'ҥKծ]fc0ͪREEE***tazEEE:yL&͛˗>~XaaaJLL48rMCo޼Y]t߮T]pї8=G%..NVU{Ѽ (IRJJ-X@s̙35tPUVV*++K~~~F l;w#F.gdd[nС^;w΍9a7ooݱ믿?Pz_RRR烸fj:5p'Z#hV׵ IWl2ŋڳgzml-,X@o233'nS\\Ο?|M>]***ѣ|r4rBfϞ>L[nu?sLi2e6mڤm۶JOOW@@QDUUUiݺuz甐kÆ j߾^{ƿ⋊ׂ Իwo͝;WqqqDhj䄆%''kǎu-;6::Zt1IW]׃%55Uvԩz ZQDZÇwk۶N9rGqQ|ULLL h~vm6}h5W*{sa9޽[߿0wj.Vm̞=[[lџ'uɱq@@ڷobmٲEw}:wO?TOr쫮կ_?;;p@ h~֭[ *44|MIRvg7N}s%%%wqW;v쐯bbbCֲeˌ,y3A;vԬY4|[nZj.\ӧСCNΝÇk՚8qvڥ,4%rkvpݻt];wlYZ1FXr4uTq:vvڥoÆ ӆ /jzpB4rZF&{C?j* @<4xzO=XV4a40ŋuqEDDpZ;>yZr$X9 SP?ރVbi@ڐLd60ad2}vGyD&ɩ-@bMdFUUU8p{1M41ڸqcl6+5kΝ;gtH՘ /+׮]n@μ@N !x%ܠX͝;W ,PPP,.]///׏cuU뮻t!IREE|}}UPP IUPP x믿p^ kQDjjjh"EDD}ٳyzoweffcǎתUl2޽[t}L uwh̘1:4h H>,ɤQee$i޽5jQhZ6ag}V֭ӯk9rD>VZ_~H-YDz?!C(;;[?2dzիW+00Po+3~)COsK!, hmx ~M8QǏ$Co>c#"##CCCUVVCR;wvX4j(k޽{5n8Y,(22RǎSll. kPD~W__W[Ҿ}Զm[mɤZUVV*441sZ#Go'|\Wbʕ+5p@W^ @!/E^@kC{ꩧdZշo_F+VД)SodV]@M6ѣGc_j۶`ڱc@+B^\5a?ڼyl٢O>DZz233G㉤<رcD:q߯_',KW9ۼy `PP7|/Њxs^HOOСCթS'+11QGusE͞=[;wM7ݤ$:9yƏ:(88X˗y)d9/e ͟?_O=&Ohԩz'^TUTT8کS1f2sN9R>zɓ'7j(8e{>@ya޽={8ݻwZƍSUUcO 2:$ '=wyGF` '.\Д)S( Df:|a_VvvV^QDp'?SO=]cǎwjͱw=Cܹڷo`=8YxVX}{2d[F?ÇkzԵkW}W曛GN֯_kqDg}Vڸqc_DD[ p\=s*,,ٳgm6%&&:v,Y 6\Ç׺uԫW/rpr%?mdI!CSppam6VS UUU8p֮][gUK/iWǎ/9Rw &(,,L&I۷owGd2Z||Әkʔ)W``OJw^Ed{G_zοoYfvҬY4w\eff9>==]x˗{ｮnk͚5Zp&NHmڴIgΜ| p(I:{NSLݻwkǎ̙3;toXN,Y_?4}tuM׍ jsjȐ!կ~%Ig}kڴi׍OMMUJJcjRHV*))رc>>ϯhd2Ln=uQ}_ս{whCZxM4I*..O?W%%% vzM6mďEd8YtO?~_W_իXƹz;siiBCCKKK5h h]hTϞ=1c4@zWEEE.]QCj۶mz7tgњ5k4eʔf=/DDDb(;;۱j*??_111Fp[oU]tѱc$]qi˗uy(KW~\wj໡(>>^:tPΝw^I?M8Q999M~{Gŋu͘10^eeTTT$􊊊tIL&͛7O˗/ןg>|X?”hpvi;wq7JLLUXXVF W '׈#W_顇RmmK.{ԧ@QQQ$(**J/$-X@s̙35tPUVV*++K~~~F h~8u8qBٚ8qn6Ix͘1C>#%''k 3: 'O?oo[ѣG+33Ӡ-]llv~ɤe˖iٲen  4zhuMuO?UffqgqZ~JNN֘1c㣤$KnEd89xe6UYYy]z R?ڵci˖-Mhm:-a릛nrcDDN zξ*mܸQFrsTBNTPPޓ$:tHo5x`}ZhQpDhܹSf?,I.IٳvܩH#CFqKGUQQ+֪gϞS9׍+//ջwow@m۶)--Md2+W^sl``6m"24sLs=;l2%$$81Lرzr*44T_~ 68*"25x^nѮ]ԦMu] d2);; 0Ed/gU[[خdj5Ed/S6FNv{3- h=qJ>F}O?>#CοrJL&͛7ϐpFNFw9ru_:x[}A+t4"2*++֭[uwjݺu6lz쩧~ZEEErJM2E6l7,>0ad2}v~ݮŋ+44T۷رcW_99L"j骬teQDFڷoOokz4x`۷9{l?^cǎmcp* 8Pk׮ժUz饗~zcǎŋcL?\w֎;3gzWǎh„ /f|W[n'|ͰllmڤPgnך5kpBM8Qi&h/ 0Ҁ+EdQkϞ=6m<?bUפ*%%űmZ%Edջwo;v,ƨPDWk1 QYYbM:PkQD܄B64&Lݻ̙3Zd|}}܈f[ BנxӧOйsԵkW1BP׮]_"2[n5: xgH8ܥ^"2+Rnŏ{Z x.("p:t:u`%&&ѣF@ w^͞=[C˗Okܸq/ԱcGUbMf`ҥJKKsקO}嗒/?n*ͦ87QHH("pYYYN VaaFiPTgv63'лﾫ? Prr&M>ȈPZq**QTgffsl[ViFz״eu]7_~:p PZ R[[yiG wsW_}0z뭚2eN<)I*,,TuuƎ۷o_uMyyy.gdZn 2{l}gڷo1JIIql[V 45P>}tY??gDڵS``kBBBTRR׭ ڱcrssu-g6e6搐wddս{wPuL:ia~ - k,gv%''k۶matH ޽{رcX,t˝ƔֹUfYN |7DfϞ-[O:u5- ѳJWTqqNm۶VRR$ѣ:ybbb a.>auIbccoܸQ</4au$ IDAT]gΜђ%Kx@>}RRR$͙3G1116lѡWC?HhVa8nt N>x@ΝS׮]5b8p@]v$ QRRl6>%nonZo֮]k׺)"0KSQDZ ctEY` x:f"\ p, O;Ed[- Ed"p x%f"ᥘ p,xf"\b&2* 5Þp %(`=xkתGStt>cCycQDGx7%KO>2C9p-ƣ kƌzGտ_^:t~;C9p-ƣ ]tI;vcƎ<#9p-p?TSS!!!/od$ `5c[F`=! 1}~O7w W? n4'Hz~O{B Fb7ye-1{B Fb0Cs]}Nϟo;d$FN:宏&u9n'/h4ڷiF]Zj^q uE*--u_ZZ*rT8kkkuyuY&Of7(,,PFsD^ p`0\v4x`egg+11Qҕ/zJNNnllvX%0:Fќ !\%wҥKעE.٬EHn&9p-c&2<߯Z/VII h k0nqsQDDEdK.QDDEdK.QDDEdK.QDDEdK.QDDEdK.QDDEdK.QD襁VgΜQNd2 e7(,,L>>[1y@^kX9sFFԩS[y漀ơSN| j*<< /F\=s*,,ٳgm6%&&:yeff:&..NYYYkΜ9zw㣤$⋺馛u^@cQDFw4u p=#BUU{L&MsL||6n6NSLٳg{nUWWG̙3e˖oy/q("\BBc6eX;r䈲tA 2D/ի16999 V>}4k,;wї@GYƎ+E:##Cn `rZT,gf{N:{mۦDGnג%KakZnze`wz+ǻ|μ@N 1fUUUjڵuZJ/֯_|uQqqqx#PhV Z|v֬Y jĉԦMtm߾݀h۳l;w#F.gdd[nС^;w΍9p-Z#0URRc:(::ZyyyF [`~meffO>mݦ8?>}UTTѣGkD h hX)))$8 qff9Vk.TUUiݺuPBB$iÆ ڽ{^{5 :i/x-X@Իwo߿_YYYnд kZ1-Nzz-<<xbUWWk}m۶՝wީ#G\7ȑ#vq9p-Z+0b$:/--u%55UvԩffaY,egg;YVlSwٳڵk>ȱZT߯_?;;p@ xKo 'E^@kŚhV:vc***RPPuyiի"""h")11~;vԬY4|٪UtM>]r?w\ >\Wĉk.8Vy3Ѭ (IRJJxbIWX:g͜9SCUeegdР+W*))ISNwܡcǎi׮]曯;l0mذA/_ .4 j@s 'E^@kdv ժUTTԻ{,hh`/㊈T{g"AKPN ކx -cDNhu G^@s`&2%("Մ &ɤ۷;?#2LN->>ޠhEdFUUU8p֮]rL||Ξ=ho#6FGBBc6eX!D%''Gӧf͚sx5f"I&)""Bz駕<&ضZ Ed1&O={*''Gcƌ5JKKsWuXx[oU]tѱc\IMMUEE:uʍ@LdN>s)44,ƨPDxn>_{*++f?~\EEE RPPҔ$Ţb-X@v ܙ [0Ed6=zc;%%E4m4[N~233U^^07N< 3@ ܹs`bhҥr?V׮]ﯻK$UTTWyxU񻳐tXY#(,:Ȧ #QADeQ0(((*( @"js?K=IT&\W]}$ݜ\+WN[ Zj~cǎ2ϟ(\R?륗^RJl gD.NG.$b 99Yڴi|I%%%飏>$]wuJOO|-[Eܹ~n5kLk׮$ڵK.K۶mSvv$iݺuСo  ӑ (iXD/hҤƏ:uhРAjٲVZ 6/ŋղeKթSGO=dIyg&(\vJ5h@6l/I|3n\@I5 4irJOO׎;<:_ѺuԵkW%$$hڵjҤ߯;[rp:r% >].rssʕ+{8]\\$/o[j4iOiӦRԩoPL4,"PZh#G(,,L&MSxxׯ+{=>%8DuEmڴQ޽g};,Ky9{W/˕+ 7E! tW,":ǣqƩf͚҅^G}Tۭr|r]~2d֭~PJuAZf;v,08 ӑ 8_ +ulҤIzgF/Ԑ!C뮻:ʒVffbcc1U&dc7ĉ:xj֬HW4w2A" ב gב pDu}zꥫZ_]_|p9 X׶m[ZJ~$iǎڰaza3 |Aeee~ ?NNrrr*t8ֽzWki֭JNNSO=Bqƺ馛4j(M<ǎL'1gJ2qs$1gpuǏWH-tEDD5 w 3P $":rN`S?WFi۶mzg4tPۭpPhh┞.I*]\.1FǏWzzj%(6ŸG.(ȅ?\XDu>ƍ;CRzGla }as;(Iȅ\PR  ' bbb4}tM>v+rrʪXN3gjڴiխ[78q»W_飏>{ｧkLd@ѣzQsM>]?z%IZ`*Ue˖믵b m޼Y-[w?JO=T`Ǚ:x9.]xnZj7J6nܨ8$uE!!!ڴi{ q&29"ITxJ9rD+Vy>,,LʕSxgeeW-D'OnժU=~ IJKKOKK>tO:_~ŻOaƎLZ|XDU͚5UVyDzi&iFԦMeddh˖-}V^\jժȯX o&2eggkU\9U^]s{1թSG5kԸqTJ[ԠAu]zfϞ'OjĈ߿Tb ( (v_~:u}ʕkD{O`> M6iݒ?C[>vZz7onM~c͚5۷kݻwW_ݻ'Z? GQbbeԲeK 0@ 6ԭުM6k_ Ȑ$:uJk׮Unݼ(33V{k"G-4guI~7TR%'WnԲeKctK/>tRkbEdhٲݫ>LqqqС q>cJ6իWqqqլY3KwׯWϞ=UJ\.-[ycyU\YQQQҥg["2pI}ڶmnZ`+nҍ7ި+**J7֗_~Yuv;vLM6լY }'̙35{lmڴI֭N8N|dddh̘1zWx#%Sl5WkN:u|x۷Oe˖-УGѣ1>}~aK`UTI˖-S* 7|jժn5Ljժi޼yޱ5k:^X<#GK.1ۭVZiƍE."(''8++^ ~.M6o5}]uM]w֭[ .@wqnVȑ#J*WT\a&O':3 ˗WڵZ /:uhʕu]w)99srrرcRSSm% 1|p-ZH~-ZhҤIj޼[oUg.tɓ'v{jժWs$Iiii>iii X P||7N999jٲn&UZU۷oլ\6l3֠A[?vX=8++d(j֬ZJ͚5wߴinv@b>:իWkھ}{\.yc~jԨQ(<۷\r^=cSj֬qƩJ*ݻŮ C֭[5vXjJnFR۶m5i$]/K/饗^r6Ku8%uرc>|222Ծ}{XBZ"2|lذA<_r%hҥ;vTfMM>][ر1E>r$?v8#!!Aʕ{ݿo,v,{Νlۭ 'NPxxj׮믿^ժUShh>.KF!b>ƌs=W>,"Ed8xF[@XDF:+==]SժUx)]:J c4zhլYSѣJg}rEd:uf̘1c裏>1V߾}[oY? sѠA4i$5k֬M4 c>RSSնm"VVV;`d IDAT`QbE-[Tzu?v&o߾={;$}ᇚ?:[3cĉ\5kAriʔ)j߾z&M衇&?a>n>s:t"##n:eddhOTtim0 DEE?lq&2|\qZjUϯYFW\q;`vZ|zz֭[ǎ,P*+&&ƏMpL ( %''+99Ӝ9s 엑;wꪫg{,rO?駟~$oѣGnMsu'xB.Ksc5;Dnv~풤5kjƌk͛/I&~ p,"IR%{Ӽ_\5p@͙3G=X~mA.11Q.KJ*}|6X;uWK.,"E +r)<#EFFߙ T? Yj߾>sKMO999֭y]|_(aaageq T -:󑑑5kf͚姎@aXDe߾}R"##զMMϟ+j˖--u@Bl7̼w˕+g{GڵE]T>999>W{@a;ݻaÆ" c"`D.^R!!!&--ѣGMHHH=U~2?S2˗7w63gӤIx̛oiׯx}cILL4~1Ƙ?ř+WaÆ+/==C;=خuLd?{=خr\ a[L!KRNNJ*.xvv>ST?ncud1Fx|rUXѱ+jvءr9V7P㊉$}۷BBBԺukח#GZj{O_vDj/=>!f.)p2A 9}<" A~`{l? :3gΔ$\.͝;Weʔ>x~zկ_߱G7nJ.SӦMj֬c%yr\r\ܹs(x<:xwX8ouxriĉ՗epCǣlvm9ڵkkٲeӧV\QFIx})gjժiŊz$Xx<ֶ}<?`sv&H3A?l?`c"IEdiӦI ٳg+44\Rٳg;V۶mvyTRjڴƌX}Iݻ$i֭OA~fctW護yTRQTX}I>}1:t&Ns6EiƱ0#nШQԹsgyח}nP:u?GԮ]ڶwv.1v&H `g\L$1\3LQ%\NolٲV2D3f̰arr﯈+U^Џ˺uԮ];3,)ȑ#:|6m~rݪWO<3f(55U7|дi[nl~9!\ cLρd7Ȅ}o߾'(0>es:^{f֬Y_xӣGc%\b/^\`2^zԩcV^]`|ڵnݺ7Ƙƍ|`4ix}s`ngwܹs1Ɯ:uʴmָ\.m֬Yx+3tPݧ_3`;L ؟۹@& \ !!!|![֯_x=~zoڴI:u*0ޱcGmڴghѢx͵g매f͚kԨKҾ}԰aoksVVw_vg^ϊK/VӦM%I׿k޽5j8^СCJNNѣGձcGկ__SLё#G<3A 3A 3A"2!P D. y 7CIEpeee9^?''GN*0~IחVZ>u*V;w*11g|ǎ*_%v+ݞu7\ohΝj۶=矕 )c]w֭ԧOGiiiZp5n8u]Æ SϞ=?C \ \ RdB` \  @~7n7xE }]zz饗 Ϟ=[_|%k׮;v233cz衇tW:^뮻fy<y<^Zw}x}Iի8p;~{ｺkok֬YUV%KhmÆ JIIq]u--cbbbn4|p-\UTI{ъ+Ǐ zi߾ڴiڵKօ^k`b;$`;$`;2^;rA"ȅ<\Ew5aaafРAff榛n2aaafҥ߰a4]v0a0aLddY~14j2nttřzꙔ믿޸\.nMhh2dq1ddd֭[0hMXXԩ_o{{x30a9vƏon_^9q1Ƙ_~ٴn/=9rL:4lDFF>cLvvMK0 ۙ`cL02!`@.C.C.C.\c{!פI}vEEEI&?~:t۷oדO>;vx;VuK})﮺O Pxxzo7n5jw>g}_[sb )SF۷$͚5KsQÆ 5k,m:uT})<M@ o. dPr\ "2ZFF,YNcƌQruVUTI\p8yUbE}^5jx}I?_|Q}6nܨ5jhڴiU?~\#GTrrwkժ#G .Ѓ>h}I?ٳg+--øq㔘aÆ9Z)SFwVbb&Lݻwkɒ%ںu*766V[nUڵ}߯-[*##Ñ3g3gq߻ˑ s EFFv&Hsf&H3A?sLE&E. @.s ;vxSvmf8`1馛vZetbJ*?ydӯ_?c?o*T`{1a޼ycǎ׿뮻_l>l2ӬY3cĉMZ… MTTEVlٲ櫯2Ӯ];/c9x𠉊r1ƚ[/M2eh=sQ[͚5!1IIIJ*&44{s:^3`;L ؟۹@& \ !!L.ưչsgs}c)S@駟5j8^u駟.PӦM .p14h{c{صk)_Wn6nX}LLL1 /4qkx}sгgOӭ[7d͏?h1fʕN:7Ƙo3N:ut/=A v3`;L ؟۹@& \ !!@a⣀e7o֋/X` .Gwڥ^{xŊuQKy7)h޼y;v?*VX`رc~ءC |4Jɓ'o{{9qZd^xG3?u4e]~W.2I'|,^/=ض`Kܹn6xӦMw^ۙ ۙ ?&2L  """U`oU||taլYg|۶m~Z͚5}_b4hx-[ȑ#%"hܹgi@qjذ>sdɒB_47sPzu{ƧMx| 6Ν;sy8=h 1]ѣygyƱ>J \ cLρ\ Ȅ@A.  \@`AkQRR|MIyax@s~hr\է~1ch#j IDATРAח;ԉ'd_|_]'Oܹs?i$C{ѩS4c ٳG}֭[x}IzG4x`:tHz7hX*n6 ++K?I~NR&MZmvN @J \ \  dB` \ iddd.]8jUf_ncnf\. 7!!!oޓ.\hj׮m\.q\ .ons% ;w1Ƭ_tǛ(Ӯ];rJ4c1.˄efM6-X|'~\S_wٲev'x”.]L:r-TR?,z(v&`; 3rL \ ȅsG.scl 6hΝV-ԥKOIIݻ͛N:~.z[(Y֭[v),,g/t~z-tM8pjϞ=U{9-_\˗/ws۷VZ?%%%iǎ#<]{-v&H dBp \  \C.r!Pұ GJ.wW5j( 4H111ڱcjժm۶G~y͹8?1F ӑ gF& pLdNG.*G3g3gq2eʨQFjժU=z}QEGGBZ[-ZЪUTlY5oC=jժKeêXXrN:zթSb/Iʕӷ~ *lٲ4SLQ&M9عs9[\|7 nedd8^?L:Uw_qǣoQJ>ۙ ۙ ?&R. dM h TM*22wQzzFSKm۶yz hƌZ|$K޽{UVo,6^{5k֜etk޽R_=$M>=,_\C і-[9h֬\.A%S,5$!!AWbb .MSNUr4l0Q{nl3A 3AL 9\ rA"$r!А . >CSBk7olbcc7&&ꧥ/Z}cIII1|äILÆ nbbb'|b.\h̙3ù*S9p@/řŋc9yӧiР9|p_c; 3`;)ٹ@& r\0\p@. ppMd ~wK?>P^oFT^7̔vҥK_K6l^z^_~ e˖~(sp&W_}Νʕ+6hҤI dB rܑ J>DFzuwkU6m$I,YiӦ;t#Խ{wedd(99YՓw!C(66V+Vp~ q@*Uk[ڵkB ޱ~3 \ cL"Ȅ\ $r |,"#hUZU>F3>k,M4Ir~||fΜ9r=h}Iʕ+վ}{O>Dݻwױc/R5߽{.>S}Ʒl٢.)'۞sQ/ +66Hz}m9Q>}zk˥Hծ][7p?aȐ!y. g;$`;$DۙC@&"r\ȯO.r\@8,##Cݻw/0޵kW=?ydҺu)KR Z}IuUZZZD]%Zj;`J*9v{_^NNfϞO?Tc,[Lqqq%I[nUFFv7xCSLѪUԮ]bW^}jٲ(>3A 3AL 9L @.H-ej= 0O>dS1b5jT{q7Ƙ_|t玮6]v5gvfffw{MFŋMjjIMM5/676#۲e̥^j6oۼyiݺYt#5m΅?|&_})]c_0~x<1cFaƎkrssMv\X9d3`# cL0&\2!,r\0\3r%@P9sYYYzꩧԮ];}駺{{ѣG{|)͟?_իWW֭%I6mRJJ g}KR}>o>z꒤EDDN:ںuk oNӧ?x<^_ʖ-ñct)ݍ_~YYjj>Nx{n5mԑOUn]oVm۶ѣGk.]veps抗LLmgd Ȅ3#@.H_E. (ʴi|-[V{ў={cqqqzWyam69p$B Pbwގ}sf%iV;v옢ϺC=r9éS4qD͜9Sْm6rH?^ uEa~{-p޽摑^ /ۙ ۙ ?&v. \ $rῑ 3 )SF_Z)rVRRϝ'L޽{^pKz!]r%͛7kҤI4c ͝;W׆ (]isL8wd  Ȁ ~<~GIywa˖-%I5*pa'edd_?tP[ǣe˖p5(44/me4|-_\:t Hy7Xhz3|r 0@xOSZZ$RJ9rx*%%E!!!J0ۙ  ۙ w. Lr\ r7CpKNN6]t0qf~xĉMll 1!!!v$ ;)--tɸ\.SlYSlYrW\ayfS\9s>}>}U˛-[8^cgԩcJ.m7on7onJ.mիgx@cIOO7O?iܸ 3W_}y뭷ɓ'oS`|Ϟ=B o71 D,Y63`;L ؟|rL8wdr\r!`"2O?mJ.m~;wyw}tgq>h?ovavaf͚eC=x}cM˖-}B2-[4w~7ɓf.s1tޱGݻ0m̙&""¸\.oƍg;X''NxN8ah&LXͻ^t3`;L ؟3ȄsG&8\ !B.&rNcA+11$''?ILLt~ʕ;S`|ٲeJ*7ƘX_ߴiqݎ׏4_uDEE9^cJ.mvY`|&::0s3eӠASti3p@zj`ӨQ#sWk(򷘘SBӹsgӹsgSBkSurs7ʕ+P>?ddd{{ cL01v&c3Ȅrrtپ`Çնmm۶Ç/~ׯ_~u~llRRR Cjjbbb/I gggTR׷=o͛+WaÆ;t7*..λO۶mՠAboׯjժk曕qƩr~ܹs3oW^ 6o3ۙ ۙ ?&2?ȄA.  x^liԨy ?梋.r^jFY`|ĈUV7Ƙk\~СCޱt#G4UV5-2)))&%%żjժv1tMQF?7&77lܸ\tEf׷=fᅞe%beʔ1۶m{'|Ҕ.]/_OOOyE"55c]jprr2aooO/Qtڕ,V.ӧF_l'l'U1Yd{Nj *䋬^`"xa=y g-Г5k`ʔ)pqq?3}W_Epp0ׯ|2RRR`4(6ld;NߘB xNj *䋨}aÆhݺ5 -- & ;vХ,書p7o3g<<d{b Ȁ^`",]0 S}RSD& & ~~~W???dffbΜ9hӦ ڴis"33ZҥqyyyHMMEjj*rssqRCYʕ+1tPs[~N"W<|PwתU w-~=F!жmr?ߑ@d:PcL(" * {A #Td0:qΝ o+@,==Wz~IlllpEZ_~(((o<^L4.X@`%ANNN8t6mjў;֭[R* ۻҿ;,, Gǎ?#Fv᫯L0Lxw_ŋrJ-Z+N䌉*9`/Tv#%셗^`/kDf ''ED4M@ܦMhG>_?kO?_ţA9rHcn3YaÆ!66+{=ￏ ԬY;?  C`` W[>z(V\),ڑ@-/v gLT {A {B ȌUQ|eYH/" A4 5j< u,YYYYpwwGƍ+X|*$W8Aq㑘_깹 C;-dM6!++ NI&Bs;w3gμs Qa,P 83&;({A>E` {yDfnݺIoܸ" ٕ>}h'yu_e"{?~>>!S@ /@z" j!{ʆX#<'2cܺu 櫉ZBxxng@Ǐ-'<{2dL&,8p`oV*{=zTh;MӰk׮J3QQQpqq]JD'Nx9u`'^xu`/Idj9|0[[[ÇҕP+gϞocǎY{VtW?~+WТQFBsq!ԭ[ע֭[h۶-Ξ=+$woT+9s`prrҡ*5qttѣG,bvd93&N8^x1*{ Df]I&XryO?ٳg"4ނ 元dff"<<,Z`0ҥKV||2<<vBa'^PPu`/09PF L6 ۷~ڵ ...0 0 ҥ ̙qUxU|5j7tj;vx$$KIDAT;w.B}0b$&&bРAر֬Y ={'wwwkaN{AuDd{{ U{a30V+رTUx={i׮]DDEԩSdG44 }h4RfӥOOOzKGGGڻwp?>kTF ۷/mذr uޝ ۓK6l@f͚ԺukZxq㬬,Y.LW^]FF%'N ?&@$;^( {AE hx: 걳Lԩk^rry%Yl{$%%a׿S"11QZmׯץ?dC… ={6ׯ/eaÇwž}СC2qwwGddΕ˭[p2W (ͪP ^@"r ;u`/E" ^x{QHv #B̜9ǽ{>sEΝ{ş'c֬YBGD 4;vnzz:`0^BkqvvF^^<<<}v̜9E 0CѢE A-C9/բpuu ɓ'#<<g϶dYv *9?&v ; ^ *^`A4 #777ڴiS7|GGGڻwhF/_6/׬KyƎK77x֭Kw%" <_d;vPN.R7Y .dҥ""9^P DDd{NP{Ad| *Ȍrƍ2hBA4h9%Aze OOOa޼y\xcƌѭBdee`ppl $$гgRi&~LL g`ř@AARRRtC6}Ç-E9^P 1@>vs j^`/^`/ȆωX- 4˩:tm6`ٲehܸЬJ߾}777٥T:{ｇsΡp($AT޽ϻuVy;w 63OOODEE!00PX 2ټyիWÇ^C͚5-ׯYT/ԠyNP Ba/{z^`TO"3Vݻѷo_4jAAA#//[nE׮]_zFJJ J@VW>8d2I&Qi͚5_|77Rsծ][h~Eg,_X~=4h^^^ҥK畤Go+[ ]G?P+T/d{8^`';A? {Q΂Zu놌 ,]O 8cƌC… ={6ׯz*1`hVUǣt邍7 ;hѤI!߯:6la#GѣG۷ocغu~ C!v^{ڝ;Q^{Q`fѣGeQ!L䟔DDwܡ;wPRRі-[h޽ԪU+ "zA۶m>hӦ ^TFjj*կ_^D\\jՊF#Fjժ\R|٬^K?z?LxAxD$ U D v {^U;͛ǩS-[QN-ZÇb t֫W/L&5 'N… .L<.]*s~)a*p̙2]6nݺK 9,XE'N\DEERLZ߽{ÇGXXʬ N |/X^`'T | a/0 ,6bHooM5"GGGڽ{;vPN(99]Fo߶x&;V===L&dkk[EhVe0 ˋRW&__J+ ZvmkRݺuuA6ѕ+WJ=z%Td}vQ;d{8zNPBa/{^`TDfcbȐ!5ZPP1c`ر8vϮ;5L߮];L: Wg L6 :tl2a5ȑ#1~xZ _~1e̘1C}K i4 zBϵ\PPXE @ U`/{A>yȌՒכ &MBBB Χ~*䱽x 646mܻw}YgѸq m׷o_MX-!OB <@pp0jժ)S ""bذał ,WXP]jŀGE>}`oooh4 NDG|/T' v**{ {Q2 c%tSN5ElܸsŁ$U7o.]4L&4i^^^Bk(,,Dbb"222͛7G޽a0ZooJ^?~,ܻw-[8@MDD_?"77aaasϕVΝ;qZ|j*!  UFuAFF\\\ M^7tz턢T*QUȓ/5`/Xx^P}a/0*s"3Vrrr BȮƏ+VSN^zd2aԨQ8q.\p!_|о}{+>` {kFqX%/0F j {.{Q>X]8HB&;;qY@ӦMq5!˖-W_}aÆ >P|,^ea*2|BN(Yl/0 2a/Xx^Pd^`/0'%ÇsAK]1^j^O>3KҮ];L: ;^iӦCLJȕǏ-[#@FD 4;vʫaaaѣ#D9# v;^`/F~ {(1 S.]۸qcڷo_gӧyd4LJ|||h4R-̙3DDRBBiӦQTTlfϞM7oެ*H{]TFAM6%MӨaÆJ+W ٥1%"^Pa<*^"5A&vBU^ b/^`ԁc jɄSNˢٳhٲ%+5E"11͛w0 ³Ǐ5kִ|kؼys횦d2I&*вeK[N U . %%wݻ777?^vip ^= NL a'T ba/P#??ް+uɍ7$U~)~9r֭O`0j! G z v;y{J?Ȅ^`ԁ` 5"Ǐx'|?RÇnݺ>ziy_=(zaÆeVXQj}aԨQ8qJTdo 77#F@KUR|e`@z0qD 8͚5]D>, *G2 7vBUҰ '|a^œ9s0zh899 {ԩSEӦMQV-!9/h(sB̜9ǽ{<;>4m&L~(<;22;w.:w ػw/?!??f^Cv0uT$$$瓺z*M:233!}-ZÇe!֭[ucǢ+@dG v;A ϐ=!d^`/0 A chF4Mx  7nɿvӜMDDÇI& ,Ɍ3W^ӧӦMhӦM4}t3fwssM6j߸q# '":}45oޜF#FjѢ9s[JHH/T`ǎԩS'JNNk׮۷-^@aa!O4|z뭷ٙlll( &L <@{AHd;H~ ; 5# {Q>X-IIIHIIIts%%%QPPmٲKZp!Æ >}P^^ۛ oN-[YZvmkRݺuתU|UӧOd_DAAm۶-ZD-۷SAA.ٲ@JaV*5D5jԠvѤIhtMeY@$ *G*xA ^Pc}0p@3r ɄniC@6l؀aÆ!44GGoٳ^PjWf*NxAHd;PdN8 jL F#1Vʙ3gпM6Yfظq#޽aÆUzѫW/mvB~p ܸq탏OgiصkTh85%8q"4x{{ȑ#~K..Qr@xji cgmĉETTW_}YYYh߻w/1" {<;A>F% r`/0a/0:IdƪHjWe# bccrJ :֯_?#"" u`ȑ?~ int main(int args, char **argv) { #if YAJL_MAJOR != 2 fail to compile #else yajl_version(); #endif return 0; } ''') try: objs = compiler.compile([fname]) compiler.link_shared_lib(objs, 'a', libraries=["yajl"]) return True finally: os.remove(compiler.library_filename('a', lib_type='shared')) for obj in objs: os.remove(obj) except: return False finally: if os.path.exists(fname): os.remove(fname) def patch_yajl_sources(): """Make yajl sources ready for direct compilation against them""" # cp cextern/yajl -R $yajl_sources_copy # mkdir $yajl_sources_copy/yajl # cp $yajl_sources_copy/src/api/*.h $yajl_sources_copy/yajl patched_sources = os.path.join(tempfile.mkdtemp(), 'yajl') shutil.copytree(os.path.join('cextern', 'yajl'), patched_sources) headers_original = os.path.join(patched_sources, 'src', 'api') headers_copy = os.path.join(patched_sources, 'yajl') shutil.copytree(headers_original, headers_copy) return patched_sources extra_sources = [] extra_include_dirs = [] libs = ['yajl'] embed_yajl = os.environ.get('IJSON_EMBED_YAJL', None) == '1' if not embed_yajl: have_yajl = yajl_present() else: yajl_sources = patch_yajl_sources() extra_sources = sorted(glob.glob(os.path.join(yajl_sources, 'src', '*.c'))) extra_sources.remove(os.path.join(yajl_sources, 'src', 'yajl_version.c')) extra_include_dirs = [yajl_sources, os.path.join(yajl_sources, 'src')] libs = [] build_yajl_default = '1' if embed_yajl or have_yajl else '0' build_yajl = os.environ.get('IJSON_BUILD_YAJL2C', build_yajl_default) == '1' if build_yajl: yajl_ext = Extension('ijson.backends._yajl2', language='c', sources=sorted(glob.glob('ijson/backends/yajl2_c/*.c')) + extra_sources, include_dirs=['ijson/backends/yajl2_c'] + extra_include_dirs, libraries=libs, depends=glob.glob('ijson/backends/yajl2_c/*.h')) setupArgs['ext_modules'] = [yajl_ext] setup(**setupArgs) ijson-3.2.3/test/000077500000000000000000000000001445666302700136355ustar00rootroot00000000000000ijson-3.2.3/test/__init__.py000066400000000000000000000000001445666302700157340ustar00rootroot00000000000000ijson-3.2.3/test/_test_async.py000066400000000000000000000007551445666302700165310ustar00rootroot00000000000000# -*- coding:utf-8 -*- import asyncio import io from ijson import compat from ._test_async_common import _get_all, _get_first class AsyncReader(object): def __init__(self, data): if type(data) == compat.bytetype: self.data = io.BytesIO(data) else: self.data = io.StringIO(data) async def read(self, n=-1): await asyncio.sleep(0) return self.data.read(n) get_all = _get_all(AsyncReader) get_first = _get_first(AsyncReader) ijson-3.2.3/test/_test_async_common.py000066400000000000000000000015201445666302700200700ustar00rootroot00000000000000# -*- coding:utf-8 -*- import asyncio import contextlib def _aiorun(f): with contextlib.closing(asyncio.new_event_loop()) as loop: loop.run_until_complete(f) def _get_all(reader): def get_all(routine, json_content, *args, **kwargs): events = [] async def run(): async for event in routine(reader(json_content), *args, **kwargs): events.append(event) _aiorun(run()) return events return get_all def _get_first(reader): def get_first(routine, json_content, *args, **kwargs): events = [] async def run(): async for event in routine(reader(json_content), *args, **kwargs): events.append(event) if events: return _aiorun(run()) return events[0] return get_first ijson-3.2.3/test/_test_async_types_coroutine.py000066400000000000000000000011201445666302700220270ustar00rootroot00000000000000# -*- coding:utf-8 -*- import io import types from ijson import compat from ._test_async_common import _get_all, _get_first class AsyncReaderTypesCoroutine(object): def __init__(self, data): if type(data) == compat.bytetype: self.data = io.BytesIO(data) else: self.data = io.StringIO(data) async def _read(self, n=-1): return self.data.read(n) @types.coroutine def read(self, n=-1): return (yield from self._read(n)) get_all = _get_all(AsyncReaderTypesCoroutine) get_first = _get_first(AsyncReaderTypesCoroutine) ijson-3.2.3/test/test_async.py000066400000000000000000000004421445666302700163630ustar00rootroot00000000000000from ijson import compat from .test_base import FileBasedTests, generate_test_cases # Generating real TestCase classes for each importable backend if compat.IS_PY35: from ._test_async import * # @UnusedWildImport generate_test_cases(globals(), 'Async', '_async', FileBasedTests)ijson-3.2.3/test/test_async_types_coroutines.py000066400000000000000000000005011445666302700220550ustar00rootroot00000000000000from ijson import compat from .test_base import FileBasedTests, generate_test_cases # Generating real TestCase classes for each importable backend if compat.IS_PY35: from ._test_async_types_coroutine import * # @UnusedWildImport generate_test_cases(globals(), 'AsyncTypesCoroutine', '_async', FileBasedTests) ijson-3.2.3/test/test_base.py000066400000000000000000000514771445666302700161760ustar00rootroot00000000000000# -*- coding:utf-8 -*- from __future__ import unicode_literals import collections import ctypes import unittest from decimal import Decimal import threading import ijson from ijson import common, compat from ijson.compat import b2s, IS_PY2 import warnings JSON = b''' { "docs": [ { "null": null, "boolean": false, "true": true, "integer": 0, "double": 0.5, "exponent": 1.0e+2, "long": 10000000000, "string": "\\u0441\\u0442\\u0440\\u043e\\u043a\\u0430 - \xd1\x82\xd0\xb5\xd1\x81\xd1\x82", "\xc3\xb1and\xc3\xba": null }, { "meta": [[1], {}] }, { "meta": {"key": "value"} }, { "meta": null }, { "meta": [] } ] } ''' JSON_OBJECT = { "docs": [ { "null": None, "boolean": False, "true": True, "integer": 0, "double": Decimal("0.5"), "exponent": 1e+2, "long": 10000000000, "string": "строка - тест", "ñandú": None }, { "meta": [[1], {}] }, { "meta": { "key": "value" } }, { "meta": None }, { "meta": [] } ] } JSON_PARSE_EVENTS = [ ('', 'start_map', None), ('', 'map_key', 'docs'), ('docs', 'start_array', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'null'), ('docs.item.null', 'null', None), ('docs.item', 'map_key', 'boolean'), ('docs.item.boolean', 'boolean', False), ('docs.item', 'map_key', 'true'), ('docs.item.true', 'boolean', True), ('docs.item', 'map_key', 'integer'), ('docs.item.integer', 'number', 0), ('docs.item', 'map_key', 'double'), ('docs.item.double', 'number', Decimal('0.5')), ('docs.item', 'map_key', 'exponent'), ('docs.item.exponent', 'number', Decimal('1.0E+2')), ('docs.item', 'map_key', 'long'), ('docs.item.long', 'number', 10000000000), ('docs.item', 'map_key', 'string'), ('docs.item.string', 'string', 'строка - тест'), ('docs.item', 'map_key', 'ñandú'), ('docs.item.ñandú', 'null', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'start_array', None), ('docs.item.meta.item', 'start_array', None), ('docs.item.meta.item.item', 'number', 1), ('docs.item.meta.item', 'end_array', None), ('docs.item.meta.item', 'start_map', None), ('docs.item.meta.item', 'end_map', None), ('docs.item.meta', 'end_array', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'start_map', None), ('docs.item.meta', 'map_key', 'key'), ('docs.item.meta.key', 'string', 'value'), ('docs.item.meta', 'end_map', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'null', None), ('docs.item', 'end_map', None), ('docs.item', 'start_map', None), ('docs.item', 'map_key', 'meta'), ('docs.item.meta', 'start_array', None), ('docs.item.meta', 'end_array', None), ('docs.item', 'end_map', None), ('docs', 'end_array', None), ('', 'end_map', None) ] JSON_KVITEMS = [ ("null", None), ("boolean", False), ("true", True), ("integer", 0), ("double", Decimal("0.5")), ("exponent", 1e+2), ("long", 10000000000), ("string", "строка - тест"), ("ñandú", None), ("meta", [[1], {}]), ("meta", {"key": "value"}), ("meta", None), ("meta", []) ] JSON_KVITEMS_META = [ ('key', 'value') ] JSON_EVENTS = [ ('start_map', None), ('map_key', 'docs'), ('start_array', None), ('start_map', None), ('map_key', 'null'), ('null', None), ('map_key', 'boolean'), ('boolean', False), ('map_key', 'true'), ('boolean', True), ('map_key', 'integer'), ('number', 0), ('map_key', 'double'), ('number', Decimal('0.5')), ('map_key', 'exponent'), ('number', 100), ('map_key', 'long'), ('number', 10000000000), ('map_key', 'string'), ('string', 'строка - тест'), ('map_key', 'ñandú'), ('null', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('start_array', None), ('start_array', None), ('number', 1), ('end_array', None), ('start_map', None), ('end_map', None), ('end_array', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('start_map', None), ('map_key', 'key'), ('string', 'value'), ('end_map', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('null', None), ('end_map', None), ('start_map', None), ('map_key', 'meta'), ('start_array', None), ('end_array', None), ('end_map', None), ('end_array', None), ('end_map', None), ] # Like JSON, but with an additional top-level array structure ARRAY_JSON = b'[' + JSON + b']' ARRAY_JSON_EVENTS = ( [('start_array', None)] + JSON_EVENTS + [('end_array', None)] ) ARRAY_JSON_PARSE_EVENTS = ( [('', 'start_array', None)] + [('.'.join(filter(None, ('item', p))), t, e) for p, t, e in JSON_PARSE_EVENTS] + [('', 'end_array', None)] ) ARRAY_JSON_OBJECT = [JSON_OBJECT] SCALAR_JSON = b'0' INVALID_JSONS = [ b'["key", "value",]', # trailing comma b'["key" "value"]', # no comma b'{"key": "value",}', # trailing comma b'{"key": "value" "key"}', # no comma b'{"key" "value"}', # no colon b'invalid', # unknown lexeme b'[1, 2] dangling junk', # dangling junk b'}', # no corresponding opening token b']', # no corresponding opening token b'"\xa8"' # invalid UTF-8 byte sequence ] YAJL1_PASSING_INVALID = INVALID_JSONS[6] INCOMPLETE_JSONS = [ b'', b'"test', b'[', b'[1', b'[1,', b'{', b'{"key"', b'{"key":', b'{"key": "value"', b'{"key": "value",', ] INCOMPLETE_JSON_TOKENS = [ b'n', b'nu', b'nul', b't', b'tr', b'tru', b'f', b'fa', b'fal', b'fals', b'[f', b'[fa', b'[fal', b'[fals', b'[t', b'[tr', b'[tru', b'[n', b'[nu', b'[nul', b'{"key": t', b'{"key": tr', b'{"key": tru', b'{"key": f', b'{"key": fa', b'{"key": fal', b'{"key": fals', b'{"key": n', b'{"key": nu', b'{"key": nul', ] STRINGS_JSON = br''' { "str1": "", "str2": "\"", "str3": "\\", "str4": "\\\\", "special\t": "\b\f\n\r\t" } ''' SURROGATE_PAIRS_JSON = br'"\uD83D\uDCA9"' PARTIAL_ARRAY_JSONS = [ (b'[1,', 1), (b'[1, 2 ', 1, 2), (b'[1, "abc"', 1, 'abc'), (b'[{"abc": [0, 1]}', {'abc': [0, 1]}), (b'[{"abc": [0, 1]},', {'abc': [0, 1]}), ] items_test_case = collections.namedtuple('items_test_case', 'json, prefix, kvitems, items') EMPTY_MEMBER_TEST_CASES = { 'simple': items_test_case( b'{"a": {"": {"b": 1, "c": 2}}}', 'a.', [("b", 1), ("c", 2)], [{"b": 1, "c": 2}] ), 'embedded': items_test_case( b'{"a": {"": {"": {"b": 1, "c": 2}}}}', 'a..', [("b", 1), ("c", 2)], [{"b": 1, "c": 2}] ), 'top_level': items_test_case( b'{"": 1, "a": 2}', '', [("", 1), ("a", 2)], [{"": 1, "a": 2}] ), 'top_level_embedded': items_test_case( b'{"": {"": 1}, "a": 2}', '', [("", {"": 1}), ("a", 2)], [{"": {"": 1}, "a": 2}] ) } class warning_catcher(object): '''Encapsulates proper warning catch-all logic in python 2.7 and 3''' def __init__(self): self.catcher = warnings.catch_warnings(record=True) def __enter__(self): ret = self.catcher.__enter__() if compat.IS_PY2: warnings.simplefilter("always") return ret def __exit__(self, *args): self.catcher.__exit__(*args) class BackendSpecificTestCase(object): ''' Base class for backend-specific tests, gives ability to easily and generically reference different methods on the backend. It requires subclasses to define a `backend` member with the backend module, and a `suffix` attribute indicating the method flavour to obtain. ''' def __getattr__(self, name): return getattr(self.backend, name + self.method_suffix) class IJsonTestsBase(object): ''' Base class with common tests for all iteration methods. Subclasses implement `all()` and `first()` to collect events coming from a particuliar method. ''' def test_basic_parse(self): events = self.get_all(self.basic_parse, JSON) self.assertEqual(events, JSON_EVENTS) def test_basic_parse_threaded(self): thread = threading.Thread(target=self.test_basic_parse) thread.start() thread.join() def test_parse(self): events = self.get_all(self.parse, JSON) self.assertEqual(events, JSON_PARSE_EVENTS) def test_items(self): events = self.get_all(self.items, JSON, '') self.assertEqual(events, [JSON_OBJECT]) def test_items_twodictlevels(self): json = b'{"meta":{"view":{"columns":[{"id": -1}, {"id": -2}]}}}' ids = self.get_all(self.items, json, 'meta.view.columns.item.id') self.assertEqual(2, len(ids)) self.assertListEqual([-2,-1], sorted(ids)) def test_items_with_dotted_name(self): json = b'{"0.1": 0}' self.assertListEqual([0], self.get_all(self.items, json, '0.1')) json = b'{"0.1": [{"a.b": 0}]}' self.assertListEqual([0], self.get_all(self.items, json, '0.1.item.a.b')) json = b'{"0.1": 0, "0": {"1": 1}}' self.assertListEqual([0, 1], self.get_all(self.items, json, '0.1')) json = b'{"abc.def": 0}' self.assertListEqual([0], self.get_all(self.items, json, 'abc.def')) self.assertListEqual([], self.get_all(self.items, json, 'abc')) self.assertListEqual([], self.get_all(self.items, json, 'def')) def test_map_type(self): obj = self.get_first(self.items, JSON, '') self.assertTrue(isinstance(obj, dict)) obj = self.get_first(self.items, JSON, '', map_type=collections.OrderedDict) self.assertTrue(isinstance(obj, collections.OrderedDict)) def test_kvitems(self): kvitems = self.get_all(self.kvitems, JSON, 'docs.item') self.assertEqual(JSON_KVITEMS, kvitems) def test_kvitems_toplevel(self): kvitems = self.get_all(self.kvitems, JSON, '') self.assertEqual(1, len(kvitems)) key, value = kvitems[0] self.assertEqual('docs', key) self.assertEqual(JSON_OBJECT['docs'], value) def test_kvitems_empty(self): kvitems = self.get_all(self.kvitems, JSON, 'docs') self.assertEqual([], kvitems) def test_kvitems_twodictlevels(self): json = b'{"meta":{"view":{"columns":[{"id": -1}, {"id": -2}]}}}' view = self.get_all(self.kvitems, json, 'meta.view') self.assertEqual(1, len(view)) key, value = view[0] self.assertEqual('columns', key) self.assertEqual([{'id': -1}, {'id': -2}], value) def test_kvitems_different_underlying_types(self): kvitems = self.get_all(self.kvitems, JSON, 'docs.item.meta') self.assertEqual(JSON_KVITEMS_META, kvitems) def test_basic_parse_array(self): events = self.get_all(self.basic_parse, ARRAY_JSON) self.assertEqual(events, ARRAY_JSON_EVENTS) def test_basic_parse_array_threaded(self): thread = threading.Thread(target=self.test_basic_parse_array) thread.start() thread.join() def test_parse_array(self): events = self.get_all(self.parse, ARRAY_JSON) self.assertEqual(events, ARRAY_JSON_PARSE_EVENTS) def test_items_array(self): events = self.get_all(self.items, ARRAY_JSON, '') self.assertEqual(events, [ARRAY_JSON_OBJECT]) def test_kvitems_array(self): kvitems = self.get_all(self.kvitems, ARRAY_JSON, 'item.docs.item') self.assertEqual(JSON_KVITEMS, kvitems) def test_scalar(self): events = self.get_all(self.basic_parse, SCALAR_JSON) self.assertEqual(events, [('number', 0)]) def test_strings(self): events = self.get_all(self.basic_parse, STRINGS_JSON) strings = [value for event, value in events if event == 'string'] self.assertEqual(strings, ['', '"', '\\', '\\\\', '\b\f\n\r\t']) self.assertTrue(('map_key', 'special\t') in events) def test_surrogate_pairs(self): event = self.get_first(self.basic_parse, SURROGATE_PAIRS_JSON) parsed_string = event[1] self.assertEqual(parsed_string, '💩') def test_numbers(self): """Check that numbers are correctly parsed""" def get_numbers(json, **kwargs): events = self.get_all(self.basic_parse, json, **kwargs) return events, [value for event, value in events if event == 'number'] def assert_numbers(json, expected_float_type, *numbers, **kwargs): events, values = get_numbers(json, **kwargs) float_types = set(type(value) for event, value in events if event == 'number') float_types -= {int} self.assertEqual(1, len(float_types)) self.assertEqual(next(iter(float_types)), expected_float_type) self.assertSequenceEqual(numbers, values) NUMBERS_JSON = b'[1, 1.0, 1E2]' assert_numbers(NUMBERS_JSON, Decimal, 1, Decimal("1.0"), Decimal("1e2")) assert_numbers(NUMBERS_JSON, float, 1, 1., 100., use_float=True) assert_numbers(b'1e400', Decimal, Decimal('1e400')) assert_numbers(b'1e-400', Decimal, Decimal('1e-400')) assert_numbers(b'1e-400', float, 0., use_float=True) # Test for 64-bit integers support when using use_float=True try: past32bits = 2 ** 32 + 1 received = get_numbers(('%d' % past32bits).encode('utf8'), use_float=True)[1][0] self.assertTrue(self.supports_64bit_integers) self.assertEqual(past32bits, received) except common.JSONError: self.assertFalse(self.supports_64bit_integers) # Check that numbers bigger than MAX_DOUBLE cannot be represented try: get_numbers(b'1e400', use_float=True) self.fail("Overflow error expected") except common.JSONError: pass def test_invalid_numbers(self): # leading zeros if self.detects_leading_zeros: for case in (b'00', b'01', b'001'): for base in (case, case + b'.0', case + b'e0', case + b'E0'): for n in (base, b'-' + base): with self.assertRaises(common.JSONError): self.get_all(self.basic_parse, n) # incomplete exponents for n in (b'1e', b'0.1e', b'0E'): with self.assertRaises(common.JSONError): self.get_all(self.basic_parse, n) # incomplete fractions for n in (b'1.', b'.1'): with self.assertRaises(common.JSONError): self.get_all(self.basic_parse, n) def test_incomplete(self): for json in INCOMPLETE_JSONS: with self.assertRaises(common.IncompleteJSONError): self.get_all(self.basic_parse, json) def test_incomplete_tokens(self): if not self.handles_incomplete_json_tokens: return for json in INCOMPLETE_JSON_TOKENS: with self.assertRaises(common.IncompleteJSONError): self.get_all(self.basic_parse, json) def test_invalid(self): for json in INVALID_JSONS: # Yajl1 doesn't complain about additional data after the end # of a parsed object. Skipping this test. if self.backend_name == 'yajl' and json == YAJL1_PASSING_INVALID: continue with self.assertRaises(common.JSONError): self.get_all(self.basic_parse, json) def test_multiple_values(self): """Test that the multiple_values flag works""" if not self.supports_multiple_values: with self.assertRaises(ValueError): self.get_all(self.basic_parse, "", multiple_values=True) return multiple_json = JSON + JSON + JSON items = lambda x, **kwargs: self.items(x, '', **kwargs) for func in (self.basic_parse, items): with self.assertRaises(common.JSONError): self.get_all(func, multiple_json) with self.assertRaises(common.JSONError): self.get_all(func, multiple_json, multiple_values=False) result = self.get_all(func, multiple_json, multiple_values=True) if func == items: self.assertEqual(result, [JSON_OBJECT, JSON_OBJECT, JSON_OBJECT]) else: self.assertEqual(result, JSON_EVENTS + JSON_EVENTS + JSON_EVENTS) def test_comments(self): json = b'{"a": 2 /* a comment */}' try: self.get_all(self.basic_parse, json, allow_comments=True) except ValueError: if self.supports_comments: raise def _test_empty_member(self, test_case): pairs = self.get_all(self.kvitems, test_case.json, test_case.prefix) self.assertEqual(test_case.kvitems, pairs) objects = self.get_all(self.items, test_case.json, test_case.prefix) self.assertEqual(test_case.items, objects) def test_empty_member(self): self._test_empty_member(EMPTY_MEMBER_TEST_CASES['simple']) def test_embedded_empty_member(self): self._test_empty_member(EMPTY_MEMBER_TEST_CASES['embedded']) def test_top_level_empty_member(self): self._test_empty_member(EMPTY_MEMBER_TEST_CASES['top_level']) def test_top_level_embedded_empty_member(self): self._test_empty_member(EMPTY_MEMBER_TEST_CASES['top_level_embedded']) class FileBasedTests(object): def test_string_stream(self): with warning_catcher() as warns: events = self.get_all(self.basic_parse, b2s(JSON)) self.assertEqual(events, JSON_EVENTS) if self.warn_on_string_stream: self.assertEqual(len(warns), 1) self.assertEqual(DeprecationWarning, warns[0].category) def test_different_buf_sizes(self): for buf_size in (1, 4, 16, 64, 256, 1024, 4098): events = self.get_all(self.basic_parse, JSON, buf_size=buf_size) self.assertEqual(events, JSON_EVENTS) def generate_backend_specific_tests(module, classname_prefix, method_suffix, *bases, **kwargs): for backend in ['python', 'yajl', 'yajl2', 'yajl2_cffi', 'yajl2_c']: try: classname = '%s%sTests' % ( ''.join(p.capitalize() for p in backend.split('_')), classname_prefix ) if IS_PY2: classname = classname.encode('ascii') _bases = bases + (BackendSpecificTestCase, unittest.TestCase) _members = { 'backend_name': backend, 'backend': ijson.get_backend(backend), 'method_suffix': method_suffix, 'warn_on_string_stream': not IS_PY2, 'supports_64bit_integers': not (backend == 'yajl' and ctypes.sizeof(ctypes.c_long) == 4) } members = kwargs.get('members', lambda _: {}) _members.update(members(backend)) module[classname] = type(classname, _bases, _members) except ImportError: pass def generate_test_cases(module, classname, method_suffix, *bases): _bases = bases + (IJsonTestsBase,) members = lambda name: { 'get_all': lambda self, *args, **kwargs: module['get_all'](*args, **kwargs), 'get_first': lambda self, *args, **kwargs: module['get_first'](*args, **kwargs), 'supports_multiple_values': name != 'yajl', 'supports_comments': name != 'python', 'detects_leading_zeros': name != 'yajl', 'handles_incomplete_json_tokens': name != 'yajl' } return generate_backend_specific_tests(module, classname, method_suffix, members=members, *_bases)ijson-3.2.3/test/test_coroutines.py000066400000000000000000000014721445666302700174440ustar00rootroot00000000000000from ijson import utils, compat from .test_base import generate_test_cases if compat.IS_PY2: def bytesiter(x): return x else: def bytesiter(x): for b in x: yield bytes([b]) def get_all(routine, json_content, *args, **kwargs): events = utils.sendable_list() coro = routine(events, *args, **kwargs) for datum in bytesiter(json_content): coro.send(datum) coro.close() return events def get_first(routine, json_content, *args, **kwargs): events = utils.sendable_list() coro = routine(events, *args, **kwargs) for datum in bytesiter(json_content): coro.send(datum) if events: return events[0] coro.close() if events: return events[0] return None generate_test_cases(globals(), 'Coroutines', '_coro')ijson-3.2.3/test/test_dump.py000066400000000000000000000026261445666302700162210ustar00rootroot00000000000000import os import subprocess import sys import unittest from ijson import compat from test.test_base import JSON class DumpTests(unittest.TestCase): def _do_test_dump(self, method, multiple_values): # Use python backend to ensure multiple_values works env = dict(os.environ) env['IJSON_BACKEND'] = 'python' # Ensure printing works on the subprocess in Windows # by using utf-8 on its stdout if 'win' in sys.platform: env = dict(os.environ) env['PYTHONIOENCODING'] = 'utf-8' cmd = [sys.executable, '-m', 'ijson.dump', '-m', method, '-p', ''] if multiple_values: cmd.append('-M') proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) input_data = JSON if multiple_values: input_data += JSON out, err = proc.communicate(input_data) status = proc.wait() self.assertEqual(0, status, "out:\n%s\nerr:%s" % (compat.b2s(out), compat.b2s(err))) def _test_dump(self, method): self._do_test_dump(method, True) self._do_test_dump(method, False) def test_basic_parse(self): self._test_dump('basic_parse') def test_parse(self): self._test_dump('parse') def test_kvitems(self): self._test_dump('kvitems') def test_items(self): self._test_dump('items')ijson-3.2.3/test/test_generators.py000066400000000000000000000121361445666302700174220ustar00rootroot00000000000000from ijson import common, compat from .test_base import (JSON, FileBasedTests, JSON_EVENTS, PARTIAL_ARRAY_JSONS, warning_catcher, INVALID_JSONS, IS_PY2, generate_test_cases) class SingleReadFile(object): '''A bytes file that can be read only once''' def __init__(self, raw_value): self.raw_value = raw_value def read(self, size=-1): if size == 0: return compat.bytetype() val = self.raw_value if not val: raise AssertionError('read twice') self.raw_value = compat.bytetype() return val class GeneratorSpecificTests(FileBasedTests): ''' Base class for parsing tests that is used to create test cases for each available backends. ''' def test_utf8_split(self): buf_size = JSON.index(b'\xd1') + 1 try: self.get_all(self.basic_parse, JSON, buf_size=buf_size) except UnicodeDecodeError: self.fail('UnicodeDecodeError raised') def test_lazy(self): # shouldn't fail since iterator is not exhausted self.backend.basic_parse(compat.BytesIO(INVALID_JSONS[0])) self.assertTrue(True) def test_boundary_lexeme(self): buf_size = JSON.index(b'false') + 1 events = self.get_all(self.basic_parse, JSON, buf_size=buf_size) self.assertEqual(events, JSON_EVENTS) def test_boundary_whitespace(self): buf_size = JSON.index(b' ') + 1 events = self.get_all(self.basic_parse, JSON, buf_size=buf_size) self.assertEqual(events, JSON_EVENTS) def test_item_building_greediness(self): self._test_item_iteration_validity(compat.BytesIO) def test_lazy_file_reading(self): if self.backend_name == 'python' and IS_PY2: # We know it doesn't work because because the decoder itself # is quite eager on its reading return self._test_item_iteration_validity(SingleReadFile) def _test_item_iteration_validity(self, file_type): for json in PARTIAL_ARRAY_JSONS: json, expected_items = json[0], json[1:] iterable = self.backend.items(file_type(json), 'item') for expected_item in expected_items: self.assertEqual(expected_item, next(iterable)) COMMON_DATA = b''' { "skip": "skip_value", "c": {"d": "e", "f": "g"}, "list": [{"o1": 1}, {"o2": 2}] }''' COMMON_PARSE = [ ('', 'start_map', None), ('', 'map_key', 'skip'), ('skip', 'string', 'skip_value'), ('', 'map_key', 'c'), ('c', 'start_map', None), ('c', 'map_key', 'd'), ('c.d', 'string', 'e'), ('c', 'map_key', 'f'), ('c.f', 'string', 'g'), ('c', 'end_map', None), ('', 'map_key', 'list'), ('list', 'start_array', None), ('list.item', 'start_map', None), ('list.item', 'map_key', 'o1'), ('list.item.o1', 'number', 1), ('list.item', 'end_map', None), ('list.item', 'start_map', None), ('list.item', 'map_key', 'o2'), ('list.item.o2', 'number', 2), ('list.item', 'end_map', None), ('list', 'end_array', None), ('', 'end_map', None), ] def _skip_parse_events(self, events): skip_value = None for prefix, _, value in events: if prefix == 'skip': skip_value = value break self.assertEqual(skip_value, 'skip_value') def _test_common_routine(self, routine, *args, **kwargs): base_routine_name = kwargs.pop('base_routine_name', 'parse') base_routine = getattr(self.backend, base_routine_name) events = base_routine(compat.BytesIO(self.COMMON_DATA)) if base_routine_name == 'parse': self._skip_parse_events(events) # Rest of events can still be used return list(routine(events, *args)) def test_common_parse(self): with warning_catcher() as warns: results = self._test_common_routine(common.parse, base_routine_name='basic_parse') self.assertEqual(self.COMMON_PARSE, results) self.assertEqual(len(warns), 1) def test_common_kvitems(self): with warning_catcher() as warns: results = self._test_common_routine(common.kvitems, 'c') self.assertEqual([("d", "e"), ("f", "g")], results) self.assertEqual(len(warns), 1) def test_common_items(self): with warning_catcher() as warns: results = self._test_common_routine(common.items, 'list.item') self.assertEqual([{"o1": 1}, {"o2": 2}], results) self.assertEqual(len(warns), 1) def _reader(json): if type(json) == compat.bytetype: return compat.BytesIO(json) return compat.StringIO(json) def get_all(routine, json_content, *args, **kwargs): return list(routine(_reader(json_content), *args, **kwargs)) def get_first(routine, json_content, *args, **kwargs): return next(routine(_reader(json_content), *args, **kwargs)) generate_test_cases(globals(), 'Generators', '_gen', GeneratorSpecificTests)ijson-3.2.3/test/test_misc.py000066400000000000000000000105231445666302700162020ustar00rootroot00000000000000import importlib.util import unittest from ijson import common, compat from .test_base import warning_catcher from test.test_base import JSON, JSON_EVENTS, JSON_PARSE_EVENTS, JSON_OBJECT,\ generate_backend_specific_tests, JSON_KVITEMS class Misc(unittest.TestCase): """Miscellaneous unit tests""" def test_common_number_is_deprecated(self): with warning_catcher() as warns: common.number("1") self.assertEqual(len(warns), 1) self.assertEqual(DeprecationWarning, warns[0].category) def test_yajl2_c_loadable(self): if compat.IS_PY2: self.skipTest("Test requires Python 3.4+") spec = importlib.util.find_spec("ijson.backends._yajl2") if spec is None: self.skipTest("yajl2_c is not built") importlib.util.module_from_spec(spec) class MainEntryPoints(object): def _assert_invalid_type(self, routine, *args, **kwargs): # Functions are not valid inputs with self.assertRaises(ValueError): routine(lambda _: JSON, *args, **kwargs) def _assert_bytes(self, expected_results, routine, *args, **kwargs): results = list(routine(JSON, *args, **kwargs)) self.assertEqual(expected_results, results) def _assert_str(self, expected_results, routine, *args, **kwargs): with warning_catcher() as warns: results = list(routine(compat.b2s(JSON), *args, **kwargs)) self.assertEqual(expected_results, results) if self.warn_on_string_stream: self.assertEqual(1, len(warns)) def _assert_unicode(self, expected_results, routine, *args, **kwargs): if not compat.IS_PY2: return with warning_catcher() as warns: results = list(routine(unicode(JSON, 'utf-8'), *args, **kwargs)) self.assertEqual(expected_results, results) self.assertEqual(1, len(warns)) def _assert_file(self, expected_results, routine, *args, **kwargs): results = list(routine(compat.BytesIO(JSON), *args, **kwargs)) self.assertEqual(expected_results, results) def _assert_async_file(self, expected_results, routine, *args, **kwargs): if not compat.IS_PY35: return from ._test_async import get_all results = get_all(routine, JSON, *args, **kwargs) self.assertEqual(expected_results, results) def _assert_async_types_coroutine(self, expected_results, routine, *args, **kwargs): if not compat.IS_PY35: return from ._test_async_types_coroutine import get_all results = get_all(routine, JSON, *args, **kwargs) self.assertEqual(expected_results, results) def _assert_events(self, expected_results, previous_routine, routine, *args, **kwargs): events = previous_routine(compat.BytesIO(JSON)) # Using a different generator to make the point that we can chain # user-provided code def event_yielder(): for evt in events: yield evt results = list(routine(event_yielder(), *args, **kwargs)) self.assertEqual(expected_results, results) def _assert_entry_point(self, expected_results, previous_routine, routine, *args, **kwargs): self._assert_invalid_type(routine, *args, **kwargs) self._assert_bytes(expected_results, routine, *args, **kwargs) self._assert_str(expected_results, routine, *args, **kwargs) self._assert_unicode(expected_results, routine, *args, **kwargs) self._assert_file(expected_results, routine, *args, **kwargs) self._assert_async_file(expected_results, routine, *args, **kwargs) self._assert_async_types_coroutine(expected_results, routine, *args, **kwargs) if previous_routine: self._assert_events(expected_results, previous_routine, routine, *args, **kwargs) def test_rich_basic_parse(self): self._assert_entry_point(JSON_EVENTS, None, self.basic_parse) def test_rich_parse(self): self._assert_entry_point(JSON_PARSE_EVENTS, self.basic_parse, self.parse) def test_rich_items(self): self._assert_entry_point([JSON_OBJECT], self.parse, self.items, '') def test_rich_kvitems(self): self._assert_entry_point(JSON_KVITEMS, self.parse, self.kvitems, 'docs.item') generate_backend_specific_tests(globals(), 'MainEntryPoints', '', MainEntryPoints)ijson-3.2.3/tox.ini000066400000000000000000000004671445666302700142000ustar00rootroot00000000000000[tox] envlist = py27, py3 [testenv] # YAJL_DLL can be used to manually point to a yajl installation passenv = YAJL_DLL commands = {envpython} -munittest discover {envpython} -mdoctest ijson/common.py pytest --cov --cov-append --cov-branch deps = cffi pytest-cov usedevelop = true