pax_global_header00006660000000000000000000000064141320026040014502gustar00rootroot0000000000000052 comment=2526165aa3237a3d6d0b6f5e984ca5ec3cb807ac cbor2-5.4.2/000077500000000000000000000000001413200260400125215ustar00rootroot00000000000000cbor2-5.4.2/.github/000077500000000000000000000000001413200260400140615ustar00rootroot00000000000000cbor2-5.4.2/.github/workflows/000077500000000000000000000000001413200260400161165ustar00rootroot00000000000000cbor2-5.4.2/.github/workflows/codeqa-test.yml000066400000000000000000000041211413200260400210500ustar00rootroot00000000000000name: Python codeqa/test on: push: branches: [master] pull_request: jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.x - uses: actions/cache@v2 with: path: ~/.cache/pip key: pip-lint - name: Install dependencies run: pip install flake8 - name: Run flake8 run: flake8 cbor2 tests test: needs: [lint] strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: [3.6, 3.7, 3.8, 3.9, pypy3] exclude: - os: macos-latest python-version: 3.7 - os: macos-latest python-version: 3.8 - os: macos-latest python-version: pypy3 - os: windows-latest python-version: 3.7 - os: windows-latest python-version: 3.8 - os: windows-latest python-version: pypy3 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 with: path: ~/.cache/pip key: pip-test-${{ matrix.python-version }}-${{ matrix.os }} - name: Install dependencies run: pip install -e .[test] coveralls - name: Test with pytest run: pytest - name: Upload Coverage run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: ${{ matrix.test-name }} COVERALLS_PARALLEL: true coveralls: name: Finish Coveralls needs: test runs-on: ubuntu-latest steps: - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.x - name: Install dependencies run: pip install coveralls - name: Notify Coveralls run: coveralls --service=github --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} cbor2-5.4.2/.github/workflows/publish.yml000066400000000000000000000040611413200260400203100ustar00rootroot00000000000000name: Publish packages to PyPI on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" - "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+" - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" jobs: linux-wheels: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Build manylinux Python wheels uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_x86_64 - uses: actions/upload-artifact@v2 with: name: linux-wheels path: dist/*-manylinux*.whl other-wheels: strategy: matrix: os: [macos-latest, windows-latest] python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 with: path: ~/.cache/pip key: pip-test-${{ matrix.python-version }}-${{ matrix.os }} - name: Install dependencies run: pip install build - name: Create packages run: python -m build --wheel . - uses: actions/upload-artifact@v2 with: name: other-wheels path: dist/*.whl sdist: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.x - name: Install dependencies run: pip install build - name: Create sdist run: python -m build --sdist . - uses: actions/upload-artifact@v2 with: name: sdist path: dist/*.tar.gz publish: needs: - linux-wheels - other-wheels - sdist runs-on: ubuntu-latest steps: - name: Download generated packaging artifacts uses: actions/download-artifact@v2 - name: Move the packages to dist/ run: | mkdir dist mv */*.whl */*.tar.gz dist - name: Upload packages uses: pypa/gh-action-pypi-publish@master with: user: __token__ password: ${{ secrets.pypi_password }} cbor2-5.4.2/.gitignore000066400000000000000000000002151413200260400145070ustar00rootroot00000000000000.project .pydevproject .idea/ .coverage coverage/ .cache/ .tox/ .eggs/ *.egg-info/ *.pyc *.pyd *.so dist/ docs/_build/ build/ virtualenv/ .vscbor2-5.4.2/.readthedocs.yml000066400000000000000000000002151413200260400156050ustar00rootroot00000000000000version: 2 formats: [htmlzip, pdf] python: version: 3.7 install: - method: pip path: . extra_requirements: [doc] cbor2-5.4.2/LICENSE.txt000066400000000000000000000020711413200260400143440ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2016 Alex Grönholm Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. cbor2-5.4.2/README.rst000066400000000000000000000102331413200260400142070ustar00rootroot00000000000000.. image:: https://github.com/agronholm/cbor2/actions/workflows/codeqa-test.yml/badge.svg :target: https://github.com/agronholm/cbor2/actions/workflows/codeqa-test.yml :alt: Testing Status .. image:: https://github.com/agronholm/cbor2/actions/workflows/publish.yml/badge.svg :target: https://github.com/agronholm/cbor2/actions/workflows/publish.yml :alt: Publish Status .. image:: https://coveralls.io/repos/github/agronholm/cbor2/badge.svg?branch=master :target: https://coveralls.io/github/agronholm/cbor2?branch=master :alt: Code Coverage .. image:: https://readthedocs.org/projects/cbor2/badge/?version=latest :target: https://cbor2.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status About ===== This library provides encoding and decoding for the Concise Binary Object Representation (CBOR) (`RFC 8949`_) serialization format. The specification is fully compatible with the original RFC 7049. `Read the docs `_ to learn more. It is implemented in pure python with an optional C backend. On PyPy, cbor2 runs with almost identical performance to the C backend. .. _RFC 8949: https://www.rfc-editor.org/rfc/rfc8949.html Features -------- * Simple api like ``json`` or ``pickle`` modules. * Support many `CBOR tags`_ with `stdlib objects`_. * Generic tag decoding. * `Shared value`_ references including cyclic references. * `String references`_ compact encoding with repeated strings replaced with indices. * Optional C module backend tested on big- and little-endian architectures. * Extensible `tagged value handling`_ using ``tag_hook`` and ``object_hook`` on decode and ``default`` on encode. * Command-line diagnostic tool, converting CBOR file or stream to JSON ``python -m cbor2.tool`` (This is a lossy conversion, for diagnostics only) * Thorough test suite. .. _CBOR tags: https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml .. _stdlib objects: https://cbor2.readthedocs.io/en/latest/usage.html#tag-support .. _Shared value: http://cbor.schmorp.de/value-sharing .. _String references: http://cbor.schmorp.de/stringref .. _tagged value handling: https://cbor2.readthedocs.io/en/latest/customizing.html#using-the-cbor-tags-for-custom-types Installation ============ :: pip install cbor2 Requirements ------------ * Python >= 3.6 (or `PyPy3`_ 3.6+) * C-extension: Any C compiler that can build Python extensions. Any modern libc with the exception of Glibc<2.9 .. _PyPy3: https://www.pypy.org/ Building the C-Extension ------------------------ To force building of the optional C-extension, set OS env ``CBOR2_BUILD_C_EXTENSION=1``. To disable building of the optional C-extension, set OS env ``CBOR2_BUILD_C_EXTENSION=0``. If this environment variable is unset, setup.py will default to auto detecting a compatible C library and attempt to compile the extension. Usage ===== `Basic Usage `_ Command-line Usage ================== ``python -m cbor2.tool`` converts CBOR data in raw binary or base64 encoding into a representation that allows printing as JSON. This is a lossy transformation as each datatype is converted into something that can be represented as a JSON value. Usage:: # Pass hexadecimal through xxd. $ echo a16568656c6c6f65776f726c64 | xxd -r -ps | python -m cbor2.tool --pretty { "hello": "world" } # Decode Base64 directly $ echo ggEC | python -m cbor2.tool --decode [1, 2] # Read from a file encoded in Base64 $ python -m cbor2.tool -d tests/examples.cbor.b64 {...} It can be used in a pipeline with json processing tools like `jq`_ to allow syntax coloring, field extraction and more. CBOR data items concatenated into a sequence can be decoded also:: $ echo ggECggMEggUG | python -m cbor2.tool -d --sequence [1, 2] [3, 4] [5, 6] Multiple files can also be sent to a single output file:: $ python -m cbor2.tool -o all_files.json file1.cbor file2.cbor ... fileN.cbor .. _jq: https://stedolan.github.io/jq/ Security ======== This library has not been tested against malicious input. In theory it should be as safe as JSON, since unlike ``pickle`` the decoder does not execute any code. cbor2-5.4.2/cbor2/000077500000000000000000000000001413200260400135305ustar00rootroot00000000000000cbor2-5.4.2/cbor2/__init__.py000066400000000000000000000036401413200260400156440ustar00rootroot00000000000000from .decoder import load, loads, CBORDecoder # noqa from .encoder import dump, dumps, CBOREncoder, shareable_encoder # noqa from .types import ( # noqa CBORError, CBOREncodeError, CBOREncodeTypeError, CBOREncodeValueError, CBORDecodeError, CBORDecodeValueError, CBORDecodeEOF, CBORTag, CBORSimpleValue, undefined ) try: from _cbor2 import * # noqa except ImportError: # Couldn't import the optimized C version; ignore the failure and leave the # pure Python implementations in place. pass else: # The pure Python implementations are replaced with the optimized C # variants, but we still need to create the encoder dictionaries for the C # variant here (this is much simpler than doing so in C, and doesn't affect # overall performance as it's a one-off initialization cost). def _init_cbor2(): from collections import OrderedDict from .encoder import default_encoders, canonical_encoders from .types import CBORTag, CBORSimpleValue, undefined # noqa import _cbor2 _cbor2.default_encoders = OrderedDict([ (( _cbor2.CBORSimpleValue if type_ is CBORSimpleValue else _cbor2.CBORTag if type_ is CBORTag else type(_cbor2.undefined) if type_ is type(undefined) else type_ ), getattr(_cbor2.CBOREncoder, method.__name__)) for type_, method in default_encoders.items() ]) _cbor2.canonical_encoders = OrderedDict([ (( _cbor2.CBORSimpleValue if type_ is CBORSimpleValue else _cbor2.CBORTag if type_ is CBORTag else type(_cbor2.undefined) if type_ is type(undefined) else type_ ), getattr(_cbor2.CBOREncoder, method.__name__)) for type_, method in canonical_encoders.items() ]) _init_cbor2() del _init_cbor2 cbor2-5.4.2/cbor2/decoder.py000066400000000000000000000525431413200260400155200ustar00rootroot00000000000000import re import struct import sys from collections.abc import Mapping from datetime import datetime, timedelta, timezone from io import BytesIO from .types import ( CBORDecodeValueError, CBORDecodeEOF, CBORTag, undefined, break_marker, CBORSimpleValue, FrozenDict) timestamp_re = re.compile(r'^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)' r'(?:\.(\d{1,6})\d*)?(?:Z|([+-]\d\d):(\d\d))$') class CBORDecoder: """ The CBORDecoder class implements a fully featured `CBOR`_ decoder with several extensions for handling shared references, big integers, rational numbers and so on. Typically the class is not used directly, but the :func:`load` and :func:`loads` functions are called to indirectly construct and use the class. When the class is constructed manually, the main entry points are :meth:`decode` and :meth:`decode_from_bytes`. :param tag_hook: callable that takes 2 arguments: the decoder instance, and the :class:`CBORTag` to be decoded. This callback is invoked for any tags for which there is no built-in decoder. The return value is substituted for the :class:`CBORTag` object in the deserialized output :param object_hook: callable that takes 2 arguments: the decoder instance, and a dictionary. This callback is invoked for each deserialized :class:`dict` object. The return value is substituted for the dict in the deserialized output. .. _CBOR: https://cbor.io/ """ __slots__ = ( '_tag_hook', '_object_hook', '_share_index', '_shareables', '_fp_read', '_immutable', '_str_errors', '_stringref_namespace') def __init__(self, fp, tag_hook=None, object_hook=None, str_errors='strict'): self.fp = fp self.tag_hook = tag_hook self.object_hook = object_hook self.str_errors = str_errors self._share_index = None self._shareables = [] self._stringref_namespace = None self._immutable = False @property def immutable(self): """ Used by decoders to check if the calling context requires an immutable type. Object_hook or tag_hook should raise an exception if this flag is set unless the result can be safely used as a dict key. """ return self._immutable @property def fp(self): return self._fp_read.__self__ @fp.setter def fp(self, value): try: if not callable(value.read): raise ValueError('fp.read is not callable') except AttributeError: raise ValueError('fp object has no read method') else: self._fp_read = value.read @property def tag_hook(self): return self._tag_hook @tag_hook.setter def tag_hook(self, value): if value is None or callable(value): self._tag_hook = value else: raise ValueError('tag_hook must be None or a callable') @property def object_hook(self): return self._object_hook @object_hook.setter def object_hook(self, value): if value is None or callable(value): self._object_hook = value else: raise ValueError('object_hook must be None or a callable') @property def str_errors(self): return self._str_errors @str_errors.setter def str_errors(self, value): if value in ('strict', 'error', 'replace'): self._str_errors = value else: raise ValueError( "invalid str_errors value {!r} (must be one of 'strict', " "'error', or 'replace')".format(value)) def set_shareable(self, value): """ Set the shareable value for the last encountered shared value marker, if any. If the current shared index is ``None``, nothing is done. :param value: the shared value :returns: the shared value to permit chaining """ if self._share_index is not None: self._shareables[self._share_index] = value return value def _stringref_namespace_add(self, string, length): if self._stringref_namespace is not None: next_index = len(self._stringref_namespace) if next_index < 24: is_referenced = length >= 3 elif next_index < 256: is_referenced = length >= 4 elif next_index < 65536: is_referenced = length >= 5 elif next_index < 4294967296: is_referenced = length >= 7 else: is_referenced = length >= 11 if is_referenced: self._stringref_namespace.append(string) def read(self, amount): """ Read bytes from the data stream. :param int amount: the number of bytes to read """ data = self._fp_read(amount) if len(data) < amount: raise CBORDecodeEOF( 'premature end of stream (expected to read {} bytes, got {} ' 'instead)'.format(amount, len(data))) return data def _decode(self, immutable=False, unshared=False): if immutable: old_immutable = self._immutable self._immutable = True if unshared: old_index = self._share_index self._share_index = None try: initial_byte = self.read(1)[0] major_type = initial_byte >> 5 subtype = initial_byte & 31 decoder = major_decoders[major_type] return decoder(self, subtype) finally: if immutable: self._immutable = old_immutable if unshared: self._share_index = old_index def decode(self): """ Decode the next value from the stream. :raises CBORDecodeError: if there is any problem decoding the stream """ return self._decode() def decode_from_bytes(self, buf): """ Wrap the given bytestring as a file and call :meth:`decode` with it as the argument. This method was intended to be used from the ``tag_hook`` hook when an object needs to be decoded separately from the rest but while still taking advantage of the shared value registry. """ with BytesIO(buf) as fp: old_fp = self.fp self.fp = fp retval = self._decode() self.fp = old_fp return retval def _decode_length(self, subtype, allow_indefinite=False): if subtype < 24: return subtype elif subtype == 24: return self.read(1)[0] elif subtype == 25: return struct.unpack('>H', self.read(2))[0] elif subtype == 26: return struct.unpack('>L', self.read(4))[0] elif subtype == 27: return struct.unpack('>Q', self.read(8))[0] elif subtype == 31 and allow_indefinite: return None else: raise CBORDecodeValueError( 'unknown unsigned integer subtype 0x%x' % subtype) def decode_uint(self, subtype): # Major tag 0 return self.set_shareable(self._decode_length(subtype)) def decode_negint(self, subtype): # Major tag 1 return self.set_shareable(-self._decode_length(subtype) - 1) def decode_bytestring(self, subtype): # Major tag 2 length = self._decode_length(subtype, allow_indefinite=True) if length is None: # Indefinite length buf = [] while True: initial_byte = self.read(1)[0] if initial_byte == 0xff: result = b''.join(buf) break elif initial_byte >> 5 == 2: length = self._decode_length(initial_byte & 0x1f) if length is None or length > sys.maxsize: raise CBORDecodeValueError( 'invalid length for indefinite bytestring chunk 0x%x' % length ) value = self.read(length) buf.append(value) else: raise CBORDecodeValueError( "non-bytestring found in indefinite length bytestring") else: if length > sys.maxsize: raise CBORDecodeValueError('invalid length for bytestring 0x%x' % length) result = self.read(length) self._stringref_namespace_add(result, length) return self.set_shareable(result) def decode_string(self, subtype): # Major tag 3 length = self._decode_length(subtype, allow_indefinite=True) if length is None: # Indefinite length # NOTE: It may seem redundant to repeat this code to handle UTF-8 # strings but there is a reason to do this separately to # byte-strings. Specifically, the CBOR spec states (in sec. 2.2): # # Text strings with indefinite lengths act the same as byte # strings with indefinite lengths, except that all their chunks # MUST be definite-length text strings. Note that this implies # that the bytes of a single UTF-8 character cannot be spread # between chunks: a new chunk can only be started at a # character boundary. # # This precludes using the indefinite bytestring decoder above as # that would happily ignore UTF-8 characters split across chunks. buf = [] while True: initial_byte = self.read(1)[0] if initial_byte == 0xff: result = ''.join(buf) break elif initial_byte >> 5 == 3: length = self._decode_length(initial_byte & 0x1f) if length is None or length > sys.maxsize: raise CBORDecodeValueError( 'invalid length for indefinite string chunk 0x%x' % length) value = self.read(length).decode('utf-8', self._str_errors) buf.append(value) else: raise CBORDecodeValueError( "non-string found in indefinite length string") else: if length > sys.maxsize: raise CBORDecodeValueError('invalid length for string 0x%x' % length) result = self.read(length).decode('utf-8', self._str_errors) self._stringref_namespace_add(result, length) return self.set_shareable(result) def decode_array(self, subtype): # Major tag 4 length = self._decode_length(subtype, allow_indefinite=True) if length is None: # Indefinite length items = [] if not self._immutable: self.set_shareable(items) while True: value = self._decode() if value is break_marker: break else: items.append(value) else: if length > sys.maxsize: raise CBORDecodeValueError('invalid length for array 0x%x' % length) items = [] if not self._immutable: self.set_shareable(items) for index in range(length): items.append(self._decode()) if self._immutable: items = tuple(items) self.set_shareable(items) return items def decode_map(self, subtype): # Major tag 5 length = self._decode_length(subtype, allow_indefinite=True) if length is None: # Indefinite length dictionary = {} self.set_shareable(dictionary) while True: key = self._decode(immutable=True, unshared=True) if key is break_marker: break else: dictionary[key] = self._decode(unshared=True) else: dictionary = {} self.set_shareable(dictionary) for _ in range(length): key = self._decode(immutable=True, unshared=True) dictionary[key] = self._decode(unshared=True) if self._object_hook: dictionary = self._object_hook(self, dictionary) self.set_shareable(dictionary) elif self._immutable: dictionary = FrozenDict(dictionary) self.set_shareable(dictionary) return dictionary def decode_semantic(self, subtype): # Major tag 6 tagnum = self._decode_length(subtype) semantic_decoder = semantic_decoders.get(tagnum) if semantic_decoder: return semantic_decoder(self) else: tag = CBORTag(tagnum, None) self.set_shareable(tag) tag.value = self._decode(unshared=True) if self._tag_hook: tag = self._tag_hook(self, tag) return self.set_shareable(tag) def decode_special(self, subtype): # Simple value if subtype < 20: # XXX Set shareable? return CBORSimpleValue(subtype) # Major tag 7 try: return special_decoders[subtype](self) except KeyError as e: raise CBORDecodeValueError( "Undefined Reserved major type 7 subtype 0x%x" % subtype) from e # # Semantic decoders (major tag 6) # def decode_datetime_string(self): # Semantic tag 0 value = self._decode() match = timestamp_re.match(value) if match: ( year, month, day, hour, minute, second, secfrac, offset_h, offset_m, ) = match.groups() if secfrac is None: microsecond = 0 else: microsecond = int('{:<06}'.format(secfrac)) if offset_h: tz = timezone(timedelta(hours=int(offset_h), minutes=int(offset_m))) else: tz = timezone.utc return self.set_shareable(datetime( int(year), int(month), int(day), int(hour), int(minute), int(second), microsecond, tz)) else: raise CBORDecodeValueError( 'invalid datetime string: {!r}'.format(value)) def decode_epoch_datetime(self): # Semantic tag 1 value = self._decode() return self.set_shareable(datetime.fromtimestamp(value, timezone.utc)) def decode_positive_bignum(self): # Semantic tag 2 from binascii import hexlify value = self._decode() return self.set_shareable(int(hexlify(value), 16)) def decode_negative_bignum(self): # Semantic tag 3 return self.set_shareable(-self.decode_positive_bignum() - 1) def decode_fraction(self): # Semantic tag 4 from decimal import Decimal try: exp, sig = self._decode() except (TypeError, ValueError) as e: raise CBORDecodeValueError("Incorrect tag 4 payload") from e tmp = Decimal(sig).as_tuple() return self.set_shareable(Decimal((tmp.sign, tmp.digits, exp))) def decode_bigfloat(self): # Semantic tag 5 from decimal import Decimal try: exp, sig = self._decode() except (TypeError, ValueError) as e: raise CBORDecodeValueError("Incorrect tag 5 payload") from e return self.set_shareable(Decimal(sig) * (2 ** Decimal(exp))) def decode_stringref(self): # Semantic tag 25 if self._stringref_namespace is None: raise CBORDecodeValueError('string reference outside of namespace') index = self._decode() try: value = self._stringref_namespace[index] except IndexError: raise CBORDecodeValueError('string reference %d not found' % index) return value def decode_shareable(self): # Semantic tag 28 old_index = self._share_index self._share_index = len(self._shareables) self._shareables.append(None) try: return self._decode() finally: self._share_index = old_index def decode_sharedref(self): # Semantic tag 29 value = self._decode(unshared=True) try: shared = self._shareables[value] except IndexError: raise CBORDecodeValueError('shared reference %d not found' % value) if shared is None: raise CBORDecodeValueError( 'shared value %d has not been initialized' % value) else: return shared def decode_rational(self): # Semantic tag 30 from fractions import Fraction return self.set_shareable(Fraction(*self._decode())) def decode_regexp(self): # Semantic tag 35 return self.set_shareable(re.compile(self._decode())) def decode_mime(self): # Semantic tag 36 from email.parser import Parser return self.set_shareable(Parser().parsestr(self._decode())) def decode_uuid(self): # Semantic tag 37 from uuid import UUID return self.set_shareable(UUID(bytes=self._decode())) def decode_stringref_namespace(self): # Semantic tag 256 old_namespace = self._stringref_namespace self._stringref_namespace = [] value = self._decode() self._stringref_namespace = old_namespace return value def decode_set(self): # Semantic tag 258 if self._immutable: return self.set_shareable(frozenset(self._decode(immutable=True))) else: return self.set_shareable(set(self._decode(immutable=True))) def decode_ipaddress(self): # Semantic tag 260 from ipaddress import ip_address buf = self.decode() if not isinstance(buf, bytes) or len(buf) not in (4, 6, 16): raise CBORDecodeValueError("invalid ipaddress value %r" % buf) elif len(buf) in (4, 16): return self.set_shareable(ip_address(buf)) elif len(buf) == 6: # MAC address return self.set_shareable(CBORTag(260, buf)) def decode_ipnetwork(self): # Semantic tag 261 from ipaddress import ip_network net_map = self.decode() if isinstance(net_map, Mapping) and len(net_map) == 1: for net in net_map.items(): try: return self.set_shareable(ip_network(net, strict=False)) except (TypeError, ValueError): break raise CBORDecodeValueError("invalid ipnetwork value %r" % net_map) def decode_self_describe_cbor(self): # Semantic tag 55799 return self._decode() # # Special decoders (major tag 7) # def decode_simple_value(self): # XXX Set shareable? return CBORSimpleValue(self.read(1)[0]) def decode_float16(self): payload = self.read(2) value = struct.unpack('>e', payload)[0] return self.set_shareable(value) def decode_float32(self): return self.set_shareable(struct.unpack('>f', self.read(4))[0]) def decode_float64(self): return self.set_shareable(struct.unpack('>d', self.read(8))[0]) major_decoders = { 0: CBORDecoder.decode_uint, 1: CBORDecoder.decode_negint, 2: CBORDecoder.decode_bytestring, 3: CBORDecoder.decode_string, 4: CBORDecoder.decode_array, 5: CBORDecoder.decode_map, 6: CBORDecoder.decode_semantic, 7: CBORDecoder.decode_special, } special_decoders = { 20: lambda self: False, 21: lambda self: True, 22: lambda self: None, 23: lambda self: undefined, 24: CBORDecoder.decode_simple_value, 25: CBORDecoder.decode_float16, 26: CBORDecoder.decode_float32, 27: CBORDecoder.decode_float64, 31: lambda self: break_marker, } semantic_decoders = { 0: CBORDecoder.decode_datetime_string, 1: CBORDecoder.decode_epoch_datetime, 2: CBORDecoder.decode_positive_bignum, 3: CBORDecoder.decode_negative_bignum, 4: CBORDecoder.decode_fraction, 5: CBORDecoder.decode_bigfloat, 25: CBORDecoder.decode_stringref, 28: CBORDecoder.decode_shareable, 29: CBORDecoder.decode_sharedref, 30: CBORDecoder.decode_rational, 35: CBORDecoder.decode_regexp, 36: CBORDecoder.decode_mime, 37: CBORDecoder.decode_uuid, 256: CBORDecoder.decode_stringref_namespace, 258: CBORDecoder.decode_set, 260: CBORDecoder.decode_ipaddress, 261: CBORDecoder.decode_ipnetwork, 55799: CBORDecoder.decode_self_describe_cbor, } def loads(s, **kwargs): """ Deserialize an object from a bytestring. :param bytes s: the bytestring to deserialize :param kwargs: keyword arguments passed to :class:`CBORDecoder` :return: the deserialized object """ with BytesIO(s) as fp: return CBORDecoder(fp, **kwargs).decode() def load(fp, **kwargs): """ Deserialize an object from an open file. :param fp: the input file (any file-like object) :param kwargs: keyword arguments passed to :class:`CBORDecoder` :return: the deserialized object """ return CBORDecoder(fp, **kwargs).decode() cbor2-5.4.2/cbor2/encoder.py000066400000000000000000000555671413200260400155430ustar00rootroot00000000000000from __future__ import division import re import math import struct from collections import OrderedDict, defaultdict from contextlib import contextmanager from functools import wraps from datetime import datetime, date, time, tzinfo from io import BytesIO from sys import modules from .types import ( CBOREncodeTypeError, CBOREncodeValueError, CBORTag, undefined, CBORSimpleValue, FrozenDict) def shareable_encoder(func): """ Wrap the given encoder function to gracefully handle cyclic data structures. If value sharing is enabled, this marks the given value shared in the datastream on the first call. If the value has already been passed to this method, a reference marker is instead written to the data stream and the wrapped function is not called. If value sharing is disabled, only infinite recursion protection is done. """ @wraps(func) def wrapper(encoder, value): encoder.encode_shared(func, value) return wrapper def container_encoder(func): """ The given encoder is a container with child values. Handle cyclic or duplicate references to the value and strings within the value efficiently. Containers may contain cyclic data structures or may contain values or themselves by referenced multiple times throughout the greater encoded value and could thus be more efficiently encoded with shared value references and string references where duplication occurs. If value sharing is enabled, this marks the given value shared in the datastream on the first call. If the value has already been passed to this method, a reference marker is instead written to the data stream and the wrapped function is not called. If value sharing is disabled, only infinite recursion protection is done. If string referencing is enabled and this is the first use of this method in encoding a value, all repeated references to long strings and bytearrays will be replaced with references to the first occurrence of those arrays. If string referencing is disabled, all strings and bytearrays will be encoded directly. """ @wraps(func) def wrapper(encoder, value): encoder.encode_container(func, value) return wrapper class CBOREncoder: """ The CBOREncoder class implements a fully featured `CBOR`_ encoder with several extensions for handling shared references, big integers, rational numbers and so on. Typically the class is not used directly, but the :func:`dump` and :func:`dumps` functions are called to indirectly construct and use the class. When the class is constructed manually, the main entry points are :meth:`encode` and :meth:`encode_to_bytes`. :param bool datetime_as_timestamp: set to ``True`` to serialize datetimes as UNIX timestamps (this makes datetimes more concise on the wire, but loses the timezone information) :param datetime.tzinfo timezone: the default timezone to use for serializing naive datetimes; if this is not specified naive datetimes will throw a :exc:`ValueError` when encoding is attempted :param bool value_sharing: set to ``True`` to allow more efficient serializing of repeated values and, more importantly, cyclic data structures, at the cost of extra line overhead :param default: a callable that is called by the encoder with two arguments (the encoder instance and the value being encoded) when no suitable encoder has been found, and should use the methods on the encoder to encode any objects it wants to add to the data stream :param bool canonical: when True, use "canonical" CBOR representation; this typically involves sorting maps, sets, etc. into a pre-determined order ensuring that serializations are comparable without decoding :param bool date_as_datetime: set to ``True`` to serialize date objects as datetimes (CBOR tag 0), which was the default behavior in previous releases (cbor2 <= 4.1.2). :param bool string_referencing: set to ``True`` to allow more efficient serializing of repeated string values .. _CBOR: https://cbor.io/ """ __slots__ = ( 'datetime_as_timestamp', '_timezone', '_default', 'value_sharing', '_fp_write', '_shared_containers', '_encoders', '_canonical', 'string_referencing', 'string_namespacing', '_string_references') def __init__(self, fp, datetime_as_timestamp=False, timezone=None, value_sharing=False, default=None, canonical=False, date_as_datetime=False, string_referencing=False): self.fp = fp self.datetime_as_timestamp = datetime_as_timestamp self.timezone = timezone self.value_sharing = value_sharing self.string_referencing = string_referencing self.string_namespacing = string_referencing self.default = default self._canonical = canonical self._shared_containers = {} # indexes used for value sharing self._string_references = {} # indexes used for string references self._encoders = default_encoders.copy() if canonical: self._encoders.update(canonical_encoders) if date_as_datetime: self._encoders[date] = CBOREncoder.encode_date def _find_encoder(self, obj_type): for type_, enc in list(self._encoders.items()): if type(type_) is tuple: try: modname, typename = type_ except (TypeError, ValueError): raise CBOREncodeValueError( "invalid deferred encoder type {!r} (must be a " "2-tuple of module name and type name, e.g. " "('collections', 'defaultdict'))".format(type_)) imported_type = getattr(modules.get(modname), typename, None) if imported_type is not None: del self._encoders[type_] self._encoders[imported_type] = enc type_ = imported_type else: # pragma: nocover continue if issubclass(obj_type, type_): self._encoders[obj_type] = enc return enc return None @property def fp(self): return self._fp_write.__self__ @fp.setter def fp(self, value): try: if not callable(value.write): raise ValueError('fp.write is not callable') except AttributeError: raise ValueError('fp object has no write method') else: self._fp_write = value.write @property def timezone(self): return self._timezone @timezone.setter def timezone(self, value): if value is None or isinstance(value, tzinfo): self._timezone = value else: raise ValueError('timezone must be None or a tzinfo instance') @property def default(self): return self._default @default.setter def default(self, value): if value is None or callable(value): self._default = value else: raise ValueError('default must be None or a callable') @property def canonical(self): return self._canonical @contextmanager def disable_value_sharing(self): """ Disable value sharing in the encoder for the duration of the context block. """ old_value_sharing = self.value_sharing self.value_sharing = False yield self.value_sharing = old_value_sharing @contextmanager def disable_string_referencing(self): """ Disable tracking of string references for the duration of the context block. """ old_string_referencing = self.string_referencing self.string_referencing = False yield self.string_referencing = old_string_referencing @contextmanager def disable_string_namespacing(self): """ Disable generation of new string namespaces for the duration of the context block. """ old_string_namespacing = self.string_namespacing self.string_namespacing = False yield self.string_namespacing = old_string_namespacing def write(self, data): """ Write bytes to the data stream. :param bytes data: the bytes to write """ self._fp_write(data) def encode(self, obj): """ Encode the given object using CBOR. :param obj: the object to encode """ obj_type = obj.__class__ encoder = ( self._encoders.get(obj_type) or self._find_encoder(obj_type) or self._default ) if not encoder: raise CBOREncodeTypeError( 'cannot serialize type %s' % obj_type.__name__) encoder(self, obj) def encode_to_bytes(self, obj): """ Encode the given object to a byte buffer and return its value as bytes. This method was intended to be used from the ``default`` hook when an object needs to be encoded separately from the rest but while still taking advantage of the shared value registry. """ with BytesIO() as fp: old_fp = self.fp self.fp = fp self.encode(obj) self.fp = old_fp return fp.getvalue() def encode_container(self, encoder, value): if self.string_namespacing: # Create a new string reference domain self.encode_length(6, 256) with self.disable_string_namespacing(): self.encode_shared(encoder, value) def encode_shared(self, encoder, value): value_id = id(value) try: index = self._shared_containers[id(value)][1] except KeyError: if self.value_sharing: # Mark the container as shareable self._shared_containers[value_id] = ( value, len(self._shared_containers) ) self.encode_length(6, 0x1c) encoder(self, value) else: self._shared_containers[value_id] = (value, None) try: encoder(self, value) finally: del self._shared_containers[value_id] else: if self.value_sharing: # Generate a reference to the previous index instead of # encoding this again self.encode_length(6, 0x1d) self.encode_int(index) else: raise CBOREncodeValueError( 'cyclic data structure detected but value sharing is ' 'disabled') def _stringref(self, value): """ Try to encode the string or bytestring as a reference. Returns True if a reference was generated, False if the string must still be emitted. """ try: index = self._string_references[value] self.encode_semantic(CBORTag(25, index)) return True except KeyError: length = len(value) next_index = len(self._string_references) if next_index < 24: is_referenced = length >= 3 elif next_index < 256: is_referenced = length >= 4 elif next_index < 65536: is_referenced = length >= 5 elif next_index < 4294967296: is_referenced = length >= 7 else: is_referenced = length >= 11 if is_referenced: self._string_references[value] = next_index return False def encode_length(self, major_tag, length): major_tag <<= 5 if length < 24: self._fp_write(struct.pack('>B', major_tag | length)) elif length < 256: self._fp_write(struct.pack('>BB', major_tag | 24, length)) elif length < 65536: self._fp_write(struct.pack('>BH', major_tag | 25, length)) elif length < 4294967296: self._fp_write(struct.pack('>BL', major_tag | 26, length)) else: self._fp_write(struct.pack('>BQ', major_tag | 27, length)) def encode_int(self, value): # Big integers (2 ** 64 and over) if value >= 18446744073709551616 or value < -18446744073709551616: if value >= 0: major_type = 0x02 else: major_type = 0x03 value = -value - 1 payload = value.to_bytes((value.bit_length() + 7) // 8, 'big') self.encode_semantic(CBORTag(major_type, payload)) elif value >= 0: self.encode_length(0, value) else: self.encode_length(1, -(value + 1)) def encode_bytestring(self, value): if self.string_referencing: if self._stringref(value): return self.encode_length(2, len(value)) self._fp_write(value) def encode_bytearray(self, value): self.encode_bytestring(bytes(value)) def encode_string(self, value): if self.string_referencing: if self._stringref(value): return encoded = value.encode('utf-8') self.encode_length(3, len(encoded)) self._fp_write(encoded) @container_encoder def encode_array(self, value): self.encode_length(4, len(value)) for item in value: self.encode(item) @container_encoder def encode_map(self, value): self.encode_length(5, len(value)) for key, val in value.items(): self.encode(key) self.encode(val) def encode_sortable_key(self, value): """ Takes a key and calculates the length of its optimal byte representation, along with the representation itself. This is used as the sorting key in CBOR's canonical representations. """ with self.disable_string_referencing(): encoded = self.encode_to_bytes(value) return len(encoded), encoded @container_encoder def encode_canonical_map(self, value): "Reorder keys according to Canonical CBOR specification" keyed_keys = ( (self.encode_sortable_key(key), key, value) for key, value in value.items() ) self.encode_length(5, len(value)) for sortkey, realkey, value in sorted(keyed_keys): if self.string_referencing: # String referencing requires that the order encoded is # the same as the order emitted so string references are # generated after an order is determined self.encode(realkey) else: self._fp_write(sortkey[1]) self.encode(value) def encode_semantic(self, value): # Nested string reference domains are distinct old_string_referencing = self.string_referencing old_string_references = self._string_references if value.tag == 256: self.string_referencing = True self._string_references = {} self.encode_length(6, value.tag) self.encode(value.value) self.string_referencing = old_string_referencing self._string_references = old_string_references # # Semantic decoders (major tag 6) # def encode_datetime(self, value): # Semantic tag 0 if not value.tzinfo: if self._timezone: value = value.replace(tzinfo=self._timezone) else: raise CBOREncodeValueError( 'naive datetime {!r} encountered and no default timezone ' 'has been set'.format(value)) if self.datetime_as_timestamp: from calendar import timegm if not value.microsecond: timestamp = timegm(value.utctimetuple()) else: timestamp = timegm(value.utctimetuple()) + value.microsecond / 1000000 self.encode_semantic(CBORTag(1, timestamp)) else: datestring = value.isoformat().replace('+00:00', 'Z') self.encode_semantic(CBORTag(0, datestring)) def encode_date(self, value): value = datetime.combine(value, time()).replace(tzinfo=self._timezone) self.encode_datetime(value) def encode_decimal(self, value): # Semantic tag 4 if value.is_nan(): self._fp_write(b'\xf9\x7e\x00') elif value.is_infinite(): self._fp_write(b'\xf9\x7c\x00' if value > 0 else b'\xf9\xfc\x00') else: dt = value.as_tuple() sig = 0 for digit in dt.digits: sig = (sig * 10) + digit if dt.sign: sig = -sig with self.disable_value_sharing(): self.encode_semantic(CBORTag(4, [dt.exponent, sig])) def encode_stringref(self, value): # Semantic tag 25 if not self._stringref(value): self.encode(value) def encode_rational(self, value): # Semantic tag 30 with self.disable_value_sharing(): self.encode_semantic(CBORTag(30, [value.numerator, value.denominator])) def encode_regexp(self, value): # Semantic tag 35 self.encode_semantic(CBORTag(35, str(value.pattern))) def encode_mime(self, value): # Semantic tag 36 self.encode_semantic(CBORTag(36, value.as_string())) def encode_uuid(self, value): # Semantic tag 37 self.encode_semantic(CBORTag(37, value.bytes)) def encode_stringref_namespace(self, value): # Semantic tag 256 with self.disable_string_namespacing(): self.encode_semantic(CBORTag(256, value)) def encode_set(self, value): # Semantic tag 258 self.encode_semantic(CBORTag(258, tuple(value))) def encode_canonical_set(self, value): # Semantic tag 258 values = sorted( (self.encode_sortable_key(key), key) for key in value ) self.encode_semantic(CBORTag(258, [key[1] for key in values])) def encode_ipaddress(self, value): # Semantic tag 260 self.encode_semantic(CBORTag(260, value.packed)) def encode_ipnetwork(self, value): # Semantic tag 261 self.encode_semantic( CBORTag(261, {value.network_address.packed: value.prefixlen})) # # Special encoders (major tag 7) # def encode_simple_value(self, value): if value.value < 20: self._fp_write(struct.pack('>B', 0xe0 | value.value)) else: self._fp_write(struct.pack('>BB', 0xf8, value.value)) def encode_float(self, value): # Handle special values efficiently if math.isnan(value): self._fp_write(b'\xf9\x7e\x00') elif math.isinf(value): self._fp_write(b'\xf9\x7c\x00' if value > 0 else b'\xf9\xfc\x00') else: self._fp_write(struct.pack('>Bd', 0xfb, value)) def encode_minimal_float(self, value): # Handle special values efficiently if math.isnan(value): self._fp_write(b'\xf9\x7e\x00') elif math.isinf(value): self._fp_write(b'\xf9\x7c\x00' if value > 0 else b'\xf9\xfc\x00') else: # Try each encoding in turn from longest to shortest encoded = struct.pack('>Bd', 0xfb, value) for format, tag in [('>Bf', 0xfa), ('>Be', 0xf9)]: try: new_encoded = struct.pack(format, tag, value) # Check if encoding as low-byte float loses precision if struct.unpack(format, new_encoded)[1] == value: encoded = new_encoded else: break except OverflowError: break self._fp_write(encoded) def encode_boolean(self, value): self._fp_write(b'\xf5' if value else b'\xf4') def encode_none(self, value): self._fp_write(b'\xf6') def encode_undefined(self, value): self._fp_write(b'\xf7') default_encoders = OrderedDict([ (bytes, CBOREncoder.encode_bytestring), (bytearray, CBOREncoder.encode_bytearray), (str, CBOREncoder.encode_string), (int, CBOREncoder.encode_int), (float, CBOREncoder.encode_float), (('decimal', 'Decimal'), CBOREncoder.encode_decimal), (bool, CBOREncoder.encode_boolean), (type(None), CBOREncoder.encode_none), (tuple, CBOREncoder.encode_array), (list, CBOREncoder.encode_array), (dict, CBOREncoder.encode_map), (defaultdict, CBOREncoder.encode_map), (OrderedDict, CBOREncoder.encode_map), (FrozenDict, CBOREncoder.encode_map), (type(undefined), CBOREncoder.encode_undefined), (datetime, CBOREncoder.encode_datetime), (type(re.compile('')), CBOREncoder.encode_regexp), (('fractions', 'Fraction'), CBOREncoder.encode_rational), (('email.message', 'Message'), CBOREncoder.encode_mime), (('uuid', 'UUID'), CBOREncoder.encode_uuid), (('ipaddress', 'IPv4Address'), CBOREncoder.encode_ipaddress), (('ipaddress', 'IPv6Address'), CBOREncoder.encode_ipaddress), (('ipaddress', 'IPv4Network'), CBOREncoder.encode_ipnetwork), (('ipaddress', 'IPv6Network'), CBOREncoder.encode_ipnetwork), (CBORSimpleValue, CBOREncoder.encode_simple_value), (CBORTag, CBOREncoder.encode_semantic), (set, CBOREncoder.encode_set), (frozenset, CBOREncoder.encode_set), ]) canonical_encoders = OrderedDict([ (float, CBOREncoder.encode_minimal_float), (dict, CBOREncoder.encode_canonical_map), (defaultdict, CBOREncoder.encode_canonical_map), (OrderedDict, CBOREncoder.encode_canonical_map), (FrozenDict, CBOREncoder.encode_canonical_map), (set, CBOREncoder.encode_canonical_set), (frozenset, CBOREncoder.encode_canonical_set), ]) def dumps(obj, **kwargs): """ Serialize an object to a bytestring. :param obj: the object to serialize :param kwargs: keyword arguments passed to :class:`~.CBOREncoder` :return: the serialized output :rtype: bytes """ with BytesIO() as fp: dump(obj, fp, **kwargs) return fp.getvalue() def dump(obj, fp, **kwargs): """ Serialize an object to a file. :param obj: the object to serialize :param fp: a file-like object :param kwargs: keyword arguments passed to :class:`~.CBOREncoder` """ CBOREncoder(fp, **kwargs).encode(obj) cbor2-5.4.2/cbor2/tool.py000066400000000000000000000144151413200260400150640ustar00rootroot00000000000000r"""Command-line tool for CBOR diagnostics and testing""" import argparse import base64 import decimal import fractions import io import ipaddress import json import re import sys import uuid from collections import OrderedDict from datetime import datetime from functools import partial from . import CBORDecoder, load from .types import FrozenDict try: from _cbor2 import CBORSimpleValue, CBORTag, undefined except ImportError: from .types import CBORSimpleValue, CBORTag, undefined default_encoders = OrderedDict( [ (bytes, lambda x: x.decode(encoding='utf-8', errors='backslashreplace')), (decimal.Decimal, str), (FrozenDict, lambda x: str(dict(x))), (CBORSimpleValue, lambda x: 'cbor_simple:{:d}'.format(x.value)), (type(undefined), lambda x: 'cbor:undef'), (datetime, lambda x: x.isoformat()), (fractions.Fraction, str), (uuid.UUID, lambda x: x.urn), (CBORTag, lambda x: {'CBORTag:{:d}'.format(x.tag): x.value}), (set, list), (re.compile('').__class__, lambda x: x.pattern), (ipaddress.IPv4Address, str), (ipaddress.IPv6Address, str), (ipaddress.IPv4Network, str), (ipaddress.IPv6Network, str) ] ) def tag_hook(decoder, tag, ignore_tags=set()): if tag.tag in ignore_tags: return tag.value if tag.tag == 24: return decoder.decode_from_bytes(tag.value) else: if decoder.immutable: return 'CBORtag:{}:{}'.format(tag.tag, tag.value) return tag class DefaultEncoder(json.JSONEncoder): def default(self, v): obj_type = v.__class__ encoder = default_encoders.get(obj_type) if encoder: return encoder(v) return json.JSONEncoder.default(self, v) def iterdecode(f, *args, **kwargs): decoder = CBORDecoder(f, *args, **kwargs) while True: try: yield decoder.decode() except EOFError: return def key_to_str(d, dict_ids=None): dict_ids = set(dict_ids or []) rval = {} if not isinstance(d, dict): if isinstance(d, CBORSimpleValue): v = 'cbor_simple:{:d}'.format(d.value) return v if isinstance(d, (tuple, list, set)): if id(d) in dict_ids: raise ValueError("Cannot convert self-referential data to JSON") else: dict_ids.add(id(d)) v = [key_to_str(x, dict_ids) for x in d] dict_ids.remove(id(d)) return v else: return d if id(d) in dict_ids: raise ValueError("Cannot convert self-referential data to JSON") else: dict_ids.add(id(d)) for k, v in d.items(): if isinstance(k, bytes): k = k.decode(encoding='utf-8', errors='backslashreplace') if isinstance(k, CBORSimpleValue): k = 'cbor_simple:{:d}'.format(k.value) if isinstance(k, (FrozenDict, frozenset, tuple)): k = str(k) if isinstance(v, dict): v = key_to_str(v, dict_ids) elif isinstance(v, (tuple, list, set)): v = [key_to_str(x, dict_ids) for x in v] rval[k] = v return rval def main(): prog = 'python -m cbor2.tool' description = ( 'A simple command line interface for cbor2 module ' 'to validate and pretty-print CBOR objects.' ) parser = argparse.ArgumentParser(prog=prog, description=description) parser.add_argument( '-o', '--outfile', type=str, help='output file', default='-' ) parser.add_argument( 'infiles', nargs='*', type=argparse.FileType('rb'), help='Collection of CBOR files to process or - for stdin', ) parser.add_argument( '-k', '--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key', ) parser.add_argument( '-p', '--pretty', action='store_true', default=False, help='indent the output to look good' ) parser.add_argument( '-s', '--sequence', action='store_true', default=False, help='Parse a sequence of concatenated CBOR items', ) parser.add_argument( '-d', '--decode', action='store_true', default=False, help='CBOR data is base64 encoded (handy for stdin)', ) parser.add_argument( '-i', '--tag-ignore', type=str, help='Comma separated list of tags to ignore and only return the value', ) options = parser.parse_args() outfile = options.outfile sort_keys = options.sort_keys pretty = options.pretty sequence = options.sequence decode = options.decode infiles = options.infiles or [sys.stdin] closefd = True if outfile == '-': outfile = 1 closefd = False opener = dict(mode='w', encoding='utf-8', errors='backslashreplace', closefd=closefd) dumpargs = dict(ensure_ascii=False) if options.tag_ignore: ignore_s = options.tag_ignore.split(',') droptags = set(int(n) for n in ignore_s if (len(n) and n[0].isdigit())) else: droptags = set() my_hook = partial(tag_hook, ignore_tags=droptags) with io.open(outfile, **opener) as outfile: for infile in infiles: if hasattr(infile, 'buffer') and not decode: infile = infile.buffer with infile: if decode: infile = io.BytesIO(base64.b64decode(infile.read())) try: if sequence: objs = iterdecode(infile, tag_hook=my_hook) else: objs = (load(infile, tag_hook=my_hook),) for obj in objs: json.dump( key_to_str(obj), outfile, sort_keys=sort_keys, indent=(None, 4)[pretty], cls=DefaultEncoder, **dumpargs ) outfile.write('\n') except (ValueError, EOFError) as e: # pragma: no cover raise SystemExit(e) if __name__ == '__main__': # pragma: no cover main() cbor2-5.4.2/cbor2/types.py000066400000000000000000000110631413200260400152470ustar00rootroot00000000000000from collections import namedtuple from collections.abc import Mapping from functools import total_ordering from reprlib import recursive_repr class CBORError(Exception): "Base class for errors that occur during CBOR encoding or decoding." class CBOREncodeError(CBORError): "Raised for exceptions occurring during CBOR encoding." class CBOREncodeTypeError(CBOREncodeError, TypeError): "Raised when attempting to encode a type that cannot be serialized." class CBOREncodeValueError(CBOREncodeError, ValueError): "Raised when the CBOR encoder encounters an invalid value." class CBORDecodeError(CBORError): "Raised for exceptions occurring during CBOR decoding." class CBORDecodeValueError(CBORDecodeError, ValueError): "Raised when the CBOR stream being decoded contains an invalid value." class CBORDecodeEOF(CBORDecodeError, EOFError): "Raised when decoding unexpectedly reaches EOF." @total_ordering class CBORTag: """ Represents a CBOR semantic tag. :param int tag: tag number :param value: encapsulated value (any object) """ __slots__ = 'tag', 'value' def __init__(self, tag, value): if not isinstance(tag, int) or tag not in range(2**64): raise TypeError('CBORTag tags must be positive integers less than 2**64') self.tag = tag self.value = value def __eq__(self, other): if isinstance(other, CBORTag): return (self.tag, self.value) == (other.tag, other.value) return NotImplemented def __le__(self, other): if isinstance(other, CBORTag): return (self.tag, self.value) <= (other.tag, other.value) return NotImplemented @recursive_repr() def __repr__(self): return 'CBORTag({self.tag}, {self.value!r})'.format(self=self) class CBORSimpleValue(namedtuple('CBORSimpleValue', ['value'])): """ Represents a CBOR "simple value". :param int value: the value (0-255) """ __slots__ = () __hash__ = namedtuple.__hash__ def __new__(cls, value): if value < 0 or value > 255: raise TypeError('simple value out of range (0..255)') return super(CBORSimpleValue, cls).__new__(cls, value) def __eq__(self, other): if isinstance(other, int): return self.value == other return super(CBORSimpleValue, self).__eq__(other) def __ne__(self, other): if isinstance(other, int): return self.value != other return super(CBORSimpleValue, self).__ne__(other) def __lt__(self, other): if isinstance(other, int): return self.value < other return super(CBORSimpleValue, self).__lt__(other) def __le__(self, other): if isinstance(other, int): return self.value <= other return super(CBORSimpleValue, self).__le__(other) def __ge__(self, other): if isinstance(other, int): return self.value >= other return super(CBORSimpleValue, self).__ge__(other) def __gt__(self, other): if isinstance(other, int): return self.value > other return super(CBORSimpleValue, self).__gt__(other) class FrozenDict(Mapping): """ A hashable, immutable mapping type. The arguments to ``FrozenDict`` are processed just like those to ``dict``. """ def __init__(self, *args, **kwargs): self._d = dict(*args, **kwargs) self._hash = None def __iter__(self): return iter(self._d) def __len__(self): return len(self._d) def __getitem__(self, key): return self._d[key] def __repr__(self): return "%s(%s)" % (self.__class__.__name__, self._d) def __hash__(self): if self._hash is None: self._hash = hash((frozenset(self), frozenset(self.values()))) return self._hash class UndefinedType: __slots__ = () def __new__(cls): try: return undefined except NameError: return super(UndefinedType, cls).__new__(cls) def __repr__(self): return "undefined" def __bool__(self): return False __nonzero__ = __bool__ # Py2.7 compat class BreakMarkerType: __slots__ = () def __new__(cls): try: return break_marker except NameError: return super(BreakMarkerType, cls).__new__(cls) def __repr__(self): return "break_marker" def __bool__(self): return True __nonzero__ = __bool__ # Py2.7 compat #: Represents the "undefined" value. undefined = UndefinedType() break_marker = BreakMarkerType() cbor2-5.4.2/docs/000077500000000000000000000000001413200260400134515ustar00rootroot00000000000000cbor2-5.4.2/docs/conf.py000066400000000000000000000012611413200260400147500ustar00rootroot00000000000000#!/usr/bin/env python import pkg_resources extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx' ] templates_path = ['_templates'] source_suffix = '.rst' master_doc = 'index' project = 'cbor2' author = 'Alex Grönholm' copyright = '2016, ' + author v = pkg_resources.get_distribution(project).parsed_version version = v.base_version release = v.public language = None exclude_patterns = ['_build'] pygments_style = 'sphinx' highlight_language = 'python' todo_include_todos = False html_theme = 'sphinx_rtd_theme' html_static_path = ['_static'] htmlhelp_basename = project.replace('-', '') + 'doc' intersphinx_mapping = {'python': ('http://docs.python.org/', None)} cbor2-5.4.2/docs/customizing.rst000066400000000000000000000140221413200260400165550ustar00rootroot00000000000000Customizing encoding and decoding ================================= Both the encoder and decoder can be customized to support a wider range of types. On the encoder side, this is accomplished by passing a callback as the ``default`` constructor argument. This callback will receive an object that the encoder could not serialize on its own. The callback should then return a value that the encoder can serialize on its own, although the return value is allowed to contain objects that also require the encoder to use the callback, as long as it won't result in an infinite loop. On the decoder side, you have two options: ``tag_hook`` and ``object_hook``. The former is called by the decoder to process any semantic tags that have no predefined decoders. The latter is called for any newly decoded ``dict`` objects, and is mostly useful for implementing a JSON compatible custom type serialization scheme. Unless your requirements restrict you to JSON compatible types only, it is recommended to use ``tag_hook`` for this purpose. Using the CBOR tags for custom types ------------------------------------ The most common way to use ``default`` is to call :meth:`~cbor2.encoder.CBOREncoder.encode` to add a custom tag in the data stream, with the payload as the value:: class Point: def __init__(self, x, y): self.x = x self.y = y def default_encoder(encoder, value): # Tag number 4000 was chosen arbitrarily encoder.encode(CBORTag(4000, [value.x, value.y])) The corresponding ``tag_hook`` would be:: def tag_hook(decoder, tag, shareable_index=None): if tag.tag != 4000: return tag # tag.value is now the [x, y] list we serialized before return Point(*tag.value) Using dicts to carry custom types --------------------------------- The same could be done with ``object_hook``, except less efficiently:: def default_encoder(encoder, value): encoder.encode(dict(typename='Point', x=value.x, y=value.y)) def object_hook(decoder, value): if value.get('typename') != 'Point': return value return Point(value['x'], value['y']) You should make sure that whatever way you decide to use for telling apart your "specially marked" dicts from arbitrary data dicts won't mistake on for the other. Value sharing with custom types ------------------------------- In order to properly encode and decode cyclic references with custom types, some special care has to be taken. Suppose you have a custom type as below, where every child object contains a reference to its parent and the parent contains a list of children:: from cbor2 import dumps, loads, shareable_encoder, CBORTag class MyType: def __init__(self, parent=None): self.parent = parent self.children = [] if parent: self.parent.children.append(self) This would not normally be serializable, as it would lead to an endless loop (in the worst case) and raise some exception (in the best case). Now, enter CBOR's extension tags 28 and 29. These tags make it possible to add special markers into the data stream which can be later referenced and substituted with the object marked earlier. To do this, in ``default`` hooks used with the encoder you will need to use the :meth:`~cbor2.encoder.shareable_encoder` decorator on your ``default`` hook function. It will automatically automatically add the object to the shared values registry on the encoder and prevent it from being serialized twice (instead writing a reference to the data stream):: @shareable_encoder def default_encoder(encoder, value): # The state has to be serialized separately so that the decoder would have a chance to # create an empty instance before the shared value references are decoded serialized_state = encoder.encode_to_bytes(value.__dict__) encoder.encode(CBORTag(3000, serialized_state)) On the decoder side, you will need to initialize an empty instance for shared value lookup before the object's state (which may contain references to it) is decoded. This is done with the :meth:`~cbor2.encoder.CBORDecoder.set_shareable` method:: def tag_hook(decoder, tag, shareable_index=None): # Return all other tags as-is if tag.tag != 3000: return tag # Create a raw instance before initializing its state to make it possible for cyclic # references to work instance = MyType.__new__(MyType) decoder.set_shareable(shareable_index, instance) # Separately decode the state of the new object and then apply it state = decoder.decode_from_bytes(tag.value) instance.__dict__.update(state) return instance You could then verify that the cyclic references have been restored after deserialization:: parent = MyType() child1 = MyType(parent) child2 = MyType(parent) serialized = dumps(parent, default=default_encoder, value_sharing=True) new_parent = loads(serialized, tag_hook=tag_hook) assert new_parent.children[0].parent is new_parent assert new_parent.children[1].parent is new_parent Decoding Tagged items as keys ----------------------------- Since the CBOR specification allows any type to be used as a key in the mapping type, the decoder provides a flag that indicates it is expecting an immutable (and by implication hashable) type. If your custom class cannot be used this way you can raise an exception if this flag is set:: def tag_hook(decoder, tag, shareable_index=None): if tag.tag != 3000: return tag if decoder.immutable: raise CBORDecodeException('MyType cannot be used as a key or set member') return MyType(*tag.value) An example where the data could be used as a dict key:: from collections import namedtuple Pair = namedtuple('Pair', 'first second') def tag_hook(decoder, tag, shareable_index=None): if tag.tag != 4000: return tag return Pair(*tag.value) The ``object_hook`` can check for the immutable flag in the same way. cbor2-5.4.2/docs/index.rst000066400000000000000000000004221413200260400153100ustar00rootroot00000000000000.. include:: ../README.rst :start-line: 10 Table of contents ----------------- .. toctree:: :maxdepth: 2 usage customizing versionhistory Encoder Decoder Types * :ref:`API reference ` cbor2-5.4.2/docs/modules/000077500000000000000000000000001413200260400151215ustar00rootroot00000000000000cbor2-5.4.2/docs/modules/decoder.rst000066400000000000000000000001271413200260400172600ustar00rootroot00000000000000:mod:`cbor2.decoder` ==================== .. automodule:: cbor2.decoder :members: cbor2-5.4.2/docs/modules/encoder.rst000066400000000000000000000001271413200260400172720ustar00rootroot00000000000000:mod:`cbor2.encoder` ==================== .. automodule:: cbor2.encoder :members: cbor2-5.4.2/docs/modules/types.rst000066400000000000000000000001211413200260400170110ustar00rootroot00000000000000:mod:`cbor2.types` ================== .. automodule:: cbor2.types :members: cbor2-5.4.2/docs/usage.rst000066400000000000000000000130721413200260400153120ustar00rootroot00000000000000Basic usage =========== Serializing and deserializing with cbor2 is pretty straightforward:: from cbor2 import dumps, loads # Serialize an object as a bytestring data = dumps(['hello', 'world']) # Deserialize a bytestring obj = loads(data) # Efficiently deserialize from a file with open('input.cbor', 'rb') as fp: obj = load(fp) # Efficiently serialize an object to a file with open('output.cbor', 'wb') as fp: dump(obj, fp) Some data types, however, require extra considerations, as detailed below. Date/time handling ------------------ The CBOR specification does not support naïve datetimes (that is, datetimes where ``tzinfo`` is missing). When the encoder encounters such a datetime, it needs to know which timezone it belongs to. To this end, you can specify a default timezone by passing a :class:`~datetime.tzinfo` instance to :func:`~cbor2.encoder.dump`/:func:`~cbor2.encoder.dumps` call as the ``timezone`` argument. Decoded datetimes are always timezone aware. By default, datetimes are serialized in a manner that retains their timezone offsets. You can optimize the data stream size by passing ``datetime_as_timestamp=False`` to :func:`~cbor2.encoder.dump`/:func:`~cbor2.encoder.dumps`, but this causes the timezone offset information to be lost. In versions prior to 4.2 the encoder would convert a ``datetime.date`` object into a ``datetime.datetime`` prior to writing. This can cause confusion on decoding so this has been disabled by default in the next version. The behaviour can be re-enabled as follows:: from cbor2 import dumps from datetime import date, timezone # Serialize dates as datetimes encoded = dumps(date(2019, 10, 28), timezone=timezone.utc, date_as_datetime=True) A default timezone offset must be provided also. Cyclic (recursive) data structures ---------------------------------- If the encoder encounters a shareable object (ie. list or dict) that it has seen before, it will by default raise :exc:`~cbor2.encoder.CBOREncodeError` indicating that a cyclic reference has been detected and value sharing was not enabled. CBOR has, however, an extension specification that allows the encoder to reference a previously encoded value without processing it again. This makes it possible to serialize such cyclic references, but value sharing has to be enabled by passing ``value_sharing=True`` to :func:`~cbor2.encoder.dump`/:func:`~cbor2.encoder.dumps`. .. warning:: Support for value sharing is rare in other CBOR implementations, so think carefully whether you want to enable it. It also causes some line overhead, as all potentially shareable values must be tagged as such. String references ----------------- When ``string_referencing=True`` is passed to :func:`~cbor2.encoder.dump`/:func:`~cbor2.encoder.dumps`, if the encoder would encode a string that it has previously encoded and where a reference would be shorter than the encoded string, it instead encodes a reference to the nth sufficiently long string already encoded. .. warning:: Support for string referencing is rare in other CBOR implementations, so think carefully whether you want to enable it. Tag support ----------- In addition to all standard CBOR tags, this library supports many extended tags: ===== ======================================== ==================================================== Tag Semantics Python type(s) ===== ======================================== ==================================================== 0 Standard date/time string datetime.date / datetime.datetime 1 Epoch-based date/time datetime.date / datetime.datetime 2 Positive bignum int / long 3 Negative bignum int / long 4 Decimal fraction decimal.Decimal 5 Bigfloat decimal.Decimal 25 String reference str / bytes 28 Mark shared value N/A 29 Reference shared value N/A 30 Rational number fractions.Fraction 35 Regular expression ``_sre.SRE_Pattern`` (result of ``re.compile(...)``) 36 MIME message email.message.Message 37 Binary UUID uuid.UUID 256 String reference namespace N/A 258 Set of unique items set 260 Network address :class:`ipaddress.IPv4Address` (or IPv6) 261 Network prefix :class:`ipaddress.IPv4Network` (or IPv6) 55799 Self-Described CBOR object ===== ======================================== ==================================================== Arbitary tags can be represented with the :class:`~cbor2.types.CBORTag` class. If you want to write a file that is detected as CBOR by the Unix ``file`` utility, wrap your data in a `~cbor2.types.CBORTag` object like so:: from cbor2 import dump, CBORTag with open('output.cbor', 'wb') as fp: dump(CBORTag(55799, obj), fp) This will be ignored on decode and the original data content will be returned. Use Cases --------- Here are some things that the cbor2 library could be (and in some cases, is being) used for: - Experimenting with network protocols based on CBOR encoding - Designing new data storage formats - Submitting binary documents to ElasticSearch without base64 encoding overhead - Storing and validating file metadata in a secure backup system - RPC which supports Decimals with low overhead cbor2-5.4.2/docs/versionhistory.rst000066400000000000000000000162141413200260400173160ustar00rootroot00000000000000Version history =============== .. currentmodule:: cbor2 This library adheres to `Semantic Versioning `_. **5.4.2** (2021-10-14) - Fix segfault when initializing CBORTag with incorrect arguments (Sekenre) - Fix sphinx build warnings (Sekenre) **5.4.1** (2021-07-23) - Fix SystemErrors when using C-backend, meaningful exceptions now raised (Sekenre) - Fix precision loss when decoding base10 decimal fractions (Sekenre) - Made CBORTag handling consistent between python and C-module (Sekenre) **5.4.0** (2021-06-04) - Fix various bounds checks in the C-backend (Sekenre) - More testing of invalid/corrupted data (Sekenre) - Support for `String References `_ (xurtis) - Update Docs to refer to new RFC8949 **5.3.0** (2021-05-18) - Removed support for Python < 3.6 **5.2.0** (2020-09-30) - Final version tested with Python 2.7 and 3.5 - README: Announce deprecation of Python 2.7, 3.5 - README: More detail and examples - Bugfix: Fix segfault on loading huge arrays with C-backend (Sekenre) - Build system: Allow packagers to force C-backend building or disable using env var (jameshilliard) - Feature: :py:mod:`cbor2.tool` Command line diagnostic tool (Sekenre) - Feature: Ignore semantic tag used for file magic 55799 AKA "Self-Described CBOR" (kalcutter) **5.1.2** (2020-07-21) - Bugfix: Refcount bug in C lib causing intermittent segfaults on shutdown (tdryer) **5.1.1** (2020-07-03) - Build system: Making C lib optional if it fails to compile (chiefnoah) - Build system: Better Glibc version detection (Sekenre and JayH5) - Tests: Positive and negative bignums (kalcutter) - Bugfix: Fractional seconds parsing in datetimes (kalcutter) **5.1.0** (2020-03-18) - Minor API change :class:`CBORSimpleValue` is now a subclass of namedtuple and allows all numeric comparisons. This brings functional parity between C and Python modules. - Fixes for C-module on big-endian systems including floating point decoding, smallint encoding, and boolean argument handling. Tested on s390x and MIPS32. - Increase version requred of setuptools during install due to unicode errors. **5.0.1** (2020-01-21) - Fix deprecation warning on python 3.7, 3.8 (mariano54) - Minor documentation tweaks **5.0.0** (2020-01-20) - **BACKWARD INCOMPATIBLE** CBOR does not have a bare DATE type, encoding dates as datetimes is disabled by default (PR by Changaco) - **BACKWARD INCOMPATIBLE** :meth:`~CBORDecoder.set_shareable` only takes the instance to share, not the shareable's index - **BACKWARD INCOMPATIBLE** :exc:`CBORError` now descends from :exc:`Exception` rather than :exc:`ValueError`; however, subordinate exceptions now descend from :exc:`ValueError` (where appropriate) so most users should notice no difference - **BACKWARD INCOMPATIBLE** :class:`CBORDecoder` can now raise :exc:`CBORDecodeEOF` which descends from :exc:`EOFError` supporting streaming applications - Optional Pure C implementation by waveform80 that functions identically to the pure Python implementation with further contributions from: toravir, jonashoechst, Changaco - Drop Python 3.3 and 3.4 support from the build process; they should still work if built from source but are no longer officially supported - Added support for encoding and decoding :class:`ipaddress.IPv4Address`, :class:`ipaddress.IPv6Address`, :class:`ipaddress.IPv4Network`, and :class:`ipaddress.IPv6Network` (semantic tags 260 and 261) **4.2.0** (2020-01-10) - **BROKEN BUILD** Removed **4.1.2** (2018-12-10) - Fixed bigint encoding taking quadratic time - Fixed overflow errors when encoding floating point numbers in canonical mode - Improved decoder performance for dictionaries - Minor documentation tweaks **4.1.1** (2018-10-14) - Fixed encoding of negative :class:`decimal.Decimal` instances (PR by Sekenre) **4.1.0** (2018-05-27) - Added canonical encoding (via ``canonical=True``) (PR by Sekenre) - Added support for encoding/decoding sets (semantic tag 258) (PR by Sekenre) - Added support for encoding `FrozenDict` (hashable dict) as map keys or set elements (PR by Sekenre) **4.0.1** (2017-08-21) - Fixed silent truncation of decoded data if there are not enough bytes in the stream for an exact read (:exc:`CBORDecodeError` is now raised instead) **4.0.0** (2017-04-24) - **BACKWARD INCOMPATIBLE** Value sharing has been disabled by default, for better compatibility with other implementations and better performance (since it is rarely needed) - **BACKWARD INCOMPATIBLE** Replaced the ``semantic_decoders`` decoder option with the :attr:`CBORDecoder.tag_hook` option - **BACKWARD INCOMPATIBLE** Replaced the ``encoders`` encoder option with the :attr:`CBOREncoder.default` option - **BACKWARD INCOMPATIBLE** Factored out the file object argument (``fp``) from all callbacks - **BACKWARD INCOMPATIBLE** The encoder no longer supports every imaginable type implementing the ``Sequence`` or ``Map`` interface, as they turned out to be too broad - Added the :attr:`CBORDecoder.object_hook` option for decoding dicts into complex objects (intended for situations where JSON compatibility is required and semantic tags cannot be used) - Added encoding and decoding of simple values (:class:`CBORSimpleValue`) (contributed by Jerry Lundström) - Replaced the decoder for bignums with a simpler and faster version (contributed by orent) - Made all relevant classes and functions available directly in the :mod:`cbor2` namespace - Added proper documentation **3.0.4** (2016-09-24) - Fixed TypeError when trying to encode extension types (regression introduced in 3.0.3) **3.0.3** (2016-09-23) - No changes, just re-releasing due to git tagging screw-up **3.0.2** (2016-09-23) - Fixed decoding failure for datetimes with microseconds (tag 0) **3.0.1** (2016-08-08) - Fixed error in the cyclic structure detection code that could mistake one container for another, sometimes causing a bogus error about cyclic data structures where there was none **3.0.0** (2016-07-03) - **BACKWARD INCOMPATIBLE** Encoder callbacks now receive three arguments: the encoder instance, the value to encode and a file-like object. The callback must must now either write directly to the file-like object or call another encoder callback instead of returning an iterable. - **BACKWARD INCOMPATIBLE** Semantic decoder callbacks now receive four arguments: the decoder instance, the primitive value, a file-like object and the shareable index for the decoded value. Decoders that support value sharing must now set the raw value at the given index in ``decoder.shareables``. - **BACKWARD INCOMPATIBLE** Removed support for iterative encoding (:meth:`CBOREncoder.encode` is no longer a generator function and always returns :data:`None`) - Significantly improved performance (encoder ~30 % faster, decoder ~60 % faster) - Fixed serialization round-trip for :data:`undefined` (simple type 23) - Added proper support for value sharing in callbacks **2.0.0** (2016-06-11) - **BACKWARD INCOMPATIBLE** Deserialize unknown tags as :class:`CBORTag` objects so as not to lose information - Fixed error messages coming from nested structures **1.1.0** (2016-06-10) - Fixed deserialization of cyclic structures **1.0.0** (2016-06-08) - Initial release cbor2-5.4.2/pyproject.toml000066400000000000000000000002271413200260400154360ustar00rootroot00000000000000[build-system] requires = [ "setuptools >= 42", "wheel >= 0.29.0", "setuptools_scm[toml] >= 3.4" ] build-backend = "setuptools.build_meta" cbor2-5.4.2/scripts/000077500000000000000000000000001413200260400142105ustar00rootroot00000000000000cbor2-5.4.2/scripts/coverage.sh000077500000000000000000000015671413200260400163530ustar00rootroot00000000000000#!/bin/bash # This script generates coverage statistics for the C-based cbor2 # implementation. It assumes you have the "lcov" package installed, and that # you are in a Python virtual-env into which the cbor2 package can be installed # with --editable. To support the C implementation, the Python interpreter for # the virtual-env must be version 3.3 or later. # # NOTE: the script "touches" all *.c files to ensure all are rebuilt with the # -coverage option. This may mess with editors notion of whether those files # have changed if they are open at the time the script is run. touch source/*.c CFLAGS="-coverage" pip install -v -e .[test] find build/temp.*/source -name "*.gcda" -delete py.test -v mkdir -p coverage/ lcov --capture -d build/temp.*/source/ --output-file coverage/coverage.info genhtml coverage/coverage.info --out coverage/ python $(dirname $0)/coverage_server.py cbor2-5.4.2/scripts/coverage_server.py000066400000000000000000000012211413200260400177370ustar00rootroot00000000000000# A trivial script for serving lcov's HTML coverage output import os import sys import webbrowser from http.server import SimpleHTTPRequestHandler from socketserver import TCPServer from threading import Thread from signal import pause class Httpd(Thread): def __init__(self): super().__init__() self.server = TCPServer(('127.0.0.1', 8000), SimpleHTTPRequestHandler) self.start() def run(self): self.server.serve_forever() os.chdir(os.path.dirname(__file__) + '/../coverage/') httpd = Httpd() webbrowser.open_new_tab('http://localhost:8000/') try: pause() finally: httpd.server.shutdown() sys.exit(0) cbor2-5.4.2/scripts/half_float_tables.py000066400000000000000000000073701413200260400202220ustar00rootroot00000000000000#!/usr/bin/python3 """ A simple script to generate the tables used in halffloat.c. The algorithms in this script are based upon the paper `Fast Half Float Conversions`_, referenced by the Wikipedia article on `half-precision floating point`_. .. _half-precision floating point: https://en.wikipedia.org/wiki/Half-precision_floating-point_format .. _Fast Half Float Conversions: ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf """ from itertools import zip_longest def grouper(iterable, n, fillvalue=None): args = [iter(iterable)] * n return zip_longest(*args, fillvalue=fillvalue) def sigtable(): print("static const uint32_t sigtable[] = {") values = ( 0 if i == 0 else convertsig(i) if 1 <= i < 1024 else 0x38000000 + ((i - 1024) << 13) for i in range(2048) ) values = ('{:#010x}'.format(i) for i in values) for row in grouper(values, 8): print(' ' + (', '.join(row)) + ',') print("};") def exptable(): values = ( 0 if i == 0 else 0x47800000 if i == 31 else 0x80000000 if i == 32 else i << 23 if 1 <= i < 31 else 0x80000000 + ((i - 32) << 23) if 33 <= i < 63 else 0xC7800000 # i == 63 for i in range(64) ) print("static const uint32_t exptable[] = {") values = ('{:#010x}'.format(i) for i in values) for row in grouper(values, 8): print(' ' + (', '.join(row)) + ',') print("};") def offsettable(): values = ( 0 if i in (0, 32) else 1024 for i in range(64) ) print("static const uint16_t offsettable[] = {") values = ('{:#06x}'.format(i) for i in values) for row in grouper(values, 8): print(' ' + (', '.join(row)) + ',') print("};") def convertsig(i): if not i: return 0 m = i << 13 e = 0 while not m & 0x00800000: e -= 0x00800000 m <<= 1 m &= ~0x00800000 e += 0x38800000 return m | e def basetable(): values = [0] * 512 for i in range(256): e = i - 127 if e < -24: # underflow to 0 values[i | 0x000] = 0 values[i | 0x100] = 0x8000 elif e < -14: # smalls to denorms values[i | 0x000] = (0x400 >> (-e - 14)) values[i | 0x100] = (0x400 >> (-e - 14)) | 0x8000 elif e < 15: # normal case values[i | 0x000] = ((e + 15) << 10) values[i | 0x100] = ((e + 15) << 10) | 0x8000 elif e < 128: # overflow to inf values[i | 0x000] = 0x7c00 values[i | 0x100] = 0xfc00 else: # inf and nan values[i | 0x000] = 0x7c00 values[i | 0x100] = 0xfc00 print("static const uint16_t basetable[] = {") values = ('{:#06x}'.format(i) for i in values) for row in grouper(values, 8): print(' ' + (', '.join(row)) + ',') print("};") def shifttable(): values = [0] * 512 for i in range(256): e = i - 127 if e < -24: # underflow to 0 values[i | 0x000] = 24 values[i | 0x100] = 24 elif e < -14: # smalls to denorms values[i | 0x000] = -e - 1 values[i | 0x100] = -e - 1 elif e < 15: # normal case values[i | 0x000] = 13 values[i | 0x100] = 13 elif e < 128: # overflow to inf values[i | 0x000] = 24 values[i | 0x100] = 24 else: # inf and nan values[i | 0x000] = 13 values[i | 0x100] = 13 print("static const uint16_t shifttable[] = {") values = ('{:#06x}'.format(i) for i in values) for row in grouper(values, 8): print(' ' + (', '.join(row)) + ',') print("};") sigtable() print() exptable() print() offsettable() print() basetable() print() shifttable() cbor2-5.4.2/scripts/ref_leak_test.py000077500000000000000000000167111413200260400174020ustar00rootroot00000000000000#!/usr/bin/env python """ This is a crude script for detecting reference leaks in the C-based cbor2 implementation. It is by no means fool-proof and won't pick up all possible ref leaks, but it is a reasonable "confidence test" that things aren't horribly wrong. The script assumes you're in an environment with objgraph and cbor2 installed. The script outputs a nicely formatted table of the tests run, and the number of "extra" objects that existed after the tests (indicating a ref-leak), or "-" if no extra objects existed. The ideal output is obviously "-" in all rows. """ import sys import objgraph import tracemalloc from datetime import datetime, timezone, timedelta from fractions import Fraction from decimal import Decimal from collections import namedtuple, OrderedDict def import_cbor2(): # Similar hack to that used in tests/conftest to get separate C and Python # implementations import cbor2 import cbor2.types import cbor2.encoder import cbor2.decoder class Module(object): # Mock module class pass py_cbor2 = Module() for source in (cbor2.types, cbor2.encoder, cbor2.decoder): for name in dir(source): setattr(py_cbor2, name, getattr(source, name)) return cbor2, py_cbor2 c_cbor2, py_cbor2 = import_cbor2() UTC = timezone.utc TEST_VALUES = [ # label, kwargs, value ('None', {}, None), ('10e0', {}, 1), ('10e12', {}, 1000000000000), ('10e29', {}, 100000000000000000000000000000), ('-10e0', {}, -1), ('-10e12', {}, -1000000000000), ('-10e29', {}, -100000000000000000000000000000), ('float1', {}, 1.0), ('float2', {}, 3.8), ('str', {}, 'foo'), ('bigstr', {}, 'foobarbaz ' * 1000), ('bytes', {}, b'foo'), ('bigbytes', {}, b'foobarbaz\x00' * 1000), ('datetime', {'timezone': UTC}, datetime(2019, 5, 9, 22, 4, 5, 123456)), ('decimal', {}, Decimal('1.1')), ('fraction', {}, Fraction(1, 5)), ('intlist', {}, [1, 2, 3]), ('bigintlist', {}, [1, 2, 3] * 1000), ('strlist', {}, ['foo', 'bar', 'baz']), ('bigstrlist', {}, ['foo', 'bar', 'baz'] * 1000), ('dict', {}, {'a': 1, 'b': 2, 'c': 3}), ('bigdict', {}, {'a' * i: i for i in range(1000)}), ('set', {}, {1, 2, 3}), ('bigset', {}, set(range(1000))), ('bigdictlist', {}, [{'a' * i: i for i in range(100)}] * 100), ('objectdict', {'timezone': UTC}, {'name': 'Foo', 'species': 'cat', 'dob': datetime(2013, 5, 20), 'weight': 4.1}), ('objectdictlist', {'timezone': UTC}, [{'name': 'Foo', 'species': 'cat', 'dob': datetime(2013, 5, 20), 'weight': 4.1}] * 100), ] Leaks = namedtuple('Leaks', ('count', 'comparison')) Tests = namedtuple('Test', ('objgraph', 'malloc')) Result = namedtuple('Result', ('encoding', 'decoding', 'roundtrip')) peak = {} def growth(): return objgraph.growth(limit=None, peak_stats=peak) def test_malloc(op): count = 0 start = datetime.now() # NOTE: Filter pointing to the op() line in the loop below, because we're # only interested in memory allocated by that line. Naturally, if this file # is edited, the lineno parameter below must be adjusted! only_op = tracemalloc.Filter(True, __file__, lineno=102, all_frames=True) tracemalloc.start(10) try: # Perform a pre-run of op so that any one-time memory allocation # (module imports, etc.) don't affect the later diffs op() before = tracemalloc.take_snapshot().filter_traces([only_op]) while True: count += 1 op() if datetime.now() - start > timedelta(seconds=0.2): break after = tracemalloc.take_snapshot().filter_traces([only_op]) diff = after.compare_to(before, 'traceback') diff = [entry for entry in diff if entry.size_diff > 0] return count, diff finally: tracemalloc.stop() def test_objgraph(op): count = 0 start = datetime.now() # See notes above op() growth() while True: count += 1 op() if datetime.now() - start > timedelta(seconds=0.2): break return count, growth() def test(op): return Tests(Leaks(*test_objgraph(op)), Leaks(*test_malloc(op))) def format_leaks(result): if result.objgraph.comparison: return '%d objs (/%d)' % ( sum(leak[-1] for leak in result.objgraph.comparison), result.objgraph.count) elif result.malloc.comparison and ( result.malloc.count < result.malloc.comparison[0].size_diff): # Running the loop always results in *some* memory allocation, but as # long as the bytes allocated are less than the number of loops it's # unlikely to be an actual leak return '%d bytes (/%d)' % ( result.malloc.comparison[0].size_diff, result.malloc.count) else: return '-' def output_table(results): # Build table content head = ('Test', 'Encoding', 'Decoding', 'Round-trip') rows = [head] + [ ( label, format_leaks(result.encoding), format_leaks(result.decoding), format_leaks(result.roundtrip), ) for label, result in results.items() ] # Format table output cols = zip(*rows) col_widths = [max(len(row) for row in col) for col in cols] sep = ''.join(( '+-', '-+-'.join('-' * width for width in col_widths), '-+', )) print(sep) print(''.join(( '| ', ' | '.join( '{value:<{width}}'.format(value=value, width=width) for value, width in zip(head, col_widths) ), ' |', ))) print(sep) for row in rows[1:]: print(''.join(( '| ', ' | '.join( '{value:<{width}}'.format(value=value, width=width) for value, width in zip(row, col_widths) ), ' |', ))) print(sep) print() print("""\ There *will* be false positives in the table above. Ignore leaks involving a tiny number of objects (e.g. 1) or a small number of bytes (e.g. < 8Kb) as such allocations are quite normal. In the case of a ref-leak of an object that can reference others (lists, sets, dicts, or anything with a __dict__), expect to see 100s or 1000s of "objs" leaked. In the case of a ref-leak of a simple object (int, str, bytes, etc.), expect to see a few hundred Kb allocated. If leaks occur across the board, it's likely to be in something universal like dump/load. If it's restricted to a type, check the encoding and decoding methods for that type. """) def main(): results = OrderedDict() sys.stderr.write("Testing") sys.stderr.flush() for name, kwargs, value in TEST_VALUES: encoded = py_cbor2.dumps(value, **kwargs) results[name] = Result( encoding=test(lambda: c_cbor2.dumps(value, **kwargs)), decoding=test(lambda: c_cbor2.loads(encoded)), roundtrip=test(lambda: c_cbor2.loads(c_cbor2.dumps(value, **kwargs))), ) sys.stderr.write(".") sys.stderr.flush() sys.stderr.write("\n") sys.stderr.write("\n") output_table(results) sys.stderr.write("\n") if __name__ == '__main__': main() cbor2-5.4.2/scripts/speed_test.py000077500000000000000000000200651413200260400167270ustar00rootroot00000000000000#!/usr/bin/env python """ A simple script for testing the two cbor2 implementations speed against each other (as well as against the C-based cbor implementation). This script assumes you're in an environment with cbor and cbor2 installed. By default the script will output a nicely formatted table comparing the speeds of the three implementations (cbor, c-cbor2, py-cbor2). Entries in the c-cbor2 columns will be color coded, with green indicating the test was at least 20% faster than the py-cbor2 implementation, red indicating the test was at least 5% slower (5% is a reasonable margin of error as timing measurements are rarely precise in a non-RTOS), and white indicating a speed between these two boundaries. If the "--csv" argument is given, the script will output the results in CSV format to stdout (for piping to whatever you want to use them in). """ import io import re import sys import csv import cbor import timeit from math import log2, ceil from datetime import datetime, timezone from fractions import Fraction from decimal import Decimal from collections import namedtuple, OrderedDict def import_cbor2(): # Similar hack to that used in tests/conftest to get separate C and Python # implementations import cbor2 import cbor2.types import cbor2.encoder import cbor2.decoder class Module(object): # Mock module class pass py_cbor2 = Module() for source in (cbor2.types, cbor2.encoder, cbor2.decoder): for name in dir(source): setattr(py_cbor2, name, getattr(source, name)) return cbor2, py_cbor2 c_cbor2, py_cbor2 = import_cbor2() UTC = timezone.utc TEST_VALUES = [ # label, kwargs, value ('None', {}, None), ('10e0', {}, 1), ('10e12', {}, 1000000000000), ('10e29', {}, 100000000000000000000000000000), ('-10e0', {}, -1), ('-10e12', {}, -1000000000000), ('-10e29', {}, -100000000000000000000000000000), ('float1', {}, 1.0), ('float2', {}, 3.8), ('str', {}, 'foo'), ('bigstr', {}, 'foobarbaz ' * 1000), ('bytes', {}, b'foo'), ('bigbytes', {}, b'foobarbaz\x00' * 1000), ('datetime', {'timezone': UTC}, datetime(2019, 5, 9, 22, 4, 5, 123456)), ('decimal', {}, Decimal('1.1')), ('fraction', {}, Fraction(1, 5)), ('intlist', {}, [1, 2, 3]), ('bigintlist', {}, [1, 2, 3] * 1000), ('strlist', {}, ['foo', 'bar', 'baz']), ('bigstrlist', {}, ['foo', 'bar', 'baz'] * 1000), ('dict', {}, {'a': 1, 'b': 2, 'c': 3}), ('bigdict', {}, {'a' * i: i for i in range(1000)}), ('set', {}, {1, 2, 3}), ('bigset', {}, set(range(1000))), ('bigdictlist', {}, [{'a' * i: i for i in range(100)}] * 100), ('objectdict', {'timezone': UTC}, {'name': 'Foo', 'species': 'cat', 'dob': datetime(2013, 5, 20), 'weight': 4.1}), ('objectdictlist', {'timezone': UTC}, [{'name': 'Foo', 'species': 'cat', 'dob': datetime(2013, 5, 20), 'weight': 4.1}] * 100), ] Codec = namedtuple('Codec', ('cbor', 'c_cbor2', 'py_cbor2')) Result = namedtuple('Result', ('encoding', 'decoding')) Timing = namedtuple('Timing', ('time', 'repeat', 'count')) def autorange(op, limit=0.2): # Adapted from the Python 3.7 version of timeit t = timeit.Timer(op) i = 1 while True: for j in 1, 2, 5: number = i * j time_taken = t.timeit(number) if time_taken >= limit: return number i *= 10 def time(op, repeat=3): try: number = autorange(op, limit=0.02) except Exception as e: return e t = timeit.Timer(op) return Timing(min(t.repeat(repeat, number)) / number, repeat, number) def format_time(t, suffixes=('s', 'ms', 'µs', 'ns'), zero='0s', template='{time:.1f}{suffix}'): if isinstance(t, Exception): return '-' else: try: index = min(len(suffixes) - 1, ceil(log2(1/t.time) / 10)) except ValueError: return zero else: return template.format(time=t.time * 2 ** (index * 10), suffix=suffixes[index]) def print_len(s): return len(re.sub(r'\x1b\[.*?m', '', s)) RED = '\x1b[1;31m' GREEN = '\x1b[1;32m' RESET = '\x1b[0m' def color_time(t, lim): time_str = format_time(t) if isinstance(t, Exception): return RED + time_str + RESET elif t.time <= lim.time * 0.8: return GREEN + time_str + RESET elif t.time > lim.time * 1.05: return RED + time_str + RESET else: return time_str def output_table(results): # Build table content head = ('Test',) + ('cbor', 'c-cbor2', 'py-cbor2') * 2 rows = [head] + [ ( value, format_time(result.cbor.encoding), color_time(result.c_cbor2.encoding, result.py_cbor2.encoding), format_time(result.py_cbor2.encoding), format_time(result.cbor.decoding), color_time(result.c_cbor2.decoding, result.py_cbor2.decoding), format_time(result.py_cbor2.decoding), ) for value, result in results.items() ] # Format table output cols = zip(*rows) col_widths = [max(print_len(row) for row in col) for col in cols] sep = ''.join(( '+-', '-+-'.join('-' * width for width in col_widths), '-+', )) print(''.join(( ' ', ' ' * col_widths[0], ' +-', '-' * (sum(col_widths[1:4]) + 6), '-+-', '-' * (sum(col_widths[4:7]) + 6), '-+', ))) print(''.join(( ' ', ' ' * col_widths[0], ' | ', '{value:^{width}}'.format(value='Encoding', width=sum(col_widths[1:4]) + 6), ' | ', '{value:^{width}}'.format(value='Decoding', width=sum(col_widths[4:7]) + 6), ' |', ))) print(sep) print(''.join(( '| ', ' | '.join( '{value:<{width}}'.format(value=value, width=width) for value, width in zip(head, col_widths) ), ' |', ))) print(sep) for row in rows[1:]: print(''.join(( '| ', ' | '.join( '{value:<{width}}'.format( value=value, width=width + len(value) - print_len(value)) for value, width in zip(row, col_widths) ), ' |', ))) print(sep) def output_csv(results): writer = csv.writer(sys.stdout) writer.writerow(( 'Title', 'cbor-encode', 'c-cbor2-encode', 'py-cbor2-encode', 'cbor-decode', 'c-cbor2-decode', 'py-cbor2-decode' )) for title, result in results.items(): writer.writerow(( title, result.cbor.encoding.time if isinstance(result.cbor.encoding, Timing) else None, result.c_cbor2.encoding.time, result.py_cbor2.encoding.time, result.cbor.decoding.time if isinstance(result.cbor.encoding, Timing) else None, result.c_cbor2.decoding.time, result.py_cbor2.decoding.time, )) def main(): results = OrderedDict() sys.stderr.write("Testing") sys.stderr.flush() for name, kwargs, value in TEST_VALUES: encoded = py_cbor2.dumps(value, **kwargs) results[name] = Codec(**{ mod_name: Result( encoding=time(lambda: mod.dumps(value, **kwargs)), decoding=time(lambda: mod.loads(encoded)) ) for mod_name, mod in { 'cbor': cbor, 'c_cbor2': c_cbor2, 'py_cbor2': py_cbor2, }.items() }) sys.stderr.write(".") sys.stderr.flush() sys.stderr.write("\n") sys.stderr.write("\n") if len(sys.argv) > 1 and sys.argv[1] == '--csv': output_csv(results) else: output_table(results) if __name__ == '__main__': main() cbor2-5.4.2/setup.cfg000066400000000000000000000024061413200260400143440ustar00rootroot00000000000000[metadata] name = cbor2 description = Pure Python CBOR (de)serializer with extensive tag support long_description = file: README.rst author = Alex Grönholm author_email = alex.gronholm@nextday.fi maintainer = Kio Smallwood (Sekenre) maintainer_email = kio@mothers-arms.co.uk project_urls = Documentation = https://cbor2.readthedocs.org/en/latest/ Source Code = https://github.com/agronholm/cbor2 Issue Tracker = https://github.com/agronholm/cbor2/issues license = MIT keywords = serialization cbor classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers License :: OSI Approved :: MIT License Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 [options] packages = find: python_requires = >= 3.6 [options.extras_require] test = pytest pytest-cov doc = sphinx_rtd_theme sphinx-autodoc-typehints >= 1.2.0 [tool:pytest] addopts = -rsx --cov --tb=short testpaths = tests [coverage:run] source = cbor2 relative_files = true [coverage:report] show_missing = true [flake8] max-line-length = 99 exclude = .tox,build,docs cbor2-5.4.2/setup.py000066400000000000000000000037501413200260400142400ustar00rootroot00000000000000import sys import os import platform from pkg_resources import parse_version from setuptools import setup, Extension min_glibc = parse_version('2.9') def check_libc(): """Return False if we have glibc < 2.9 and should not build the C extension.""" # Borrowed from pip internals # https://github.com/pypa/pip/blob/20.1.1/src/pip/_internal/utils/glibc.py#L21-L36 try: # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": libc, version = os.confstr("CS_GNU_LIBC_VERSION").split() except (AttributeError, OSError, ValueError): # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... return True if libc != 'glibc': # Attempt to build with musl or other libc return True return parse_version(version) >= min_glibc cpython = platform.python_implementation() == 'CPython' windows = sys.platform.startswith('win') use_c_ext = os.environ.get("CBOR2_BUILD_C_EXTENSION", None) if use_c_ext == "1": build_c_ext = True elif use_c_ext == "0": build_c_ext = False else: build_c_ext = cpython and (windows or check_libc()) # Enable GNU features for libc's like musl, should have no effect # on Apple/BSDs if build_c_ext and not windows: gnu_flag = ['-D_GNU_SOURCE'] else: gnu_flag = [] if build_c_ext: _cbor2 = Extension( '_cbor2', # math.h routines are built-in to MSVCRT libraries=['m'] if not windows else [], extra_compile_args=['-std=c99'] + gnu_flag, sources=[ 'source/module.c', 'source/encoder.c', 'source/decoder.c', 'source/tags.c', 'source/halffloat.c', ], optional=True ) kwargs = {'ext_modules': [_cbor2]} else: kwargs = {} setup( use_scm_version={ 'version_scheme': 'post-release', 'local_scheme': 'dirty-tag' }, setup_requires=[ 'setuptools >= 40.7.0', 'setuptools_scm >= 1.7.0' ], **kwargs ) cbor2-5.4.2/source/000077500000000000000000000000001413200260400140215ustar00rootroot00000000000000cbor2-5.4.2/source/decoder.c000066400000000000000000001703721413200260400156040ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include #include #if __FreeBSD__ #include #elif __APPLE__ #include #elif ! _WIN32 #include #endif #include #include #include #include #include "module.h" #include "halffloat.h" #include "tags.h" #include "decoder.h" #if __APPLE__ #define be16toh(x) OSSwapBigToHostInt16(x) #define be32toh(x) OSSwapBigToHostInt32(x) #define be64toh(x) OSSwapBigToHostInt64(x) #elif _WIN32 // All windows platforms are (currently) little-endian so byteswap is required #define be16toh(x) _byteswap_ushort(x) #define be32toh(x) _byteswap_ulong(x) #define be64toh(x) _byteswap_uint64(x) #endif // copied from cpython/Objects/bytesobject.c for bounds checks #define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) enum DecodeOption { DECODE_NORMAL = 0, DECODE_IMMUTABLE = 1, DECODE_UNSHARED = 2 }; typedef uint8_t DecodeOptions; static int _CBORDecoder_set_fp(CBORDecoderObject *, PyObject *, void *); static int _CBORDecoder_set_tag_hook(CBORDecoderObject *, PyObject *, void *); static int _CBORDecoder_set_object_hook(CBORDecoderObject *, PyObject *, void *); static int _CBORDecoder_set_str_errors(CBORDecoderObject *, PyObject *, void *); static PyObject * decode(CBORDecoderObject *, DecodeOptions); static PyObject * decode_bytestring(CBORDecoderObject *, uint8_t); static PyObject * decode_string(CBORDecoderObject *, uint8_t); static PyObject * CBORDecoder_decode_datetime_string(CBORDecoderObject *); static PyObject * CBORDecoder_decode_epoch_datetime(CBORDecoderObject *); static PyObject * CBORDecoder_decode_fraction(CBORDecoderObject *); static PyObject * CBORDecoder_decode_bigfloat(CBORDecoderObject *); static PyObject * CBORDecoder_decode_rational(CBORDecoderObject *); static PyObject * CBORDecoder_decode_regexp(CBORDecoderObject *); static PyObject * CBORDecoder_decode_uuid(CBORDecoderObject *); static PyObject * CBORDecoder_decode_mime(CBORDecoderObject *); static PyObject * CBORDecoder_decode_positive_bignum(CBORDecoderObject *); static PyObject * CBORDecoder_decode_negative_bignum(CBORDecoderObject *); static PyObject * CBORDecoder_decode_simple_value(CBORDecoderObject *); static PyObject * CBORDecoder_decode_float16(CBORDecoderObject *); static PyObject * CBORDecoder_decode_float32(CBORDecoderObject *); static PyObject * CBORDecoder_decode_float64(CBORDecoderObject *); static PyObject * CBORDecoder_decode_ipaddress(CBORDecoderObject *); static PyObject * CBORDecoder_decode_ipnetwork(CBORDecoderObject *); static PyObject * CBORDecoder_decode_self_describe_cbor(CBORDecoderObject *); static PyObject * CBORDecoder_decode_shareable(CBORDecoderObject *); static PyObject * CBORDecoder_decode_sharedref(CBORDecoderObject *); static PyObject * CBORDecoder_decode_set(CBORDecoderObject *); static PyObject * CBORDecoder_decode_stringref(CBORDecoderObject *); static PyObject * CBORDecoder_decode_stringref_ns(CBORDecoderObject *); // Constructors and destructors ////////////////////////////////////////////// static int CBORDecoder_traverse(CBORDecoderObject *self, visitproc visit, void *arg) { Py_VISIT(self->read); Py_VISIT(self->tag_hook); Py_VISIT(self->object_hook); Py_VISIT(self->shareables); Py_VISIT(self->stringref_namespace); // No need to visit str_errors; it's only a string and can't reference us // or other objects return 0; } static int CBORDecoder_clear(CBORDecoderObject *self) { Py_CLEAR(self->read); Py_CLEAR(self->tag_hook); Py_CLEAR(self->object_hook); Py_CLEAR(self->shareables); Py_CLEAR(self->stringref_namespace); Py_CLEAR(self->str_errors); return 0; } // CBORDecoder.__del__(self) static void CBORDecoder_dealloc(CBORDecoderObject *self) { PyObject_GC_UnTrack(self); CBORDecoder_clear(self); Py_TYPE(self)->tp_free((PyObject *) self); } // CBORDecoder.__new__(cls, *args, **kwargs) PyObject * CBORDecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { CBORDecoderObject *self; PyDateTime_IMPORT; if (!PyDateTimeAPI) return NULL; self = (CBORDecoderObject *) type->tp_alloc(type, 0); if (self) { // self.shareables = [] self->shareables = PyList_New(0); if (!self->shareables) goto error; Py_INCREF(Py_None); self->stringref_namespace = Py_None; Py_INCREF(Py_None); self->read = Py_None; Py_INCREF(Py_None); self->tag_hook = Py_None; Py_INCREF(Py_None); self->object_hook = Py_None; self->str_errors = PyBytes_FromString("strict"); self->immutable = false; self->shared_index = -1; } return (PyObject *) self; error: Py_DECREF(self); return NULL; } // CBORDecoder.__init__(self, fp=None, tag_hook=None, object_hook=None, // str_errors='strict') int CBORDecoder_init(CBORDecoderObject *self, PyObject *args, PyObject *kwargs) { static char *keywords[] = { "fp", "tag_hook", "object_hook", "str_errors", NULL }; PyObject *fp = NULL, *tag_hook = NULL, *object_hook = NULL, *str_errors = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOO", keywords, &fp, &tag_hook, &object_hook, &str_errors)) return -1; if (_CBORDecoder_set_fp(self, fp, NULL) == -1) return -1; if (tag_hook && _CBORDecoder_set_tag_hook(self, tag_hook, NULL) == -1) return -1; if (object_hook && _CBORDecoder_set_object_hook(self, object_hook, NULL) == -1) return -1; if (str_errors && _CBORDecoder_set_str_errors(self, str_errors, NULL) == -1) return -1; if (!_CBOR2_FrozenDict && _CBOR2_init_FrozenDict() == -1) return -1; return 0; } // Property accessors //////////////////////////////////////////////////////// // CBORDecoder._get_fp(self) static PyObject * _CBORDecoder_get_fp(CBORDecoderObject *self, void *closure) { PyObject *ret = PyMethod_GET_SELF(self->read); Py_INCREF(ret); return ret; } // CBORDecoder._set_fp(self, value) static int _CBORDecoder_set_fp(CBORDecoderObject *self, PyObject *value, void *closure) { PyObject *tmp, *read; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete fp attribute"); return -1; } read = PyObject_GetAttr(value, _CBOR2_str_read); if (!(read && PyCallable_Check(read))) { PyErr_SetString(PyExc_ValueError, "fp object must have a callable read method"); return -1; } // See notes in encoder.c / _CBOREncoder_set_fp tmp = self->read; self->read = read; Py_DECREF(tmp); return 0; } // CBORDecoder._get_tag_hook(self) static PyObject * _CBORDecoder_get_tag_hook(CBORDecoderObject *self, void *closure) { Py_INCREF(self->tag_hook); return self->tag_hook; } // CBORDecoder._set_tag_hook(self, value) static int _CBORDecoder_set_tag_hook(CBORDecoderObject *self, PyObject *value, void *closure) { PyObject *tmp; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete tag_hook attribute"); return -1; } if (value != Py_None && !PyCallable_Check(value)) { PyErr_Format(PyExc_ValueError, "invalid tag_hook value %R (must be callable or " "None", value); return -1; } tmp = self->tag_hook; Py_INCREF(value); self->tag_hook = value; Py_DECREF(tmp); return 0; } // CBORDecoder._get_object_hook(self) static PyObject * _CBORDecoder_get_object_hook(CBORDecoderObject *self, void *closure) { Py_INCREF(self->object_hook); return self->object_hook; } // CBORDecoder._set_object_hook(self, value) static int _CBORDecoder_set_object_hook(CBORDecoderObject *self, PyObject *value, void *closure) { PyObject *tmp; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete object_hook attribute"); return -1; } if (value != Py_None && !PyCallable_Check(value)) { PyErr_Format(PyExc_ValueError, "invalid object_hook value %R (must be callable or " "None)", value); return -1; } tmp = self->object_hook; Py_INCREF(value); self->object_hook = value; Py_DECREF(tmp); return 0; } // CBORDecoder._get_str_errors(self) static PyObject * _CBORDecoder_get_str_errors(CBORDecoderObject *self, void *closure) { return PyUnicode_DecodeASCII( PyBytes_AS_STRING(self->str_errors), PyBytes_GET_SIZE(self->str_errors), "strict"); } // CBORDecoder._set_str_errors(self, value) static int _CBORDecoder_set_str_errors(CBORDecoderObject *self, PyObject *value, void *closure) { PyObject *tmp, *bytes; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete str_errors attribute"); return -1; } if (PyUnicode_Check(value)) { bytes = PyUnicode_AsASCIIString(value); if (bytes) { if (!strcmp(PyBytes_AS_STRING(bytes), "strict") || !strcmp(PyBytes_AS_STRING(bytes), "error") || !strcmp(PyBytes_AS_STRING(bytes), "replace")) { tmp = self->str_errors; self->str_errors = bytes; Py_DECREF(tmp); return 0; } Py_DECREF(bytes); } } PyErr_Format(PyExc_ValueError, "invalid str_errors value %R (must be one of 'strict', " "'error', or 'replace')", value); return -1; } // CBORDecoder._get_immutable(self, value) static PyObject * _CBORDecoder_get_immutable(CBORDecoderObject *self, void *closure) { if (self->immutable) Py_RETURN_TRUE; else Py_RETURN_FALSE; } // Utility functions ///////////////////////////////////////////////////////// static int fp_read(CBORDecoderObject *self, char *buf, const Py_ssize_t size) { PyObject *obj, *size_obj; char *data; int ret = -1; size_obj = PyLong_FromSsize_t(size); if (size_obj) { obj = PyObject_CallFunctionObjArgs(self->read, size_obj, NULL); if (obj) { assert(PyBytes_CheckExact(obj)); if (PyBytes_GET_SIZE(obj) == (Py_ssize_t) size) { data = PyBytes_AS_STRING(obj); memcpy(buf, data, size); ret = 0; } else { PyErr_Format( _CBOR2_CBORDecodeEOF, "premature end of stream (expected to read %zd bytes, " "got %zd instead)", size, PyBytes_GET_SIZE(obj)); } Py_DECREF(obj); } Py_DECREF(size_obj); } return ret; } // CBORDecoder.read(self, length) -> bytes static PyObject * CBORDecoder_read(CBORDecoderObject *self, PyObject *length) { PyObject *ret = NULL; Py_ssize_t len; len = PyLong_AsSsize_t(length); if (PyErr_Occurred()) return NULL; ret = PyBytes_FromStringAndSize(NULL, len); if (ret) { if (fp_read(self, PyBytes_AS_STRING(ret), len) == -1) { Py_DECREF(ret); ret = NULL; } } return ret; } static inline void set_shareable(CBORDecoderObject *self, PyObject *value) { if (value && self->shared_index != -1) { Py_INCREF(value); // PyList_SetItem "steals" reference // TODO use weakrefs? or explicitly empty list? #ifndef NDEBUG int ret = #endif PyList_SetItem(self->shareables, self->shared_index, value); assert(!ret); } } // CBORDecoder.set_shareable(self, value) static PyObject * CBORDecoder_set_shareable(CBORDecoderObject *self, PyObject *value) { set_shareable(self, value); Py_RETURN_NONE; } static int decode_length(CBORDecoderObject *self, uint8_t subtype, uint64_t *length, bool *indefinite) { union { union { uint64_t value; char buf[sizeof(uint64_t)]; } u64; union { uint32_t value; char buf[sizeof(uint32_t)]; } u32; union { uint16_t value; char buf[sizeof(uint16_t)]; } u16; union { uint8_t value; char buf[sizeof(uint8_t)]; } u8; } value; if (subtype < 28) { if (subtype < 24) { *length = subtype; } else if (subtype == 24) { if (fp_read(self, value.u8.buf, sizeof(uint8_t)) == -1) return -1; *length = value.u8.value; } else if (subtype == 25) { if (fp_read(self, value.u16.buf, sizeof(uint16_t)) == -1) return -1; *length = be16toh(value.u16.value); } else if (subtype == 26) { if (fp_read(self, value.u32.buf, sizeof(uint32_t)) == -1) return -1; *length = be32toh(value.u32.value); } else { if (fp_read(self, value.u64.buf, sizeof(uint64_t)) == -1) return -1; *length = be64toh(value.u64.value); } if (indefinite) *indefinite = false; return 0; } else if (subtype == 31 && indefinite && *indefinite) { // well, indefinite is already true so nothing to see here... return 0; } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "unknown unsigned integer subtype 0x%x", subtype); return -1; } } static int string_namespace_add(CBORDecoderObject *self, PyObject *string, uint64_t length) { if (self->stringref_namespace != Py_None) { uint64_t next_index = PyList_GET_SIZE(self->stringref_namespace); bool is_referenced = true; if (next_index < 24) { is_referenced = length >= 3; } else if (next_index < 256) { is_referenced = length >= 4; } else if (next_index < 65536) { is_referenced = length >= 5; } else if (next_index < 4294967296ull) { is_referenced = length >= 7; } else { is_referenced = length >= 11; } if (is_referenced) { return PyList_Append(self->stringref_namespace, string); } } return 0; } // Major decoders //////////////////////////////////////////////////////////// static PyObject * decode_uint(CBORDecoderObject *self, uint8_t subtype) { // major type 0 uint64_t length; PyObject *ret; if (decode_length(self, subtype, &length, NULL) == -1) return NULL; ret = PyLong_FromUnsignedLongLong(length); set_shareable(self, ret); return ret; } static PyObject * decode_negint(CBORDecoderObject *self, uint8_t subtype) { // major type 1 PyObject *value, *one, *ret = NULL; value = decode_uint(self, subtype); if (value) { one = PyLong_FromLong(1); if (one) { ret = PyNumber_Negative(value); if (ret) { Py_DECREF(value); value = ret; ret = PyNumber_Subtract(value, one); set_shareable(self, ret); } Py_DECREF(one); } Py_DECREF(value); } return ret; } static PyObject * decode_definite_bytestring(CBORDecoderObject *self, Py_ssize_t length) { PyObject *ret = NULL; ret = PyBytes_FromStringAndSize(NULL, length); if (!ret) return NULL; if (fp_read(self, PyBytes_AS_STRING(ret), length) == -1) { Py_DECREF(ret); return NULL; } if (string_namespace_add(self, ret, length) == -1) { Py_DECREF(ret); return NULL; } return ret; } static PyObject * decode_indefinite_bytestrings(CBORDecoderObject *self) { PyObject *list, *ret = NULL; LeadByte lead; list = PyList_New(0); if (list) { while (1) { if (fp_read(self, &lead.byte, 1) == -1) break; if (lead.major == 2 && lead.subtype != 31) { ret = decode_bytestring(self, lead.subtype); if (ret) { PyList_Append(list, ret); Py_DECREF(ret); ret = NULL; } else { break; } } else if (lead.major == 7 && lead.subtype == 31) { // break-code ret = PyObject_CallMethodObjArgs( _CBOR2_empty_bytes, _CBOR2_str_join, list, NULL); break; } else { PyErr_SetString( _CBOR2_CBORDecodeValueError, "non-bytestring found in indefinite length bytestring"); break; } } Py_DECREF(list); } return ret; } static PyObject * decode_bytestring(CBORDecoderObject *self, uint8_t subtype) { // major type 2 uint64_t length = 0; bool indefinite = true; PyObject *ret; char length_hex[17]; if (decode_length(self, subtype, &length, &indefinite) == -1) return NULL; if (length > (uint64_t)PY_SSIZE_T_MAX - (uint64_t)PyBytesObject_SIZE) { sprintf(length_hex, "%llX", length); PyErr_Format( _CBOR2_CBORDecodeValueError, "excessive bytestring size 0x%s", length_hex); return NULL; } if (indefinite) ret = decode_indefinite_bytestrings(self); else ret = decode_definite_bytestring(self, (Py_ssize_t)length); set_shareable(self, ret); return ret; } // NOTE: It may seem redundant to repeat the definite and indefinite routines // to handle UTF-8 strings but there is a reason to do this separately. // Specifically, the CBOR spec states (in sec. 2.2): // // Text strings with indefinite lengths act the same as byte strings with // indefinite lengths, except that all their chunks MUST be definite-length // text strings. Note that this implies that the bytes of a single UTF-8 // character cannot be spread between chunks: a new chunk can only be // started at a character boundary. // // This precludes using the indefinite bytestring decoder above as that would // happily ignore UTF-8 characters split across chunks. static PyObject * decode_definite_string(CBORDecoderObject *self, Py_ssize_t length) { PyObject *ret = NULL; char *buf; buf = PyMem_Malloc(length); if (!buf) return PyErr_NoMemory(); if (fp_read(self, buf, length) == 0) ret = PyUnicode_DecodeUTF8( buf, length, PyBytes_AS_STRING(self->str_errors)); PyMem_Free(buf); if (string_namespace_add(self, ret, length) == -1) { Py_DECREF(ret); return NULL; } return ret; } static PyObject * decode_indefinite_strings(CBORDecoderObject *self) { PyObject *list, *ret = NULL; LeadByte lead; list = PyList_New(0); if (list) { while (1) { if (fp_read(self, &lead.byte, 1) == -1) break; if (lead.major == 3 && lead.subtype != 31) { ret = decode_string(self, lead.subtype); if (ret) { PyList_Append(list, ret); Py_DECREF(ret); ret = NULL; } else { break; } } else if (lead.major == 7 && lead.subtype == 31) { // break-code ret = PyObject_CallMethodObjArgs( _CBOR2_empty_str, _CBOR2_str_join, list, NULL); break; } else { PyErr_SetString( _CBOR2_CBORDecodeValueError, "non-string found in indefinite length string"); break; } } Py_DECREF(list); } return ret; } static PyObject * decode_string(CBORDecoderObject *self, uint8_t subtype) { // major type 3 uint64_t length = 0; bool indefinite = true; PyObject *ret; char length_hex[17]; if (decode_length(self, subtype, &length, &indefinite) == -1) return NULL; if (length > (uint64_t)PY_SSIZE_T_MAX - (uint64_t)PyBytesObject_SIZE) { sprintf(length_hex, "%llX", length); PyErr_Format( _CBOR2_CBORDecodeValueError, "excessive string size 0x%s", length_hex); return NULL; } if (indefinite) ret = decode_indefinite_strings(self); else ret = decode_definite_string(self, (Py_ssize_t)length); set_shareable(self, ret); return ret; } static PyObject * decode_indefinite_array(CBORDecoderObject *self) { PyObject *array, *item, *ret = NULL; array = PyList_New(0); if (array) { ret = array; set_shareable(self, array); while (ret) { item = decode(self, DECODE_UNSHARED); if (item == break_marker) { Py_DECREF(item); break; } else if (item) { if (PyList_Append(array, item) == -1) ret = NULL; Py_DECREF(item); } else ret = NULL; } if (ret && self->immutable) { ret = PyList_AsTuple(array); if (ret) { Py_DECREF(array); // There's a potential here for an indefinite length recursive // array to wind up with a strange representation (the outer // being a tuple, the inners all being a list). However, a // recursive tuple isn't valid in the first place so it's a bit // of a waste of time searching for recursive references just // to throw an error set_shareable(self, ret); } else ret = NULL; } if (!ret) Py_DECREF(array); } return ret; } static PyObject * decode_definite_array(CBORDecoderObject *self, Py_ssize_t length) { Py_ssize_t i; PyObject *array, *item, *ret = NULL; if (length > 65536) { // Let cPython manage allocation of huge lists by appending // items one-by-one array = PyList_New(0); if (array) { ret = array; set_shareable(self, array); for (i = 0; i < length; ++i) { item = decode(self, DECODE_UNSHARED); if (item) { if (PyList_Append(array, item) == -1) { ret = NULL; Py_DECREF(item); break; } Py_DECREF(item); } else { ret = NULL; break; } } if (ret && self->immutable) { ret = PyList_AsTuple(array); if (ret) { Py_DECREF(array); // There's a potential here for an indefinite length recursive // array to wind up with a strange representation (the outer // being a tuple, the inners all being a list). However, a // recursive tuple isn't valid in the first place so it's a bit // of a waste of time searching for recursive references just // to throw an error set_shareable(self, ret); } else ret = NULL; } if (!ret) Py_DECREF(array); } } else { if (self->immutable) { array = PyTuple_New(length); if (array) { ret = array; for (i = 0; i < length; ++i) { item = decode(self, DECODE_UNSHARED); if (item) PyTuple_SET_ITEM(array, i, item); else { ret = NULL; break; } } } // This is done *after* the construction of the tuple because while // it's valid for a tuple object to be shared, it's not valid for it to // contain a reference to itself (because a reference to it can't exist // during its own construction ... in Python at least; as can be seen // above this *is* theoretically possible at the C level). set_shareable(self, ret); } else { array = PyList_New(length); if (array) { ret = array; set_shareable(self, array); for (i = 0; i < length; ++i) { item = decode(self, DECODE_UNSHARED); if (item) PyList_SET_ITEM(array, i, item); else { ret = NULL; break; } } } } if (!ret) Py_DECREF(array); } return ret; } static PyObject * decode_array(CBORDecoderObject *self, uint8_t subtype) { // major type 4 uint64_t length; bool indefinite = true; char length_hex[17]; if (decode_length(self, subtype, &length, &indefinite) == -1) return NULL; if (indefinite) return decode_indefinite_array(self); if (length > (uint64_t)PY_SSIZE_T_MAX) { sprintf(length_hex, "%llX", length); PyErr_Format( _CBOR2_CBORDecodeValueError, "excessive array size 0x%s", length_hex); return NULL; } else return decode_definite_array(self, (Py_ssize_t) length); } static PyObject * decode_map(CBORDecoderObject *self, uint8_t subtype) { // major type 5 uint64_t length; bool indefinite = true; PyObject *map, *key, *value, *ret = NULL; map = PyDict_New(); if (map) { ret = map; set_shareable(self, map); if (decode_length(self, subtype, &length, &indefinite) == 0) { if (indefinite) { while (ret) { key = decode(self, DECODE_IMMUTABLE | DECODE_UNSHARED); if (key == break_marker) { Py_DECREF(key); break; } else if (key) { value = decode(self, DECODE_UNSHARED); if (value) { if (PyDict_SetItem(map, key, value) == -1) ret = NULL; Py_DECREF(value); } else ret = NULL; Py_DECREF(key); } else ret = NULL; } } else { while (ret && length--) { key = decode(self, DECODE_IMMUTABLE | DECODE_UNSHARED); if (key) { value = decode(self, DECODE_UNSHARED); if (value) { if (PyDict_SetItem(map, key, value) == -1) ret = NULL; Py_DECREF(value); } else ret = NULL; Py_DECREF(key); } else ret = NULL; } } } else ret = NULL; if (!ret) Py_DECREF(map); } if (ret && self->immutable) { // _CBOR2_FrozenDict is initialized in CBORDecoder_init map = PyObject_CallFunctionObjArgs(_CBOR2_FrozenDict, ret, NULL); if (map) { set_shareable(self, map); Py_DECREF(ret); ret = map; } } if (ret && self->object_hook != Py_None) { map = PyObject_CallFunctionObjArgs(self->object_hook, self, ret, NULL); if (map) { set_shareable(self, map); Py_DECREF(ret); ret = map; } } return ret; } // Semantic decoders ///////////////////////////////////////////////////////// static PyObject * decode_semantic(CBORDecoderObject *self, uint8_t subtype) { // major type 6 uint64_t tagnum; PyObject *tag, *value, *ret = NULL; if (decode_length(self, subtype, &tagnum, NULL) == 0) { switch (tagnum) { case 0: ret = CBORDecoder_decode_datetime_string(self); break; case 1: ret = CBORDecoder_decode_epoch_datetime(self); break; case 2: ret = CBORDecoder_decode_positive_bignum(self); break; case 3: ret = CBORDecoder_decode_negative_bignum(self); break; case 4: ret = CBORDecoder_decode_fraction(self); break; case 5: ret = CBORDecoder_decode_bigfloat(self); break; case 25: ret = CBORDecoder_decode_stringref(self); break; case 28: ret = CBORDecoder_decode_shareable(self); break; case 29: ret = CBORDecoder_decode_sharedref(self); break; case 30: ret = CBORDecoder_decode_rational(self); break; case 35: ret = CBORDecoder_decode_regexp(self); break; case 36: ret = CBORDecoder_decode_mime(self); break; case 37: ret = CBORDecoder_decode_uuid(self); break; case 256: ret = CBORDecoder_decode_stringref_ns(self); break; case 258: ret = CBORDecoder_decode_set(self); break; case 260: ret = CBORDecoder_decode_ipaddress(self); break; case 261: ret = CBORDecoder_decode_ipnetwork(self); break; case 55799: ret = CBORDecoder_decode_self_describe_cbor(self); break; default: tag = CBORTag_New(tagnum); if (tag) { set_shareable(self, tag); value = decode(self, DECODE_UNSHARED); if (value) { if (CBORTag_SetValue(tag, value) == 0) { if (self->tag_hook == Py_None) { Py_INCREF(tag); ret = tag; } else { ret = PyObject_CallFunctionObjArgs( self->tag_hook, self, tag, NULL); set_shareable(self, ret); } } Py_DECREF(value); } Py_DECREF(tag); } break; } } return ret; } static PyObject * parse_datestr(CBORDecoderObject *self, PyObject *str) { const char* buf; char *p; Py_ssize_t size; PyObject *tz, *delta, *ret = NULL; bool offset_sign; unsigned long int Y, m, d, H, M, S, offset_H, offset_M, uS; if (!_CBOR2_timezone_utc && _CBOR2_init_timezone_utc() == -1) return NULL; buf = PyUnicode_AsUTF8AndSize(str, &size); if ( size < 20 || buf[4] != '-' || buf[7] != '-' || buf[10] != 'T' || buf[13] != ':' || buf[16] != ':') { PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid datetime string %R", str); return NULL; } if (buf) { Y = strtoul(buf, NULL, 10); m = strtoul(buf + 5, NULL, 10); d = strtoul(buf + 8, NULL, 10); H = strtoul(buf + 11, NULL, 10); M = strtoul(buf + 14, NULL, 10); S = strtoul(buf + 17, &p, 10); uS = 0; if (*p == '.') { unsigned long int scale = 100000; p++; while (*p >= '0' && *p <= '9') { uS += (*p++ - '0') * scale; scale /= 10; } } if (*p == 'Z') { offset_sign = false; Py_INCREF(_CBOR2_timezone_utc); tz = _CBOR2_timezone_utc; } else { tz = NULL; offset_sign = *p == '-'; if (offset_sign || *p == '+') { p++; offset_H = strtoul(p, &p, 10); offset_M = strtoul(p + 1, &p, 10); delta = PyDelta_FromDSU(0, (offset_sign ? -1 : 1) * (offset_H * 3600 + offset_M * 60), 0); if (delta) { #if PY_VERSION_HEX >= 0x03070000 tz = PyTimeZone_FromOffset(delta); #else tz = PyObject_CallFunctionObjArgs( _CBOR2_timezone, delta, NULL); #endif Py_DECREF(delta); } } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid datetime string %R", str); } if (tz) { ret = PyDateTimeAPI->DateTime_FromDateAndTime( Y, m, d, H, M, S, uS, tz, PyDateTimeAPI->DateTimeType); Py_DECREF(tz); } } return ret; } // CBORDecoder.decode_datetime_string(self) static PyObject * CBORDecoder_decode_datetime_string(CBORDecoderObject *self) { // semantic type 0 PyObject *match, *str, *ret = NULL; if (!_CBOR2_datestr_re && _CBOR2_init_re_compile() == -1) return NULL; str = decode(self, DECODE_NORMAL); if (str) { if (PyUnicode_Check(str)) { match = PyObject_CallMethodObjArgs( _CBOR2_datestr_re, _CBOR2_str_match, str, NULL); if (match) { if (match != Py_None) ret = parse_datestr(self, str); else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid datetime string: %R", str); Py_DECREF(match); } } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid datetime value: %R", str); Py_DECREF(str); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_epoch_datetime(self) static PyObject * CBORDecoder_decode_epoch_datetime(CBORDecoderObject *self) { // semantic type 1 PyObject *num, *tuple, *ret = NULL; if (!_CBOR2_timezone_utc && _CBOR2_init_timezone_utc() == -1) return NULL; num = decode(self, DECODE_NORMAL); if (num) { if (PyNumber_Check(num)) { tuple = PyTuple_Pack(2, num, _CBOR2_timezone_utc); if (tuple) { ret = PyDateTime_FromTimestamp(tuple); Py_DECREF(tuple); } } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid timestamp value %R", num); } Py_DECREF(num); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_positive_bignum(self) static PyObject * CBORDecoder_decode_positive_bignum(CBORDecoderObject *self) { // semantic type 2 PyObject *bytes, *ret = NULL; bytes = decode(self, DECODE_NORMAL); if (bytes) { if (PyBytes_CheckExact(bytes)) ret = PyObject_CallMethod( (PyObject*) &PyLong_Type, "from_bytes", "Os", bytes, "big"); else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid bignum value %R", bytes); Py_DECREF(bytes); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_negative_bignum(self) static PyObject * CBORDecoder_decode_negative_bignum(CBORDecoderObject *self) { // semantic type 3 PyObject *value, *one, *neg, *ret = NULL; value = CBORDecoder_decode_positive_bignum(self); if (value) { one = PyLong_FromLong(1); if (one) { neg = PyNumber_Negative(value); if (neg) { ret = PyNumber_Subtract(neg, one); Py_DECREF(neg); } Py_DECREF(one); } Py_DECREF(value); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_fraction(self) static PyObject * CBORDecoder_decode_fraction(CBORDecoderObject *self) { // semantic type 4 PyObject *payload_t, *tmp, *sig, *exp, *ret = NULL; PyObject *decimal_t, *sign, *digits, *args = NULL; if (!_CBOR2_Decimal && _CBOR2_init_Decimal() == -1) return NULL; // NOTE: There's no particular necessity for this to be immutable, it's // just a performance choice payload_t = decode(self, DECODE_IMMUTABLE | DECODE_UNSHARED); if (payload_t) { if (PyTuple_CheckExact(payload_t) && PyTuple_GET_SIZE(payload_t) == 2) { exp = PyTuple_GET_ITEM(payload_t, 0); sig = PyTuple_GET_ITEM(payload_t, 1); tmp = PyObject_CallFunction(_CBOR2_Decimal, "O", sig); if (tmp) { decimal_t = PyObject_CallMethod(tmp, "as_tuple", NULL); if (decimal_t) { sign = PyTuple_GET_ITEM(decimal_t, 0); digits = PyTuple_GET_ITEM(decimal_t, 1); args = PyTuple_Pack(3, sign, digits, exp); ret = PyObject_CallFunction(_CBOR2_Decimal, "(O)", args); Py_DECREF(decimal_t); Py_DECREF(args); } Py_DECREF(tmp); } } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "Incorrect tag 4 payload"); } Py_DECREF(payload_t); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_bigfloat static PyObject * CBORDecoder_decode_bigfloat(CBORDecoderObject *self) { // semantic type 5 PyObject *tuple, *tmp, *sig, *exp, *two, *ret = NULL; if (!_CBOR2_Decimal && _CBOR2_init_Decimal() == -1) return NULL; // NOTE: see semantic type 4 tuple = decode(self, DECODE_IMMUTABLE | DECODE_UNSHARED); if (tuple) { if (PyTuple_CheckExact(tuple) && PyTuple_GET_SIZE(tuple) == 2) { exp = PyTuple_GET_ITEM(tuple, 0); sig = PyTuple_GET_ITEM(tuple, 1); two = PyObject_CallFunction(_CBOR2_Decimal, "i", 2); if (two) { tmp = PyNumber_Power(two, exp, Py_None); if (tmp) { ret = PyNumber_Multiply(sig, tmp); Py_DECREF(tmp); } Py_DECREF(two); } } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "Incorrect tag 5 payload"); } Py_DECREF(tuple); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_stringref(self) static PyObject * CBORDecoder_decode_stringref(CBORDecoderObject *self) { // semantic type 25 PyObject *index, *ret = NULL; if (self->stringref_namespace == Py_None) { PyErr_Format( _CBOR2_CBORDecodeValueError, "string reference outside of namespace"); return NULL; } index = decode(self, DECODE_UNSHARED); if (index) { if (PyLong_CheckExact(index)) { ret = PyList_GetItem(self->stringref_namespace, PyLong_AsSsize_t(index)); if (ret) { // convert borrowed reference to new reference Py_INCREF(ret); } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "string reference %R not found", index); } } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid string reference %R", index); } } return ret; } // CBORDecoder.decode_shareable(self) static PyObject * CBORDecoder_decode_shareable(CBORDecoderObject *self) { // semantic type 28 Py_ssize_t old_index; PyObject *ret = NULL; old_index = self->shared_index; self->shared_index = PyList_GET_SIZE(self->shareables); if (PyList_Append(self->shareables, Py_None) == 0) ret = decode(self, DECODE_NORMAL); self->shared_index = old_index; return ret; } // CBORDecoder.decode_sharedref(self) static PyObject * CBORDecoder_decode_sharedref(CBORDecoderObject *self) { // semantic type 29 PyObject *index, *ret = NULL; index = decode(self, DECODE_UNSHARED); if (index) { if (PyLong_CheckExact(index)) { ret = PyList_GetItem(self->shareables, PyLong_AsSsize_t(index)); if (ret) { if (ret == Py_None) { PyErr_Format( _CBOR2_CBORDecodeValueError, "shared value %R has not been initialized", index); ret = NULL; } else { // convert borrowed reference to new reference Py_INCREF(ret); } } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "shared reference %R not found", index); } } else { PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid shared reference %R", index); } Py_DECREF(index); } return ret; } // CBORDecoder.decode_rational(self) static PyObject * CBORDecoder_decode_rational(CBORDecoderObject *self) { // semantic type 30 PyObject *tuple, *ret = NULL; if (!_CBOR2_Fraction && _CBOR2_init_Fraction() == -1) return NULL; // NOTE: see semantic type 4 tuple = decode(self, DECODE_IMMUTABLE | DECODE_UNSHARED); if (tuple) { if (PyTuple_CheckExact(tuple) && PyTuple_GET_SIZE(tuple) == 2) { ret = PyObject_CallFunctionObjArgs( _CBOR2_Fraction, PyTuple_GET_ITEM(tuple, 0), PyTuple_GET_ITEM(tuple, 1), NULL); } Py_DECREF(tuple); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_regexp(self) static PyObject * CBORDecoder_decode_regexp(CBORDecoderObject *self) { // semantic type 35 PyObject *pattern, *ret = NULL; if (!_CBOR2_re_compile && _CBOR2_init_re_compile() == -1) return NULL; pattern = decode(self, DECODE_UNSHARED); if (pattern) { ret = PyObject_CallFunctionObjArgs(_CBOR2_re_compile, pattern, NULL); Py_DECREF(pattern); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_mime(self) static PyObject * CBORDecoder_decode_mime(CBORDecoderObject *self) { // semantic type 36 PyObject *value, *parser, *ret = NULL; if (!_CBOR2_Parser && _CBOR2_init_Parser() == -1) return NULL; value = decode(self, DECODE_UNSHARED); if (value) { parser = PyObject_CallFunctionObjArgs(_CBOR2_Parser, NULL); if (parser) { ret = PyObject_CallMethodObjArgs(parser, _CBOR2_str_parsestr, value, NULL); Py_DECREF(parser); } Py_DECREF(value); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_uuid(self) static PyObject * CBORDecoder_decode_uuid(CBORDecoderObject *self) { // semantic type 37 PyObject *bytes, *ret = NULL; if (!_CBOR2_UUID && _CBOR2_init_UUID() == -1) return NULL; bytes = decode(self, DECODE_UNSHARED); if (bytes) { ret = PyObject_CallFunctionObjArgs(_CBOR2_UUID, Py_None, bytes, NULL); Py_DECREF(bytes); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_stringref_namespace(self) static PyObject * CBORDecoder_decode_stringref_ns(CBORDecoderObject *self) { // semantic type 256 PyObject *old_namespace, *ret = NULL; old_namespace = self->stringref_namespace; self->stringref_namespace = PyList_New(0); if (self->stringref_namespace) { ret = decode(self, DECODE_NORMAL); Py_CLEAR(self->stringref_namespace); } self->stringref_namespace = old_namespace; return ret; } // CBORDecoder.decode_set(self) static PyObject * CBORDecoder_decode_set(CBORDecoderObject *self) { // semantic type 258 PyObject *array, *ret = NULL; array = decode(self, DECODE_IMMUTABLE); if (array) { if (PyList_CheckExact(array) || PyTuple_CheckExact(array)) { if (self->immutable) ret = PyFrozenSet_New(array); else ret = PySet_New(array); } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid set array %R", array); Py_DECREF(array); } // This can be done after construction of the set/frozenset because, // unlike lists/dicts a set cannot contain a reference to itself (a set // is unhashable). Nor can a frozenset contain a reference to itself // because it can't refer to itself during its own construction. set_shareable(self, ret); return ret; } // CBORDecoder.decode_ipaddress(self) static PyObject * CBORDecoder_decode_ipaddress(CBORDecoderObject *self) { // semantic type 260 PyObject *tag, *bytes, *ret = NULL; if (!_CBOR2_ip_address && _CBOR2_init_ip_address() == -1) return NULL; bytes = decode(self, DECODE_UNSHARED); if (bytes) { if (PyBytes_CheckExact(bytes)) { if (PyBytes_GET_SIZE(bytes) == 4 || PyBytes_GET_SIZE(bytes) == 16) ret = PyObject_CallFunctionObjArgs(_CBOR2_ip_address, bytes, NULL); else if (PyBytes_GET_SIZE(bytes) == 6) { // MAC address tag = CBORTag_New(260); if (tag) { if (CBORTag_SetValue(tag, bytes) == 0) { if (self->tag_hook == Py_None) { Py_INCREF(tag); ret = tag; } else { ret = PyObject_CallFunctionObjArgs( self->tag_hook, self, tag, NULL); } } Py_DECREF(tag); } } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid ipaddress value %R", bytes); } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid ipaddress value %R", bytes); Py_DECREF(bytes); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_ipnetwork(self) static PyObject * CBORDecoder_decode_ipnetwork(CBORDecoderObject *self) { // semantic type 261 PyObject *map, *tuple, *bytes, *prefixlen, *ret = NULL; Py_ssize_t pos = 0; if (!_CBOR2_ip_network && _CBOR2_init_ip_address() == -1) return NULL; map = decode(self, DECODE_UNSHARED); if (map) { if (PyDict_CheckExact(map) && PyDict_Size(map) == 1) { if (PyDict_Next(map, &pos, &bytes, &prefixlen)) { if ( PyBytes_CheckExact(bytes) && PyLong_CheckExact(prefixlen) && (PyBytes_GET_SIZE(bytes) == 4 || PyBytes_GET_SIZE(bytes) == 16)) { tuple = PyTuple_Pack(2, bytes, prefixlen); if (tuple) { ret = PyObject_CallFunctionObjArgs( _CBOR2_ip_network, tuple, Py_False, NULL); Py_DECREF(tuple); } } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid ipnetwork value %R", map); } else // We've already checked the size is 1 so this shouldn't be // possible assert(0); } else PyErr_Format( _CBOR2_CBORDecodeValueError, "invalid ipnetwork value %R", map); Py_DECREF(map); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_self_describe_cbor(self) static PyObject * CBORDecoder_decode_self_describe_cbor(CBORDecoderObject *self) { // semantic tag 55799 return decode(self, DECODE_NORMAL); } // Special decoders ////////////////////////////////////////////////////////// static PyObject * decode_special(CBORDecoderObject *self, uint8_t subtype) { // major type 7 PyObject *tag, *ret = NULL; if ((subtype) < 20) { tag = PyStructSequence_New(&CBORSimpleValueType); if (tag) { PyStructSequence_SET_ITEM(tag, 0, PyLong_FromLong(subtype)); if (PyStructSequence_GET_ITEM(tag, 0)) { Py_INCREF(tag); ret = tag; } Py_DECREF(tag); // XXX Set shareable? } } else { switch (subtype) { case 20: Py_RETURN_FALSE; case 21: Py_RETURN_TRUE; case 22: Py_RETURN_NONE; case 23: CBOR2_RETURN_UNDEFINED; case 24: return CBORDecoder_decode_simple_value(self); case 25: return CBORDecoder_decode_float16(self); case 26: return CBORDecoder_decode_float32(self); case 27: return CBORDecoder_decode_float64(self); case 31: CBOR2_RETURN_BREAK; default: PyErr_Format( _CBOR2_CBORDecodeValueError, "Undefined Reserved major type 7 subtype 0x%x", subtype); break; } } return ret; } // CBORDecoder.decode_simple_value(self) static PyObject * CBORDecoder_decode_simple_value(CBORDecoderObject *self) { PyObject *tag, *ret = NULL; uint8_t buf; if (fp_read(self, (char*)&buf, sizeof(uint8_t)) == 0) { tag = PyStructSequence_New(&CBORSimpleValueType); if (tag) { PyStructSequence_SET_ITEM(tag, 0, PyLong_FromLong(buf)); if (PyStructSequence_GET_ITEM(tag, 0)) { Py_INCREF(tag); ret = tag; } Py_DECREF(tag); } } // XXX Set shareable? return ret; } // CBORDecoder.decode_float16(self) static PyObject * CBORDecoder_decode_float16(CBORDecoderObject *self) { PyObject *ret = NULL; union { uint16_t i; char buf[sizeof(uint16_t)]; } u; if (fp_read(self, u.buf, sizeof(uint16_t)) == 0) ret = PyFloat_FromDouble(unpack_float16(u.i)); set_shareable(self, ret); return ret; } // CBORDecoder.decode_float32(self) static PyObject * CBORDecoder_decode_float32(CBORDecoderObject *self) { PyObject *ret = NULL; union { uint32_t i; float f; char buf[sizeof(float)]; } u; if (fp_read(self, u.buf, sizeof(float)) == 0) { u.i = be32toh(u.i); ret = PyFloat_FromDouble(u.f); } set_shareable(self, ret); return ret; } // CBORDecoder.decode_float64(self) static PyObject * CBORDecoder_decode_float64(CBORDecoderObject *self) { PyObject *ret = NULL; union { uint64_t i; double f; char buf[sizeof(double)]; } u; if (fp_read(self, u.buf, sizeof(double)) == 0) { u.i = be64toh(u.i); ret = PyFloat_FromDouble(u.f); } set_shareable(self, ret); return ret; } PyObject * decode(CBORDecoderObject *self, DecodeOptions options) { bool old_immutable; Py_ssize_t old_index; PyObject *ret = NULL; LeadByte lead; if (options & DECODE_IMMUTABLE) { old_immutable = self->immutable; self->immutable = true; } if (options & DECODE_UNSHARED) { old_index = self->shared_index; self->shared_index = -1; } if (Py_EnterRecursiveCall(" in CBORDecoder.decode")) return NULL; if (fp_read(self, &lead.byte, 1) == 0) { switch (lead.major) { case 0: ret = decode_uint(self, lead.subtype); break; case 1: ret = decode_negint(self, lead.subtype); break; case 2: ret = decode_bytestring(self, lead.subtype); break; case 3: ret = decode_string(self, lead.subtype); break; case 4: ret = decode_array(self, lead.subtype); break; case 5: ret = decode_map(self, lead.subtype); break; case 6: ret = decode_semantic(self, lead.subtype); break; case 7: ret = decode_special(self, lead.subtype); break; default: assert(0); } } Py_LeaveRecursiveCall(); if (options & DECODE_IMMUTABLE) self->immutable = old_immutable; if (options & DECODE_UNSHARED) self->shared_index = old_index; return ret; } // CBORDecoder.decode(self) -> obj PyObject * CBORDecoder_decode(CBORDecoderObject *self) { return decode(self, DECODE_NORMAL); } // CBORDecoder.decode_from_bytes(self, data) static PyObject * CBORDecoder_decode_from_bytes(CBORDecoderObject *self, PyObject *data) { PyObject *save_read, *buf, *ret = NULL; if (!_CBOR2_BytesIO && _CBOR2_init_BytesIO() == -1) return NULL; save_read = self->read; buf = PyObject_CallFunctionObjArgs(_CBOR2_BytesIO, data, NULL); if (buf) { self->read = PyObject_GetAttr(buf, _CBOR2_str_read); if (self->read) { ret = decode(self, DECODE_NORMAL); Py_DECREF(self->read); } Py_DECREF(buf); } self->read = save_read; return ret; } // Decoder class definition ////////////////////////////////////////////////// #define PUBLIC_MAJOR(type) \ static PyObject * \ CBORDecoder_decode_##type(CBORDecoderObject *self, PyObject *subtype) \ { \ return decode_##type(self, (uint8_t) PyLong_AsUnsignedLong(subtype));\ } PUBLIC_MAJOR(uint); PUBLIC_MAJOR(negint); PUBLIC_MAJOR(bytestring); PUBLIC_MAJOR(string); PUBLIC_MAJOR(array); PUBLIC_MAJOR(map); PUBLIC_MAJOR(semantic); PUBLIC_MAJOR(special); #undef PUBLIC_MAJOR static PyGetSetDef CBORDecoder_getsetters[] = { {"fp", (getter) _CBORDecoder_get_fp, (setter) _CBORDecoder_set_fp, "input file-like object", NULL}, {"tag_hook", (getter) _CBORDecoder_get_tag_hook, (setter) _CBORDecoder_set_tag_hook, "hook called when decoding an unknown semantic tag", NULL}, {"object_hook", (getter) _CBORDecoder_get_object_hook, (setter) _CBORDecoder_set_object_hook, "hook called when decoding any dict", NULL}, {"str_errors", (getter) _CBORDecoder_get_str_errors, (setter) _CBORDecoder_set_str_errors, "the error mode to use when decoding UTF-8 encoded strings"}, {"immutable", (getter) _CBORDecoder_get_immutable, NULL, "when True, the next item decoded should be made immutable (a " "tuple instead of a list, a frozenset instead of a set, etc.)"}, {NULL} }; static PyMethodDef CBORDecoder_methods[] = { {"read", (PyCFunction) CBORDecoder_read, METH_O, "read the specified number of bytes from the input"}, // Decoding methods {"decode", (PyCFunction) CBORDecoder_decode, METH_NOARGS, "decode the next value from the input"}, {"decode_from_bytes", (PyCFunction) CBORDecoder_decode_from_bytes, METH_O, "decode the specified byte-string"}, {"decode_uint", (PyCFunction) CBORDecoder_decode_uint, METH_O, "decode an unsigned integer from the input"}, {"decode_negint", (PyCFunction) CBORDecoder_decode_negint, METH_O, "decode a negative integer from the input"}, {"decode_bytestring", (PyCFunction) CBORDecoder_decode_bytestring, METH_O, "decode a bytes string from the input"}, {"decode_string", (PyCFunction) CBORDecoder_decode_string, METH_O, "decode a unicode string from the input"}, {"decode_array", (PyCFunction) CBORDecoder_decode_array, METH_O, "decode a list or tuple from the input"}, {"decode_map", (PyCFunction) CBORDecoder_decode_map, METH_O, "decode a dict from the input"}, {"decode_semantic", (PyCFunction) CBORDecoder_decode_semantic, METH_O, "decode a semantically tagged value from the input"}, {"decode_special", (PyCFunction) CBORDecoder_decode_special, METH_O, "decode a special value from the input"}, {"decode_datetime_string", (PyCFunction) CBORDecoder_decode_datetime_string, METH_NOARGS, "decode a date-time string from the input"}, {"decode_epoch_datetime", (PyCFunction) CBORDecoder_decode_epoch_datetime, METH_NOARGS, "decode a timestamp offset from the input"}, {"decode_positive_bignum", (PyCFunction) CBORDecoder_decode_positive_bignum, METH_NOARGS, "decode a positive big-integer from the input"}, {"decode_negative_bignum", (PyCFunction) CBORDecoder_decode_negative_bignum, METH_NOARGS, "decode a negative big-integer from the input"}, {"decode_fraction", (PyCFunction) CBORDecoder_decode_fraction, METH_NOARGS, "decode a fractional number from the input"}, {"decode_rational", (PyCFunction) CBORDecoder_decode_rational, METH_NOARGS, "decode a rational value from the input"}, {"decode_bigfloat", (PyCFunction) CBORDecoder_decode_bigfloat, METH_NOARGS, "decode a large floating-point value from the input"}, {"decode_regexp", (PyCFunction) CBORDecoder_decode_regexp, METH_NOARGS, "decode a regular expression from the input"}, {"decode_mime", (PyCFunction) CBORDecoder_decode_mime, METH_NOARGS, "decode a MIME message from the input"}, {"decode_uuid", (PyCFunction) CBORDecoder_decode_uuid, METH_NOARGS, "decode a UUID from the input"}, {"decode_shareable", (PyCFunction) CBORDecoder_decode_shareable, METH_NOARGS, "decode a shareable value from the input"}, {"decode_sharedref", (PyCFunction) CBORDecoder_decode_sharedref, METH_NOARGS, "decode a shared reference from the input"}, {"decode_stringref", (PyCFunction) CBORDecoder_decode_stringref, METH_NOARGS, "decode a string reference from the input"}, {"decode_stringref_namespace", (PyCFunction) CBORDecoder_decode_stringref_ns, METH_NOARGS, "decode a string reference namespace from the input"}, {"decode_set", (PyCFunction) CBORDecoder_decode_set, METH_NOARGS, "decode a set or frozenset from the input"}, {"decode_ipaddress", (PyCFunction) CBORDecoder_decode_ipaddress, METH_NOARGS, "decode an IPv4Address or IPv6Address from the input"}, {"decode_ipnetwork", (PyCFunction) CBORDecoder_decode_ipnetwork, METH_NOARGS, "decode an IPv4Network or IPv6Network from the input"}, {"decode_self_describe_cbor", (PyCFunction) CBORDecoder_decode_self_describe_cbor, METH_NOARGS, "decode a data item after a self-describe CBOR tag"}, {"decode_simple_value", (PyCFunction) CBORDecoder_decode_simple_value, METH_NOARGS, "decode a CBORSimpleValue from the input"}, {"decode_float16", (PyCFunction) CBORDecoder_decode_float16, METH_NOARGS, "decode a half-precision floating-point value from the input"}, {"decode_float32", (PyCFunction) CBORDecoder_decode_float32, METH_NOARGS, "decode a floating-point value from the input"}, {"decode_float64", (PyCFunction) CBORDecoder_decode_float64, METH_NOARGS, "decode a double-precision floating-point value from the input"}, {"set_shareable", (PyCFunction) CBORDecoder_set_shareable, METH_O, "set the specified object as the current shareable reference"}, {NULL} }; PyDoc_STRVAR(CBORDecoder__doc__, "The CBORDecoder class implements a fully featured `CBOR`_ decoder with\n" "several extensions for handling shared references, big integers,\n" "rational numbers and so on. Typically the class is not used directly,\n" "but the :func:`cbor2.load` and :func:`cbor2.loads` functions are called\n" "to indirectly construct and use the class.\n" "\n" "When the class is constructed manually, the main entry points are\n" ":meth:`decode` and :meth:`decode_from_bytes`.\n" "\n" ":param tag_hook:\n" " callable that takes 2 arguments: the decoder instance, and the\n" " :class:`_cbor2.CBORTag` to be decoded. This callback is invoked for\n" " any tags for which there is no built-in decoder. The return value is\n" " substituted for the :class:`_cbor2.CBORTag` object in the\n" " deserialized output\n" ":param object_hook:\n" " callable that takes 2 arguments: the decoder instance, and a\n" " dictionary. This callback is invoked for each deserialized\n" " :class:`dict` object. The return value is substituted for the dict\n" " in the deserialized output.\n" "\n" ".. _CBOR: https://cbor.io/\n" ); PyTypeObject CBORDecoderType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_cbor2.CBORDecoder", .tp_doc = CBORDecoder__doc__, .tp_basicsize = sizeof(CBORDecoderObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, .tp_new = CBORDecoder_new, .tp_init = (initproc) CBORDecoder_init, .tp_dealloc = (destructor) CBORDecoder_dealloc, .tp_traverse = (traverseproc) CBORDecoder_traverse, .tp_clear = (inquiry) CBORDecoder_clear, .tp_getset = CBORDecoder_getsetters, .tp_methods = CBORDecoder_methods, }; cbor2-5.4.2/source/decoder.h000066400000000000000000000011411413200260400155740ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include typedef struct { PyObject_HEAD PyObject *read; // cached read() method of fp PyObject *tag_hook; PyObject *object_hook; PyObject *shareables; PyObject *stringref_namespace; PyObject *str_errors; bool immutable; Py_ssize_t shared_index; } CBORDecoderObject; extern PyTypeObject CBORDecoderType; PyObject * CBORDecoder_new(PyTypeObject *, PyObject *, PyObject *); int CBORDecoder_init(CBORDecoderObject *, PyObject *, PyObject *); PyObject * CBORDecoder_decode(CBORDecoderObject *); cbor2-5.4.2/source/encoder.c000066400000000000000000002076031413200260400156140ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include #if __FreeBSD__ #include #elif __APPLE__ #include #elif ! _WIN32 #include #endif #include #include #include #include #include "module.h" #include "halffloat.h" #include "tags.h" #include "encoder.h" #if __APPLE__ #define htobe16(x) OSSwapHostToBigInt16(x) #define htobe32(x) OSSwapHostToBigInt32(x) #define htobe64(x) OSSwapHostToBigInt64(x) #elif _WIN32 // All windows platforms are (currently) little-endian so byteswap is required #define htobe16(x) _byteswap_ushort(x) #define htobe32(x) _byteswap_ulong(x) #define htobe64(x) _byteswap_uint64(x) #endif typedef PyObject * (EncodeFunction)(CBOREncoderObject *, PyObject *); static int encode_semantic(CBOREncoderObject *, const uint64_t, PyObject *); static PyObject * encode_shared(CBOREncoderObject *, EncodeFunction *, PyObject *); static PyObject * encode_container(CBOREncoderObject *, EncodeFunction *, PyObject *); static PyObject * CBOREncoder_encode_to_bytes(CBOREncoderObject *, PyObject *); static PyObject * CBOREncoder_encode_int(CBOREncoderObject *, PyObject *); static PyObject * CBOREncoder_encode_float(CBOREncoderObject *, PyObject *); static int _CBOREncoder_set_fp(CBOREncoderObject *, PyObject *, void *); static int _CBOREncoder_set_default(CBOREncoderObject *, PyObject *, void *); static int _CBOREncoder_set_timezone(CBOREncoderObject *, PyObject *, void *); // Constructors and destructors ////////////////////////////////////////////// static int CBOREncoder_traverse(CBOREncoderObject *self, visitproc visit, void *arg) { Py_VISIT(self->write); Py_VISIT(self->encoders); Py_VISIT(self->default_handler); Py_VISIT(self->shared); Py_VISIT(self->tz); Py_VISIT(self->shared_handler); Py_VISIT(self->string_references); return 0; } static int CBOREncoder_clear(CBOREncoderObject *self) { Py_CLEAR(self->write); Py_CLEAR(self->encoders); Py_CLEAR(self->default_handler); Py_CLEAR(self->shared); Py_CLEAR(self->tz); Py_CLEAR(self->shared_handler); Py_CLEAR(self->string_references); return 0; } // CBOREncoder.__del__(self) static void CBOREncoder_dealloc(CBOREncoderObject *self) { PyObject_GC_UnTrack(self); CBOREncoder_clear(self); Py_TYPE(self)->tp_free((PyObject *) self); } // CBOREncoder.__new__(cls, *args, **kwargs) PyObject * CBOREncoder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { CBOREncoderObject *self; PyDateTime_IMPORT; if (!PyDateTimeAPI) return NULL; self = (CBOREncoderObject *) type->tp_alloc(type, 0); if (self) { Py_INCREF(Py_None); self->encoders = Py_None; Py_INCREF(Py_None); self->shared = Py_None; Py_INCREF(Py_None); self->write = Py_None; Py_INCREF(Py_None); self->default_handler = Py_None; Py_INCREF(Py_None); self->tz = Py_None; Py_INCREF(Py_None); self->string_references = Py_None; self->enc_style = 0; self->timestamp_format = false; self->value_sharing = false; self->shared_handler = NULL; self->string_referencing = false; self->string_namespacing = false; } return (PyObject *) self; } // CBOREncoder.__init__(self, fp=None, datetime_as_timestamp=0, timezone=None, // value_sharing=False, default=None, canonical=False, // date_as_datetime=False) int CBOREncoder_init(CBOREncoderObject *self, PyObject *args, PyObject *kwargs) { static char *keywords[] = { "fp", "datetime_as_timestamp", "timezone", "value_sharing", "default", "canonical", "date_as_datetime", "string_referencing", NULL }; PyObject *tmp, *fp = NULL, *default_handler = NULL, *tz = NULL; int value_sharing = 0, timestamp_format = 0, enc_style = 0, date_as_datetime = 0, string_referencing = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|pOpOppp", keywords, &fp, ×tamp_format, &tz, &value_sharing, &default_handler, &enc_style, &date_as_datetime, &string_referencing)) return -1; // Predicate values are returned as ints, but need to be stored as bool or ubyte if (timestamp_format == 1) self->timestamp_format = true; if (value_sharing == 1) self->value_sharing = true; if (enc_style == 1) self->enc_style = 1; if (string_referencing == 1) { self->string_referencing = true; self->string_namespacing = true; } if (_CBOREncoder_set_fp(self, fp, NULL) == -1) return -1; if (default_handler && _CBOREncoder_set_default(self, default_handler, NULL) == -1) return -1; if (tz && _CBOREncoder_set_timezone(self, tz, NULL) == -1) return -1; self->shared = PyDict_New(); if (!self->shared) return -1; self->string_references = PyDict_New(); if (!self->string_references) return -1; if (!_CBOR2_default_encoders && init_default_encoders() == -1) return -1; tmp = self->encoders; self->encoders = PyObject_CallMethodObjArgs( _CBOR2_default_encoders, _CBOR2_str_copy, NULL); Py_DECREF(tmp); if (!self->encoders) return -1; if (self->enc_style) { if (!_CBOR2_canonical_encoders && init_canonical_encoders() == -1) return -1; if (!PyObject_CallMethodObjArgs(self->encoders, _CBOR2_str_update, _CBOR2_canonical_encoders, NULL)) return -1; } if (date_as_datetime == 1) { PyObject *encode_date = PyObject_GetAttr((PyObject *) &CBOREncoderType, _CBOR2_str_encode_date); if (!encode_date) return -1; PyObject *datetime_class = PyDateTimeAPI->DateType; if (PyObject_SetItem(self->encoders, datetime_class, encode_date) == -1) return -1; Py_DECREF(encode_date); } return 0; } // Property accessors //////////////////////////////////////////////////////// // CBOREncoder._get_fp(self) static PyObject * _CBOREncoder_get_fp(CBOREncoderObject *self, void *closure) { PyObject *ret = PyMethod_GET_SELF(self->write); Py_INCREF(ret); return ret; } // CBOREncoder._set_fp(self, value) static int _CBOREncoder_set_fp(CBOREncoderObject *self, PyObject *value, void *closure) { PyObject *tmp, *write; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete fp attribute"); return -1; } write = PyObject_GetAttr(value, _CBOR2_str_write); if (!(write && PyCallable_Check(write))) { PyErr_SetString(PyExc_ValueError, "fp object must have a callable write method"); return -1; } // It's a bit naughty caching the write method, but it does provide a // notable speed boost avoiding the lookup of the method on every write. // Still, it is theoretically valid for an object to change its write() // method in the middle of a dump. But unless someone actually complains // about this I'm loathe to change it... tmp = self->write; // NOTE: no need to INCREF write here as GetAttr returns a new ref self->write = write; Py_DECREF(tmp); return 0; } // CBOREncoder._get_default(self) static PyObject * _CBOREncoder_get_default(CBOREncoderObject *self, void *closure) { Py_INCREF(self->default_handler); return self->default_handler; } // CBOREncoder._set_default(self, value) static int _CBOREncoder_set_default(CBOREncoderObject *self, PyObject *value, void *closure) { PyObject *tmp; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete default attribute"); return -1; } if (value != Py_None && !PyCallable_Check(value)) { PyErr_Format(PyExc_ValueError, "invalid default value %R (must be callable " "or None)", value); return -1; } tmp = self->default_handler; Py_INCREF(value); self->default_handler = value; Py_DECREF(tmp); return 0; } // CBOREncoder._get_timezone(self) static PyObject * _CBOREncoder_get_timezone(CBOREncoderObject *self, void *closure) { Py_INCREF(self->tz); return self->tz; } // CBOREncoder._set_timezone(self, value) static int _CBOREncoder_set_timezone(CBOREncoderObject *self, PyObject *value, void *closure) { PyObject *tmp; if (!value) { PyErr_SetString(PyExc_AttributeError, "cannot delete timezone attribute"); return -1; } if (!PyTZInfo_Check(value) && value != Py_None) { PyErr_Format(PyExc_ValueError, "invalid timezone value %R (must be tzinfo instance " "or None)", value); return -1; } tmp = self->tz; Py_INCREF(value); self->tz = value; Py_DECREF(tmp); return 0; } // CBOREncoder._get_canonical(self) static PyObject * _CBOREncoder_get_canonical(CBOREncoderObject *self, void *closure) { if (self->enc_style) Py_RETURN_TRUE; else Py_RETURN_FALSE; } // Utility methods /////////////////////////////////////////////////////////// static int fp_write(CBOREncoderObject *self, const char *buf, const Py_ssize_t length) { PyObject *bytes, *ret = NULL; bytes = PyBytes_FromStringAndSize(buf, length); if (bytes) { ret = PyObject_CallFunctionObjArgs(self->write, bytes, NULL); Py_XDECREF(ret); Py_DECREF(bytes); } return ret ? 0 : -1; } // CBOREncoder.write(self, data) static PyObject * CBOREncoder_write(CBOREncoderObject *self, PyObject *data) { if (!PyBytes_Check(data)) { PyErr_SetString(PyExc_TypeError, "expected bytes for writing"); return NULL; } if (fp_write(self, PyBytes_AS_STRING(data), PyBytes_GET_SIZE(data)) == -1) return NULL; Py_RETURN_NONE; } static int encode_length(CBOREncoderObject *self, const uint8_t major_tag, const uint64_t length) { LeadByte *lead; char buf[sizeof(LeadByte) + sizeof(uint64_t)]; lead = (LeadByte*)buf; lead->major = major_tag; if (length < 24) { lead->subtype = (uint8_t) length; return fp_write(self, buf, 1); } else if (length <= UCHAR_MAX) { lead->subtype = 24; buf[1] = (uint8_t) length; return fp_write(self, buf, sizeof(uint8_t) + 1); } else if (length <= USHRT_MAX) { lead->subtype = 25; *((uint16_t*)(buf + 1)) = htobe16((uint16_t) length); return fp_write(self, buf, sizeof(uint16_t) + 1); } else if (length <= UINT_MAX) { lead->subtype = 26; *((uint32_t*)(buf + 1)) = htobe32((uint32_t) length); return fp_write(self, buf, sizeof(uint32_t) + 1); } else { lead->subtype = 27; *((uint64_t*)(buf + 1)) = htobe64(length); return fp_write(self, buf, sizeof(uint64_t) + 1); } } // CBOREncoder.encode_length(self, major_tag, length) static PyObject * CBOREncoder_encode_length(CBOREncoderObject *self, PyObject *args) { uint8_t major_tag; uint64_t length; if (!PyArg_ParseTuple(args, "BK", &major_tag, &length)) return NULL; if (encode_length(self, major_tag, length) == -1) return NULL; Py_RETURN_NONE; } // Given a deferred type tuple (module-name, type-name), find the specified // module in sys.modules, get the specified type from within it and return it // as a new reference. Returns NULL without setting an error if the module // cannot be found (indicating it hasn't been loaded and therefore cannot be // the type we're looking for), or sets an error if the specified type cannot // be found within it static PyObject * find_deferred(PyObject *type_tuple) { PyObject *mod_name, *mod, *type_name; if (PyTuple_GET_SIZE(type_tuple) == 2) { mod_name = PyTuple_GET_ITEM(type_tuple, 0); type_name = PyTuple_GET_ITEM(type_tuple, 1); if (PyUnicode_Check(mod_name) && PyUnicode_Check(type_name)) { mod = PyDict_GetItem(PyImport_GetModuleDict(), mod_name); if (!mod) return NULL; return PyObject_GetAttr(mod, type_name); } } PyErr_Format(_CBOR2_CBOREncodeValueError, "invalid deferred encoder type %R (must be a 2-tuple of module " "name and type name, e.g. ('collections', 'defaultdict'))", type_tuple); return NULL; } // Given a deferred type item tuple from the self->encoders dictionary, attempt // to find the specified type (by calling find_deferred) and replace the entry // in the dictionary with the discovered type, mapping to the same handler static PyObject * replace_deferred(CBOREncoderObject *self, PyObject *item) { PyObject *enc_type, *encoder, *ret = NULL; enc_type = PyTuple_GET_ITEM(item, 0); encoder = PyTuple_GET_ITEM(item, 1); ret = find_deferred(enc_type); if (ret) { if (PyObject_DelItem(self->encoders, enc_type) == -1) { Py_DECREF(ret); ret = NULL; } else if (PyObject_SetItem(self->encoders, ret, encoder) == -1) { Py_DECREF(ret); ret = NULL; } } return ret; } // Try to encode a string reference if the string has already been // emitted. static int stringref(CBOREncoderObject *self, PyObject *value) { PyObject *index, *ret = NULL; int retcode = -1; index = PyDict_GetItem(self->string_references, value); if (index) { if (encode_length(self, 6, 25) == 0) { ret = CBOREncoder_encode_int(self, index); if (ret) { Py_DECREF(ret); retcode = 1; } } } else { uint64_t length = PyObject_Length(value); uint64_t next_index = PyDict_Size(self->string_references); bool is_referenced = true; if (next_index < 24) { is_referenced = length >= 3; } else if (next_index < 256) { is_referenced = length >= 4; } else if (next_index < 65536) { is_referenced = length >= 5; } else if (next_index < 4294967296ull) { is_referenced = length >= 7; } else { is_referenced = length >= 11; } if (is_referenced) { index = PyLong_FromLongLong(next_index); if (index && PyDict_SetItem(self->string_references, value, index) == 0) retcode = 0; } else { retcode = 0; } } return retcode; } // CBOREncoder._find_encoder(type) static PyObject * CBOREncoder_find_encoder(CBOREncoderObject *self, PyObject *type) { PyObject *enc_type, *items, *iter, *item, *ret; ret = PyObject_GetItem(self->encoders, type); if (!ret && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); items = PyMapping_Items(self->encoders); if (items) { iter = PyObject_GetIter(items); if (iter) { while (!ret && (item = PyIter_Next(iter))) { enc_type = PyTuple_GET_ITEM(item, 0); if (PyTuple_Check(enc_type)) { enc_type = replace_deferred(self, item); // This DECREF might look strange but at this point, // enc_type is a new reference rather than borrowed as // it was previously. However, we know a reference to // it must exist in the encoders dictionary so it's // safe to convert without it being destroyed if (enc_type) Py_DECREF(enc_type); } if (enc_type) switch (PyObject_IsSubclass(type, enc_type)) { case 1: ret = PyTuple_GET_ITEM(item, 1); if (PyObject_SetItem(self->encoders, type, ret) == 0) break; // fall-thru to error case case -1: enc_type = NULL; ret = NULL; break; } Py_DECREF(item); // We need to check PyErr_Occurred here as enc_type can be // NULL with no error in the case replace_deferred found // no loaded module with the specified name in which case // we should simply continue to the next entry if (!enc_type && PyErr_Occurred()) break; } Py_DECREF(iter); } Py_DECREF(items); } if (!ret && !PyErr_Occurred()) ret = Py_None; if (ret) Py_INCREF(ret); } return ret; } // Major encoders //////////////////////////////////////////////////////////// static PyObject * encode_negative_int(PyObject *value) { PyObject *neg, *one, *ret = NULL; // return -value - 1 one = PyLong_FromLong(1); if (one) { neg = PyNumber_Negative(value); if (neg) { ret = PyNumber_Subtract(neg, one); Py_DECREF(neg); } Py_DECREF(one); } return ret; } static PyObject * encode_larger_int(CBOREncoderObject *self, PyObject *value) { PyObject *zero, *bits, *buf, *tmp, *ret = NULL; uint8_t major_tag; unsigned long long val; zero = PyLong_FromLong(0); if (zero) { major_tag = 0; // This isn't strictly required for the positive case, but ensuring // value is a new (instead of "borrowed") reference simplifies the // ref counting later Py_INCREF(value); switch (PyObject_RichCompareBool(value, zero, Py_LT)) { case 1: major_tag = 1; tmp = encode_negative_int(value); Py_DECREF(value); value = tmp; // fall-thru to positive case case 0: val = PyLong_AsUnsignedLongLong(value); if (!PyErr_Occurred()) { if (encode_length(self, major_tag, val) == 0) { Py_INCREF(Py_None); ret = Py_None; break; } } // fall-thru to error case case -1: // if error is overflow encode a big-num if (PyErr_ExceptionMatches(PyExc_OverflowError)) { PyErr_Clear(); major_tag += 2; bits = PyObject_CallMethodObjArgs( value, _CBOR2_str_bit_length, NULL); if (bits) { long length = PyLong_AsLong(bits); if (!PyErr_Occurred()) { buf = PyObject_CallMethod( value, "to_bytes", "ls", (length + 7) / 8, "big"); if (buf) { if (encode_semantic(self, major_tag, buf) == 0) { Py_INCREF(Py_None); ret = Py_None; } Py_DECREF(buf); } } Py_DECREF(bits); } } break; default: assert(0); } Py_DECREF(value); } return ret; } // CBOREncoder.encode_int(self, value) static PyObject * CBOREncoder_encode_int(CBOREncoderObject *self, PyObject *value) { // major types 0 and 1 PyObject *ret = NULL; long val; int overflow; val = PyLong_AsLongAndOverflow(value, &overflow); if (overflow == 0) { // fast-path: technically this branch isn't needed, but longs are much // faster than long longs on some archs and it's likely the *vast* // majority of ints encoded will fall into this size if (val != -1 || !PyErr_Occurred()) { if (val >= 0) { if (encode_length(self, 0, val) == 0) { Py_INCREF(Py_None); ret = Py_None; } } else { // avoid overflow in the case where int_value == -2^31 val = -(val + 1); if (encode_length(self, 1, val) == 0) { Py_INCREF(Py_None); ret = Py_None; } } } } else ret = encode_larger_int(self, value); return ret; } // CBOREncoder.encode_bytestring(self, value) static PyObject * CBOREncoder_encode_bytestring(CBOREncoderObject *self, PyObject *value) { // major type 2 char *buf; Py_ssize_t length; if (PyBytes_AsStringAndSize(value, &buf, &length) == -1) return NULL; if (self->string_referencing) { switch (stringref(self, value)) { case -1: return NULL; case 1: Py_RETURN_NONE; } } if (encode_length(self, 2, length) == -1) return NULL; if (fp_write(self, buf, length) == -1) return NULL; Py_RETURN_NONE; } // CBOREncoder.encode_bytearray(self, value) static PyObject * CBOREncoder_encode_bytearray(CBOREncoderObject *self, PyObject *value) { // major type 2 (again) Py_ssize_t length; if (!PyByteArray_Check(value)) { PyErr_Format(_CBOR2_CBOREncodeValueError, "invalid bytearray value %R", value); return NULL; } if (self->string_referencing) { switch (stringref(self, value)) { case -1: return NULL; case 1: Py_RETURN_NONE; } } length = PyByteArray_GET_SIZE(value); if (encode_length(self, 2, length) == -1) return NULL; if (fp_write(self, PyByteArray_AS_STRING(value), length) == -1) return NULL; Py_RETURN_NONE; } // CBOREncoder.encode_string(self, value) static PyObject * CBOREncoder_encode_string(CBOREncoderObject *self, PyObject *value) { // major type 3 const char *buf; Py_ssize_t length; buf = PyUnicode_AsUTF8AndSize(value, &length); if (!buf) return NULL; if (self->string_referencing) switch (stringref(self, value)) { case -1: return NULL; case 1: Py_RETURN_NONE; } if (encode_length(self, 3, length) == -1) return NULL; if (fp_write(self, buf, length) == -1) return NULL; Py_RETURN_NONE; } static PyObject * encode_array(CBOREncoderObject *self, PyObject *value) { PyObject **items, *fast, *ret = NULL; Py_ssize_t length; fast = PySequence_Fast(value, "argument must be iterable"); if (fast) { length = PySequence_Fast_GET_SIZE(fast); items = PySequence_Fast_ITEMS(fast); if (encode_length(self, 4, length) == 0) { while (length) { ret = CBOREncoder_encode(self, *items); if (ret) Py_DECREF(ret); else goto error; items++; length--; } Py_INCREF(Py_None); ret = Py_None; } error: Py_DECREF(fast); } return ret; } // CBOREncoder.encode_array(self, value) static PyObject * CBOREncoder_encode_array(CBOREncoderObject *self, PyObject *value) { // major type 4 return encode_container(self, &encode_array, value); } static PyObject * encode_dict(CBOREncoderObject *self, PyObject *value) { PyObject *key, *val, *ret; Py_ssize_t pos = 0; if (encode_length(self, 5, PyDict_Size(value)) == 0) { while (PyDict_Next(value, &pos, &key, &val)) { Py_INCREF(key); ret = CBOREncoder_encode(self, key); Py_DECREF(key); if (ret) Py_DECREF(ret); else return NULL; Py_INCREF(val); ret = CBOREncoder_encode(self, val); Py_DECREF(val); if (ret) Py_DECREF(ret); else return NULL; } } Py_RETURN_NONE; } static PyObject * encode_mapping(CBOREncoderObject *self, PyObject *value) { PyObject **items, *list, *fast, *ret = NULL; Py_ssize_t length; list = PyMapping_Items(value); if (list) { fast = PySequence_Fast(list, "internal error"); if (fast) { length = PySequence_Fast_GET_SIZE(fast); items = PySequence_Fast_ITEMS(fast); if (encode_length(self, 5, length) == 0) { while (length) { ret = CBOREncoder_encode(self, PyTuple_GET_ITEM(*items, 0)); if (ret) Py_DECREF(ret); else goto error; ret = CBOREncoder_encode(self, PyTuple_GET_ITEM(*items, 1)); if (ret) Py_DECREF(ret); else goto error; items++; length--; } ret = Py_None; Py_INCREF(ret); } error: Py_DECREF(fast); } Py_DECREF(list); } return ret; } static PyObject * CBOREncoder__encode_map(CBOREncoderObject *self, PyObject *value) { if (PyDict_Check(value)) return encode_dict(self, value); else return encode_mapping(self, value); } // CBOREncoder.encode_map(self, value) static PyObject * CBOREncoder_encode_map(CBOREncoderObject *self, PyObject *value) { // major type 5 return encode_container(self, &CBOREncoder__encode_map, value); } // Semantic encoders ///////////////////////////////////////////////////////// static int encode_semantic(CBOREncoderObject *self, const uint64_t tag, PyObject *value) { PyObject *obj; if (encode_length(self, 6, tag) == -1) return -1; obj = CBOREncoder_encode(self, value); Py_XDECREF(obj); return obj == NULL ? -1 : 0; } // CBOREncoder.encode_semantic(self, tag) static PyObject * CBOREncoder_encode_semantic(CBOREncoderObject *self, PyObject *value) { // major type 6 CBORTagObject *tag; PyObject *ret = NULL; PyObject *old_string_references = self->string_references; bool old_string_referencing = self->string_referencing; if (!CBORTag_CheckExact(value)) return NULL; tag = (CBORTagObject *) value; if (tag->tag == 256) { PyObject *string_references = PyDict_New(); if (!string_references) return NULL; self->string_referencing = true; self->string_references = string_references; } if (encode_semantic(self, tag->tag, tag->value) == 0) { Py_INCREF(Py_None); ret = Py_None; } if (self->string_references != old_string_references) { Py_DECREF(self->string_references); } self->string_references = old_string_references; self->string_referencing = old_string_referencing; return ret; } static PyObject * encode_datestr(CBOREncoderObject *self, PyObject *datestr) { const char *buf; Py_ssize_t length, match; match = PyUnicode_Tailmatch( datestr, _CBOR2_str_utc_suffix, PyUnicode_GET_LENGTH(datestr) - 6, PyUnicode_GET_LENGTH(datestr), 1); if (match != -1) { buf = PyUnicode_AsUTF8AndSize(datestr, &length); if (buf) { if (fp_write(self, "\xC0", 1) == 0) { if (match) { if (encode_length(self, 3, length - 5) == 0) if (fp_write(self, buf, length - 6) == 0) if (fp_write(self, "Z", 1) == 0) Py_RETURN_NONE; } else { if (encode_length(self, 3, length) == 0) if (fp_write(self, buf, length) == 0) Py_RETURN_NONE; } } } } return NULL; } static PyObject * encode_timestamp(CBOREncoderObject *self, PyObject *timestamp) { PyObject *ret = NULL; if (fp_write(self, "\xC1", 1) == 0) { double d = PyFloat_AS_DOUBLE(timestamp); if (d == trunc(d)) { PyObject *i = PyLong_FromDouble(d); if (i) { ret = CBOREncoder_encode_int(self, i); Py_DECREF(i); } } else { ret = CBOREncoder_encode_float(self, timestamp); } } return ret; } // CBOREncoder.encode_datetime(self, value) static PyObject * CBOREncoder_encode_datetime(CBOREncoderObject *self, PyObject *value) { // semantic type 0 or 1 PyObject *tmp, *ret = NULL; if (PyDateTime_Check(value)) { if (!((PyDateTime_DateTime*)value)->hastzinfo) { if (self->tz != Py_None) { value = PyDateTimeAPI->DateTime_FromDateAndTime( PyDateTime_GET_YEAR(value), PyDateTime_GET_MONTH(value), PyDateTime_GET_DAY(value), PyDateTime_DATE_GET_HOUR(value), PyDateTime_DATE_GET_MINUTE(value), PyDateTime_DATE_GET_SECOND(value), PyDateTime_DATE_GET_MICROSECOND(value), self->tz, PyDateTimeAPI->DateTimeType); } else { PyErr_Format(_CBOR2_CBOREncodeValueError, "naive datetime %R encountered and no default " "timezone has been set", value); value = NULL; } } else { // convert value from borrowed to a new reference to simplify our // cleanup later Py_INCREF(value); } if (value) { if (self->timestamp_format) { tmp = PyObject_CallMethodObjArgs( value, _CBOR2_str_timestamp, NULL); if (tmp) ret = encode_timestamp(self, tmp); } else { tmp = PyObject_CallMethodObjArgs( value, _CBOR2_str_isoformat, NULL); if (tmp) ret = encode_datestr(self, tmp); } Py_XDECREF(tmp); Py_DECREF(value); } } return ret; } // CBOREncoder.encode_date(self, value) static PyObject * CBOREncoder_encode_date(CBOREncoderObject *self, PyObject *value) { PyObject *datetime, *ret = NULL; if (PyDate_Check(value)) { datetime = PyDateTimeAPI->DateTime_FromDateAndTime( PyDateTime_GET_YEAR(value), PyDateTime_GET_MONTH(value), PyDateTime_GET_DAY(value), 0, 0, 0, 0, self->tz, PyDateTimeAPI->DateTimeType); if (datetime) { ret = CBOREncoder_encode_datetime(self, datetime); Py_DECREF(datetime); return ret; } } return NULL; } // A variant of fp_classify for the decimal.Decimal type static int decimal_classify(PyObject *value) { PyObject *tmp; tmp = PyObject_CallMethodObjArgs(value, _CBOR2_str_is_nan, NULL); if (tmp) { if (PyObject_IsTrue(tmp)) { Py_DECREF(tmp); return DC_NAN; } else { Py_DECREF(tmp); tmp = PyObject_CallMethodObjArgs( value, _CBOR2_str_is_infinite, NULL); if (tmp) { if (PyObject_IsTrue(tmp)) { Py_DECREF(tmp); return DC_INFINITE; } else { Py_DECREF(tmp); return DC_NORMAL; } } } } return DC_ERROR; } // Returns 1 if the decimal.Decimal value is < 0, 0 if it's >= 0, and -1 on // error static int decimal_negative(PyObject *value) { PyObject *zero; int ret = -1; zero = PyLong_FromLong(0); if (zero) { ret = PyObject_RichCompareBool(value, zero, Py_GT); Py_DECREF(zero); } return ret; } static PyObject * encode_decimal_digits(CBOREncoderObject *self, PyObject *value) { PyObject *tuple, *digits, *exp, *sig, *ten, *tmp, *ret = NULL; int sign = 0; bool sharing; tuple = PyObject_CallMethodObjArgs(value, _CBOR2_str_as_tuple, NULL); if (tuple) { if (PyArg_ParseTuple(tuple, "pOO", &sign, &digits, &exp)) { sig = PyLong_FromLong(0); if (sig) { ten = PyLong_FromLong(10); if (ten) { Py_ssize_t length = PyTuple_GET_SIZE(digits); // for digit in digits: sig = (sig * 10) + digit for (Py_ssize_t i = 0; i < length; ++i) { tmp = PyNumber_Multiply(sig, ten); if (!tmp) break; Py_DECREF(sig); sig = tmp; tmp = PyNumber_Add(sig, PyTuple_GET_ITEM(digits, i)); if (!tmp) break; Py_DECREF(sig); sig = tmp; } Py_DECREF(ten); // if sign: sig = -sig if (tmp && sign) { tmp = PyNumber_Negative(sig); if (tmp) { Py_DECREF(sig); sig = tmp; } } if (tmp) { sharing = self->value_sharing; self->value_sharing = false; value = PyTuple_Pack(2, exp, sig); if (value) { if (encode_semantic(self, 4, value) == 0) { Py_INCREF(Py_None); ret = Py_None; } Py_DECREF(value); } self->value_sharing = sharing; } } Py_DECREF(sig); } } Py_DECREF(tuple); } return ret; } // CBOREncoder.encode_decimal(self, value) static PyObject * CBOREncoder_encode_decimal(CBOREncoderObject *self, PyObject *value) { // semantic type 4 switch (decimal_classify(value)) { case DC_NAN: if (fp_write(self, "\xF9\x7E\x00", 3) == -1) return NULL; break; case DC_INFINITE: switch (decimal_negative(value)) { case 1: if (fp_write(self, "\xF9\x7C\x00", 3) == -1) return NULL; break; case 0: if (fp_write(self, "\xF9\xFC\x00", 3) == -1) return NULL; break; case -1: return NULL; default: assert(0); } break; case DC_NORMAL: return encode_decimal_digits(self, value); case DC_ERROR: return NULL; default: assert(0); } Py_RETURN_NONE; } static PyObject * encode_container(CBOREncoderObject *self, EncodeFunction *encoder, PyObject *value) { PyObject *ret = NULL; bool old_string_namespacing = self->string_namespacing; if (self->string_namespacing) { self->string_namespacing = false; if (encode_semantic(self, 256, value) == 0) { Py_INCREF(Py_None); ret = Py_None; } } else { ret = encode_shared(self, encoder, value); } self->string_namespacing = old_string_namespacing; return ret; } static PyObject * encode_shared(CBOREncoderObject *self, EncodeFunction *encoder, PyObject *value) { PyObject *id, *index, *tuple, *ret = NULL; id = PyLong_FromVoidPtr(value); if (id) { tuple = PyDict_GetItem(self->shared, id); if (self->value_sharing) { if (tuple) { if (encode_length(self, 6, 29) == 0) ret = CBOREncoder_encode_int( self, PyTuple_GET_ITEM(tuple, 1)); } else { index = PyLong_FromSsize_t(PyDict_Size(self->shared)); if (index) { tuple = PyTuple_Pack(2, value, index); if (tuple) { if (PyDict_SetItem(self->shared, id, tuple) == 0) if (encode_length(self, 6, 28) == 0) ret = encoder(self, value); Py_DECREF(tuple); } Py_DECREF(index); } } } else { if (tuple) { PyErr_SetString( _CBOR2_CBOREncodeValueError, "cyclic data structure detected but value sharing is " "disabled"); } else { tuple = PyTuple_Pack(2, value, Py_None); if (tuple) { if (PyDict_SetItem(self->shared, id, tuple) == 0) { ret = encoder(self, value); PyDict_DelItem(self->shared, id); } Py_DECREF(tuple); } } } Py_DECREF(id); } return ret; } static PyObject * shared_callback(CBOREncoderObject *self, PyObject *value) { if (PyCallable_Check(self->shared_handler)) { return PyObject_CallFunctionObjArgs( self->shared_handler, self, value, NULL); } else { PyErr_Format( _CBOR2_CBOREncodeTypeError, "non-callable passed as shared encoding method"); return NULL; } } // CBOREncoder.encode_shared(self, encode_method, value) static PyObject * CBOREncoder_encode_shared(CBOREncoderObject *self, PyObject *args) { // semantic type 28 or 29 PyObject *method, *value, *tmp, *ret = NULL; if (PyArg_ParseTuple(args, "OO", &method, &value)) { Py_INCREF(method); tmp = self->shared_handler; self->shared_handler = method; ret = encode_shared(self, &shared_callback, value); self->shared_handler = tmp; Py_DECREF(method); } return ret; } // CBOREncoder.encode_stringref(self, value) static PyObject * CBOREncoder_encode_stringref(CBOREncoderObject *self, PyObject *value) { // semantic type 25 PyObject *ret = NULL; switch (stringref(self, value)) { case 1: Py_RETURN_NONE; case 0: ret = CBOREncoder_encode(self, value); } return ret; } // CBOREncoder.encode_rational(self, value) static PyObject * CBOREncoder_encode_rational(CBOREncoderObject *self, PyObject *value) { // semantic type 30 PyObject *tuple, *num, *den, *ret = NULL; bool sharing; num = PyObject_GetAttr(value, _CBOR2_str_numerator); if (num) { den = PyObject_GetAttr(value, _CBOR2_str_denominator); if (den) { tuple = PyTuple_Pack(2, num, den); if (tuple) { sharing = self->value_sharing; self->value_sharing = false; if (encode_semantic(self, 30, tuple) == 0) { Py_INCREF(Py_None); ret = Py_None; } self->value_sharing = sharing; Py_DECREF(tuple); } Py_DECREF(den); } Py_DECREF(num); } return ret; } // CBOREncoder.encode_regexp(self, value) static PyObject * CBOREncoder_encode_regexp(CBOREncoderObject *self, PyObject *value) { // semantic type 35 PyObject *pattern, *ret = NULL; pattern = PyObject_GetAttr(value, _CBOR2_str_pattern); if (pattern) { if (encode_semantic(self, 35, pattern) == 0) { Py_INCREF(Py_None); ret = Py_None; } Py_DECREF(pattern); } return ret; } // CBOREncoder.encode_mime(self, value) static PyObject * CBOREncoder_encode_mime(CBOREncoderObject *self, PyObject *value) { // semantic type 36 PyObject *buf, *ret = NULL; buf = PyObject_CallMethodObjArgs(value, _CBOR2_str_as_string, NULL); if (buf) { if (encode_semantic(self, 36, buf) == 0) { Py_INCREF(Py_None); ret = Py_None; } Py_DECREF(buf); } return ret; } // CBOREncoder.encode_uuid(self, value) static PyObject * CBOREncoder_encode_uuid(CBOREncoderObject *self, PyObject *value) { // semantic type 37 PyObject *bytes, *ret = NULL; bytes = PyObject_GetAttr(value, _CBOR2_str_bytes); if (bytes) { if (encode_semantic(self, 37, bytes) == 0) { Py_INCREF(Py_None); ret = Py_None; } Py_DECREF(bytes); } return ret; } // CBOREncoder.encode_stringref_namespace(self, value) static PyObject * CBOREncoder_encode_stringref_ns(CBOREncoderObject *self, PyObject *value) { // semantic type 256 PyObject *ret = NULL; bool old_string_namespacing = self->string_namespacing; self->string_namespacing = false; if (encode_semantic(self, 256, value) == 0) { Py_INCREF(Py_None); ret = Py_None; } self->string_namespacing = old_string_namespacing; return ret; } static PyObject * encode_set(CBOREncoderObject *self, PyObject *value) { Py_ssize_t length; PyObject *iter, *item, *ret = NULL; length = PySet_Size(value); if (length != -1) { iter = PyObject_GetIter(value); if (iter) { if (encode_length(self, 6, 258) == 0) { if (encode_length(self, 4, length) == 0) { while ((item = PyIter_Next(iter))) { ret = CBOREncoder_encode(self, item); Py_DECREF(item); if (ret) Py_DECREF(ret); else goto error; } if (!PyErr_Occurred()) { Py_INCREF(Py_None); ret = Py_None; } } } error: Py_DECREF(iter); } } return ret; } // CBOREncoder.encode_set(self, value) static PyObject * CBOREncoder_encode_set(CBOREncoderObject *self, PyObject *value) { // semantic type 258 return encode_container(self, &encode_set, value); } static PyObject * encode_ipaddress(CBOREncoderObject *self, PyObject *value) { PyObject *bytes, *ret = NULL; bytes = PyObject_GetAttr(value, _CBOR2_str_packed); if (bytes) { if (encode_semantic(self, 260, bytes) == 0) { Py_INCREF(Py_None); ret = Py_None; } Py_DECREF(bytes); } return ret; } // CBOREncoder.encode_ipaddress(self, value) static PyObject * CBOREncoder_encode_ipaddress(CBOREncoderObject *self, PyObject *value) { // semantic type 260 return encode_container(self, &encode_ipaddress, value); } static PyObject * encode_ipnetwork(CBOREncoderObject *self, PyObject *value) { PyObject *map, *addr, *bytes, *prefixlen, *ret = NULL; addr = PyObject_GetAttr(value, _CBOR2_str_network_address); if (addr) { bytes = PyObject_GetAttr(addr, _CBOR2_str_packed); if (bytes) { prefixlen = PyObject_GetAttr(value, _CBOR2_str_prefixlen); if (prefixlen) { map = PyDict_New(); if (map) { if (PyDict_SetItem(map, bytes, prefixlen) == 0) { if (encode_semantic(self, 261, map) == 0) { Py_INCREF(Py_None); ret = Py_None; } } Py_DECREF(map); } Py_DECREF(prefixlen); } Py_DECREF(bytes); } Py_DECREF(addr); } return ret; } // CBOREncoder.encode_ipnetwork(self, value) static PyObject * CBOREncoder_encode_ipnetwork(CBOREncoderObject *self, PyObject *value) { // semantic type 261 return encode_container(self, &encode_ipnetwork, value); } // Special encoders ////////////////////////////////////////////////////////// // CBOREncoder.encode_float(self, value) static PyObject * CBOREncoder_encode_float(CBOREncoderObject *self, PyObject *value) { // major type 7 union { double f; uint64_t i; char buf[sizeof(double)]; } u; u.f = PyFloat_AS_DOUBLE(value); if (u.f == -1.0 && PyErr_Occurred()) return NULL; switch (fpclassify(u.f)) { case FP_NAN: if (fp_write(self, "\xF9\x7E\x00", 3) == -1) return NULL; break; case FP_INFINITE: if (u.f > 0) { if (fp_write(self, "\xF9\x7C\x00", 3) == -1) return NULL; } else { if (fp_write(self, "\xF9\xFC\x00", 3) == -1) return NULL; } break; default: if (fp_write(self, "\xFB", 1) == -1) return NULL; u.i = htobe64(u.i); if (fp_write(self, u.buf, sizeof(double)) == -1) return NULL; break; } Py_RETURN_NONE; } // CBOREncoder.encode_boolean(self, value) static PyObject * CBOREncoder_encode_boolean(CBOREncoderObject *self, PyObject *value) { // special type 20 or 21 if (PyObject_IsTrue(value)) { if (fp_write(self, "\xF5", 1) == -1) return NULL; } else { if (fp_write(self, "\xF4", 1) == -1) return NULL; } Py_RETURN_NONE; } // CBOREncoder.encode_none(self, value) static PyObject * CBOREncoder_encode_none(CBOREncoderObject *self, PyObject *value) { // special type 22 if (fp_write(self, "\xF6", 1) == -1) return NULL; Py_RETURN_NONE; } // CBOREncoder.encode_undefined(self, value) static PyObject * CBOREncoder_encode_undefined(CBOREncoderObject *self, PyObject *value) { // special type 23 if (fp_write(self, "\xF7", 1) == -1) return NULL; Py_RETURN_NONE; } // CBOREncoder.encode_simple_value(self, (value,)) static PyObject * CBOREncoder_encode_simple_value(CBOREncoderObject *self, PyObject *args) { // special types 0..255 uint8_t value; if (!PyArg_ParseTuple(args, "B", &value)) return NULL; if (value < 20) { value |= 0xE0; if (fp_write(self, (char *)&value, 1) == -1) return NULL; } else { if (fp_write(self, "\xF8", 1) == -1) return NULL; if (fp_write(self, (char *)&value, 1) == -1) return NULL; } Py_RETURN_NONE; } // Canonical encoding methods //////////////////////////////////////////////// // CBOREncoder.encode_minimal_float(self, value) static PyObject * CBOREncoder_encode_minimal_float(CBOREncoderObject *self, PyObject *value) { union { double f; uint64_t i; char buf[sizeof(double)]; } u_double; union { float f; uint32_t i; char buf[sizeof(float)]; } u_single; union { uint16_t i; char buf[sizeof(uint16_t)]; } u_half; u_double.f = PyFloat_AS_DOUBLE(value); if (u_double.f == -1.0 && PyErr_Occurred()) return NULL; switch (fpclassify(u_double.f)) { case FP_NAN: if (fp_write(self, "\xF9\x7E\x00", 3) == -1) return NULL; break; case FP_INFINITE: if (u_double.f > 0) { if (fp_write(self, "\xF9\x7C\x00", 3) == -1) return NULL; } else { if (fp_write(self, "\xF9\xFC\x00", 3) == -1) return NULL; } break; default: u_single.f = (float) u_double.f; if (u_single.f == u_double.f) { u_half.i = pack_float16(u_single.f); if (unpack_float16(u_half.i) == u_single.f) { if (fp_write(self, "\xF9", 1) == -1) return NULL; if (fp_write(self, u_half.buf, sizeof(uint16_t)) == -1) return NULL; } else { if (fp_write(self, "\xFA", 1) == -1) return NULL; u_single.i = htobe32(u_single.i); if (fp_write(self, u_single.buf, sizeof(float)) == -1) return NULL; } } else { if (fp_write(self, "\xFB", 1) == -1) return NULL; u_double.i = htobe64(u_double.i); if (fp_write(self, u_double.buf, sizeof(double)) == -1) return NULL; } break; } Py_RETURN_NONE; } static PyObject * encode_canonical_map_list(CBOREncoderObject *self, PyObject *list) { PyObject *bytes, *ret; Py_ssize_t index; if (PyList_Sort(list) == -1) return NULL; if (encode_length(self, 5, PyList_GET_SIZE(list)) == -1) return NULL; for (index = 0; index < PyList_GET_SIZE(list); ++index) { // If we are encoding string references, the order of the keys // needs to match the order we encode. if (self->string_referencing) { ret = CBOREncoder_encode(self, PyTuple_GET_ITEM(PyList_GET_ITEM(list, index), 2)); if (ret) Py_DECREF(ret); else return NULL; } else { // We already have the encoded form of the key so just write it out bytes = PyTuple_GET_ITEM(PyList_GET_ITEM(list, index), 1); if (fp_write(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)) == -1) return NULL; } ret = CBOREncoder_encode(self, PyTuple_GET_ITEM(PyList_GET_ITEM(list, index), 3)); if (ret) Py_DECREF(ret); else return NULL; } Py_RETURN_NONE; } static PyObject * dict_to_canonical_list(CBOREncoderObject *self, PyObject *value) { PyObject *bytes, *length, *key, *val, *tuple, *ret, *list; Py_ssize_t index, pos; ret = list = PyList_New(PyDict_Size(value)); if (list) { pos = 0; index = 0; while (ret && PyDict_Next(value, &pos, &key, &val)) { Py_INCREF(key); bytes = CBOREncoder_encode_to_bytes(self, key); Py_DECREF(key); if (bytes) { length = PyLong_FromSsize_t(PyBytes_GET_SIZE(bytes)); if (length) { tuple = PyTuple_Pack(4, length, bytes, key, val); if (tuple) PyList_SET_ITEM(list, index, tuple); // steals ref else ret = NULL; index++; Py_DECREF(length); } else ret = NULL; Py_DECREF(bytes); } else ret = NULL; } if (!ret) Py_DECREF(list); } return ret; } static PyObject * mapping_to_canonical_list(CBOREncoderObject *self, PyObject *value) { PyObject **items, *map_items, *fast, *bytes, *length, *tuple, *ret, *list; Py_ssize_t fast_len, index; ret = list = PyList_New(PyMapping_Size(value)); if (list) { map_items = PyMapping_Items(value); if (map_items) { fast = PySequence_Fast(map_items, "internal error"); if (fast) { index = 0; fast_len = PySequence_Fast_GET_SIZE(fast); items = PySequence_Fast_ITEMS(fast); while (ret && fast_len) { bytes = CBOREncoder_encode_to_bytes(self, PyTuple_GET_ITEM(*items, 0)); if (bytes) { length = PyLong_FromSsize_t(PyBytes_GET_SIZE(bytes)); if (length) { tuple = PyTuple_Pack(4, length, bytes, PyTuple_GET_ITEM(*items, 0), PyTuple_GET_ITEM(*items, 1)); if (tuple) PyList_SET_ITEM(list, index, tuple); // steals ref else ret = NULL; Py_DECREF(length); } else ret = NULL; Py_DECREF(bytes); } else ret = NULL; items++; index++; fast_len--; } Py_DECREF(fast); } else ret = NULL; Py_DECREF(map_items); } else ret = NULL; if (!ret) Py_DECREF(list); } return ret; } static PyObject * encode_canonical_map(CBOREncoderObject *self, PyObject *value) { PyObject *list, *ret = NULL; bool string_referencing_old = self->string_referencing; // Don't generate string references when sorting keys self->string_referencing = false; if (PyDict_Check(value)) list = dict_to_canonical_list(self, value); else list = mapping_to_canonical_list(self, value); self->string_referencing = string_referencing_old; if (list) { ret = encode_canonical_map_list(self, list); Py_DECREF(list); } return ret; } static PyObject * CBOREncoder_encode_canonical_map(CBOREncoderObject *self, PyObject *value) { return encode_container(self, &encode_canonical_map, value); } static PyObject * encode_canonical_set_list(CBOREncoderObject *self, PyObject *list) { PyObject *bytes; Py_ssize_t index; if (PyList_Sort(list) == -1) return NULL; if (encode_length(self, 6, 258) == -1) return NULL; if (encode_length(self, 4, PyList_GET_SIZE(list)) == -1) return NULL; for (index = 0; index < PyList_GET_SIZE(list); ++index) { // We already have the encoded form, so just write it out bytes = PyTuple_GET_ITEM(PyList_GET_ITEM(list, index), 1); if (fp_write(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)) == -1) return NULL; } Py_RETURN_NONE; } static PyObject * set_to_canonical_list(CBOREncoderObject *self, PyObject *value) { PyObject *iter, *bytes, *length, *item, *tuple, *list, *ret; Py_ssize_t index; ret = list = PyList_New(PySet_GET_SIZE(value)); if (list) { iter = PyObject_GetIter(value); if (iter) { index = 0; while (ret && (item = PyIter_Next(iter))) { bytes = CBOREncoder_encode_to_bytes(self, item); if (bytes) { length = PyLong_FromSsize_t(PyBytes_GET_SIZE(bytes)); if (length) { tuple = PyTuple_Pack(3, length, bytes, item); if (tuple) PyList_SET_ITEM(list, index, tuple); // steals ref else ret = NULL; index++; Py_DECREF(length); } else ret = NULL; Py_DECREF(bytes); } else ret = NULL; Py_DECREF(item); } Py_DECREF(iter); } if (!ret) Py_DECREF(list); } return ret; } static PyObject * encode_canonical_set(CBOREncoderObject *self, PyObject *value) { PyObject *list, *ret = NULL; list = set_to_canonical_list(self, value); if (list) { ret = encode_canonical_set_list(self, list); Py_DECREF(list); } return ret; } static PyObject * CBOREncoder_encode_canonical_set(CBOREncoderObject *self, PyObject *value) { return encode_container(self, &encode_canonical_set, value); } // Main entry points ///////////////////////////////////////////////////////// static inline PyObject * encode(CBOREncoderObject *self, PyObject *value) { PyObject *encoder, *ret = NULL; switch (self->enc_style) { case 1: // canonical encoders if (PyFloat_CheckExact(value)) return CBOREncoder_encode_minimal_float(self, value); else if (PyDict_CheckExact(value)) return CBOREncoder_encode_canonical_map(self, value); else if (PyAnySet_CheckExact(value)) return CBOREncoder_encode_canonical_set(self, value); // fall-thru case 0: // regular encoders if (PyBytes_CheckExact(value)) return CBOREncoder_encode_bytestring(self, value); else if (PyByteArray_CheckExact(value)) return CBOREncoder_encode_bytearray(self, value); else if (PyUnicode_CheckExact(value)) return CBOREncoder_encode_string(self, value); else if (PyLong_CheckExact(value)) return CBOREncoder_encode_int(self, value); else if (PyFloat_CheckExact(value)) return CBOREncoder_encode_float(self, value); else if (PyBool_Check(value)) return CBOREncoder_encode_boolean(self, value); else if (value == Py_None) return CBOREncoder_encode_none(self, value); else if (value == undefined) return CBOREncoder_encode_undefined(self, value); else if (PyTuple_CheckExact(value)) return CBOREncoder_encode_array(self, value); else if (PyList_CheckExact(value)) return CBOREncoder_encode_array(self, value); else if (PyDict_CheckExact(value)) return CBOREncoder_encode_map(self, value); else if (PyDateTime_CheckExact(value)) return CBOREncoder_encode_datetime(self, value); else if (PyAnySet_CheckExact(value)) return CBOREncoder_encode_set(self, value); // fall-thru default: // lookup type (or subclass) in self->encoders encoder = CBOREncoder_find_encoder(self, (PyObject *)Py_TYPE(value)); if (encoder) { if (encoder != Py_None) ret = PyObject_CallFunctionObjArgs( encoder, self, value, NULL); else if (self->default_handler != Py_None) ret = PyObject_CallFunctionObjArgs( self->default_handler, self, value, NULL); else PyErr_Format( _CBOR2_CBOREncodeTypeError, "cannot serialize type %R", (PyObject *)Py_TYPE(value)); Py_DECREF(encoder); } } return ret; } // CBOREncoder.encode(self, value) PyObject * CBOREncoder_encode(CBOREncoderObject *self, PyObject *value) { PyObject *ret; // TODO reset shared dict? if (Py_EnterRecursiveCall(" in CBOREncoder.encode")) return NULL; ret = encode(self, value); Py_LeaveRecursiveCall(); return ret; } static PyObject * CBOREncoder_encode_to_bytes(CBOREncoderObject *self, PyObject *value) { PyObject *save_write, *buf, *ret = NULL; if (!_CBOR2_BytesIO && _CBOR2_init_BytesIO() == -1) return NULL; save_write = self->write; buf = PyObject_CallFunctionObjArgs(_CBOR2_BytesIO, NULL); if (buf) { self->write = PyObject_GetAttr(buf, _CBOR2_str_write); if (self->write) { ret = CBOREncoder_encode(self, value); if (ret) { assert(ret == Py_None); Py_DECREF(ret); ret = PyObject_CallMethodObjArgs(buf, _CBOR2_str_getvalue, NULL); } Py_DECREF(self->write); } Py_DECREF(buf); } self->write = save_write; return ret; } // Encoder class definition ////////////////////////////////////////////////// static PyMemberDef CBOREncoder_members[] = { {"_encoders", T_OBJECT_EX, offsetof(CBOREncoderObject, encoders), READONLY, "the ordered dict mapping types to encoder functions"}, {"enc_style", T_UBYTE, offsetof(CBOREncoderObject, enc_style), 0, "the optimized encoder lookup to use (0=regular, 1=canonical, " "anything else is custom)"}, {"datetime_as_timestamp", T_BOOL, offsetof(CBOREncoderObject, timestamp_format), 0, "the sub-type to use when encoding datetime objects"}, {"value_sharing", T_BOOL, offsetof(CBOREncoderObject, value_sharing), 0, "if True, then efficiently encode recursive structures"}, {NULL} }; static PyGetSetDef CBOREncoder_getsetters[] = { {"fp", (getter) _CBOREncoder_get_fp, (setter) _CBOREncoder_set_fp, "output file-like object", NULL}, {"default", (getter) _CBOREncoder_get_default, (setter) _CBOREncoder_set_default, "default handler called when encoding unknown objects", NULL}, {"timezone", (getter) _CBOREncoder_get_timezone, (setter) _CBOREncoder_set_timezone, "the timezone to use when encoding naive datetime objects", NULL}, {"canonical", (getter) _CBOREncoder_get_canonical, NULL, "if True, then CBOR canonical encoding will be generated", NULL}, {NULL} }; static PyMethodDef CBOREncoder_methods[] = { {"_find_encoder", (PyCFunction) CBOREncoder_find_encoder, METH_O, "find an encoding function for the specified type"}, {"write", (PyCFunction) CBOREncoder_write, METH_O, "write the specified data to the output"}, // Standard encoding methods {"encode", (PyCFunction) CBOREncoder_encode, METH_O, "encode the specified *value* to the output"}, {"encode_to_bytes", (PyCFunction) CBOREncoder_encode_to_bytes, METH_O, "encode the specified *value* to a bytestring"}, {"encode_length", (PyCFunction) CBOREncoder_encode_length, METH_VARARGS, "encode the specified *major_tag* with the specified *length* to " "the output"}, {"encode_int", (PyCFunction) CBOREncoder_encode_int, METH_O, "encode the specified integer *value* to the output"}, {"encode_float", (PyCFunction) CBOREncoder_encode_float, METH_O, "encode the specified floating-point *value* to the output"}, {"encode_boolean", (PyCFunction) CBOREncoder_encode_boolean, METH_O, "encode the specified boolean *value* to the output"}, {"encode_none", (PyCFunction) CBOREncoder_encode_none, METH_O, "encode the None value to the output"}, {"encode_undefined", (PyCFunction) CBOREncoder_encode_undefined, METH_O, "encode the undefined value to the output"}, {"encode_datetime", (PyCFunction) CBOREncoder_encode_datetime, METH_O, "encode the datetime *value* to the output"}, {"encode_date", (PyCFunction) CBOREncoder_encode_date, METH_O, "encode the date *value* to the output"}, {"encode_bytestring", (PyCFunction) CBOREncoder_encode_bytestring, METH_O, "encode the specified bytes *value* to the output"}, {"encode_bytearray", (PyCFunction) CBOREncoder_encode_bytearray, METH_O, "encode the specified bytearray *value* to the output"}, {"encode_string", (PyCFunction) CBOREncoder_encode_string, METH_O, "encode the specified string *value* to the output"}, {"encode_array", (PyCFunction) CBOREncoder_encode_array, METH_O, "encode the specified sequence *value* to the output"}, {"encode_map", (PyCFunction) CBOREncoder_encode_map, METH_O, "encode the specified mapping *value* to the output"}, {"encode_semantic", (PyCFunction) CBOREncoder_encode_semantic, METH_O, "encode the specified CBORTag to the output"}, {"encode_simple_value", (PyCFunction) CBOREncoder_encode_simple_value, METH_O, "encode the specified CBORSimpleValue to the output"}, {"encode_rational", (PyCFunction) CBOREncoder_encode_rational, METH_O, "encode the specified fraction to the output"}, {"encode_decimal", (PyCFunction) CBOREncoder_encode_decimal, METH_O, "encode the specified Decimal to the output"}, {"encode_regexp", (PyCFunction) CBOREncoder_encode_regexp, METH_O, "encode the specified regular expression object to the output"}, {"encode_mime", (PyCFunction) CBOREncoder_encode_mime, METH_O, "encode the specified MIME message object to the output"}, {"encode_uuid", (PyCFunction) CBOREncoder_encode_uuid, METH_O, "encode the specified UUID to the output"}, {"encode_set", (PyCFunction) CBOREncoder_encode_set, METH_O, "encode the specified set to the output"}, {"encode_ipaddress", (PyCFunction) CBOREncoder_encode_ipaddress, METH_O, "encode the specified IPv4 or IPv6 address to the output"}, {"encode_ipnetwork", (PyCFunction) CBOREncoder_encode_ipnetwork, METH_O, "encode the specified IPv4 or IPv6 network prefix to the output"}, {"encode_shared", (PyCFunction) CBOREncoder_encode_shared, METH_VARARGS, "encode the specified CBORTag to the output"}, {"encode_stringref", (PyCFunction) CBOREncoder_encode_stringref, METH_O, "encode the string potentially referencing an existing occurrence"}, {"encode_stringref_namespace", (PyCFunction) CBOREncoder_encode_stringref_ns, METH_O, "encode all string and bytestring descendants with stringrefs"}, // Canonical encoding methods {"encode_minimal_float", (PyCFunction) CBOREncoder_encode_minimal_float, METH_O, "encode the specified float to a minimal representation in the output"}, {"encode_canonical_map", (PyCFunction) CBOREncoder_encode_canonical_map, METH_O, "encode the specified map to a canonical representation in the output"}, {"encode_canonical_set", (PyCFunction) CBOREncoder_encode_canonical_set, METH_O, "encode the specified set to a canonical representation in the output"}, {NULL} }; PyDoc_STRVAR(CBOREncoder__doc__, "The CBOREncoder class implements a fully featured `CBOR`_ encoder with\n" "several extensions for handling shared references, big integers,\n" "rational numbers and so on. Typically the class is not used directly,\n" "but the :func:`cbor2.dump` and :func:`cbor2.dumps` functions are called\n" "to indirectly construct and use the class.\n" "\n" "When the class is constructed manually, the main entry points are\n" ":meth:`encode` and :meth:`encode_to_bytes`.\n" "\n" ":param bool datetime_as_timestamp:\n" " set to ``True`` to serialize datetimes as UNIX timestamps (this\n" " makes datetimes more concise on the wire, but loses the timezone\n" " information)\n" ":param datetime.tzinfo timezone:\n" " the default timezone to use for serializing naive datetimes; if\n" " this is not specified naive datetimes will throw a :exc:`ValueError`\n" " when encoding is attempted\n" ":param bool value_sharing:\n" " set to ``True`` to allow more efficient serializing of repeated\n" " values and, more importantly, cyclic data structures, at the cost\n" " of extra line overhead\n" ":param default:\n" " a callable that is called by the encoder with two arguments (the\n" " encoder instance and the value being encoded) when no suitable\n" " encoder has been found, and should use the methods on the encoder\n" " to encode any objects it wants to add to the data stream\n" ":param int canonical:\n" " when True, use \"canonical\" CBOR representation; this typically\n" " involves sorting maps, sets, etc. into a pre-determined order ensuring\n" " that serializations are comparable without decoding\n" "\n" ".. _CBOR: https://cbor.io/\n" ); PyTypeObject CBOREncoderType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_cbor2.CBOREncoder", .tp_doc = CBOREncoder__doc__, .tp_basicsize = sizeof(CBOREncoderObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, .tp_new = CBOREncoder_new, .tp_init = (initproc) CBOREncoder_init, .tp_dealloc = (destructor) CBOREncoder_dealloc, .tp_traverse = (traverseproc) CBOREncoder_traverse, .tp_clear = (inquiry) CBOREncoder_clear, .tp_members = CBOREncoder_members, .tp_getset = CBOREncoder_getsetters, .tp_methods = CBOREncoder_methods, }; cbor2-5.4.2/source/encoder.h000066400000000000000000000016511413200260400156140ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include // Constants for decimal_classify #define DC_NORMAL 0 #define DC_INFINITE 1 #define DC_NAN 2 #define DC_ERROR -1 typedef struct { PyObject_HEAD PyObject *write; // cached write() method of fp PyObject *encoders; PyObject *default_handler; PyObject *shared; PyObject *string_references; PyObject *tz; // renamed from timezone to avoid Python issue #24643 PyObject *shared_handler; uint8_t enc_style; // 0=regular, 1=canonical, 2=custom bool timestamp_format; bool value_sharing; bool string_referencing; bool string_namespacing; } CBOREncoderObject; extern PyTypeObject CBOREncoderType; PyObject * CBOREncoder_new(PyTypeObject *, PyObject *, PyObject *); int CBOREncoder_init(CBOREncoderObject *, PyObject *, PyObject *); PyObject * CBOREncoder_encode(CBOREncoderObject *, PyObject *); cbor2-5.4.2/source/halffloat.c000066400000000000000000001123311413200260400161260ustar00rootroot00000000000000#include #include #include #include #if __FreeBSD__ #include #elif __APPLE__ #include #elif _GNU_SOURCE #include #endif #include "halffloat.h" #if __APPLE__ #define be16toh(x) OSSwapBigToHostInt16(x) #define htobe16(x) OSSwapHostToBigInt16(x) #elif _WIN32 // All windows platforms are (currently) little-endian so byteswap is required #define be16toh(x) _byteswap_ushort(x) #define htobe16(x) _byteswap_ushort(x) #endif // Based upon ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf ("Fast // Half Float Conversions") referenced by the Wikipedia article on // half-precision floating point: // // https://en.wikipedia.org/wiki/Half-precision_floating-point_format static const uint32_t sigtable[] = { 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34a00000, 0x34c00000, 0x34e00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35a00000, 0x35a80000, 0x35b00000, 0x35b80000, 0x35c00000, 0x35c80000, 0x35d00000, 0x35d80000, 0x35e00000, 0x35e80000, 0x35f00000, 0x35f80000, 0x36000000, 0x36040000, 0x36080000, 0x360c0000, 0x36100000, 0x36140000, 0x36180000, 0x361c0000, 0x36200000, 0x36240000, 0x36280000, 0x362c0000, 0x36300000, 0x36340000, 0x36380000, 0x363c0000, 0x36400000, 0x36440000, 0x36480000, 0x364c0000, 0x36500000, 0x36540000, 0x36580000, 0x365c0000, 0x36600000, 0x36640000, 0x36680000, 0x366c0000, 0x36700000, 0x36740000, 0x36780000, 0x367c0000, 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368a0000, 0x368c0000, 0x368e0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369a0000, 0x369c0000, 0x369e0000, 0x36a00000, 0x36a20000, 0x36a40000, 0x36a60000, 0x36a80000, 0x36aa0000, 0x36ac0000, 0x36ae0000, 0x36b00000, 0x36b20000, 0x36b40000, 0x36b60000, 0x36b80000, 0x36ba0000, 0x36bc0000, 0x36be0000, 0x36c00000, 0x36c20000, 0x36c40000, 0x36c60000, 0x36c80000, 0x36ca0000, 0x36cc0000, 0x36ce0000, 0x36d00000, 0x36d20000, 0x36d40000, 0x36d60000, 0x36d80000, 0x36da0000, 0x36dc0000, 0x36de0000, 0x36e00000, 0x36e20000, 0x36e40000, 0x36e60000, 0x36e80000, 0x36ea0000, 0x36ec0000, 0x36ee0000, 0x36f00000, 0x36f20000, 0x36f40000, 0x36f60000, 0x36f80000, 0x36fa0000, 0x36fc0000, 0x36fe0000, 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370a0000, 0x370b0000, 0x370c0000, 0x370d0000, 0x370e0000, 0x370f0000, 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371a0000, 0x371b0000, 0x371c0000, 0x371d0000, 0x371e0000, 0x371f0000, 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372a0000, 0x372b0000, 0x372c0000, 0x372d0000, 0x372e0000, 0x372f0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373a0000, 0x373b0000, 0x373c0000, 0x373d0000, 0x373e0000, 0x373f0000, 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374a0000, 0x374b0000, 0x374c0000, 0x374d0000, 0x374e0000, 0x374f0000, 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375a0000, 0x375b0000, 0x375c0000, 0x375d0000, 0x375e0000, 0x375f0000, 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376a0000, 0x376b0000, 0x376c0000, 0x376d0000, 0x376e0000, 0x376f0000, 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377a0000, 0x377b0000, 0x377c0000, 0x377d0000, 0x377e0000, 0x377f0000, 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378a0000, 0x378a8000, 0x378b0000, 0x378b8000, 0x378c0000, 0x378c8000, 0x378d0000, 0x378d8000, 0x378e0000, 0x378e8000, 0x378f0000, 0x378f8000, 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379a0000, 0x379a8000, 0x379b0000, 0x379b8000, 0x379c0000, 0x379c8000, 0x379d0000, 0x379d8000, 0x379e0000, 0x379e8000, 0x379f0000, 0x379f8000, 0x37a00000, 0x37a08000, 0x37a10000, 0x37a18000, 0x37a20000, 0x37a28000, 0x37a30000, 0x37a38000, 0x37a40000, 0x37a48000, 0x37a50000, 0x37a58000, 0x37a60000, 0x37a68000, 0x37a70000, 0x37a78000, 0x37a80000, 0x37a88000, 0x37a90000, 0x37a98000, 0x37aa0000, 0x37aa8000, 0x37ab0000, 0x37ab8000, 0x37ac0000, 0x37ac8000, 0x37ad0000, 0x37ad8000, 0x37ae0000, 0x37ae8000, 0x37af0000, 0x37af8000, 0x37b00000, 0x37b08000, 0x37b10000, 0x37b18000, 0x37b20000, 0x37b28000, 0x37b30000, 0x37b38000, 0x37b40000, 0x37b48000, 0x37b50000, 0x37b58000, 0x37b60000, 0x37b68000, 0x37b70000, 0x37b78000, 0x37b80000, 0x37b88000, 0x37b90000, 0x37b98000, 0x37ba0000, 0x37ba8000, 0x37bb0000, 0x37bb8000, 0x37bc0000, 0x37bc8000, 0x37bd0000, 0x37bd8000, 0x37be0000, 0x37be8000, 0x37bf0000, 0x37bf8000, 0x37c00000, 0x37c08000, 0x37c10000, 0x37c18000, 0x37c20000, 0x37c28000, 0x37c30000, 0x37c38000, 0x37c40000, 0x37c48000, 0x37c50000, 0x37c58000, 0x37c60000, 0x37c68000, 0x37c70000, 0x37c78000, 0x37c80000, 0x37c88000, 0x37c90000, 0x37c98000, 0x37ca0000, 0x37ca8000, 0x37cb0000, 0x37cb8000, 0x37cc0000, 0x37cc8000, 0x37cd0000, 0x37cd8000, 0x37ce0000, 0x37ce8000, 0x37cf0000, 0x37cf8000, 0x37d00000, 0x37d08000, 0x37d10000, 0x37d18000, 0x37d20000, 0x37d28000, 0x37d30000, 0x37d38000, 0x37d40000, 0x37d48000, 0x37d50000, 0x37d58000, 0x37d60000, 0x37d68000, 0x37d70000, 0x37d78000, 0x37d80000, 0x37d88000, 0x37d90000, 0x37d98000, 0x37da0000, 0x37da8000, 0x37db0000, 0x37db8000, 0x37dc0000, 0x37dc8000, 0x37dd0000, 0x37dd8000, 0x37de0000, 0x37de8000, 0x37df0000, 0x37df8000, 0x37e00000, 0x37e08000, 0x37e10000, 0x37e18000, 0x37e20000, 0x37e28000, 0x37e30000, 0x37e38000, 0x37e40000, 0x37e48000, 0x37e50000, 0x37e58000, 0x37e60000, 0x37e68000, 0x37e70000, 0x37e78000, 0x37e80000, 0x37e88000, 0x37e90000, 0x37e98000, 0x37ea0000, 0x37ea8000, 0x37eb0000, 0x37eb8000, 0x37ec0000, 0x37ec8000, 0x37ed0000, 0x37ed8000, 0x37ee0000, 0x37ee8000, 0x37ef0000, 0x37ef8000, 0x37f00000, 0x37f08000, 0x37f10000, 0x37f18000, 0x37f20000, 0x37f28000, 0x37f30000, 0x37f38000, 0x37f40000, 0x37f48000, 0x37f50000, 0x37f58000, 0x37f60000, 0x37f68000, 0x37f70000, 0x37f78000, 0x37f80000, 0x37f88000, 0x37f90000, 0x37f98000, 0x37fa0000, 0x37fa8000, 0x37fb0000, 0x37fb8000, 0x37fc0000, 0x37fc8000, 0x37fd0000, 0x37fd8000, 0x37fe0000, 0x37fe8000, 0x37ff0000, 0x37ff8000, 0x38000000, 0x38004000, 0x38008000, 0x3800c000, 0x38010000, 0x38014000, 0x38018000, 0x3801c000, 0x38020000, 0x38024000, 0x38028000, 0x3802c000, 0x38030000, 0x38034000, 0x38038000, 0x3803c000, 0x38040000, 0x38044000, 0x38048000, 0x3804c000, 0x38050000, 0x38054000, 0x38058000, 0x3805c000, 0x38060000, 0x38064000, 0x38068000, 0x3806c000, 0x38070000, 0x38074000, 0x38078000, 0x3807c000, 0x38080000, 0x38084000, 0x38088000, 0x3808c000, 0x38090000, 0x38094000, 0x38098000, 0x3809c000, 0x380a0000, 0x380a4000, 0x380a8000, 0x380ac000, 0x380b0000, 0x380b4000, 0x380b8000, 0x380bc000, 0x380c0000, 0x380c4000, 0x380c8000, 0x380cc000, 0x380d0000, 0x380d4000, 0x380d8000, 0x380dc000, 0x380e0000, 0x380e4000, 0x380e8000, 0x380ec000, 0x380f0000, 0x380f4000, 0x380f8000, 0x380fc000, 0x38100000, 0x38104000, 0x38108000, 0x3810c000, 0x38110000, 0x38114000, 0x38118000, 0x3811c000, 0x38120000, 0x38124000, 0x38128000, 0x3812c000, 0x38130000, 0x38134000, 0x38138000, 0x3813c000, 0x38140000, 0x38144000, 0x38148000, 0x3814c000, 0x38150000, 0x38154000, 0x38158000, 0x3815c000, 0x38160000, 0x38164000, 0x38168000, 0x3816c000, 0x38170000, 0x38174000, 0x38178000, 0x3817c000, 0x38180000, 0x38184000, 0x38188000, 0x3818c000, 0x38190000, 0x38194000, 0x38198000, 0x3819c000, 0x381a0000, 0x381a4000, 0x381a8000, 0x381ac000, 0x381b0000, 0x381b4000, 0x381b8000, 0x381bc000, 0x381c0000, 0x381c4000, 0x381c8000, 0x381cc000, 0x381d0000, 0x381d4000, 0x381d8000, 0x381dc000, 0x381e0000, 0x381e4000, 0x381e8000, 0x381ec000, 0x381f0000, 0x381f4000, 0x381f8000, 0x381fc000, 0x38200000, 0x38204000, 0x38208000, 0x3820c000, 0x38210000, 0x38214000, 0x38218000, 0x3821c000, 0x38220000, 0x38224000, 0x38228000, 0x3822c000, 0x38230000, 0x38234000, 0x38238000, 0x3823c000, 0x38240000, 0x38244000, 0x38248000, 0x3824c000, 0x38250000, 0x38254000, 0x38258000, 0x3825c000, 0x38260000, 0x38264000, 0x38268000, 0x3826c000, 0x38270000, 0x38274000, 0x38278000, 0x3827c000, 0x38280000, 0x38284000, 0x38288000, 0x3828c000, 0x38290000, 0x38294000, 0x38298000, 0x3829c000, 0x382a0000, 0x382a4000, 0x382a8000, 0x382ac000, 0x382b0000, 0x382b4000, 0x382b8000, 0x382bc000, 0x382c0000, 0x382c4000, 0x382c8000, 0x382cc000, 0x382d0000, 0x382d4000, 0x382d8000, 0x382dc000, 0x382e0000, 0x382e4000, 0x382e8000, 0x382ec000, 0x382f0000, 0x382f4000, 0x382f8000, 0x382fc000, 0x38300000, 0x38304000, 0x38308000, 0x3830c000, 0x38310000, 0x38314000, 0x38318000, 0x3831c000, 0x38320000, 0x38324000, 0x38328000, 0x3832c000, 0x38330000, 0x38334000, 0x38338000, 0x3833c000, 0x38340000, 0x38344000, 0x38348000, 0x3834c000, 0x38350000, 0x38354000, 0x38358000, 0x3835c000, 0x38360000, 0x38364000, 0x38368000, 0x3836c000, 0x38370000, 0x38374000, 0x38378000, 0x3837c000, 0x38380000, 0x38384000, 0x38388000, 0x3838c000, 0x38390000, 0x38394000, 0x38398000, 0x3839c000, 0x383a0000, 0x383a4000, 0x383a8000, 0x383ac000, 0x383b0000, 0x383b4000, 0x383b8000, 0x383bc000, 0x383c0000, 0x383c4000, 0x383c8000, 0x383cc000, 0x383d0000, 0x383d4000, 0x383d8000, 0x383dc000, 0x383e0000, 0x383e4000, 0x383e8000, 0x383ec000, 0x383f0000, 0x383f4000, 0x383f8000, 0x383fc000, 0x38400000, 0x38404000, 0x38408000, 0x3840c000, 0x38410000, 0x38414000, 0x38418000, 0x3841c000, 0x38420000, 0x38424000, 0x38428000, 0x3842c000, 0x38430000, 0x38434000, 0x38438000, 0x3843c000, 0x38440000, 0x38444000, 0x38448000, 0x3844c000, 0x38450000, 0x38454000, 0x38458000, 0x3845c000, 0x38460000, 0x38464000, 0x38468000, 0x3846c000, 0x38470000, 0x38474000, 0x38478000, 0x3847c000, 0x38480000, 0x38484000, 0x38488000, 0x3848c000, 0x38490000, 0x38494000, 0x38498000, 0x3849c000, 0x384a0000, 0x384a4000, 0x384a8000, 0x384ac000, 0x384b0000, 0x384b4000, 0x384b8000, 0x384bc000, 0x384c0000, 0x384c4000, 0x384c8000, 0x384cc000, 0x384d0000, 0x384d4000, 0x384d8000, 0x384dc000, 0x384e0000, 0x384e4000, 0x384e8000, 0x384ec000, 0x384f0000, 0x384f4000, 0x384f8000, 0x384fc000, 0x38500000, 0x38504000, 0x38508000, 0x3850c000, 0x38510000, 0x38514000, 0x38518000, 0x3851c000, 0x38520000, 0x38524000, 0x38528000, 0x3852c000, 0x38530000, 0x38534000, 0x38538000, 0x3853c000, 0x38540000, 0x38544000, 0x38548000, 0x3854c000, 0x38550000, 0x38554000, 0x38558000, 0x3855c000, 0x38560000, 0x38564000, 0x38568000, 0x3856c000, 0x38570000, 0x38574000, 0x38578000, 0x3857c000, 0x38580000, 0x38584000, 0x38588000, 0x3858c000, 0x38590000, 0x38594000, 0x38598000, 0x3859c000, 0x385a0000, 0x385a4000, 0x385a8000, 0x385ac000, 0x385b0000, 0x385b4000, 0x385b8000, 0x385bc000, 0x385c0000, 0x385c4000, 0x385c8000, 0x385cc000, 0x385d0000, 0x385d4000, 0x385d8000, 0x385dc000, 0x385e0000, 0x385e4000, 0x385e8000, 0x385ec000, 0x385f0000, 0x385f4000, 0x385f8000, 0x385fc000, 0x38600000, 0x38604000, 0x38608000, 0x3860c000, 0x38610000, 0x38614000, 0x38618000, 0x3861c000, 0x38620000, 0x38624000, 0x38628000, 0x3862c000, 0x38630000, 0x38634000, 0x38638000, 0x3863c000, 0x38640000, 0x38644000, 0x38648000, 0x3864c000, 0x38650000, 0x38654000, 0x38658000, 0x3865c000, 0x38660000, 0x38664000, 0x38668000, 0x3866c000, 0x38670000, 0x38674000, 0x38678000, 0x3867c000, 0x38680000, 0x38684000, 0x38688000, 0x3868c000, 0x38690000, 0x38694000, 0x38698000, 0x3869c000, 0x386a0000, 0x386a4000, 0x386a8000, 0x386ac000, 0x386b0000, 0x386b4000, 0x386b8000, 0x386bc000, 0x386c0000, 0x386c4000, 0x386c8000, 0x386cc000, 0x386d0000, 0x386d4000, 0x386d8000, 0x386dc000, 0x386e0000, 0x386e4000, 0x386e8000, 0x386ec000, 0x386f0000, 0x386f4000, 0x386f8000, 0x386fc000, 0x38700000, 0x38704000, 0x38708000, 0x3870c000, 0x38710000, 0x38714000, 0x38718000, 0x3871c000, 0x38720000, 0x38724000, 0x38728000, 0x3872c000, 0x38730000, 0x38734000, 0x38738000, 0x3873c000, 0x38740000, 0x38744000, 0x38748000, 0x3874c000, 0x38750000, 0x38754000, 0x38758000, 0x3875c000, 0x38760000, 0x38764000, 0x38768000, 0x3876c000, 0x38770000, 0x38774000, 0x38778000, 0x3877c000, 0x38780000, 0x38784000, 0x38788000, 0x3878c000, 0x38790000, 0x38794000, 0x38798000, 0x3879c000, 0x387a0000, 0x387a4000, 0x387a8000, 0x387ac000, 0x387b0000, 0x387b4000, 0x387b8000, 0x387bc000, 0x387c0000, 0x387c4000, 0x387c8000, 0x387cc000, 0x387d0000, 0x387d4000, 0x387d8000, 0x387dc000, 0x387e0000, 0x387e4000, 0x387e8000, 0x387ec000, 0x387f0000, 0x387f4000, 0x387f8000, 0x387fc000, 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800a000, 0x3800c000, 0x3800e000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801a000, 0x3801c000, 0x3801e000, 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802a000, 0x3802c000, 0x3802e000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803a000, 0x3803c000, 0x3803e000, 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804a000, 0x3804c000, 0x3804e000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805a000, 0x3805c000, 0x3805e000, 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806a000, 0x3806c000, 0x3806e000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807a000, 0x3807c000, 0x3807e000, 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808a000, 0x3808c000, 0x3808e000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809a000, 0x3809c000, 0x3809e000, 0x380a0000, 0x380a2000, 0x380a4000, 0x380a6000, 0x380a8000, 0x380aa000, 0x380ac000, 0x380ae000, 0x380b0000, 0x380b2000, 0x380b4000, 0x380b6000, 0x380b8000, 0x380ba000, 0x380bc000, 0x380be000, 0x380c0000, 0x380c2000, 0x380c4000, 0x380c6000, 0x380c8000, 0x380ca000, 0x380cc000, 0x380ce000, 0x380d0000, 0x380d2000, 0x380d4000, 0x380d6000, 0x380d8000, 0x380da000, 0x380dc000, 0x380de000, 0x380e0000, 0x380e2000, 0x380e4000, 0x380e6000, 0x380e8000, 0x380ea000, 0x380ec000, 0x380ee000, 0x380f0000, 0x380f2000, 0x380f4000, 0x380f6000, 0x380f8000, 0x380fa000, 0x380fc000, 0x380fe000, 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810a000, 0x3810c000, 0x3810e000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811a000, 0x3811c000, 0x3811e000, 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812a000, 0x3812c000, 0x3812e000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813a000, 0x3813c000, 0x3813e000, 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814a000, 0x3814c000, 0x3814e000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815a000, 0x3815c000, 0x3815e000, 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816a000, 0x3816c000, 0x3816e000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817a000, 0x3817c000, 0x3817e000, 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818a000, 0x3818c000, 0x3818e000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819a000, 0x3819c000, 0x3819e000, 0x381a0000, 0x381a2000, 0x381a4000, 0x381a6000, 0x381a8000, 0x381aa000, 0x381ac000, 0x381ae000, 0x381b0000, 0x381b2000, 0x381b4000, 0x381b6000, 0x381b8000, 0x381ba000, 0x381bc000, 0x381be000, 0x381c0000, 0x381c2000, 0x381c4000, 0x381c6000, 0x381c8000, 0x381ca000, 0x381cc000, 0x381ce000, 0x381d0000, 0x381d2000, 0x381d4000, 0x381d6000, 0x381d8000, 0x381da000, 0x381dc000, 0x381de000, 0x381e0000, 0x381e2000, 0x381e4000, 0x381e6000, 0x381e8000, 0x381ea000, 0x381ec000, 0x381ee000, 0x381f0000, 0x381f2000, 0x381f4000, 0x381f6000, 0x381f8000, 0x381fa000, 0x381fc000, 0x381fe000, 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820a000, 0x3820c000, 0x3820e000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821a000, 0x3821c000, 0x3821e000, 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822a000, 0x3822c000, 0x3822e000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823a000, 0x3823c000, 0x3823e000, 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824a000, 0x3824c000, 0x3824e000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825a000, 0x3825c000, 0x3825e000, 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826a000, 0x3826c000, 0x3826e000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827a000, 0x3827c000, 0x3827e000, 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828a000, 0x3828c000, 0x3828e000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829a000, 0x3829c000, 0x3829e000, 0x382a0000, 0x382a2000, 0x382a4000, 0x382a6000, 0x382a8000, 0x382aa000, 0x382ac000, 0x382ae000, 0x382b0000, 0x382b2000, 0x382b4000, 0x382b6000, 0x382b8000, 0x382ba000, 0x382bc000, 0x382be000, 0x382c0000, 0x382c2000, 0x382c4000, 0x382c6000, 0x382c8000, 0x382ca000, 0x382cc000, 0x382ce000, 0x382d0000, 0x382d2000, 0x382d4000, 0x382d6000, 0x382d8000, 0x382da000, 0x382dc000, 0x382de000, 0x382e0000, 0x382e2000, 0x382e4000, 0x382e6000, 0x382e8000, 0x382ea000, 0x382ec000, 0x382ee000, 0x382f0000, 0x382f2000, 0x382f4000, 0x382f6000, 0x382f8000, 0x382fa000, 0x382fc000, 0x382fe000, 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830a000, 0x3830c000, 0x3830e000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831a000, 0x3831c000, 0x3831e000, 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832a000, 0x3832c000, 0x3832e000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833a000, 0x3833c000, 0x3833e000, 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834a000, 0x3834c000, 0x3834e000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835a000, 0x3835c000, 0x3835e000, 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836a000, 0x3836c000, 0x3836e000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837a000, 0x3837c000, 0x3837e000, 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838a000, 0x3838c000, 0x3838e000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839a000, 0x3839c000, 0x3839e000, 0x383a0000, 0x383a2000, 0x383a4000, 0x383a6000, 0x383a8000, 0x383aa000, 0x383ac000, 0x383ae000, 0x383b0000, 0x383b2000, 0x383b4000, 0x383b6000, 0x383b8000, 0x383ba000, 0x383bc000, 0x383be000, 0x383c0000, 0x383c2000, 0x383c4000, 0x383c6000, 0x383c8000, 0x383ca000, 0x383cc000, 0x383ce000, 0x383d0000, 0x383d2000, 0x383d4000, 0x383d6000, 0x383d8000, 0x383da000, 0x383dc000, 0x383de000, 0x383e0000, 0x383e2000, 0x383e4000, 0x383e6000, 0x383e8000, 0x383ea000, 0x383ec000, 0x383ee000, 0x383f0000, 0x383f2000, 0x383f4000, 0x383f6000, 0x383f8000, 0x383fa000, 0x383fc000, 0x383fe000, 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840a000, 0x3840c000, 0x3840e000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841a000, 0x3841c000, 0x3841e000, 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842a000, 0x3842c000, 0x3842e000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843a000, 0x3843c000, 0x3843e000, 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844a000, 0x3844c000, 0x3844e000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845a000, 0x3845c000, 0x3845e000, 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846a000, 0x3846c000, 0x3846e000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847a000, 0x3847c000, 0x3847e000, 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848a000, 0x3848c000, 0x3848e000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849a000, 0x3849c000, 0x3849e000, 0x384a0000, 0x384a2000, 0x384a4000, 0x384a6000, 0x384a8000, 0x384aa000, 0x384ac000, 0x384ae000, 0x384b0000, 0x384b2000, 0x384b4000, 0x384b6000, 0x384b8000, 0x384ba000, 0x384bc000, 0x384be000, 0x384c0000, 0x384c2000, 0x384c4000, 0x384c6000, 0x384c8000, 0x384ca000, 0x384cc000, 0x384ce000, 0x384d0000, 0x384d2000, 0x384d4000, 0x384d6000, 0x384d8000, 0x384da000, 0x384dc000, 0x384de000, 0x384e0000, 0x384e2000, 0x384e4000, 0x384e6000, 0x384e8000, 0x384ea000, 0x384ec000, 0x384ee000, 0x384f0000, 0x384f2000, 0x384f4000, 0x384f6000, 0x384f8000, 0x384fa000, 0x384fc000, 0x384fe000, 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850a000, 0x3850c000, 0x3850e000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851a000, 0x3851c000, 0x3851e000, 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852a000, 0x3852c000, 0x3852e000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853a000, 0x3853c000, 0x3853e000, 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854a000, 0x3854c000, 0x3854e000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855a000, 0x3855c000, 0x3855e000, 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856a000, 0x3856c000, 0x3856e000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857a000, 0x3857c000, 0x3857e000, 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858a000, 0x3858c000, 0x3858e000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859a000, 0x3859c000, 0x3859e000, 0x385a0000, 0x385a2000, 0x385a4000, 0x385a6000, 0x385a8000, 0x385aa000, 0x385ac000, 0x385ae000, 0x385b0000, 0x385b2000, 0x385b4000, 0x385b6000, 0x385b8000, 0x385ba000, 0x385bc000, 0x385be000, 0x385c0000, 0x385c2000, 0x385c4000, 0x385c6000, 0x385c8000, 0x385ca000, 0x385cc000, 0x385ce000, 0x385d0000, 0x385d2000, 0x385d4000, 0x385d6000, 0x385d8000, 0x385da000, 0x385dc000, 0x385de000, 0x385e0000, 0x385e2000, 0x385e4000, 0x385e6000, 0x385e8000, 0x385ea000, 0x385ec000, 0x385ee000, 0x385f0000, 0x385f2000, 0x385f4000, 0x385f6000, 0x385f8000, 0x385fa000, 0x385fc000, 0x385fe000, 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860a000, 0x3860c000, 0x3860e000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861a000, 0x3861c000, 0x3861e000, 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862a000, 0x3862c000, 0x3862e000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863a000, 0x3863c000, 0x3863e000, 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864a000, 0x3864c000, 0x3864e000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865a000, 0x3865c000, 0x3865e000, 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866a000, 0x3866c000, 0x3866e000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867a000, 0x3867c000, 0x3867e000, 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868a000, 0x3868c000, 0x3868e000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869a000, 0x3869c000, 0x3869e000, 0x386a0000, 0x386a2000, 0x386a4000, 0x386a6000, 0x386a8000, 0x386aa000, 0x386ac000, 0x386ae000, 0x386b0000, 0x386b2000, 0x386b4000, 0x386b6000, 0x386b8000, 0x386ba000, 0x386bc000, 0x386be000, 0x386c0000, 0x386c2000, 0x386c4000, 0x386c6000, 0x386c8000, 0x386ca000, 0x386cc000, 0x386ce000, 0x386d0000, 0x386d2000, 0x386d4000, 0x386d6000, 0x386d8000, 0x386da000, 0x386dc000, 0x386de000, 0x386e0000, 0x386e2000, 0x386e4000, 0x386e6000, 0x386e8000, 0x386ea000, 0x386ec000, 0x386ee000, 0x386f0000, 0x386f2000, 0x386f4000, 0x386f6000, 0x386f8000, 0x386fa000, 0x386fc000, 0x386fe000, 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870a000, 0x3870c000, 0x3870e000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871a000, 0x3871c000, 0x3871e000, 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872a000, 0x3872c000, 0x3872e000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873a000, 0x3873c000, 0x3873e000, 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874a000, 0x3874c000, 0x3874e000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875a000, 0x3875c000, 0x3875e000, 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876a000, 0x3876c000, 0x3876e000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877a000, 0x3877c000, 0x3877e000, 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878a000, 0x3878c000, 0x3878e000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879a000, 0x3879c000, 0x3879e000, 0x387a0000, 0x387a2000, 0x387a4000, 0x387a6000, 0x387a8000, 0x387aa000, 0x387ac000, 0x387ae000, 0x387b0000, 0x387b2000, 0x387b4000, 0x387b6000, 0x387b8000, 0x387ba000, 0x387bc000, 0x387be000, 0x387c0000, 0x387c2000, 0x387c4000, 0x387c6000, 0x387c8000, 0x387ca000, 0x387cc000, 0x387ce000, 0x387d0000, 0x387d2000, 0x387d4000, 0x387d6000, 0x387d8000, 0x387da000, 0x387dc000, 0x387de000, 0x387e0000, 0x387e2000, 0x387e4000, 0x387e6000, 0x387e8000, 0x387ea000, 0x387ec000, 0x387ee000, 0x387f0000, 0x387f2000, 0x387f4000, 0x387f6000, 0x387f8000, 0x387fa000, 0x387fc000, 0x387fe000, }; static const uint32_t exptable[] = { 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0a000000, 0x0a800000, 0x0b000000, 0x0b800000, 0x0c000000, 0x0c800000, 0x0d000000, 0x0d800000, 0x0e000000, 0x0e800000, 0x0f000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8a000000, 0x8a800000, 0x8b000000, 0x8b800000, 0x8c000000, 0x8c800000, 0x8d000000, 0x8d800000, 0x8e000000, 0x8e800000, 0x8f000000, 0xc7800000, }; static const uint16_t offsettable[] = { 0x0000, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0000, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, }; static const uint16_t basetable[] = { 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, 0x0c00, 0x1000, 0x1400, 0x1800, 0x1c00, 0x2000, 0x2400, 0x2800, 0x2c00, 0x3000, 0x3400, 0x3800, 0x3c00, 0x4000, 0x4400, 0x4800, 0x4c00, 0x5000, 0x5400, 0x5800, 0x5c00, 0x6000, 0x6400, 0x6800, 0x6c00, 0x7000, 0x7400, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x7c00, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, 0x8200, 0x8400, 0x8800, 0x8c00, 0x9000, 0x9400, 0x9800, 0x9c00, 0xa000, 0xa400, 0xa800, 0xac00, 0xb000, 0xb400, 0xb800, 0xbc00, 0xc000, 0xc400, 0xc800, 0xcc00, 0xd000, 0xd400, 0xd800, 0xdc00, 0xe000, 0xe400, 0xe800, 0xec00, 0xf000, 0xf400, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, 0xfc00, }; static const uint16_t shifttable[] = { 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0017, 0x0016, 0x0015, 0x0014, 0x0013, 0x0012, 0x0011, 0x0010, 0x000f, 0x000e, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x000d, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0017, 0x0016, 0x0015, 0x0014, 0x0013, 0x0012, 0x0011, 0x0010, 0x000f, 0x000e, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x000d, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x0018, 0x000d, }; float unpack_float16(uint16_t i) { // This function assumes that i contains the big-endian representation of // an IEEE754 16-bit floating point value (1 sign bit, 5-bits of exponent, // and 10-bits of mantissa). It returns the IEEE754 single-precision // equivalent value (handling infinity and NaN cases accordingly) union { struct { #if PY_BIG_ENDIAN unsigned int exp: 6; // yes, this includes the sign unsigned int sig: 10; #else unsigned int sig: 10; unsigned int exp: 6; // yes, this includes the sign #endif }; uint16_t value; } in; union { float value; uint32_t i; } ret; in.value = be16toh(i); ret.i = sigtable[offsettable[in.exp] + in.sig] + exptable[in.exp]; return ret.value; } uint16_t pack_float16(float f) { // This function returns the big-endian representation of an IEEE754 // 16-bit floating point value as a 16-bit integer (handling infinity and // NaN cases accordingly); overflow returns infinity, underflow returns 0.0 union { struct { #if PY_BIG_ENDIAN unsigned int exp: 9; unsigned int sig: 23; #else unsigned int sig: 23; unsigned int exp: 9; // again, this includes the sign #endif }; float f; } in; uint16_t ret; in.f = f; ret = basetable[in.exp] + (in.sig >> shifttable[in.exp]); return htobe16(ret); } cbor2-5.4.2/source/halffloat.h000066400000000000000000000001251413200260400161300ustar00rootroot00000000000000#include float unpack_float16(uint16_t); uint16_t pack_float16(float f); cbor2-5.4.2/source/module.c000066400000000000000000000673031413200260400154630ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include "module.h" #include "tags.h" #include "encoder.h" #include "decoder.h" // Some notes on conventions in this code. All methods conform to a couple of // return styles: // // * PyObject* (mostly for methods accessible from Python) in which case a // return value of NULL indicates an error, or // // * int (mostly for internal methods) in which case 0 indicates success and -1 // an error. This is in keeping with most of Python's C-API. // // In an attempt to avoid leaks a particular coding style is used where // possible: // // 1. As soon as a new reference to an object is generated / returned, a // block like this follows: if (ref) { ... Py_DECREF(ref); } // // 2. The result is calculated in the "ret" local and returned only at the // end of the function, once we're sure all references have been accounted // for. // // 3. No "return" is permitted before the end of the function, and "break" or // "goto" should be used over a minimal distance to ensure Py_DECREFs aren't // jumped over. // // 4. Wherever possible, functions that return a PyObject pointer return a // *new* reference (like the majority of the CPython API) as opposed to // a borrowed reference. // // 5. The above rules are broken occasionally where necessary for clarity :) // // While this style helps ensure fewer leaks, it's worth noting it results in // rather "nested" code which looks a bit unusual / ugly for C. Furthermore, // it's not fool-proof; there's probably some leaks left. Please file bugs for // any leaks you detect! // break_marker singleton //////////////////////////////////////////////////// static PyObject * break_marker_repr(PyObject *op) { return PyUnicode_FromString("break_marker"); } static void break_marker_dealloc(PyObject *ignore) { Py_FatalError("deallocating break_marker"); } static PyObject * break_marker_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_Size(kwargs))) { PyErr_SetString(PyExc_TypeError, "break_marker_type takes no arguments"); return NULL; } Py_INCREF(break_marker); return break_marker; } static int break_marker_bool(PyObject *v) { return 1; } static PyNumberMethods break_marker_as_number = { .nb_bool = (inquiry) break_marker_bool, }; PyTypeObject break_marker_type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "break_marker_type", .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = break_marker_new, .tp_dealloc = break_marker_dealloc, .tp_repr = break_marker_repr, .tp_as_number = &break_marker_as_number, }; PyObject _break_marker_obj = { _PyObject_EXTRA_INIT 1, &break_marker_type }; // undefined singleton /////////////////////////////////////////////////////// static PyObject * undefined_repr(PyObject *op) { return PyUnicode_FromString("undefined"); } static void undefined_dealloc(PyObject *ignore) { Py_FatalError("deallocating undefined"); } static PyObject * undefined_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_Size(kwargs))) { PyErr_SetString(PyExc_TypeError, "undefined_type takes no arguments"); return NULL; } Py_INCREF(undefined); return undefined; } static int undefined_bool(PyObject *v) { return 0; } static PyNumberMethods undefined_as_number = { .nb_bool = (inquiry) undefined_bool, }; PyTypeObject undefined_type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "undefined_type", .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = undefined_new, .tp_dealloc = undefined_dealloc, .tp_repr = undefined_repr, .tp_as_number = &undefined_as_number, }; PyObject _undefined_obj = { _PyObject_EXTRA_INIT 1, &undefined_type }; // CBORSimpleValue namedtuple //////////////////////////////////////////////// PyTypeObject CBORSimpleValueType; static PyStructSequence_Field CBORSimpleValueFields[] = { {.name = "value"}, {NULL}, }; PyDoc_STRVAR(_CBOR2_CBORSimpleValue__doc__, "Represents a CBOR \"simple value\", with a value of 0 to 255." ); static PyStructSequence_Desc CBORSimpleValueDesc = { .name = "CBORSimpleValue", .doc = _CBOR2_CBORSimpleValue__doc__, .fields = CBORSimpleValueFields, .n_in_sequence = 1, }; static PyObject * CBORSimpleValue_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { static char *keywords[] = {"value", NULL}; PyObject *value = NULL, *ret; Py_ssize_t val; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "n", keywords, &val)) return NULL; if (val < 0 || val > 255) { PyErr_SetString(PyExc_TypeError, "simple value out of range (0..255)"); return NULL; } ret = PyStructSequence_New(type); if (ret) { value = PyLong_FromSsize_t(val); if (value) PyStructSequence_SET_ITEM(ret, 0, value); } return ret; } static PyObject * CBORSimpleValue_richcompare(PyObject *a, PyObject *b, int op) { switch (PyObject_IsInstance(b, (PyObject *) &CBORSimpleValueType)) { case 1: return PyObject_RichCompare( PyStructSequence_GET_ITEM(a, 0), PyStructSequence_GET_ITEM(b, 0), op); case -1: return NULL; } switch (PyObject_IsInstance(b, (PyObject *) &PyLong_Type)) { case 1: return PyObject_RichCompare( PyStructSequence_GET_ITEM(a, 0), b, op); case -1: return NULL; } Py_RETURN_NOTIMPLEMENTED; } // dump/load functions /////////////////////////////////////////////////////// static PyObject * CBOR2_dump(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *obj = NULL, *ret = NULL; CBOREncoderObject *self; bool decref_args = false; if (PyTuple_GET_SIZE(args) == 0) { if (kwargs) obj = PyDict_GetItem(kwargs, _CBOR2_str_obj); if (!obj) { PyErr_SetString(PyExc_TypeError, "dump missing 1 required argument: 'obj'"); return NULL; } Py_INCREF(obj); if (PyDict_DelItem(kwargs, _CBOR2_str_obj) == -1) { Py_DECREF(obj); return NULL; } } else { obj = PyTuple_GET_ITEM(args, 0); args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); if (!args) return NULL; Py_INCREF(obj); decref_args = true; } self = (CBOREncoderObject *)CBOREncoder_new(&CBOREncoderType, NULL, NULL); if (self) { if (CBOREncoder_init(self, args, kwargs) == 0) { ret = CBOREncoder_encode(self, obj); } Py_DECREF(self); } Py_DECREF(obj); if (decref_args) Py_DECREF(args); return ret; } static PyObject * CBOR2_dumps(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *fp, *result, *new_args = NULL, *obj = NULL, *ret = NULL; Py_ssize_t i; if (!_CBOR2_BytesIO && _CBOR2_init_BytesIO() == -1) return NULL; fp = PyObject_CallFunctionObjArgs(_CBOR2_BytesIO, NULL); if (fp) { if (PyTuple_GET_SIZE(args) == 0) { if (kwargs) obj = PyDict_GetItem(kwargs, _CBOR2_str_obj); if (obj) { if (PyDict_DelItem(kwargs, _CBOR2_str_obj) == 0) new_args = PyTuple_Pack(2, obj, fp); } else { PyErr_SetString(PyExc_TypeError, "dumps missing required argument: 'obj'"); } } else { obj = PyTuple_GET_ITEM(args, 0); new_args = PyTuple_New(PyTuple_GET_SIZE(args) + 1); if (new_args) { Py_INCREF(obj); Py_INCREF(fp); PyTuple_SET_ITEM(new_args, 0, obj); // steals ref PyTuple_SET_ITEM(new_args, 1, fp); // steals ref for (i = 1; i < PyTuple_GET_SIZE(args); ++i) { Py_INCREF(PyTuple_GET_ITEM(args, i)); PyTuple_SET_ITEM(new_args, i + 1, PyTuple_GET_ITEM(args, i)); } } } if (new_args) { result = CBOR2_dump(module, new_args, kwargs); if (result) { ret = PyObject_CallMethodObjArgs(fp, _CBOR2_str_getvalue, NULL); Py_DECREF(result); } Py_DECREF(new_args); } Py_DECREF(fp); } return ret; } static PyObject * CBOR2_load(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *ret = NULL; CBORDecoderObject *self; self = (CBORDecoderObject *)CBORDecoder_new(&CBORDecoderType, NULL, NULL); if (self) { if (CBORDecoder_init(self, args, kwargs) == 0) { ret = CBORDecoder_decode(self); } Py_DECREF(self); } return ret; } static PyObject * CBOR2_loads(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *fp, *new_args = NULL, *s = NULL, *ret = NULL; Py_ssize_t i; if (!_CBOR2_BytesIO && _CBOR2_init_BytesIO() == -1) return NULL; if (PyTuple_GET_SIZE(args) == 0) { if (kwargs) { new_args = PyTuple_New(1); if (new_args) { s = PyDict_GetItem(kwargs, _CBOR2_str_s); Py_INCREF(s); if (PyDict_DelItem(kwargs, _CBOR2_str_s) == -1) { Py_DECREF(s); Py_CLEAR(new_args); } } } else { PyErr_SetString(PyExc_TypeError, "dump missing 1 required argument: 's'"); } } else { new_args = PyTuple_New(PyTuple_GET_SIZE(args)); if (new_args) { s = PyTuple_GET_ITEM(args, 0); Py_INCREF(s); for (i = 1; i < PyTuple_GET_SIZE(args); ++i) { // inc. ref because PyTuple_SET_ITEM steals a ref Py_INCREF(PyTuple_GET_ITEM(args, i)); PyTuple_SET_ITEM(new_args, i, PyTuple_GET_ITEM(args, i)); } } } if (new_args) { fp = PyObject_CallFunctionObjArgs(_CBOR2_BytesIO, s, NULL); if (fp) { PyTuple_SET_ITEM(new_args, 0, fp); ret = CBOR2_load(module, new_args, kwargs); // no need to dec. ref fp here because SET_ITEM above stole the ref } Py_DECREF(s); Py_DECREF(new_args); } return ret; } // Cache-init functions ////////////////////////////////////////////////////// int _CBOR2_init_BytesIO(void) { PyObject *io; // from io import BytesIO io = PyImport_ImportModule("io"); if (!io) goto error; _CBOR2_BytesIO = PyObject_GetAttr(io, _CBOR2_str_BytesIO); Py_DECREF(io); if (!_CBOR2_BytesIO) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import BytesIO from io"); return -1; } int _CBOR2_init_FrozenDict(void) { PyObject *cbor2_types; // from cbor2.types import FrozenDict cbor2_types = PyImport_ImportModule("cbor2.types"); if (!cbor2_types) goto error; _CBOR2_FrozenDict = PyObject_GetAttr(cbor2_types, _CBOR2_str_FrozenDict); Py_DECREF(cbor2_types); if (!_CBOR2_FrozenDict) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import FrozenDict from cbor2.types"); return -1; } int _CBOR2_init_Decimal(void) { PyObject *decimal; // from decimal import Decimal decimal = PyImport_ImportModule("decimal"); if (!decimal) goto error; _CBOR2_Decimal = PyObject_GetAttr(decimal, _CBOR2_str_Decimal); Py_DECREF(decimal); if (!_CBOR2_Decimal) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import Decimal from decimal"); return -1; } int _CBOR2_init_Fraction(void) { PyObject *fractions; // from fractions import Fraction fractions = PyImport_ImportModule("fractions"); if (!fractions) goto error; _CBOR2_Fraction = PyObject_GetAttr(fractions, _CBOR2_str_Fraction); Py_DECREF(fractions); if (!_CBOR2_Fraction) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import Fraction from fractions"); return -1; } int _CBOR2_init_UUID(void) { PyObject *uuid; // from uuid import UUID uuid = PyImport_ImportModule("uuid"); if (!uuid) goto error; _CBOR2_UUID = PyObject_GetAttr(uuid, _CBOR2_str_UUID); Py_DECREF(uuid); if (!_CBOR2_UUID) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import UUID from uuid"); return -1; } int _CBOR2_init_re_compile(void) { PyObject *re; // import re // datestr_re = re.compile("long-date-time-regex...") re = PyImport_ImportModule("re"); if (!re) goto error; _CBOR2_re_compile = PyObject_GetAttr(re, _CBOR2_str_compile); Py_DECREF(re); if (!_CBOR2_re_compile) goto error; _CBOR2_datestr_re = PyObject_CallFunctionObjArgs( _CBOR2_re_compile, _CBOR2_str_datestr_re, NULL); if (!_CBOR2_datestr_re) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import compile from re"); return -1; } int _CBOR2_init_timezone_utc(void) { #if PY_VERSION_HEX >= 0x03070000 Py_INCREF(PyDateTime_TimeZone_UTC); _CBOR2_timezone_utc = PyDateTime_TimeZone_UTC; _CBOR2_timezone = NULL; return 0; #else PyObject* datetime; // from datetime import timezone // utc = timezone.utc datetime = PyImport_ImportModule("datetime"); if (!datetime) goto error; _CBOR2_timezone = PyObject_GetAttr(datetime, _CBOR2_str_timezone); Py_DECREF(datetime); if (!_CBOR2_timezone) goto error; _CBOR2_timezone_utc = PyObject_GetAttr(_CBOR2_timezone, _CBOR2_str_utc); if (!_CBOR2_timezone_utc) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import timezone from datetime"); return -1; #endif } int _CBOR2_init_Parser(void) { PyObject *parser; // from email.parser import Parser parser = PyImport_ImportModule("email.parser"); if (!parser) goto error; _CBOR2_Parser = PyObject_GetAttr(parser, _CBOR2_str_Parser); Py_DECREF(parser); if (!_CBOR2_Parser) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import Parser from email.parser"); return -1; } int _CBOR2_init_ip_address(void) { PyObject *ipaddress; // from ipaddress import ip_address ipaddress = PyImport_ImportModule("ipaddress"); if (!ipaddress) goto error; _CBOR2_ip_address = PyObject_GetAttr(ipaddress, _CBOR2_str_ip_address); _CBOR2_ip_network = PyObject_GetAttr(ipaddress, _CBOR2_str_ip_network); Py_DECREF(ipaddress); if (!_CBOR2_ip_address) goto error; if (!_CBOR2_ip_network) goto error; return 0; error: PyErr_SetString(PyExc_ImportError, "unable to import ip_address from ipaddress"); return -1; } // Module definition ///////////////////////////////////////////////////////// PyObject *_CBOR2_empty_bytes = NULL; PyObject *_CBOR2_empty_str = NULL; PyObject *_CBOR2_str_as_string = NULL; PyObject *_CBOR2_str_as_tuple = NULL; PyObject *_CBOR2_str_bit_length = NULL; PyObject *_CBOR2_str_bytes = NULL; PyObject *_CBOR2_str_BytesIO = NULL; PyObject *_CBOR2_str_canonical_encoders = NULL; PyObject *_CBOR2_str_compile = NULL; PyObject *_CBOR2_str_copy = NULL; PyObject *_CBOR2_str_datestr_re = NULL; PyObject *_CBOR2_str_Decimal = NULL; PyObject *_CBOR2_str_default_encoders = NULL; PyObject *_CBOR2_str_denominator = NULL; PyObject *_CBOR2_str_encode_date = NULL; PyObject *_CBOR2_str_Fraction = NULL; PyObject *_CBOR2_str_fromtimestamp = NULL; PyObject *_CBOR2_str_FrozenDict = NULL; PyObject *_CBOR2_str_getvalue = NULL; PyObject *_CBOR2_str_groups = NULL; PyObject *_CBOR2_str_ip_address = NULL; PyObject *_CBOR2_str_ip_network = NULL; PyObject *_CBOR2_str_is_infinite = NULL; PyObject *_CBOR2_str_is_nan = NULL; PyObject *_CBOR2_str_isoformat = NULL; PyObject *_CBOR2_str_join = NULL; PyObject *_CBOR2_str_match = NULL; PyObject *_CBOR2_str_network_address = NULL; PyObject *_CBOR2_str_numerator = NULL; PyObject *_CBOR2_str_obj = NULL; PyObject *_CBOR2_str_packed = NULL; PyObject *_CBOR2_str_Parser = NULL; PyObject *_CBOR2_str_parsestr = NULL; PyObject *_CBOR2_str_pattern = NULL; PyObject *_CBOR2_str_prefixlen = NULL; PyObject *_CBOR2_str_read = NULL; PyObject *_CBOR2_str_s = NULL; PyObject *_CBOR2_str_timestamp = NULL; PyObject *_CBOR2_str_timezone = NULL; PyObject *_CBOR2_str_update = NULL; PyObject *_CBOR2_str_utc = NULL; PyObject *_CBOR2_str_utc_suffix = NULL; PyObject *_CBOR2_str_UUID = NULL; PyObject *_CBOR2_str_write = NULL; PyObject *_CBOR2_CBORError = NULL; PyObject *_CBOR2_CBOREncodeError = NULL; PyObject *_CBOR2_CBOREncodeTypeError = NULL; PyObject *_CBOR2_CBOREncodeValueError = NULL; PyObject *_CBOR2_CBORDecodeError = NULL; PyObject *_CBOR2_CBORDecodeValueError = NULL; PyObject *_CBOR2_CBORDecodeEOF = NULL; PyObject *_CBOR2_timezone = NULL; PyObject *_CBOR2_timezone_utc = NULL; PyObject *_CBOR2_BytesIO = NULL; PyObject *_CBOR2_Decimal = NULL; PyObject *_CBOR2_Fraction = NULL; PyObject *_CBOR2_FrozenDict = NULL; PyObject *_CBOR2_UUID = NULL; PyObject *_CBOR2_Parser = NULL; PyObject *_CBOR2_re_compile = NULL; PyObject *_CBOR2_datestr_re = NULL; PyObject *_CBOR2_ip_address = NULL; PyObject *_CBOR2_ip_network = NULL; PyObject *_CBOR2_default_encoders = NULL; PyObject *_CBOR2_canonical_encoders = NULL; static void cbor2_free(PyObject *m) { Py_CLEAR(_CBOR2_timezone_utc); Py_CLEAR(_CBOR2_timezone); Py_CLEAR(_CBOR2_BytesIO); Py_CLEAR(_CBOR2_Decimal); Py_CLEAR(_CBOR2_Fraction); Py_CLEAR(_CBOR2_UUID); Py_CLEAR(_CBOR2_Parser); Py_CLEAR(_CBOR2_re_compile); Py_CLEAR(_CBOR2_datestr_re); Py_CLEAR(_CBOR2_ip_address); Py_CLEAR(_CBOR2_ip_network); Py_CLEAR(_CBOR2_CBOREncodeError); Py_CLEAR(_CBOR2_CBOREncodeTypeError); Py_CLEAR(_CBOR2_CBOREncodeValueError); Py_CLEAR(_CBOR2_CBORDecodeError); Py_CLEAR(_CBOR2_CBORDecodeValueError); Py_CLEAR(_CBOR2_CBORDecodeEOF); Py_CLEAR(_CBOR2_CBORError); Py_CLEAR(_CBOR2_default_encoders); Py_CLEAR(_CBOR2_canonical_encoders); } static PyMethodDef _cbor2methods[] = { {"dump", (PyCFunction) CBOR2_dump, METH_VARARGS | METH_KEYWORDS, "encode a value to the stream"}, {"dumps", (PyCFunction) CBOR2_dumps, METH_VARARGS | METH_KEYWORDS, "encode a value to a byte-string"}, {"load", (PyCFunction) CBOR2_load, METH_VARARGS | METH_KEYWORDS, "decode a value from the stream"}, {"loads", (PyCFunction) CBOR2_loads, METH_VARARGS | METH_KEYWORDS, "decode a value from a byte-string"}, {NULL} }; PyDoc_STRVAR(_cbor2__doc__, "The _cbor2 module is the C-extension backing the cbor2 Python module. It\n" "defines the base :exc:`CBORError`, :exc:`CBOREncodeError`,\n" ":exc:`CBOREncodeTypeError`, :exc:`CBOREncodeValueError`,\n" ":exc:`CBORDecodeError`, :exc:`CBORDecodeValueError`, :exc:`CBORDecodeEOF`,\n" ":class:`CBOREncoder`, :class:`CBORDecoder`, :class:`CBORTag`, and undefined\n" "types which are operational in and of themselves." ); PyDoc_STRVAR(_cbor2_CBORError__doc__, "Base class for errors that occur during CBOR encoding or decoding." ); PyDoc_STRVAR(_cbor2_CBOREncodeError__doc__, "Raised for exceptions occurring during CBOR encoding." ); PyDoc_STRVAR(_cbor2_CBOREncodeTypeError__doc__, "Raised when attempting to encode a type that cannot be serialized." ); PyDoc_STRVAR(_cbor2_CBOREncodeValueError__doc__, "Raised when the CBOR encoder encounters an invalid value." ); PyDoc_STRVAR(_cbor2_CBORDecodeError__doc__, "Raised for exceptions occurring during CBOR decoding." ); PyDoc_STRVAR(_cbor2_CBORDecodeValueError__doc__, "Raised when the CBOR stream being decoded contains an invalid value." ); PyDoc_STRVAR(_cbor2_CBORDecodeEOF__doc__, "Raised when decoding unexpectedly reaches EOF." ); static struct PyModuleDef _cbor2module = { PyModuleDef_HEAD_INIT, .m_name = "_cbor2", .m_doc = _cbor2__doc__, .m_size = -1, .m_free = (freefunc) cbor2_free, .m_methods = _cbor2methods, }; int init_default_encoders(void) { PyObject *mod, *dict; // NOTE: All functions below return borrowed references, hence the lack of // DECREF calls if (_CBOR2_default_encoders) return 0; mod = PyState_FindModule(&_cbor2module); if (!mod) return -1; dict = PyModule_GetDict(mod); if (!dict) return -1; _CBOR2_default_encoders = PyDict_GetItem( dict, _CBOR2_str_default_encoders); if (_CBOR2_default_encoders) { Py_INCREF(_CBOR2_default_encoders); return 0; } return -1; } int init_canonical_encoders(void) { PyObject *mod, *dict; // NOTE: All functions below return borrowed references, hence the lack of // DECREF calls if (_CBOR2_canonical_encoders) return 0; mod = PyState_FindModule(&_cbor2module); if (!mod) return -1; dict = PyModule_GetDict(mod); if (!dict) return -1; _CBOR2_canonical_encoders = PyDict_GetItem( dict, _CBOR2_str_canonical_encoders); if (_CBOR2_canonical_encoders) { Py_INCREF(_CBOR2_canonical_encoders); return 0; } return -1; } PyMODINIT_FUNC PyInit__cbor2(void) { PyObject *module, *base; PyDateTime_IMPORT; if (!PyDateTimeAPI) return NULL; if (PyType_Ready(&break_marker_type) < 0) return NULL; if (PyType_Ready(&undefined_type) < 0) return NULL; if (PyType_Ready(&CBORTagType) < 0) return NULL; if (PyType_Ready(&CBOREncoderType) < 0) return NULL; if (PyType_Ready(&CBORDecoderType) < 0) return NULL; module = PyModule_Create(&_cbor2module); if (!module) return NULL; _CBOR2_CBORError = PyErr_NewExceptionWithDoc( "_cbor2.CBORError", _cbor2_CBORError__doc__, NULL, NULL); if (!_CBOR2_CBORError) goto error; Py_INCREF(_CBOR2_CBORError); if (PyModule_AddObject(module, "CBORError", _CBOR2_CBORError) == -1) goto error; _CBOR2_CBOREncodeError = PyErr_NewExceptionWithDoc( "_cbor2.CBOREncodeError", _cbor2_CBOREncodeError__doc__, _CBOR2_CBORError, NULL); if (!_CBOR2_CBOREncodeError) goto error; Py_INCREF(_CBOR2_CBOREncodeError); if (PyModule_AddObject(module, "CBOREncodeError", _CBOR2_CBOREncodeError) == -1) goto error; base = PyTuple_Pack(2, _CBOR2_CBOREncodeError, PyExc_TypeError); _CBOR2_CBOREncodeTypeError = PyErr_NewExceptionWithDoc( "_cbor2.CBOREncodeTypeError", _cbor2_CBOREncodeTypeError__doc__, base, NULL); Py_DECREF(base); if (!_CBOR2_CBOREncodeTypeError) goto error; Py_INCREF(_CBOR2_CBOREncodeTypeError); if (PyModule_AddObject(module, "CBOREncodeTypeError", _CBOR2_CBOREncodeTypeError) == -1) goto error; base = PyTuple_Pack(2, _CBOR2_CBOREncodeError, PyExc_ValueError); _CBOR2_CBOREncodeValueError = PyErr_NewExceptionWithDoc( "_cbor2.CBOREncodeValueError", _cbor2_CBOREncodeValueError__doc__, base, NULL); Py_DECREF(base); if (!_CBOR2_CBOREncodeValueError) goto error; Py_INCREF(_CBOR2_CBOREncodeValueError); if (PyModule_AddObject(module, "CBOREncodeValueError", _CBOR2_CBOREncodeValueError) == -1) goto error; _CBOR2_CBORDecodeError = PyErr_NewExceptionWithDoc( "_cbor2.CBORDecodeError", _cbor2_CBORDecodeError__doc__, _CBOR2_CBORError, NULL); if (!_CBOR2_CBORDecodeError) goto error; Py_INCREF(_CBOR2_CBORDecodeError); if (PyModule_AddObject(module, "CBORDecodeError", _CBOR2_CBORDecodeError) == -1) goto error; base = PyTuple_Pack(2, _CBOR2_CBORDecodeError, PyExc_ValueError); _CBOR2_CBORDecodeValueError = PyErr_NewExceptionWithDoc( "_cbor2.CBORDecodeValueError", _cbor2_CBORDecodeValueError__doc__, base, NULL); Py_DECREF(base); if (!_CBOR2_CBORDecodeValueError) goto error; Py_INCREF(_CBOR2_CBORDecodeValueError); if (PyModule_AddObject(module, "CBORDecodeValueError", _CBOR2_CBORDecodeValueError) == -1) goto error; base = PyTuple_Pack(2, _CBOR2_CBORDecodeError, PyExc_EOFError); _CBOR2_CBORDecodeEOF = PyErr_NewExceptionWithDoc( "_cbor2.CBORDecodeEOF", _cbor2_CBORDecodeEOF__doc__, base, NULL); Py_DECREF(base); if (!_CBOR2_CBORDecodeEOF) goto error; Py_INCREF(_CBOR2_CBORDecodeEOF); if (PyModule_AddObject(module, "CBORDecodeEOF", _CBOR2_CBORDecodeEOF) == -1) goto error; // Use PyStructSequence_InitType2 to workaround #34784 (dup of #28709) if (PyStructSequence_InitType2(&CBORSimpleValueType, &CBORSimpleValueDesc) == -1) goto error; Py_INCREF((PyObject *) &CBORSimpleValueType); CBORSimpleValueType.tp_new = CBORSimpleValue_new; CBORSimpleValueType.tp_richcompare = CBORSimpleValue_richcompare; if (PyModule_AddObject( module, "CBORSimpleValue", (PyObject *) &CBORSimpleValueType) == -1) goto error; Py_INCREF(&CBORTagType); if (PyModule_AddObject(module, "CBORTag", (PyObject *) &CBORTagType) == -1) goto error; Py_INCREF(&CBOREncoderType); if (PyModule_AddObject(module, "CBOREncoder", (PyObject *) &CBOREncoderType) == -1) goto error; Py_INCREF(&CBORDecoderType); if (PyModule_AddObject(module, "CBORDecoder", (PyObject *) &CBORDecoderType) == -1) goto error; Py_INCREF(break_marker); if (PyModule_AddObject(module, "break_marker", break_marker) == -1) goto error; Py_INCREF(undefined); if (PyModule_AddObject(module, "undefined", undefined) == -1) goto error; #define INTERN_STRING(name) \ if (!_CBOR2_str_##name && \ !(_CBOR2_str_##name = PyUnicode_InternFromString(#name))) \ goto error; INTERN_STRING(as_string); INTERN_STRING(as_tuple); INTERN_STRING(bit_length); INTERN_STRING(bytes); INTERN_STRING(BytesIO); INTERN_STRING(canonical_encoders); INTERN_STRING(compile); INTERN_STRING(copy); INTERN_STRING(Decimal); INTERN_STRING(default_encoders); INTERN_STRING(denominator); INTERN_STRING(encode_date); INTERN_STRING(Fraction); INTERN_STRING(fromtimestamp); INTERN_STRING(FrozenDict); INTERN_STRING(getvalue); INTERN_STRING(groups); INTERN_STRING(ip_address); INTERN_STRING(ip_network); INTERN_STRING(is_infinite); INTERN_STRING(is_nan); INTERN_STRING(isoformat); INTERN_STRING(join); INTERN_STRING(match); INTERN_STRING(network_address); INTERN_STRING(numerator); INTERN_STRING(obj); INTERN_STRING(packed); INTERN_STRING(Parser); INTERN_STRING(parsestr); INTERN_STRING(pattern); INTERN_STRING(prefixlen); INTERN_STRING(read); INTERN_STRING(s); INTERN_STRING(timestamp); INTERN_STRING(timezone); INTERN_STRING(update); INTERN_STRING(utc); INTERN_STRING(UUID); INTERN_STRING(write); #undef INTERN_STRING if (!_CBOR2_str_utc_suffix && !(_CBOR2_str_utc_suffix = PyUnicode_InternFromString("+00:00"))) goto error; if (!_CBOR2_str_datestr_re && !(_CBOR2_str_datestr_re = PyUnicode_InternFromString( "^(\\d{4})-(\\d\\d)-(\\d\\d)T" // Y-m-d "(\\d\\d):(\\d\\d):(\\d\\d)" // H:M:S "(?:\\.(\\d{1,6})\\d*)?" // .uS "(?:Z|([+-]\\d\\d):(\\d\\d))$"))) // +-TZ goto error; if (!_CBOR2_empty_bytes && !(_CBOR2_empty_bytes = PyBytes_FromStringAndSize(NULL, 0))) goto error; if (!_CBOR2_empty_str && !(_CBOR2_empty_str = PyUnicode_FromStringAndSize(NULL, 0))) goto error; return module; error: Py_DECREF(module); return NULL; } cbor2-5.4.2/source/module.h000066400000000000000000000073651413200260400154720ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include // structure of the lead-byte for all CBOR records typedef union { struct { #if PY_BIG_ENDIAN unsigned int major: 3; unsigned int subtype: 5; #else unsigned int subtype: 5; unsigned int major: 3; #endif }; char byte; } LeadByte; // break_marker singleton extern PyObject _break_marker_obj; #define break_marker (&_break_marker_obj) #define CBOR2_RETURN_BREAK return Py_INCREF(break_marker), break_marker // undefined singleton extern PyObject _undefined_obj; #define undefined (&_undefined_obj) #define CBOR2_RETURN_UNDEFINED return Py_INCREF(undefined), undefined // CBORSimpleValue namedtuple type extern PyTypeObject CBORSimpleValueType; // Various interned strings extern PyObject *_CBOR2_empty_bytes; extern PyObject *_CBOR2_empty_str; extern PyObject *_CBOR2_str_as_string; extern PyObject *_CBOR2_str_as_tuple; extern PyObject *_CBOR2_str_bit_length; extern PyObject *_CBOR2_str_bytes; extern PyObject *_CBOR2_str_BytesIO; extern PyObject *_CBOR2_str_canonical_encoders; extern PyObject *_CBOR2_str_compile; extern PyObject *_CBOR2_str_copy; extern PyObject *_CBOR2_str_datestr_re; extern PyObject *_CBOR2_str_Decimal; extern PyObject *_CBOR2_str_default_encoders; extern PyObject *_CBOR2_str_denominator; extern PyObject *_CBOR2_str_encode_date; extern PyObject *_CBOR2_str_Fraction; extern PyObject *_CBOR2_str_fromtimestamp; extern PyObject *_CBOR2_str_FrozenDict; extern PyObject *_CBOR2_str_getvalue; extern PyObject *_CBOR2_str_groups; extern PyObject *_CBOR2_str_ip_address; extern PyObject *_CBOR2_str_ip_network; extern PyObject *_CBOR2_str_is_infinite; extern PyObject *_CBOR2_str_is_nan; extern PyObject *_CBOR2_str_isoformat; extern PyObject *_CBOR2_str_join; extern PyObject *_CBOR2_str_match; extern PyObject *_CBOR2_str_network_address; extern PyObject *_CBOR2_str_numerator; extern PyObject *_CBOR2_str_obj; extern PyObject *_CBOR2_str_packed; extern PyObject *_CBOR2_str_Parser; extern PyObject *_CBOR2_str_parsestr; extern PyObject *_CBOR2_str_pattern; extern PyObject *_CBOR2_str_prefixlen; extern PyObject *_CBOR2_str_read; extern PyObject *_CBOR2_str_s; extern PyObject *_CBOR2_str_timestamp; extern PyObject *_CBOR2_str_timezone; extern PyObject *_CBOR2_str_update; extern PyObject *_CBOR2_str_utc; extern PyObject *_CBOR2_str_utc_suffix; extern PyObject *_CBOR2_str_UUID; extern PyObject *_CBOR2_str_write; // Exception classes extern PyObject *_CBOR2_CBORError; extern PyObject *_CBOR2_CBOREncodeError; extern PyObject *_CBOR2_CBOREncodeTypeError; extern PyObject *_CBOR2_CBOREncodeValueError; extern PyObject *_CBOR2_CBORDecodeError; extern PyObject *_CBOR2_CBORDecodeValueError; extern PyObject *_CBOR2_CBORDecodeEOF; // Global references (initialized by functions declared below) extern PyObject *_CBOR2_timezone; extern PyObject *_CBOR2_timezone_utc; extern PyObject *_CBOR2_BytesIO; extern PyObject *_CBOR2_Decimal; extern PyObject *_CBOR2_Fraction; extern PyObject *_CBOR2_FrozenDict; extern PyObject *_CBOR2_UUID; extern PyObject *_CBOR2_Parser; extern PyObject *_CBOR2_re_compile; extern PyObject *_CBOR2_datestr_re; extern PyObject *_CBOR2_ip_address; extern PyObject *_CBOR2_ip_network; // Initializers for the cached references above int _CBOR2_init_timezone_utc(void); // also handles timezone int _CBOR2_init_BytesIO(void); int _CBOR2_init_Decimal(void); int _CBOR2_init_Fraction(void); int _CBOR2_init_FrozenDict(void); int _CBOR2_init_UUID(void); int _CBOR2_init_Parser(void); int _CBOR2_init_re_compile(void); // also handles datestr_re int _CBOR2_init_ip_address(void); int init_default_encoders(void); int init_canonical_encoders(void); // Encoder registries extern PyObject *_CBOR2_default_encoders; extern PyObject *_CBOR2_canonical_encoders; cbor2-5.4.2/source/tags.c000066400000000000000000000130321413200260400151220ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include #include "structmember.h" #include "tags.h" // Constructors and destructors ////////////////////////////////////////////// static int CBORTag_traverse(CBORTagObject *self, visitproc visit, void *arg) { Py_VISIT(self->value); return 0; } static int CBORTag_clear(CBORTagObject *self) { Py_CLEAR(self->value); return 0; } // CBORTag.__del__(self) static void CBORTag_dealloc(CBORTagObject *self) { PyObject_GC_UnTrack(self); CBORTag_clear(self); Py_TYPE(self)->tp_free((PyObject *) self); } // CBORTag.__new__(cls, *args, **kwargs) static PyObject * CBORTag_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { CBORTagObject *self; self = (CBORTagObject *) type->tp_alloc(type, 0); if (self) { self->tag = 0; Py_INCREF(Py_None); self->value = Py_None; } return (PyObject *) self; } // CBORTag.__init__(self, tag=None, value=None) static int CBORTag_init(CBORTagObject *self, PyObject *args, PyObject *kwargs) { static char *keywords[] = {"tag", "value", NULL}; PyObject *tmp, *value, *tmp_tag = NULL; uint64_t tag = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO", keywords, &tmp_tag, &value)) return -1; // Raises an overflow error if it doesn't work tag = PyLong_AsUnsignedLongLong(tmp_tag); if (tag == (uint64_t)-1) { if (PyErr_Occurred()){ if (PyErr_ExceptionMatches(PyExc_OverflowError)) { PyErr_Clear(); // clear the overflow error PyErr_SetString(PyExc_TypeError, "CBORTag tags must be positive integers less than 2**64"); } // otherwise must be some other exception probably type err return -1; } // otherwise it's 2**64-1 which is fine :) } self->tag = tag; if (value) { tmp = self->value; Py_INCREF(value); self->value = value; Py_XDECREF(tmp); } return 0; } // Special methods /////////////////////////////////////////////////////////// static PyObject * CBORTag_repr(CBORTagObject *self) { PyObject *ret = NULL; if (Py_ReprEnter((PyObject *)self)) ret = PyUnicode_FromString("..."); else ret = PyUnicode_FromFormat("CBORTag(%llu, %R)", self->tag, self->value); Py_ReprLeave((PyObject *)self); return ret; } static PyObject * CBORTag_richcompare(PyObject *aobj, PyObject *bobj, int op) { PyObject *ret = NULL; CBORTagObject *a, *b; if (!(CBORTag_CheckExact(aobj) && CBORTag_CheckExact(bobj))) { Py_RETURN_NOTIMPLEMENTED; } else { a = (CBORTagObject *)aobj; b = (CBORTagObject *)bobj; if (a == b) { // Special case: both are the same object switch (op) { case Py_EQ: case Py_LE: case Py_GE: ret = Py_True; break; case Py_NE: case Py_LT: case Py_GT: ret = Py_False; break; default: assert(0); } Py_INCREF(ret); } else if (a->tag == b->tag) { // Tags are equal, rich-compare the value ret = PyObject_RichCompare(a->value, b->value, op); } else { // Tags differ; simple integer comparison switch (op) { case Py_EQ: ret = Py_False; break; case Py_NE: ret = Py_True; break; case Py_LT: ret = a->tag < b->tag ? Py_True : Py_False; break; case Py_LE: ret = a->tag <= b->tag ? Py_True : Py_False; break; case Py_GE: ret = a->tag >= b->tag ? Py_True : Py_False; break; case Py_GT: ret = a->tag > b->tag ? Py_True : Py_False; break; default: assert(0); } Py_INCREF(ret); } } return ret; } // C API ///////////////////////////////////////////////////////////////////// PyObject * CBORTag_New(uint64_t tag) { CBORTagObject *ret = NULL; ret = PyObject_GC_New(CBORTagObject, &CBORTagType); if (ret) { ret->tag = tag; Py_INCREF(Py_None); ret->value = Py_None; } return (PyObject *)ret; } int CBORTag_SetValue(PyObject *tag, PyObject *value) { PyObject *tmp; CBORTagObject *self; if (!CBORTag_CheckExact(tag)) return -1; if (!value) return -1; self = (CBORTagObject*)tag; tmp = self->value; Py_INCREF(value); self->value = value; Py_XDECREF(tmp); return 0; } // Tag class definition ////////////////////////////////////////////////////// static PyMemberDef CBORTag_members[] = { {"tag", T_ULONGLONG, offsetof(CBORTagObject, tag), 0, "the semantic tag associated with the value"}, {"value", T_OBJECT_EX, offsetof(CBORTagObject, value), 0, "the tagged value"}, {NULL} }; PyDoc_STRVAR(CBORTag__doc__, "The CBORTag class represents a semantically tagged value in a CBOR\n" "encoded stream. The :attr:`tag` attribute holds the numeric tag\n" "associated with the stored :attr:`value`.\n" ); PyTypeObject CBORTagType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_cbor2.CBORTag", .tp_doc = CBORTag__doc__, .tp_basicsize = sizeof(CBORTagObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_new = CBORTag_new, .tp_init = (initproc) CBORTag_init, .tp_dealloc = (destructor) CBORTag_dealloc, .tp_traverse = (traverseproc) CBORTag_traverse, .tp_clear = (inquiry) CBORTag_clear, .tp_members = CBORTag_members, .tp_repr = (reprfunc) CBORTag_repr, .tp_richcompare = CBORTag_richcompare, }; cbor2-5.4.2/source/tags.h000066400000000000000000000005161413200260400151320ustar00rootroot00000000000000#define PY_SSIZE_T_CLEAN #include #include typedef struct { PyObject_HEAD uint64_t tag; PyObject *value; } CBORTagObject; extern PyTypeObject CBORTagType; PyObject * CBORTag_New(uint64_t); int CBORTag_SetValue(PyObject *, PyObject *); #define CBORTag_CheckExact(op) (Py_TYPE(op) == &CBORTagType) cbor2-5.4.2/tests/000077500000000000000000000000001413200260400136635ustar00rootroot00000000000000cbor2-5.4.2/tests/conftest.py000066400000000000000000000025061413200260400160650ustar00rootroot00000000000000import platform import struct import pytest import cbor2.types import cbor2.encoder import cbor2.decoder load_exc = '' try: import _cbor2 except ModuleNotFoundError as e: if not str(e).startswith('No module'): load_exc = str(e) _cbor2 = None cpython = pytest.mark.skipif( platform.python_implementation() != "CPython" or _cbor2 is None, reason=(load_exc or "requires CPython"), ) @pytest.fixture def will_overflow(): ''' Construct an array/string/bytes length which would cause a memory error on decode. This should be less than sys.maxsize (the max integer index) ''' bit_size = struct.calcsize('P') * 8 huge_length = 1 << (bit_size - 8) return struct.pack('Q', huge_length) class Module(object): # Mock module class pass @pytest.fixture(params=[pytest.param("c", marks=cpython), "python"], scope="session") def impl(request): if request.param == "c": return _cbor2 else: # Make a mock module of cbor2 which always contains the pure Python # implementations, even if the top-level package has imported the # _cbor2 module module = Module() for source in (cbor2.types, cbor2.encoder, cbor2.decoder): for name in dir(source): setattr(module, name, getattr(source, name)) return module cbor2-5.4.2/tests/examples.cbor.b64000066400000000000000000000016201413200260400167410ustar00rootroot00000000000000s2VieXRlc4NARAECAwRCwsJoZGVjaW1hbHOFxIIiGTcrxIIiOTcq+X4A+XwA+fwAaGRpY3Rfa2V5 gaGhAgFgZmZsb2F0c4b7P/GZmZmZmZr7fjfkPIgAdZz7wBBmZmZmZmb5fAD5fgD5/ABoZnJhY3Rp b26B2B6CAgVoaW50ZWdlcnORAAEKFxgYGGQZA+gaAA9CQBsAAADo1KUQABv//////////8JJAQAA AAAAAAAAO///////////w0kBAAAAAAAAAAAgKThjOQPnZmlwYWRkcoLZAQREwAoKAdkBBFAgAQ24 haMAAAAAii4DcHM0ZWlwbmV0gtkBBaFEwKgAABgY2QEFoVAgAQ24haMAAAAAii4AAAAAGGBlcmVn ZXiB2CNtaGVsbG8gKHdvcmxkKWdzaW1wbGVzhODi8/ggaHNwZWNpYWxzhPT19vdnc3RyaW5nc4Zg YWFkSUVURmIiXGLDvGPmsLRmdGFnZ2VkgdkXcGVIZWxsb2p0aW1lc3RhbXBzh8B0MjAxMy0wMy0y MVQyMDowNDowMFrAeBsyMDEzLTAzLTIxVDIwOjA0OjAwLjM4MDg0MVrAeBkyMDEzLTAzLTIxVDIy OjA0OjAwKzAyOjAwwHQyMDEzLTAzLTIxVDIwOjA0OjAwWsB0MjAxMy0wMy0yMVQyMDowNDowMFrA eBsyMDEzLTAzLTIxVDIwOjA0OjAwLjEyMzQ1NlrAdDIwMTMtMDMtMjFUMjA6MDQ6MDBaaXR1cGxl X2tleYGhggIBYGR1dWlkgdglUF6v+si1HkgFgSd/3MeEL69qc2ltcGxlX2tleYGh+GP2anRhZ19h c19rZXmBodkXd2xub3RpbXBvcnRhbnT2aGVtYmVkZGVkgdgYSYRhYWFiYWNhZA== cbor2-5.4.2/tests/examples.json000066400000000000000000000041061413200260400163750ustar00rootroot00000000000000{ "bytes": [ "", "\u0001\u0002\u0003\u0004", "\\xc2\\xc2" ], "decimals": [ "14.123", "-14.123", NaN, Infinity, -Infinity ], "dict_key": [ { "FrozenDict({2: 1})": "" } ], "embedded": [ [ "a", "b", "c", "d" ] ], "floats": [ 1.1, 1e+300, -4.1, Infinity, NaN, -Infinity ], "fraction": [ "2/5" ], "integers": [ 0, 1, 10, 23, 24, 100, 1000, 1000000, 1000000000000, 18446744073709551615, 18446744073709551616, -18446744073709551616, -18446744073709551617, -1, -10, -100, -1000 ], "ipaddr": [ "192.10.10.1", "2001:db8:85a3::8a2e:370:7334" ], "ipnet": [ "192.168.0.0/24", "2001:db8:85a3::8a2e:0:0/96" ], "regex": [ "hello (world)" ], "simple_key": [ { "cbor_simple:99": null } ], "simples": [ "cbor_simple:0", "cbor_simple:2", "cbor_simple:19", "cbor_simple:32" ], "specials": [ false, true, null, "cbor:undef" ], "strings": [ "", "a", "IETF", "\"\\", "ü", "水" ], "tag_as_key": [ { "CBORtag:6007:notimportant": null } ], "tagged": [ { "CBORTag:6000": "Hello" } ], "timestamps": [ "2013-03-21T20:04:00+00:00", "2013-03-21T20:04:00.380841+00:00", "2013-03-21T22:04:00+02:00", "2013-03-21T20:04:00+00:00", "2013-03-21T20:04:00+00:00", "2013-03-21T20:04:00.123456+00:00", "2013-03-21T20:04:00+00:00" ], "tuple_key": [ { "(2, 1)": "" } ], "uuid": [ "urn:uuid:5eaffac8-b51e-4805-8127-7fdcc7842faf" ] } cbor2-5.4.2/tests/test_decoder.py000066400000000000000000000574561413200260400167220ustar00rootroot00000000000000import math import re from binascii import unhexlify from datetime import datetime, timedelta, timezone from decimal import Decimal from email.message import Message from fractions import Fraction from io import BytesIO from ipaddress import ip_address, ip_network import struct import sys from uuid import UUID import pytest from cbor2.types import FrozenDict def test_fp_attr(impl): with pytest.raises(ValueError): impl.CBORDecoder(None) with pytest.raises(ValueError): class A(object): pass foo = A() foo.read = None impl.CBORDecoder(foo) with BytesIO(b'foobar') as stream: decoder = impl.CBORDecoder(stream) assert decoder.fp is stream with pytest.raises(AttributeError): del decoder.fp def test_tag_hook_attr(impl): with BytesIO(b'foobar') as stream: with pytest.raises(ValueError): impl.CBORDecoder(stream, tag_hook='foo') decoder = impl.CBORDecoder(stream) tag_hook = lambda decoder, tag: None # noqa: E731 decoder.tag_hook = tag_hook assert decoder.tag_hook is tag_hook with pytest.raises(AttributeError): del decoder.tag_hook def test_object_hook_attr(impl): with BytesIO(b'foobar') as stream: with pytest.raises(ValueError): impl.CBORDecoder(stream, object_hook='foo') decoder = impl.CBORDecoder(stream) object_hook = lambda decoder, data: None # noqa: E731 decoder.object_hook = object_hook assert decoder.object_hook is object_hook with pytest.raises(AttributeError): del decoder.object_hook def test_str_errors_attr(impl): with BytesIO(b'foobar') as stream: with pytest.raises(ValueError): impl.CBORDecoder(stream, str_errors=False) with pytest.raises(ValueError): impl.CBORDecoder(stream, str_errors='foo') decoder = impl.CBORDecoder(stream) decoder.str_errors = 'replace' assert decoder.str_errors == 'replace' with pytest.raises(AttributeError): del decoder.str_errors def test_read(impl): with BytesIO(b'foobar') as stream: decoder = impl.CBORDecoder(stream) assert decoder.read(3) == b'foo' assert decoder.read(3) == b'bar' with pytest.raises(TypeError): decoder.read('foo') with pytest.raises(impl.CBORDecodeError): decoder.read(10) def test_decode_from_bytes(impl): with BytesIO(b'foobar') as stream: decoder = impl.CBORDecoder(stream) assert decoder.decode_from_bytes(b'\x01') == 1 with pytest.raises(TypeError): decoder.decode_from_bytes(u'foo') def test_immutable_attr(impl): with BytesIO(unhexlify('d917706548656c6c6f')) as stream: decoder = impl.CBORDecoder(stream) assert not decoder.immutable def tag_hook(decoder, tag): assert decoder.immutable return tag.value decoder.decode() def test_load(impl): with pytest.raises(TypeError): impl.load() with pytest.raises(TypeError): impl.loads() assert impl.loads(s=b'\x01') == 1 with BytesIO(b'\x01') as stream: assert impl.load(fp=stream) == 1 @pytest.mark.parametrize('payload, expected', [ ('00', 0), ('01', 1), ('0a', 10), ('17', 23), ('1818', 24), ('1819', 25), ('1864', 100), ('1903e8', 1000), ('1a000f4240', 1000000), ('1b000000e8d4a51000', 1000000000000), ('1bffffffffffffffff', 18446744073709551615), ('c249010000000000000000', 18446744073709551616), ('3bffffffffffffffff', -18446744073709551616), ('c349010000000000000000', -18446744073709551617), ('20', -1), ('29', -10), ('3863', -100), ('3903e7', -1000) ]) def test_integer(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected def test_invalid_integer_subtype(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(b'\x1c') assert str(exc.value).endswith('unknown unsigned integer subtype 0x1c') assert isinstance(exc, ValueError) @pytest.mark.parametrize('payload, expected', [ ('f90000', 0.0), ('f98000', -0.0), ('f93c00', 1.0), ('fb3ff199999999999a', 1.1), ('f93e00', 1.5), ('f97bff', 65504.0), ('fa47c35000', 100000.0), ('fa7f7fffff', 3.4028234663852886e+38), ('fb7e37e43c8800759c', 1.0e+300), ('f90001', 5.960464477539063e-8), ('f90400', 0.00006103515625), ('f9c400', -4.0), ('fbc010666666666666', -4.1), ('f97c00', float('inf')), ('f9fc00', float('-inf')), ('fa7f800000', float('inf')), ('faff800000', float('-inf')), ('fb7ff0000000000000', float('inf')), ('fbfff0000000000000', float('-inf')) ]) def test_float(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload', ['f97e00', 'fa7fc00000', 'fb7ff8000000000000']) def test_float_nan(impl, payload): decoded = impl.loads(unhexlify(payload)) assert math.isnan(decoded) @pytest.fixture(params=[ ('f4', False), ('f5', True), ('f6', None), ('f7', 'undefined') ], ids=['false', 'true', 'null', 'undefined']) def special_values(request, impl): payload, expected = request.param if expected == 'undefined': expected = impl.undefined return payload, expected def test_special(impl, special_values): payload, expected = special_values decoded = impl.loads(unhexlify(payload)) assert decoded is expected @pytest.mark.parametrize('payload, expected', [ ('40', b''), ('4401020304', b'\x01\x02\x03\x04'), ]) def test_binary(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload, expected', [ ('60', u''), ('6161', u'a'), ('6449455446', u'IETF'), ('62225c', u'\"\\'), ('62c3bc', u'\u00fc'), ('63e6b0b4', u'\u6c34') ]) def test_string(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload, expected', [ ('80', []), ('83010203', [1, 2, 3]), ('8301820203820405', [1, [2, 3], [4, 5]]), ('98190102030405060708090a0b0c0d0e0f101112131415161718181819', list(range(1, 26))) ]) def test_array(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload, expected', [ ('a0', {}), ('a201020304', {1: 2, 3: 4}) ]) def test_map(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload, expected', [ ('a26161016162820203', {'a': 1, 'b': [2, 3]}), ('826161a161626163', ['a', {'b': 'c'}]), ('a56161614161626142616361436164614461656145', {'a': 'A', 'b': 'B', 'c': 'C', 'd': 'D', 'e': 'E'}) ]) def test_mixed_array_map(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload, expected', [ ('5f42010243030405ff', b'\x01\x02\x03\x04\x05'), ('7f657374726561646d696e67ff', 'streaming'), ('9fff', []), ('9f018202039f0405ffff', [1, [2, 3], [4, 5]]), ('9f01820203820405ff', [1, [2, 3], [4, 5]]), ('83018202039f0405ff', [1, [2, 3], [4, 5]]), ('83019f0203ff820405', [1, [2, 3], [4, 5]]), ('9f0102030405060708090a0b0c0d0e0f101112131415161718181819ff', list(range(1, 26))), ('bf61610161629f0203ffff', {'a': 1, 'b': [2, 3]}), ('826161bf61626163ff', ['a', {'b': 'c'}]), ('bf6346756ef563416d7421ff', {'Fun': True, 'Amt': -2}), ('d901029f010203ff', {1, 2, 3}), ]) def test_streaming(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected @pytest.mark.parametrize('payload', [ '5f42010200', '7f63737472a0', ]) def test_bad_streaming_strings(impl, payload): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify(payload)) assert exc.match( r"non-(byte)?string found in indefinite length \1string") assert isinstance(exc, ValueError) @pytest.fixture(params=[ ('e0', 0), ('e2', 2), ('f3', 19), ('f820', 32), ]) def simple_value(request, impl): payload, expected = request.param return payload, expected, impl.CBORSimpleValue(expected) def test_simple_value(impl, simple_value): payload, expected, wrapped = simple_value decoded = impl.loads(unhexlify(payload)) assert decoded == expected assert decoded == wrapped def test_simple_val_as_key(impl): decoded = impl.loads(unhexlify('A1F86301')) assert decoded == {impl.CBORSimpleValue(99): 1} # # Tests for extension tags # @pytest.mark.parametrize('payload, expected', [ ('c074323031332d30332d32315432303a30343a30305a', datetime(2013, 3, 21, 20, 4, 0, tzinfo=timezone.utc)), ('c0781b323031332d30332d32315432303a30343a30302e3338303834315a', datetime(2013, 3, 21, 20, 4, 0, 380841, tzinfo=timezone.utc)), ('c07819323031332d30332d32315432323a30343a30302b30323a3030', datetime(2013, 3, 21, 22, 4, 0, tzinfo=timezone(timedelta(hours=2)))), ('c11a514b67b0', datetime(2013, 3, 21, 20, 4, 0, tzinfo=timezone.utc)), ('c11a514b67b0', datetime(2013, 3, 21, 22, 4, 0, tzinfo=timezone(timedelta(hours=2)))) ], ids=['datetime/utc', 'datetime+micro/utc', 'datetime/eet', 'timestamp/utc', 'timestamp/eet']) def test_datetime(impl, payload, expected): decoded = impl.loads(unhexlify(payload)) assert decoded == expected def test_datetime_secfrac(impl): decoded = impl.loads(b'\xc0\x78\x162018-08-02T07:00:59.1Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 100000, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x172018-08-02T07:00:59.01Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 10000, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x182018-08-02T07:00:59.001Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 1000, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x192018-08-02T07:00:59.0001Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 100, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x1a2018-08-02T07:00:59.00001Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 10, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x1b2018-08-02T07:00:59.000001Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 1, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x1c2018-08-02T07:00:59.0000001Z') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 0, tzinfo=timezone.utc) def test_datetime_secfrac_naive_float_to_int_cast(impl): # A secfrac that would have rounding errors if naively parsed as # `int(float(secfrac) * 1000000)`. decoded = impl.loads(b'\xc0\x78\x202018-08-02T07:00:59.000251+00:00') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 251, tzinfo=timezone.utc) def test_datetime_secfrac_overflow(impl): decoded = impl.loads(b'\xc0\x78\x2c2018-08-02T07:00:59.100500999999999999+00:00') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 100500, tzinfo=timezone.utc) decoded = impl.loads(b'\xc0\x78\x2c2018-08-02T07:00:59.999999999999999999+00:00') assert decoded == datetime(2018, 8, 2, 7, 0, 59, 999999, tzinfo=timezone.utc) def test_datetime_secfrac_requires_digit(impl): with pytest.raises(impl.CBORDecodeError) as excinfo: impl.loads(b'\xc0\x78\x1a2018-08-02T07:00:59.+00:00') assert isinstance(excinfo.value, ValueError) assert str(excinfo.value) == "invalid datetime string: '2018-08-02T07:00:59.+00:00'" with pytest.raises(impl.CBORDecodeError) as excinfo: impl.loads(b'\xc0\x78\x152018-08-02T07:00:59.Z') assert isinstance(excinfo.value, ValueError) assert str(excinfo.value) == "invalid datetime string: '2018-08-02T07:00:59.Z'" def test_bad_datetime(impl): with pytest.raises(impl.CBORDecodeError) as excinfo: impl.loads(unhexlify('c06b303030302d3132332d3031')) assert isinstance(excinfo.value, ValueError) assert str(excinfo.value) == "invalid datetime string: '0000-123-01'" def test_positive_bignum(impl): # Example from RFC 8949 section 3.4.3. decoded = impl.loads(unhexlify('c249010000000000000000')) assert decoded == 18446744073709551616 def test_negative_bignum(impl): decoded = impl.loads(unhexlify('c349010000000000000000')) assert decoded == -18446744073709551617 def test_fraction(impl): decoded = impl.loads(unhexlify('c48221196ab3')) assert decoded == Decimal('273.15') def test_decimal_precision(impl): decoded = impl.loads(unhexlify('c482384dc252011f1fe37d0c70ff50456ba8b891997b07d6')) assert decoded == Decimal('9.7703426561852468194804075821069770622934E-38') def test_bigfloat(impl): decoded = impl.loads(unhexlify('c5822003')) assert decoded == Decimal('1.5') def test_rational(impl): decoded = impl.loads(unhexlify('d81e820205')) assert decoded == Fraction(2, 5) def test_regex(impl): decoded = impl.loads(unhexlify('d8236d68656c6c6f2028776f726c6429')) expr = re.compile(u'hello (world)') assert decoded == expr def test_mime(impl): decoded = impl.loads(unhexlify( 'd824787b436f6e74656e742d547970653a20746578742f706c61696e3b20636861727365743d2269736f2d38' '3835392d3135220a4d494d452d56657273696f6e3a20312e300a436f6e74656e742d5472616e736665722d45' '6e636f64696e673a2071756f7465642d7072696e7461626c650a0a48656c6c6f203d413475726f')) assert isinstance(decoded, Message) assert decoded.get_payload() == 'Hello =A4uro' def test_uuid(impl): decoded = impl.loads(unhexlify('d825505eaffac8b51e480581277fdcc7842faf')) assert decoded == UUID(hex='5eaffac8b51e480581277fdcc7842faf') @pytest.mark.parametrize('payload, expected', [ ('d9010444c00a0a01', ip_address(u'192.10.10.1')), ('d901045020010db885a3000000008a2e03707334', ip_address(u'2001:db8:85a3::8a2e:370:7334')), ('d9010446010203040506', (260, b'\x01\x02\x03\x04\x05\x06')), ], ids=[ 'ipv4', 'ipv6', 'mac', ]) def test_ipaddress(impl, payload, expected): if isinstance(expected, tuple): expected = impl.CBORTag(*expected) payload = unhexlify(payload) assert impl.loads(payload) == expected def test_bad_ipaddress(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('d9010443c00a0a')) assert str(exc.value).endswith('invalid ipaddress value %r' % b'\xc0\x0a\x0a') assert isinstance(exc, ValueError) with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('d9010401')) assert str(exc.value).endswith('invalid ipaddress value 1') assert isinstance(exc, ValueError) @pytest.mark.parametrize('payload, expected', [ ('d90105a144c0a800641818', ip_network('192.168.0.100/24', False)), ('d90105a15020010db885a3000000008a2e000000001860', ip_network(u'2001:db8:85a3:0:0:8a2e::/96', False)), ], ids=[ 'ipv4', 'ipv6', ]) def test_ipnetwork(impl, payload, expected): # XXX The following pytest.skip is only included to work-around a bug in # pytest under python 3.3 (which prevents the decorator above from skipping # correctly); remove when 3.3 support is dropped payload = unhexlify(payload) assert impl.loads(payload) == expected def test_bad_ipnetwork(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('d90105a244c0a80064181844c0a800001818')) assert str(exc.value).endswith( 'invalid ipnetwork value %r' % {b'\xc0\xa8\x00d': 24, b'\xc0\xa8\x00\x00': 24}) assert isinstance(exc, ValueError) with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('d90105a144c0a80064420102')) assert str(exc.value).endswith( 'invalid ipnetwork value %r' % {b'\xc0\xa8\x00d': b'\x01\x02'}) assert isinstance(exc, ValueError) def test_bad_shared_reference(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('d81d05')) assert str(exc.value).endswith('shared reference 5 not found') assert isinstance(exc, ValueError) def test_uninitialized_shared_reference(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('D81CA1D81D014161')) assert str(exc.value).endswith('shared value 0 has not been initialized') assert isinstance(exc, ValueError) def test_immutable_shared_reference(impl): # a = (1, 2, 3) # b = ((a, a), a) # data = dumps(set(b)) decoded = impl.loads(unhexlify('d90102d81c82d81c82d81c83010203d81d02d81d02')) a = [item for item in decoded if len(item) == 3][0] b = [item for item in decoded if len(item) == 2][0] assert decoded == set(((a, a), a)) assert b[0] is a assert b[1] is a def test_cyclic_array(impl): decoded = impl.loads(unhexlify('d81c81d81d00')) assert decoded == [decoded] def test_cyclic_map(impl): decoded = impl.loads(unhexlify('d81ca100d81d00')) assert decoded == {0: decoded} def test_string_ref(impl): decoded = impl.loads(unhexlify('d9010085656669727374d81900667365636f6e64d81900d81901')) assert isinstance(decoded, list) assert decoded[0] == "first" assert decoded[1] == "first" assert decoded[2] == "second" assert decoded[3] == "first" assert decoded[4] == "second" def test_outside_string_ref_namespace(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('85656669727374d81900667365636f6e64d81900d81901')) assert str(exc.value).endswith('string reference outside of namespace') assert isinstance(exc, ValueError) def test_invalid_string_ref(impl): with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('d9010086656669727374d81900667365636f6e64d81900d81901d81903')) assert str(exc.value).endswith('string reference 3 not found') assert isinstance(exc, ValueError) @pytest.mark.parametrize('payload, expected', [ ('d9d9f71903e8', 1000), ('d9d9f7c249010000000000000000', 18446744073709551616), ], ids=['self_describe_cbor+int', 'self_describe_cbor+positive_bignum']) def test_self_describe_cbor(impl, payload, expected): assert impl.loads(unhexlify(payload)) == expected def test_unhandled_tag(impl): """ Test that a tag is simply ignored and its associated value returned if there is no special handling available for it. """ decoded = impl.loads(unhexlify('d917706548656c6c6f')) assert decoded == impl.CBORTag(6000, u'Hello') def test_premature_end_of_stream(impl): """ Test that the decoder detects a situation where read() returned fewer than expected bytes. """ with pytest.raises(impl.CBORDecodeError) as exc: impl.loads(unhexlify('437879')) exc.match(r'premature end of stream \(expected to read 3 bytes, got 2 instead\)') assert isinstance(exc, EOFError) def test_tag_hook(impl): def reverse(decoder, tag): return tag.value[::-1] decoded = impl.loads(unhexlify('d917706548656c6c6f'), tag_hook=reverse) assert decoded == u'olleH' def test_tag_hook_cyclic(impl): class DummyType(object): def __init__(self, value): self.value = value def unmarshal_dummy(decoder, tag): instance = DummyType.__new__(DummyType) decoder.set_shareable(instance) instance.value = decoder.decode_from_bytes(tag.value) return instance decoded = impl.loads(unhexlify('D81CD90BB849D81CD90BB843D81D00'), tag_hook=unmarshal_dummy) assert isinstance(decoded, DummyType) assert decoded.value.value is decoded def test_object_hook(impl): class DummyType(object): def __init__(self, state): self.state = state payload = unhexlify('A2616103616205') decoded = impl.loads(payload, object_hook=lambda decoder, value: DummyType(value)) assert isinstance(decoded, DummyType) assert decoded.state == {'a': 3, 'b': 5} def test_load_from_file(impl, tmpdir): path = tmpdir.join('testdata.cbor') path.write_binary(b'\x82\x01\x0a') with path.open('rb') as fp: obj = impl.load(fp) assert obj == [1, 10] def test_nested_exception(impl): with pytest.raises((impl.CBORDecodeError, TypeError)) as exc: impl.loads(unhexlify('A1D9177082010201')) exc.match( r"(unhashable type: '(_?cbor2\.)?CBORTag'" r"|" r"'(_?cbor2\.)?CBORTag' objects are unhashable)") assert isinstance(exc, TypeError) def test_set(impl): payload = unhexlify('d9010283616361626161') value = impl.loads(payload) assert type(value) is set assert value == {u'a', u'b', u'c'} @pytest.mark.parametrize('payload, expected', [ ('a1a1616161626163', {FrozenDict({'a': 'b'}): 'c'}), ('A1A1A10101A1666E6573746564F5A1666E6573746564F4', {FrozenDict({FrozenDict({1: 1}): FrozenDict({"nested": True})}): {"nested": False}}), ('a182010203', {(1, 2): 3}), ('a1d901028301020304', {frozenset({1, 2, 3}): 4}), ('A17f657374726561646d696e67ff01', {"streaming": 1}), ('d9010282d90102820102d90102820304', {frozenset({1, 2}), frozenset({3, 4})}) ]) def test_immutable_keys(impl, payload, expected): value = impl.loads(unhexlify(payload)) assert value == expected # Corrupted or invalid data checks def test_huge_truncated_array(impl, will_overflow): with pytest.raises(impl.CBORDecodeError): impl.loads(unhexlify('9b') + will_overflow) def test_huge_truncated_string(impl): huge_index = struct.pack('Q', sys.maxsize + 1) with pytest.raises((impl.CBORDecodeError, MemoryError)): impl.loads(unhexlify('7b') + huge_index + unhexlify('70717273')) @pytest.mark.parametrize('dtype_prefix', ['7B', '5b'], ids=['string', 'bytes']) def test_huge_truncated_data(impl, dtype_prefix, will_overflow): with pytest.raises((impl.CBORDecodeError, MemoryError)): impl.loads(unhexlify(dtype_prefix) + will_overflow) @pytest.mark.parametrize('tag_dtype', ['7F7B', '5f5B'], ids=['string', 'bytes']) def test_huge_truncated_indefinite_data(impl, tag_dtype, will_overflow): huge_index = struct.pack('Q', sys.maxsize + 1) with pytest.raises((impl.CBORDecodeError, MemoryError)): impl.loads(unhexlify(tag_dtype) + huge_index + unhexlify('70717273ff')) @pytest.mark.parametrize('data', ['7f61777f6177ffff', '5f41775f4177ffff'], ids=['string', 'bytes']) def test_embedded_indefinite_data(impl, data): with pytest.raises(impl.CBORDecodeValueError): impl.loads(unhexlify(data)) @pytest.mark.parametrize('data', ['7f01ff', '5f01ff'], ids=['string', 'bytes']) def test_invalid_indefinite_data_item(impl, data): with pytest.raises(impl.CBORDecodeValueError): impl.loads(unhexlify(data)) @pytest.mark.parametrize('data', [ '7f7bff0000000000000471717272ff', '5f5bff0000000000000471717272ff' ], ids=['string', 'bytes'] ) def test_indefinite_overflow(impl, data): with pytest.raises(impl.CBORDecodeValueError): impl.loads(unhexlify(data)) def test_invalid_cbor(impl): with pytest.raises(impl.CBORDecodeError): impl.loads(unhexlify( 'c788370016b8965bdb2074bff82e5a20e09bec21f8406e86442b87ec3ff245b70a47624dc9cdc6824b2a' '4c52e95ec9d6b0534b71c2b49e4bf9031500cee6869979c297bb5a8b381e98db714108415e5c50db7897' '4c271579b01633a3ef6271be5c225eb2' ) ) @pytest.mark.parametrize('data, expected', [ ('fc', '1c'), ('fd', '1d'), ('fe', '1e') ], ) def test_reserved_special_tags(impl, data, expected): with pytest.raises(impl.CBORDecodeValueError) as exc_info: impl.loads(unhexlify(data)) assert exc_info.value.args[0] == "Undefined Reserved major type 7 subtype 0x" + expected @pytest.mark.parametrize('data, expected', [ ('c400', '4'), ('c500', '5') ], ) def test_decimal_payload_unpacking(impl, data, expected): with pytest.raises(impl.CBORDecodeValueError) as exc_info: impl.loads(unhexlify(data)) assert exc_info.value.args[0] == f"Incorrect tag {expected} payload" cbor2-5.4.2/tests/test_encoder.py000066400000000000000000000414051413200260400167170ustar00rootroot00000000000000import re from io import BytesIO from binascii import unhexlify from collections import OrderedDict from datetime import datetime, timedelta, date, timezone from decimal import Decimal from email.mime.text import MIMEText from fractions import Fraction from ipaddress import ip_address, ip_network from uuid import UUID import pytest from cbor2 import shareable_encoder from cbor2.types import FrozenDict def test_fp_attr(impl): with pytest.raises(ValueError): impl.CBOREncoder(None) with pytest.raises(ValueError): class A(object): pass foo = A() foo.write = None impl.CBOREncoder(foo) with BytesIO() as stream: encoder = impl.CBOREncoder(stream) assert encoder.fp is stream with pytest.raises(AttributeError): del encoder.fp def test_default_attr(impl): with BytesIO() as stream: encoder = impl.CBOREncoder(stream) assert encoder.default is None with pytest.raises(ValueError): encoder.default = 1 with pytest.raises(AttributeError): del encoder.default def test_timezone_attr(impl): with BytesIO() as stream: encoder = impl.CBOREncoder(stream) assert encoder.timezone is None with pytest.raises(ValueError): encoder.timezone = 1 with pytest.raises(AttributeError): del encoder.timezone def test_write(impl): with BytesIO() as stream: encoder = impl.CBOREncoder(stream) encoder.write(b'foo') assert stream.getvalue() == b'foo' with pytest.raises(TypeError): encoder.write(1) def test_encoders_load_type(impl): with BytesIO() as stream: encoder = impl.CBOREncoder(stream) encoder._encoders[(1, 2, 3)] = lambda self, value: None with pytest.raises(ValueError) as exc: encoder.encode(object()) assert str(exc.value).endswith( 'invalid deferred encoder type (1, 2, 3) (must be a 2-tuple ' "of module name and type name, e.g. ('collections', " "'defaultdict'))") def test_encode_length(impl): # This test is purely for coverage in the C variant with BytesIO() as stream: encoder = impl.CBOREncoder(stream) encoder.encode_length(0, 1) assert stream.getvalue() == b'\x01' def test_canonical_attr(impl): # Another test purely for coverage in the C variant with BytesIO() as stream: enc = impl.CBOREncoder(stream) assert not enc.canonical enc = impl.CBOREncoder(stream, canonical=True) assert enc.canonical def test_dump(impl): with pytest.raises(TypeError): impl.dump() with pytest.raises(TypeError): impl.dumps() assert impl.dumps(obj=1) == b'\x01' with BytesIO() as stream: impl.dump(fp=stream, obj=1) assert stream.getvalue() == b'\x01' @pytest.mark.parametrize('value, expected', [ (0, '00'), (1, '01'), (10, '0a'), (23, '17'), (24, '1818'), (100, '1864'), (1000, '1903e8'), (1000000, '1a000f4240'), (1000000000000, '1b000000e8d4a51000'), (18446744073709551615, '1bffffffffffffffff'), (18446744073709551616, 'c249010000000000000000'), (-18446744073709551616, '3bffffffffffffffff'), (-18446744073709551617, 'c349010000000000000000'), (-1, '20'), (-10, '29'), (-100, '3863'), (-1000, '3903e7') ]) def test_integer(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected @pytest.mark.parametrize('value, expected', [ (1.1, 'fb3ff199999999999a'), (1.0e+300, 'fb7e37e43c8800759c'), (-4.1, 'fbc010666666666666'), (float('inf'), 'f97c00'), (float('nan'), 'f97e00'), (float('-inf'), 'f9fc00') ]) def test_float(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected @pytest.mark.parametrize('value, expected', [ (b'', '40'), (b'\x01\x02\x03\x04', '4401020304'), ]) def test_bytestring(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected def test_bytearray(impl): expected = unhexlify('4401020304') assert impl.dumps(bytearray(b'\x01\x02\x03\x04')) == expected @pytest.mark.parametrize('value, expected', [ (u'', '60'), (u'a', '6161'), (u'IETF', '6449455446'), (u'"\\', '62225c'), (u'\u00fc', '62c3bc'), (u'\u6c34', '63e6b0b4') ]) def test_string(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected @pytest.fixture(params=[ (False, 'f4'), (True, 'f5'), (None, 'f6'), ('undefined', 'f7') ], ids=['false', 'true', 'null', 'undefined']) def special_values(request, impl): value, expected = request.param if value == 'undefined': value = impl.undefined return value, expected def test_special(impl, special_values): value, expected = special_values expected = unhexlify(expected) assert impl.dumps(value) == expected @pytest.fixture(params=[ (0, 'e0'), (2, 'e2'), (19, 'f3'), (32, 'f820') ]) def simple_values(request, impl): value, expected = request.param return impl.CBORSimpleValue(value), expected def test_simple_value(impl, simple_values): value, expected = simple_values expected = unhexlify(expected) assert impl.dumps(value) == expected def test_simple_val_as_key(impl): payload = {impl.CBORSimpleValue(99): 1} result = impl.dumps(payload) assert result == unhexlify('A1F86301') # # Tests for extension tags # @pytest.mark.parametrize('value, as_timestamp, expected', [ (datetime(2013, 3, 21, 20, 4, 0, tzinfo=timezone.utc), False, 'c074323031332d30332d32315432303a30343a30305a'), (datetime(2013, 3, 21, 20, 4, 0, 380841, tzinfo=timezone.utc), False, 'c0781b323031332d30332d32315432303a30343a30302e3338303834315a'), (datetime(2013, 3, 21, 22, 4, 0, tzinfo=timezone(timedelta(hours=2))), False, 'c07819323031332d30332d32315432323a30343a30302b30323a3030'), (datetime(2013, 3, 21, 20, 4, 0), False, 'c074323031332d30332d32315432303a30343a30305a'), (datetime(2013, 3, 21, 20, 4, 0, tzinfo=timezone.utc), True, 'c11a514b67b0'), (datetime(2013, 3, 21, 20, 4, 0, 123456, tzinfo=timezone.utc), True, 'c1fb41d452d9ec07e6b4'), (datetime(2013, 3, 21, 22, 4, 0, tzinfo=timezone(timedelta(hours=2))), True, 'c11a514b67b0') ], ids=[ 'datetime/utc', 'datetime+micro/utc', 'datetime/eet', 'naive', 'timestamp/utc', 'timestamp+micro/utc', 'timestamp/eet' ]) def test_datetime(impl, value, as_timestamp, expected): expected = unhexlify(expected) assert impl.dumps(value, datetime_as_timestamp=as_timestamp, timezone=timezone.utc) == expected @pytest.mark.parametrize('tz', [None, timezone.utc], ids=['no timezone', 'utc']) def test_date_fails(impl, tz): encoder = impl.CBOREncoder(BytesIO(b''), timezone=tz, date_as_datetime=False) assert date not in encoder._encoders with pytest.raises(impl.CBOREncodeError) as exc: encoder.encode(date(2013, 3, 21)) assert isinstance(exc, ValueError) def test_date_as_datetime(impl): expected = unhexlify('c074323031332d30332d32315430303a30303a30305a') assert impl.dumps(date(2013, 3, 21), timezone=timezone.utc, date_as_datetime=True) == expected def test_naive_datetime(impl): """Test that naive datetimes are gracefully rejected when no timezone has been set.""" with pytest.raises(impl.CBOREncodeError) as exc: impl.dumps(datetime(2013, 3, 21)) exc.match('naive datetime datetime.datetime(2013, 3, 21) encountered ' 'and no default timezone has been set') assert isinstance(exc, ValueError) @pytest.mark.parametrize('value, expected', [ (Decimal('14.123'), 'c4822219372b'), (Decimal('-14.123'), 'C4822239372A'), (Decimal('NaN'), 'f97e00'), (Decimal('Infinity'), 'f97c00'), (Decimal('-Infinity'), 'f9fc00') ], ids=['normal', 'negative', 'nan', 'inf', 'neginf']) def test_decimal(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected def test_rational(impl): expected = unhexlify('d81e820205') assert impl.dumps(Fraction(2, 5)) == expected def test_regex(impl): expected = unhexlify('d8236d68656c6c6f2028776f726c6429') assert impl.dumps(re.compile(u'hello (world)')) == expected def test_mime(impl): expected = unhexlify( 'd824787b436f6e74656e742d547970653a20746578742f706c61696e3b20636861727365743d2269736f2d38' '3835392d3135220a4d494d452d56657273696f6e3a20312e300a436f6e74656e742d5472616e736665722d456' 'e636f64696e673a2071756f7465642d7072696e7461626c650a0a48656c6c6f203d413475726f') message = MIMEText(u'Hello \u20acuro', 'plain', 'iso-8859-15') assert impl.dumps(message) == expected def test_uuid(impl): expected = unhexlify('d825505eaffac8b51e480581277fdcc7842faf') assert impl.dumps(UUID(hex='5eaffac8b51e480581277fdcc7842faf')) == expected @pytest.mark.parametrize('value, expected', [ (ip_address(u'192.10.10.1'), 'd9010444c00a0a01'), (ip_address(u'2001:db8:85a3::8a2e:370:7334'), 'd901045020010db885a3000000008a2e03707334'), ], ids=[ 'ipv4', 'ipv6', ]) def test_ipaddress(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected @pytest.mark.parametrize('value, expected', [ (ip_network(u'192.168.0.100/24', False), 'd90105a144c0a800001818'), (ip_network(u'2001:db8:85a3:0:0:8a2e::/96', False), 'd90105a15020010db885a3000000008a2e000000001860'), ], ids=[ 'ipv4', 'ipv6', ]) def test_ipnetwork(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value) == expected def test_custom_tag(impl): expected = unhexlify('d917706548656c6c6f') assert impl.dumps(impl.CBORTag(6000, u'Hello')) == expected def test_cyclic_array(impl): """Test that an array that contains itself can be serialized with value sharing enabled.""" expected = unhexlify('d81c81d81c81d81d00') a = [[]] a[0].append(a) assert impl.dumps(a, value_sharing=True) == expected def test_cyclic_array_nosharing(impl): """Test that serializing a cyclic structure w/o value sharing will blow up gracefully.""" a = [] a.append(a) with pytest.raises(impl.CBOREncodeError) as exc: impl.dumps(a) exc.match('cyclic data structure detected but value sharing is disabled') assert isinstance(exc, ValueError) def test_cyclic_map(impl): """Test that a dict that contains itself can be serialized with value sharing enabled.""" expected = unhexlify('d81ca100d81d00') a = {} a[0] = a assert impl.dumps(a, value_sharing=True) == expected def test_cyclic_map_nosharing(impl): """Test that serializing a cyclic structure w/o value sharing will fail gracefully.""" a = {} a[0] = a with pytest.raises(impl.CBOREncodeError) as exc: impl.dumps(a) exc.match('cyclic data structure detected but value sharing is disabled') assert isinstance(exc, ValueError) @pytest.mark.parametrize('value_sharing, expected', [ (False, '828080'), (True, 'd81c82d81c80d81d01') ], ids=['nosharing', 'sharing']) def test_not_cyclic_same_object(impl, value_sharing, expected): """Test that the same shareable object can be included twice if not in a cyclic structure.""" expected = unhexlify(expected) a = [] b = [a, a] assert impl.dumps(b, value_sharing=value_sharing) == expected def test_unsupported_type(impl): with pytest.raises(impl.CBOREncodeError) as exc: impl.dumps(lambda: None) exc.match('cannot serialize type function') assert isinstance(exc, TypeError) def test_default(impl): class DummyType(object): def __init__(self, state): self.state = state def default_encoder(encoder, value): encoder.encode(value.state) expected = unhexlify('820305') obj = DummyType([3, 5]) serialized = impl.dumps(obj, default=default_encoder) assert serialized == expected def test_default_cyclic(impl): class DummyType(object): def __init__(self, value=None): self.value = value @shareable_encoder def default_encoder(encoder, value): state = encoder.encode_to_bytes(value.value) encoder.encode(impl.CBORTag(3000, state)) expected = unhexlify('D81CD90BB849D81CD90BB843D81D00') obj = DummyType() obj2 = DummyType(obj) obj.value = obj2 serialized = impl.dumps(obj, value_sharing=True, default=default_encoder) assert serialized == expected def test_dump_to_file(impl, tmpdir): path = tmpdir.join('testdata.cbor') with path.open('wb') as fp: impl.dump([1, 10], fp) assert path.read_binary() == b'\x82\x01\x0a' @pytest.mark.parametrize('value, expected', [ ({}, 'a0'), (OrderedDict([(b'a', b''), (b'b', b'')]), 'A2416140416240'), (OrderedDict([(b'b', b''), (b'a', b'')]), 'A2416140416240'), (OrderedDict([(u'a', u''), (u'b', u'')]), 'a2616160616260'), (OrderedDict([(u'b', u''), (u'a', u'')]), 'a2616160616260'), (OrderedDict([(b'00001', u''), (b'002', u'')]), 'A2433030326045303030303160'), (OrderedDict([(255, 0), (2, 0)]), 'a2020018ff00'), (FrozenDict([(b'a', b''), (b'b', b'')]), 'A2416140416240') ], ids=['empty', 'bytes in order', 'bytes out of order', 'text in order', 'text out of order', 'byte length', 'integer keys', 'frozendict']) def test_ordered_map(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value, canonical=True) == expected @pytest.mark.parametrize('value, expected', [ (3.5, 'F94300'), (100000.0, 'FA47C35000'), (3.8, 'FB400E666666666666'), (float('inf'), 'f97c00'), (float('nan'), 'f97e00'), (float('-inf'), 'f9fc00'), (float.fromhex('0x1.0p-24'), 'f90001'), (float.fromhex('0x1.4p-24'), 'fa33a00000'), (float.fromhex('0x1.ff8p-63'), 'fa207fc000'), (1e300, 'fb7e37e43c8800759c') ], ids=['float 16', 'float 32', 'float 64', 'inf', 'nan', '-inf', 'float 16 minimum positive subnormal', 'mantissa o/f to 32', 'exponent o/f to 32', 'oversize float']) def test_minimal_floats(impl, value, expected): expected = unhexlify(expected) assert impl.dumps(value, canonical=True) == expected def test_tuple_key(impl): assert impl.dumps({(2, 1): u''}) == unhexlify('a182020160') def test_dict_key(impl): assert impl.dumps({FrozenDict({2: 1}): u''}) == unhexlify('a1a1020160') @pytest.mark.parametrize('frozen', [False, True], ids=['set', 'frozenset']) def test_set(impl, frozen): value = {u'a', u'b', u'c'} if frozen: value = frozenset(value) serialized = impl.dumps(value) assert len(serialized) == 10 assert serialized.startswith(unhexlify('d9010283')) @pytest.mark.parametrize('frozen', [False, True], ids=['set', 'frozenset']) def test_canonical_set(impl, frozen): value = {u'y', u'x', u'aa', u'a'} if frozen: value = frozenset(value) serialized = impl.dumps(value, canonical=True) assert serialized == unhexlify('d9010284616161786179626161') @pytest.mark.parametrize('value', [ u'', u'a', u'abcde', b'\x01\x02\x03\x04', [u'a', u'bb', u'a', u'bb'], [u'a', u'bb', u'ccc', u'dddd', u'a', u'bb'], {u'a': u'm', u'bb': u'nn', u'e': u'm', u'ff': u'nn'}, {u'a': u'm', u'bb': u'nn', u'ccc': u'ooo', u'dddd': u'pppp', u'e': u'm', u'ff': u'nn'}, ], ids=['empty string', 'short string', 'long string', 'bytestring', 'array of short strings', 'no repeated long strings', 'dict with short keys and strings', 'dict with no repeated long strings']) def test_encode_stringrefs_unchanged(impl, value): expected = impl.dumps(value) if isinstance(value, list) or isinstance(value, dict): expected = b'\xd9\x01\x00' + expected assert impl.dumps(value, string_referencing=True) == expected def test_encode_stringrefs_array(impl): value = [u'aaaa', u'aaaa', u'bbbb', u'aaaa', u'bbbb'] equivalent = [u'aaaa', impl.CBORTag(25, 0), u'bbbb', impl.CBORTag(25, 0), impl.CBORTag(25, 1)] assert impl.dumps(value, string_referencing=True) == b'\xd9\x01\x00' + impl.dumps(equivalent) def test_encode_stringrefs_dict(impl): value = {u'aaaa': u'mmmm', u'bbbb': u'bbbb', u'cccc': u'aaaa', u'mmmm': u'aaaa'} expected = unhexlify( 'd90100' 'a4' '6461616161' '646d6d6d6d' '6462626262' 'd81902' '6463636363' 'd81900' 'd81901' 'd81900' ) assert impl.dumps(value, string_referencing=True, canonical=True) == expected @pytest.mark.parametrize('tag', [-1, 2**64, 'f'], ids=['too small', 'too large', 'wrong type']) def test_invalid_tag(impl, tag): with pytest.raises(TypeError): impl.dumps(impl.CBORTag(tag, 'value')) def test_largest_tag(impl): expected = unhexlify('dbffffffffffffffff6176') assert impl.dumps(impl.CBORTag(2**64-1, 'v')) == expected cbor2-5.4.2/tests/test_tool.py000066400000000000000000000100361413200260400162510ustar00rootroot00000000000000import pytest import binascii import json from io import BytesIO, TextIOWrapper import cbor2.tool @pytest.mark.parametrize( 'value, expected', [ ((1, 2, 3), [1, 2, 3]), ({b"\x01\x02\x03": "b"}, {"\x01\x02\x03": "b"}), ({"dict": {"b": 17}}, {"dict": {"b": 17}}), ], ids=['tuple', 'byte_key', 'recursion'], ) def test_key_to_str(value, expected): assert cbor2.tool.key_to_str(value) == expected def test_default(): with pytest.raises(TypeError): json.dumps(BytesIO(b''), cls=cbor2.tool.DefaultEncoder) @pytest.mark.parametrize( 'payload', ["D81CA16162D81CA16161D81D00", "d81c81d81c830102d81d00"], ids=['dict', 'list'] ) def test_self_referencing(payload): decoded = cbor2.loads(binascii.unhexlify(payload)) with pytest.raises(ValueError, match="Cannot convert self-referential data to JSON"): cbor2.tool.key_to_str(decoded) def test_nonrecursive_ref(): payload = 'd81c83d81ca26162d81ca16161016163d81d02d81d01d81d01' decoded = cbor2.loads(binascii.unhexlify(payload)) result = cbor2.tool.key_to_str(decoded) expected = [ {"b": {"a": 1}, "c": {"a": 1}}, {"b": {"a": 1}, "c": {"a": 1}}, {"b": {"a": 1}, "c": {"a": 1}}, ] assert result == expected def test_stdin(monkeypatch, tmpdir): f = tmpdir.join('outfile') argv = ['-o', str(f)] inbuf = TextIOWrapper(BytesIO(binascii.unhexlify('02'))) with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) m.setattr('sys.stdin', inbuf) cbor2.tool.main() assert f.read() == '2\n' def test_stdout(monkeypatch, tmpdir): argv = ['-o', '-'] inbuf = TextIOWrapper(BytesIO(binascii.unhexlify('02'))) outbuf = BytesIO() with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) m.setattr('sys.stdin', inbuf) m.setattr('sys.stdout', outbuf) cbor2.tool.main() def test_readfrom(monkeypatch, tmpdir): f = tmpdir.join('infile') outfile = tmpdir.join('outfile') f.write_binary(binascii.unhexlify('02')) argv = ['-o', str(outfile), str(f)] with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) cbor2.tool.main() assert outfile.read() == '2\n' def test_b64(monkeypatch, tmpdir): f = tmpdir.join('outfile') argv = ['-d', '-o', str(f)] inbuf = TextIOWrapper(BytesIO(b'oQID')) with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) m.setattr('sys.stdin', inbuf) cbor2.tool.main() assert f.read() == '{"2": 3}\n' def test_stream(monkeypatch, tmpdir): f = tmpdir.join('outfile') argv = ['--sequence', '-o', str(f)] inbuf = TextIOWrapper(BytesIO(binascii.unhexlify('0203'))) with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) m.setattr('sys.stdin', inbuf) cbor2.tool.main() assert f.read() == '2\n3\n' def test_embed_bytes(monkeypatch, tmpdir): f = tmpdir.join('outfile') argv = ['-o', str(f)] inbuf = TextIOWrapper(BytesIO(binascii.unhexlify('42C2C2'))) with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) m.setattr('sys.stdin', inbuf) cbor2.tool.main() assert f.read() == '"\\\\xc2\\\\xc2"\n' def test_dtypes_from_file(monkeypatch, tmpdir): infile = 'tests/examples.cbor.b64' expected = open('tests/examples.json', 'r').read() outfile = tmpdir.join('outfile.json') argv = ['--sort-keys', '--pretty', '-d', '-o', str(outfile), infile] with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) cbor2.tool.main() assert outfile.read() == expected def test_ignore_tag(monkeypatch, tmpdir): f = tmpdir.join('outfile') argv = ['-o', str(f), '-i', '6000'] inbuf = TextIOWrapper(BytesIO(binascii.unhexlify('D917706548656C6C6F'))) expected = '"Hello"\n' with monkeypatch.context() as m: m.setattr('sys.argv', [''] + argv) m.setattr('sys.stdin', inbuf) cbor2.tool.main() assert f.read() == expected cbor2-5.4.2/tests/test_types.py000066400000000000000000000060151413200260400164420ustar00rootroot00000000000000import pytest from cbor2.types import FrozenDict def test_undefined_bool(impl): assert not impl.undefined def test_undefined_repr(impl): assert repr(impl.undefined) == 'undefined' def test_undefined_singleton(impl): assert type(impl.undefined)() is impl.undefined def test_undefined_init(impl): with pytest.raises(TypeError): type(impl.undefined)('foo') def test_break_bool(impl): assert impl.break_marker def test_break_repr(impl): assert repr(impl.break_marker) == 'break_marker' def test_break_singleton(impl): assert type(impl.break_marker)() is impl.break_marker def test_break_init(impl): with pytest.raises(TypeError): type(impl.break_marker)('foo') def test_tag_init(impl): with pytest.raises(TypeError): impl.CBORTag('foo', 'bar') def test_tag_attr(impl): tag = impl.CBORTag(1, 'foo') assert tag.tag == 1 assert tag.value == 'foo' def test_tag_compare(impl): tag1 = impl.CBORTag(1, 'foo') tag2 = impl.CBORTag(1, 'foo') tag3 = impl.CBORTag(2, 'bar') tag4 = impl.CBORTag(2, 'baz') assert tag1 is not tag2 assert tag1 == tag2 assert not (tag1 == tag3) assert tag1 != tag3 assert tag3 >= tag2 assert tag3 > tag2 assert tag2 < tag3 assert tag2 <= tag3 assert tag4 >= tag3 assert tag4 > tag3 assert tag3 < tag4 assert tag3 <= tag4 def test_tag_compare_unimplemented(impl): tag = impl.CBORTag(1, 'foo') assert not tag == (1, 'foo') with pytest.raises(TypeError): tag <= (1, 'foo') def test_tag_recursive(impl): tag = impl.CBORTag(1, None) tag.value = tag assert repr(tag) == 'CBORTag(1, ...)' assert tag is tag.value assert tag == tag.value assert not (tag != tag.value) def test_tag_repr(impl): assert repr(impl.CBORTag(600, 'blah')) == "CBORTag(600, 'blah')" def test_simple_value_repr(impl): assert repr(impl.CBORSimpleValue(1)) == "CBORSimpleValue(value=1)" def test_simple_value_equals(impl): tag1 = impl.CBORSimpleValue(1) tag2 = impl.CBORSimpleValue(1) tag3 = impl.CBORSimpleValue(21) tag4 = impl.CBORSimpleValue(99) assert tag1 == tag2 assert tag1 == 1 assert not tag2 == "21" assert tag1 != tag3 assert tag1 != 21 assert tag2 != "21" assert tag4 > tag1 assert tag4 >= tag3 assert 99 <= tag4 assert 100 > tag4 assert tag4 <= 100 assert 2 < tag4 assert tag4 >= 99 assert tag1 <= tag4 def test_simple_ordering(impl): randints = [9, 7, 3, 8, 4, 0, 2, 5, 6, 1] expected = [impl.CBORSimpleValue(v) for v in range(10)] disordered = [impl.CBORSimpleValue(v) for v in randints] assert expected == sorted(disordered) assert expected == sorted(randints) def test_simple_value_too_big(impl): with pytest.raises(TypeError) as exc: impl.CBORSimpleValue(256) assert str(exc.value) == 'simple value out of range (0..255)' def test_frozendict(): assert len(FrozenDict({1: 2, 3: 4})) == 2 assert repr(FrozenDict({1: 2})) == "FrozenDict({1: 2})" cbor2-5.4.2/tox.ini000066400000000000000000000003761413200260400140420ustar00rootroot00000000000000[tox] envlist = py36, py37, py38, py39, pypy3, flake8 skip_missing_interpreters = true isolated_build = true [testenv] commands = python -m pytest {posargs} extras = test [testenv:flake8] deps = flake8 commands = flake8 cbor2 tests skip_install = true