pax_global_header00006660000000000000000000000064144164635170014525gustar00rootroot0000000000000052 comment=4b7bc78f320d8bfb77b6793b64c88b0f22ce64c8 bpack-1.1.0/000077500000000000000000000000001441646351700126045ustar00rootroot00000000000000bpack-1.1.0/.github/000077500000000000000000000000001441646351700141445ustar00rootroot00000000000000bpack-1.1.0/.github/workflows/000077500000000000000000000000001441646351700162015ustar00rootroot00000000000000bpack-1.1.0/.github/workflows/ci.yml000066400000000000000000000015411441646351700173200ustar00rootroot00000000000000name: CI on: [push, pull_request] jobs: build: name: Build ${{ matrix.os }} ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: matrix: # os: [ubuntu-latest, macos-latest, windows-latest] os: [ubuntu-latest] python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] include: - os: windows-latest python-version: '3.x' - os: macos-latest python-version: '3.x' steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install .[test] - name: Test with pytest run: | python -m pytest -v -W=error bpack-1.1.0/.github/workflows/docs.yml000066400000000000000000000016611441646351700176600ustar00rootroot00000000000000name: Docs on: [push, pull_request] jobs: build-docs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.x uses: actions/setup-python@v4 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install .[test,docs] python -m pip install sphinxcontrib-spelling - name: Build docs run: | mkdir -p docs/_static python3 -m sphinx -W -b html docs docs/_build/html - name: Link check run: | mkdir -p docs/_static python3 -m sphinx -W -b linkcheck docs docs/_build/linkcheck - name: Doctest run: | mkdir -p docs/_static python3 -m sphinx -W -b doctest docs docs/_build/doctest - name: Spell check run: | mkdir -p docs/_static python3 -m sphinx -W -b spelling docs docs/_build/spelling bpack-1.1.0/.github/workflows/lint.yml000066400000000000000000000014071441646351700176740ustar00rootroot00000000000000name: Lint on: [push, pull_request] jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.x uses: actions/setup-python@v4 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install .[test] python -m pip install flake8 pydocstyle isort black - name: Lint with flake8 run: | python -m flake8 --count --show-source --statistics bpack - name: Lint with pydocstyle run: | python -m pydocstyle --count bpack - name: Lint with isort run: | python -m isort --check bpack - name: Lint with black run: | python -m black --check bpack bpack-1.1.0/.gitignore000066400000000000000000000034561441646351700146040ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # PyCharm /.idea # VS Code /.vscode bpack-1.1.0/.readthedocs.yml000066400000000000000000000002141441646351700156670ustar00rootroot00000000000000version: 2 build: image: latest python: install: - method: pip path: . extra_requirements: - docs formats: []bpack-1.1.0/LICENSE000066400000000000000000000261351441646351700136200ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. bpack-1.1.0/MANIFEST.in000066400000000000000000000004441441646351700143440ustar00rootroot00000000000000exclude .gitignore prune .github include requirements-dev.txt tox.ini include docs/conf.py docs/Makefile docs/make.bat docs/spelling_wordlist.txt recursive-include docs *.rst *.rst_t # recursive-include examples *.py *.rst *.txt include examples/s1isp.py include examples/packet_decoding.py bpack-1.1.0/Makefile000066400000000000000000000023741441646351700142520ustar00rootroot00000000000000#!/usr/bin/make -f PYTHON=python3 .PHONY: default help dist check clean distclean api lint default: help help: @echo "Usage: make " @echo "Available targets:" @echo " help - print this help message" @echo " dist - generate the distribution packages (source and wheel)" @echo " check - run a full test (using tox)" @echo " clean - clean build artifacts" @echo " distclean - clean all generated and cache files" @echo " api - update the API source files in the documentation" @echo " lint - perform check with code linter (flake8, black)" dist: $(PYTHON) -m build $(PYTHON) -m twine check dist/* check: $(PYTHON) -m tox clean: $(MAKE) -C docs clean $(RM) -r build bpack.egg-info docs/_build $(RM) -r __pycache__ */__pycache__ */*/__pycache__ distclean: clean $(RM) -r dist .coverage htmlcov .pytest_cache .tox @# $(RM) .DS_Store */.DS_Store */*/.DS_Store @# $(RM) -r .idea api: $(RM) -r docs/api sphinx-apidoc --module-first --separate --no-toc --doc-project "bpack API" \ -o docs/api --templatedir docs/_templates/apidoc bpack bpack/tests lint: $(PYTHON) -m flake8 --count --statistics bpack $(PYTHON) -m pydocstyle --count bpack $(PYTHON) -m isort --check bpack $(PYTHON) -m black --check bpack bpack-1.1.0/README.rst000066400000000000000000000032671441646351700143030ustar00rootroot00000000000000Binary data structures (un-)Packing library =========================================== :Copyright: 2020-2023, Antonio Valentino .. badges |PyPI Status| |GHA Status| |Documentation Status| .. |PyPI Status| image:: https://img.shields.io/pypi/v/bpack.svg :target: https://pypi.org/project/bpack :alt: PyPI Status .. |GHA Status| image:: https://github.com/avalentino/bpack/workflows/Build/badge.svg :target: https://github.com/avalentino/bpack/actions :alt: GitHub Actions Status .. |Documentation Status| image:: https://readthedocs.org/projects/bpack/badge/?version=latest :target: https://bpack.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status .. description The *bpack* Python package provides tools to describe and encode/decode binary data. Binary data are assumed to be organized in *records*, each composed by a sequence of fields. Fields are characterized by a known size, offset (w.r.t. the beginning of the record) and datatype. The package provides classes and functions that can be used to: * describe binary data structures in a declarative way (structures can be specified up to the bit level) * automatically generate encoders/decoders for a specified data descriptor Encoders/decoders (*backends*) rely on well known Python packages like: * |struct| (form the standard library) * bitstruct_ (optional) * numpy_ (optional) * bitarray_ (optional) - partial implementation .. _struct: https://docs.python.org/3/library/struct.html .. _bitstruct: https://github.com/eerimoq/bitstruct .. _numpy: https://numpy.org .. _bitarray: https://github.com/ilanschnell/bitarray .. local-definitions .. |struct| replace:: struct_ bpack-1.1.0/bpack/000077500000000000000000000000001441646351700136645ustar00rootroot00000000000000bpack-1.1.0/bpack/__init__.py000066400000000000000000000007231441646351700157770ustar00rootroot00000000000000"""Binary data structures (un-)Packing library. bpack provides tools to describe, in a *declarative* way, and encode/decode binary data. """ from .enums import EBaseUnits, EByteOrder, EBitOrder # noqa: F401 from .typing import T # noqa: F401 from .descriptors import ( # noqa: F401 descriptor, field, fields, asdict, astuple, is_descriptor, is_field, baseunits, byteorder, bitorder, calcsize, ) __version__ = "1.1.0" bpack-1.1.0/bpack/ba.py000066400000000000000000000135631441646351700146300ustar00rootroot00000000000000"""Bitarray based codec for binary data structures.""" import struct import itertools from typing import Any, Callable, Optional import bitarray from bitarray.util import ba2int import bpack import bpack.utils import bpack.codecs from .enums import EBaseUnits, EBitOrder, EByteOrder from .descriptors import field_descriptors __all__ = ["Decoder", "decoder", "BACKEND_NAME", "BACKEND_TYPE"] BACKEND_NAME = "bitarray" BACKEND_TYPE = EBaseUnits.BITS FactoryType = Callable[[bitarray.bitarray], Any] def ba_to_float_factory( size, byteorder: str = ">", bitorder: str = "big" ) -> FactoryType: """Convert a bitarray into a float.""" assert bitorder == "big" if size == 16: fmt = f"{byteorder}e" elif size == 32: fmt = f"{byteorder}f" elif size == 64: fmt = f"{byteorder}d" else: raise ValueError("floating point item size must be 16, 32 or 64 bits") codec_ = struct.Struct(fmt) def func(ba): return codec_.unpack(ba.tobytes())[0] return func def converter_factory( type_, size: Optional[int] = None, signed: bool = False, byteorder: str = ">", bitorder: str = "big", ) -> FactoryType: if bpack.utils.is_sequence_type(type_, error=True): raise TypeError( f"backend {BACKEND_NAME!r} does not supports sequence types: " f"'{type_}'." ) etype = bpack.utils.effective_type(type_) if etype is int: def func(ba): return ba2int(ba, signed) elif etype is float: func = ba_to_float_factory(size, byteorder, bitorder) elif etype is bytes: def func(ba): return ba.tobytes() elif etype is str: def func(ba): return ba.tobytes().decode("ascii") elif etype is bool: def func(ba): return bool(bitarray.util.ba2int(ba)) else: raise TypeError( f"type '{type_}' is not supported by the {__name__} backend" f"({BACKEND_NAME})" ) if etype is not type_: def converter(x, conv_func=func): return type_(conv_func(x)) else: converter = func return converter def _bitorder_to_baorder(bitorder: EBitOrder) -> str: if bitorder in {EBitOrder.MSB, EBitOrder.DEFAULT}: s = "big" elif bitorder is EBitOrder.LSB: s = "little" else: raise ValueError(f'invalid bit order: "{bitorder}"') return s class Decoder(bpack.codecs.Decoder): """Bitarray based data decoder. Only supports "big endian" byte-order and MSB bit-order. """ baseunits = EBaseUnits.BITS def __init__(self, descriptor, converters=converter_factory): """Initialize the decoder. The *descriptor* parameter* is a bpack record descriptor. """ super().__init__(descriptor) byteorder = bpack.byteorder(descriptor) if byteorder in {EByteOrder.LE, EByteOrder.NATIVE}: raise NotImplementedError( f"byte order '{byteorder}' is not supported by the {__name__} " f"backend ({BACKEND_NAME})" ) bitorder = _bitorder_to_baorder(bpack.bitorder(descriptor)) if bitorder != "big": raise NotImplementedError( f"bit order '{bitorder}' is not supported by the {__name__} " f"backend ({BACKEND_NAME})" ) if callable(converters): conv_factory = converters byteorder_str = byteorder.value if byteorder.value else ">" converters = [ conv_factory( field_descr.type, field_descr.size, field_descr.signed, byteorder_str, ) for field_descr in field_descriptors(descriptor) ] if converters is not None: converters = list(converters) n_fields = len(list(bpack.fields(descriptor))) if len(converters) != n_fields: raise ValueError( f"the number of converters ({len(converters)}) does not " f"match the number of fields ({n_fields})" ) self._converters = converters self._slices = [ slice(field_descr.offset, field_descr.offset + field_descr.size) for field_descr in field_descriptors(descriptor) ] def decode(self, data: bytes): """Decode binary data and return a record object.""" ba = bitarray.bitarray() ba.frombytes(data) values = [ba[slice_] for slice_ in self._slices] if self._converters is not None: values = [ convert(value) if convert is not None else value for convert, value in zip(self._converters, values) ] return self.descriptor(*values) decoder = bpack.codecs.make_codec_decorator(Decoder) def _pairwise(it): a, b = itertools.tee(it) next(b, None) return itertools.zip_longest(a, b, fillvalue=None) def unpackbits(data: bytes, bits_per_sample: int, signed: bool = False): """Unpack packed (integer) values form a string of bytes. Takes in input a string of bytes in which (integer) samples have been stored using ``bits_per_sample`` bit for each sample, and returns the sequence of corresponding Python integers. Example:: 3 bytes 4 samples |------|------|------|------| --> [samp_1, samp_2, samp_3, samp_4] 4 samples (6 bits per sample) If ``signed`` is set to True integers are assumed to be stored as signed integers. """ nbits = len(data) * 8 # assert nbits % bits_per_sample == 0 slices = [ slice(s, e) for s, e in _pairwise(range(0, nbits, bits_per_sample)) ] ba = bitarray.bitarray() ba.frombytes(data) return [ba2int(ba[slice_], signed) for slice_ in slices] bpack-1.1.0/bpack/bs.py000066400000000000000000000165271441646351700146550ustar00rootroot00000000000000"""Bitstruct based codec for binary data structures.""" import math import warnings import functools from typing import Optional try: import cbitstruct as bitstruct except ImportError: import bitstruct try: import bitstruct.c except ImportError: pass import bpack import bpack.utils import bpack.codecs from .enums import EBaseUnits, EByteOrder from .codecs import has_codec, get_codec from .descriptors import field_descriptors __all__ = [ "Decoder", "decoder", "Encoder", "encoder", "Codec", "codec", "BACKEND_NAME", "BACKEND_TYPE", "packbits", "unpackbits", ] BACKEND_NAME = "bitstruct" BACKEND_TYPE = EBaseUnits.BITS class BitStruct: @staticmethod def _simplified_fmt(format_: str) -> Optional[str]: fmt = format_.replace(">", "") if "<" in fmt: return None else: return fmt def __init__(self, format_: str, names=None): codec_ = None if hasattr(bitstruct, "c"): fmt = self._simplified_fmt(format_) if fmt is not None: try: codec_ = bitstruct.c.compile(fmt, names) except NotImplementedError: pass if codec_ is None: codec_ = bitstruct.compile(format_, names) self._bitstruct = codec_ self._format: str = format_ @property def format(self) -> str: # noqa: A003 return self._format def __getattr__(self, name): return getattr(self._bitstruct, name) _TYPE_TO_STR = { bool: "b", int: "u", (int, False): "u", (int, True): "s", float: "f", bytes: "r", str: "t", None: "p", } def _format_string_without_order(fmt: str, order: str) -> str: # NOTE: in the current implementation the byte order is handled # externally to _to_fmt if order != "": fmt = fmt[:-1] if fmt.endswith(order) else fmt return fmt def _to_fmt( type_, size: int, bitorder: str = "", byteorder: str = "", signed: Optional[bool] = None, repeat: Optional[int] = None, ) -> str: assert size > 0, f"invalid size: {size:r}" assert bitorder in ("", ">", "<"), f"invalid order: {bitorder:r}" if repeat is None: repeat = 1 assert repeat > 0, f"invalid repeat: {repeat:r}" if has_codec(type_, bpack.codecs.Decoder): decoder_ = get_codec(type_) if isinstance(decoder_, Decoder): return _format_string_without_order(decoder_.format, byteorder) elif ( bpack.is_descriptor(type_) and bpack.baseunits(type_) is Decoder.baseunits ): decoder_ = Decoder(type_) return _format_string_without_order(decoder_.format, byteorder) etype = bpack.utils.effective_type(type_) key = (etype, signed) if etype is int and signed is not None else etype try: fmt = f"{bitorder}{_TYPE_TO_STR[key]}{size}" * repeat except KeyError: raise TypeError(f"unsupported type: {etype:!r}") # fmt += byteorder # NOTE: handled externally return fmt def _endianess_to_str(order: EByteOrder) -> str: if order is EByteOrder.NATIVE: return EByteOrder.get_native().value return order.value class Codec(bpack.codecs.BaseStructCodec): """Bitstruct based codec. Default bit-order: MSB. """ baseunits = EBaseUnits.BITS @staticmethod def _get_base_codec(descriptor): byteorder = bpack.byteorder(descriptor) byteorder = _endianess_to_str(byteorder) bitorder = bpack.bitorder(descriptor).value fmt = "".join( _to_fmt( field_descr.type, size=field_descr.size, bitorder=bitorder, byteorder=byteorder, signed=field_descr.signed, repeat=field_descr.repeat, ) for field_descr in field_descriptors(descriptor, pad=True) ) fmt = fmt + byteorder # byte order return BitStruct(fmt) @staticmethod def _get_decode_converters_map(descriptor): return { field_descr.type: field_descr.type for field_descr in field_descriptors(descriptor) if bpack.utils.is_enum_type(field_descr.type) } @staticmethod def _get_encode_converters_map(descriptor): def from_enum(x): return x.value converters_map = { field_descr.type: from_enum for field_descr in field_descriptors(descriptor) if ( bpack.utils.is_enum_type(field_descr.type) and not issubclass(field_descr.type, int) ) } return converters_map codec = bpack.codecs.make_codec_decorator(Codec) Decoder = Encoder = Codec decoder = encoder = codec @functools.lru_cache() # @COPMPATIBILITY with Python 3.7 def _get_sequence_codec( nsamples: int, bits_per_sample, signed=False, byteorder: str = "" ) -> BitStruct: nbits = nsamples * bits_per_sample outsize = math.ceil(nbits / 8) npad = outsize * 8 - nbits if signed: fmt = f"s{bits_per_sample:d}" * nsamples else: fmt = f"u{bits_per_sample:d}" * nsamples if npad > 0: fmt += f"p{npad:d}" fmt += byteorder return BitStruct(fmt) def packbits( values, bits_per_sample: int, signed: bool = False, byteorder: str = "" ) -> bytes: """Pack integer values using the specified number of bits for each sample. Converts a sequence of values into a string of bytes in which each sample is stored according to the specified number of bits. Example:: 4 samples 3 bytes [samp_1, samp_2, samp_3, samp_4] --> |------|------|------|------| 4 samples (6 bits per sample) Please note that no check that the input values actually fits in the specified number of bits is performed is performed. The function return a sting of bytes including same number of samples of the input plus possibly some padding bit (at the end) to fill an integer number of bytes. If ``signed`` is set to True integers are stored as signed integers. """ nsamples = len(values) if (nsamples * bits_per_sample) % 8: warnings.warn( f"packing {nsamples} with {bits_per_sample} bits per " f"sample requires padding" ) encoder_ = _get_sequence_codec( nsamples, bits_per_sample, signed=signed, byteorder=byteorder ) return encoder_.pack(*values) def unpackbits( data: bytes, bits_per_sample: int, signed: bool = False, byteorder: str = "", ): """Unpack packed (integer) values form a string of bytes. Takes in input a string of bytes in which (integer) samples have been stored using ``bits_per_sample`` bit for each sample, and returns the sequence of corresponding Python integers. Example:: 3 bytes 4 samples |------|------|------|------| --> [samp_1, samp_2, samp_3, samp_4] 4 samples (6 bits per sample) If ``signed`` is set to True integers are assumed to be stored as signed integers. """ nsamples = len(data) * 8 // bits_per_sample decoder_ = _get_sequence_codec( nsamples, bits_per_sample, signed=signed, byteorder=byteorder ) return decoder_.unpack(data) bpack-1.1.0/bpack/codecs.py000066400000000000000000000233531441646351700155040ustar00rootroot00000000000000"""Base classes and utility functions for codecs.""" import abc from typing import Callable, NamedTuple, Optional, Type, Union import bpack.utils import bpack.descriptors from .enums import EBaseUnits from .descriptors import field_descriptors __all__ = [ "Codec", "Encoder", "Decoder", "has_codec", ] CODEC_ATTR_NAME = "__bpack_decoder__" class BaseCodec: """Base class for codecs, encoders and decoders.""" baseunits: EBaseUnits @classmethod def _check_descriptor(cls, descriptor): if bpack.baseunits(descriptor) is not cls.baseunits: raise ValueError( f"{cls.__module__}.{cls.__name__} " f"only accepts descriptors with base units " f"'{cls.baseunits.value}'" ) def __init__(self, descriptor): self._check_descriptor(descriptor) self._descriptor = descriptor @property def descriptor(self): """Return the descriptor associated to the codec.""" return self._descriptor class Decoder(BaseCodec, abc.ABC): """Base class for decoders.""" @abc.abstractmethod def decode(self, data: bytes): """Decode binary data and return Python object.""" pass class Encoder(BaseCodec, abc.ABC): """Base class for encoders.""" @abc.abstractmethod def encode(self, record) -> bytes: """Encode python objects into binary data.""" pass class Codec(Decoder, Encoder, abc.ABC): """Base class for codecs.""" pass CodecType = Union[Decoder, Encoder, Codec] def make_codec_decorator(codec_type: Type[CodecType]): """Generate a codec decorator for the input decoder class.""" @bpack.utils.classdecorator def codec(cls): """Class decorator to add (de)coding methods to a descriptor class. The decorator automatically generates a *Codec* object form the input descriptor class and attach to it methods for conversion form/to bytes. """ codec_ = codec_type(descriptor=cls) bpack.utils.set_new_attribute(cls, CODEC_ATTR_NAME, codec_) if isinstance(codec_, Decoder): decode_func = bpack.utils.create_fn( name="frombytes", args=("cls", "data"), body=[f"return cls.{CODEC_ATTR_NAME}.decode(data)"], ) decode_func = classmethod(decode_func) bpack.utils.set_new_attribute(cls, "frombytes", decode_func) if isinstance(codec_, Encoder): encode_func = bpack.utils.create_fn( name="tobytes", args=("self",), body=[f"return self.{CODEC_ATTR_NAME}.encode(self)"], ) bpack.utils.set_new_attribute(cls, "tobytes", encode_func) return cls return codec def has_codec( descriptor, codec_type: Optional[Type[CodecType]] = None ) -> bool: """Return True if the input descriptor has a codec attached. A descriptor decorated with a *codec* decorator has an attached codec instance and "frombytes"/"tobytes" methods (depending on the kind of codec). The *codec_type* parameter can be used to query for specific codec features: * codec_type=None: return True for any king of codec * codec_type=:class:`Decoder`: return True if the attached coded has decoding capabilities * codec_type=:class:`Encoder`: return True if the attached coded has encoding capabilities * codec_type=:class:`Codec`: return True if the attached coded has both encoding and decoding capabilities """ if hasattr(descriptor, CODEC_ATTR_NAME): assert isinstance(get_codec(descriptor), (Codec, Decoder, Encoder)) if codec_type is None: return True elif issubclass(codec_type, Codec): return hasattr(descriptor, "frombytes") and hasattr( descriptor, "tobytes" ) elif issubclass(codec_type, Decoder): return hasattr(descriptor, "frombytes") elif issubclass(codec_type, Encoder): return hasattr(descriptor, "tobytes") return False def get_codec(descriptor) -> CodecType: """Return the codec instance attached to the input descriptor.""" return getattr(descriptor, CODEC_ATTR_NAME, None) # TODO: remove def get_codec_type(descriptor) -> Type[CodecType]: """Return the type of the codec attached to the input descriptor.""" codec_ = getattr(descriptor, CODEC_ATTR_NAME, None) if codec_ is not None: return type(codec_) def _get_flat_len(descriptor): count = 0 for field_descr in field_descriptors(descriptor): if bpack.is_descriptor(field_descr.type): count += _get_flat_len(field_descr.type) elif field_descr.repeat is not None: count += field_descr.repeat else: count += 1 return count class ConverterInfo(NamedTuple): func: Callable src: Union[int, slice] dst: Union[int, slice] class BaseStructCodec(Codec): """Base class for codecs base on struct like backends.""" @staticmethod @abc.abstractmethod def _get_base_codec(descriptor): pass def __init__( self, descriptor, codec=None, decode_converters=None, encode_converters=None, ): """Initialize the BaseStructCodec. The *descriptor* parameter* is a bpack record descriptor. """ super().__init__(descriptor) if codec is None: codec = self._get_base_codec(descriptor) if decode_converters is None: decode_converters = self._get_decode_converters(descriptor) if encode_converters is None: encode_converters = self._get_encode_converters(descriptor) self._codec = codec self._decode_converters = decode_converters self._encode_converters = encode_converters self._flat_len = _get_flat_len(descriptor) @property def format(self) -> str: # noqa: A003 """Return the format string.""" return self._codec.format @classmethod def _get_decoder(cls, descr): assert ( bpack.is_descriptor(descr) and bpack.baseunits(descr) is cls.baseunits ) if has_codec(descr, Decoder): decoder_ = get_codec(descr) return decoder_ decoder_ = cls(descr) return decoder_ @staticmethod def _get_decode_converters_map(descriptor): return {} @classmethod def _get_decode_converters(cls, descriptor): converters_map = cls._get_decode_converters_map(descriptor) converters = [] for idx, field_descr in enumerate(field_descriptors(descriptor)): if field_descr.type in converters_map: func = converters_map[field_descr.type] converters.append(ConverterInfo(func, idx, idx)) elif bpack.is_descriptor(field_descr.type): decoder_ = cls._get_decoder(field_descr.type) n_items = decoder_._flat_len src = slice(idx, idx + n_items) func = decoder_._from_flat_list converters.append(ConverterInfo(func, src, idx)) elif field_descr.repeat is not None: sequence_type = bpack.utils.sequence_type( field_descr.type, error=True ) src = slice(idx, idx + field_descr.repeat) converters.append(ConverterInfo(sequence_type, src, idx)) return converters def _from_flat_list(self, values): for func, src, dst in self._decode_converters: if isinstance(src, int): values[dst] = func(values[src]) else: value = func(values[src]) del values[src] values.insert(dst, value) return self.descriptor(*values) def decode(self, data: bytes): """Decode binary data and return a record object.""" values = list(self._codec.unpack(data)) return self._from_flat_list(values) @classmethod def _get_encoder(cls, descr): assert ( bpack.is_descriptor(descr) and bpack.baseunits(descr) is cls.baseunits ) if has_codec(descr, Encoder): encoder_ = get_codec(descr) return encoder_ encoder_ = cls(descr) return encoder_ @staticmethod def _get_encode_converters_map(descriptor): return {} @classmethod def _get_encode_converters(cls, descriptor): converters_map = cls._get_encode_converters_map(descriptor) def nullop(x): return x converters = [] for idx, field_descr in enumerate(field_descriptors(descriptor)): if field_descr.type in converters_map: func = converters_map[field_descr.type] converters.append(ConverterInfo(func, idx, idx)) elif bpack.is_descriptor(field_descr.type): encoder_ = cls._get_encoder(field_descr.type) func = encoder_._to_flat_list slice_ = slice(idx, idx + 1) converters.append(ConverterInfo(func, idx, slice_)) elif field_descr.repeat is not None: slice_ = slice(idx, idx + 1) converters.append(ConverterInfo(nullop, idx, slice_)) return converters def _to_flat_list(self, record): values = [ getattr(record, field.name) for field in bpack.fields(record) ] for func, src, dst in self._encode_converters[::-1]: values[dst] = func(values[src]) return values def encode(self, record) -> bytes: """Encode a record object into binary data.""" values = self._to_flat_list(record) return self._codec.pack(*values) bpack-1.1.0/bpack/descriptors.py000066400000000000000000000524021441646351700166020ustar00rootroot00000000000000"""Descriptors for binary records.""" import copy import enum import math import types import warnings import dataclasses from typing import Iterable, Optional, Sequence, Type, Union import bpack.utils import bpack.typing from .enums import EBaseUnits, EByteOrder, EBitOrder from .utils import classdecorator __all__ = [ "descriptor", "is_descriptor", "fields", "asdict", "astuple", "calcsize", "baseunits", "byteorder", "bitorder", "field", "Field", "is_field", "field_descriptors", "BinFieldDescriptor", "get_field_descriptor", "set_field_descriptor", "BASEUNITS_ATTR_NAME", "BYTEORDER_ATTR_NAME", "BITORDER_ATTR_NAME", "METADATA_KEY", ] BASEUNITS_ATTR_NAME = "__bpack_baseunits__" BYTEORDER_ATTR_NAME = "__bpack_byteorder__" BITORDER_ATTR_NAME = "__bpack_bitorder__" SIZE_ATTR_NAME = "__bpack_size__" METADATA_KEY = "__bpack_metadata__" class DescriptorConsistencyError(ValueError): pass class NotFieldDescriptorError(TypeError): pass def _resolve_type(type_): """Remove type annotations. Replace :class:`typing.Annotated` types with the corresponding not-annotated ones. """ if bpack.utils.is_sequence_type(type_): etype = bpack.utils.effective_type(type_) rtype = copy.copy(type_) rtype.__args__ = (etype,) elif bpack.typing.is_annotated(type_): rtype = bpack.utils.effective_type(type_) else: rtype = type_ return rtype @dataclasses.dataclass class BinFieldDescriptor: """Descriptor for bpack fields. See also :func:`bpack.filed` for a description of the attributes. """ type: Optional[Type] = None # noqa: A003 size: Optional[int] = None #: item size offset: Optional[int] = None signed: Optional[bool] = None repeat: Optional[int] = None #: number of items # converter: Optional[Callable] = None def _validate_type(self): if self.type is None: raise TypeError(f"invalid type '{self.type}'") def _validate_size(self): msg = f"invalid size: {self.size!r} (must be a positive integer)" if not isinstance(self.size, int): raise TypeError(msg) if self.size <= 0: raise ValueError(msg) def _validate_offset(self): msg = f"invalid offset: {self.offset!r} (must be an integer >= 0)" if not isinstance(self.offset, int): raise TypeError(msg) if self.offset < 0: raise ValueError(msg) def _validate_signed(self): if not isinstance(self.signed, bool): raise TypeError( f"invalid 'signed' parameter: {self.signed!r} " f"(must be a bool or None)" ) def _validate_repeat(self): msg = f"invalid repeat: {self.repeat!r} (must be a positive)" if not isinstance(self.repeat, int): raise TypeError(msg) if self.repeat < 1: raise ValueError(msg) def _validate_enum_type(self): assert issubclass(self.type, enum.Enum) # perform checks on supported enum.Enum types bpack.utils.enum_item_type(self.type) def __post_init__(self): """Finalize BinFieldDescriptor instance initialization.""" if self.offset is not None: self._validate_offset() if self.size is not None: self._validate_size() if self.signed is not None: self._validate_signed() if self.repeat is not None: self._validate_repeat() def validate(self): """Perform validity check on the BinFieldDescriptor instance.""" self._validate_type() self._validate_size() if self.offset is not None: self._validate_offset() if self.signed is not None: self._validate_signed() if not self.is_int_type(): warnings.warn( f"the 'signed' parameter will be ignored for non-integer " f"type: '{self.type}'" ) if self.repeat is not None: self._validate_repeat() if not self.is_sequence_type() and self.repeat is not None: raise TypeError( f"repeat parameter specified for non-sequence type: " f"{self.type}" ) if bpack.utils.is_enum_type(self.type): self._validate_enum_type() elif self.is_sequence_type() and self.repeat is None: raise TypeError( f"no 'repeat' parameter specified for sequence type " f"{self.type}" ) def is_int_type(self) -> bool: """Return True if the field is an integer or a sub-type of integer.""" return bpack.utils.is_int_type(self.type) def is_sequence_type(self) -> bool: """Return True if the field is a sequence.""" return bpack.utils.is_sequence_type(self.type, error=True) def is_enum_type(self) -> bool: """Return True if the field is an enum.""" return bpack.utils.is_enum_type(self.type) @property def total_size(self): """Total size in bytes of the field (considering all item).""" repeat = self.repeat if self.repeat is not None else 1 return self.size * repeat @staticmethod def _is_compatible_param(old, new): if old is not None and new is not None and old != new: return False return True def update_from_type(self, type_: Type): """Update the field descriptor according to the specified type.""" if self.type is not None: raise TypeError("the type attribute is already set") if bpack.typing.is_annotated(type_): _, params = bpack.typing.get_args(type_) valid = True if not self._is_compatible_param(self.size, params.size): valid = False if not self._is_compatible_param(self.signed, params.signed): valid = False if not valid: raise DescriptorConsistencyError( f"type string '{params}' is incompatible with the " f"field descriptor {self}" ) self.type = params.type if self.signed is None: self.signed = params.signed if self.size is None: self.size = params.size elif bpack.utils.is_sequence_type(type_): etype = bpack.utils.effective_type(type_, keep_annotations=True) self.update_from_type(etype) self.type = _resolve_type(type_) else: self.type = type_ Field = dataclasses.Field def field( *, size: Optional[int] = None, offset: Optional[int] = None, signed: Optional[bool] = None, repeat: Optional[int] = None, metadata=None, **kwargs, ) -> Field: """Initialize a field descriptor. Returned object is a :class:`Field` instance with metadata properly initialized to describe the field of a binary record. :param size: int size of the field in :class:`EBaseUnits` :param offset: int offset of the field w.r.t. the beginning of the record (expressed in :class:`EBaseUnits`) :param signed: bool True if an `int` field is signed, False otherwise. This parameter must not be specified for non `int` fields. :param repeat: int length of the sequence for `sequence` fields, i.e. fields consisting in multiple items having the same data type. This parameter must not be specified if the data type is not a sequence type (e.g. `List`). :param metadata: additional metadata to be attached the the field descriptor. :param kwargs: additional keyword arguments for the :func:`dataclasses.field` function. """ field_descr = BinFieldDescriptor( size=size, offset=offset, signed=signed, repeat=repeat ) metadata = metadata.copy() if metadata is not None else {} metadata[METADATA_KEY] = types.MappingProxyType( dataclasses.asdict(field_descr) ) return dataclasses.field(metadata=metadata, **kwargs) def is_field(obj) -> bool: """Return true if an ``obj`` can be considered is a field descriptor.""" return ( isinstance(obj, Field) and obj.metadata and METADATA_KEY in obj.metadata ) def _update_field_metadata(field_, **kwargs): metadata = field_.metadata.copy() if field_.metadata is not None else {} metadata.update(**kwargs) field_.metadata = types.MappingProxyType(metadata) return field_ def get_field_descriptor( field: Field, validate: bool = True ) -> BinFieldDescriptor: """Return the field descriptor attached to a :class:`Field`.""" if not is_field(field): raise NotFieldDescriptorError(f"not a field descriptor: {field}") field_descr = BinFieldDescriptor(**field.metadata[METADATA_KEY]) field_descr.update_from_type(field.type) if validate: field_descr.validate() return field_descr def set_field_descriptor( field: Field, descriptor: BinFieldDescriptor, validate: bool = True ) -> Field: """Set the field metadata according to the specified descriptor.""" if validate: descriptor.validate() field_descr_metadata = { k: v for k, v in dataclasses.asdict(descriptor).items() if v is not None } type_ = field_descr_metadata.pop("type", None) if bpack.typing.is_annotated(field.type): field_type, _ = bpack.typing.get_args(field.type) else: field_type = field.type if type_ != _resolve_type(field_type): raise TypeError( f"type mismatch between BinFieldDescriptor.type ({type_!r}) and " f"filed.type ({field.type!r})" ) new_metadata = { METADATA_KEY: types.MappingProxyType(field_descr_metadata), } _update_field_metadata(field, **new_metadata) return field _DEFAULT_SIZE_MAP = { EBaseUnits.BYTES: { bool: 1, # int: 4, # float: 4, }, EBaseUnits.BITS: { bool: 1, # int: 32, # float: 32, }, } def _get_default_size(type_, baseunits: EBaseUnits) -> Union[int, None]: if is_descriptor(type_): return calcsize(type_, baseunits) etype = bpack.utils.effective_type(type_) # if bpack.utils.is_enum_type(type_): # if bpack.utils.is_int_type(type_): # signbit = 1 if any(item.value < 0 for item in type_) else 0 # bits = signbit + max(item.value.bit_lenght() for item in type_) # if baseunits is EBaseUnits.BITS: # if bits <= 8: # return 1 # elif bits <= 16: # return 2 # elif bits <= 32: # return 4 # else: # return 8 # else: # return bits # elif issubclass(etype, str): # length = max(len(item.value.encode('utf-8')) for item in type_) # return length * bytes_to_baseunits # elif issubclass(etype, bytes): # length = max(len(item.value) for item in type_) # return length * bytes_to_baseunits # else: # return None return _DEFAULT_SIZE_MAP[baseunits].get(etype) def _get_effective_byteorder( byteorder: EByteOrder, baseunits: EBaseUnits ) -> EByteOrder: byteorder = EByteOrder(byteorder) effective_byteorder = byteorder if baseunits is EBaseUnits.BYTES: if byteorder in {EByteOrder.NATIVE, EByteOrder.DEFAULT}: effective_byteorder = EByteOrder.get_native() else: if byteorder is EByteOrder.DEFAULT: effective_byteorder = EByteOrder.BE elif byteorder in EByteOrder.NATIVE: effective_byteorder = EByteOrder.get_native() return effective_byteorder @classdecorator def descriptor( cls, *, size: Optional[int] = None, byteorder: Union[str, EByteOrder] = EByteOrder.DEFAULT, bitorder: Optional[Union[str, EBitOrder]] = None, baseunits: EBaseUnits = EBaseUnits.BYTES, **kwargs, ): """Class decorator to define descriptors for binary records. It converts a dataclass into a descriptor object for binary records. * ensures that all fields are :class:`bpack.descriptor.Field` descriptors * offsets are automatically computed if necessary * consistency checks on offsets and sizes are performed :param cls: class to be decorated :param size: the size (expressed in *base units*) of the binary record :param byteorder: the byte-order of the binary record :param bitorder: the bit-order of the binary record (must be ``None`` if the *base units* are bytes). If set to none in bit-based records it is assumed :data:`bpack.enums.EBitOrder.DEFAULT` which corresponds to :data:`bpack.enums.EBitOrder.MSB` in all decoders currently implemented. :param baseunits: the base units (:data:`bpack.enums.EBaseUnits.BITS` or :data:`bpack.enums.EBaseUnits.BYTES`) used to specify the binary record descriptor It is also possible to specify as additional keyword arguments all the parameters accepted by :func:`dataclasses.dataclass`. """ baseunits = EBaseUnits(baseunits) byteorder = EByteOrder(byteorder) if dataclasses.is_dataclass(cls): warnings.warn( "the explicit use of dataclasses is deprecated", category=DeprecationWarning, ) else: cls = dataclasses.dataclass(cls, **kwargs) fields_ = dataclasses.fields(cls) # Initialize to a dummy value with initial offset + size = 0 prev_field_descr = BinFieldDescriptor(size=None, offset=0) prev_field_descr.size = 0 # trick to bypass checks on BinFieldDescriptor content_size = 0 for idx, field_ in enumerate(fields_): assert isinstance(field_, Field) # NOTE: this is ensured by dataclasses but not by attr assert field_.type is not None if bpack.typing.is_annotated(field_.type): # check byteorder _, params = bpack.typing.get_args(field_.type) if params.byteorder: effective_byteorder = _get_effective_byteorder( byteorder, baseunits ) if params.byteorder != effective_byteorder: raise DescriptorConsistencyError( f"the byteorder of field {field_.name} " f"('{params.byteorder}' is not consistent with the " f"descriptor byteorder ({byteorder}) )" ) try: field_descr = get_field_descriptor(field_, validate=False) except NotFieldDescriptorError: field_descr = BinFieldDescriptor() if isinstance(field_, Field): field_descr.update_from_type(field_.type) if field_descr.size is None: field_descr.size = _get_default_size(field_descr.type, baseunits) if field_descr.size is None: raise TypeError(f'size not specified for field: "{field_.name}"') if ( is_descriptor(field_descr.type) and calcsize(field_descr.type, baseunits) != field_descr.size ): raise DescriptorConsistencyError( f"mismatch between field.size ({field_descr.size}) and size " f"of field.type ({calcsize(field_descr.type, baseunits)}) " f"in field '{field_.name}'" ) auto_offset = prev_field_descr.offset + prev_field_descr.total_size if field_descr.offset is None: field_descr.offset = auto_offset elif field_descr.offset < auto_offset: raise DescriptorConsistencyError( f"invalid offset for filed n. {idx}: {field_}" ) set_field_descriptor(field_, field_descr) prev_field_descr = field_descr content_size += field_descr.size field_descr = get_field_descriptor(fields_[-1]) auto_size = field_descr.offset + field_descr.total_size assert auto_size >= content_size # this should be already checked above if size is None: size = auto_size elif int(size) != size: raise TypeError(f"invalid size: {size!r}") elif size < auto_size: raise DescriptorConsistencyError( f"the specified size ({size}) is smaller than total size of " f"fields ({auto_size})" ) if baseunits is EBaseUnits.BITS: if size % 8 != 0: warnings.warn("bit struct not aligned to bytes") if baseunits is not EBaseUnits.BITS and bitorder is not None: raise ValueError( "it is not possible to specify the 'bitorder' " "if 'baseunits' is not 'BITS'" ) elif baseunits is EBaseUnits.BITS and bitorder is None: bitorder = EBitOrder.DEFAULT setattr(cls, BASEUNITS_ATTR_NAME, baseunits) setattr(cls, BYTEORDER_ATTR_NAME, byteorder) setattr( cls, BITORDER_ATTR_NAME, EBitOrder(bitorder) if bitorder is not None else None, ) setattr(cls, SIZE_ATTR_NAME, size) return cls def fields(obj) -> Sequence[Field]: """Return a tuple describing the fields of this descriptor.""" return dataclasses.fields(obj) def is_descriptor(obj) -> bool: """Return true if ``obj`` is a descriptor or a descriptor instance.""" try: return hasattr(obj, BASEUNITS_ATTR_NAME) and is_field(fields(obj)[0]) except (TypeError, ValueError): # dataclass.fields(...) --> TypeError # attr.fields(...) --> NotAnAttrsClassError(ValueError) return False except IndexError: # no fields return False def asdict(obj, *, dict_factory=dict) -> dict: """Return the fields of a record as a new dictionary. The returned dictionary maps field names to field values. If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: tuples, lists, and dicts. """ return dataclasses.asdict(obj, dict_factory=dict_factory) def astuple(obj, *, tuple_factory=tuple) -> Sequence: """Return the fields of a dataclass instance as new tuple of field values. If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: tuples, lists, and dicts. """ return dataclasses.astuple(obj, tuple_factory=tuple_factory) def calcsize(obj, units: Optional[EBaseUnits] = None) -> int: """Return the size of the ``obj`` record. If the *units* parameter is not specified (default) then the returned *size* is expressed in the same *base units* of the descriptor. """ if not is_descriptor(obj): raise TypeError(f"{obj!r} is not a descriptor") size = getattr(obj, SIZE_ATTR_NAME) if units: baseunits_ = getattr(obj, BASEUNITS_ATTR_NAME) units = EBaseUnits(units) if units is not baseunits_: if units is EBaseUnits.BYTES: # baseunits is BITS and units is BYTES size = math.ceil(size / 8) else: # baseunits is BYTES and units is BITS size *= 8 return size def baseunits(obj) -> EBaseUnits: """Return the base units of a binary record descriptor.""" try: return getattr(obj, BASEUNITS_ATTR_NAME) except AttributeError: raise TypeError(f'"{obj}" is not a descriptor') def byteorder(obj) -> EByteOrder: """Return the byte order of a binary record descriptor (endianess).""" try: return getattr(obj, BYTEORDER_ATTR_NAME) except AttributeError: raise TypeError(f'"{obj}" is not a descriptor') def bitorder(obj) -> Union[EBitOrder, None]: """Return the bit order of a binary record descriptor.""" try: return getattr(obj, BITORDER_ATTR_NAME) except AttributeError: raise TypeError(f'"{obj}" is not a descriptor') def field_descriptors( descriptor, pad: bool = False ) -> Iterable[BinFieldDescriptor]: """Return the list of field descriptors for the input record descriptor. Items are instances of the :class:`BinFieldDescriptor` class describing characteristics of each field of the input binary record descriptor. If the ``pad`` parameter is set to True then also generate dummy field descriptors for padding elements necessary to take into account offsets between fields. """ if pad: offset = 0 for field_ in fields(descriptor): field_descr = get_field_descriptor(field_) assert field_descr.offset >= offset if field_descr.offset > offset: # padding yield BinFieldDescriptor( size=field_descr.offset - offset, offset=offset ) # offset = field_.offset yield field_descr offset = field_descr.offset + field_descr.total_size size = calcsize(descriptor) if offset < size: # padding yield BinFieldDescriptor(size=size - offset, offset=offset) else: for field_ in fields(descriptor): yield get_field_descriptor(field_) bpack-1.1.0/bpack/enums.py000066400000000000000000000016141441646351700153670ustar00rootroot00000000000000"""Enumeration types for the bpack package.""" import sys import enum class EBaseUnits(enum.Enum): """Base units used to specify size and offset parameters in descriptors.""" BITS = "bits" BYTES = "bytes" class EByteOrder(enum.Enum): """Enumeration for byte order (endianess). .. note:: the :data:`EByteOrder.DEFAULT` is equivalent to :data:`EByteOrder.NATIVE` for binary structures having :data:`EBaseUnits.BYTE` base units, and :data:`EByteOrder.BE` for binary structures having :data:`EBaseUnits.BIT` base units. """ BE = ">" LE = "<" NATIVE = "=" DEFAULT = "" @classmethod def get_native(cls): """Return the native byte order.""" return cls.LE if sys.byteorder == "little" else cls.BE class EBitOrder(enum.Enum): """Enumeration for bit order.""" MSB = ">" LSB = "<" DEFAULT = "" bpack-1.1.0/bpack/np.py000066400000000000000000000416621441646351700146640ustar00rootroot00000000000000"""Numpy based codec for binary data structures.""" import enum import functools import collections from typing import NamedTuple, Optional import numpy as np import bpack import bpack.utils import bpack.codecs from .enums import EBaseUnits from .descriptors import ( field_descriptors, get_field_descriptor, BinFieldDescriptor, ) __all__ = [ "Decoder", "decoder", "Encoder", "encoder", "Codec", "codec", "BACKEND_NAME", "BACKEND_TYPE", "descriptor_to_dtype", "unpackbits", "ESignMode", ] BACKEND_NAME = "numpy" BACKEND_TYPE = EBaseUnits.BYTES def bin_field_descripor_to_dtype(field_descr: BinFieldDescriptor) -> np.dtype: """Convert a field descriptor into a :class:`numpy.dtype`. .. seealso:: :class:`bpack.descriptors.BinFieldDescriptor`. """ # TODO: add byteorder size = field_descr.size etype = bpack.utils.effective_type(field_descr.type) typecode = np.dtype(etype).kind if etype in (bytes, str): typecode = "S" elif etype is int and not field_descr.signed: typecode = "u" if typecode == "O": raise TypeError(f"unsupported type: {field_descr.type:!r}") repeat = field_descr.repeat repeat = repeat if repeat and repeat > 1 else "" return np.dtype(f"{repeat}{typecode}{size}") def descriptor_to_dtype(descriptor) -> np.dtype: """Convert the descriptor of a binary record into a :class:`numpy.dtype`. Please note that (unicode) strings are treated as "utf-8" encoded byte strings. UCS4 encoded strings are not supported. Sequences (:class:`typing.Sequence` and :class:`typing.List`) are always converted into :class:`numpy.ndarray`. .. seealso:: :func:`bpack.descriptors.descriptor`. """ params = collections.defaultdict(list) for field in bpack.fields(descriptor): field_descr = get_field_descriptor(field) if bpack.is_descriptor(field_descr.type): dtype = descriptor_to_dtype(field_descr.type) else: dtype = bin_field_descripor_to_dtype(field_descr) params["names"].append(field.name) params["formats"].append(dtype) params["offsets"].append(field_descr.offset) # params['titles'].append('...') params = dict(params) # numpy do not accept defaultdict params["itemsize"] = bpack.calcsize(descriptor) dt = np.dtype(dict(params)) byteorder = bpack.byteorder(descriptor).value if byteorder: dt = dt.newbyteorder(byteorder) return dt def _decode_converter_factory(type_): etype = bpack.utils.effective_type(type_) if bpack.utils.is_enum_type(type_): if etype is str: def converter(x, cls=type_): # TODO: harmonize with other backends that use 'ascii' return cls(x.tobytes().decode("utf-8")) else: def converter(x, cls=type_): return cls(x) elif etype is str: def converter(x): # TODO: harmonize with other backends that use 'ascii' return x.tobytes().decode("utf-8") elif bpack.is_descriptor(type_): def converter(x, cls=type_): return cls(*x) else: converter = None return converter def _encode_converter_factory(type_): converter = None etype = bpack.utils.effective_type(type_) if bpack.utils.is_enum_type(type_): if etype is str: def converter(x): # TODO: harmonize with other backends that use 'ascii' return x.value.encode("utf-8") elif not issubclass(type_, int): def converter(x): return x.value elif etype is str: def converter(x): # TODO: harmonize with other backends that use 'ascii' return x.encode("utf-8") # TODO: cleanup # elif bpack.is_descriptor(type_): # # astuple works recursively so nested descriptors have been # # already converted into sequences # # # # def converter(x): # # return bpack.astuple(x, tuple_factory=list) # pass return converter class Codec(bpack.codecs.Codec): """Numpy based codec. (Unicode) strings are treated as "utf-8" encoded byte strings. UCS4 encoded strings are not supported. """ baseunits = EBaseUnits.BYTES def __init__(self, descriptor): """Initialize the codec. The *descriptor* parameter* is a bpack record descriptor. """ super().__init__(descriptor) assert bpack.bitorder(descriptor) is None decode_converters = [ (idx, _decode_converter_factory(field_descr.type)) for idx, field_descr in enumerate(field_descriptors(descriptor)) ] encode_converters = [ (idx, _encode_converter_factory(field_descr.type)) for idx, field_descr in enumerate(field_descriptors(descriptor)) ] self._dtype = descriptor_to_dtype(descriptor) self._decode_converters = [ (idx, func) for idx, func in decode_converters if func ] self._encode_converters = [ (idx, func) for idx, func in encode_converters if func ] @property def dtype(self): """Return the numpy `dtype` corresponding to the `codec.descriptor`.""" return self._dtype def decode(self, data: bytes, count: int = 1): """Decode binary data and return a record object.""" v = np.frombuffer(data, dtype=self._dtype, count=count) if self._decode_converters: out = [] for item in v: item = list(item) # fields of the np record for idx, func in self._decode_converters: item[idx] = func(item[idx]) out.append(self.descriptor(*item)) else: out = [self.descriptor(*item) for item in v] if len(v) == 1: out = out[0] return out def encode(self, record): """Encode record (Python object) into binary data.""" # exploit the recursive behaviour of astuple values = bpack.astuple(record) # , tuple_factory=list) values = list(values) # nested record and sequences stay tuples for idx, func in self._encode_converters: values[idx] = func(values[idx]) return np.array(tuple(values), dtype=self.dtype).tobytes() codec = bpack.codecs.make_codec_decorator(Codec) Decoder = Encoder = Codec decoder = encoder = codec # --- bits packing/unpacking -------------------------------------------------- class EMaskMode(enum.Enum): """Mask mode. :STANDARD: mask the lower nbits, e.g. 0b00001111 for nbit=4 :COMPLEMENT: mask the upper bits by complementing the STANDARD mask, e.g. 0b11110000 for nbit=4 and dtype"unit8" :SINGLE_BIT: mask only the n-th bit (conunting form zero), e.g. 0b00001000 form nbit=4 """ STANDARD = 0 COMPLEMENT = 1 SINGLE_BIT = 2 def _get_item_size(bits_per_sample: int) -> int: """Item size of the integer type that can take requested bits.""" if bits_per_sample > 64 or bits_per_sample < 1: raise ValueError(f"bits_per_sample: {bits_per_sample}") elif bits_per_sample <= 8: return 1 else: return 2 ** int(np.ceil(np.log2(bits_per_sample)) - 3) def _get_buffer_size(bits_per_sample: int) -> int: """Item size of the integer type that can take requested bits and shift.""" return _get_item_size(bits_per_sample + 7) # @COMPATIBILITY: lru_cache without parenteses requires Python > 3.7 @functools.lru_cache() def make_bitmask( bits_per_sample: int, dtype=None, mode: EMaskMode = EMaskMode.STANDARD, ) -> np.ndarray: """Return a mask for dtype according to the specified nbits and mask mode. .. sealso:: :class:`EMaskMode`. """ mode = EMaskMode(mode) assert 0 < bits_per_sample <= 64 if dtype is None: dtype = f"u{_get_item_size(bits_per_sample)}" if mode == EMaskMode.SINGLE_BIT: mask = 2 ** (bits_per_sample - 1) if bits_per_sample > 0 else 0 mask = np.asarray(mask) else: shift = np.array(64 - bits_per_sample, dtype=np.uint32) mask = np.array(0xFFFFFFFFFFFFFFFF) >> shift if mode == EMaskMode.COMPLEMENT: mask = np.invert(mask) return mask.astype(dtype) class BitUnpackParams(NamedTuple): samples: int dtype: str buf_itemsize: int buf_dtype: str index_map: np.ndarray shifts: np.ndarray mask: np.ndarray @functools.lru_cache() # @COPMPATIBILITY with Python 3.7 def _unpackbits_params( nbits: int, bits_per_sample: int, samples_per_block: int, bit_offset: int, blockstride: int, signed: bool = False, byteorder: str = ">", ) -> BitUnpackParams: assert nbits >= bit_offset if samples_per_block is None: if blockstride is not None: raise ValueError( "'samples_per_block' cannot be computed automatically " "when 'blockstride' is provided" ) samples_per_block = (nbits - bit_offset) // bits_per_sample blocksize = bits_per_sample * samples_per_block if blockstride is None: blockstride = blocksize else: assert blockstride >= blocksize nstrides = (nbits - bit_offset) // blockstride extrabits = nbits - bit_offset - nstrides * blockstride if extrabits >= blocksize: nblocks = nstrides + 1 extra_samples = 0 else: nblocks = nstrides extra_samples = extrabits // bits_per_sample assert nblocks >= 0 pad = blockstride - blocksize sizes = [bit_offset] if nblocks > 0: sizes.extend([bits_per_sample] * (samples_per_block - 1)) block_sizes = [bits_per_sample + pad] + [bits_per_sample] * ( samples_per_block - 1 ) sizes.extend(block_sizes * (nblocks - 1)) if extra_samples: sizes.append(bits_per_sample + pad) sizes.extend([bits_per_sample] * (extra_samples - 1)) bit_offsets = np.cumsum(sizes) byte_offsets = bit_offsets // 8 samples = len(bit_offsets) itemsize = _get_item_size(bits_per_sample) buf_itemsize = _get_buffer_size(bits_per_sample) dtype = f"{byteorder}{'i' if signed else 'u'}{itemsize}" buf_dtype = f"{byteorder}u{buf_itemsize}" index = np.arange(buf_itemsize) + byte_offsets[:, None] index = np.clip(index, 0, nbits // 8 - 1) mask = make_bitmask(bits_per_sample, buf_dtype) shifts = bit_offsets - byte_offsets * 8 + bits_per_sample shifts = buf_itemsize * 8 - shifts return BitUnpackParams( samples=samples, dtype=dtype, buf_itemsize=buf_itemsize, buf_dtype=buf_dtype, index_map=index, shifts=shifts, mask=mask, ) class ESignMode(enum.IntEnum): """Enumeration for sign encoding convention.""" UNSIGNED = 0 SIGNED = 1 SIGN_AND_MOD = 2 def unsigned_to_signed( data, bits_per_sample: int, dtype=None, sign_mode: ESignMode = ESignMode.SIGNED, inplace: bool = False, ) -> np.ndarray: """Convert unpacked unsigned integers into signed integers. .. sealso:: :class:`ESignMode`. """ if dtype is None: dtype = f"i{_get_item_size(bits_per_sample)}" sign_mode = ESignMode(sign_mode) if inplace: if not isinstance(data, np.ndarray): raise TypeError( f"The input 'data' ({data!r}) parameter is not a " f"'numpy.ndarray'" ) out = data else: out = np.array(data) out = out.astype(dtype) sign_mask = make_bitmask(bits_per_sample, dtype, EMaskMode.SINGLE_BIT) is_negative = (out & sign_mask).astype(bool) if sign_mode == ESignMode.SIGNED: cmask = make_bitmask( bits_per_sample - 1, dtype, mode=EMaskMode.COMPLEMENT ) out[is_negative] = out[is_negative] | cmask elif sign_mode == ESignMode.SIGN_AND_MOD: mask = make_bitmask(bits_per_sample - 1, dtype) sign = (-1) ** is_negative out = sign * (out & mask) return out @functools.lru_cache() # @COMPATIBILITY: parenteses not needed in Python>=3.8 def make_unsigned_to_signed_lut( bits_per_sample: int, dtype=None, sign_mode: ESignMode = ESignMode.SIGNED, ) -> np.ndarray: """Build a look-up table (LUT) for unsigned to signed integer conversion. .. sealso:: :class:`ESignMode`. """ assert bits_per_sample <= 16 idtype = f"u{_get_item_size(bits_per_sample)}" data = np.arange(2**bits_per_sample, dtype=idtype) return unsigned_to_signed( data, bits_per_sample, dtype, sign_mode, inplace=True ) def unpackbits( data: bytes, bits_per_sample: int, samples_per_block: Optional[int] = None, bit_offset: int = 0, blockstride: Optional[int] = None, sign_mode: ESignMode = ESignMode.UNSIGNED, byteorder: str = ">", use_lut: bool = True, ) -> np.ndarray: """Unpack packed (integer) values form a string of bytes. Takes in input a string of bytes in which (integer) samples have been stored using ``bits_per_sample`` bit for each sample, and returns the sequence of corresponding Python integers. Example:: 3 bytes 4 samples |------|------|------|------| --> [samp_1, samp_2, samp_3, samp_4] 4 samples (6 bits per sample) :param data: bytes string of bytes containing the packed data :param bits_per_sample: int the number of bits used to encode each sample :param samples_per_block: int, optional the number of samples in each data block contained in the input string of bytes. This parameter is mostly relevant if the data block contains other information (or padding bits) in addition to the data samples. The number of blocks is deduced from the length of the input string of bytes, the number of samples per block and the number of bits per sample. If `samples_per_block` is not provided it is assumed a single block, and the number of samples is derived from the length of the input string of bytes and the number of bits per sample. :param bit_offset: int, optional the number of bits after which the sequence of samples (data blocks) starts (default: 0). It can be used e.g. to take into account of a possible binary header at the beginning of the sequence of samples. :param blockstride: int, optional the number of bits between the start of a data block and the start of the following one. This parameter is mostly relevant if the data block contains other information (or padding bits) in addition to the data samples. If not provided the `blockstride` is assumed to be equal to the size of the data block i.e. `bits_per_sample * samples_per_block`. :param sign_mode: ESignMode, optional specifies how the sign of the integer samples shall is encoded. Dy default unsigned samples are assumed. .. seealso:: :class:`ESignMode`. :param byteorder: str, optional Byte order of the encoded integers. Only relevant for multi byte samples. Default: ">" (big endian). :param use_lut: bool, optional specifies whenever the decoding of signed samples shall exploit look-up tables (typically faster). Default: True. """ signed = bool(sign_mode in {ESignMode.SIGNED, ESignMode.SIGN_AND_MOD}) if bit_offset == 0 and blockstride is None: if bits_per_sample == 1 and sign_mode == ESignMode.UNSIGNED: return np.unpackbits(np.frombuffer(data, dtype="uint8")) elif ( bits_per_sample in {8, 16, 32, 64} and sign_mode != ESignMode.SIGN_AND_MOD ): size = bits_per_sample // 8 kind = "i" if signed else "u" typestr = f"{byteorder}{kind}{size}" return np.frombuffer(data, dtype=np.dtype(typestr)) nbits = len(data) * 8 params = _unpackbits_params( nbits, bits_per_sample, samples_per_block, bit_offset, blockstride, signed, byteorder, ) samples, dtype, buf_itemsize, buf_dtype, index_map, shifts, mask = params npdata = np.frombuffer(data, dtype="u1") buf = np.empty(samples, dtype=buf_dtype) bytesview = buf.view(dtype="u1").reshape(samples, buf_itemsize) bytesview[...] = npdata[index_map] outdata = ((buf >> shifts) & mask).astype(dtype) if sign_mode == ESignMode.UNSIGNED: pass elif sign_mode in {ESignMode.SIGNED, ESignMode.SIGN_AND_MOD}: if not use_lut: outdata = unsigned_to_signed( outdata, bits_per_sample, dtype, sign_mode, inplace=True ) else: lut = make_unsigned_to_signed_lut( bits_per_sample, dtype, sign_mode ) outdata = lut[outdata] else: raise ValueError(f"Invalid 'sign_mode' parameter: '{sign_mode}'") return outdata bpack-1.1.0/bpack/st.py000066400000000000000000000143231441646351700146670ustar00rootroot00000000000000"""Struct based codec for binary data structures.""" import struct from typing import Optional import bpack import bpack.utils import bpack.codecs from .enums import EBaseUnits from .codecs import has_codec, get_codec from .descriptors import field_descriptors __all__ = [ "Decoder", "decoder", "Encoder", "encoder", "Codec", "codec", "BACKEND_NAME", "BACKEND_TYPE", ] BACKEND_NAME = "bitstruct" BACKEND_TYPE = EBaseUnits.BYTES _TYPE_SIGNED_AND_SIZE_TO_STR = { # type, signed, size (bool, None, 1): "?", (int, None, 1): "b", # default (int, False, 1): "B", (int, True, 1): "b", (int, None, 2): "h", # default (int, False, 2): "H", (int, True, 2): "h", (int, None, 4): "i", # default (int, False, 4): "I", (int, True, 4): "i", (int, None, 8): "q", # default (int, False, 8): "Q", (int, True, 8): "q", (float, None, 2): "e", (float, None, 4): "f", (float, None, 8): "d", (bytes, None, None): "s", (str, None, None): "s", (None, None, None): "x", # padding } _DEFAULT_SIZE = { bool: 1, int: 4, float: 4, bytes: 1, # str: 1, } def _format_string_without_order(fmt: str, order: str) -> str: # NOTE: in the current implementation the byte order is handled # externally to _to_fmt # if order != '': # fmt = fmt[1:] if fmt.startswith(order) else fmt # TODO: improve. # This is mainly a hack necessary because, in the current # implementation, _to_fmt is always called by Decoder.__init__ # with order='' so here it is not possible to rely on the value # of order if fmt[0] in {">", "<", "=", "@", "!"}: if order and fmt[0] != order: raise ValueError( f"inconsistent byteorder for nested record: " f"record byteorder is '{order}', " f"nested record byteorder is '{fmt[0]}'" ) fmt = fmt[1:] # else: # # TODO: how to check consistency when order=''? return fmt def _to_fmt( type_, size: Optional[int] = None, order: str = "", signed: Optional[bool] = None, repeat: Optional[int] = None, ) -> str: size = _DEFAULT_SIZE.get(type_, size) if size is None else size assert size is not None and size > 0 assert order in ("", ">", "<", "=", "@", "!"), f"invalid order: {order!r}" assert signed in (True, False, None) if has_codec(type_, bpack.codecs.Decoder): decoder_ = get_codec(type_) if isinstance(decoder_, Decoder): return _format_string_without_order(decoder_.format, order) elif ( bpack.is_descriptor(type_) and bpack.baseunits(type_) is Decoder.baseunits ): decoder_ = Decoder(type_) return _format_string_without_order(decoder_.format, order) etype = bpack.utils.effective_type(type_) repeat = 1 if repeat is None else repeat try: if etype in (str, bytes, None): # none is for padding bytes key = (etype, signed, None) return f"{order}{size}{_TYPE_SIGNED_AND_SIZE_TO_STR[key]}" * repeat else: key = (etype, signed, size) return f"{order}{repeat}{_TYPE_SIGNED_AND_SIZE_TO_STR[key]}" except KeyError: raise TypeError( f"unable to generate format string for " f"type='{type_}', size='{size}', order='{order}', " f"signed='{signed}', repeat='{repeat}'" ) def _enum_decode_converter_factory(type_, converters_map=None): converters_map = converters_map if converters_map is not None else dict() enum_item_type = bpack.utils.enum_item_type(type_) if enum_item_type in converters_map: base_converter = converters_map[enum_item_type] def to_enum(x): return type_(base_converter(x)) else: to_enum = type_ return to_enum def _enum_encode_converter_factory(type_, converters_map=None): converters_map = converters_map if converters_map is not None else dict() enum_item_type = bpack.utils.enum_item_type(type_) if enum_item_type in converters_map: base_converter = converters_map[enum_item_type] def from_enum(x): return base_converter(x.value) else: def from_enum(x): return x.value return from_enum class Codec(bpack.codecs.BaseStructCodec): """Struct based codec. Default byte-order: MSB. """ baseunits = EBaseUnits.BYTES @staticmethod def _get_base_codec(descriptor): byteorder = bpack.byteorder(descriptor) # assert all(descr.order for descr in field_descriptors(descriptor)) byteorder = byteorder.value # NOTE: struct expects that the byteorder specifier is used only # once at the beginning of the format string fmt = byteorder + "".join( _to_fmt( field_descr.type, field_descr.size, order="", repeat=field_descr.repeat, ) for field_descr in field_descriptors(descriptor, pad=True) ) return struct.Struct(fmt) @staticmethod def _get_decode_converters_map(descriptor): converters_map = { str: lambda s: s.decode("ascii"), } converters_map.update( ( field_descr.type, _enum_decode_converter_factory( field_descr.type, converters_map ), ) for field_descr in field_descriptors(descriptor) if bpack.utils.is_enum_type(field_descr.type) ) return converters_map @staticmethod def _get_encode_converters_map(descriptor): converters_map = { str: lambda s: s.encode("ascii"), } converters_map.update( ( field_descr.type, _enum_encode_converter_factory( field_descr.type, converters_map ), ) for field_descr in field_descriptors(descriptor) if bpack.utils.is_enum_type(field_descr.type) ) return converters_map codec = bpack.codecs.make_codec_decorator(Codec) Decoder = Encoder = Codec decoder = encoder = codec bpack-1.1.0/bpack/tests/000077500000000000000000000000001441646351700150265ustar00rootroot00000000000000bpack-1.1.0/bpack/tests/__init__.py000066400000000000000000000000271441646351700171360ustar00rootroot00000000000000"""Tests for bpack.""" bpack-1.1.0/bpack/tests/test_backends_codec.py000066400000000000000000001555321441646351700213610ustar00rootroot00000000000000"""Test bpack codecs.""" import sys import enum import struct import functools from typing import List, Sequence import pytest import bpack import bpack.st import bpack.codecs try: import bpack.bs as bpack_bs except ImportError: # pragma: no cover bpack_bs = None try: import bpack.ba as bpack_ba except ImportError: # pragma: no cover bpack_ba = None try: import bpack.np as bpack_np except ImportError: # pragma: no cover bpack_np = None skipif = pytest.mark.skipif BITS_BACKENDS = [ pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available") ), pytest.param( bpack_ba, id="ba", marks=skipif(not bpack_ba, reason="not available") ), ] BYTES_BACKENDS = [ pytest.param(bpack.st, id="st"), pytest.param( bpack_np, id="np", marks=skipif(not bpack_np, reason="not available") ), ] ALL_BACKENDS = BITS_BACKENDS + BYTES_BACKENDS @pytest.mark.parametrize("backend", ALL_BACKENDS) def test_backend(backend): assert hasattr(backend, "BACKEND_NAME") assert hasattr(backend, "BACKEND_TYPE") @pytest.mark.parametrize("backend", ALL_BACKENDS) def test_attrs(backend): codec = getattr(backend, "codec", backend.decoder) @codec @bpack.descriptor(baseunits=backend.BACKEND_TYPE) class Record: field_1: int = bpack.field(size=4, default=0) field_2: int = bpack.field(size=4, default=1) assert hasattr(Record, bpack.descriptors.BASEUNITS_ATTR_NAME) assert hasattr(Record, bpack.descriptors.BYTEORDER_ATTR_NAME) assert hasattr(Record, bpack.descriptors.BITORDER_ATTR_NAME) assert hasattr(Record, bpack.descriptors.SIZE_ATTR_NAME) assert hasattr(Record, bpack.codecs.CODEC_ATTR_NAME) @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.BE, bitorder=bpack.EBitOrder.MSB, frozen=True, ) class BitRecordBeMsb: # default (unsigned) field_01: bool = bpack.field(size=1, default=True) field_02: int = bpack.field(size=3, default=4) field_03: int = bpack.field(size=12, default=2048) field_04: float = bpack.field(size=32, default=1.0) field_05: bytes = bpack.field(size=24, default=b"abc") field_06: str = bpack.field(size=24, default="ABC") # 4 padding bits ([96:100]) field_08: int = bpack.field(size=28, default=134217727, offset=100) # signed field_11: bool = bpack.field(size=1, default=False) field_12: int = bpack.field(size=3, default=-4, signed=True) field_13: int = bpack.field(size=12, default=-2048, signed=True) field_18: int = bpack.field(size=32, default=-(2**31), signed=True) # unsigned field_21: bool = bpack.field(size=1, default=True) field_22: int = bpack.field(size=3, default=4, signed=False) field_23: int = bpack.field(size=12, default=2048, signed=False) field_28: int = bpack.field(size=32, default=2**31, signed=False) # fmt: off BIT_ENCODED_DATA_BE_MSB = [ # default (unsigned) bytes([ 0b11001000, 0b00000000, # fields 1 to 3 0b00111111, 0b10000000, 0b00000000, 0b00000000, # field_4 (float32) ]), b'abc', # field_5 (bytes) b'ABC', # field_6 (str) bytes([ # 4 padding bits + field_8 0b00000111, 0b11111111, 0b11111111, 0b11111111, ]), # signed bytes([ 0b01001000, 0b00000000, # fields 11 to 13 0b10000000, 0b00000000, 0b00000000, 0b00000000, # field_18 (uint32) ]), # unsigned bytes([ 0b11001000, 0b00000000, # fields 21 to 23 0b10000000, 0b00000000, 0b00000000, 0b00000000, # field_28 (sint32) ]), ] BIT_ENCODED_DATA_BE_MSB = b''.join(BIT_ENCODED_DATA_BE_MSB) # fmt: on @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.LE, bitorder=bpack.EBitOrder.MSB, frozen=True, ) class BitRecordLeMsb: # default (unsigned) field_01: bool = bpack.field(size=1, default=True) field_02: int = bpack.field(size=3, default=4) field_03: int = bpack.field(size=12, default=2048) field_04: float = bpack.field(size=32, default=1.0) field_05: bytes = bpack.field(size=24, default=b"abc") field_06: str = bpack.field(size=24, default="ABC") # 4 padding bits ([96:100]) field_08: int = bpack.field(size=28, default=134217727, offset=100) # signed field_11: bool = bpack.field(size=1, default=False) field_12: int = bpack.field(size=3, default=-4, signed=True) field_13: int = bpack.field(size=12, default=-2048, signed=True) field_18: int = bpack.field(size=32, default=-(2**31), signed=True) # unsigned field_21: bool = bpack.field(size=1, default=True) field_22: int = bpack.field(size=3, default=4, signed=False) field_23: int = bpack.field(size=12, default=2048, signed=False) field_28: int = bpack.field(size=32, default=2**31, signed=False) # fmt: off BIT_ENCODED_DATA_LE_MSB = [ # default (unsigned) bytes([ 0b11000000, 0b10000000, # fields 1 to 3 0b00000000, 0b00000000, 0b10000000, 0b00111111, # field_4 (float32) ]), b'abc', # field_5 (bytes) b'ABC', # field_6 (str) bytes([ # 4 padding bits + field_8 0b00001111, 0b11111111, 0b11111111, 0b01111111, ]), # signed bytes([ 0b01000000, 0b10000000, # fields 11 to 13 0b00000000, 0b00000000, 0b00000000, 0b10000000, # field_18 (sint32) ]), # unsigned bytes([ 0b11000000, 0b10000000, # fields 21 to 23 0b00000000, 0b00000000, 0b00000000, 0b10000000, # field_28 (uint32) ]), ] BIT_ENCODED_DATA_LE_MSB = b''.join(BIT_ENCODED_DATA_LE_MSB) # fmt: on @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.BE, bitorder=bpack.EBitOrder.LSB, frozen=True, ) class BitRecordBeLsb: # default (unsigned) field_01: bool = bpack.field(size=1, default=True) field_02: int = bpack.field(size=3, default=4) field_03: int = bpack.field(size=12, default=2048) field_04: float = bpack.field(size=32, default=1.0) field_05: bytes = bpack.field(size=24, default=b"abc") field_06: str = bpack.field(size=24, default="ABC") # 4 padding bits ([96:100]) field_08: int = bpack.field(size=28, default=134217727, offset=100) # signed field_11: bool = bpack.field(size=1, default=False) field_12: int = bpack.field(size=3, default=-4, signed=True) field_13: int = bpack.field(size=12, default=-2048, signed=True) field_18: int = bpack.field(size=32, default=-(2**31), signed=True) # unsigned field_21: bool = bpack.field(size=1, default=True) field_22: int = bpack.field(size=3, default=4, signed=False) field_23: int = bpack.field(size=12, default=2048, signed=False) field_28: int = bpack.field(size=32, default=2**31, signed=False) # fmt: off BIT_ENCODED_DATA_BE_LSB = [ # default (unsigned) bytes([ 0b10010000, 0b00000001, # fields 1 to 3 0b00000000, 0b00000000, 0b00000001, 0b11111100, # field_4 (float32) ]), bytes([0b11000110, 0b01000110, 0b10000110]), # field_5 (bytes) - b'abc' bytes([0b11000010, 0b01000010, 0b10000010]), # field_6 (str) - 'ABC' bytes([ # 4 padding bits + field_8 0b00001111, 0b11111111, 0b11111111, 0b11111110, ]), # signed bytes([ 0b00010000, 0b00000001, # fields 11 to 13 0b00000000, 0b00000000, 0b00000000, 0b00000001, # field_18 (uint32) ]), # unsigned bytes([ 0b10010000, 0b00000001, # fields 21 to 23 0b00000000, 0b00000000, 0b00000000, 0b00000001, # field_28 (sint32) ]), ] BIT_ENCODED_DATA_BE_LSB = b''.join(BIT_ENCODED_DATA_BE_LSB) # fmt: on @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.LE, bitorder=bpack.EBitOrder.LSB, frozen=True, ) class BitRecordLeLsb: # default (unsigned) field_01: bool = bpack.field(size=1, default=True) field_02: int = bpack.field(size=3, default=4) field_03: int = bpack.field(size=12, default=2048) field_04: float = bpack.field(size=32, default=1.0) field_05: bytes = bpack.field(size=24, default=b"abc") field_06: str = bpack.field(size=24, default="ABC") # 4 padding bits ([96:100]) field_08: int = bpack.field(size=28, default=134217727, offset=100) # signed field_11: bool = bpack.field(size=1, default=False) field_12: int = bpack.field(size=3, default=-4, signed=True) field_13: int = bpack.field(size=12, default=-2048, signed=True) field_18: int = bpack.field(size=32, default=-(2**31), signed=True) # unsigned field_21: bool = bpack.field(size=1, default=True) field_22: int = bpack.field(size=3, default=4, signed=False) field_23: int = bpack.field(size=12, default=2048, signed=False) field_28: int = bpack.field(size=32, default=2**31, signed=False) # fmt: off BIT_ENCODED_DATA_LE_LSB = [ # default (unsigned) bytes([ 0b10010001, 0b00000000, # fields 1 to 3 0b11111100, 0b00000001, 0b00000000, 0b00000000, # field_4 (float32) ]), bytes([0b11000110, 0b01000110, 0b10000110]), # field_5 (bytes) - b'abc' bytes([0b11000010, 0b01000010, 0b10000010]), # field_6 (str) - 'ABC' bytes([ # 4 padding bits + field_8 0b00001110, 0b11111111, 0b11111111, 0b11111111, ]), # signed bytes([ 0b00010001, 0b00000000, # fields 11 to 13 0b00000001, 0b00000000, 0b00000000, 0b00000000, # field_18 (sint32) ]), # unsigned bytes([ 0b10010001, 0b00000000, # fields 21 to 23 0b00000001, 0b00000000, 0b00000000, 0b00000000, # field_28 (uint32) ]), ] BIT_ENCODED_DATA_LE_LSB = b''.join(BIT_ENCODED_DATA_LE_LSB) # fmt: on @bpack.descriptor( baseunits=bpack.EBaseUnits.BYTES, byteorder=bpack.EByteOrder.BE, frozen=True, ) class ByteRecordBe: field_01: bool = bpack.field(size=1, default=False) field_02: int = bpack.field(size=1, default=1) field_03: int = bpack.field(size=1, default=-1, signed=True) field_04: int = bpack.field(size=1, default=+1, signed=False) field_05: int = bpack.field(size=2, default=2) field_06: int = bpack.field(size=2, default=-2, signed=True) field_07: int = bpack.field(size=2, default=+2, signed=False) field_08: int = bpack.field(size=4, default=4) field_09: int = bpack.field(size=4, default=-4, signed=True) field_10: int = bpack.field(size=4, default=+4, signed=False) field_11: int = bpack.field(size=8, default=8) field_12: int = bpack.field(size=8, default=-8, signed=True) field_13: int = bpack.field(size=8, default=+8, signed=False) field_14: float = bpack.field(size=2, default=10.0) field_15: float = bpack.field(size=4, default=100.0) field_16: float = bpack.field(size=8, default=1000.0) field_17: bytes = bpack.field(size=3, default=b"abc") field_18: str = bpack.field(size=3, default="ABC") # 4 padding bytes ([66:70]) b'xxxx' field_20: bytes = bpack.field(size=4, offset=70, default=b"1234") # fmt: off BYTE_ENCODED_DATA_BE = [ False, # bool (8 bit) 0b00000001, # +1 sint8 0b11111111, # -1 sint8 0b00000001, # +1 uint8 0b00000000, 0b00000010, # +2 sint16 0b11111111, 0b11111110, # -2 sint16 0b00000000, 0b00000010, # +2 uint16 0b00000000, 0b00000000, 0b00000000, 0b00000100, # +4 sint32 0b11111111, 0b11111111, 0b11111111, 0b11111100, # -4 sint32 0b00000000, 0b00000000, 0b00000000, 0b00000100, # +4 uint32 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00001000, # +8 sint64 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111000, # -8 sint64 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00001000, # +8 uint64 0b01001001, 0b00000000, # 10 float16 0b01000010, 0b11001000, 0b00000000, 0b00000000, # 100 float32 0b01000000, 0b10001111, 0b01000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, # 1000 float64 0b01100001, 0b01100010, 0b01100011, # b'abc' bytes 0b01000001, 0b01000010, 0b01000011, # 'ABC' str # 4 padding bytes ([36:40]) 0b01111000, 0b01111000, 0b01111000, 0b01111000, # b'xxxx' bytes # field_20 0b00110001, 0b00110010, 0b00110011, 0b00110100, # b'1234' bytes ] BYTE_ENCODED_DATA_BE = bytes(BYTE_ENCODED_DATA_BE) # fmt: on @bpack.descriptor( baseunits=bpack.EBaseUnits.BYTES, byteorder=bpack.EByteOrder.LE, frozen=True, ) class ByteRecordLe: field_01: bool = bpack.field(size=1, default=False) field_02: int = bpack.field(size=1, default=1) field_03: int = bpack.field(size=1, default=-1, signed=True) field_04: int = bpack.field(size=1, default=+1, signed=False) field_05: int = bpack.field(size=2, default=2) field_06: int = bpack.field(size=2, default=-2, signed=True) field_07: int = bpack.field(size=2, default=+2, signed=False) field_08: int = bpack.field(size=4, default=4) field_09: int = bpack.field(size=4, default=-4, signed=True) field_10: int = bpack.field(size=4, default=+4, signed=False) field_11: int = bpack.field(size=8, default=8) field_12: int = bpack.field(size=8, default=-8, signed=True) field_13: int = bpack.field(size=8, default=+8, signed=False) field_14: float = bpack.field(size=2, default=10.0) field_15: float = bpack.field(size=4, default=100.0) field_16: float = bpack.field(size=8, default=1000.0) field_17: bytes = bpack.field(size=3, default=b"abc") field_18: str = bpack.field(size=3, default="ABC") # 4 padding bytes ([66:70]) b'xxxx' field_20: bytes = bpack.field(size=4, offset=70, default=b"1234") # fmt: off BYTE_ENCODED_DATA_LE = [ False, # bool (8 bit) 0b00000001, # +1 sint8 0b11111111, # -1 sint8 0b00000001, # +1 uint8 0b00000010, 0b00000000, # +2 sint16 0b11111110, 0b11111111, # -2 sint16 0b00000010, 0b00000000, # +2 uint16 0b00000100, 0b00000000, 0b00000000, 0b00000000, # +4 sint32 0b11111100, 0b11111111, 0b11111111, 0b11111111, # -4 sint32 0b00000100, 0b00000000, 0b00000000, 0b00000000, # +4 uint32 0b00001000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, # +8 sint64 0b11111000, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, # -8 sint64 0b00001000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, # +8 uint64 0b00000000, 0b01001001, # 10 float16 0b00000000, 0b00000000, 0b11001000, 0b01000010, # 100 float32 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b01000000, 0b10001111, 0b01000000, # 1000 float64 0b01100001, 0b01100010, 0b01100011, # b'abc' bytes 0b01000001, 0b01000010, 0b01000011, # 'ABC' str # 4 padding bytes ([66:70]) 0b01111000, 0b01111000, 0b01111000, 0b01111000, # b'xxxx' bytes # field_20 0b00110001, 0b00110010, 0b00110011, 0b00110100, # b'1234' bytes ] BYTE_ENCODED_DATA_LE = bytes(BYTE_ENCODED_DATA_LE) # fmt: on def _fix_padding(data, refdata): if refdata in {BYTE_ENCODED_DATA_BE, BYTE_ENCODED_DATA_LE}: return data[:66] + b"xxxx" + data[70:] return refdata @pytest.mark.parametrize( "backend, Record, encoded_data", [ pytest.param(bpack.st, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="st BE"), pytest.param(bpack.st, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="st LE"), pytest.param( bpack_np, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="np BE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_np, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="np LE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeMsb, BIT_ENCODED_DATA_BE_MSB, id="bs BE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeMsb, BIT_ENCODED_DATA_LE_MSB, id="bs LE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeLsb, BIT_ENCODED_DATA_BE_LSB, id="bs BE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeLsb, BIT_ENCODED_DATA_LE_LSB, id="bs LE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_ba, BitRecordBeMsb, BIT_ENCODED_DATA_BE_MSB, id="ba BE MSB", marks=pytest.mark.skipif(not bpack_ba, reason="not available"), ), ], ) def test_decoder(backend, Record, encoded_data): # noqa: N803 decoded_data = Record() decoder = backend.Decoder(Record) assert hasattr(decoder, "baseunits") assert decoder.baseunits is bpack.baseunits(Record) record = decoder.decode(encoded_data) assert record == decoded_data if hasattr(backend, "Codec"): codec = backend.Codec(Record) assert hasattr(codec, "baseunits") assert codec.baseunits is bpack.baseunits(Record) record = codec.decode(encoded_data) assert record == decoded_data @pytest.mark.parametrize( "backend, Record, encoded_data", [ pytest.param(bpack.st, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="st BE"), pytest.param(bpack.st, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="st LE"), pytest.param( bpack_np, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="np BE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_np, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="np LE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeMsb, BIT_ENCODED_DATA_BE_MSB, id="bs BE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeMsb, BIT_ENCODED_DATA_LE_MSB, id="bs LE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeLsb, BIT_ENCODED_DATA_BE_LSB, id="bs BE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeLsb, BIT_ENCODED_DATA_LE_LSB, id="bs LE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_ba, BitRecordBeMsb, BIT_ENCODED_DATA_BE_MSB, id="ba BE MSB", marks=pytest.mark.skipif(not bpack_ba, reason="not available"), ), ], ) def test_decoder_func(backend, Record, encoded_data): # noqa: N803 decoded_data = Record() record_type = backend.decoder(Record) record = record_type.frombytes(encoded_data) assert record == decoded_data if hasattr(backend, "codec"): record_type = backend.codec(Record) record = record_type.frombytes(encoded_data) assert record == decoded_data @pytest.mark.parametrize( "backend, Record, encoded_data", [ pytest.param(bpack.st, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="st BE"), pytest.param(bpack.st, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="st LE"), pytest.param( bpack_np, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="np BE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_np, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="np LE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeMsb, BIT_ENCODED_DATA_BE_MSB, id="bs BE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeMsb, BIT_ENCODED_DATA_LE_MSB, id="bs LE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeLsb, BIT_ENCODED_DATA_BE_LSB, id="bs BE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeLsb, BIT_ENCODED_DATA_LE_LSB, id="bs LE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), ], ) def test_encoder(backend, Record, encoded_data): # noqa: N803 record = Record() encoder = backend.Encoder(Record) assert hasattr(encoder, "baseunits") assert encoder.baseunits is bpack.baseunits(Record) data = encoder.encode(record) if backend.BACKEND_TYPE is bpack.EBaseUnits.BYTES: data = _fix_padding(data, encoded_data) assert data == encoded_data codec = backend.Codec(Record) assert hasattr(codec, "baseunits") assert codec.baseunits is bpack.baseunits(Record) data = codec.encode(record) if backend.BACKEND_TYPE is bpack.EBaseUnits.BYTES: data = _fix_padding(data, encoded_data) assert data == encoded_data @pytest.mark.parametrize( "backend, Record, encoded_data", [ pytest.param(bpack.st, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="st BE"), pytest.param(bpack.st, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="st LE"), pytest.param( bpack_np, ByteRecordBe, BYTE_ENCODED_DATA_BE, id="np BE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_np, ByteRecordLe, BYTE_ENCODED_DATA_LE, id="np LE", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeMsb, BIT_ENCODED_DATA_BE_MSB, id="bs BE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeMsb, BIT_ENCODED_DATA_LE_MSB, id="bs LE MSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordBeLsb, BIT_ENCODED_DATA_BE_LSB, id="bs BE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_bs, BitRecordLeLsb, BIT_ENCODED_DATA_LE_LSB, id="bs LE LSB", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), ], ) def test_encoder_func(backend, Record, encoded_data): # noqa: N803 record = Record() record_type = backend.encoder(Record) data = record_type.tobytes(record) if backend.BACKEND_TYPE is bpack.EBaseUnits.BYTES: data = _fix_padding(data, encoded_data) assert data == encoded_data record_type = backend.codec(Record) data = record_type.tobytes(record) if backend.BACKEND_TYPE is bpack.EBaseUnits.BYTES: data = _fix_padding(data, encoded_data) assert data == encoded_data @pytest.mark.parametrize("backend", BITS_BACKENDS) def test_bit_decoder_decorator_frombytes(backend): @backend.decoder @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS, frozen=True) class Record: field_01: bool = bpack.field(size=1, default=True) field_02: int = bpack.field(size=3, default=4) field_03: int = bpack.field(size=12, default=2048) field_04: float = bpack.field(size=32, default=1.0) field_05: bytes = bpack.field(size=24, default=b"abc") field_06: str = bpack.field(size=24, default="ABC") # 4 padding bits ([96:100]) 0b1111 field_08: int = bpack.field(size=28, default=134217727, offset=100) decoded_data = Record() encoded_data = BIT_ENCODED_DATA_BE_MSB[: bpack.calcsize(Record)] record = Record.frombytes(encoded_data) assert record.field_01 == decoded_data.field_01 assert record.field_02 == decoded_data.field_02 assert record.field_03 == decoded_data.field_03 assert record.field_04 == decoded_data.field_04 assert record.field_05 == decoded_data.field_05 assert record.field_06 == decoded_data.field_06 assert record.field_08 == decoded_data.field_08 @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ) ], ) def test_bit_encoder_decorator_tobytes(backend): @backend.encoder @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS, frozen=True) class Record: field_01: bool = bpack.field(size=1, default=True) field_02: int = bpack.field(size=3, default=4) field_03: int = bpack.field(size=12, default=2048) field_04: float = bpack.field(size=32, default=1.0) field_05: bytes = bpack.field(size=24, default=b"abc") field_06: str = bpack.field(size=24, default="ABC") # 4 padding bits ([96:100]) 0b1111 field_08: int = bpack.field(size=28, default=134217727, offset=100) encoded_data = BIT_ENCODED_DATA_BE_MSB[: bpack.calcsize(Record)] record = Record() data = record.tobytes() data = _fix_padding(data, encoded_data) assert data == encoded_data @pytest.mark.parametrize("backend", BYTES_BACKENDS) def test_byte_decoder_decorator_frombytes(backend): @backend.decoder @bpack.descriptor( baseunits=bpack.EBaseUnits.BYTES, byteorder=bpack.EByteOrder.BE, frozen=True, ) class Record: field_1: int = bpack.field(size=1, default=1) field_2: int = bpack.field(size=2, default=2) decoded_data = Record() encoded_data = bytes([0b00000001, 0b00000000, 0b00000010]) record = Record.frombytes(encoded_data) assert record.field_1 == decoded_data.field_1 assert record.field_2 == decoded_data.field_2 @pytest.mark.parametrize("backend", BYTES_BACKENDS) def test_byte_encoder_decorator_tobytes(backend): @backend.encoder @bpack.descriptor( baseunits=bpack.EBaseUnits.BYTES, byteorder=bpack.EByteOrder.BE, frozen=True, ) class Record: field_1: int = bpack.field(size=1, default=1) field_2: int = bpack.field(size=2, default=2) encoded_data = bytes([0b00000001, 0b00000000, 0b00000010]) record = Record() data = record.tobytes() assert data == encoded_data @pytest.mark.parametrize("backend", ALL_BACKENDS) def test_unsupported_type(backend): class CustomType: pass codec = getattr(backend, "codec", backend.decoder) with pytest.raises(TypeError): @codec @bpack.descriptor(baseunits=backend.Decoder.baseunits, frozen=True) class Record: field_1: CustomType = bpack.field(size=8) @pytest.mark.parametrize("backend", BYTES_BACKENDS) def test_byte_decoder_native_byteorder_frombytes(backend): size = 4 value = 1 @backend.decoder @bpack.descriptor(byteorder=bpack.EByteOrder.NATIVE, frozen=True) class Record: field_1: int = bpack.field(size=size, default=value) data = value.to_bytes(size, sys.byteorder) assert Record.frombytes(data) == Record() @pytest.mark.parametrize("backend", BYTES_BACKENDS) def test_byte_encoder_native_byteorder_tobytes(backend): size = 4 value = 1 @backend.encoder @bpack.descriptor(byteorder=bpack.EByteOrder.NATIVE, frozen=True) class Record: field_1: int = bpack.field(size=size, default=value) record = Record() data = value.to_bytes(size, sys.byteorder) assert record.tobytes() == data @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ) ], ) def test_bit_decoder_native_byteorder_frombytes(backend): size = 64 value = 1 @backend.decoder @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.NATIVE, frozen=True, ) class Record: field_1: int = bpack.field(size=size, default=value) data = value.to_bytes(size // 8, sys.byteorder) assert Record.frombytes(data) == Record() @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ) ], ) def test_bit_encoder_native_byteorder_tobytes(backend): size = 64 value = 1 @backend.encoder @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.NATIVE, frozen=True, ) class Record: field_1: int = bpack.field(size=size, default=value) record = Record() data = value.to_bytes(size // 8, sys.byteorder) assert record.tobytes() == data @pytest.mark.parametrize("backend", BITS_BACKENDS) def test_bit_decoder_default_byteorder_frombytes(backend): size = 64 value = 1 @backend.decoder @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.DEFAULT, frozen=True, ) class Record: field_1: int = bpack.field(size=size, default=value) # default byte order is big for bit descriptors data = value.to_bytes(size // 8, "big") assert Record.frombytes(data) == Record() @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ) ], ) def test_bit_encoder_default_byteorder_tobytes(backend): size = 64 value = 1 @backend.encoder @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.DEFAULT, frozen=True, ) class Record: field_1: int = bpack.field(size=size, default=value) record = Record() # default byte order is big for bit descriptors data = value.to_bytes(size // 8, "big") # default is "big" for bits assert record.tobytes() == data @pytest.mark.parametrize("backend", BITS_BACKENDS) def test_wrong_baseunits_bit(backend): codec = getattr(backend, "codec", backend.decoder) with pytest.raises(ValueError): @codec @bpack.descriptor(baseunits=bpack.EBaseUnits.BYTES) class Record: field_1: int = bpack.field(size=8, default=1) @pytest.mark.parametrize("backend", BYTES_BACKENDS) def test_wrong_baseunits_byte(backend): codec = getattr(backend, "codec", backend.decoder) with pytest.raises(ValueError): @codec @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=8, default=1) @pytest.mark.parametrize("backend", ALL_BACKENDS) def test_enum_decoding_bytes(backend): class EStrEnumType(enum.Enum): A = "a" B = "b" class EBytesEnumType(enum.Enum): A = b"a" B = b"b" class EIntEnumType(enum.Enum): A = 1 B = 2 class EFlagEnumType(enum.Enum): A = 1 B = 2 if backend.Decoder.baseunits is bpack.EBaseUnits.BYTES: bitorder = None ssize = 1 isize = 1 encoded_data = b"".join( [ EStrEnumType.A.value.encode("ascii"), EBytesEnumType.A.value, EIntEnumType.A.value.to_bytes(1, "little", signed=False), EFlagEnumType.A.value.to_bytes(1, "little", signed=False), ] ) else: bitorder = bpack.EBitOrder.MSB ssize = 8 isize = 4 encoded_data = b"".join( [ EStrEnumType.A.value.encode("ascii"), EBytesEnumType.A.value, bytes([0b00010001]), ] ) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, bitorder=bitorder) class Record: field_1: EStrEnumType = bpack.field(size=ssize, default=EStrEnumType.A) field_2: EBytesEnumType = bpack.field( size=ssize, default=EBytesEnumType.A ) field_3: EIntEnumType = bpack.field(size=isize, default=EIntEnumType.A) field_4: EFlagEnumType = bpack.field( size=isize, default=EFlagEnumType.A ) record = Record.frombytes(encoded_data) assert record == Record() @pytest.mark.parametrize( "backend", [ pytest.param(bpack.st, id="st"), pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_np, id="np", marks=skipif(not bpack_np, reason="not available"), ), ], ) def test_enum_encoding_bytes(backend): class EStrEnumType(enum.Enum): A = "a" B = "b" class EBytesEnumType(enum.Enum): A = b"a" B = b"b" class EIntEnumType(enum.Enum): A = 1 B = 2 class EFlagEnumType(enum.Enum): A = 1 B = 2 if backend.Decoder.baseunits is bpack.EBaseUnits.BYTES: bitorder = None ssize = 1 isize = 1 encoded_data = [ EStrEnumType.A.value.encode("ascii"), EBytesEnumType.A.value, EIntEnumType.A.value.to_bytes(1, "little", signed=False), EFlagEnumType.A.value.to_bytes(1, "little", signed=False), ] encoded_data = b"".join(encoded_data) else: bitorder = bpack.EBitOrder.MSB ssize = 8 isize = 4 encoded_data = [ EStrEnumType.A.value.encode("ascii"), EBytesEnumType.A.value, bytes([0b00010001]), ] encoded_data = b"".join(encoded_data) @backend.encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, bitorder=bitorder) class Record: field_1: EStrEnumType = bpack.field(size=ssize, default=EStrEnumType.A) field_2: EBytesEnumType = bpack.field( size=ssize, default=EBytesEnumType.A ) field_3: EIntEnumType = bpack.field(size=isize, default=EIntEnumType.A) field_4: EFlagEnumType = bpack.field( size=isize, default=EFlagEnumType.A ) record = Record() data = record.tobytes() assert data == encoded_data @pytest.mark.parametrize( "backend", [ pytest.param(bpack.st, id="st"), pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ), ], ) def test_decode_sequence(backend): if backend.Decoder.baseunits is bpack.EBaseUnits.BYTES: bitorder = None size = 1 repeat = 2 encoded_data = bytes([3, 3, 4, 4]) else: bitorder = bpack.EBitOrder.MSB size = 4 repeat = 2 encoded_data = bytes([0b00110011, 0b01000100]) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, bitorder=bitorder) class Record: field_1: List[int] = bpack.field(size=size, repeat=repeat) field_2: Sequence[int] = bpack.field(size=size, repeat=repeat) ref_record = Record([3, 3], (4, 4)) record = Record.frombytes(encoded_data) assert record == ref_record assert type(record.field_1) is list assert type(record.field_2) is tuple for field, sequence_type in zip( bpack.fields(Record), (List[int], Sequence[int]) ): assert field.type == sequence_type @pytest.mark.parametrize( "backend", [ pytest.param(bpack.st, id="st"), pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ), ], ) def test_encode_sequence(backend): if backend.Decoder.baseunits is bpack.EBaseUnits.BYTES: bitorder = None size = 1 repeat = 2 encoded_data = bytes([3, 3, 4, 4]) else: bitorder = bpack.EBitOrder.MSB size = 4 repeat = 2 encoded_data = bytes([0b00110011, 0b01000100]) @backend.encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, bitorder=bitorder) class Record: field_1: List[int] = bpack.field(size=size, repeat=repeat) field_2: Sequence[int] = bpack.field(size=size, repeat=repeat) record = Record([3, 3], (4, 4)) data = record.tobytes() assert data == encoded_data @pytest.mark.parametrize( "backend", [ pytest.param(bpack.st, id="st"), pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_np, id="np", marks=skipif(not bpack_np, reason="not available"), ), ], ) class TestNestedRecord: @staticmethod def get_encoded_data(baseunits): if baseunits is bpack.EBaseUnits.BYTES: # TODO: use the default byte order # fmt: off encoded_data = [ 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000001, 0b00000000, 0b00000000, 0b00000000, 0b00000010, 0b00000000, 0b00000000, 0b00000000, 0b00000011, 0b00000000, 0b00000000, 0b00000000, 0b00000001, 0b00000000, 0b00000000, 0b00000000, 0b00000010, 0b00000000, 0b00000000, 0b00000000, ] encoded_data = bytes(encoded_data) # fmt: on else: # baseunits is bpack.EBaseUnits.BITS: encoded_data = bytes([0b00000001, 0b00100011, 0b000010010]) return encoded_data def test_nested_record_decoder(self, backend): encoded_data = self.get_encoded_data(backend.Decoder.baseunits) @backend.decoder # NOTE: this is a decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: Record = bpack.field(default_factory=Record) field_3: int = bpack.field(size=4, default=3) field_4: Record = bpack.field(default_factory=Record) assert NestedRecord.frombytes(encoded_data) == NestedRecord() def test_nested_record_encoder(self, backend): encoded_data = self.get_encoded_data(backend.Decoder.baseunits) @backend.encoder # NOTE: this is an encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @backend.encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: Record = bpack.field(default_factory=Record) field_3: int = bpack.field(size=4, default=3) field_4: Record = bpack.field(default_factory=Record) record = NestedRecord() assert record.tobytes() == encoded_data def test_nested_record_frombytes(self, backend): encoded_data = self.get_encoded_data(backend.Decoder.baseunits) # NOTE: this time the inner record is not a decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: Record = bpack.field(default_factory=Record) field_3: int = bpack.field(size=4, default=3) field_4: Record = bpack.field(default_factory=Record) assert NestedRecord.frombytes(encoded_data) == NestedRecord() def test_nested_record_tobytes(self, backend): encoded_data = self.get_encoded_data(backend.Decoder.baseunits) # NOTE: this time the inner record is not a decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @backend.encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: Record = bpack.field(default_factory=Record) field_3: int = bpack.field(size=4, default=3) field_4: Record = bpack.field(default_factory=Record) record = NestedRecord() assert record.tobytes() == encoded_data def test_nested_record_decoder_with_order(self, backend): encoded_data = self.get_encoded_data(backend.Decoder.baseunits) if backend.Decoder.baseunits is bpack.EBaseUnits.BITS: kwargs = dict(bitorder=">", byteorder=">") else: # TODO: use the default byteorder (see get_encoded_data) kwargs = dict(byteorder=bpack.EByteOrder.LE) @backend.decoder # NOTE: this is a decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, **kwargs) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, **kwargs) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: Record = bpack.field(default_factory=Record) field_3: int = bpack.field(size=4, default=3) field_4: Record = bpack.field(default_factory=Record) assert NestedRecord.frombytes(encoded_data) == NestedRecord() def test_nested_record_encoder_with_order(self, backend): encoded_data = self.get_encoded_data(backend.Decoder.baseunits) if backend.Decoder.baseunits is bpack.EBaseUnits.BITS: kwargs = dict(bitorder=">", byteorder=">") else: # TODO: use the default byteorder (see get_encoded_data) kwargs = dict(byteorder=bpack.EByteOrder.LE) @backend.encoder # NOTE: this is a encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, **kwargs) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @backend.encoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, **kwargs) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: Record = bpack.field(default_factory=Record) field_3: int = bpack.field(size=4, default=3) field_4: Record = bpack.field(default_factory=Record) record = NestedRecord() assert record.tobytes() == encoded_data @pytest.mark.parametrize( "backend", [ pytest.param(bpack.st, id="st"), # pytest.param(bpack_np, id='np', # marks=skipif(not bpack_np, reason='not available')) pytest.param( bpack_bs, id="bs", marks=skipif(not bpack_bs, reason="not available"), ), ], ) class TestMultiNestedRecord: def _record_to_list(self, record): from bpack.utils import is_sequence_type out = [] for filed in bpack.fields(record): value = getattr(record, filed.name) if bpack.is_descriptor(value): out.extend(self._record_to_list(value)) elif is_sequence_type(filed.type): out.extend(value) else: out.append(value) return out def _bytes_record_to_data(self, record): from bpack import st as _st values = self._record_to_list(record) decoder = _st.Decoder(type(record)) fmt = decoder.format values = [getattr(v, "value", v) for v in values] return struct.pack(fmt, *values) def _bits_record_to_data(self, record): # TODO: check import bitstruct from bpack import bs as _bs values = self._record_to_list(record) decoder = _bs.Decoder(type(record)) fmt = decoder.format values = [getattr(v, "value", v) for v in values] return bitstruct.pack(fmt, *values) def _record_to_data(self, record): baseunits = bpack.baseunits(record) if baseunits is bpack.EBaseUnits.BYTES: data = self._bytes_record_to_data(record) else: data = self._bits_record_to_data(record) nbytes = bpack.calcsize(record, bpack.EBaseUnits.BYTES) assert len(data) == nbytes return data def test_decode_nested_record_two_levels(self, backend): class EEnum(enum.Enum): ONE = 1 FOUR = 4 @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel02: field_01: EEnum = bpack.field(size=4, default=EEnum.ONE) field_02: int = bpack.field(size=4, default=2) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: RecordLevel02 = bpack.field(default_factory=RecordLevel02) field_3: int = bpack.field(size=4, default=3) field_4: RecordLevel02 = bpack.field( default_factory=functools.partial(RecordLevel02, EEnum.FOUR, 5) ) record = NestedRecord() encoded_data = self._record_to_data(record) assert NestedRecord.frombytes(encoded_data) == record def test_encode_nested_record_two_levels(self, backend): class EEnum(enum.Enum): ONE = 1 FOUR = 4 @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel02: field_01: EEnum = bpack.field(size=4, default=EEnum.ONE) field_02: int = bpack.field(size=4, default=2) @backend.codec @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: RecordLevel02 = bpack.field(default_factory=RecordLevel02) field_3: int = bpack.field(size=4, default=3) field_4: RecordLevel02 = bpack.field( default_factory=functools.partial(RecordLevel02, EEnum.FOUR, 5) ) record = NestedRecord() encoded_data = self._record_to_data(record) assert record.tobytes() == encoded_data def test_decode_nested_record_three_levels(self, backend): class EEnum(enum.Enum): ONE = 1 TWO = 2 SEVEN = 7 @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel03: field_001: EEnum = bpack.field(size=4, default=EEnum.ONE) field_002: int = bpack.field(size=4, default=2) @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel02: field_01: int = bpack.field(size=4, default=1) field_02: RecordLevel03 = bpack.field( default_factory=functools.partial(RecordLevel03, EEnum.TWO, 3) ) field_03: int = bpack.field(size=4, default=4) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: RecordLevel02 = bpack.field(default_factory=RecordLevel02) field_3: int = bpack.field(size=4, default=5) field_4: RecordLevel02 = bpack.field( default_factory=functools.partial( RecordLevel02, 6, RecordLevel03(EEnum.SEVEN, 8), 9 ) ) record = NestedRecord() encoded_data = self._record_to_data(record) assert NestedRecord.frombytes(encoded_data) == record def test_encode_nested_record_three_levels(self, backend): class EEnum(enum.Enum): ONE = 1 TWO = 2 SEVEN = 7 @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel03: field_001: EEnum = bpack.field(size=4, default=EEnum.ONE) field_002: int = bpack.field(size=4, default=2) @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel02: field_01: int = bpack.field(size=4, default=1) field_02: RecordLevel03 = bpack.field( default_factory=functools.partial(RecordLevel03, EEnum.TWO, 3) ) field_03: int = bpack.field(size=4, default=4) @backend.codec @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: RecordLevel02 = bpack.field(default_factory=RecordLevel02) field_3: int = bpack.field(size=4, default=5) field_4: RecordLevel02 = bpack.field( default_factory=functools.partial( RecordLevel02, 6, RecordLevel03(EEnum.SEVEN, 8), 9 ) ) record = NestedRecord() encoded_data = self._record_to_data(record) assert record.tobytes() == encoded_data def test_decode_nested_record_four_levels(self, backend): class EEnum(enum.Enum): ONE = 1 THREE = 3 TEN = 10 @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel04: field_0001: EEnum = bpack.field(size=4, default=EEnum.ONE) field_0002: int = bpack.field(size=4, default=2) @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel03: field_001: int = bpack.field(size=4, default=1) field_002: RecordLevel04 = bpack.field( default_factory=RecordLevel04 ) field_003: int = bpack.field(size=4, default=3) @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel02: field_01: int = bpack.field(size=4, default=1) field_02: RecordLevel03 = bpack.field( default_factory=functools.partial( RecordLevel03, 2, RecordLevel04(EEnum.THREE, 4), 5 ) ) field_03: int = bpack.field(size=4, default=6) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: RecordLevel02 = bpack.field(default_factory=RecordLevel02) field_3: int = bpack.field(size=4, default=7) field_4: RecordLevel02 = bpack.field( default_factory=functools.partial( RecordLevel02, 8, RecordLevel03(9, RecordLevel04(EEnum.TEN, 11), 12), 13, ) ) record = NestedRecord() encoded_data = self._record_to_data(record) assert NestedRecord.frombytes(encoded_data) == record def test_encode_nested_record_four_levels(self, backend): class EEnum(enum.Enum): ONE = 1 THREE = 3 TEN = 10 @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel04: field_0001: EEnum = bpack.field(size=4, default=EEnum.ONE) field_0002: int = bpack.field(size=4, default=2) @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel03: field_001: int = bpack.field(size=4, default=1) field_002: RecordLevel04 = bpack.field( default_factory=RecordLevel04 ) field_003: int = bpack.field(size=4, default=3) @bpack.descriptor(baseunits=backend.Decoder.baseunits) class RecordLevel02: field_01: int = bpack.field(size=4, default=1) field_02: RecordLevel03 = bpack.field( default_factory=functools.partial( RecordLevel03, 2, RecordLevel04(EEnum.THREE, 4), 5 ) ) field_03: int = bpack.field(size=4, default=6) @backend.codec @bpack.descriptor(baseunits=backend.Decoder.baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=0) field_2: RecordLevel02 = bpack.field(default_factory=RecordLevel02) field_3: int = bpack.field(size=4, default=7) field_4: RecordLevel02 = bpack.field( default_factory=functools.partial( RecordLevel02, 8, RecordLevel03(9, RecordLevel04(EEnum.TEN, 11), 12), 13, ) ) record = NestedRecord() encoded_data = self._record_to_data(record) assert record.tobytes() == encoded_data bpack-1.1.0/bpack/tests/test_codecs.py000066400000000000000000000032621441646351700177020ustar00rootroot00000000000000"""Tests for codec utils.""" import pytest import bpack import bpack.st from bpack.codecs import has_codec, get_codec, get_codec_type try: import bpack.bs as bpack_bs except ImportError: # pragma: no cover bpack_bs = None @pytest.mark.parametrize( "backend", [ pytest.param(bpack.st, id="st"), pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), ], ) def test_codec_helpers(backend): @backend.codec @bpack.descriptor(baseunits=backend.Decoder.baseunits) class Record: field_1: int = bpack.field(size=4, default=0) field_2: int = bpack.field(size=4, default=0) assert has_codec(Record) assert has_codec(Record, bpack.codecs.Decoder) assert has_codec(Record, backend.Decoder) assert has_codec(Record, bpack.codecs.Encoder) assert has_codec(Record, backend.Encoder) assert has_codec(Record, bpack.codecs.Codec) assert has_codec(Record, backend.Codec) assert get_codec_type(Record) is backend.Codec assert isinstance(get_codec(Record), backend.Codec) assert isinstance(get_codec(Record), bpack.codecs.Codec) record = Record() assert has_codec(record) assert has_codec(record, bpack.codecs.Decoder) assert has_codec(record, backend.Decoder) assert has_codec(record, bpack.codecs.Encoder) assert has_codec(record, backend.Encoder) assert has_codec(record, bpack.codecs.Codec) assert has_codec(record, backend.Codec) assert get_codec_type(record) is backend.Codec assert isinstance(get_codec(record), backend.Codec) assert isinstance(get_codec(record), bpack.codecs.Codec) bpack-1.1.0/bpack/tests/test_decoder_ba.py000066400000000000000000000046031441646351700205110ustar00rootroot00000000000000"""Specific tests for the bitarray based decoder.""" from typing import List, Sequence import pytest import bpack bpack_ba = pytest.importorskip("bpack.ba") @pytest.mark.parametrize( "size, data", [ # fmt: off (16, bytes([0b00111100, 0b00000000])), (32, bytes([0b00111111, 0b10000000, 0b00000000, 0b00000000])), (64, bytes([0b00111111, 0b11110000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000])), # fmt: on ], ids=["float16", "float32", "float64"], ) def test_float(size, data): backend = bpack_ba codec = getattr(backend, "codec", backend.decoder) @codec @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS) class Record: field_1: float = bpack.field(size=size) record = Record.frombytes(data) assert record.field_1 == 1.0 def test_invalid_float_size(): backend = bpack_ba codec = getattr(backend, "codec", backend.decoder) with pytest.raises(ValueError): @codec @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS) class Record: field_1: float = bpack.field(size=80) def test_little_endian(): backend = bpack_ba codec = getattr(backend, "codec", backend.decoder) with pytest.raises(NotImplementedError): @codec @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, byteorder=bpack.EByteOrder.LE, frozen=True, ) class Record: field_1: int = bpack.field(size=8, default=1) def test_invalid_bitorder(): backend = bpack_ba codec = getattr(backend, "codec", backend.decoder) with pytest.raises(NotImplementedError): @codec @bpack.descriptor( baseunits=bpack.EBaseUnits.BITS, bitorder=bpack.EBitOrder.LSB, frozen=True, ) class Record: field_1: int = bpack.field(size=8, default=1) def test_sequence(): backend = bpack_ba codec = getattr(backend, "codec", backend.decoder) with pytest.raises(TypeError): @codec @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS) class Record: field_1: List[int] = bpack.field( size=4, signed=False, repeat=2, default=3 ) field_2: Sequence[int] = bpack.field( size=4, signed=False, repeat=2, default=4 ) bpack-1.1.0/bpack/tests/test_decoder_np.py000066400000000000000000000016751441646351700205520ustar00rootroot00000000000000"""Specific tests for the bitarray based decoder.""" from typing import List, Sequence import pytest import bpack bpack_np = pytest.importorskip("bpack.np") def test_decode_sequence(): backend = bpack_np bitorder = None size = 1 repeat = 2 encoded_data = bytes([3, 3, 4, 4]) @backend.decoder @bpack.descriptor(baseunits=backend.Decoder.baseunits, bitorder=bitorder) class Record: field_1: List[int] = bpack.field(size=size, repeat=repeat) field_2: Sequence[int] = bpack.field(size=size, repeat=repeat) ref_record = Record([3, 3], (4, 4)) record = Record.frombytes(encoded_data) assert list(record.field_1) == list(ref_record.field_1) assert list(record.field_2) == list(ref_record.field_2) for field, sequence_type in zip( bpack.fields(Record), (List[int], Sequence[int]) ): assert field.type == sequence_type # TODO # def test_encode_sequence(): # pass bpack-1.1.0/bpack/tests/test_desctiptor_utils.py000066400000000000000000000325071441646351700220460ustar00rootroot00000000000000"""Test utility functions for descriptors.""" import dataclasses import collections.abc from typing import List import pytest import bpack from bpack import EBaseUnits, EByteOrder, EBitOrder from bpack.descriptors import BinFieldDescriptor, METADATA_KEY def test_is_descriptor(): assert not bpack.is_descriptor(1) assert not bpack.is_descriptor("x") class C: pass assert not bpack.is_descriptor(C) assert not bpack.is_descriptor(C()) @dataclasses.dataclass class D: field_1: int = 0 assert not bpack.is_descriptor(D) assert not bpack.is_descriptor(D()) @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.is_descriptor(Record) assert bpack.is_descriptor(Record()) class Record: field_1: int = 0 setattr(Record, bpack.descriptors.BASEUNITS_ATTR_NAME, "dummy") assert not bpack.is_descriptor(Record) assert not bpack.is_descriptor(Record()) class Record: pass setattr(Record, bpack.descriptors.BASEUNITS_ATTR_NAME, "dummy") assert not bpack.is_descriptor(Record) assert not bpack.is_descriptor(Record()) class Record: pass setattr(Record, bpack.descriptors.BASEUNITS_ATTR_NAME, "dummy") assert not bpack.is_descriptor(Record) assert not bpack.is_descriptor(Record()) def test_is_field(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) for field_ in bpack.fields(Record): assert bpack.is_field(field_) for field_ in bpack.fields(Record()): assert bpack.is_field(field_) dataclasses_field_ = dataclasses.field() assert not bpack.is_field(dataclasses_field_) def test_calcsize(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) field_3: List[int] = bpack.field(size=1, default=0, repeat=4) assert bpack.baseunits(Record) is EBaseUnits.BYTES assert bpack.calcsize(Record) == 16 assert bpack.calcsize(Record()) == 16 assert bpack.calcsize(Record, EBaseUnits.BYTES) == 16 assert bpack.calcsize(Record, EBaseUnits.BITS) == 16 * 8 assert bpack.calcsize(Record, "bytes") == 16 assert bpack.calcsize(Record, "bits") == 16 * 8 @bpack.descriptor(baseunits=EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=16, default=1 / 3) field_3: int = bpack.field(size=4, default=3) field_4: List[int] = bpack.field(size=4, default=0, repeat=2) assert bpack.baseunits(Record) is EBaseUnits.BITS assert bpack.calcsize(Record) == 4 * 8 assert bpack.calcsize(Record()) == 4 * 8 assert bpack.calcsize(Record, EBaseUnits.BYTES) == 4 assert bpack.calcsize(Record, EBaseUnits.BITS) == 4 * 8 @dataclasses.dataclass class Record: field_1: int = 0 with pytest.raises(TypeError): bpack.calcsize(Record) with pytest.raises(TypeError): bpack.calcsize(Record()) class Record: pass with pytest.raises(TypeError): bpack.calcsize(Record) with pytest.raises(TypeError): bpack.calcsize(Record()) def test_fields(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert isinstance(bpack.fields(Record), tuple) assert len(bpack.fields(Record)) == 2 assert isinstance(bpack.fields(Record()), tuple) assert len(bpack.fields(Record())) == 2 def test_get_baseunits(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.baseunits(Record) == EBaseUnits.BYTES assert bpack.baseunits(Record()) == EBaseUnits.BYTES @bpack.descriptor(baseunits=EBaseUnits.BYTES) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.baseunits(Record) == EBaseUnits.BYTES assert bpack.baseunits(Record()) == EBaseUnits.BYTES @bpack.descriptor(baseunits=EBaseUnits.BITS, size=16) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.baseunits(Record) == EBaseUnits.BITS assert bpack.baseunits(Record()) == EBaseUnits.BITS @dataclasses.dataclass class Record: field_1: int = 0 with pytest.raises(TypeError): bpack.baseunits(Record) @pytest.mark.parametrize( "byteorder", [EByteOrder.LE, EByteOrder.BE, EByteOrder.NATIVE, EByteOrder.DEFAULT], ) def test_byteorder_explicit(byteorder): @bpack.descriptor(byteorder=byteorder) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.byteorder(Record) is byteorder assert bpack.byteorder(Record()) is byteorder def test_byteorder(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.byteorder(Record) is EByteOrder.DEFAULT assert bpack.byteorder(Record()) is EByteOrder.DEFAULT @dataclasses.dataclass class Dummy: x: int = 0 with pytest.raises(TypeError): bpack.byteorder(Dummy) class Dummy: pass with pytest.raises(TypeError): bpack.byteorder(Dummy) @pytest.mark.parametrize( "bitorder", [EBitOrder.MSB, EBitOrder.LSB, EBitOrder.DEFAULT] ) def test_bitorder(bitorder): @bpack.descriptor(bitorder=bitorder, baseunits=EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) field_3: int = bpack.field(size=4, default=0) assert bpack.bitorder(Record) is bitorder assert bpack.bitorder(Record()) is bitorder def test_bitorder_error(): @dataclasses.dataclass class Dummy: x: int = 0 with pytest.raises(TypeError): bpack.bitorder(Dummy) class Dummy: pass with pytest.raises(TypeError): bpack.bitorder(Dummy) def test_field_descriptors_iter(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) field_descriptors = bpack.descriptors.field_descriptors(Record) assert isinstance(field_descriptors, collections.abc.Iterable) field_descriptors = list(field_descriptors) assert all( isinstance(field_descr, BinFieldDescriptor) for field_descr in field_descriptors ) assert len(field_descriptors) == 2 field_descriptors = bpack.descriptors.field_descriptors(Record()) assert isinstance(field_descriptors, collections.abc.Iterable) field_descriptors = list(field_descriptors) assert all( isinstance(field_descr, BinFieldDescriptor) for field_descr in field_descriptors ) assert len(field_descriptors) == 2 def test_field_descriptors_iter_with_pad(): @bpack.descriptor(size=24) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3, offset=8) types_ = [int, None, float, None] field_descriptors = bpack.descriptors.field_descriptors(Record, pad=True) assert isinstance(field_descriptors, collections.abc.Iterable) field_descriptors = list(field_descriptors) assert all( isinstance(field_descr, BinFieldDescriptor) for field_descr in field_descriptors ) assert len(field_descriptors) == 4 assert [descr.type for descr in field_descriptors] == types_ assert sum(descr.size for descr in field_descriptors) == 24 assert bpack.calcsize(Record) == 24 field_descriptors = bpack.descriptors.field_descriptors(Record(), pad=True) assert isinstance(field_descriptors, collections.abc.Iterable) field_descriptors = list(field_descriptors) assert all( isinstance(field_descr, BinFieldDescriptor) for field_descr in field_descriptors ) assert len(field_descriptors) == 4 assert [descr.type for descr in field_descriptors] == types_ assert sum(descr.size for descr in field_descriptors) == 24 assert bpack.calcsize(Record()) == 24 def test_get_field_descriptor_01(): field = bpack.field(size=1, offset=2, signed=True) with pytest.raises(TypeError): bpack.descriptors.get_field_descriptor(field) descr = bpack.descriptors.get_field_descriptor(field, validate=False) assert descr.type is None assert descr.size == 1 assert descr.offset == 2 assert descr.signed is True field.type = int descr = bpack.descriptors.get_field_descriptor(field) assert descr.type is int assert descr.size == 1 assert descr.offset == 2 assert descr.signed is True def test_get_field_descriptor_02(): @bpack.descriptor class Record: field_1: int = bpack.field(size=1, offset=2, default=0, signed=True) field_2: float = bpack.field(size=4, offset=3, default=0.1) data = [(int, 1, 2, True), (float, 4, 3, None)] for field, (type_, size, offset, signed) in zip( bpack.fields(Record), data ): descr = bpack.descriptors.get_field_descriptor(field) assert descr.type is type_ assert descr.size == size assert descr.offset == offset assert descr.signed is signed record = Record() for field, (type_, size, offset, signed) in zip( bpack.fields(record), data ): descr = bpack.descriptors.get_field_descriptor(field) assert descr.type is type_ assert descr.size == size assert descr.offset == offset assert descr.signed is signed def test_set_field_descriptor(): field = dataclasses.field() assert not bpack.is_field(field) descr = bpack.descriptors.BinFieldDescriptor() with pytest.raises(TypeError): bpack.descriptors.set_field_descriptor(field, descr) bpack.descriptors.set_field_descriptor(field, descr, validate=False) assert bpack.is_field(field) def test_set_field_descriptor_type_mismatch(): field = bpack.field() field.type = int descr = bpack.descriptors.BinFieldDescriptor(type=float, size=1) with pytest.raises(TypeError, match="mismatch"): bpack.descriptors.set_field_descriptor(field, descr) def test_set_field_descriptor_values(): field = dataclasses.field() field.type = int assert not bpack.is_field(field) descr = bpack.descriptors.BinFieldDescriptor( type=field.type, size=1, offset=2, signed=True ) bpack.descriptors.set_field_descriptor(field, descr) assert bpack.is_field(field) descr_out = bpack.descriptors.get_field_descriptor(field) assert descr_out.type is field.type assert descr_out.size == 1 assert descr_out.offset == 2 assert descr_out.signed is True def test_field_descriptor_metadata(): field = bpack.field() field.type = int descr = bpack.descriptors.BinFieldDescriptor( type=field.type, size=1, offset=2, signed=True ) bpack.descriptors.set_field_descriptor(field, descr) assert field.metadata is not None assert METADATA_KEY in field.metadata descr_metadata = field.metadata[METADATA_KEY] assert isinstance(descr_metadata, collections.abc.Mapping) with pytest.raises(TypeError): # immutable (types.MappingProxyType) descr_metadata["new_key"] = "new_value" assert "type" not in descr_metadata assert "size" in descr_metadata assert descr_metadata["size"] == 1 assert "offset" in descr_metadata assert descr_metadata["offset"] == 2 assert "signed" in descr_metadata assert descr_metadata["signed"] is True assert len(descr_metadata) == 3 def test_field_descriptor_minimal_metadata(): field = bpack.field() field.type = int descr = bpack.descriptors.BinFieldDescriptor(type=field.type, size=1) bpack.descriptors.set_field_descriptor(field, descr) descr_metadata = field.metadata[bpack.descriptors.METADATA_KEY] assert "type" not in descr_metadata assert "size" in descr_metadata assert descr_metadata["size"] == 1 assert len(descr_metadata) == 1 def test_asdict(): @bpack.descriptor class Record: field_1: int = bpack.field(size=2, default=0, signed=True) field_2: float = bpack.field(size=4, default=0.1) record = Record() assert bpack.asdict(record) == dict( field_1=record.field_1, field_2=record.field_2 ) def test_astuple(): @bpack.descriptor class SubRecord: field_2_1: int = bpack.field(size=2, default=0, signed=True) field_2_2: float = bpack.field(size=4, default=0.1) @bpack.descriptor class Record: field_1: str = bpack.field(size=20, default="field_3_value") field_2: SubRecord = bpack.field(default_factory=SubRecord) record = Record() values = ( record.field_1, (record.field_2.field_2_1, record.field_2.field_2_2), ) assert bpack.astuple(record) == values assert type(bpack.astuple(record)) is tuple assert type(bpack.astuple(record, tuple_factory=list)) is list bpack-1.1.0/bpack/tests/test_field_descriptor.py000066400000000000000000000467231441646351700217740ustar00rootroot00000000000000"""Test bpack field descriptors.""" import sys import enum from typing import List import pytest import bpack from bpack.descriptors import get_field_descriptor class TestFieldFactory: @staticmethod def test_base(): bpack.field(size=1, offset=0, signed=False, default=0, repeat=1) @staticmethod def test_field_vs_field_class(): field_ = bpack.field(size=1) assert bpack.descriptors.is_field(field_) assert isinstance(field_, bpack.descriptors.Field) @staticmethod @pytest.mark.parametrize(argnames="size", argvalues=[1.3, "x"]) def test_invalid_size_type(size): with pytest.raises(TypeError): bpack.field(size=size, default=1 / 3) @staticmethod @pytest.mark.parametrize(argnames="size", argvalues=[0, -8]) def test_invalid_size(size): with pytest.raises(ValueError): bpack.field(size=size, default=1 / 3) @staticmethod @pytest.mark.parametrize(argnames="offset", argvalues=[1.3, "x"]) def test_invalid_offset_type(offset): with pytest.raises(TypeError): bpack.field(size=8, default=1 / 3, offset=offset) @staticmethod def test_invalid_offset(): with pytest.raises(ValueError): bpack.field(size=8, default=1 / 3, offset=-8) @staticmethod @pytest.mark.parametrize("value", [-8, "a"]) def test_invalid_signed_type(value): with pytest.raises(TypeError): bpack.field(size=8, default=1, signed=value) @staticmethod @pytest.mark.parametrize(argnames="repeat", argvalues=[1.3, "x"]) def test_invalid_repeat_type(repeat): with pytest.raises(TypeError): bpack.field(size=8, default=1 / 3, repeat=repeat) @staticmethod def test_invalid_repeat(): with pytest.raises(ValueError): bpack.field(size=8, default=1 / 3, repeat=0) @staticmethod def test_metadata_key(): field_ = bpack.field(size=1) assert bpack.descriptors.METADATA_KEY in field_.metadata class TestRecordFields: @staticmethod def test_field_properties_01(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0, signed=True) field_2: float = bpack.field(size=8, default=1 / 3) field_3: List[int] = bpack.field(size=1, default=1, repeat=1) # name, type, size, offset, repeat field_data = [ ("field_1", int, 4, 0, True, None), ("field_2", float, 8, 4, None, None), ("field_3", List[int], 1, 12, None, 1), ] for field_, data in zip(bpack.fields(Record), field_data): name, type_, size, offset, signed, repeat = data assert field_.name == name assert field_.type == type_ field_descr = get_field_descriptor(field_) assert field_descr.type == type_ assert field_descr.size == size assert field_descr.offset == offset assert field_descr.signed == signed assert field_descr.repeat == repeat @staticmethod def test_field_properties_02(): @bpack.descriptor class Record: field_1: int = bpack.field( size=4, offset=1, default=0, signed=False ) field_2: float = bpack.field(size=8, default=1 / 3) field_3: List[int] = bpack.field(size=1, default=1, repeat=1) # name, type, size, offset, repeat field_data = [ ("field_1", int, 4, 1, False, None), ("field_2", float, 8, 5, None, None), ("field_3", List[int], 1, 13, None, 1), ] for field_, data in zip(bpack.fields(Record), field_data): name, type_, size, offset, signed, repeat = data assert field_.name == name assert field_.type == type_ field_descr = get_field_descriptor(field_) assert field_descr.type == type_ assert field_descr.size == size assert field_descr.offset == offset assert field_descr.signed == signed assert field_descr.repeat == repeat @staticmethod def test_field_properties_03(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, offset=1, default=0) field_2: float = bpack.field(size=8, offset=6, default=1 / 3) # name, type, size, offset, repeat field_data = [ ("field_1", int, 4, 1, None, None), ("field_2", float, 8, 6, None, None), ] for field_, data in zip(bpack.fields(Record), field_data): name, type_, size, offset, signed, repeat = data assert field_.name == name assert field_.type == type_ field_descr = get_field_descriptor(field_) assert field_descr.type == type_ assert field_descr.size == size assert field_descr.offset == offset assert field_descr.signed == signed assert field_descr.repeat == repeat @staticmethod def test_finvalid_field_type(): with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: "invalid" = bpack.field(size=4) # noqa: F821 class TestEnumFields: @staticmethod def test_enum(): class EEnumType(enum.Enum): A = "a" B = "b" C = "c" @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: EEnumType = bpack.field(size=1, default=EEnumType.A) field_2 = bpack.fields(Record)[1] assert field_2.name == "field_2" assert field_2.type is EEnumType assert field_2.default is EEnumType.A assert isinstance(Record().field_2, EEnumType) @staticmethod def test_int_enum(): class EEnumType(enum.IntEnum): A = 1 B = 2 C = 4 @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: EEnumType = bpack.field( size=1, signed=True, default=EEnumType.A ) field_2 = bpack.fields(Record)[1] assert field_2.name == "field_2" assert field_2.type is EEnumType assert field_2.default is EEnumType.A assert isinstance(Record().field_2, EEnumType) @staticmethod def test_intflag_enum(): class EEnumType(enum.IntFlag): A = 1 B = 2 C = 4 @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: EEnumType = bpack.field(size=1, default=EEnumType.A) field_2 = bpack.fields(Record)[1] assert field_2.name == "field_2" assert field_2.type is EEnumType assert field_2.default is EEnumType.A assert isinstance(Record().field_2, EEnumType) @staticmethod def test_invalid_enum(): class EEnumType(enum.Enum): A = 1 B = "b" C = 4 with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: EEnumType = bpack.field(size=1, default=EEnumType.A) @staticmethod def test_invalid_signed_qualifier(): class EEnumType(enum.Enum): A = "a" B = "b" C = "c" with pytest.warns(UserWarning): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: EEnumType = bpack.field( size=1, signed=True, default=EEnumType.A ) class TestFieldDescriptor: @staticmethod def test_empty_init(): descr = bpack.descriptors.BinFieldDescriptor() assert descr.type is None assert descr.size is None assert descr.offset is None assert descr.signed is None assert descr.repeat is None assert len(vars(descr)) == 5 @staticmethod def test_init(): descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2, True, 1) assert descr.type is int assert descr.size == 1 assert descr.offset == 2 assert descr.signed is True assert descr.repeat == 1 assert len(vars(descr)) == 5 @staticmethod def test_init_kw(): descr = bpack.descriptors.BinFieldDescriptor( type=int, size=1, offset=2, signed=True, repeat=1 ) assert descr.type is int assert descr.size == 1 assert descr.offset == 2 assert descr.signed is True assert descr.repeat == 1 assert len(vars(descr)) == 5 @staticmethod def test_init_invalid_type(): with pytest.raises(TypeError): bpack.descriptors.BinFieldDescriptor(size=1.1) with pytest.raises(TypeError): bpack.descriptors.BinFieldDescriptor(offset=2.1) with pytest.raises(TypeError): bpack.descriptors.BinFieldDescriptor(signed=complex(3.1, 0)) with pytest.raises(TypeError): bpack.descriptors.BinFieldDescriptor(repeat=1.1) @staticmethod def test_init_invalid_value(): with pytest.raises(ValueError): bpack.descriptors.BinFieldDescriptor(size=-1) with pytest.raises(ValueError): bpack.descriptors.BinFieldDescriptor(size=0) with pytest.raises(ValueError): bpack.descriptors.BinFieldDescriptor(offset=-1) with pytest.raises(ValueError): bpack.descriptors.BinFieldDescriptor(repeat=0) @staticmethod def test_validate(): descr = bpack.descriptors.BinFieldDescriptor(int, 1) descr.validate() descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2) descr.validate() descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2, True) descr.validate() descr = bpack.descriptors.BinFieldDescriptor(List[int], 1, 2, True, 1) descr.validate() @staticmethod def test_validation_warning(): descr = bpack.descriptors.BinFieldDescriptor( type=float, size=4, signed=True ) with pytest.warns(UserWarning, match="ignore"): descr.validate() @staticmethod def test_validation_error(): descr = bpack.descriptors.BinFieldDescriptor() with pytest.raises(TypeError): descr.validate() descr = bpack.descriptors.BinFieldDescriptor(type=int) with pytest.raises(TypeError): descr.validate() descr = bpack.descriptors.BinFieldDescriptor(size=1) with pytest.raises(TypeError): descr.validate() descr = bpack.descriptors.BinFieldDescriptor( type=int, size=1, repeat=2 ) with pytest.raises(TypeError): descr.validate() @staticmethod def test_post_validation_error_on_type(): descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2) descr.validate() descr.type = None with pytest.raises(TypeError): descr.validate() @staticmethod @pytest.mark.parametrize( "size, error_type", [ (None, TypeError), (0, ValueError), (-1, ValueError), (1.1, TypeError), ], ids=["None", "zero", "negative", "float"], ) def test_post_validation_error_on_size(size, error_type): descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2) descr.validate() descr.size = size with pytest.raises(error_type): descr.validate() @staticmethod @pytest.mark.parametrize( "offset, error_type", [(-1, ValueError), (1.1, TypeError)], ids=["negative", "float"], ) def test_post_validation_error_on_offset(offset, error_type): descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2) descr.validate() descr.offset = offset with pytest.raises(error_type): descr.validate() @staticmethod def test_post_validation_warning_on_signed(): descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2, signed=True) descr.validate() descr.type = float with pytest.warns(UserWarning, match="ignore"): descr.validate() @staticmethod def test_post_validation_error_on_repeat(): descr = bpack.descriptors.BinFieldDescriptor(int, 1, 2, signed=True) descr.validate() descr.repeat = 2 with pytest.raises(TypeError): descr.validate() descr = bpack.descriptors.BinFieldDescriptor( List[int], 1, 2, signed=True, repeat=2 ) descr.validate() descr.repeat = 0 with pytest.raises(ValueError): descr.validate() def test_methods(self): descr = bpack.descriptors.BinFieldDescriptor(int, 1) descr.validate() assert descr.is_int_type() assert not descr.is_sequence_type() assert not descr.is_enum_type() descr = bpack.descriptors.BinFieldDescriptor(float, 1) descr.validate() assert not descr.is_int_type() assert not descr.is_sequence_type() assert not descr.is_enum_type() descr = bpack.descriptors.BinFieldDescriptor(List[int], 1, repeat=10) descr.validate() assert descr.is_int_type() assert descr.is_sequence_type() assert not descr.is_enum_type() descr = bpack.descriptors.BinFieldDescriptor(List[float], 1, repeat=10) descr.validate() assert not descr.is_int_type() assert descr.is_sequence_type() assert not descr.is_enum_type() class EEnumType(enum.Enum): A = "a" descr = bpack.descriptors.BinFieldDescriptor(EEnumType, 1) descr.validate() assert not descr.is_int_type() assert not descr.is_sequence_type() assert descr.is_enum_type() class EEnumType(enum.IntEnum): A = 1 descr = bpack.descriptors.BinFieldDescriptor(EEnumType, 1) descr.validate() assert descr.is_int_type() assert not descr.is_sequence_type() assert descr.is_enum_type() class EEnumType(enum.IntFlag): A = 1 descr = bpack.descriptors.BinFieldDescriptor(EEnumType, 1) descr.validate() assert descr.is_int_type() assert not descr.is_sequence_type() assert descr.is_enum_type() class TestAnnotatedType: @staticmethod @pytest.mark.parametrize( "byteorder", [">", "<", "|", ""], ids=[">", "<", "|", "None"] ) def test_annotated_type(byteorder): @bpack.descriptor(byteorder=byteorder if byteorder != "|" else "") class Record: field_1: bpack.T[f"{byteorder}i4"] # noqa: F821 field_2: bpack.T[f"{byteorder}u4"] # noqa: F821 field_3: bpack.T[f"{byteorder}f4"] # noqa: F821 field_4: bpack.T[f"{byteorder}c4"] # noqa: F821 field_5: bpack.T[f"{byteorder}S4"] # noqa: F821 fields = { field.name: get_field_descriptor(field) for field in bpack.fields(Record) } assert fields["field_1"].type == int assert fields["field_1"].size == 4 assert fields["field_1"].signed is True assert fields["field_1"].repeat is None assert fields["field_2"].type == int assert fields["field_2"].size == 4 assert fields["field_2"].signed is False assert fields["field_2"].repeat is None assert fields["field_3"].type == float assert fields["field_3"].size == 4 assert fields["field_3"].signed is None assert fields["field_3"].repeat is None assert fields["field_4"].type == complex assert fields["field_4"].size == 4 assert fields["field_4"].signed is None assert fields["field_4"].repeat is None assert fields["field_5"].type == bytes assert fields["field_5"].size == 4 assert fields["field_5"].signed is None assert fields["field_5"].repeat is None @staticmethod def test_list_with_annotated_type(): typestr = "i4" @bpack.descriptor class Record: field_1: List[bpack.T[typestr]] = bpack.field(repeat=2) field = bpack.fields(Record)[0] assert field.type == List[bpack.T[typestr]] field_descr = get_field_descriptor(field) assert field_descr.type == List[int] assert field_descr.size == 4 assert field_descr.signed is True assert field_descr.repeat == 2 @staticmethod def test_byteorder_consistency(): typestr = ">i8" with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor(byteorder=bpack.EByteOrder.LE) class Record: field: bpack.T[typestr] typestr = "i8" @bpack.descriptor class Record: # noqa: F811 field: bpack.T[typestr] typestr = "|i8" @bpack.descriptor(byteorder=bpack.EByteOrder.BE) class Record: # noqa: F811 field: bpack.T[typestr] @staticmethod def test_size_consistency(): typestr = "i8" @bpack.descriptor class Record: field: bpack.T[typestr] = bpack.field(size=8, signed=True) descr = get_field_descriptor(bpack.fields(Record)[0]) assert descr.type is int assert descr.size == 8 assert descr.repeat is None assert descr.signed is True typestr = "u8" @bpack.descriptor class Record: field: bpack.T[typestr] = bpack.field(size=8, signed=False) descr = get_field_descriptor(bpack.fields(Record)[0]) assert descr.type is int assert descr.size == 8 assert descr.repeat is None assert descr.signed is False typestr = "f8" @bpack.descriptor class Record: field: bpack.T[typestr] = bpack.field(size=8) descr = get_field_descriptor(bpack.fields(Record)[0]) assert descr.type is float assert descr.size == 8 assert descr.repeat is None assert descr.signed is None @staticmethod def test_size_consistency_error(): typestr = "i8" with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor class Record: field: bpack.T[typestr] = bpack.field(size=3) @staticmethod def test_signed_consistency_error(): typestr = "i8" with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor class Record: field: bpack.T[typestr] = bpack.field(size=8, signed=False) @staticmethod @pytest.mark.parametrize( "typestr", ["invalid", "@i8", "|x8", "|i0", "|ii"] ) def test_invalid_typestr(typestr): with pytest.raises(ValueError): @bpack.descriptor class Record: field: bpack.T[typestr] = bpack.field(size=1) bpack-1.1.0/bpack/tests/test_packbits.py000066400000000000000000000357111441646351700202460ustar00rootroot00000000000000"""Unit tests for packbit and unpackbut functions.""" import math from typing import Sequence, Tuple import pytest try: import bpack.ba as bpack_ba except ImportError: # pragma: no cover bpack_ba = None try: import bitstruct import bpack.bs as bpack_bs except ImportError: # pragma: no cover bitstruct = None bpack_bs = None try: import numpy as np import bpack.np as bpack_np except ImportError: # pragma: no cover np = None bpack_np = None def _sample_data( bits_per_sample: int, nsamples: int = 256 ) -> Tuple[bytes, Sequence[int]]: """Generate a packed data block having spb samples bps bits each.""" # fmt: off elementary_range = { 2: bytes([0b00011011]), 3: bytes([0b00000101, 0b00111001, 0b01110111]), 4: bytes([0b00000001, 0b00100011, 0b01000101, 0b01100111, 0b10001001, 0b10101011, 0b11001101, 0b11101111]), 5: bytes([0b00000000, 0b01000100, 0b00110010, 0b00010100, 0b11000111, 0b01000010, 0b01010100, 0b10110110, 0b00110101, 0b11001111, 0b10000100, 0b01100101, 0b00111010, 0b01010110, 0b11010111, 0b11000110, 0b01110101, 0b10111110, 0b01110111, 0b11011111]), 6: bytes([0b00000000, 0b00010000, 0b10000011, 0b00010000, 0b01010001, 0b10000111, 0b00100000, 0b10010010, 0b10001011, 0b00110000, 0b11010011, 0b10001111, 0b01000001, 0b00010100, 0b10010011, 0b01010001, 0b01010101, 0b10010111, 0b01100001, 0b10010110, 0b10011011, 0b01110001, 0b11010111, 0b10011111, 0b10000010, 0b00011000, 0b10100011, 0b10010010, 0b01011001, 0b10100111, 0b10100010, 0b10011010, 0b10101011, 0b10110010, 0b11011011, 0b10101111, 0b11000011, 0b00011100, 0b10110011, 0b11010011, 0b01011101, 0b10110111, 0b11100011, 0b10011110, 0b10111011, 0b11110011, 0b11011111, 0b10111111]), 7: bytes([0b00000000, 0b00000100, 0b00010000, 0b00110000, 0b10000001, 0b01000011, 0b00000111, 0b00010000, 0b00100100, 0b01010000, 0b10110001, 0b10000011, 0b01000111, 0b00001111, 0b00100000, 0b01000100, 0b10010001, 0b00110010, 0b10000101, 0b01001011, 0b00010111, 0b00110000, 0b01100100, 0b11010001, 0b10110011, 0b10000111, 0b01001111, 0b00011111, 0b01000000, 0b10000101, 0b00010010, 0b00110100, 0b10001001, 0b01010011, 0b00100111, 0b01010000, 0b10100101, 0b01010010, 0b10110101, 0b10001011, 0b01010111, 0b00101111, 0b01100000, 0b11000101, 0b10010011, 0b00110110, 0b10001101, 0b01011011, 0b00110111, 0b01110000, 0b11100101, 0b11010011, 0b10110111, 0b10001111, 0b01011111, 0b00111111, 0b10000001, 0b00000110, 0b00010100, 0b00111000, 0b10010001, 0b01100011, 0b01000111, 0b10010001, 0b00100110, 0b01010100, 0b10111001, 0b10010011, 0b01100111, 0b01001111, 0b10100001, 0b01000110, 0b10010101, 0b00111010, 0b10010101, 0b01101011, 0b01010111, 0b10110001, 0b01100110, 0b11010101, 0b10111011, 0b10010111, 0b01101111, 0b01011111, 0b11000001, 0b10000111, 0b00010110, 0b00111100, 0b10011001, 0b01110011, 0b01100111, 0b11010001, 0b10100111, 0b01010110, 0b10111101, 0b10011011, 0b01110111, 0b01101111, 0b11100001, 0b11000111, 0b10010111, 0b00111110, 0b10011101, 0b01111011, 0b01110111, 0b11110001, 0b11100111, 0b11010111, 0b10111111, 0b10011111, 0b01111111, 0b01111111]), 8: bytes(range(2 ** 8)), } # fmt: on assert (nsamples * bits_per_sample) % 8 == 0 base_range = elementary_range[bits_per_sample] nreplica = math.ceil(nsamples / 2**bits_per_sample) nbytes = nsamples * bits_per_sample // 8 data = (base_range * nreplica)[:nbytes] base_range = list(range(2**bits_per_sample)) values = (base_range * nreplica)[:nsamples] return data, values @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ) ], ) @pytest.mark.parametrize("bits_per_sample", [2, 3, 4, 5, 6, 7, 8]) @pytest.mark.parametrize("nsamples", [256]) def test_packbits(backend, bits_per_sample, nsamples): data, values = _sample_data(bits_per_sample, nsamples) odata = backend.packbits(values, bits_per_sample) assert odata == data @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ) ], ) @pytest.mark.parametrize("bits_per_sample", [3]) @pytest.mark.parametrize("nsamples", [256]) def test_packbits_bad_values(backend, bits_per_sample, nsamples): values = [2**bits_per_sample] * nsamples with pytest.raises(Exception): # TODO: improve error handling backend.packbits(values, bits_per_sample) @pytest.mark.parametrize( "backend", [ pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ) ], ) @pytest.mark.parametrize("bits_per_sample", [3]) def test_packbits_nsanples_requires_pad(backend, bits_per_sample): values = [1] nsamples = len(values) # the number of samples does not fits an integer number of bytes assert (nsamples * bits_per_sample % 8) != 0 with pytest.warns(UserWarning): backend.packbits(values, bits_per_sample) @pytest.mark.parametrize( "backend", [ pytest.param( bpack_ba, id="ba", marks=pytest.mark.skipif(not bpack_ba, reason="not available"), ), pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_np, id="np", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), ], ) @pytest.mark.parametrize("bits_per_sample", [2, 3, 4, 5, 6, 7, 8]) @pytest.mark.parametrize("nsamples", [256]) def test_unpackbits(backend, bits_per_sample, nsamples): data, values = _sample_data(bits_per_sample, nsamples) ovalues = backend.unpackbits(data, bits_per_sample) assert list(ovalues) == values @pytest.mark.parametrize( "backend", [ pytest.param( bpack_ba, id="ba", marks=pytest.mark.skipif(not bpack_ba, reason="not available"), ), pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_np, id="np", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), ], ) def test_unpackbits_1_bit_per_sample(backend): bits_per_sample = 1 values = [1, 0, 1, 0, 1, 0, 1, 0] data = bytes([0b10101010]) ovalues = backend.unpackbits(data, bits_per_sample) assert list(ovalues) == values @pytest.mark.skipif(not bpack_bs, reason="bitstruct not available") @pytest.mark.parametrize( "backend", [ pytest.param( bpack_ba, id="ba", marks=pytest.mark.skipif(not bpack_ba, reason="not available"), ), pytest.param( bpack_bs, id="bs", marks=pytest.mark.skipif(not bpack_bs, reason="not available"), ), pytest.param( bpack_np, id="np", marks=pytest.mark.skipif(not bpack_np, reason="not available"), ), ], ) @pytest.mark.parametrize("bits_per_sample", [10, 12, 14, 16, 32, 64]) @pytest.mark.parametrize("nsamples", [256]) def test_unpackbits_large(backend, bits_per_sample, nsamples): values = [item % (2**bits_per_sample) for item in range(nsamples)] data = bpack_bs.packbits(values, bits_per_sample) ovalues = backend.unpackbits(data, bits_per_sample) assert list(ovalues) == values def _make_sample_data_block( header_size, bits_per_sample, samples_per_block, bit_offset=0, nblocks=1, sign_mode=0, ): bits_per_block = header_size + bits_per_sample * samples_per_block nbytes = math.ceil((bit_offset + bits_per_block) / 8) typechar = "s" if sign_mode == 1 else "u" block_fmt = f"{typechar}{bits_per_sample}" * samples_per_block if header_size > 0: block_fmt = f"u{header_size}" + block_fmt leading_pad = f"p{bit_offset}" if bit_offset > 0 else "" trailing_pad = nbytes * 8 - bits_per_block * nblocks - bit_offset trailing_pad = f"p{trailing_pad}" if trailing_pad > 0 else "" fmt = f"{leading_pad}{block_fmt * nblocks}{trailing_pad}" n = 2**bits_per_sample if sign_mode == 0: ramp_values = list(range(n)) * math.ceil(samples_per_block / n) out_values = ramp_values elif sign_mode == 1: hs = 2 ** (bits_per_sample - 1) ramp_values = list(range(-hs, hs)) * math.ceil(samples_per_block / n) out_values = ramp_values elif sign_mode == 2: hs = 2 ** (bits_per_sample - 1) ramp_values = list(range(hs)) out_values = ramp_values + [-item for item in ramp_values] out_values *= math.ceil(samples_per_block / n) sign_mask = 2 ** (bits_per_sample - 1) ramp_values += [item | sign_mask for item in ramp_values] ramp_values *= math.ceil(samples_per_block / n) else: raise ValueError(f"Unexpected 'sign_mode': {sign_mode}") values = ramp_values[:samples_per_block] out_values = out_values[:samples_per_block] if header_size > 0: values = [113] + values out_values = [113] + out_values values *= nblocks out_values *= nblocks data = bitstruct.pack(fmt, *values) return data, out_values @pytest.mark.skipif(not bitstruct, reason="bitstruct not available") @pytest.mark.skipif(not np, reason="numpy not available") @pytest.mark.parametrize("bit_offset", [0, 1, 2]) @pytest.mark.parametrize("header_size", [9, 13]) @pytest.mark.parametrize("bits_per_sample", [3, 4, 5, 6, 12, 13, 14]) @pytest.mark.parametrize("samples_per_block", [64, 128, 256]) @pytest.mark.parametrize("nblocks", [1, 3, 20]) def test_headers( bit_offset, header_size, bits_per_sample, samples_per_block, nblocks ): data, values = _make_sample_data_block( header_size, bits_per_sample, samples_per_block, bit_offset, nblocks=nblocks, ) block_size = header_size + bits_per_sample * samples_per_block headers = bpack_np.unpackbits( data, bits_per_sample=header_size, samples_per_block=1, bit_offset=bit_offset, blockstride=block_size, ) assert len(headers) == nblocks assert all(headers == values[0]) @pytest.mark.skipif(not bitstruct, reason="bitstruct not available") @pytest.mark.skipif(not np, reason="numpy not available") @pytest.mark.parametrize("bit_offset", [0, 1, 2]) @pytest.mark.parametrize("header_size", [0, 9, 13]) @pytest.mark.parametrize("bits_per_sample", [3, 4, 5, 8, 13]) @pytest.mark.parametrize("samples_per_block", [64, 128]) @pytest.mark.parametrize("nblocks", [1, 3, 20]) @pytest.mark.parametrize("sign_mode", [0, 1, 2]) def test_data( bit_offset, header_size, bits_per_sample, samples_per_block, nblocks, sign_mode, ): data, values = _make_sample_data_block( header_size, bits_per_sample, samples_per_block, bit_offset, nblocks=nblocks, sign_mode=sign_mode, ) assert len(values) == ( samples_per_block * nblocks + (nblocks if header_size else 0) ) block_size = header_size + bits_per_sample * samples_per_block odata = bpack_np.unpackbits( data, bits_per_sample, samples_per_block, bit_offset=bit_offset + header_size, blockstride=block_size, sign_mode=sign_mode, ) extra_bits = len(data) * 8 - bit_offset - block_size * nblocks extra_bits = max(extra_bits - header_size, 0) extra_samples = extra_bits // bits_per_sample assert len(odata) == nblocks * samples_per_block + extra_samples k = 1 if header_size > 0 else 0 for idx in range(nblocks): oslice = slice(samples_per_block * idx, samples_per_block * (idx + 1)) vslice = slice( k + (samples_per_block + k) * idx, k + (samples_per_block + k) * idx + samples_per_block, ) assert all(odata[oslice] == values[vslice]) @pytest.mark.skipif(not bitstruct, reason="bitstruct not available") @pytest.mark.skipif(not np, reason="numpy not available") @pytest.mark.parametrize( "header_size, dtype", [ (7, ">u1"), (8, ">u1"), (9, ">u2"), (15, ">u2"), (16, ">u2"), (17, ">u4"), ], ) def test_headers_dtype(header_size, dtype): bits_per_sample = 8 samples_per_block = 64 data, _ = _make_sample_data_block( header_size, bits_per_sample, samples_per_block ) headers = bpack_np.unpackbits( data, bits_per_sample=header_size, samples_per_block=1 ) assert headers.dtype == np.dtype(dtype) @pytest.mark.skipif(not bitstruct, reason="bitstruct not available") @pytest.mark.skipif(not np, reason="numpy not available") @pytest.mark.parametrize( "bits_per_sample, dtype", [ (7, ">u1"), (8, ">u1"), (9, ">u2"), (15, ">u2"), (16, ">u2"), (17, ">u4"), ], ) def test_data_dtype(bits_per_sample, dtype): header_size = 0 samples_per_block = 64 data, _ = _make_sample_data_block( header_size, bits_per_sample, samples_per_block ) odata = bpack_np.unpackbits(data, bits_per_sample, samples_per_block) assert odata.dtype == np.dtype(dtype) @pytest.mark.skipif(not bitstruct, reason="bitstruct not available") @pytest.mark.skipif(not np, reason="numpy not available") def test_auto_sample_per_block(): header_size = 0 samples_per_block = 64 bits_per_sample = 3 data, _ = _make_sample_data_block( header_size, bits_per_sample, samples_per_block ) odata = bpack_np.unpackbits(data, bits_per_sample) assert len(odata) == samples_per_block with pytest.raises(ValueError): bpack_np.unpackbits( data, bits_per_sample, blockstride=bits_per_sample * samples_per_block, ) @pytest.mark.skipif(not np, reason="numpy not available") @pytest.mark.parametrize( ["bits_per_sample", "mode", "ref_mask"], [ (3, 0, 0b00000111), (20, 0, 0b000011111111111111111111), (34, 0, 0b0000001111111111111111111111111111111111), (3, 1, 0b11111000), (3, 2, 0b00000100), ], ) def test_make_bitmask(bits_per_sample, mode, ref_mask): mask = bpack_np.make_bitmask(bits_per_sample, mode=mode) assert mask == ref_mask bpack-1.1.0/bpack/tests/test_record_descriptor.py000066400000000000000000000360621441646351700221620ustar00rootroot00000000000000"""Test bpack (record) descriptors.""" import dataclasses from typing import List, Sequence, Tuple import pytest import bpack import bpack.descriptors from bpack import EBaseUnits, EByteOrder, EBitOrder from bpack.descriptors import Field as BPackField def test_base(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert dataclasses.is_dataclass(Record) assert len(bpack.fields(Record)) == 2 assert bpack.baseunits(Record) is EBaseUnits.BYTES # default assert all(isinstance(f, BPackField) for f in bpack.fields(Record)) def test_frozen(): @bpack.descriptor(frozen=True) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert dataclasses.is_dataclass(Record) assert len(bpack.fields(Record)) == 2 assert bpack.baseunits(Record) is EBaseUnits.BYTES # default assert all(isinstance(f, BPackField) for f in bpack.fields(Record)) def test_dataclass(): with pytest.warns(DeprecationWarning): @bpack.descriptor @dataclasses.dataclass class Record: field_1: int = bpack.field(size=8, default=0) field_2: float = bpack.field(size=8, default=1 / 3) @pytest.mark.parametrize( argnames="baseunits", argvalues=[EBaseUnits.BYTES, EBaseUnits.BITS, "bits", "bytes"], ) def test_base_units(baseunits): @bpack.descriptor(baseunits=baseunits) class Record: field_1: int = bpack.field(size=8, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.baseunits(Record) is EBaseUnits(baseunits) assert bpack.baseunits(Record()) is EBaseUnits(baseunits) def test_byte_alignment_warning(): with pytest.warns(UserWarning, match="bit struct not aligned to bytes"): @bpack.descriptor(baseunits=EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) def test_attrs(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert hasattr(Record, bpack.descriptors.BASEUNITS_ATTR_NAME) assert hasattr(Record, bpack.descriptors.BYTEORDER_ATTR_NAME) assert hasattr(Record, bpack.descriptors.BITORDER_ATTR_NAME) assert hasattr(Record, bpack.descriptors.SIZE_ATTR_NAME) @pytest.mark.parametrize(argnames="baseunits", argvalues=[None, 8, "x"]) def test_invalid_baseunits(baseunits): with pytest.raises(ValueError): @bpack.descriptor(baseunits=baseunits) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) @pytest.mark.parametrize( argnames="order", argvalues=[ EByteOrder.LE, EByteOrder.BE, EByteOrder.NATIVE, EByteOrder.DEFAULT, "<", ">", "=", "", ], ) def test_byteorder(order): @bpack.descriptor(byteorder=order) class Record: field_1: int = bpack.field(size=8, default=0) field_2: float = bpack.field(size=8, default=1 / 3) if isinstance(order, str): assert bpack.byteorder(Record) is EByteOrder(order) assert bpack.byteorder(Record()) is EByteOrder(order) else: assert bpack.byteorder(Record) is order assert bpack.byteorder(Record()) is order @pytest.mark.parametrize(argnames="order", argvalues=["invalid", None]) def test_invalid_byteorder(order): with pytest.raises(ValueError): @bpack.descriptor(byteorder=order) class Record: field_1: int = bpack.field(size=8, default=0) @pytest.mark.parametrize( argnames="order", argvalues=[EBitOrder.LSB, EBitOrder.MSB, EBitOrder.DEFAULT, "<", ">", ""], ) def test_bitorder(order): @bpack.descriptor(bitorder=order, baseunits=EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=8, default=0) field_2: float = bpack.field(size=8, default=1 / 3) if isinstance(order, str): assert bpack.bitorder(Record) is EBitOrder(order) assert bpack.bitorder(Record()) is EBitOrder(order) else: assert bpack.bitorder(Record) is order assert bpack.bitorder(Record()) is order def test_default_bitorder(): @bpack.descriptor(baseunits=EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=8, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.bitorder(Record) is EBitOrder.DEFAULT assert bpack.bitorder(Record()) is EBitOrder.DEFAULT def test_invalid_bitorder(): with pytest.raises(ValueError): @bpack.descriptor(byteorder="invalid", baseunits=EBaseUnits.BITS) class Record: field_1: int = bpack.field(size=8, default=0) def test_bitorder_in_byte_descriptors(): @bpack.descriptor(baseunits=EBaseUnits.BYTES) class Record: field_1: int = bpack.field(size=8, default=0) assert bpack.byteorder(Record) is EByteOrder.DEFAULT assert bpack.byteorder(Record()) is EByteOrder.DEFAULT with pytest.raises(ValueError): @bpack.descriptor( bitorder=EBitOrder.DEFAULT, baseunits=EBaseUnits.BYTES ) class Record: field_1: int = bpack.field(size=8, default=0) def test_invalid_field_class(): with pytest.raises(TypeError, match="size not specified"): @bpack.descriptor class Record: field_1: int = 0 field_2: float = 1 / 3 def test_no_field_type(): with pytest.raises(TypeError): @bpack.descriptor class Record: field_1 = bpack.field(size=4, default=0) def test_invalid_field_size_type(): with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=None, default=1 / 3) def test_inconsistent_field_offset_and_size(): with pytest.raises(ValueError): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, offset=1, default=1 / 3) def test_inconsistent_field_type_and_signed(): with pytest.warns(UserWarning, match="ignore"): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3, signed=True) def test_repeat(): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=4, default=0, repeat=2) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.calcsize(Record()) == 16 def test_no_repeat(): with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) def test_inconsistent_field_type_and_repeat(): with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0, repeat=2) field_2: float = bpack.field(size=8, default=1 / 3, signed=True) class TestExplicitSize: @staticmethod def test_explicit_size(): size = 16 @bpack.descriptor(size=size) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.calcsize(Record) == size assert bpack.calcsize(Record()) == size @staticmethod def test_invalid_explicit_size(): with pytest.raises(ValueError): @bpack.descriptor(size=10) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) @staticmethod def test_inconsistent_explicit_size_and_offset(): with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor(size=16) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3, offset=12) @staticmethod def test_inconsistent_explicit_size_and_repeat(): with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor(size=12) class Record: field_1: List[int] = bpack.field(size=4, default=0, repeat=2) field_2: float = bpack.field(size=8, default=1 / 3) @staticmethod def test_inconsistent_explicit_size_offset_and_repeat(): with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor(size=20) class Record: field_1: int = bpack.field(size=4, default=0) field_2: List[float] = bpack.field( size=8, default=1 / 3, offset=8, repeat=2 ) @staticmethod def test_invalid_explicit_size_type(): with pytest.raises(TypeError): @bpack.descriptor(size=10.5) class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, default=1 / 3) class TestSize: # NOTE: basic tests are in test_descriptor_utils.test_calcsize. # Here corner cases are addressed @staticmethod def test_len_with_offset_01(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: float = bpack.field(size=8, offset=10, default=1 / 3) assert bpack.calcsize(Record) == 18 @staticmethod def test_len_with_offset_02(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, offset=10, default=0) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.calcsize(Record) == 22 @staticmethod def test_len_with_offset_03(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, offset=10, default=0) field_2: float = bpack.field(size=8, offset=20, default=1 / 3) assert bpack.calcsize(Record) == 28 @staticmethod def test_len_with_offset_04(): size = 30 @bpack.descriptor(size=size) class Record: field_1: int = bpack.field(size=4, offset=10, default=0) field_2: float = bpack.field(size=8, offset=20, default=1 / 3) assert bpack.calcsize(Record) == size @staticmethod def test_len_with_repeat_01(): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=4, default=0, repeat=2) field_2: float = bpack.field(size=8, default=1 / 3) assert bpack.calcsize(Record) == 16 @staticmethod def test_len_with_repeat_02(): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=4, default=0, repeat=2) field_2: List[float] = bpack.field(size=8, default=1 / 3, repeat=2) assert bpack.calcsize(Record) == 24 @staticmethod def test_len_with_repeat_and_offset_01(): @bpack.descriptor class Record: field_1: List[int] = bpack.field( size=4, default=0, repeat=2, offset=6 ) field_2: List[float] = bpack.field(size=8, default=1 / 3, repeat=2) assert bpack.calcsize(Record) == 30 @staticmethod def test_len_with_repeat_and_offset_02(): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=4, default=0, repeat=2) field_2: List[float] = bpack.field( size=8, default=1 / 3, repeat=2, offset=14 ) assert bpack.calcsize(Record) == 30 @staticmethod def test_len_with_repeat_and_offset_03(): with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=4, default=0, repeat=2) field_2: List[float] = bpack.field( size=8, default=1 / 3, repeat=2, offset=6 ) def test_sequence_type(): @bpack.descriptor class Record: field_1: List[int] = bpack.field(size=1, repeat=4) field_1 = bpack.fields(Record)[0] assert field_1.type is List[int] assert bpack.calcsize(Record) == 4 @bpack.descriptor class Record: field_1: Sequence[int] = bpack.field(size=1, repeat=4) field_1 = bpack.fields(Record)[0] assert field_1.type is Sequence[int] with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: Tuple[int, int] = bpack.field(size=1, repeat=2) with pytest.raises(TypeError): @bpack.descriptor class Record: field_1: Sequence = bpack.field(size=1, repeat=2) def test_field_auto_size(): @bpack.descriptor class Record: field_1: bool assert bpack.calcsize(Record) == 1 with pytest.warns(UserWarning): @bpack.descriptor(baseunits=EBaseUnits.BITS) class Record: field_1: bool assert bpack.calcsize(Record) == 1 def test_nested_records(): @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=11) field_2: float = bpack.field(size=4, default=22.22) @bpack.descriptor class NestedRecord: field_1: str = bpack.field(size=10, default="0123456789") field_2: Record = bpack.field( size=bpack.calcsize(Record), default_factory=Record ) field_3: int = bpack.field(size=4, default=3) record = Record() nested_record = NestedRecord() assert nested_record.field_1 == "0123456789" assert nested_record.field_2 == record assert nested_record.field_2.field_1 == record.field_1 assert nested_record.field_2.field_2 == record.field_2 assert nested_record.field_3 == 3 with pytest.raises(bpack.descriptors.DescriptorConsistencyError): @bpack.descriptor class NestedRecord: field_1: str = bpack.field(size=10, default="0123456789") field_2: Record = bpack.field( size=bpack.calcsize(Record) + 1, default_factory=Record ) field_3: int = bpack.field(size=4, default=3) @pytest.mark.parametrize("baseunits", [EBaseUnits.BYTES, EBaseUnits.BITS]) def test_nested_records_autosize(baseunits): @bpack.descriptor(baseunits=baseunits) class Record: field_1: int = bpack.field(size=4, default=1) field_2: int = bpack.field(size=4, default=2) @bpack.descriptor(baseunits=baseunits) class NestedRecord: field_1: int = bpack.field(size=4, default=1) field_2: Record = bpack.field(default_factory=Record) # autosize bytes field_3: int = bpack.field(size=4, default=4) nested_record_size = bpack.calcsize(NestedRecord, baseunits) record_size = bpack.calcsize(Record, baseunits) assert nested_record_size == 8 + record_size bpack-1.1.0/bpack/tests/test_typing.py000066400000000000000000000103061441646351700177510ustar00rootroot00000000000000"""Test bpack.typing.""" import string import pytest import bpack.typing from bpack.typing import TypeParams TYPE_CODES = "iufcS" UNSUPPORTED_TYPE_CODES = "?bBOatmMUV" INVALID_TYPE_CODES = set(string.printable) - set( TYPE_CODES + UNSUPPORTED_TYPE_CODES ) class TestStrToTypeParams: @staticmethod @pytest.mark.parametrize("byteorder", [">", "<", "|", ""]) def test_byteorder(byteorder): s = f"{byteorder}i4" if byteorder in ("", "|"): byteorder = None else: byteorder = bpack.EByteOrder(byteorder if byteorder != "|" else "") params = bpack.typing.str_to_type_params(s) assert params.byteorder is byteorder @staticmethod @pytest.mark.parametrize("size", [1, 2, 4, 8, 3, 120]) def test_size(size): s = f"i{size}" params = bpack.typing.str_to_type_params(s) assert params.size == size assert isinstance(params.size, int) @staticmethod @pytest.mark.parametrize("size", [0, -1]) def test_invalid_size(size): s = f"i{size}" with pytest.raises(ValueError): bpack.typing.str_to_type_params(s) @staticmethod def test_no_size(): s = "i" params = bpack.typing.str_to_type_params(s) assert params.size is None @staticmethod @pytest.mark.parametrize("typecode", INVALID_TYPE_CODES) def test_invalid_typecode(typecode): s = f"{typecode}4" with pytest.raises(ValueError, match="invalid"): bpack.typing.str_to_type_params(s) @staticmethod @pytest.mark.parametrize("typecode", UNSUPPORTED_TYPE_CODES) def test_unsupported_typecode(typecode): s = f"{typecode}4" with pytest.raises(TypeError, match="not supported"): bpack.typing.str_to_type_params(s) @staticmethod @pytest.mark.parametrize("byteorder", [">", "<", "|", ""]) @pytest.mark.parametrize("typecode", ["S"]) @pytest.mark.parametrize("size", ["2", "4", "8"]) def test_bytes_typecode(byteorder, typecode, size): s = f"{byteorder}{typecode}{size}" params = bpack.typing.str_to_type_params(s) assert params.type is bytes assert params.signed is None @staticmethod @pytest.mark.parametrize("byteorder", [">", "<", "|", ""]) @pytest.mark.parametrize("typecode", ["f"]) @pytest.mark.parametrize("size", ["2", "4", "8"]) def test_float_typecode(byteorder, typecode, size): s = f"{byteorder}{typecode}{size}" params = bpack.typing.str_to_type_params(s) assert params.type is float assert params.signed is None @staticmethod @pytest.mark.parametrize("byteorder", [">", "<", "|", ""]) @pytest.mark.parametrize("typecode", ["c"]) @pytest.mark.parametrize("size", ["4", "8", "16"]) def test_complex_typecode(byteorder, typecode, size): s = f"{byteorder}{typecode}{size}" params = bpack.typing.str_to_type_params(s) assert params.type is complex assert params.signed is None @staticmethod @pytest.mark.parametrize("byteorder", [">", "<", "|", ""]) @pytest.mark.parametrize("typecode", ["i", "u"]) @pytest.mark.parametrize("size", ["2", "4", "8"]) def test_int_typecode(byteorder, typecode, size): s = f"{byteorder}{typecode}{size}" signed = bool(typecode == "i") params = bpack.typing.str_to_type_params(s) assert params.type is int assert params.signed == signed @pytest.mark.parametrize( "typestr, type_, params", [ ("i4", int, TypeParams(None, int, 4, True)), ("u2", int, TypeParams(None, int, 2, False)), ("f8", float, TypeParams(None, float, 8, None)), ("c16", complex, TypeParams(None, complex, 16, None)), ("S128", bytes, TypeParams(None, bytes, 128, None)), (">i8", int, TypeParams(bpack.EByteOrder.BE, int, 8, True)), ("f", float, TypeParams(None, float, None, None)), ], ids=["i4", "u2", "f8", "c16", "S128", ">i8", "f"], ) def test_type_annotation(typestr, type_, params): T = bpack.typing.T[typestr] # noqa: N806 assert isinstance(T(), type_) atype, metadata = bpack.typing.get_args(T) assert metadata == params assert atype == type_ == metadata.type bpack-1.1.0/bpack/tests/test_utils.py000066400000000000000000000122761441646351700176070ustar00rootroot00000000000000"""Test bpack.utils.""" import enum import typing import pytest import bpack import bpack.utils class TestEnumType: @staticmethod @pytest.mark.parametrize("value", [1, 1.1, None, "a", b"a"]) def test_enum_type(value): class EEnumType(enum.Enum): A = value assert bpack.utils.enum_item_type(EEnumType) is type(value) @staticmethod def test_invalid_enum_type(): class EInvalidEnumType: A = "a" with pytest.raises(TypeError): assert bpack.utils.enum_item_type(EInvalidEnumType) @staticmethod def test_unsupported_enum_type(): class EEnumType(enum.Enum): A = "a" B = "b" c = 1 with pytest.raises(TypeError): assert bpack.utils.enum_item_type(EEnumType) @staticmethod def test_int_enum_type(): class EEnumType(enum.IntEnum): A = 1 assert bpack.utils.enum_item_type(EEnumType) is int @staticmethod def test_flag_enum_type(): class EEnumType(enum.IntFlag): A = 1 assert bpack.utils.enum_item_type(EEnumType) is int @staticmethod def test_subclass_types(): class EFlagEnumType(enum.IntFlag): A = 1 B = 2 class EEnumType(enum.Enum): A = EFlagEnumType.A B = 0 C = EFlagEnumType.B assert bpack.utils.enum_item_type(EEnumType) is int def test_effective_type(): assert bpack.utils.effective_type(bool) is bool assert bpack.utils.effective_type(int) is int assert bpack.utils.effective_type(str) is str assert bpack.utils.effective_type(bytes) is bytes assert bpack.utils.effective_type(float) is float assert bpack.utils.effective_type(None) is None for type_ in ( typing.Type[typing.Any], typing.Tuple[int, float], typing.Tuple[int], ): assert bpack.utils.effective_type(type_) == type_ def test_effective_enum(): class EStrEnumType(enum.Enum): A = "a" B = "b" assert bpack.utils.effective_type(EStrEnumType) is str class EBytesEnumType(enum.Enum): A = b"a" B = b"b" assert bpack.utils.effective_type(EBytesEnumType) is bytes class EIntEnumType(enum.Enum): A = 1 B = 2 assert bpack.utils.effective_type(EIntEnumType) is int class EFlagEnumType(enum.Enum): A = 1 B = 2 assert bpack.utils.effective_type(EFlagEnumType) is int def test_effective_type_from_seq(): assert bpack.utils.effective_type(typing.List[bool]) is bool assert bpack.utils.effective_type(typing.List[int]) is int assert bpack.utils.effective_type(typing.List[str]) is str assert bpack.utils.effective_type(typing.List[bytes]) is bytes assert bpack.utils.effective_type(typing.List[float]) is float def test_effective_type_from_annotated_type(): assert bpack.utils.effective_type(bpack.T["i"]) is int assert bpack.utils.effective_type(bpack.T["u"]) is int assert bpack.utils.effective_type(bpack.T["f"]) is float assert bpack.utils.effective_type(bpack.T["c"]) is complex assert bpack.utils.effective_type(bpack.T["S"]) is bytes def test_invalid_effective_type(): with pytest.raises(TypeError): bpack.utils.effective_type("i8") def test_effective_type_keep_annotations(): atype = bpack.T["i8"] etype = bpack.utils.effective_type(atype, keep_annotations=True) assert etype == atype def test_get_sequence_type(): type_ = typing.List[int] assert bpack.utils.sequence_type(type_) is list type_ = typing.Sequence[int] assert bpack.utils.sequence_type(type_) is tuple type_ = typing.Tuple[int, int] assert bpack.utils.sequence_type(type_) is None with pytest.raises(TypeError): bpack.utils.sequence_type(type_, error=True) assert bpack.utils.sequence_type(list) is None assert bpack.utils.sequence_type(typing.List) is None assert bpack.utils.sequence_type(typing.Sequence) is None assert bpack.utils.sequence_type(typing.Tuple) is None assert bpack.utils.sequence_type(typing.Type[typing.Any]) is None def test_get_sequence_type_from_annotated_type(): typestr = "i8" type_ = typing.List[bpack.T[typestr]] assert bpack.utils.sequence_type(type_) is list type_ = typing.Sequence[bpack.T[typestr]] assert bpack.utils.sequence_type(type_) is tuple def test_is_sequence_type(): type_ = typing.List[int] assert bpack.utils.is_sequence_type(type_) type_ = typing.Sequence[int] assert bpack.utils.is_sequence_type(type_) type_ = typing.Tuple[int, int] assert not bpack.utils.is_sequence_type(type_) with pytest.raises(TypeError): bpack.utils.is_sequence_type(type_, error=True) assert not bpack.utils.is_sequence_type(list) assert not bpack.utils.is_sequence_type(typing.List) assert not bpack.utils.is_sequence_type(typing.Sequence) assert not bpack.utils.is_sequence_type(typing.Tuple) def test_is_sequence_type_from_annotated_type(): typestr = "i8" type_ = typing.List[bpack.T[typestr]] assert bpack.utils.is_sequence_type(type_) type_ = typing.Sequence[bpack.T[typestr]] assert bpack.utils.is_sequence_type(type_) bpack-1.1.0/bpack/typing.py000066400000000000000000000153621441646351700155570ustar00rootroot00000000000000"""bpack support for type annotations.""" import re from typing import NamedTuple, Optional, Type, Union # @COMPATIBILITY: available in Python 3.7 ... 3.11 try: from typing import _tp_cache except ImportError: def _tp_cache(x): return x try: # pragma: no cover from typing_extensions import ( # isort:skip # @COMPATIBILITY: Python < 3.9 Annotated, # @COMPATIBILITY: Python < 3.7 (and Python < 3.8.3) get_args, get_origin, ) except ImportError: # pragma: no cover from typing import Annotated, get_origin, get_args from .enums import EByteOrder __all__ = ["T", "TypeParams", "is_annotated"] _DTYPE_RE = re.compile( r"^(?P[<|>])?" r"(?P[?bBiufcmMUVOSat])" r"(?P\d+)?$" ) FieldTypes = Type[Union[bool, int, float, complex, bytes, str]] class TypeParams(NamedTuple): """Named tuple describing type parameters.""" byteorder: Optional[EByteOrder] type: FieldTypes # noqa: A003 size: Optional[int] signed: Optional[bool] def __repr__(self): """Return the string representation of the TypeParams object.""" byteorder = self.byteorder byteorder = repr(byteorder) if byteorder is not None else byteorder size = str(self.size) if self.size is not None else self.size return ( f"{self.__class__.__name__}(byteorder={byteorder}, " f"type={self.type.__name__!r}, size={size}, signed={self.signed})" ) def str_to_type_params(typestr: str) -> TypeParams: """Convert a string describing a data type into type parameters. The ``typestr`` parameter is a string describing a data type. The *typestr* string format consists of 3 parts: * an (optional) character describing the byte order of the data - ``<``: little-endian, - ``>``: big-endian, - ``|``: not-relevant * a character code giving the basic type of the array, and * an integer providing the number of bytes the type uses The basic type character codes are: * ``i``: sighed integer * ``u``: unsigned integer * ``f``: float * ``c``: complex * ``S``: bytes (string) .. note:: *typestr* the format described above is a sub-set of the one used in the numpy "array interface". .. seealso:: https://numpy.org/doc/stable/reference/arrays.dtypes.html and https://numpy.org/doc/stable/reference/arrays.interface.html """ mobj = _DTYPE_RE.match(typestr) if mobj is None: raise ValueError(f"invalid data type specifier: '{typestr}'") byteorder = mobj.group("byteorder") stype = mobj.group("type") size = mobj.group("size") signed = None if size is not None: size = int(size) if size <= 0: raise ValueError(f"invalid size: '{size}'") if byteorder == "|": byteorder = None elif byteorder is not None: byteorder = EByteOrder(byteorder) # if stype == '?' or (stype == 'b' and size == 1): # type_ = bool # elif stype in 'bB': # type_ = bytes # elif stype == 'i': if stype == "i": type_ = int signed = True elif stype == "u": type_ = int signed = False elif stype == "f": type_ = float elif stype == "c": type_ = complex # elif stype == 'm': # type_ = datetime.timedelta # elif stype == 'M': # type_ = datetime.datetime # elif stype == 'U': # type_ = str elif stype == "S": type_ = bytes # elif stype == 'V': # type_ = bytes else: # '?': bool # 'b': (signed) byte (single item) # 'B': (unsigned) byte (single item) # 't': bitfield # 'O': object # 'U': (unicode) str (32bit UCS4 encoding) # 'a' : null terminated strings # 'm', 'M': timedelta and datetime raise TypeError( f"type specifier '{stype}' is valid for the 'array protocol' but " f"not supported by bpack" ) return TypeParams(byteorder, type_, size, signed) class T: """Allow to specify numeric type annotations using string descriptors. Example:: >>> T['u4'] # doctest: +NORMALIZE_WHITESPACE typing.Annotated[int, TypeParams(byteorder=None, type='int', size=4, signed=False)] The resulting type annotation is a :class:`typing.Annotated` numeric type with attached a :class:`bpack.typing.TypeParams` instance. String descriptors, or *typestr*, are compatible with numpy (a sub-set of one used in the numpy "array interface"). The *typestr* string format consists of 3 parts: * an (optional) character describing the byte order of the data - ``<``: little-endian, - ``>``: big-endian, - ``|``: not-relevant * a character code giving the basic type of the array, and * an integer providing the number of bytes the type uses The basic type character codes are: * ``i``: sighed integer * ``u``: unsigned integer * ``f``: float * ``c``: complex * ``S``: bytes (string) .. note:: *typestr* the format described above is a sub-set of the one used in the numpy "array interface". .. seealso:: :func:`str_to_type_params`, :class:`TypeParams`, https://numpy.org/doc/stable/reference/arrays.dtypes.html and https://numpy.org/doc/stable/reference/arrays.interface.html """ __slots__ = () def __new__(cls, *args, **kwargs): """Initialize a new `T` descriptor.""" raise TypeError(f"Type '{cls.__name__}' cannot be instantiated.") @_tp_cache def __class_getitem__(cls, params): # noqa: D105, N805 if not isinstance(params, str): raise TypeError( f"{cls.__name__}[...] should be used with a single argument " f"(a string describing a basic numeric type)." ) typestr = params metadata = str_to_type_params(typestr) return Annotated[metadata.type, metadata] def __init_subclass__(cls, *args, **kwargs): """Subclass initializer. Alway raise a TypeError to prevent sub-classing. """ raise TypeError(f"Cannot subclass {cls.__module__}.{cls.__name__}") def is_annotated(type_: Type) -> bool: """Return True if the input is an annotated numeric type. An *annotated numeric type* is assumed to be a :class:`typing.Annotated` type annotation of a basic numeric type with attached a :class:`bpack.typing.TypeParams` instance. .. seealso:: :class:`bpack.typing.T`. """ if get_origin(type_) is Annotated: args = get_args(type_) if len(args) == 2: etype, params = args return isinstance(etype, type) and isinstance(params, TypeParams) return False bpack-1.1.0/bpack/utils.py000066400000000000000000000115261441646351700154030ustar00rootroot00000000000000"""Utility functions and classes.""" import enum import typing import functools import dataclasses import collections.abc from typing import Type, Union from .typing import is_annotated, get_origin, get_args, Annotated def classdecorator(func): """Class decorator that can be used with or without parameters.""" @functools.wraps(func) def wrapper(cls=None, **kwargs): def wrap(klass): return func(klass, **kwargs) # Check if called as @decorator or @decorator(...). if cls is None: # Called with parentheses return wrap # Called as @decorator without parentheses return wrap(cls) return wrapper def create_fn( name, args, body, *, globals=None, # noqa: A002 locals=None, # noqa: A002 return_type=dataclasses.MISSING, ): """Create a function object.""" return dataclasses._create_fn( name, args, body, globals=globals, locals=locals, return_type=return_type, ) def set_new_attribute(cls, name, value): """Programmatically add a new attribute/method to a class.""" return dataclasses._set_new_attribute(cls, name, value) def sequence_type(type_: Type, error: bool = False) -> Union[Type, None]: """Return the sequence type associated to a typed sequence. The function return :class:`list` or :class:`tuple` if the input is considered a valid typed sequence, ``None`` otherwise. Please note that fields annotated with :class:`typing.Tuple` are not considered homogeneous sequences even if all items are specified to have the same type. """ sequence_type_ = get_origin(type_) if sequence_type_ is None: return None if not issubclass(sequence_type_, typing.Sequence): return None args = get_args(type_) if len(args) < 1: return None if len(args) > 1: if error: raise TypeError(f"{type_} is not supported") else: return None if not is_annotated(args[0]) and not isinstance(args[0], type): # COMPATIBILITY: with typing_extensions and Python v3.7 # need to be a concrete type return None # pragma: no cover if not issubclass(sequence_type_, collections.abc.MutableSequence): sequence_type_ = tuple assert sequence_type_ in {list, tuple} return sequence_type_ def is_sequence_type(type_: Type, error: bool = False) -> bool: """Return True if the input is an homogeneous typed sequence. Please note that fields annotated with :class:`typing.Tuple` are not considered homogeneous sequences even if all items are specified to have the same type. """ seq_type = sequence_type(type_, error=error) return seq_type is not None def is_enum_type(type_: Type) -> bool: """Return True if the input is and :class:`enum.Enum`.""" return get_origin(type_) is None and issubclass(type_, enum.Enum) def enum_item_type(enum_cls: Type[enum.Enum]) -> Type: """Return the type of the items of an enum.Enum. This function also checks that all items of an enum have the same (or compatible) type. """ if not is_enum_type(enum_cls): raise TypeError(f'"{enum_cls}" is not an enum. Enum') elif issubclass(enum_cls, int): return int else: types_ = [type(item.value) for item in enum_cls] type_ = types_.pop() for item in types_: if issubclass(item, type_): continue elif issubclass(type_, item): type_ = item else: raise TypeError( "only Enum with homogeneous values are supported" ) return type_ def effective_type( type_: Union[Type, Type[enum.Enum], Type], keep_annotations: bool = False ) -> Type: """Return the effective type. In case of enums or sequences return the item type. """ origin = get_origin(type_) if origin is None: if type_ is not None and issubclass(type_, enum.Enum): etype = enum_item_type(type_) else: etype = type_ elif origin is Annotated: # TODO: check issubclass(origin, Annotated): if keep_annotations: etype = type_ else: etype, _ = get_args(type_) elif not issubclass(origin, typing.Sequence): etype = type_ elif issubclass(origin, typing.Tuple): etype = type_ else: # is a sequence args = get_args(type_) assert len(args) == 1 etype = effective_type(args[0], keep_annotations=keep_annotations) return etype def is_int_type(type_: Type) -> bool: """Return true if the effective type is an integer.""" if is_sequence_type(type_): etype = effective_type(type_) return issubclass(etype, int) else: return issubclass(type_, int) bpack-1.1.0/docs/000077500000000000000000000000001441646351700135345ustar00rootroot00000000000000bpack-1.1.0/docs/Makefile000066400000000000000000000011721441646351700151750ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) bpack-1.1.0/docs/TODO.rst000066400000000000000000000007101441646351700150310ustar00rootroot00000000000000:orphan: Miscellanea *To Do* list ------------------------ * check alignment in bpack.st (native '@' and '' behaves differently form '=') * improve documentation * improve typing * benchmarks * :class:`EBaseUnits` shall become a :class:`IntFlag` to allow the *decoders* to declare base-units as follows:: baseunits = bpack.EBaseUnits.BITS | bpack.EBaseUnits.BYTES * nested records and repeat (test) * typing in generated methods (.frombytes) TBD bpack-1.1.0/docs/_templates/000077500000000000000000000000001441646351700156715ustar00rootroot00000000000000bpack-1.1.0/docs/_templates/apidoc/000077500000000000000000000000001441646351700171305ustar00rootroot00000000000000bpack-1.1.0/docs/_templates/apidoc/package.rst_t000066400000000000000000000021771441646351700216070ustar00rootroot00000000000000{%- macro automodule(modname, options) -%} .. automodule:: {{ modname }} {%- for option in options %} :{{ option }}: {%- endfor %} {%- endmacro %} {%- macro toctree(docnames) -%} .. toctree:: :maxdepth: {{ maxdepth }} {% for docname in docnames %} {{ docname }} {%- endfor %} {%- endmacro %} {%- if is_namespace %} {{- [pkgname, "namespace"] | join(" ") | e | heading }} {% else %} {{- [pkgname, "package"] | join(" ") | e | heading }} {% endif %} {%- if modulefirst and not is_namespace %} {{ automodule(pkgname, automodule_options) }} {% endif %} {%- if subpackages %} .. only:: html Subpackages ----------- {{ toctree(subpackages) }} {% endif %} {%- if submodules %} .. only:: html Submodules ---------- {% if separatemodules %} {{ toctree(submodules) }} {% else %} {%- for submodule in submodules %} {% if show_headings %} {{- [submodule, "module"] | join(" ") | e | heading(2) }} {% endif %} {{ automodule(submodule, automodule_options) }} {% endfor %} {%- endif %} {%- endif %} {%- if not modulefirst and not is_namespace %} Module contents --------------- {{ automodule(pkgname, automodule_options) }} {% endif %} bpack-1.1.0/docs/api/000077500000000000000000000000001441646351700143055ustar00rootroot00000000000000bpack-1.1.0/docs/api/bpack.ba.rst000066400000000000000000000001601441646351700164750ustar00rootroot00000000000000bpack.ba module =============== .. automodule:: bpack.ba :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.bs.rst000066400000000000000000000001601441646351700165170ustar00rootroot00000000000000bpack.bs module =============== .. automodule:: bpack.bs :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.codecs.rst000066400000000000000000000001741441646351700173600ustar00rootroot00000000000000bpack.codecs module =================== .. automodule:: bpack.codecs :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.descriptors.rst000066400000000000000000000002131441646351700204530ustar00rootroot00000000000000bpack.descriptors module ======================== .. automodule:: bpack.descriptors :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.enums.rst000066400000000000000000000001711441646351700172440ustar00rootroot00000000000000bpack.enums module ================== .. automodule:: bpack.enums :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.np.rst000066400000000000000000000001601441646351700165300ustar00rootroot00000000000000bpack.np module =============== .. automodule:: bpack.np :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.rst000066400000000000000000000004701441646351700161200ustar00rootroot00000000000000bpack package ============= .. automodule:: bpack :members: :undoc-members: :show-inheritance: .. only:: html Submodules ---------- .. toctree:: :maxdepth: 4 bpack.ba bpack.bs bpack.codecs bpack.descriptors bpack.enums bpack.np bpack.st bpack.typing bpack.utils bpack-1.1.0/docs/api/bpack.st.rst000066400000000000000000000001601441646351700165410ustar00rootroot00000000000000bpack.st module =============== .. automodule:: bpack.st :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.typing.rst000066400000000000000000000001741441646351700174320ustar00rootroot00000000000000bpack.typing module =================== .. automodule:: bpack.typing :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/api/bpack.utils.rst000066400000000000000000000001711441646351700172550ustar00rootroot00000000000000bpack.utils module ================== .. automodule:: bpack.utils :members: :undoc-members: :show-inheritance: bpack-1.1.0/docs/bpack.rst000066400000000000000000000007331441646351700153510ustar00rootroot00000000000000=========================================== Binary data structures (un-)Packing library =========================================== :HomePage: https://github.com/avalentino/bpack :Author: Antonio Valentino :Contact: antonio.valentino@tiscali.it :Copyright: 2020-2023, Antonio Valentino :Version: |release| .. toctree:: :maxdepth: 2 overview installation userguide api/bpack development license release_notes bpack-1.1.0/docs/conf.py000066400000000000000000000063161441646351700150410ustar00rootroot00000000000000# Configuration file for the Sphinx documentation builder. # # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- import os import sys sys.path.insert(0, os.path.abspath('..')) # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = 'bpack' copyright = '2020-2023, Antonio Valentino' # noqa: D100 author = 'Antonio Valentino' # The full version, including alpha/beta/rc tags import bpack release = bpack.__version__ master_doc = 'index' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.extlinks', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx_rtd_theme', ] try: import sphinxcontrib.spelling # noqa: F401 except ImportError: pass else: extensions.append('sphinxcontrib.spelling') templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "sphinx_rtd_theme" html_static_path = ['_static'] html_theme_options = { # 'vcs_pageview_mode': 'blob', } html_context = { # 'github_url': 'https://github.com/avalentino/bpack/', 'display_github': True, 'github_user': 'avalentino', 'github_repo': 'bpack', 'github_version': 'main', 'conf_py_path': '/docs/', # Path in the checkout to the docs root } html_last_updated_fmt = '' # -- Options for LaTeX output ------------------------------------------------ latex_documents = [ # (startdocname, targetname, title, author, theme, toctree_only) (project, project + '.tex', 'Binary data structures (un-)Packing library', author, 'manual', False), ] latex_domain_indices = False latex_elements = { # 'papersize': 'a4paper', 'pointsize': '12pt', } # -- Extension configuration ------------------------------------------------- # -- Options for autodoc extension ------------------------------------------- # autoclass_content = 'both' autodoc_member_order = 'groupwise' # autodoc_default_options = {} autodoc_mock_imports = [] for module_name in ['bitarray', 'bitstruct', 'numpy']: try: __import__(module_name) except ImportError: autodoc_mock_imports.append(module_name) # -- Options for intersphinx extension --------------------------------------- intersphinx_mapping = { 'https://docs.python.org/3/': None, 'https://numpy.org/doc/stable/': None, } # -- Options for extlinks extension ------------------------------------------ extlinks = { 'issue': ('https://github.com/avalentino/bpack/issues/%s', 'gh-%s'), } # -- Options for todo extension ---------------------------------------------- todo_include_todos = True bpack-1.1.0/docs/development.rst000066400000000000000000000060331441646351700166120ustar00rootroot00000000000000Developers Guide ================ Project links ------------- :PyPI page: https://pypi.org/project/bpack :repository: https://github.com/avalentino/bpack :issue tracker: https://github.com/avalentino/bpack/issues :CI: https://github.com/avalentino/bpack/actions :HTML documentation: https://bpack.readthedocs.io Set-up the development environment ---------------------------------- Pip ~~~ .. code-block:: shell $ python3 -m venv --prompt venv .venv $ source .venv/bin/activate (venv) $ python3 -m pip install -r requirements-dev-txt Conda ~~~~~ .. code-block:: shell $ conda create -c conda-forge -n bpack \ --file requirements-dev.txt python=3 Debian/Ubuntu ~~~~~~~~~~~~~ .. code-block:: shell $ sudo apt install python3-bitstruct python3-bitarray \ python3-pytest python3-pytest-cov \ python3-sphinx python3-sphinx-rtd-theme Testing the code ---------------- Basic testing ~~~~~~~~~~~~~ .. code-block:: shell $ python3 -m pytest It is also recommended to use the ``-W=error`` option. Advanced testing ~~~~~~~~~~~~~~~~ Tox_ is used to run a comprehensive test suite on multiple Python version. I also check formatting, coverage and checks that the documentation builds properly. .. code-block:: shell $ tox Test coverage ------------- .. code-block:: shell $ python3 -m pytest --cov --cov-report=html --cov-report=term bpack Check code style and formatting ------------------------------- The code style and formatting shall be checked with flake8_ as follows: .. code-block:: shell $ python3 -m flake8 --statistics --count bpack Moreover, also the correct formatting of "docstrings" shall be checked, using pydocstyle_ this time: .. code-block:: shell $ python3 -m pydocstyle --count bpack A more strict check of formatting can be done using black_: .. code-block:: shell $ python3 -m black --check bpack Finally the ordering of imports can be checked with isort_ as follows: .. code-block:: shell $ python3 -m isort --check bpack Please note that all the relevant configuration for the above mentioned tools are in the `pyproject.toml` file. Build the documentation ----------------------- .. code-block:: shell $ make -C docs html Test code snippets in the documentation --------------------------------------- .. code-block:: shell $ make -C docs doctest Check documentation links ------------------------- .. code-block:: shell $ make -C docs linkcheck Check documentation spelling ---------------------------- .. code-block:: shell $ make -C docs spelling Update the API documentation ---------------------------- .. code-block:: shell $ rm -rf docs/api $ sphinx-apidoc --module-first --separate --no-toc \ --doc-project "bpack API" -o docs/api \ --templatedir docs/_templates/apidoc \ bpack bpack/tests .. _Tox: https://tox.readthedocs.io .. _Python: https://www.python.org .. _flake8: https://flake8.pycqa.org .. _pydocstyle: http://www.pydocstyle.org .. _black: https://black.readthedocs.io .. _isort: https://pycqa.github.io/isort bpack-1.1.0/docs/index.rst000066400000000000000000000011331441646351700153730ustar00rootroot00000000000000.. bpack documentation master file, created by sphinx-quickstart on Sun Nov 29 10:40:46 2020. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. .. title and metadata .. include:: bpack.rst :end-before: .. toctree:: .. description .. include:: ../README.rst :start-after: .. badges :end-before: .. local-definitions .. |struct| replace:: :mod:`struct` Documentation ============= .. toctree:: :maxdepth: 2 overview installation userguide api/bpack development license release_notes indices bpack-1.1.0/docs/indices.rst000066400000000000000000000001721441646351700157040ustar00rootroot00000000000000.. only:: html Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` bpack-1.1.0/docs/installation.rst000066400000000000000000000015111441646351700167650ustar00rootroot00000000000000Installation ============ Pip --- Basic installation: .. code-block:: shell $ python3 -m pip install bpack Recommended: .. code-block:: shell $ python3 -m pip install bpack[bs] to install also dependencies necessary to use the :mod:`bpack.bs` backend and (for binary structures defined up to the bit level). Conda ----- .. code-block:: shell $ conda install -c conda-forge -c avalentino bpack Testing ------- To run the test suite it is necessary to have pytest_ installed: .. code-block:: shell $ python3 -m pytest --pyargs bpack This only tests codec backends for which the necessary dependencies are available. To run a complete test please make sure to install all optional dependencies and testing libraries: .. code-block:: shell $ python3 -m pip install bpack[test] .. _pytest: https://docs.pytest.org bpack-1.1.0/docs/license.rst000066400000000000000000000013231441646351700157070ustar00rootroot00000000000000Copyright and License ===================== Copyright 2020-2023 Antonio Valentino Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Integral license text --------------------- .. include:: ../LICENSE :literal: bpack-1.1.0/docs/make.bat000066400000000000000000000014401441646351700151400ustar00rootroot00000000000000@ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.https://www.sphinx-doc.org/ exit /b 1 ) if "%1" == "" goto help %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd bpack-1.1.0/docs/overview.rst000066400000000000000000000051631441646351700161410ustar00rootroot00000000000000Overview ======== What is bpack? -------------- .. include:: ../README.rst :start-after: .. description :end-before: .. local-definitions .. |struct| replace:: :mod:`struct` Features -------- * declarative description of binary data structures * specification of data structures up to *bit* level * automatic *codec* generation form data descriptors * decoding (form binary data to Python objects) * encoding (form Python objects to binary data) * backend: - :mod:`bpack.st` backend based on the :mod:`struct` module of standard Python_ library - :mod:`bpack.bs` backend based on bitstruct_ - :mod:`bpack.np` backend based on numpy_ - :mod:`bpack.ba` backend based on bitarray_ (only included for benchmarking purposes) * support for signed/unsigned integer types * support for :class:`enum.Enum` types * support for sequence types, i.e. fields with multiple (homogeneous) items * both bit and byte order can be specified by the user * automatic size determination for some data types * record nesting (the field in a record descriptor can be another record) * possibility to specify data types using the special type annotation class :class:`bpack.typing.T` that accepts annotations and string specifiers compatible with the numpy_ "Array Interface" and ``dtype`` * comprehensive test suite .. _limitations-label: Limitations ----------- * only fixed size binary records are supported by design, the size of the record shall be known at the moment of the record descriptor definition. It should be easy for the user to leverage tools provided by the *bpack* Python package to support more complex decoding scenarios. * currently it is assumed that all fields in a binary record share the same bit/byte order. The management of different byte order in the same binary record is, in principle, possible but not planned at the moment. * sequence types can only contain basic numeric types; nested sequences, sequences of enums or sequences of records are not allowed at the moment. * record nesting is only possible for records having the same base-units, bits or bytes, and compatible decoder types eventually. * currently the :mod:`bpack.np` has a limited (incomplete) support to record nesting and encoding capabilities. Possible additional features still not implemented -------------------------------------------------- * user defined converters * support for complex and datetime data types * conversion to CSV, HDF5 and :func:`numpy.dtype` .. _Python: https://www.python.org .. _bitstruct: https://github.com/eerimoq/bitstruct .. _numpy: https://numpy.org .. _bitarray: https://github.com/ilanschnell/bitarray bpack-1.1.0/docs/release_notes.rst000066400000000000000000000102351441646351700171170ustar00rootroot00000000000000Release Notes ============= bpack v1.1.0 (15/04/2023) ------------------------- * Added support for signed integers to :func:`bpack.np.unpackbits`. Both standard signed integers and integers encoded with sign and module are now supported. * Use uppercase enums in `s1isp.py` example. * Improved docstrings in :mod:`bpack.np`. * Fixed several typos. bpack v1.0.0 (05/02/2023) ------------------------- * Fix compatibility with Python v3.11. * Move setup configuration form `setup.cfg` to `pyproject.toml`. bpack v0.8.2 (20/03/2022) ------------------------- * Fallback to standard bitstruct if the bitstruct.c extension does not support the format string bpack v0.8.1 (30/11/2021) ------------------------- * Drop ``setup.py``, no longer needed. * Improve compatibility with `typing-extensions`_ v4.0 (closes :issue:`1`). * Use the compiled extension of `bitstruct`_ when available (and compatible with the specified format string). * Use `cbitsturct`_ when available (preferred over the compiled extension of `bitstruct`_). .. _`typing-extensions`: https://pypi.org/project/typing-extensions .. _bitstruct: https://github.com/eerimoq/bitstruct .. _cbitsturct: https://github.com/qchateau/cbitstruct bpack v0.8.0 (03/06/2021) ------------------------- * New "encoding" feature. Records can be now encoded into binary strings using the :mod:`bpack.st` and :mod:`bpack.bs` backends. Previously only "decoding" was supported. The :mod:`bpack.np` only implements a partial support to encoding currently. bpack v0.7.1 (08/03/2021) ------------------------- * Improved User Guide * :func:`bpack.np.unpackbits` has been generalized and optimized. * New example for packet decoding. * Improved support for nested records. bpack v0.7.0 (21/01/2021) ------------------------- * New *packbit*/*unpackbit* functions (provisional API). * Fixed a bug in decoding of nested records. * Added example program for Sentinel-1 space packets decoding bpack v0.6.0 (15/01/2021) ------------------------- * New numpy_ based backend. * New :meth:`bpack.enums.EByteOrder.get_native` method. * Now data types in descriptor definition can also be specified by means of special type annotation type (:class:`bpack.typing.T`) that accepts numpy-like format strings. * Now it is no longer necessary to use the :func:`dataclasses.dataclass` decorator to define a descriptor. That way to define descriptors is **depercated**. All parameters previously specified via :func:`dataclasses.dataclass` (like e.g. *frozen*) shall now be passed directly to the :func:`bpack.descriptors.descriptor` decorator. With this change the use of :mod:`dataclasses` becomes an implementation detail. * The ``size`` parameter of the :func:`bpack.descriptors.field` factory function is now optional. * General improvements and code refactoring. * Improved CI testing. * Added automatic spell checking of documentation in CI. * Backward incompatible changes: - :class:`bpack.enums.EBaseUnits`, :class:`bpack.enums.EByteOrder` and :class:`bpack.enums.EBitOrder` enums moved to the new :mod:`bpack.enums` module (the recommended way to access enums is directly form :mod:`bpack`, e.g. ``bpack.EByteOrder``) - :data:`bpack.enums.EByteOrder.BIG` and :data:`bpack.enums.EByteOrder.LITTLE` enumerates have been renamed into :data:`bpack.enums.EByteOrder.BE` and :data:`bpack.enums.EByteOrder.LE` respectively - classes decorated with the :func:`bpack.descriptors.descriptor` decorator no longer have the ``__len__`` method automatically added; the recommended way to compute the size of a descriptors (class or instance) is to use the :func:`bpack.descriptros.calcsize` function - the default behavior of the :func:`bpack.decorators.calcsize` has been changed to return the size of the input *descriptor* in the same *base units* of the descriptor itself; previously the default behavior was to return the size in bytes .. _numpy: https://numpy.org bpack v0.5.0 (31/12/2020) ------------------------- * Initial release. The package implements all core functionalities but - the API is still not stable - the documentation is incomplete - some advanced feature is still missing bpack-1.1.0/docs/spelling_wordlist.txt000066400000000000000000000004111441646351700200350ustar00rootroot00000000000000ba backend backends benchmarking bitarray bitstruct bpack bs codec codecs conda dataclass de dev dicts docstring docstrings endian endianess enum enums frombytes instantiation miscellanea np numpy programmatically python seealso struct tobytes utils un unicode utf bpack-1.1.0/docs/userguide.rst000066400000000000000000000633611441646351700162730ustar00rootroot00000000000000User Guide ========== Core concepts ------------- *bpack* is a lightweight Python package intended to help users to * describe binary data structures * encode/decode binary data to/form Python object Descriptors ~~~~~~~~~~~ The user can define binary data structure in a declarative way, as follows: .. testcode:: import bpack @bpack.descriptor class BinaryRecord: field_1: float = bpack.field(size=8) field_2: int = bpack.field(size=4, signed=True) Key concepts for definition of binary data structures are * the declaration of the data structure by means of the :func:`bpack.descriptors.descriptor` class decorator. It allows to specify the main properties of the data structure. * the specification of the characteristics of each field, mainly the data type, the size and (optionally) the offset with respect to the beginning of the record. This can be done using the :func:`bpack.descriptors.field` factory function. In the above example the ``BinaryRecord`` has been defined to have two fields: :field_1: a double precision floating point (8 bytes) :field_2: a 32bit signed integer (``size`` is expressed in bytes in this case) The offset of the fields have not been explicitly specified so they are computed automatically. In the example ``field_1`` has ``offset=0``, while ``field_2`` has ``offset=4`` i.e. data belonging to ``field_2`` immediately follow the ones of the previous field. The design is strongly inspired to the one of the :mod:`dataclasses` package of the Python standard library. Codecs ~~~~~~ Once a binary structure is defined, the *bpack* package allows to automatically generate :class:`Codec` objects that are able to convert binary data into a Python objects and vice versa: .. testcode:: import bpack.st binary_data = b'\x18-DT\xfb!\t@\x15\xcd[\x07' codec = bpack.st.Codec(BinaryRecord) record = codec.decode(binary_data) assert record.field_1 == 3.141592653589793 assert record.field_2 == 123456789 print(record) .. testoutput:: BinaryRecord(field_1=3.141592653589793, field_2=123456789) .. testcode:: encoded_data = codec.encode(record) assert binary_data == encoded_data print('binary_data: ', binary_data) print('encoded_data:', encoded_data) .. testoutput:: binary_data: b'\x18-DT\xfb!\t@\x15\xcd[\x07' encoded_data: b'\x18-DT\xfb!\t@\x15\xcd[\x07' In the example above it has been used the :class:`bpack.st.Codec` class form the :mod:`bpack.st` module. Please note that the decoder class (:class:`bpack.st.Codec`) * takes in input the *descriptor* (i.e. the type) of the binary data structure, and * return a *codec* object which is capable to encode/decode only binary data organized according to the *descriptor* received at the instantiation time. If one need to encode/decode a differed data structure than it is necessary to instantiate a different codec. The :mod:`bpack.st` module used in the example is just one of the, so called, *backends* available in *bpack*. See the Backends_ section below for more details. Binary data structures declaration ---------------------------------- As anticipated above the declaration of a binary data structure and its main properties is done using the :func:`bpack.descriptors.descriptor` class decorator. Bit vs byte structures ~~~~~~~~~~~~~~~~~~~~~~ One of the properties that the :func:`bpack.descriptors.descriptor` class decorator allows to specify is *baseunits*. It allows to specify the elementary units used to describe the binary structure itself. A structure can be described in terms of *bytes* or in terms of *bits*, i.e. if field size and offsets have to be intended as number of bytes of as number of bits. This is an important distinction for two reasons: * it is fundamental for *decoders* (see below) to know much data have to be converted and where this data are exactly located in a string of bytes * not all *backends* are capable of decoding both kinds of structures .. note:: Currently available *backends* do not support nested data structures (see `Record nesting`_) described using different *baseunits* (see :ref:`limitations-label`). Anyway it is in the plans to overcome this limitation. *Baseunits* can be specified as follows: .. testcode:: @bpack.descriptor(baseunits=bpack.EBaseUnits.BITS) class BitRecord: field_1: bool = bpack.field(size=1) field_2: int = bpack.field(size=3) field_3: int = bpack.field(size=4) The ``baseunits`` parameter has been specified as a parameter of the :func:`bpack.descriptors.descriptor` class decorator and its possible values are enumerated by the :class:`bpack.enums.EBaseUnits` :class:`enum.Enum`: * :data:`bpack.enums.EBaseUnits.BITS`, or * :data:`bpack.enums.EBaseUnits.BYTES` If the ``baseunits`` parameter is not specified than it is assumed to be equal to :data:`bpack.enums.EBaseUnits.BYTES` by default. Please note that the entire data structure of the above example is only 8 bits (1 byte) large. .. note:: Please note that *baseunits* and many of the function and method parameters whose valued is supposed to be an :class:`enum.Enum` can also accept a string value. E.g. the above example can also be written as follows: .. testcode:: @bpack.descriptor(baseunits='bits') class BitRecord: field_1: bool = bpack.field(size=1) field_2: int = bpack.field(size=3) field_3: int = bpack.field(size=4) Please refer to the specific enum documentation (in this case :class:`bpack.enums.EBaseUnits`) to know which are string values corresponding to the desired enumerated value. Specifying bit/byte order ~~~~~~~~~~~~~~~~~~~~~~~~~ Other important parameters for the :func:`bpack.descriptors.descriptor` class decorator are: :byteorder: whose possible values are described by :class:`bpack.enums.EByteOrder`. By the fault the native byte order is assumed. :bitorder: whose possible values are described by :class:`bpack.enums.EBitOrder`. The *bitorder* parameter shall always be set to ``None`` the if *baseunits* value is :data:`bpack.enums.EBaseUnits.BYTES`. Both this parameters describe the internal organization of binary data of each field. Descriptor size ~~~~~~~~~~~~~~~ The :func:`bpack.descriptors.descriptor` class decorator also allows to specify *explicitly* the overall size of the binary data structure: .. testcode:: @bpack.descriptor(baseunits='bits', size=8) class BinaryRecord: field_1: bool = bpack.field(size=1) field_2: int = bpack.field(size=3) In this case the the overall size of ``BitRecord`` is 8 bits (1 bytes) .. doctest:: >>> bpack.calcsize(BinaryRecord) 8 even if the sum of sizes of all fields is only 4 bits. Usually explicitly specifying the *size* of a binary data structure is not necessary because the *bpack* is able to compute it automatically by looking at the size of fields. In some cases, anyway, it can be useful to specify it, e.g. when one want to use a descriptor like the one defined in the above example as field of a larger descriptor (see `Record nesting`_). In this case it is important tho know the correct size of each field in order to be able to automatically compute the *offset* of the following fields. Fields specification -------------------- As anticipated in the previous section there are three main elements that the *bpack* package need to know about fields in order to have a complete description of a binary data structure: * the field data **type**, * the filed **size** (expressed in *baseunits*, see `Bit vs byte structures`_), and * the field **offset** with respect to the beginning of the binary data structure (also in this case expressed in *baseunits*, see `Bit vs byte structures`_) .. testcode:: @bpack.descriptor class BinaryRecord: field: int = bpack.field(size=4, offset=0) Please note, anyway, that in some case it is possible to infer some of the above information from the context so it is not always to specify all of them explicitly. More details will be provided in the following. As shown in the example above the main way to specify a field descriptor is to use the :func:`bpack.descriptors.field` factory function together with Python type annotations to specify the data type. Type ~~~~ The data type of a field is the only parameter that is always mandatory, and also it is the only parameter that is not specified by means of the :func:`bpack.descriptors.field` factory function, rather it is specified the standard Python syntax for type annotations. Currently supported data types are: :basic types: basic Python types like ``bool``, ``int``, ``float``, ``bytes``, ``str`` (``complex`` is not supported currently) :enums: enumeration types defined using the :mod:`enum` module of the standard Python library. Please refer to the `Enumeration fields`_ section for more details about features and limitations :sequences: used to define fields containing a sequence of homogeneous values (i.e. values having the same data type). A *sequence* data type in *bpack* can be defined using the standard type annotations classes like :class:`typing.Sequence` or :class:`typing.List`. Please refer to the `Sequence fields`_ section for more details about features and limitations :descriptors: i.e. any binary data structure defined using the :func:`bpack.descriptors.descriptor` class decorator (see also `Record nesting`_) :type annotations: annotated data types defined by means of the :class:`bpack.typing.T` type annotation. Please refer to the `Special type annotations`_ section for a more detailed description .. note:: The ``str`` type in Python is used to represent unicode strings. The conversion of this kind of strings form/to binary format requires some form of decoding/encoding. *Bpack* codecs (see `Data codecs`_) convert ``str`` data form/to ``bytes`` strings using the "UTF-8" encoding. Please note that the *size* of a ``str`` field still describes the number of bits/bytes if its binary representation, not the length of the string (which in principle could require a number of bytes larger that the number of characters). Size ~~~~ The field *size* is specified as a positive integer in *baseunits* (see the `Bit vs byte structures`_ section). It is a fundamental information and it must be always specified by means of the :func:`bpack.descriptors.field` factory function unless it is absolutely clear and unambiguous how to determine the fields size from the data type. This is only possible in the following cases: * the data type is ``bool`` in which case the size is assumed to be ``1`` (at the moment no other basic type has a default size associated) * the data type is a record descriptor, in which case the field size is computed as follows: .. testcode:: bpack.calcsize(BinaryRecord, units=bpack.baseunits(BinaryRecord)) * the data type is specified using special type annotations also including size information: .. testcode:: from bpack import T @bpack.descriptor class BinaryRecord: field: T['u3'] The ``T['i3']`` type annotation specifier defines an unsigned integer type (``u``) having size 3 (for the specific example this means 3 bytes) Please refer to the `Special type annotations`_ section for more details. Please note that the size of the field must not necessarily correspond to the size of one of the data types supported by the platform. In the example above it has been specified a type ``T['i3']`` which corresponds to a 24 bits unsigned integer. It is represented using a standard Python ``int`` in the Python code but the binary representation will always take only 3 bytes. Offset ~~~~~~ The field *offset* is specified as a not-negative integer in *baseunits* (see the `Bit vs byte structures`_ section), and it represent the amount of *baseunits* for the beginning pf the record to the beginning of the field. It is a fundamental information and it can be specified by means of the :func:`bpack.descriptors.field` factory function. The *bpack* package, anyway, implements a mechanism to automatically compute the field offset exploiting information of the other fields in the record. For this reason it is necessary to specify the field *offset* explicitly only in very specific cases. For example the *verbose* definition of a record with 5 integer fields looks like the following: .. testcode:: @bpack.descriptor class BinaryRecord: field_1: int = bpack.field(size=4, offset=0) field_2: int = bpack.field(size=4, offset=4) field_3: int = bpack.field(size=4, offset=8) field_4: int = bpack.field(size=4, offset=12) field_5: int = bpack.field(size=4, offset=16) If not specified the offset of the first field is assumed to be ``0``, and the offset of the following fields is assumed to be equal to the offset of the previous field plus the size of the previous field itself:: field[n].offset = field[n - 1].offset + field[n - 1].size In short the automatic offset computation works assuming that all fields are stored contiguously and without holes. .. testcode:: @bpack.descriptor class BinaryRecord: field_1: int = bpack.field(size=4) # offset = 0 first field field_2: int = bpack.field(size=4) # offset = 4 # field_1.offset + field_1.size field_3: int = bpack.field(size=4) # offset = 8 # field_2.offset + field_2.size field_4: int = bpack.field(size=4) # offset = 12 # field_3.offset + field_3.size field_5: int = bpack.field(size=4) # offset = 16 # field_4.offset + field_4.size Now suppose that the user is not interested in the field n. 2 and wants to remove it form the descriptor. This creates a *gap* in the binary data which makes not possible to exploit the automatic offset computation mechanism: .. testcode:: @bpack.descriptor class BinaryRecord: field_1: int = bpack.field(size=4) # offset = 0 first field # field_2: int = bpack.field(size=4) field_3: int = bpack.field(size=4) # offset = 4 != 8 NOT CORRECT field_4: int = bpack.field(size=4) # offset = 8 != 12 NOT CORRECT field_5: int = bpack.field(size=4) # offset = 12 != 16 NOT CORRECT The automatic computation of the offset fails, in this case, because of the missing information about ``field_2``. Indeed, since ``field_2`` has not been specified, for the computation of the offset of ``field_3`` *bpack* assumes that the previous field is ``field_1`` and performs the computation computes accordingly:: field_3.offest = fielf_1.offset + field_i.size == 4 != 8 # INCORRECT The incorrect offset of ``field_3`` causes the incorrect computation of the offset all the fields that follow. One option to recover the correct behavior (without falling back to the *verbose* description shown at the beginning of the section) is to specify explicitly **only** the offset of the first field after the gap: .. testcode:: @bpack.descriptor class BinaryRecord: field_1: int = bpack.field(size=4) # offset = 0 first field # field_2: int = bpack.field(size=4) field_3: int = bpack.field(size=4, offset=8) field_4: int = bpack.field(size=4) # offset = 12 field_5: int = bpack.field(size=4) # offset = 16 In this way the correct offset can be computed automatically for all fields but the one(s) immediately following a *gap* in the data descriptor. Signed integer types ~~~~~~~~~~~~~~~~~~~~ Only for integer types, it is possible to specify if the integer value is *signed* or not. Although this distinction is not relevant in the Python code, it is necessary to have this information when data have to be stored in binary form. .. testcode:: @bpack.descriptor class BinaryRecord: field: int = bpack.field(size=4, offset=0, signed=True) If *signed* is not specified for a field having and integer type, then it is assumed to be ``False`` (*unsigned*). The *signed* parameter is ignored if the data type is not ``int``. Default values ~~~~~~~~~~~~~~ The :func:`bpack.descriptors.field` factory function also allows to specify default values using the ``default`` parameter: .. testcode:: @bpack.descriptor class BinaryRecord: field: int = bpack.field(size=4, default=0) This allows to instantiate the record without specifying the value of each field: .. doctest:: >>> BinaryRecord() BinaryRecord(field=0) In cases in which the :func:`bpack.descriptors.field` factory function is not used for field definition, the default value can be specified by direct assignment: .. testcode:: @bpack.descriptor class BinaryRecord: field_1: bool = False field_2: bpack.T['i4'] = 33 .. note:: No check is performed by *bpack* to ensure that the default value specified for a field is consistent with the corresponding data type. Enumeration fields ------------------ The *bpack* package supports direct mapping of integer types, strings of ``bytes`` and Python ``str`` (unicode) into enumerated values of Python :class:`Enum` types (including also :class:`IntEnum` and :class:`IntFlag`). Example: .. testcode:: import enum class EColor(enum.IntEnum): RED = 1 GREEN = 2 BLUE = 3 BLACK = 10 WHITE = 11 @bpack.descriptor(baseunits='bits') class BinaryRecord: foreground: EColor = bpack.field(size=4, default=EColor.BLACK) background: EColor = bpack.field(size=4, default=EColor.WHITE) record = BinaryRecord() print(record) .. testoutput:: BinaryRecord(foreground=, background=) The ``EColor`` enum values are lower that 16 so they can be represented with only 4 bits. In particular the binary representation of ``BLACK`` and ``WHITE`` is: .. doctest:: >>> format(EColor.BLACK, '04b') '1010' >>> format(EColor.WHITE, '04b') '1011' and the binary string representing it is: .. testcode:: data = bytes([0b10101011]) print(data) .. testoutput:: b'\xab' The data string can be decoded using the :mod:`bpack.bs` backend that is suitable to handle based binary data structures with ``bits`` as *baseunits*: .. testcode:: import bpack.bs decoder = bpack.bs.Decoder(BinaryRecord) record = decoder.decode(data) print(record) .. testoutput:: BinaryRecord(foreground=, background=) The result is directly mapped into Python enum values: ``EColor:BLACK`` and ``EColor:WHITE``. .. note:: If the :class:`Enum` sub-classes are accepted as field type only if all the enumeration values have the same type (``int``, ``bytes`` or ``str``). Sequence fields --------------- *bpack* provides a basic support to homogeneous *sequence* fields i.e. fields containing a sequence of values having the same data type. The sequence is specified using the standard Python type annotation classes :class:`typing.Sequence` or :class:`typing.List`. The data type of a sequence item can be any of the basic data types described in `Type`_. .. testcode:: from typing import Sequence, List @bpack.descriptor class BinaryRecord: sequence: Sequence[int] = bpack.field(size=1, repeat=2) list: List[float] = bpack.field(size=4, repeat=3) Please note that the *size* parameter of the :func:`bpack.descriptors.field` factory function describes the size of the sequence *item*, while the *repeat* parameter described the number of elements in the *sequence*. The :mod:`bpack.bs` and :mod:`bpack.st` backend map ``Sequence[T]`` onto Python :class:`tuple` instances and ``List[T]`` onto :class:`list` instances. The :mod:`bpack.np` instead maps all kind of sequences onto :class:`numpy.ndarray` instances. Record nesting -------------- Descriptors of binary structures (record types) can gave fields that are binary structure descriptors in their turn (sub-records). Example: .. testcode:: @bpack.descriptor class SubRecord: field_21: int = bpack.field(size=2, default=1) field_22: int = bpack.field(size=2, default=2) @bpack.descriptor class Record: field_1: int = bpack.field(size=4, default=0) field_2: SubRecord = bpack.field(default_factory=SubRecord) print(Record()) .. testoutput:: Record(field_1=0, field_2=SubRecord(field_21=1, field_22=2)) Decoding of the ``Record`` structure will automatically decode also data belonging to the sub-record and assign to ``filed_2`` a ``SubRecord`` instance. Special type annotations ------------------------ Using the :func:`bpack.descriptors.field` factory function to defile fields can be sometime very verbose and boring. The *bpack* package provides an typing annotation helper, :class:`bpack.typing.T`, that allow to specify basic types annotated with additional information like the *size* or the *signed* attribute for integers. This helps to reduce the amount of typesetting required to specify a binary structure. The :class:`bpack.typing.T` type annotation class take in input a string argument and converts it into an annotated basic type. .. doctest:: >>> T['u4'] # doctest: +NORMALIZE_WHITESPACE typing.Annotated[int, TypeParams(byteorder=None, type='int', size=4, signed=False)] The resulting type annotation is a :class:`typing.Annotated` basic type with attached a :class:`bpack.typing.TypeParams` instance. For example the following descriptor: .. testcode:: @bpack.descriptor class BinaryRecord: field_1: int = bpack.field(size=4, signed=True, default=0) field_2: int = bpack.field(size=4, signed=False, default=1) Can be specified in a more synthetic form as follows: .. testcode:: @bpack.descriptor class BinaryRecord: field_1: T['i4'] = 0 field_2: T['u4'] = 1 String descriptors, or *typestr*, are compatible with numpy (a sub-set of one used in the numpy `array interface`_). The *typestr* string format consists of 3 parts: * an (optional) character describing the bit/byte order of the data - ``<``: little-endian, - ``>``: big-endian, - ``|``: not-relevant * a character code giving the basic type of the array, and * an integer providing the number of bytes the type uses The basic type character codes are: * ``i``: sighed integer * ``u``: unsigned integer * ``f``: float * ``c``: complex * ``S``: bytes (string) .. note:: Although the *typestr* format allows to specify the bit/byte *order* of the datatype it is usually not necessary to do it because descriptor object already have this information. .. seealso:: :func:`bpack.typing.str_to_type_params`, :class:`bpack.typing.TypeParams`, https://numpy.org/doc/stable/reference/arrays.dtypes.html and https://numpy.org/doc/stable/reference/arrays.interface.html .. _`array interface`: https://numpy.org/doc/stable/reference/arrays.interface.html Data codecs ----------- Backends ~~~~~~~~ Backends provide encoding/decoding capabilities for binary data *descriptors* exploiting external packages to do the low level job. Currently *bpack* provides the: * :mod:`bpack.st` backend, based on the :mod:`struct` package, and * :mod:`bpack.bs` backend, based on the bitstruct_ package to decode binary data described at bit level, i.e. with fields that can have size expressed in terms of number of bits (also smaller that 8). * :mod:`bpack.np` backend, based on numpy_ (limited encoding capabilities) Additionally a :mod:`bpack.ba` backend, feature incomplete, is also provided mainly for benchmarking purposes. The :mod:`bpack.ba` backend is based on the bitarray_ package. .. _bitstruct: https://github.com/eerimoq/bitstruct .. _bitarray: https://github.com/ilanschnell/bitarray .. _numpy: https://numpy.org Codec objects ~~~~~~~~~~~~~ Each backend provides a ``Codec`` class that can be used to instantiate a *codec* objects. Please refer to the `Codecs`_ section for a description of basic concepts of how decoders work. Decoders are instantiated passing to the ``Codec`` class a binary data record *descriptor*. Each *codec* has * a ``descriptor`` property by which it is possible to access the *descriptor* associated to the ``Decoder`` instance * a ``baseunits`` property that indicates the kind of *descriptors* supported by the ``Decoder`` class * a ``decode(data: bytes)`` method that takes in input a string of :class:`bytes` and returns an instance of the record type specified at the instantiation of the *codec* object * a ``encode(record)`` method that takes in input an instance of the record type specified at the instantiation of the *codec* object (a Python object) and returns a string of :class:`bytes` Details on the ``Codec`` API can be found in: * :class:`bpack.bs.Codec`, * :class:`bpack.np.Codec`, * :class:`bpack.st.Codec` .. note:: the :mod:`bpack.ba` backend does not provides encoding capabilities so no :class:`bpack.ba.Codec` class exists. A :class:`bpack.ba.Decoder` class exists instead providing only decoding capabilities. Codec decorator ~~~~~~~~~~~~~~~ Each backend provides also a ``@codec`` decorator the can be used to add to a ^descriptor^ direct decoding capabilities. In particular the ``frombytes(data: bytes)`` class method and the ``tobytes()`` method are added to the descriptor to be able to write code as the following: .. testcode:: import bpack import bpack.st @bpack.st.codec @bpack.descriptor class BinaryRecord: field_1: float = bpack.field(size=8) field_2: int = bpack.field(size=4, signed=True) binary_data = b'\x18-DT\xfb!\t@\x15\xcd[\x07' record = BinaryRecord.frombytes(binary_data) print(record) .. testoutput:: BinaryRecord(field_1=3.141592653589793, field_2=123456789) .. testcode:: encoded_data = record.tobytes() assert binary_data == encoded_data print(encoded_data) .. testoutput:: b'\x18-DT\xfb!\t@\x15\xcd[\x07' bpack-1.1.0/examples/000077500000000000000000000000001441646351700144225ustar00rootroot00000000000000bpack-1.1.0/examples/packet_decoding.py000066400000000000000000000035071441646351700201040ustar00rootroot00000000000000import dataclasses from typing import Tuple import numpy as np from bpack.np import unpackbits @dataclasses.dataclass(frozen=True) class PacketDescriptor: header_size: int bits_per_sample: int nsamples: int @property def packet_size(self): return self.header_size + self.bits_per_sample * self.nsamples def decode_packet( data: bytes, descr: PacketDescriptor ) -> Tuple[np.ndarray, np.ndarray]: if descr.header_size > 0: headers = unpackbits( data, descr.header_size, samples_per_block=1, blockstride=descr.packet_size, ) else: headers = np.empty(shape=(), dtype="u1") samples = unpackbits( data, descr.bits_per_sample, samples_per_block=descr.nsamples, bit_offset=descr.header_size, blockstride=descr.packet_size, ) return samples, headers def test(): from bpack.tests.test_packbits import ( _make_sample_data_block as mk_sample_data, ) descr = PacketDescriptor(header_size=13, bits_per_sample=5, nsamples=64) print("descr", descr) data, values = mk_sample_data( descr.header_size, descr.bits_per_sample, descr.nsamples, nblocks=2 ) values_per_block = 1 + descr.nsamples header_values = np.asarray(values[::values_per_block]) sample_values = np.asarray( [ values[start : start + descr.nsamples] for start in range(1, len(values), values_per_block) ] ).ravel() print("header_values:", header_values) print("sample_values:", sample_values) samples, headers = decode_packet(data, descr) print("headers:", headers) print("samples:", samples) assert (headers == header_values).all() assert (samples == sample_values).all() if __name__ == "__main__": test() bpack-1.1.0/examples/s1isp.py000066400000000000000000000212031441646351700160310ustar00rootroot00000000000000"""Sentinel-1 Instrument Source Packets (ISP) decoding example.""" import enum import logging import datetime import bpack import bpack.bs from bpack import T BITS = bpack.EBaseUnits.BITS BE = bpack.EByteOrder.BE SYNK_MARKER = 0x352EF853 FREF = 37.53472224 class SyncMarkerException(RuntimeError): pass class EEccNumber(enum.IntEnum): # TODO: check "NOT_SET" NOT_SET = 0 # CONTINGENCY: RESERVED FOR GROUND TESTING OR MODE UPGRADING S1 = 1 S2 = 2 S3 = 3 S4 = 4 S5_N = 5 S6 = 6 IW = 8 WM = 9 S5_S = 10 S1_NO_ICAL = 11 S2_NO_ICAL = 12 S3_NO_ICAL = 13 S4_NO_ICAL = 14 RFC = 15 TEST = 16 EN_S3 = 17 AN_S1 = 18 AN_S2 = 19 AN_S3 = 20 AN_S4 = 21 AN_S5_N = 22 AN_S5_S = 23 AN_S6 = 24 S5_N_NO_ICAL = 25 S5_S_NO_ICAL = 26 S6_NO_ICAL = 27 EN_S3_NO_ICAL = 31 EN = 32 AN_S1_NO_ICAL = 33 AN_S3_NO_ICAL = 34 AN_S6_NO_ICAL = 35 NC_S1 = 37 NC_S2 = 38 NC_S3 = 39 NC_S4 = 40 NC_S5_N = 41 NC_S5_S = 42 NC_S6 = 43 NC_EW = 44 NC_IW = 45 NC_WM = 46 class ETestMode(enum.IntEnum): DEFAULT = 0 CONTINGENCY_RXM_FULLY_OPERATIONAL = 4 # 100 CONTINGENCY_RXM_FULLY_BYPASSED = 5 # 101 OPER = 6 # 110 BYPASS = 7 # 111 class ERxChannelId(enum.IntEnum): V = 0 H = 1 class EBaqMode(enum.IntEnum): BYPASS = 0 BAQ3 = 3 BAQ4 = 4 BAQ5 = 5 FDBAQ_MODE_0 = 12 FDBAQ_MODE_1 = 13 FDBAQ_MODE_2 = 14 class ERangeDecimation(enum.IntEnum): X3_ON_4 = 0 X2_ON_3 = 1 X5_ON_9 = 3 X4_ON_9 = 4 X3_ON_8 = 5 X1_ON_3 = 6 X1_ON_6 = 7 X3_ON_7 = 8 X5_ON_16 = 9 X3_ON_26 = 10 X4_ON_11 = 11 @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class PacketPrimaryHeader: version: T["u3"] = 0 packet_type: T["u1"] = 0 secondary_header_flag: bool = True pid: T["u7"] = 0 pcat: T["u4"] = 0 sequence_flags: T["u2"] = 0 sequence_counter: T["u14"] = 0 packet_data_length: T["u16"] = 0 @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class DatationService: coarse_time: T["u32"] = 0 fine_time: T["u16"] = 0 @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class FixedAncillaryDataService: sync_marker: T["u32"] = SYNK_MARKER data_take_id: T["u32"] = 0 ecc_num: EEccNumber = bpack.field(size=8, default=EEccNumber.NOT_SET) # n. 1 bit n/a test_mode: ETestMode = bpack.field( size=3, offset=73, default=ETestMode.DEFAULT ) rx_channel_id: ERxChannelId = bpack.field(size=4, default=ERxChannelId.V) instrument_configuration_id: T["u32"] = 0 # NOTE: the data type is TBD @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class SubCommunicationAncillaryDataService: word_index: T["u8"] = 0 word_data: T["u16"] = 0 @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class CounterService: space_packet_count: T["u32"] = 0 pri_count: T["u32"] = 0 @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class RadarConfigurationSupportService: error_flag: bool = False baq_mode: EBaqMode = bpack.field(size=5, offset=3, default=EBaqMode.BYPASS) baq_block_len: T["u8"] = 0 # n. 8 bits padding range_decimation: ERangeDecimation = bpack.field( size=8, offset=24, default=0 ) rx_gain: T["u8"] = 0 tx_ramp_rate: T["u16"] = 0 tx_pulse_start_freq: T["u16"] = 0 tx_pulse_length: T["u24"] = 0 # n. 3 bits pad rank: T["u5"] = bpack.field(offset=99, default=0) pri: T["u24"] = 0 swst: T["u24"] = 0 swl: T["u24"] = 0 sas_sbb_message: T["S24"] = 0 # TODO: replace with SAS sub-record ses_sbb_message: T["S24"] = 0 # TODO: replace with SES sub-record @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE, size=24) class RadarSampleCountService: number_of_quads: T["u16"] = 0 # n. 8 bits pad @bpack.bs.decoder @bpack.descriptor(baseunits=BITS, byteorder=BE) class PacketSecondaryHeader: datation_service: DatationService fixed_ancillary_data_service: FixedAncillaryDataService subcom_ancillary_data_service: SubCommunicationAncillaryDataService counters_service: CounterService radar_configuration_support_service: RadarConfigurationSupportService radar_sample_count_service: RadarSampleCountService def sequential_stream_decoder(filename, maxcount=None): """Decode packet headers and store them into a pandas data-frame.""" import tqdm import pandas as pd log = logging.getLogger(__name__) log.info(f'start decoding: "{filename}"') t0 = datetime.datetime.now() primary_header_size = bpack.calcsize( PacketPrimaryHeader, bpack.EBaseUnits.BYTES ) secondary_header_size = bpack.calcsize( PacketSecondaryHeader, bpack.EBaseUnits.BYTES ) records = [] packet_counter = 0 pbar = tqdm.tqdm(unit=" packets", desc="decoded") with open(filename, "rb") as fd, pbar: while fd: # primary header data = fd.read(primary_header_size) if len(data) == 0 or (maxcount and len(records) > maxcount): break # type - PacketPrimaryHeader primary_header = PacketPrimaryHeader.frombytes(data) # print(primary_header) assert primary_header.version == 0 assert primary_header.packet_type == 0 assert primary_header.sequence_flags == 3 # assert primary_header.sequence_counter == packet_counter % 2**14 # secondary header assert primary_header.secondary_header_flag data_field_size = primary_header.packet_data_length + 1 data = fd.read(data_field_size) # type - PacketSecondaryHeader secondary_header = PacketSecondaryHeader.frombytes( data[:secondary_header_size] ) # print(secondary_header) sync = secondary_header.fixed_ancillary_data_service.sync_marker if sync != SYNK_MARKER: raise SyncMarkerException( f"packat count: {packet_counter + 1}" ) radar_cfg = secondary_header.radar_configuration_support_service assert radar_cfg.error_flag is False # baq_block_len = 8 * (radar_cfg.baq_block_len + 1) # assert baq_block_len == 256, ( # f'baq_block_len: {radar_cfg.baq_block_len}, ' # f'baq_mode: {radar_cfg.baq_mode}' # ) counters_service = secondary_header.counters_service assert packet_counter == counters_service.space_packet_count packet_counter += 1 # update the dataframe r = bpack.asdict(primary_header) r.update(bpack.asdict(secondary_header.datation_service)) r.update( bpack.asdict(secondary_header.fixed_ancillary_data_service) ) r.update( bpack.asdict(secondary_header.subcom_ancillary_data_service) ) r.update(bpack.asdict(secondary_header.counters_service)) r.update( bpack.asdict( secondary_header.radar_configuration_support_service ) ) r.update(bpack.asdict(secondary_header.radar_sample_count_service)) records.append(r) # user data # TBW pbar.update() elapsed = datetime.datetime.now() - t0 log.info(f"decoding complete (elapsed time: {elapsed})") return pd.DataFrame(records) if __name__ == "__main__": import os import sys logging.basicConfig( level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s" ) filename = ( "S1B_S3_RAW__0SDV_20210530T130904_20210530T130929_027134_033DC1_1AF2.SAFE/" "s1b-s3-raw-s-vv-20210530t130904-20210530t130929-027134-033dc1.dat" ) if not os.path.exists(filename): sys.exit( """ERROR: sample product not available. You can download it as follows (scihub.copernicus.eu authentication needed): $ sentinelsat --name 'S1B_S3_RAW__0SDV_20200615T162409_20200615T162435_022046_029D76*' --download """ ) df = sequential_stream_decoder(filename) # , maxcount=10) # print() # print(df.head()) """ $ env PYTHONPATH=.. python3 s1isp.py 2021-06-03 08:56:31,783 INFO: start decoding: "S1B_S3_RAW__0SDV_20210530T130904_20210530T130929_027134_033DC1_1AF2.SAFE/s1b-s3-raw-s-vv-20210530t130904-20210530t130929-027134-033dc1.dat" decoded: 48941 packets [00:19, 2506.83 packets/s] 2021-06-03 08:56:51,320 INFO: decoding complete (elapsed time: 0:00:19.537022) """ bpack-1.1.0/pyproject.toml000066400000000000000000000033761441646351700155310ustar00rootroot00000000000000[build-system] requires = ["setuptools>=61.0.0", "wheel"] build-backend = "setuptools.build_meta" [project] name = "bpack" authors = [ {name = "Antonio Valentino", email = "antonio.valentino@tiscali.it"}, ] description = "Binary data structures (un-)packing library" readme = {file = "README.rst", content-type = "text/x-rst"} requires-python = ">=3.7" keywords = [ "binary", "struct", "descriptor", "declarative", "bit", "unpack", "pack" ] license = {text = "Apache-2.0"} classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering", "Topic :: Utilities", ] dependencies = [ 'typing-extensions; python_version < "3.9"', ] dynamic = ["version"] [project.optional-dependencies] test = ["pytest", "bitarray>=1.5.1", "bitstruct", "numpy"] bs = ["bitstruct"] ba = ["bitarray>=1.5.1"] np = ["numpy"] docs = ["sphinx", "sphinx_rtd_theme"] [project.urls] homepage = "https://github.com/avalentino/bpack" documentation = "https://bpack.readthedocs.io" repository = "https://github.com/avalentino/bpack.git" changelog = "https://github.com/avalentino/bpack/blob/main/docs/release_notes.rst" [tool.setuptools] packages = ["bpack", "bpack.tests"] zip-safe = true # license-files = ["LICENSE"] [tool.setuptools.dynamic] version = {attr = "bpack.__version__"} [tool.pytest.ini_options] addopts = "--ignore=examples/" [tool.black] line-length = 79 # target-version = ['py311'] [tool.isort] profile = "black" length_sort = true no_inline_sort = true include_trailing_comma = true use_parentheses = true line_length = 79 bpack-1.1.0/requirements-dev.txt000066400000000000000000000002541441646351700166450ustar00rootroot00000000000000typing-extensions # ; python_version < "3.9" bitstruct numpy bitarray >= 1.5.1 pytest pytest-cov tox >= 4 flake8 sphinx sphinx_rtd_theme sphinxcontrib-spelling build twine bpack-1.1.0/tox.ini000066400000000000000000000030061441646351700141160ustar00rootroot00000000000000[tox] envlist = py37,py38,py39,py310,py311,pypy3,strict,coverage,docs,linkcheck,codestyle,spelling isolated_build = True [testenv] deps = pytest extras = test commands = python3 -m pytest --doctest-modules [testenv:pypy3] deps = pytest typing-extensions commands = python3 -m pytest [testenv:py{37,38}] commands = python3 -m pytest [testenv:strict] commands = python3 -m pytest -W error [testenv:coverage] deps = pytest pytest-cov commands = python3 -m pytest --cov bpack --cov-report=html --cov-report=term bpack [testenv:docs] allowlist_externals = mkdir changedir = docs extras = docs deps = bitstruct commands = mkdir -p _static python3 -m sphinx -W -b html . _build/html python3 -m sphinx -W -b doctest . _build/doctest [testenv:linkcheck] allowlist_externals = mkdir changedir = docs extras = docs commands = mkdir -p _static python3 -m sphinx -W -b linkcheck . _build/linkcheck [testenv:spelling] allowlist_externals = mkdir changedir = docs extras = docs deps = sphinxcontrib-spelling commands = mkdir -p _static python3 -m sphinx -W -b spelling . _build/spelling [testenv:codestyle] skip_install = true deps = flake8 changedir = {toxinidir} commands = python3 -m flake8 --version python3 -m flake8 --statistics --count bpack [flake8] doctests = true extend-ignore = D107 per-file-ignores = bpack/tests/test_*.py:D bpack/utils.py:D103 # [pytest] # addopts = --ignore=examples/ --doctest-modules # doctest_optionflags = --doctest-glob="*.rst"