pax_global_header00006660000000000000000000000064145345172730014525gustar00rootroot0000000000000052 comment=78196f68b6a2a437783d99162a262708f4c233a9 sdwilsh-siobrultech-protocols-78196f6/000077500000000000000000000000001453451727300200415ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/.github/000077500000000000000000000000001453451727300214015ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/.github/workflows/000077500000000000000000000000001453451727300234365ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/.github/workflows/build.yml000066400000000000000000000020541453451727300252610ustar00rootroot00000000000000--- name: Build on: # yamllint disable-line rule:truthy push: branches: - "main" - "v**" pull_request: branches: - "main" - "v**" jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: - "3.8" - "3.9" - "3.10" - "3.11" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install Dependencies run: | python -m pip install --upgrade pip pip install pep517 pip install -r requirements.txt - name: Build a binary wheel and a source tarball run: >- python -m pep517.build --source --binary --out-dir dist/ . - name: Install Test Dependencies run: | pip install -r requirements-dev.txt - name: Test with pytest run: | pytest sdwilsh-siobrultech-protocols-78196f6/.github/workflows/lint.yml000066400000000000000000000010461453451727300251300ustar00rootroot00000000000000--- name: Lint on: # yamllint disable-line rule:truthy push: branches: - "main" - "v**" pull_request: branches: - "main" - "v**" jobs: lint: runs-on: ubuntu-latest steps: - uses: earthly/actions-setup@v1.0.8 with: github-token: ${{ secrets.GITHUB_TOKEN }} # renovate: datasource=docker depName=earthly/earthly version: "v0.7.22" - uses: actions/checkout@v4 - name: Earthly Lint env: EARTHLY_CI: true run: earthly +lint sdwilsh-siobrultech-protocols-78196f6/.github/workflows/publish.yml000066400000000000000000000022021453451727300256230ustar00rootroot00000000000000--- # This workflows will upload a Python Package using Twine when a release is # created on GitHub. name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI on: # yamllint disable-line rule:truthy release: types: [created] jobs: deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip pip install pep517 --user - name: Build a binary wheel and a source tarball run: >- python -m pep517.build --source --binary --out-dir dist/ . - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@master with: password: ${{ secrets.TESTPYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@master with: password: ${{ secrets.PYPI_API_TOKEN }} sdwilsh-siobrultech-protocols-78196f6/.gitignore000066400000000000000000000023711453451727300220340ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # PyCharm /.idea/tasks.xml /.idea/workspace.xml # Ruff /.ruff_cache sdwilsh-siobrultech-protocols-78196f6/.vscode/000077500000000000000000000000001453451727300214025ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/.vscode/settings.json000066400000000000000000000007561453451727300241450ustar00rootroot00000000000000{ "python.testing.unittestArgs": [ "-v", "-s", "./tests", "-p", "test_*.py" ], "python.testing.pytestEnabled": true, "python.testing.nosetestsEnabled": false, "python.testing.unittestEnabled": false, "python.formatting.provider": "black", "editor.formatOnSave": true, "python.testing.pytestArgs": [ "tests" ], "python.analysis.typeCheckingMode": "basic", "python.analysis.diagnosticMode": "workspace" }sdwilsh-siobrultech-protocols-78196f6/Earthfile000066400000000000000000000031321453451727300216660ustar00rootroot00000000000000VERSION 0.6 FROM alpine # renovate: datasource=docker depName=python versioning=docker ARG PYTHON_VERSION=3.12 python-requirements: FROM python:$PYTHON_VERSION WORKDIR /usr/src/app COPY requirements.txt . COPY setup.cfg . COPY setup.py . RUN pip install --no-cache-dir -r requirements.txt python-dev-requirements: FROM +python-requirements WORKDIR /usr/src/app COPY requirements-dev.txt . RUN pip install --no-cache-dir -r requirements-dev.txt black-validate: FROM +python-dev-requirements WORKDIR /usr/src/app COPY --dir scripts . COPY --dir siobrultech_protocols . COPY --dir tests . RUN black . --check --diff --color pyright-image: FROM +python-dev-requirements RUN nodeenv /.cache/nodeenv ENV PYRIGHT_PYTHON_ENV_DIR=/.cache/nodeenv WORKDIR /usr/src/app pyright-validate: FROM +pyright-image WORKDIR /usr/src/app COPY pyproject.toml . COPY --dir scripts . COPY --dir siobrultech_protocols . COPY --dir tests . RUN pyright renovate-validate: # renovate: datasource=docker depName=renovate/renovate versioning=docker ARG RENOVATE_VERSION=37 FROM renovate/renovate:$RENOVATE_VERSION WORKDIR /usr/src/app COPY renovate.json . RUN renovate-config-validator ruff-validate: FROM +python-dev-requirements WORKDIR /usr/src/app COPY pyproject.toml . COPY --dir scripts . COPY --dir siobrultech_protocols . COPY --dir tests . RUN ruff check . --diff lint: BUILD +black-validate BUILD +pyright-validate BUILD +renovate-validate BUILD +ruff-validate sdwilsh-siobrultech-protocols-78196f6/LICENSE000066400000000000000000000021201453451727300210410ustar00rootroot00000000000000MIT License Copyright (c) 2018 Jonathan Keljo Copyright (c) 2021 Shawn Wilsher Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. sdwilsh-siobrultech-protocols-78196f6/README.md000066400000000000000000000144221453451727300213230ustar00rootroot00000000000000![Lint](https://github.com/sdwilsh/siobrultech-protocols/workflows/Lint/badge.svg) ![Build](https://github.com/sdwilsh/siobrultech-protocols/workflows/Build/badge.svg) # What is siobrultech-protocols? This library is a collection of protcols that decode various packet formats from [Brultech Research](https://www.brultech.com/). # What is Sans-I/O? Sans-I/O is a philosophy for developing protocol processing libraries in which the library does not do any I/O. Instead, a user of the library is responsible for transferring blocks of bytes between the socket or pipe and the protocol library, and for receiving application-level protocol items from and sending them to the library. This obviously makes a sans-I/O library a little more difficult to use, but comes with the advantage that the same library can be used with any I/O and concurrency mechanism: the same library should be usable in a single-request-at-a-time server, a process-per-request or thread-per-request blocking server, a server using select/poll and continuations, or a server using asyncio, Twisted, or any other asynchronous framework. See [SansIO](https://sans-io.readthedocs.io/) for more information. ## Installation ``` pip install siobrultech-protocols ``` ## Usage ### Receiving data packets ```python import functools from siobrultech_protocols.gem.protocol import PacketProtocol, PacketReceivedMessage # Queue to get received packets from. queue = asyncio.Queue() # Pass this Protocol to whatever receives data from the device. protocol_factory = functools.partial(PacketProtocol, queue=queue) # Dequeue and look for packet received messages. (Typically do this in a loop.) message = await queue.get() if isinstance(message, PacketReceivedMessage): packet = message.packet queue.task_done() ``` ### Receiving data packets AND sending API commands If you want to send API commands as well, use a `BidirectionalProtocol` instead of a `PacketProtocol`. Then given the `protocol` instance for a given connection, do the API call as follows: ```python from siobrultech_protocols.gem.api import get_serial_number serial = await get_serial_number(protocol) ``` `siobrultech_protocols` provides direct support for a small set of API calls. Some of these calls work for both GEM and ECM monitors, others are GEM-only. #### Methods to Get Information from a Device | Method | GEM | ECM | Description | | ------------------- | --- | --- | ---------------------------------------- | | `get_serial_number` | ✅︎ | ✅︎ | Obtains the serial number of the device. | #### Methods to Setup a Device | Method | GEM | ECM | Description | | ----------------------------- | --- | --- | --------------------------------------------------------------------------- | | `set_date_and_time` | ✅︎ | ❌ | Sets the GEM's clock to the specified `datetime`. | | `set_packet_format` | ✅︎ | ❌ | Sets the GEM's packet format to the specified `PacketFormatType`. | | `set_packet_send_interval` | ✅︎ | ✅︎ | Sets the frequency (seconds) that the monitor should send packets. | | `set_secondary_packet_format` | ✅︎ | ❌ | Sets the GEM's secondary packet format to the specified `PacketFormatType`. | | `synchronize_time` | ✅︎ | ❌ | Synchronizes the GEM's clock to the time of the local device. | ### Calling API endpoints that aren't supported by this library `siobrultech_protocols` has built-in support for just a tiny subset of the full API exposed by GEM and ECM. If you want to call an API endpoint for which this library doesn't provide a helper, you can make your own. For example, the following outline could be filled in to support the "get all settings" endpoint; you could define `GET_ALL_SETTINGS`: ```python from siobrultech_protocols.gem import api # Define a Python data type for the response. It can be whatever you want; a simple Dict, a custom dataclass, etc. AllSettings = Dict[str, Any] def _parse_all_gem_settings(response: str) -> AllSettings: # Here you would parse the GEM response into the python type you defined above def _parse_all_ecm_settings(response: bytes) -> AllSettings: # Here you would parse the ECM response into the python type you defined above GET_ALL_SETTINGS = api.ApiCall[None, AllSettings]( gem_formatter=lambda _: "^^^RQSALL", gem_parser=_parse_all_gem_settings, ecm_formatter=lambda _: [b"\xfc", b"SET", b"RCV"], ecm_parser=_parse_all_ecm_settings, ) ``` Given an `ApiCall` (whether one of those in `siobrultech_protocols.api` or defined yourself as above), you can make the request by working with the protocol directly as follows: ```python # Start the API request; do this once for each API call. Each protocol instance can only support one # API call at a time. delay = protocol.begin_api_request() sleep(delay) # Wait for the specified delay, using whatever mechanism is appropriate for your environment # Send the API request. We use asyncio's implementation of Future, but you can use whatever you like. result = asyncio.get_event_loop().create_future() protocol.invoke_api(GET_ALL_SETTINGS, None, result) settings = await asyncio.wait_for(result, timeout=5) # End the API request protocol.end_api_request() ``` Alternatively, we also provide a context wrapper that works with `asyncio` as well: ```python from siobrultech_protocols.gem import api async with api.call_api(GET_ALL_SETTINGS, protocol) as f: settings = await f(None) ``` Take a look at some usage examples from [libraries that use this](https://github.com/sdwilsh/siobrultech-protocols/network/dependents). ### Calling API endpoints when multiple devices share a connection All of the API helper methods take an optional `serial_number` parameter to target a specific device if there are multiple devices on the same connection. This has no effect for ECM devices. ## Development ### Setup ``` python3.11 -m venv .venv source .venv/bin/activate # Install Requirements pip install -r requirements.txt # Install Dev Requirements pip install -r requirements-dev.txt ``` ### Testing Tests are run with `pytest`. ### Linting Lint can be run with [Earthly](https://earthly.dev/) with `./earthly.sh +lint` sdwilsh-siobrultech-protocols-78196f6/earthly.sh000077500000000000000000000003661453451727300220550ustar00rootroot00000000000000#!/bin/bash docker run \ --privileged \ -v /var/run/docker.sock:/var/run/docker.sock \ --rm \ -t \ -v "$(pwd)":/workspace \ -v earthly-tmp:/tmp/earthly:rw \ earthly/earthly:v0.7.22 \ --allow-privileged \ "$@" sdwilsh-siobrultech-protocols-78196f6/pyproject.toml000066400000000000000000000017221453451727300227570ustar00rootroot00000000000000[build-system] requires = ["setuptools >= 40.6.0", "wheel"] build-backend = "setuptools.build_meta" [tool.pyright] include = [ "siobrultech_protocols", "scripts", "tests" ] exclude = [ "**/__pycache__" ] pythonVersion = "3.8" reportFunctionMemberAccess = "error" reportInvalidTypeVarUse = "error" reportMissingImports = "error" reportMissingParameterType = "error" reportMissingTypeArgument = "error" reportPrivateUsage = "error" reportUnknownMemberType = "error" reportUnknownParameterType = "error" reportUntypedBaseClass = "error" reportUntypedClassDecorator = "error" reportUntypedFunctionDecorator = "error" reportUntypedNamedTuple = "error" reportUnusedClass = "error" reportUnusedFunction = "error" reportUnusedImport = "error" reportUnusedVariable = "error" typeCheckingMode = "basic" [tool.ruff] select = [ # Pycodestyle "E", # Pyflakes "F", # isort "I001", ] ignore = [ # Rely on Black to handle line length "E501", ] sdwilsh-siobrultech-protocols-78196f6/renovate.json000066400000000000000000000024351453451727300225630ustar00rootroot00000000000000{ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:base" ], "packageRules": [ { "matchUpdateTypes": [ "minor", "patch" ], "matchCurrentVersion": "!/^0/", "automerge": true } ], "pip_requirements": { "fileMatch": [ "(^|/)requirements(-.*)?.txt$" ] }, "regexManagers": [ { "customType": "regex", "fileMatch": [ "^\\.github/workflows/.*\\.yml$" ], "matchStrings": [ "#\\s*renovate:\\s*datasource=(?.*?)\\s+depName=(?.*?)\\s+(?:[a-z\\-_]+)?version:\\s+\"(?.*?)\"" ] }, { "fileMatch": [ "^Earthfile$" ], "matchStrings": [ "#\\s*renovate:\\s*datasource=(?.*?)\\s+depName=(?.*?)(?:\\s+versioning=(?.*?))?\\n\\s*ARG\\s+.+_VERSION=(?.*?)\\s" ], "versioningTemplate": "{{#if versioning}}{{versioning}}{{else}}semver{{/if}}" }, { "fileMatch": [ "^earthly\\.sh$" ], "datasourceTemplate": "docker", "depNameTemplate": "earthly/earthly", "matchStrings": [ "earthly\\/earthly:(?.*?)\\s" ], "versioningTemplate": "semver-coerced" } ] }sdwilsh-siobrultech-protocols-78196f6/requirements-dev.txt000066400000000000000000000001371453451727300241020ustar00rootroot00000000000000black==23.11.0 pyright==1.1.339 pytest==7.4.3 pytest-asyncio==0.21.1 pyyaml==6.0.1 ruff==0.1.6 sdwilsh-siobrultech-protocols-78196f6/requirements.txt000066400000000000000000000000051453451727300233200ustar00rootroot00000000000000-e . sdwilsh-siobrultech-protocols-78196f6/scripts/000077500000000000000000000000001453451727300215305ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/scripts/version_bump.py000066400000000000000000000127361453451727300246230ustar00rootroot00000000000000import argparse import configparser import os import subprocess from packaging.version import Version CONFIG_FILE = os.path.join( subprocess.run( ["git", "rev-parse", "--show-toplevel"], check=True, capture_output=True, text=True, ).stdout.strip(), "setup.cfg", ) def init_argparse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Cuts a new release branch.", ) parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument( "--ignore-untracked-files", action="store_true", help="Ignores any untracked files from `git status` and cuts a release anyway.", ) return parser def has_clean_git_status(ignore_untracked: bool) -> bool: r = subprocess.run( ["git", "status", "--porcelain"], check=True, capture_output=True, text=True ) for line in r.stdout.strip().splitlines(): if ignore_untracked and line.startswith("??"): continue return False return True def checkout_main_branch() -> None: subprocess.run( ["git", "fetch"], check=True, capture_output=True, ) subprocess.run( ["git", "checkout", "-b", "version-bump", "origin/main"], check=True, capture_output=True, ) def get_current_revision(verbose: bool) -> str: r = subprocess.run( ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True ) rev = r.stdout.strip() if verbose: print(f"`git rev-parse HEAD`: {rev}") return rev def get_current_version() -> Version: config = configparser.ConfigParser() config.read(CONFIG_FILE) return Version(config["metadata"]["version"]) def get_release_version(current_version: Version) -> Version: return Version(current_version.base_version) def get_next_version(current_version: Version) -> Version: return Version( ".".join([str(current_version.major), str(current_version.minor + 1), "0a"]) ) def write_version_to_file(version: Version, verbose: bool) -> None: config = configparser.ConfigParser() config.read(CONFIG_FILE) config["metadata"]["version"] = str(version) with open(CONFIG_FILE, "w") as config_file: config.write(config_file) if verbose: print(f"Updated {CONFIG_FILE} to version {version}") def cut_release_branch(release_version: Version, verbose: bool) -> None: pre_release_rev = get_current_revision(verbose) print(f"Cutting release {release_version} from {pre_release_rev}") write_version_to_file(release_version, verbose) subprocess.run( ["git", "add", CONFIG_FILE], check=True, capture_output=True, ) subprocess.run( [ "git", "commit", "-m", f"""Cut release for {release_version} Based on {pre_release_rev} """, ], check=True, capture_output=True, ) subprocess.run( [ "git", "push", "origin", f"HEAD:refs/heads/v{release_version.major}.{release_version.minor}", ], check=True, capture_output=True, ) print( f"Release branch for v{release_version.major}.{release_version.minor} has been created." ) print( f"Please go to https://github.com/sdwilsh/siobrultech-protocols/releases/new to create a new release for v{release_version.major}.{release_version.minor}." ) subprocess.run( ["git", "reset", "--hard", pre_release_rev], check=True, capture_output=True, ) def update_main_branch( next_version: Version, release_version: Version, verbose: bool ) -> None: print( f"Updating main to use version {next_version} now that {release_version} is branched." ) subprocess.run( ["git", "branch", "-m", "version-bump", f"v{next_version}-version-bump"], check=True, capture_output=True, ) write_version_to_file(next_version, verbose) subprocess.run(["git", "add", CONFIG_FILE]) subprocess.run( [ "git", "commit", "-m", f"""Version bump to {next_version} v{release_version.major}.{release_version.minor} branch has been cut.""", ], check=True, capture_output=True, ) subprocess.run( [ "git", "push", "origin", f"HEAD:refs/heads/v{next_version}-version-bump", ], check=True, capture_output=True, ) print( f"Please go to https://github.com/sdwilsh/siobrultech-protocols/pull/new/v{next_version}-version-bump to open a pull request." ) if __name__ == "__main__": try: parser = init_argparse() args = parser.parse_args() if not has_clean_git_status(args.ignore_untracked_files): print("`git status` shows untracked files!") exit(1) checkout_main_branch() current_version = get_current_version() release_version = get_release_version(current_version) next_version = get_next_version(current_version) cut_release_branch(release_version, args.verbose) assert has_clean_git_status(args.ignore_untracked_files) update_main_branch(next_version, release_version, args.verbose) assert has_clean_git_status(args.ignore_untracked_files) except subprocess.CalledProcessError as e: print(f"Command failed: {e.cmd}\nstderr: {e.stderr.decode()}") sdwilsh-siobrultech-protocols-78196f6/setup.cfg000066400000000000000000000017211453451727300216630ustar00rootroot00000000000000[metadata] name = siobrultech-protocols version = 0.14.0 author = Shawn Wilsher author_email = me@shawnwilsher.com description = A Sans-I/O Python client library for Brultech Devices long_description = file: README.md long_description_content_type = text/markdown license_file = LICENSE classifiers = Development Status :: 4 - Beta License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 project_urls = Bug Reports = https://github.com/sdwilsh/siobrultech-protocols/issues Release Notes = https://github.com/sdwilsh/siobrultech-protocols/releases/ Source = https://github.com/sdwilsh/siobrultech-protocols [options] python_requires = >3.8, <4 packages = find: [options.package_data] siobrultech_protocols = py.typed [flake8] max-line-length = 88 max-complexity = 10 select = E9,F63,F7,F82 sdwilsh-siobrultech-protocols-78196f6/setup.py000066400000000000000000000001641453451727300215540ustar00rootroot00000000000000"""siobrultech-protocols stub setup script.""" import setuptools if __name__ == "__main__": setuptools.setup() sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/000077500000000000000000000000001453451727300244705ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/__init__.py000066400000000000000000000000001453451727300265670ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/000077500000000000000000000000001453451727300252405ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/__init__.py000066400000000000000000000000001453451727300273370ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/api.py000066400000000000000000000165711453451727300263750ustar00rootroot00000000000000from __future__ import annotations import asyncio import struct from contextlib import asynccontextmanager from datetime import datetime, timedelta from typing import Any, AsyncIterator, Callable, Coroutine, Generic, Optional from siobrultech_protocols.gem.packets import PacketFormatType from .const import ( CMD_GET_SERIAL_NUMBER, CMD_SET_DATE_AND_TIME, CMD_SET_PACKET_FORMAT, CMD_SET_PACKET_SEND_INTERVAL, CMD_SET_SECONDARY_PACKET_FORMAT, ) from .protocol import ApiCall, BidirectionalProtocol, R, T TIMEOUT = timedelta(seconds=15) @asynccontextmanager async def call_api( api: ApiCall[T, R], protocol: BidirectionalProtocol, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> AsyncIterator[Callable[[T], Coroutine[Any, None, R]]]: if timeout is None: timeout = TIMEOUT async def send(arg: T) -> R: future = asyncio.get_event_loop().create_future() protocol.invoke_api(api, arg, future, serial_number) return await asyncio.wait_for(future, timeout=timeout.total_seconds()) delay = protocol.begin_api_request() try: await asyncio.sleep(delay.seconds) yield send finally: protocol.end_api_request() class NewlineTerminatedStringResponseParser(Generic[R]): """ ApiCall requires response parsers to return None if there is not enough data to parse yet. This is a helper class for writing parsers for API responses that are terminated with \r\n (which is most of the GEM API calls). This parser will return None if the string it is given does not contain \r\n. If the string does contain \r\n, this parser will pass everything up to and including that \r\n to the parser that it wraps. """ def __init__(self, parser: Callable[[str], R]) -> None: """ Initialize a NewlineTerminatedStringResponseParser. parser - a callable that parses a string into an R. The string passed to the callable is guaranteed to end in \r\n. The callable should raise if the string cannot be parsed into an R. """ self._parser = parser def __call__(self, arg: str) -> R | None: if not arg.endswith("\r\n"): return None return self._parser(arg) def parse_ecm_serial_number_from_settings(binary: bytes) -> int | None: """ Unlike GEM, ECM-1240 doesn't have a specific get-serial-number API. Instead, it returns the serial number as part of the device settings response. This parser understands enough of that format to extract the serial number from the device settings. """ ECM_SETTINGS_STRUCT_LENGTH = 32 ECM_SETTINGS_CHECKSUM_SIZE = 1 ECM_SETTINGS_RESPONSE_LENGTH = ( ECM_SETTINGS_STRUCT_LENGTH + ECM_SETTINGS_CHECKSUM_SIZE ) BITS_PER_BYTE = 8 ECM_SETTINGS_CHECKSUM_MODULUS = 1 << (ECM_SETTINGS_CHECKSUM_SIZE * BITS_PER_BYTE) if len(binary) < ECM_SETTINGS_RESPONSE_LENGTH: return None # Unpacking just what we need from the settings struct # device_id - the device ID code # serial_number - the serial number # zero - a byte whose value is supposed to be 0 (just for correctness checking) # [device_id, serial_number, zero, checksum] = struct.unpack_from( ">10xBH18xBB", binary, 0 ) actual_sum = ( sum(binary[:ECM_SETTINGS_STRUCT_LENGTH]) % ECM_SETTINGS_CHECKSUM_MODULUS ) if zero != 0 or actual_sum != checksum: raise ValueError() # Following the GEM convention of just slamming device ID together with serial number # to get what the user considers the serial number. return int(f"{device_id}{serial_number:05}") GET_SERIAL_NUMBER = ApiCall[None, int]( gem_formatter=lambda _: CMD_GET_SERIAL_NUMBER, gem_parser=NewlineTerminatedStringResponseParser(lambda response: int(response)), ecm_formatter=lambda _: [b"\xfc", b"SET", b"RCV"], ecm_parser=parse_ecm_serial_number_from_settings, ) async def get_serial_number( protocol: BidirectionalProtocol, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> int: async with call_api(GET_SERIAL_NUMBER, protocol, serial_number, timeout) as f: return await f(None) SET_DATE_AND_TIME = ApiCall[datetime, bool]( gem_formatter=lambda dt: f"{CMD_SET_DATE_AND_TIME}{dt.strftime('%y,%m,%d,%H,%M,%S')}\r", gem_parser=NewlineTerminatedStringResponseParser( lambda response: response == "DTM\r\n" ), ecm_formatter=None, ecm_parser=None, ) SET_PACKET_FORMAT = ApiCall[int, bool]( gem_formatter=lambda pf: f"{CMD_SET_PACKET_FORMAT}{pf:02}", gem_parser=NewlineTerminatedStringResponseParser( lambda response: response == "PKT\r\n" ), ecm_formatter=None, ecm_parser=None, ) SET_PACKET_SEND_INTERVAL = ApiCall[int, bool]( gem_formatter=lambda si: f"{CMD_SET_PACKET_SEND_INTERVAL}{si:03}", gem_parser=NewlineTerminatedStringResponseParser( lambda response: response == "IVL\r\n" ), ecm_formatter=lambda si: [b"\xfc", b"SET", b"IV2", bytes([si])], ecm_parser=None, ) SET_SECONDARY_PACKET_FORMAT = ApiCall[int, bool]( gem_formatter=lambda pf: f"{CMD_SET_SECONDARY_PACKET_FORMAT}{pf:02}", gem_parser=NewlineTerminatedStringResponseParser( lambda response: response == "PKF\r\n" ), ecm_formatter=None, ecm_parser=None, ) async def set_date_and_time( protocol: BidirectionalProtocol, time: datetime, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> bool: async with call_api(SET_DATE_AND_TIME, protocol, serial_number, timeout) as f: return await f(time) async def set_packet_format( protocol: BidirectionalProtocol, format: PacketFormatType, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> bool: async with call_api(SET_PACKET_FORMAT, protocol, serial_number, timeout) as f: return await f(format) async def set_packet_send_interval( protocol: BidirectionalProtocol, send_interval_seconds: int, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> bool: if send_interval_seconds < 0 or send_interval_seconds > 256: raise ValueError("send_interval must be a postive number no greater than 256") async with call_api( SET_PACKET_SEND_INTERVAL, protocol, serial_number, timeout ) as f: r = await f(send_interval_seconds) # The GEM will give us a True or False response here, but the ECM has no response # and returns None. In order to keep API compatibility between the two, we assume # the the ECM succeeded. return r is not False async def set_secondary_packet_format( protocol: BidirectionalProtocol, format: PacketFormatType, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> bool: async with call_api( SET_SECONDARY_PACKET_FORMAT, protocol, serial_number, timeout ) as f: return await f(format) async def synchronize_time( protocol: BidirectionalProtocol, serial_number: Optional[int] = None, timeout: Optional[timedelta] = None, ) -> bool: """ Synchronizes the clock on the device to the time on the local device, accounting for the time waited for packets to clear. """ time = datetime.now() + protocol.packet_delay_clear_time return await set_date_and_time(protocol, time, serial_number, timeout) sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/const.py000066400000000000000000000012151453451727300267370ustar00rootroot00000000000000from datetime import timedelta ESCAPE_SEQUENCE = "^^^" _SYSTEM_PREFIX = ESCAPE_SEQUENCE + "SYS" _REQUEST_PREFIX = ESCAPE_SEQUENCE + "RQS" TARGET_SERIAL_NUMBER_PREFIX = ESCAPE_SEQUENCE + "NMB" CMD_DELAY_NEXT_PACKET = _SYSTEM_PREFIX + "PDL" CMD_GET_SERIAL_NUMBER = _REQUEST_PREFIX + "SRN" CMD_SET_DATE_AND_TIME = _SYSTEM_PREFIX + "DTM" CMD_SET_PACKET_FORMAT = _SYSTEM_PREFIX + "PKT" CMD_SET_PACKET_SEND_INTERVAL = _SYSTEM_PREFIX + "IVL" CMD_SET_SECONDARY_PACKET_FORMAT = _SYSTEM_PREFIX + "PKF" PACKET_DELAY_CLEAR_TIME_DEFAULT = timedelta( seconds=3 ) # Time to wait after a packet delay request so that GEM can finish sending any pending packets sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/fields.py000066400000000000000000000152561453451727300270710ustar00rootroot00000000000000""" Field types used in https://www.brultech.com/software/files/downloadSoft/GEM-PKT_Packet_Format_2_1.pdf """ from abc import ABC, abstractmethod from datetime import datetime from enum import Enum, unique from typing import Any, List @unique class ByteOrder(Enum): # Big-endian (the name comes from the GEM packet format spec) HiToLo = 1 # Little endian (the name comes from the GEM packet format spec) LoToHi = 2 @unique class Sign(Enum): Signed = 1 Unsigned = 2 class Field(ABC): def __init__(self, size: int): self._size = size @property def size(self) -> int: return self._size @abstractmethod def read(self, buffer: bytes, offset: int) -> Any: """Convert the buffer at the given offset to the proper value.""" @abstractmethod def write(self, value: Any, buffer: bytearray) -> None: """Write the given value to the given buffer.""" def write_padding(self, buffer: bytearray) -> None: """Writes a dummy value whose size matches this field's size.""" buffer.extend([0xFE] * self.size) class ByteField(Field): def __init__(self): super().__init__(size=1) def read(self, buffer: bytes, offset: int) -> bytes: return buffer[offset : offset + self.size] def write(self, value: int, buffer: bytearray) -> None: buffer.append(value) class BytesField(Field): def read(self, buffer: bytes, offset: int) -> bytes: return buffer[offset : offset + self.size] def write(self, value: bytes, buffer: bytearray) -> None: buffer.extend(value) class NumericField(Field): def __init__(self, size: int, order: ByteOrder, signed: Sign): super().__init__(size=size) self.order: ByteOrder = order self.signed: Sign = signed def read(self, buffer: bytes, offset: int) -> int: return _parse(buffer[offset : offset + self.size], self.order, self.signed) def write(self, value: int, buffer: bytearray) -> None: temp = bytearray(self.size) _format(value, self.order, self.signed, temp) buffer.extend(temp) @property def max(self) -> int: """The maximum value that can be encoded in this field.""" bits = 8 * self.size if self.signed == Sign.Unsigned: return (1 << bits) - 1 else: return (1 << (bits - 1)) - 1 class FloatingPointField(Field): def __init__(self, size: int, order: ByteOrder, signed: Sign, divisor: float): self.raw_field: NumericField = NumericField(size, order, signed) super().__init__(size=self.raw_field.size) self.divisor: float = divisor def read(self, buffer: bytes, offset: int) -> float: return self.raw_field.read(buffer, offset) / self.divisor def write(self, value: float, buffer: bytearray) -> None: int_value = round(value * self.divisor) self.raw_field.write(int_value, buffer) class DateTimeField(Field): def __init__(self): super().__init__(size=6) def read(self, buffer: bytes, offset: int) -> datetime: year, month, day, hour, minute, second = buffer[offset : offset + self.size] return datetime(2000 + year, month, day, hour, minute, second) def write(self, value: datetime, buffer: bytearray) -> None: buffer.extend( [ value.year - 2000, value.month, value.day, value.hour, value.minute, value.second, ] ) class ArrayField(Field): def __init__(self, num_elems: int, elem_field: Field): super().__init__(size=num_elems * elem_field.size) self.elem_field: Field = elem_field self.num_elems: int = num_elems def read(self, buffer: bytes, offset: int) -> List[Any]: return [ self.elem_field.read(buffer, offset + i * self.elem_field.size) for i in range(self.num_elems) ] def write(self, value: List[Any], buffer: bytearray) -> None: for item in value[0 : self.num_elems]: self.elem_field.write(item, buffer) class FloatingPointArrayField(ArrayField): elem_field: FloatingPointField def __init__( self, num_elems: int, size: int, order: ByteOrder, signed: Sign, divisor: float, ): super().__init__( num_elems=num_elems, elem_field=FloatingPointField( size=size, order=order, signed=signed, divisor=divisor ), ) def read(self, buffer: bytes, offset: int) -> List[float]: return super().read(buffer, offset) def write(self, value: List[float], buffer: bytearray) -> None: super().write(value, buffer) class NumericArrayField(ArrayField): elem_field: NumericField def __init__(self, num_elems: int, size: int, order: ByteOrder, signed: Sign): super().__init__( num_elems=num_elems, elem_field=NumericField(size=size, order=order, signed=signed), ) def read(self, buffer: bytes, offset: int) -> List[int]: return super().read(buffer, offset) def write(self, value: List[int], buffer: bytearray) -> None: super().write(value, buffer) @property def max(self) -> int: return self.elem_field.max def _parse( raw_octets: bytes, order: ByteOrder = ByteOrder.HiToLo, signed: Sign = Sign.Unsigned ) -> int: """Reads the given octets as a big-endian value. The function name comes from how such values are described in the packet format spec.""" octets = list(raw_octets) if len(octets) == 0: return 0 if order == ByteOrder.LoToHi: octets.reverse() # If this is a signed field (i.e., temperature), the highest-order # bit indicates sign. Detect this (and clear the bit so we can # compute the magnitude). # # This isn't documented in the protocol spec, but matches other # implementations. sign = 1 if signed == Sign.Signed and (octets[0] & 0x80): octets[0] &= ~0x80 sign = -1 result = 0 for octet in octets: result = (result << 8) + octet return sign * result def _format(value: int, order: ByteOrder, signed: Sign, buffer: bytearray) -> None: """Writes the given value to the buffer with the given byte order and signed-ness.""" sign = 1 if signed == Sign.Signed and value < 0: value = -value sign = -1 index = 0 if order == ByteOrder.LoToHi else -1 while value > 0: buffer[index] = value & 0xFF index += 1 if order == ByteOrder.LoToHi else -1 value = value >> 8 if sign == -1: buffer[0 if order == ByteOrder.HiToLo else -1] |= 0x80 sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/packets.py000066400000000000000000000400771453451727300272540ustar00rootroot00000000000000""" Packet formats defined in https://www.brultech.com/software/files/downloadSoft/GEM-PKT_Packet_Format_2_1.pdf """ from __future__ import annotations import codecs import json from collections import OrderedDict from copy import copy from datetime import datetime from enum import IntEnum, unique from typing import Any, Dict, List, Optional from .fields import ( ByteField, ByteOrder, BytesField, DateTimeField, Field, FloatingPointArrayField, FloatingPointField, NumericArrayField, NumericField, Sign, ) class MalformedPacketException(Exception): pass class Packet(object): def __init__( self, packet_format: PacketFormat, voltage: float, absolute_watt_seconds: List[int], device_id: int, serial_number: int, seconds: int, pulse_counts: Optional[List[int]] = None, temperatures: Optional[List[Optional[float]]] = None, polarized_watt_seconds: Optional[List[int]] = None, currents: Optional[List[float]] = None, time_stamp: Optional[datetime] = None, aux: Optional[List[int]] = None, dc_voltage: Optional[int] = None, **kwargs: Dict[str, Any], ): self.packet_format: PacketFormat = packet_format self.voltage: float = voltage self.absolute_watt_seconds: List[int] = absolute_watt_seconds self.polarized_watt_seconds: Optional[List[int]] = polarized_watt_seconds self.currents: Optional[List[float]] = currents self.device_id: int = device_id self.serial_number: int = serial_number self.seconds: int = seconds self.pulse_counts: List[int] = pulse_counts or [] self.temperatures: List[float | None] = temperatures or [] if time_stamp: self.time_stamp: datetime = time_stamp else: self.time_stamp: datetime = datetime.now() self.aux: List[int] = aux or [] self.dc_voltage = dc_voltage def __str__(self) -> str: return json.dumps( { "device_id": self.device_id, "serial_number": self.serial_number, "seconds": self.seconds, "voltage": self.voltage, "absolute_watt_seconds": self.absolute_watt_seconds, "polarized_watt_seconds": self.polarized_watt_seconds, "currents": self.currents, "pulse_counts": self.pulse_counts, "temperatures": self.temperatures, "time_stamp": self.time_stamp.isoformat(), } ) @property def num_channels(self) -> int: """The number of channels in the packet given the format. There may be fewer on the device.""" return self.packet_format.num_channels @property def type(self) -> str: """The packet format type's name.""" return self.packet_format.name def delta_seconds(self, prev: int) -> int: field = self.packet_format.fields["seconds"] assert isinstance(field, NumericField) return self._delta_value(field, self.seconds, prev) def delta_pulse_count(self, index: int, prev: int) -> int: field = self.packet_format.fields["pulse_counts"] assert isinstance(field, NumericArrayField) return self._delta_value(field.elem_field, self.pulse_counts[index], prev) def delta_aux_count(self, index: int, prev: int) -> int: field = self.packet_format.fields["aux"] assert isinstance(field, NumericArrayField) return self._delta_value(field.elem_field, self.aux[index], prev) def delta_absolute_watt_seconds(self, index: int, prev: int) -> int: field = self.packet_format.fields["absolute_watt_seconds"] assert isinstance(field, NumericArrayField) return self._delta_value( field.elem_field, self.absolute_watt_seconds[index], prev ) def delta_polarized_watt_seconds(self, index: int, prev: int) -> int: field = self.packet_format.fields["polarized_watt_seconds"] assert isinstance(field, NumericArrayField) if self.polarized_watt_seconds is not None: return self._delta_value( field.elem_field, self.polarized_watt_seconds[index], prev ) else: return 0 def _delta_value(self, field: NumericField, cur: int, prev: int) -> int: if prev > cur: diff = field.max + 1 - prev diff += cur else: diff = cur - prev return diff @staticmethod def _packets_sorted(packet_a: Packet, packet_b: Packet) -> tuple[Packet, Packet]: if packet_a.seconds < packet_b.seconds: oldest_packet = packet_a newest_packet = packet_b else: oldest_packet = packet_b newest_packet = packet_a return oldest_packet, newest_packet def get_average_power( self, index: int, other_packet: Packet, ) -> float: oldest_packet, newest_packet = self._packets_sorted(self, other_packet) elapsed_seconds = newest_packet.delta_seconds(oldest_packet.seconds) # The Brultech devices measure one or two things with their counters: # * Absolute Watt-seconds, which is incoming and outgoing Watt-seconds, combined. # * Polarized Watt-seconds (only when NET metering is enabled), which is just outgoing Watt-seconds. # # Therefore, in order to compute the average power (Watts) between packets flowing through the point that is # being measured, we need to compute two things: # * Produced Watt-seconds, which is just Polarized Watt-seconds. # * Consumed Watt-seconds, which is Absolute Watt-seconds minus Polarized Watt-seconds. # # Given those two values, the average power is just the (consumed - produced) / elapsed time. In this way, a # negative flow of power occurs if more power was produced than consumed. delta_absolute_watt_seconds = newest_packet.delta_absolute_watt_seconds( index, oldest_packet.absolute_watt_seconds[index] ) # It is only possible to produce if the given channel has NET metering enabled. delta_produced_watt_seconds = ( newest_packet.delta_polarized_watt_seconds( index, oldest_packet.polarized_watt_seconds[index] ) if oldest_packet.polarized_watt_seconds is not None and newest_packet.polarized_watt_seconds is not None else 0.0 ) delta_consumed_watt_seconds = ( delta_absolute_watt_seconds - delta_produced_watt_seconds ) return ( (delta_consumed_watt_seconds - delta_produced_watt_seconds) / elapsed_seconds if elapsed_seconds else 0 ) def get_average_pulse_rate(self, index: int, other_packet: Packet) -> float: oldest_packet, newest_packet = self._packets_sorted(self, other_packet) elapsed_seconds = newest_packet.delta_seconds(oldest_packet.seconds) return ( ( newest_packet.delta_pulse_count( index, oldest_packet.pulse_counts[index] ) / elapsed_seconds ) if elapsed_seconds else 0 ) def get_average_aux_rate_of_change(self, index: int, other_packet: Packet) -> float: oldest_packet, newest_packet = self._packets_sorted(self, other_packet) elapsed_seconds = newest_packet.delta_seconds(oldest_packet.seconds) return ( ( newest_packet.delta_aux_count(index, oldest_packet.aux[index]) / elapsed_seconds ) if elapsed_seconds else 0 ) @unique class PacketFormatType(IntEnum): ECM_1220 = 1 ECM_1240 = 3 BIN48_NET_TIME = 4 BIN48_NET = 5 BIN48_ABS = 7 BIN32_NET = 8 BIN32_ABS = 9 class PacketFormat(object): def __init__( self, name: str, type: PacketFormatType, code: int, num_channels: int, ): self.name: str = name self.type: PacketFormatType = type self.code = code self.num_channels: int = num_channels self.fields: OrderedDict[str, Field] = OrderedDict() @property def size(self) -> int: result = 0 for value in self.fields.values(): result += value.size return result def parse(self, data: bytes) -> Packet: if len(data) < self.size: raise MalformedPacketException( "Packet too short. Expected {0} bytes, found {1} bytes.".format( self.size, len(data) ) ) _checksum(data, self.size) offset = 0 args = { "packet_format": self, } for key, value in self.fields.items(): args[key] = value.read(data, offset) offset += value.size if args["code"] != self.code: raise MalformedPacketException( "bad code {0} im packet: {1}".format( args["code"], codecs.encode(data, "hex") ) ) if args["footer"] != 0xFFFE: raise MalformedPacketException( "bad footer {0} in packet: {1}".format( hex(args["footer"]), codecs.encode(data, "hex") # type: ignore ) ) return Packet(**args) # type: ignore def format(self, packet: Packet) -> bytes: result = bytearray() for key, field in self.fields.items(): if key == "footer": value = 0xFFFE elif key == "header": value = 0xFEFF elif key == "code": value = self.code else: value = getattr(packet, key) if hasattr(packet, key) else None if value is not None: field.write(value, result) else: field.write_padding(result) result[-1] = _compute_checksum(result, self.size) assert len(result) == self.size return bytes(result) class ECMPacketFormat(PacketFormat): def __init__( self, name: str, type: PacketFormatType, code: int, has_aux_channels: bool = False, ): super().__init__(name, type, code=code, num_channels=2) self.fields["header"] = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) self.fields["code"] = NumericField(1, ByteOrder.HiToLo, Sign.Unsigned) self.fields["voltage"] = FloatingPointField( 2, ByteOrder.HiToLo, Sign.Unsigned, 10.0 ) self.fields["absolute_watt_seconds"] = NumericArrayField( self.num_channels, 5, ByteOrder.LoToHi, Sign.Unsigned ) self.fields["polarized_watt_seconds"] = NumericArrayField( self.num_channels, 5, ByteOrder.LoToHi, Sign.Unsigned ) self.fields["reserved"] = BytesField(size=4) self.fields["serial_number"] = NumericField(2, ByteOrder.LoToHi, Sign.Unsigned) self.fields["flag"] = ByteField() self.fields["device_id"] = NumericField(1, ByteOrder.HiToLo, Sign.Unsigned) self.fields["currents"] = FloatingPointArrayField( self.num_channels, 2, ByteOrder.LoToHi, Sign.Unsigned, 100.0 ) self.fields["seconds"] = NumericField(3, ByteOrder.LoToHi, Sign.Unsigned) self.num_aux_channels = 0 if has_aux_channels: self.num_aux_channels = 5 self.fields["aux"] = NumericArrayField( self.num_aux_channels, 4, ByteOrder.LoToHi, Sign.Unsigned ) self.fields["dc_voltage"] = NumericField(2, ByteOrder.LoToHi, Sign.Unsigned) self.fields["footer"] = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) self.fields["checksum"] = ByteField() class GEMPacketFormat(PacketFormat): NUM_PULSE_COUNTERS: int = 4 NUM_TEMPERATURE_SENSORS: int = 8 def __init__( self, name: str, type: PacketFormatType, code: int, num_channels: int, has_net_metering: bool = False, has_time_stamp: bool = False, ): super().__init__(name, type, code=code, num_channels=num_channels) self.fields["header"] = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) self.fields["code"] = NumericField(1, ByteOrder.HiToLo, Sign.Unsigned) self.fields["voltage"] = FloatingPointField( 2, ByteOrder.HiToLo, Sign.Unsigned, 10.0 ) self.fields["absolute_watt_seconds"] = NumericArrayField( num_channels, 5, ByteOrder.LoToHi, Sign.Unsigned ) if has_net_metering: self.fields["polarized_watt_seconds"] = NumericArrayField( num_channels, 5, ByteOrder.LoToHi, Sign.Unsigned ) self.fields["serial_number"] = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) self.fields["reserved"] = ByteField() self.fields["device_id"] = NumericField(1, ByteOrder.HiToLo, Sign.Unsigned) self.fields["currents"] = FloatingPointArrayField( num_channels, 2, ByteOrder.LoToHi, Sign.Unsigned, 50.0 ) self.fields["seconds"] = NumericField(3, ByteOrder.LoToHi, Sign.Unsigned) self.fields["pulse_counts"] = NumericArrayField( GEMPacketFormat.NUM_PULSE_COUNTERS, 3, ByteOrder.LoToHi, Sign.Unsigned ) self.fields["temperatures"] = FloatingPointArrayField( GEMPacketFormat.NUM_TEMPERATURE_SENSORS, 2, ByteOrder.LoToHi, Sign.Signed, 2.0, ) if num_channels == 32: self.fields["spare_bytes"] = BytesField(2) if has_time_stamp: self.fields["time_stamp"] = DateTimeField() self.fields["footer"] = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) self.fields["checksum"] = ByteField() def parse(self, data: bytes) -> Packet: packet = super().parse(data) packet.temperatures = [ # Above 255 means it wasn't able to read the sensor (though we sometimes also get 0 for that) temperature if temperature is not None and temperature <= 255.0 else None for temperature in packet.temperatures ] return packet def format(self, packet: Packet) -> bytes: packet = copy(packet) packet.temperatures = [ temperature if temperature is not None else 256 for temperature in packet.temperatures ] return super().format(packet) def _compute_checksum(packet: bytes, size: int) -> int: checksum = 0 for i in packet[: size - 1]: checksum += i checksum = checksum % 256 return checksum def _checksum(packet: bytes, size: int) -> None: checksum = _compute_checksum(packet, size) if checksum != packet[size - 1]: raise MalformedPacketException( "bad checksum for packet: {0}".format(codecs.encode(packet[:size], "hex")) ) BIN48_NET_TIME = GEMPacketFormat( name="BIN48-NET-TIME", type=PacketFormatType.BIN48_NET_TIME, code=5, num_channels=48, has_net_metering=True, has_time_stamp=True, ) BIN48_NET = GEMPacketFormat( name="BIN48-NET", type=PacketFormatType.BIN48_NET, code=5, num_channels=48, has_net_metering=True, has_time_stamp=False, ) BIN48_ABS = GEMPacketFormat( name="BIN48-ABS", type=PacketFormatType.BIN48_ABS, code=6, num_channels=48, has_net_metering=False, has_time_stamp=False, ) BIN32_NET = GEMPacketFormat( name="BIN32-NET", type=PacketFormatType.BIN32_NET, code=7, num_channels=32, has_net_metering=True, has_time_stamp=False, ) BIN32_ABS = GEMPacketFormat( name="BIN32-ABS", type=PacketFormatType.BIN32_ABS, code=8, num_channels=32, has_net_metering=False, has_time_stamp=False, ) ECM_1240 = ECMPacketFormat( name="ECM-1240", type=PacketFormatType.ECM_1240, code=3, has_aux_channels=True ) ECM_1220 = ECMPacketFormat( name="ECM-1220", type=PacketFormatType.ECM_1220, code=1, has_aux_channels=False ) sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/gem/protocol.py000066400000000000000000000477411453451727300274700ustar00rootroot00000000000000from __future__ import annotations import asyncio import logging from collections import deque from dataclasses import dataclass from datetime import timedelta from enum import Enum, unique from typing import Any, Callable, Deque, Generic, List, Optional, Set, TypeVar, Union from .const import ( CMD_DELAY_NEXT_PACKET, ESCAPE_SEQUENCE, PACKET_DELAY_CLEAR_TIME_DEFAULT, TARGET_SERIAL_NUMBER_PREFIX, ) from .packets import ( BIN32_ABS, BIN32_NET, BIN48_ABS, BIN48_NET, BIN48_NET_TIME, ECM_1220, ECM_1240, MalformedPacketException, Packet, PacketFormatType, ) LOG = logging.getLogger(__name__) PACKET_HEADER = bytes.fromhex("feff") API_RESPONSE_WAIT_TIME = timedelta(seconds=3) # Time to wait for an API response @dataclass(frozen=True) class PacketProtocolMessage: """Base class for messages sent by a PacketProtocol.""" protocol: PacketProtocol @dataclass(frozen=True) class ConnectionMadeMessage(PacketProtocolMessage): """Message sent when a new connection has been made to a protocol. Sent once shortly after creation of the protocol instance.""" pass @dataclass(frozen=True) class PacketReceivedMessage(PacketProtocolMessage): """Message sent when a packet has been received by the protocol.""" packet: Packet @dataclass(frozen=True) class ConnectionLostMessage(PacketProtocolMessage): """Message sent when a protocol loses its connection. exc is the exception that caused the connection to drop, if any.""" exc: Optional[BaseException] class PacketProtocol(asyncio.Protocol): """Protocol implementation for processing a stream of data packets from a GreenEye Monitor.""" def __init__( self, queue: asyncio.Queue[PacketProtocolMessage], ): """ Create a new protocol instance. Whenever a data packet is received from the GEM, a `Packet` instance will be enqueued to `queue`. """ self._buffer = bytearray() self._queue = queue self._transport: Optional[asyncio.BaseTransport] = None self._packet_type: PacketFormatType | None = None def connection_made(self, transport: asyncio.BaseTransport) -> None: LOG.info("%d: Connection opened", id(self)) assert self._transport is None self._transport = transport self._queue.put_nowait(ConnectionMadeMessage(protocol=self)) def connection_lost(self, exc: Optional[BaseException]) -> None: if exc is not None: LOG.warning("%d: Connection lost due to exception", id(self), exc_info=exc) else: LOG.info("%d: Connection closed", id(self)) self._transport = None self._queue.put_nowait(ConnectionLostMessage(protocol=self, exc=exc)) def data_received(self, data: bytes) -> None: LOG.debug("%d: Received %d bytes", id(self), len(data)) self._buffer.extend(data) try: should_continue = True while should_continue: should_continue = self._process_buffer() self._ensure_transport() except Exception: LOG.exception("%d: Exception while attempting to parse a packet.", id(self)) def close(self) -> None: """Closes the underlying transport, if any.""" if self._transport: self._transport.close() self._transport = None def _process_buffer(self) -> bool: """ Attempts to process one chunk of data in the buffer. - If the buffer starts with a complete packet, delivers that packet to the queue and returns True - If the buffer starts with an incomplete packet, returns False - If the buffer starts with data that is not a packet, removes that data from the buffer, passes it to unknown_data_received(), and returns True Subclasses override unknown_data_received to process data in the buffer that is not packets (e.g. responses to API calls). Returns True if another call to _process_buffer might be able to process more of the buffer, False if the caller should wait for more data to be added to the buffer before calling again. """ def skip_malformed_packet(msg: str, *args: Any, **kwargs: Any): header_index = self._buffer.find(PACKET_HEADER, 1) end = header_index if header_index != -1 else len(self._buffer) LOG.debug( "%d Skipping malformed packet due to " + msg + ". Buffer contents: %s", id(self), *args, self._buffer[0:end], ) del self._buffer[0:end] header_index = self._buffer.find(PACKET_HEADER) if header_index != 0: end = header_index if header_index != -1 else len(self._buffer) self.unknown_data_received(self._buffer[0:end]) del self._buffer[0:end] return len(self._buffer) > 0 if len(self._buffer) < len(PACKET_HEADER) + 1: # Not enough length yet LOG.debug( "%d: Not enough data in buffer yet (%d bytes): %s", id(self), len(self._buffer), self._buffer, ) return False format_code = self._buffer[len(PACKET_HEADER)] if format_code == 8: packet_format = BIN32_ABS elif format_code == 7: packet_format = BIN32_NET elif format_code == 6: packet_format = BIN48_ABS elif format_code == 5: packet_format = BIN48_NET elif format_code == 3: packet_format = ECM_1240 elif format_code == 1: packet_format = ECM_1220 else: skip_malformed_packet("unknown format code 0x%x", format_code) return len(self._buffer) > 0 if len(self._buffer) < packet_format.size: # Not enough length yet LOG.debug( "%d: Not enough data in buffer yet (%d bytes)", id(self), len(self._buffer), ) return False try: packet = None try: packet = packet_format.parse(self._buffer) except MalformedPacketException: if packet_format != BIN48_NET: raise if packet is None: if len(self._buffer) < BIN48_NET_TIME.size: # Not enough length yet LOG.debug( "%d: Not enough data in buffer yet (%d bytes)", id(self), len(self._buffer), ) return False packet = BIN48_NET_TIME.parse(self._buffer) self._packet_type = packet.packet_format.type LOG.debug("%d: Parsed one %s packet.", id(self), packet.packet_format.name) del self._buffer[0 : packet.packet_format.size] self._queue.put_nowait(PacketReceivedMessage(protocol=self, packet=packet)) except MalformedPacketException as e: skip_malformed_packet(e.args[0]) return len(self._buffer) > 0 def unknown_data_received(self, data: bytes) -> None: LOG.debug( "%d: No header found. Discarding junk data: %s", id(self), data, ) def _ensure_transport(self) -> asyncio.BaseTransport: if not self._transport: raise EOFError return self._transport @unique class ProtocolState(Enum): RECEIVING_PACKETS = 1 # Receiving packets from the GEM SENT_PACKET_DELAY_REQUEST = 2 # Sent the packet delay request prior to an API request, waiting for any in-flight packets SENDING_API_REQUEST = 3 # Sending a multi-part request SENT_API_REQUEST = 4 # Sent an API request, waiting for a response RECEIVED_API_RESPONSE = 5 # Received an API response, waiting for end call class ProtocolStateException(Exception): def __init__( self, actual: ProtocolState, expected: Union[ProtocolState, Set[ProtocolState]], *args: object, ) -> None: self._actual = actual self._expected = expected super().__init__(*args) def __str__(self) -> str: if isinstance(self._expected, set): expected = [s.name for s in self._expected] if len(expected) > 1: expected_str = ", ".join(expected[:-1]) + f", or {expected[-1]}" else: expected_str = expected[0] else: expected_str = self._expected.name return f"Expected state to be {expected_str}; but got {self._actual.name}!" # Argument type of an ApiCall. T = TypeVar("T") # Return type of an ApiCall response parser. R = TypeVar("R") @unique class ApiType(Enum): ECM = 1 GEM = 2 @dataclass class ApiCall(Generic[T, R]): """ Helper class for making API calls with BidirectionalProtocol. There is one instance of this class for each supported API call. This class handles the send_api_request and receive_api_response parts of driving the protocol, since those are specific to each API request type. """ def __init__( self, gem_formatter: Callable[[T], str], gem_parser: Callable[[str], R | None] | None, ecm_formatter: Callable[[T], List[bytes]] | None, ecm_parser: Callable[[bytes], R | None] | None, ) -> None: """ Create a new APICall. gem_formatter - a callable that, given a parameter of type T, returns the string to send to the GEM to make the API call gem_parser - a callable that, given a string, parses it into a value of type R. If there is not enough data to parse yet, it should return None. If there is enough data to parse, but it is malformed, it should raise an Exception. ecm_formatter - a callable that, given a parameter of type T, returns the series of bytes chunks to send to the ECM to make the API call ecm_parser - a callable that, given a bytes, parses it into a value of type R. If there is not enough data to parse yet, it should return None. If there is enough data to parse, but it is malformed, it should raise an Exception. """ self._gem_formatter = gem_formatter self._gem_parser = gem_parser self._ecm_formatter = ecm_formatter self._ecm_parser = ecm_parser def format( self, api_type: ApiType, arg: T, serial_number: int | None, ) -> List[bytes]: if api_type == ApiType.GEM: result = self._gem_formatter(arg) if serial_number: result = result.replace( ESCAPE_SEQUENCE, f"{TARGET_SERIAL_NUMBER_PREFIX}{serial_number%100000:05}", ) return [result.encode()] elif api_type == ApiType.ECM: assert self._ecm_formatter result = self._ecm_formatter(arg) return result else: assert False def has_parser(self, api_type: ApiType) -> bool: if api_type == ApiType.GEM: return self._gem_parser is not None elif api_type == ApiType.ECM: return self._ecm_parser is not None else: assert False def parse(self, api_type: ApiType, response: bytes) -> R | None: if api_type == ApiType.GEM: return self._gem_parser(response.decode()) if self._gem_parser else None elif api_type == ApiType.ECM: return self._ecm_parser(response) if self._ecm_parser else None else: assert False class BidirectionalProtocol(PacketProtocol): """Protocol implementation for bi-directional communication with a GreenEye Monitor.""" """ Create a new BidirectionalProtocol The passed in queue contains full packets that have been received. The packet_delay_clear_time plus API_RESPONSE_WAIT_TIME must be less than 15 seconds. """ def __init__( self, queue: asyncio.Queue[PacketProtocolMessage], packet_delay_clear_time: timedelta = PACKET_DELAY_CLEAR_TIME_DEFAULT, send_packet_delay: bool = True, api_type: ApiType | None = None, ): # Ensure that the clear time and the response wait time fit within the 15 second packet delay interval that is requested. assert (packet_delay_clear_time + API_RESPONSE_WAIT_TIME) < timedelta( seconds=15 ) super().__init__(queue) self._api_buffer = bytearray() self.send_packet_delay = send_packet_delay self._packet_delay_clear_time = packet_delay_clear_time self._state = ProtocolState.RECEIVING_PACKETS self._api_call: ApiCall[Any, Any] | None = None self._api_result: asyncio.Future[Any] | None = None self._api_type = api_type self._api_requests: Deque[bytes] = deque() @property def packet_delay_clear_time(self) -> timedelta: return self._packet_delay_clear_time @property def api_type(self) -> ApiType: if self._api_type is None: if ( self._packet_type == PacketFormatType.ECM_1220 or self._packet_type == PacketFormatType.ECM_1240 ): self._api_type = ApiType.ECM elif self._packet_type: self._api_type = ApiType.GEM result = self._api_type assert result return result @api_type.setter def api_type(self, type: ApiType) -> None: self._api_type = type def unknown_data_received(self, data: bytes) -> None: if self._state == ProtocolState.SENDING_API_REQUEST: # We're in the middle of an ECM API call, which # has multiple roundtrips to send all the request chunks assert self._api_call if data.startswith(b"\xfc"): # ECM acks each chunk with \xfc if len(self._api_requests) > 0: self._send_next_api_request_chunk() else: # No more chunks means that we've now completely sent the request self._state = ProtocolState.SENT_API_REQUEST if self._api_call.has_parser(self.api_type): # This API call is expecting a response, and # the ACK character might be immediately followed # by response data, so we pull off the ACK character # and fall through to the rest of the method, # which then pulls the response from the data data = data[1:] else: # Last ACK of a request with no response self._set_result(result=None) else: self._set_result( exception=Exception("Bad response from device: {data}") ) if self._state == ProtocolState.SENT_API_REQUEST: assert self._api_call is not None self._api_buffer.extend(data) response = bytes(self._api_buffer) LOG.debug("%d: Attempting to parse API response: %s", id(self), response) result = self._api_call.parse(self.api_type, response) if result: if self.api_type == ApiType.ECM: self._ensure_write_transport().write(b"\xfc") self._set_result(result=result) elif self._state == ProtocolState.RECEIVING_PACKETS: super().unknown_data_received(data) def begin_api_request(self) -> timedelta: """ Begin the process of sending an API request. Calls WriteTransport.write on the associated transport with bytes that need to be sent. Returns a timedelta. Callers must wait for that amount of time, then call send_api_request with the actual request. """ self._expect_state(ProtocolState.RECEIVING_PACKETS) self._state = ProtocolState.SENT_PACKET_DELAY_REQUEST if self.api_type == ApiType.GEM and self.send_packet_delay: LOG.debug("%d: Starting API request. Requesting packet delay...", id(self)) self._ensure_write_transport().write( CMD_DELAY_NEXT_PACKET.encode() ) # Delay packets for 15 seconds return self._packet_delay_clear_time else: return timedelta(seconds=0) def invoke_api( self, api: ApiCall[T, R], arg: T, result: asyncio.Future[R], serial_number: Optional[int] = None, ) -> None: """ Send the given API request, after having called begin_api_request. Calls WriteTransport.write on the associated transport with bytes that need to be sent. Returns a timedelta. Callers must wait for that amount of time, then call receive_api_response to receive the response. """ self._expect_state(ProtocolState.SENT_PACKET_DELAY_REQUEST) assert len(self._api_requests) == 0 self._api_call = api self._api_result = result self._api_requests.extend(api.format(self.api_type, arg, serial_number)) self._send_next_api_request_chunk() def _send_next_api_request_chunk(self) -> None: assert len(self._api_requests) > 0 assert self._api_call assert self._api_result request = self._api_requests.popleft() LOG.debug("%d: Sending API request %s...", id(self), request) self._ensure_write_transport().write(request) self._state = ( ProtocolState.SENT_API_REQUEST if self.api_type == ApiType.GEM else ProtocolState.SENDING_API_REQUEST ) if ( not self._api_call.has_parser(self.api_type) and self.api_type == ApiType.GEM ): # GEM API calls without a response are just a single send and you're done # (ECM API calls without a response still have multiple request chunks to send, # and even the last chunk is still acked with \xfc, so we don't advance the # state for them here) assert len(self._api_requests) == 0 self._set_result(result=None) def _set_result( self, result: Any | None = None, exception: Exception | None = None ) -> None: assert ( self._state == ProtocolState.SENT_API_REQUEST or self._state == ProtocolState.SENDING_API_REQUEST ) assert self._api_result assert not self._api_result.done() if exception is None: self._state = ProtocolState.RECEIVED_API_RESPONSE self._api_result.set_result(result) else: assert result is None self._api_result.set_exception(exception) self._api_result = None self._api_call = None def end_api_request(self) -> None: """ Ends an API request. Every begin_api_request call must have a matching end_api_request call, even if an error occurred in between. """ self._expect_state( { ProtocolState.RECEIVED_API_RESPONSE, ProtocolState.SENDING_API_REQUEST, ProtocolState.SENT_API_REQUEST, ProtocolState.SENT_PACKET_DELAY_REQUEST, } ) self._api_buffer.clear() self._api_call = None self._api_requests.clear() self._api_result = None LOG.debug("%d: Ended API request", id(self)) self._state = ProtocolState.RECEIVING_PACKETS def _ensure_write_transport(self) -> asyncio.WriteTransport: transport = self._ensure_transport() assert isinstance(transport, asyncio.WriteTransport) return transport def _expect_state(self, expected_state: Union[ProtocolState, Set[ProtocolState]]): if not isinstance(expected_state, set): expected_state = {expected_state} assert len(expected_state) > 0 if self._state not in expected_state: raise ProtocolStateException(actual=self._state, expected=expected_state) sdwilsh-siobrultech-protocols-78196f6/siobrultech_protocols/py.typed000066400000000000000000000000001453451727300261550ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/tests/000077500000000000000000000000001453451727300212035ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/tests/__init__.py000066400000000000000000000000001453451727300233020ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/tests/gem/000077500000000000000000000000001453451727300217535ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/tests/gem/data/000077500000000000000000000000001453451727300226645ustar00rootroot00000000000000sdwilsh-siobrultech-protocols-78196f6/tests/gem/data/BIN32-ABS.bin000066400000000000000000000004151453451727300245360ustar00rootroot00000000000000Щ/#b (lyƣ[HV!5OK;}pK_)(>)=Kn1 0)&[ '[N S" `% 8 t8 (sdwilsh-siobrultech-protocols-78196f6/tests/gem/data/BIN32-NET.bin000066400000000000000000000006551453451727300245650ustar00rootroot00000000000000/b!b٤ l`zMVHx V!>5O;DYI_"'>)=Ρ#1 0 0&[ '\N S"_% 9 '8 (Zsdwilsh-siobrultech-protocols-78196f6/tests/gem/data/BIN48-ABS.bin000066400000000000000000000005731453451727300245520ustar00rootroot000000000000003/n lNX &gSH?U!5O>w;`PRd 6 [ K&- M   Un< 1KO ( :#sdwilsh-siobrultech-protocols-78196f6/tests/gem/data/BIN48-NET.bin000066400000000000000000000011531453451727300245660ustar00rootroot00000000000000"/WC lhFDQH/nU!4Od;?H_8'l)<ճ̷M1 w0&  QO US [ &\M  T" _$ ; <  657 (sdwilsh-siobrultech-protocols-78196f6/tests/gem/data/ECM-1240.bin000066400000000000000000000001011453451727300243360ustar00rootroot00000000000000a2A..~T%F?A?Asdwilsh-siobrultech-protocols-78196f6/tests/gem/mock_transport.py000066400000000000000000000014571453451727300254010ustar00rootroot00000000000000import asyncio from typing import List from siobrultech_protocols.gem.protocol import BidirectionalProtocol class MockTransport(asyncio.WriteTransport): def __init__(self) -> None: self.writes: List[bytes] = [] self.closed: bool = False def write(self, data: bytes) -> None: self.writes.append(data) def close(self) -> None: self.closed = True class MockRespondingTransport(asyncio.WriteTransport): def __init__( self, protocol: BidirectionalProtocol, encoded_response: bytes, ) -> None: self._protocol = protocol self._encoded_response = encoded_response def write(self, data: bytes) -> None: loop = asyncio.get_event_loop() loop.call_soon(self._protocol.data_received, self._encoded_response) sdwilsh-siobrultech-protocols-78196f6/tests/gem/packet_test_data.py000066400000000000000000000401701453451727300256260ustar00rootroot00000000000000import datetime import os from io import StringIO from typing import Iterable from siobrultech_protocols.gem.protocol import Packet greeneye_dir = os.path.dirname(os.path.abspath(__file__)) greeneye_data_dir = os.path.join(greeneye_dir, "data") PACKETS = { "BIN32-ABS.bin": { "absolute_watt_seconds": [ 3123664, 9249700, 195388151, 100917236, 7139112, 1440, 4, 3, 14645520, 111396601, 33259670, 38296448, 1108415, 2184858, 5191049, 1, 71032651, 60190845, 47638292, 12017483, 36186563, 14681918, 69832947, 37693, 60941899, 1685614, 902, 799182, 302590, 3190972, 5, 647375119, ], "currents": [ 0.42, 0.44, 3.86, 0.4, 0.14, 0.0, 0.0, 0.0, 0.78, 1.82, 1.56, 0.26, 0.38, 0.08, 0.16, 0.0, 1.66, 0.68, 0.18, 0.12, 1.92, 0.74, 0.2, 0.12, 1.12, 0.1, 0.08, 0.4, 0.08, 0.18, 0.0, 14.42, ], "pulse_counts": [0, 0, 0, 0], "seconds": 997492, "serial_number": 603, "device_id": 11, "temperatures": [None, -5, 20, 255, 0, 0, 0, 0], "voltage": 121.1, }, "BIN32-NET.bin": { "absolute_watt_seconds": [ 3123588, 9249122, 195352930, 100916608, 7139048, 1440, 4, 3, 14639320, 111380602, 33246631, 38295282, 1108344, 2184716, 5190974, 1, 71017653, 60184900, 47637526, 12017481, 36168994, 14675409, 69832510, 37693, 60935828, 1685539, 902, 799127, 302590, 3190447, 5, 647245834, ], "currents": [ 0.42, 0.44, 3.88, 0.46, 0.14, 0.0, 0.0, 0.0, 0.78, 1.84, 1.56, 0.26, 0.38, 0.08, 0.16, 0.0, 1.66, 0.68, 0.16, 0.12, 1.9, 0.74, 0.2, 0.1, 1.14, 0.1, 0.08, 0.4, 0.1, 0.18, 0.0, 14.42, ], "polarized_watt_seconds": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], "pulse_counts": [0, 0, 0, 0], "seconds": 997415, "serial_number": 603, "device_id": 11, "temperatures": [None, -5, 20, 255, 0, 0, 0, 0], "voltage": 121.5, }, "BIN48-ABS.bin": { "absolute_watt_seconds": [ 3123507, 9248674, 195325612, 100916122, 7139001, 1440, 4, 3, 14634511, 111368220, 33236493, 38294375, 1108287, 2184600, 5190920, 1, 71006014, 60180284, 47636927, 12017481, 36155362, 14670370, 69832182, 37692, 60930993, 1685472, 902, 799075, 302590, 3190044, 5, 647145389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], "currents": [ 0.42, 0.44, 3.86, 0.4, 0.14, 0.0, 0.0, 0.0, 0.78, 1.8, 1.58, 0.26, 0.42, 0.08, 0.18, 0.0, 1.66, 0.68, 0.16, 0.12, 1.92, 0.74, 0.2, 0.1, 1.22, 0.1, 0.08, 0.36, 0.08, 0.18, 0.0, 14.42, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1230.0, 993.52, 127.88, 471.04, 0.08, 63.3, 314.88, 11.0, ], "pulse_counts": [0, 0, 0, 0], "seconds": 997354, "serial_number": 603, "device_id": 11, "temperatures": [None, -5, 20, 255, 0, 0, 0, 0], "voltage": 121.3, }, "BIN48-NET.bin": { "absolute_watt_seconds": [ 3123490, 9248489, 195314519, 100915929, 7138977, 1440, 4, 3, 14632552, 111363102, 33232371, 38294008, 1108271, 2184558, 5190900, 1, 71001276, 60178405, 47636683, 12017480, 36149816, 14668312, 69832044, 37692, 60928981, 1685452, 902, 799053, 302590, 3189879, 5, 647104414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], "currents": [ 0.44, 0.44, 3.96, 0.46, 0.14, 0.0, 0.0, 0.0, 0.76, 1.84, 1.54, 0.26, 0.4, 0.08, 0.18, 0.0, 1.68, 0.68, 0.18, 0.12, 1.9, 0.72, 0.2, 0.1, 1.18, 0.12, 0.08, 0.4, 0.1, 0.18, 0.0, 14.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1230.0, 993.52, 127.88, 1290.24, 0.14, 64.7, 279.04, 11.3, ], "polarized_watt_seconds": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893353787397, 16779521, 0, 3489681664, 1048746, 33554688, 339315458048, 2816, 21480407271, 1392508928, 1, 117440512, 68831281152, 47446822415, 25887770890, 4328719365, ], "pulse_counts": [0, 0, 0, 0], "seconds": 997327, "serial_number": 603, "device_id": 11, "temperatures": [None, -5, 20, 255, 0, 0, 0, 0], "voltage": 121.3, }, "BIN48-NET-TIME.bin": { "absolute_watt_seconds": [ 2973101, 8059708, 156428334, 85168830, 3701996, 1417, 4, 3, 10122331, 102754614, 25349121, 23541326, 910505, 1899678, 3136543, 1, 50182738, 46920840, 37015191, 7826156, 19817953, 11755830, 61617610, 35109, 51981008, 1519294, 760, 663522, 229936, 2635054, 5, 451930676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], "currents": [ 0.42, 0.44, 1.36, 0.54, 0.1, 0.0, 0.0, 0.0, 0.22, 1.28, 0.92, 0.26, 0.4, 0.1, 0.84, 0.0, 3.02, 1.1, 0.16, 0.86, 0.08, 0.2, 0.22, 0.12, 1.16, 0.12, 0.08, 0.42, 0.08, 0.12, 0.0, 10.66, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1230.0, 993.52, 127.88, 302.08, 0.0, 53.76, 1052.16, 3.16, ], "polarized_watt_seconds": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554051239936, 4352, 0, 2097152768, 8590917732, 1728053504, 549779603456, 1677724160, 21474902016, 1275068416, 0, 50331648, 68801396736, 47446822415, 25887770890, 4328719365, ], "pulse_counts": [0, 0, 0, 0], "seconds": 841707, "serial_number": 603, "device_id": 11, "temperatures": [None, -5, 20, 255, 0, 0, 0, 0], "time_stamp": datetime.datetime(2017, 12, 20, 5, 7, 26), "voltage": 121.7, }, "BIN48-NET-TIME_tricky.bin": { "absolute_watt_seconds": [ 231291827, 375937488, 2000191302, 884282444, 217533987, 26818, 83, 64, 235203561, 660892780, 516638549, 590071215, 16739113, 46163811, 120489725, 22, 651996464, 1115855892, 443956599, 187609729, 418355582, 186555196, 553111232, 1396693, 868815981, 27838920, 2307787, 11390311, 9790746, 48429661, 62, 11386703968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], "currents": [ 0.36, 0.3, 1.5, 0.34, 0.08, 0.0, 0.0, 0.0, 0.32, 0.76, 0.9, 0.28, 0.36, 0.08, 0.14, 0.0, 0.2, 1.54, 0.18, 0.1, 0.18, 0.16, 0.2, 0.08, 1.7, 0.08, 0.0, 0.3, 0.08, 0.12, 0.0, 7.32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1230.0, 993.52, 127.88, 250.88, 655.52, 41.28, 384.0, 659.16, ], "polarized_watt_seconds": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665720258565, 0, 0, 1375737600, 4296278116, 256, 790274048000, 2816, 25769803791, 2667577600, 0, 67108864, 68773347328, 47446822415, 25887770890, 4328719365, ], "pulse_counts": [0, 0, 0, 0], "seconds": 11988815, "serial_number": 603, "device_id": 11, "temperatures": [None, -5, 20, 255, 0, 0, 0, 0], "time_stamp": datetime.datetime(2018, 6, 11, 21, 16, 58), "voltage": 122.2, }, "ECM-1240.bin": { "absolute_watt_seconds": [3325793, 519631], "polarized_watt_seconds": [3050305, 97838], "currents": [0.21, 62.28], "seconds": 9700, "serial_number": 434, "device_id": 5, "voltage": 128.8, "aux": [70, 63, 65, 63, 65], "dc_voltage": 511, }, } def read_packets(packet_file_names: Iterable[str]): result = bytearray() for packet_file_name in packet_file_names: result.extend(read_packet(packet_file_name)) return bytes(result) def read_packet(packet_file_name: str): with open(os.path.join(greeneye_data_dir, packet_file_name), "rb") as data_file: return data_file.read() def assert_packet(packet_file_name: str, parsed_packet: Packet): expected_packet = PACKETS[packet_file_name] expected = StringIO() actual = StringIO() for key, expected_value in sorted(expected_packet.items(), key=lambda x: x[0]): actual_value = getattr(parsed_packet, key) if expected_value != actual_value: expected.write( "{key}={expected},\n".format(key=key, expected=repr(expected_value)) ) actual.write("{key}={actual},\n".format(key=key, actual=repr(actual_value))) if expected.getvalue() != "": raise AssertionError( """ Some packet fields did not match. Expected: {expected} Actual: {actual}""".format( expected=expected.getvalue(), actual=actual.getvalue() ) ) sdwilsh-siobrultech-protocols-78196f6/tests/gem/test_api.py000066400000000000000000000254601453451727300241440ustar00rootroot00000000000000from __future__ import annotations import asyncio from datetime import datetime, timedelta from typing import List, Optional from unittest.async_case import IsolatedAsyncioTestCase from unittest.mock import patch from siobrultech_protocols.gem.api import ( GET_SERIAL_NUMBER, SET_DATE_AND_TIME, SET_PACKET_FORMAT, SET_PACKET_SEND_INTERVAL, SET_SECONDARY_PACKET_FORMAT, ApiCall, R, T, call_api, get_serial_number, set_date_and_time, set_packet_format, set_packet_send_interval, set_secondary_packet_format, synchronize_time, ) from siobrultech_protocols.gem.packets import PacketFormatType from siobrultech_protocols.gem.protocol import ( ApiType, BidirectionalProtocol, PacketProtocolMessage, ) from tests.gem.mock_transport import MockRespondingTransport, MockTransport class TestApi(IsolatedAsyncioTestCase): def setUp(self): self._queue: asyncio.Queue[PacketProtocolMessage] = asyncio.Queue() self._transport = MockTransport() self._protocol = BidirectionalProtocol( self._queue, packet_delay_clear_time=timedelta(seconds=0), api_type=ApiType.GEM, ) self._protocol.connection_made(self._transport) async def testApiCallWithoutResponse(self): await self.assertCall( ApiCall(lambda _: "REQUEST", None, None, None), "REQUEST", None, None, None, None, ) async def testApiCall(self): await self.assertCall( ApiCall(lambda _: "REQUEST", lambda response: response, None, None), "REQUEST", None, None, "RESPONSE".encode(), "RESPONSE", ) async def testApiCallWithSerialNumber(self): await self.assertCall( ApiCall(lambda _: "^^^REQUEST", lambda response: response, None, None), "^^^NMB02345REQUEST", None, 1002345, "RESPONSE".encode(), "RESPONSE", ) async def testApiCallIgnored(self): call = ApiCall(lambda _: "REQUEST", lambda response: response, None, None) async with call_api(call, self._protocol, timeout=timedelta(seconds=0)) as f: with self.assertRaises(asyncio.exceptions.TimeoutError): await f(None) async def testGEMGetSerialNumber(self): await self.assertCall( GET_SERIAL_NUMBER, "^^^RQSSRN", None, None, "1234567\r\n".encode(), 1234567, ) async def testECMGetSerialNumber(self): await self.assertECMCall( GET_SERIAL_NUMBER, [b"\xfc", b"SET", b"RCV"], None, None, b"\xa7\x04\xa7\x04\xf4\x03\x01\x0c\x04\x0b\x05\x01\xb2\x90\x00\x00\x00\x00\x8d\x8b\x8c\x8b\xce\x00\xff\xff\xff\xff\x05\x80>\x00m", 500434, ) async def testSetDateTime(self): await self.assertCall( SET_DATE_AND_TIME, "^^^SYSDTM12,08,23,13,30,28\r", datetime.fromisoformat("2012-08-23 13:30:28"), None, "DTM\r\n".encode(), True, ) async def testSetPacketFormat(self): await self.assertCall( SET_PACKET_FORMAT, "^^^SYSPKT02", 2, None, "PKT\r\n".encode(), True, ) async def testGEMSetPacketSendInterval(self): await self.assertCall( SET_PACKET_SEND_INTERVAL, "^^^SYSIVL042", 42, None, "IVL\r\n".encode(), True, ) async def testECMSetPacketSendInterval(self): await self.assertECMCall( SET_PACKET_SEND_INTERVAL, [b"\xfc", b"SET", b"IV2", bytes([42])], 42, None, None, None, ) async def testSetSecondaryPacketFormat(self): await self.assertCall( SET_SECONDARY_PACKET_FORMAT, "^^^SYSPKF00", 0, None, "PKF\r\n".encode(), True, ) async def assertCall( self, call: ApiCall[T, R], request: str, arg: T, serial_number: Optional[int], encoded_response: bytes | None, parsed_response: R | None, ): self._protocol.begin_api_request() self._transport.writes.clear() # Ignore the packet delay result = asyncio.get_event_loop().create_future() self._protocol.invoke_api(call, arg, result, serial_number) self.assertEqual( self._transport.writes, [request.encode()], f"{request.encode()} should be written to the transport", ) if encoded_response is not None: self._protocol.data_received(encoded_response) result = await asyncio.wait_for(result, 0) self.assertEqual( result, parsed_response, f"{parsed_response} should be the parsed value returned", ) async def assertECMCall( self, call: ApiCall[T, R], request: List[bytes], arg: T, serial_number: Optional[int], encoded_response: bytes | None, parsed_response: R | None, ): self._protocol.api_type = ApiType.ECM self._transport.writes.clear() self._protocol.begin_api_request() result = asyncio.get_event_loop().create_future() self._protocol.invoke_api(call, arg, result, serial_number) for _ in range(0, len(request)): self._protocol.data_received(b"\xfc") self.assertEqual( self._transport.writes, request, f"{request} should be written to the transport", ) if encoded_response is not None: self._transport.writes.clear() self._protocol.data_received(encoded_response) result = await asyncio.wait_for(result, 0) self.assertEqual( result, parsed_response, f"{parsed_response} should be the parsed value returned", ) self._protocol.end_api_request() if encoded_response is not None: self.assertEqual( self._transport.writes, [b"\xfc"], "ECM API calls with a response should be acked by the caller", ) class TestContextManager(IsolatedAsyncioTestCase): def setUp(self): self._queue: asyncio.Queue[PacketProtocolMessage] = asyncio.Queue() self._transport = MockTransport() self._protocol = BidirectionalProtocol( self._queue, packet_delay_clear_time=timedelta(seconds=0), api_type=ApiType.GEM, ) self._protocol.connection_made(self._transport) @patch( "siobrultech_protocols.gem.protocol.API_RESPONSE_WAIT_TIME", timedelta(seconds=0), ) async def testApiCall(self): call = ApiCall(lambda _: "REQUEST", lambda response: response, None, None) async with call_api(call, self._protocol) as f: self.setApiResponse("RESPONSE".encode()) response = await f(None) self.assertEqual(response, "RESPONSE") async def testApiCallWithSerialNumber(self): call = ApiCall(lambda _: "^^^REQUEST", lambda response: response, None, None) async with call_api(call, self._protocol, serial_number=1234567) as f: self.setApiResponse("RESPONSE".encode()) response = await f(None) self.assertEqual(response, "RESPONSE") self.assertEqual(self._transport.writes, [b"^^^SYSPDL", b"^^^NMB34567REQUEST"]) async def testTaskCanceled(self): call = ApiCall(lambda _: "REQUEST", lambda response: response, None, None) with self.assertRaises(asyncio.CancelledError): with patch("asyncio.sleep") as mock_sleep: mock_sleep.side_effect = asyncio.CancelledError async with call_api(call, self._protocol): raise AssertionError("this should not be reached") def setApiResponse(self, ecnoded_response: bytes) -> asyncio.Task[None]: async def notify_data_received() -> None: self._protocol.data_received(ecnoded_response) return asyncio.create_task( notify_data_received(), name=f"{__name__}:send_api_resonse" ) class TestApiHelpers(IsolatedAsyncioTestCase): def setUp(self): self._protocol = BidirectionalProtocol( asyncio.Queue(), packet_delay_clear_time=timedelta(seconds=0), api_type=ApiType.GEM, ) patcher_API_RESPONSE_WAIT_TIME = patch( "siobrultech_protocols.gem.protocol.API_RESPONSE_WAIT_TIME", timedelta(seconds=0), ) patcher_API_RESPONSE_WAIT_TIME.start() self.addCleanup(lambda: patcher_API_RESPONSE_WAIT_TIME.stop()) async def test_get_serial_number(self): transport = MockRespondingTransport(self._protocol, "1234567\r\n".encode()) self._protocol.connection_made(transport) serial = await get_serial_number(self._protocol) self.assertEqual(serial, 1234567) async def test_set_date_and_time(self): transport = MockRespondingTransport(self._protocol, "DTM\r\n".encode()) self._protocol.connection_made(transport) success = await set_date_and_time(self._protocol, datetime(2020, 3, 11)) self.assertTrue(success) async def test_set_packet_format(self): transport = MockRespondingTransport(self._protocol, "PKT\r\n".encode()) self._protocol.connection_made(transport) success = await set_packet_format(self._protocol, PacketFormatType.BIN32_ABS) self.assertTrue(success) async def test_set_packet_send_interval(self): with self.assertRaises(ValueError): await set_packet_send_interval(self._protocol, -1) with self.assertRaises(ValueError): await set_packet_send_interval(self._protocol, 257) transport = MockRespondingTransport(self._protocol, "IVL\r\n".encode()) self._protocol.connection_made(transport) success = await set_packet_send_interval(self._protocol, 42) self.assertTrue(success) async def test_set_secondary_packet_format(self): transport = MockRespondingTransport(self._protocol, "PKF\r\n".encode()) self._protocol.connection_made(transport) success = await set_secondary_packet_format( self._protocol, PacketFormatType.BIN32_ABS ) self.assertTrue(success) async def test_synchronize_time(self): transport = MockRespondingTransport(self._protocol, "DTM\r\n".encode()) self._protocol.connection_made(transport) success = await synchronize_time(self._protocol) self.assertTrue(success) sdwilsh-siobrultech-protocols-78196f6/tests/gem/test_bidirectional_protocol.py000066400000000000000000000202461453451727300301210ustar00rootroot00000000000000import asyncio import unittest from siobrultech_protocols.gem.const import CMD_DELAY_NEXT_PACKET from siobrultech_protocols.gem.protocol import ( ApiCall, ApiType, BidirectionalProtocol, ConnectionLostMessage, ConnectionMadeMessage, PacketProtocolMessage, PacketReceivedMessage, ProtocolStateException, ) from tests.gem.mock_transport import MockTransport from tests.gem.packet_test_data import assert_packet, read_packet TestCall = ApiCall[str, str]( gem_formatter=lambda x: x, gem_parser=lambda x: x if x.endswith("\n") else None, ecm_formatter=lambda x: [ (x + "1").encode(), (x + "2").encode(), (x + "3").encode(), ], ecm_parser=lambda x: x.decode() if x.endswith(b"\n") else None, ) class TestBidirectionalProtocol(unittest.IsolatedAsyncioTestCase): def setUp(self): self._queue: asyncio.Queue[PacketProtocolMessage] = asyncio.Queue() self._transport = MockTransport() self._protocol = BidirectionalProtocol(self._queue, api_type=ApiType.GEM) self._protocol.connection_made(self._transport) self._result: asyncio.Future[str] = asyncio.get_event_loop().create_future() message = self._queue.get_nowait() assert isinstance(message, ConnectionMadeMessage) assert message.protocol is self._protocol def tearDown(self) -> None: if not self._transport.closed: exc = Exception("Test") self._protocol.connection_lost(exc=exc) message = self._queue.get_nowait() assert isinstance(message, ConnectionLostMessage) assert message.protocol is self._protocol assert message.exc is exc self._protocol.close() # Close after connection_lost is not required, but at least should not crash def testClose(self): self._protocol.close() assert self._transport.closed def testBeginApi(self): self._protocol.begin_api_request() self.assertEqual(self._transport.writes, [CMD_DELAY_NEXT_PACKET.encode()]) def testBeginApiWithoutDelay(self): self._protocol.send_packet_delay = False self._protocol.begin_api_request() self.assertEqual(self._transport.writes, []) def testSendWithoutBeginFails(self): with self.assertRaises(ProtocolStateException): self._protocol.invoke_api(TestCall, "request", self._result) def testSendRequest(self): self._protocol.begin_api_request() self._transport.writes.clear() self._protocol.invoke_api(TestCall, "request", self._result) self.assertEqual(self._transport.writes, ["request".encode()]) async def testEcmApiCall(self): self._protocol.api_type = ApiType.ECM self._protocol.begin_api_request() self._protocol.invoke_api(TestCall, "request", self._result) self._protocol.data_received(b"\xfc") self._protocol.data_received(b"\xfc") self._protocol.data_received(b"\xfcRESPONSE\n") response = await self.get_response() self.assertEqual( self._transport.writes, [b"request1", b"request2", b"request3", b"\xfc"] ) self.assertEqual(response, "RESPONSE\n") async def testFailureDuringEcmApiCallDoesNotPreventNextCall(self): self._protocol.api_type = ApiType.ECM self._protocol.begin_api_request() self._protocol.invoke_api(TestCall, "request", self._result) self._protocol.data_received(b"\xfc") self._protocol.data_received(b"X") with self.assertRaises(Exception): await self.get_response() self._protocol.end_api_request() self.assertEqual(self._transport.writes, [b"request1", b"request2"]) self._transport.writes.clear() self._result = asyncio.get_event_loop().create_future() self._protocol.begin_api_request() self._protocol.invoke_api(TestCall, "request2", self._result) self._protocol.data_received(b"\xfc") self._protocol.data_received(b"\xfc") self._protocol.data_received(b"\xfcRESPONSE\n") response = await self.get_response() self._protocol.end_api_request() self.assertEqual( self._transport.writes, [b"request21", b"request22", b"request23", b"\xfc"] ) self.assertEqual(response, "RESPONSE\n") async def testPacketRacingWithApi(self): """Tests that the protocol can handle a packet coming in right after it has requested a packet delay from the GEM.""" self._protocol.begin_api_request() self._protocol.data_received(read_packet("BIN32-ABS.bin")) self._protocol.invoke_api(TestCall, "REQUEST", self._result) self._protocol.data_received(b"RESPONSE\n") response = await self.get_response() self._protocol.end_api_request() self.assertEqual(response, "RESPONSE\n") self.assertPacket("BIN32-ABS.bin") async def testPacketInterleavingWithApi(self): """Tests that the protocol can handle a packet coming in in the middle of the API response. (I don't know whether this can happen in practice.)""" self._protocol.begin_api_request() self._protocol.data_received(read_packet("BIN32-ABS.bin")) self._protocol.invoke_api(TestCall, "REQUEST", self._result) self._protocol.data_received(b"RES") self._protocol.data_received(read_packet("BIN32-ABS.bin")) self._protocol.data_received(b"PONSE\n") response = await self.get_response() self._protocol.end_api_request() self.assertEqual(response, "RESPONSE\n") self.assertPacket("BIN32-ABS.bin") self.assertPacket("BIN32-ABS.bin") def testDeviceIgnoresApi(self): """Tests that the protocol fails appropriately if a device ignores API calls and just keeps sending packets.""" self._protocol.begin_api_request() self._protocol.data_received(read_packet("BIN32-ABS.bin")) self._protocol.invoke_api(TestCall, "REQUEST", self._result) self._protocol.data_received(read_packet("BIN32-ABS.bin")) assert not self._result.done() self._protocol.end_api_request() self.assertPacket("BIN32-ABS.bin") self.assertPacket("BIN32-ABS.bin") async def testApiCallWithPacketInProgress(self): """Tests that the protocol can handle a packet that's partially arrived when it requested a packet delay from the GEM.""" packet = read_packet("BIN32-ABS.bin") bytes_sent_before_packet_delay_command = 32 self._protocol.data_received(packet[0:bytes_sent_before_packet_delay_command]) self._protocol.begin_api_request() self._protocol.data_received(packet[bytes_sent_before_packet_delay_command:]) self._protocol.invoke_api(TestCall, "REQUEST", self._result) self._protocol.data_received(b"RESPONSE\n") response = await self.get_response() self._protocol.end_api_request() self.assertEqual(response, "RESPONSE\n") self.assertPacket("BIN32-ABS.bin") async def testApiCallToIdleGem(self): """Tests that the protocol can handle no packets arriving after it has requested a packet delay from the GEM.""" self._protocol.begin_api_request() self._protocol.invoke_api(TestCall, "REQUEST", self._result) self._protocol.data_received(b"RESPONSE\n") response = await self.get_response() self._protocol.end_api_request() self.assertEqual(response, "RESPONSE\n") self.assertNoPacket() def testEndAfterBegin(self): """Checks for the case where user-code may fail, and we just call end_api_request after calling begin_api_request.""" self._protocol.begin_api_request() self._protocol.end_api_request() async def get_response(self) -> str: return await asyncio.wait_for(self._result, 0) def assertNoPacket(self): self.assertTrue(self._queue.empty()) def assertPacket(self, expected_packet: str): message = self._queue.get_nowait() assert isinstance(message, PacketReceivedMessage) assert message.protocol is self._protocol assert_packet(expected_packet, message.packet) if __name__ == "__main__": unittest.main() sdwilsh-siobrultech-protocols-78196f6/tests/gem/test_fields.py000066400000000000000000000134501453451727300246350ustar00rootroot00000000000000import unittest from datetime import datetime from siobrultech_protocols.gem.fields import ( ArrayField, ByteField, ByteOrder, BytesField, DateTimeField, FloatingPointArrayField, FloatingPointField, NumericArrayField, NumericField, Sign, ) class TestFieldParsing(unittest.TestCase): def testByteFieldRead(self): self.assertEqual(b"c", ByteField().read(b"abcdefg", 2)) def testBytesFieldRead(self): self.assertEqual(b"cdef", BytesField(4).read(b"abcdefg", 2)) def testNumericFieldHiToLoRead(self): self.assertEqual( 1, NumericField(2, ByteOrder.HiToLo, Sign.Unsigned).read(b"\x02\x00\x01", 1) ) def testNumericFieldHiToLoSignedRead(self): self.assertEqual( -1, NumericField(2, ByteOrder.HiToLo, Sign.Signed).read(b"\x02\x80\x01", 1) ) def testNumericFieldLoToHiRead(self): self.assertEqual( 256, NumericField(2, ByteOrder.LoToHi, Sign.Unsigned).read(b"\x02\x00\x01", 1), ) def testNumericFieldLoToHiSignedRead(self): self.assertEqual( -256, NumericField(2, ByteOrder.LoToHi, Sign.Signed).read(b"\x02\x00\x81", 1), ) def testNumericFieldUnsignedMax(self): field = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) self.assertEqual( field.read(b"\xff\xff", 0), field.max, ) def testNumericFieldSignedMax(self): field = NumericField(2, ByteOrder.HiToLo, Sign.Signed) self.assertEqual( field.read(b"\x7f\xff", 0), field.max, ) def testFloatingPointFieldRead(self): self.assertEqual( 0.5, FloatingPointField(2, ByteOrder.HiToLo, Sign.Unsigned, 2.0).read( b"\x02\x00\x01", 1 ), ) def testFloatingPointFieldReadDiv100(self): self.assertEqual( 1.16, FloatingPointField(2, ByteOrder.HiToLo, Sign.Unsigned, 100.0).read( b"\x02\x00\x74", 1 ), ) def testDateTimeFieldRead(self): self.assertEqual( datetime(2020, 1, 1, 0, 0, 0), DateTimeField().read(b"\x02\x14\x01\x01\x00\x00\x00", 1), ) def testArrayFieldRead(self): self.assertEqual( [1, 2, 3, 4], ArrayField(4, NumericField(2, ByteOrder.HiToLo, Sign.Unsigned)).read( b"\x05\x00\x01\x00\x02\x00\x03\x00\x04", 1 ), ) def testNumericArrayFieldRead(self): self.assertEqual( [1, 2, 3, 4], NumericArrayField(4, 2, ByteOrder.HiToLo, Sign.Unsigned).read( b"\x05\x00\x01\x00\x02\x00\x03\x00\x04", 1 ), ) def testFloatingPointArrayFieldRead(self): self.assertEqual( [0.5, 1.0, 1.5, 2.0], FloatingPointArrayField(4, 2, ByteOrder.HiToLo, Sign.Unsigned, 2.0).read( b"\x05\x00\x01\x00\x02\x00\x03\x00\x04", 1 ), ) class TestFieldFormatting(unittest.TestCase): def setUp(self): self._buffer = bytearray() def testByteFieldWrite(self): ByteField().write(ord("c"), self._buffer) self.assertEqual(b"c", self._buffer) def testBytesFieldWrite(self): BytesField(4).write(b"cdef", self._buffer) self.assertEqual(b"cdef", self._buffer) def testNumericFieldHiToLoWrite(self): NumericField(2, ByteOrder.HiToLo, Sign.Signed).write(1, self._buffer) self.assertEqual(b"\x00\x01", self._buffer) def testNumericFieldHiToLoSignedWrite(self): NumericField(2, ByteOrder.HiToLo, Sign.Signed).write(-1, self._buffer) self.assertEqual(b"\x80\x01", self._buffer) def testNumericFieldLoToHiWrite(self): NumericField(2, ByteOrder.LoToHi, Sign.Signed).write(256, self._buffer) self.assertEqual(b"\x00\x01", self._buffer) def testNumericFieldLoToHiSignedWrite(self): NumericField(2, ByteOrder.LoToHi, Sign.Signed).write(-256, self._buffer) self.assertEqual(b"\x00\x81", self._buffer) def testNumericFieldUnsignedMax(self): field = NumericField(2, ByteOrder.HiToLo, Sign.Unsigned) field.write(field.max, self._buffer) self.assertEqual(b"\xff\xff", self._buffer) def testNumericFieldSignedMax(self): field = NumericField(2, ByteOrder.HiToLo, Sign.Signed) field.write(field.max, self._buffer) self.assertEqual(b"\x7f\xff", self._buffer) def testFloatingPointFieldWrite(self): FloatingPointField(2, ByteOrder.HiToLo, Sign.Unsigned, 2.0).write( 0.5, self._buffer ) self.assertEqual(b"\x00\x01", self._buffer) def testFloatingPointFieldWriteDiv100(self): FloatingPointField(2, ByteOrder.HiToLo, Sign.Unsigned, 100.0).write( 1.16, self._buffer ) self.assertEqual(b"\x00\x74", self._buffer) def testDateTimeFieldWrite(self): DateTimeField().write(datetime(2020, 1, 1, 0, 0, 0), self._buffer) self.assertEqual(b"\x14\x01\x01\x00\x00\x00", self._buffer) def testArrayFieldWrite(self): ArrayField(4, NumericField(2, ByteOrder.HiToLo, Sign.Unsigned)).write( [1, 2, 3, 4], self._buffer ) self.assertEqual(b"\x00\x01\x00\x02\x00\x03\x00\x04", self._buffer) def testNumericArrayFieldWrite(self): NumericArrayField(4, 2, ByteOrder.HiToLo, Sign.Unsigned).write( [1, 2, 3, 4], self._buffer ) self.assertEqual(b"\x00\x01\x00\x02\x00\x03\x00\x04", self._buffer) def testFloatingPointArrayFieldWrite(self): FloatingPointArrayField(4, 2, ByteOrder.HiToLo, Sign.Unsigned, 2.0).write( [0.5, 1.0, 1.5, 2.0], self._buffer ) self.assertEqual(b"\x00\x01\x00\x02\x00\x03\x00\x04", self._buffer) sdwilsh-siobrultech-protocols-78196f6/tests/gem/test_packets.py000066400000000000000000000225371453451727300250270ustar00rootroot00000000000000import functools import unittest from siobrultech_protocols.gem import packets from tests.gem.packet_test_data import assert_packet, read_packet packet_maker = functools.partial( packets.Packet, packet_format=packets.BIN32_ABS, voltage=120.0, absolute_watt_seconds=[0] * packets.BIN32_NET.num_channels, device_id=123456, serial_number=123456, seconds=0, pulse_counts=[0] * packets.GEMPacketFormat.NUM_PULSE_COUNTERS, temperatures=list([0.0] * packets.GEMPacketFormat.NUM_TEMPERATURE_SENSORS), aux=None, ) class TestPacketFormats(unittest.TestCase): def test_bin32_abs(self): check_packet("BIN32-ABS.bin", packets.BIN32_ABS) def test_bin32_net(self): check_packet("BIN32-NET.bin", packets.BIN32_NET) def test_bin48_abs(self): check_packet("BIN48-ABS.bin", packets.BIN48_ABS) def test_bin48_net(self): check_packet("BIN48-NET.bin", packets.BIN48_NET) def test_bin48_net_time(self): check_packet("BIN48-NET-TIME.bin", packets.BIN48_NET_TIME) def test_bin48_net_time_tricky(self): """BIN48_NET and BIN48_NET_TIME packets both have the same packet type code, so in order to detect the difference you must try to parse as BIN48_NET first, and if that fails try BIN48_NET_TIME. However, if the parser just checks the checksum and not the footer, it's possible for a BIN48_NET_TIME packet to be mistaken for a BIN48_NET. This is one such packet.""" try: parse_packet("BIN48-NET-TIME_tricky.bin", packets.BIN48_NET) self.fail("Should have thrown") except packets.MalformedPacketException: pass check_packet("BIN48-NET-TIME_tricky.bin", packets.BIN48_NET_TIME) def test_ecm_1240(self): check_packet("ECM-1240.bin", packets.ECM_1240) def test_short_packet(self): packet = read_packet("BIN32-NET.bin") with self.assertRaisesRegex( packets.MalformedPacketException, "Packet too short." ): packets.BIN32_NET.parse(packet[:-1]) def test_packet_with_extra_after(self): data = bytearray() data.extend(read_packet("BIN32-NET.bin")) data.extend(read_packet("BIN32-ABS.bin")) packet = packets.BIN32_NET.parse(data) assert_packet("BIN32-NET.bin", packet) class TestPacketDeltaComputation(unittest.TestCase): def test_packet_delta_seconds(self): packet = parse_packet("BIN32-ABS.bin", packets.BIN32_ABS) self.assertEqual(997492, packet.seconds) self.assertEqual(997493, packet.delta_seconds(2**24 - 1)) self.assertEqual(1000000, packet.delta_seconds(2**24 - (1000000 - 997492))) def test_packet_delta_pulses(self): packet = parse_packet("BIN48-NET-TIME_tricky.bin", packets.BIN48_NET_TIME) # All the pulse counts in our packets are 0, so let's fake some out packet.pulse_counts = [100, 200, 300, 400] self.assertEqual( [1100, 1200, 1300, 1400], [ packet.delta_pulse_count(i, 2**24 - 1000) for i in range(0, len(packet.pulse_counts)) ], ) def test_packet_delta_aux(self): packet = parse_packet("ECM-1240.bin", packets.ECM_1240) self.assertEqual( [10, 3, 5, 3, 5], [packet.delta_aux_count(i, 60) for i in range(0, len(packet.aux))], ) def test_packet_delta_absolute_watt_seconds(self): packet = parse_packet("BIN32-ABS.bin", packets.BIN32_ABS) self.assertEqual( [ 3123664, 9249700, 195388151, 100917236, 7139112, 1440, 4, 3, 14645520, 111396601, 33259670, 38296448, 1108415, 2184858, 5191049, 1, 71032651, 60190845, 47638292, 12017483, 36186563, 14681918, 69832947, 37693, 60941899, 1685614, 902, 799182, 302590, 3190972, 5, 647375119, ], packet.absolute_watt_seconds, ) self.assertEqual( [ packet.absolute_watt_seconds[i] + 1000 for i in range(0, len(packet.absolute_watt_seconds)) ], [ packet.delta_absolute_watt_seconds(i, 2**40 - 1000) for i in range(0, len(packet.absolute_watt_seconds)) ], ) def test_packet_delta_polarized_watt_seconds(self): packet = parse_packet("BIN32-NET.bin", packets.BIN32_NET) # Packet didn't have any negative numbers, so let's do some manual ones packet.polarized_watt_seconds = [ -1600 + 100 * i for i in range(0, packet.num_channels) ] self.assertEqual( [ packet.polarized_watt_seconds[i] + 1000 + 2**39 for i in range(0, len(packet.polarized_watt_seconds)) ], [ packet.delta_polarized_watt_seconds(i, 2**39 - 1000) for i in range(0, len(packet.polarized_watt_seconds)) ], ) class TestPacketAverageComputation(unittest.TestCase): def test_packet_average_power_no_time_passed(self): packet_a = packet_maker( absolute_watt_seconds=[10] * packets.BIN32_ABS.num_channels, ) self.assertEqual(packet_a.get_average_power(0, packet_a), 0) def test_packet_average_power(self): packet_a = packet_maker( absolute_watt_seconds=[10] * packets.BIN32_ABS.num_channels, ) packet_b = packet_maker( absolute_watt_seconds=[20] * packets.BIN32_ABS.num_channels, seconds=10, ) self.assertEqual(packet_a.get_average_power(0, packet_b), 1.0) self.assertEqual(packet_b.get_average_power(0, packet_a), 1.0) def test_packet_average_power_net_metering_mixed(self): packet_a = packet_maker( absolute_watt_seconds=[10] * packets.BIN32_NET.num_channels, packet_format=packets.BIN32_NET, polarized_watt_seconds=[0] * packets.BIN32_NET.num_channels, ) packet_b = packet_maker( absolute_watt_seconds=[40] * packets.BIN32_NET.num_channels, packet_format=packets.BIN32_NET, polarized_watt_seconds=[10] * packets.BIN32_NET.num_channels, seconds=10, ) self.assertEqual(packet_a.get_average_power(0, packet_b), 1.0) self.assertEqual(packet_b.get_average_power(0, packet_a), 1.0) def test_produced_power(self): packet_a = packet_maker( absolute_watt_seconds=[0] * packets.BIN32_NET.num_channels, packet_format=packets.BIN32_NET, polarized_watt_seconds=[0] * packets.BIN32_NET.num_channels, ) packet_b = packet_maker( absolute_watt_seconds=[10] * packets.BIN32_NET.num_channels, packet_format=packets.BIN32_NET, polarized_watt_seconds=[10] * packets.BIN32_NET.num_channels, seconds=10, ) self.assertEqual(packet_a.get_average_power(0, packet_b), -1.0) self.assertEqual(packet_b.get_average_power(0, packet_a), -1.0) def test_pulse_rate(self): packet_a = packet_maker( pulse_counts=[0] * packets.BIN32_ABS.NUM_PULSE_COUNTERS, ) packet_b = packet_maker( pulse_counts=[15] * packets.BIN32_ABS.NUM_PULSE_COUNTERS, seconds=10, ) self.assertEqual(packet_a.get_average_pulse_rate(0, packet_b), 1.5) self.assertEqual(packet_b.get_average_pulse_rate(0, packet_a), 1.5) def test_pulse_rate_no_time_passed(self): packet_a = packet_maker( pulse_counts=[0] * packets.BIN32_ABS.NUM_PULSE_COUNTERS, ) self.assertEqual(packet_a.get_average_pulse_rate(0, packet_a), 0) def test_aux_rate(self): packet_a = packet_maker( packet_format=packets.ECM_1240, aux=[0] * packets.ECM_1240.num_aux_channels ) packet_b = packet_maker( packet_format=packets.ECM_1240, aux=[15] * packets.ECM_1240.num_aux_channels, seconds=10, ) self.assertEqual(packet_a.get_average_aux_rate_of_change(0, packet_b), 1.5) self.assertEqual(packet_b.get_average_aux_rate_of_change(0, packet_a), 1.5) def test_aux_rate_no_time_passed(self): packet_a = packet_maker( packet_format=packets.ECM_1240, aux=[0] * packets.ECM_1240.num_aux_channels ) self.assertEqual(packet_a.get_average_aux_rate_of_change(0, packet_a), 0) def check_packet(packet_file_name: str, packet_format: packets.PacketFormat): packet = parse_packet(packet_file_name, packet_format) assert_packet(packet_file_name, packet) raw_data = packet_format.format(packet) reparsed_packet = packet_format.parse(raw_data) assert_packet(packet_file_name, reparsed_packet) def parse_packet(packet_file_name: str, packet_format: packets.PacketFormat): return packet_format.parse(read_packet(packet_file_name)) if __name__ == "__main__": unittest.main() sdwilsh-siobrultech-protocols-78196f6/tests/gem/test_protocol.py000066400000000000000000000117331453451727300252320ustar00rootroot00000000000000import asyncio import logging import sys import unittest from siobrultech_protocols.gem.packets import BIN48_NET, Packet from siobrultech_protocols.gem.protocol import ( ConnectionLostMessage, ConnectionMadeMessage, PacketProtocol, PacketProtocolMessage, PacketReceivedMessage, ) from tests.gem.mock_transport import MockTransport from tests.gem.packet_test_data import assert_packet, read_packet, read_packets logging.basicConfig( stream=sys.stderr, level=logging.DEBUG, format="%(asctime)s [%(name)s](%(levelname)s) %(message)s", ) class TestPacketAccumulator(unittest.IsolatedAsyncioTestCase): def setUp(self): self._queue: asyncio.Queue[PacketProtocolMessage] = asyncio.Queue() self._transport = MockTransport() self._protocol = PacketProtocol(queue=self._queue) self._protocol.connection_made(self._transport) message = self._queue.get_nowait() assert isinstance(message, ConnectionMadeMessage) assert message.protocol is self._protocol def tearDown(self) -> None: if not self._transport.closed: exc = Exception("Test") self._protocol.connection_lost(exc=exc) message = self._queue.get_nowait() assert isinstance(message, ConnectionLostMessage) assert message.protocol is self._protocol assert message.exc is exc self._protocol.close() # Close after connection_lost is not required, but at least should not crash def testClose(self): self._protocol.close() assert self._transport.closed def test_single_packet(self): packet_data = read_packet("BIN32-ABS.bin") self._protocol.data_received(packet_data) packet = self.expect_packet_recieved() assert_packet("BIN32-ABS.bin", packet) def test_header_only(self): packet_data = read_packet("BIN32-ABS.bin") self._protocol.data_received(packet_data[:2]) with self.assertRaises(asyncio.queues.QueueEmpty): self._queue.get_nowait() self._protocol.data_received(packet_data[2:]) packet = self.expect_packet_recieved() assert_packet("BIN32-ABS.bin", packet) def test_partial_packet(self): packet_data = read_packet("BIN32-ABS.bin") self._protocol.data_received(packet_data[:100]) with self.assertRaises(asyncio.queues.QueueEmpty): self._queue.get_nowait() self._protocol.data_received(packet_data[100:]) packet = self.expect_packet_recieved() assert_packet("BIN32-ABS.bin", packet) def test_time_packet(self): packet_data = read_packet("BIN48-NET-TIME_tricky.bin") self._protocol.data_received(packet_data) packet = self.expect_packet_recieved() assert_packet("BIN48-NET-TIME_tricky.bin", packet) def test_partial_time_packet(self): packet_data = read_packet("BIN48-NET-TIME_tricky.bin") self._protocol.data_received(packet_data[: BIN48_NET.size]) with self.assertRaises(asyncio.queues.QueueEmpty): self._queue.get_nowait() self._protocol.data_received(packet_data[BIN48_NET.size :]) packet = self.expect_packet_recieved() assert_packet("BIN48-NET-TIME_tricky.bin", packet) def test_multiple_packets(self): packet_data = read_packets( ["BIN32-ABS.bin", "BIN32-NET.bin", "BIN48-NET.bin", "BIN48-NET-TIME.bin"] ) self._protocol.data_received(packet_data) packet = self.expect_packet_recieved() assert_packet("BIN32-ABS.bin", packet) packet = self.expect_packet_recieved() assert_packet("BIN32-NET.bin", packet) packet = self.expect_packet_recieved() assert_packet("BIN48-NET.bin", packet) packet = self.expect_packet_recieved() assert_packet("BIN48-NET-TIME.bin", packet) def test_multiple_packets_with_junk(self): self._protocol.data_received(read_packet("BIN32-ABS.bin")) self._protocol.data_received(bytes.fromhex("feff05")) self._protocol.data_received(read_packet("BIN32-NET.bin")) self._protocol.data_received(bytes.fromhex("feff01")) self._protocol.data_received(read_packet("BIN48-NET.bin")) self._protocol.data_received(bytes.fromhex("23413081afb134870dacea")) self._protocol.data_received(read_packet("BIN48-NET-TIME.bin")) packet = self.expect_packet_recieved() assert_packet("BIN32-ABS.bin", packet) packet = self.expect_packet_recieved() assert_packet("BIN32-NET.bin", packet) packet = self.expect_packet_recieved() assert_packet("BIN48-NET.bin", packet) packet = self.expect_packet_recieved() assert_packet("BIN48-NET-TIME.bin", packet) def expect_packet_recieved(self) -> Packet: message = self._queue.get_nowait() assert isinstance(message, PacketReceivedMessage) assert message.protocol is self._protocol return message.packet if __name__ == "__main__": unittest.main()