pax_global_header00006660000000000000000000000064150151036550014513gustar00rootroot0000000000000052 comment=9f40e3bb74c971248acba83de69b6452417ac92b xarray-ceos-alos2-2025.05.0/000077500000000000000000000000001501510365500153015ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/.github/000077500000000000000000000000001501510365500166415ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/.github/dependabot.yml000066400000000000000000000001701501510365500214670ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "monthly" xarray-ceos-alos2-2025.05.0/.github/release.yml000066400000000000000000000001141501510365500210000ustar00rootroot00000000000000changelog: exclude: authors: - dependabot - pre-commit-ci xarray-ceos-alos2-2025.05.0/.github/workflows/000077500000000000000000000000001501510365500206765ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/.github/workflows/ci.yaml000066400000000000000000000043641501510365500221640ustar00rootroot00000000000000name: CI on: push: branches: [main] pull_request: branches: [main] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: detect-skip-ci-trigger: name: "Detect CI Trigger: [skip-ci]" if: | github.repository == 'umr-lops/xarray-ceos-alos2' && github.event_name == 'push' || github.event_name == 'pull_request' runs-on: ubuntu-latest outputs: triggered: ${{ steps.detect-trigger.outputs.trigger-found }} steps: - uses: actions/checkout@v4 with: fetch-depth: 2 - uses: xarray-contrib/ci-trigger@v1 id: detect-trigger with: keyword: "[skip-ci]" ci: name: ${{ matrix.os }} py${{ matrix.python-version }} runs-on: ${{ matrix.os }} needs: detect-skip-ci-trigger if: needs.detect-skip-ci-trigger.outputs.triggered == 'false' defaults: run: shell: bash -l {0} strategy: fail-fast: false matrix: python-version: ["3.10", "3.12", "3.13"] os: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: - name: Checkout the repository uses: actions/checkout@v4 with: # need to fetch all tags to get a correct version fetch-depth: 0 # fetch all branches and tags - name: Setup environment variables run: | echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV echo "CONDA_ENV_FILE=ci/requirements/environment.yaml" >> $GITHUB_ENV - name: Setup micromamba uses: mamba-org/setup-micromamba@v2 with: environment-file: ${{ env.CONDA_ENV_FILE }} environment-name: xarray-ceos-alos2-tests cache-environment: true cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" create-args: >- python=${{matrix.python-version}} conda - name: Install xarray-ceos-alos2 run: | python -m pip install --no-deps -e . - name: Import xarray-ceos-alos2 run: | python -c "import ceos_alos2" - name: Run tests run: | python -m pytest --cov=ceos_alos2 xarray-ceos-alos2-2025.05.0/.github/workflows/pypi.yaml000066400000000000000000000024321501510365500225440ustar00rootroot00000000000000name: Upload Package to PyPI on: release: types: [created] jobs: build: name: Build packages runs-on: ubuntu-latest if: github.repository == 'umr-lops/xarray-ceos-alos2' steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install build twine - name: Build run: | python -m build --sdist --wheel --outdir dist/ . - name: Check the built archives run: | twine check dist/* - name: Upload build artifacts uses: actions/upload-artifact@v4 with: name: packages path: dist/* pypi-publish: name: Upload to PyPI runs-on: ubuntu-latest needs: build environment: name: pypi url: https://pypi.org/p/xarray-ceos-alos2 permissions: id-token: write steps: - name: Download build artifacts uses: actions/download-artifact@v4 with: name: packages path: dist/ - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc xarray-ceos-alos2-2025.05.0/.github/workflows/upstream-dev.yaml000066400000000000000000000050371501510365500242030ustar00rootroot00000000000000name: upstream-dev CI on: push: branches: [main] pull_request: branches: [main] schedule: - cron: "0 18 * * 0" # Weekly "On Sundays at 18:00" UTC workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: detect-test-upstream-trigger: name: "Detect CI Trigger: [test-upstream]" if: github.event_name == 'push' || github.event_name == 'pull_request' runs-on: ubuntu-latest outputs: triggered: ${{ steps.detect-trigger.outputs.trigger-found }} steps: - uses: actions/checkout@v4 with: fetch-depth: 2 - uses: xarray-contrib/ci-trigger@v1.2 id: detect-trigger with: keyword: "[test-upstream]" upstream-dev: name: upstream-dev runs-on: ubuntu-latest needs: detect-test-upstream-trigger if: | always() && github.repository == 'umr-lops/xarray-ceos-alos2' && ( github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.detect-test-upstream-trigger.outputs.triggered == 'true' || contains(github.event.pull_request.labels.*.name, 'run-upstream') ) defaults: run: shell: bash -l {0} strategy: fail-fast: false matrix: python-version: ["3.12"] steps: - name: checkout the repository uses: actions/checkout@v4 with: # need to fetch all tags to get a correct version fetch-depth: 0 # fetch all branches and tags - name: set up conda environment uses: mamba-org/setup-micromamba@v2 with: environment-file: ci/requirements/environment.yaml environment-name: tests create-args: >- python=${{ matrix.python-version }} pytest-reportlog conda - name: install upstream-dev dependencies run: bash ci/install-upstream-dev.sh - name: install the package run: python -m pip install --no-deps -e . - name: show versions run: python -m pip list - name: import run: | python -c 'import ceos_alos2' - name: run tests if: success() id: status run: | python -m pytest -rf --report-log=pytest-log.jsonl - name: report failures if: | failure() && steps.tests.outcome == 'failure' && github.event_name == 'schedule' uses: xarray-contrib/issue-from-pytest-log@v1 with: log-path: pytest-log.jsonl xarray-ceos-alos2-2025.05.0/.gitignore000066400000000000000000000003661501510365500172760ustar00rootroot00000000000000# editor files *~ \#*\# # python bytecode *.py[co] __pycache__/ # install artifacts /build /dist /*.egg-info # tools .ipynb_checkpoints/ .hypothesis/ .pytest_cache .coverage .coverage.* .cache /docs/_build/ /docs/generated/ /docs/warnings.log xarray-ceos-alos2-2025.05.0/.pre-commit-config.yaml000066400000000000000000000020561501510365500215650ustar00rootroot00000000000000ci: autoupdate_schedule: monthly # https://pre-commit.com/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-docstring-first - repo: https://github.com/psf/black rev: 25.1.0 hooks: - id: black - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: - id: blackdoc additional_dependencies: ["black==25.1.0"] - id: blackdoc-autoupdate-black - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.8 hooks: - id: ruff args: [--fix] - repo: https://github.com/kynan/nbstripout rev: 0.8.1 hooks: - id: nbstripout args: [--extra-keys=metadata.kernelspec metadata.language_info.version] - repo: https://github.com/rbubley/mirrors-prettier rev: v3.5.3 hooks: - id: prettier - repo: https://github.com/ComPWA/taplo-pre-commit rev: v0.9.3 hooks: - id: taplo-format - id: taplo-lint args: [--no-schema] xarray-ceos-alos2-2025.05.0/.readthedocs.yaml000066400000000000000000000007771501510365500205430ustar00rootroot00000000000000version: 2 build: os: ubuntu-lts-latest tools: python: "3.12" jobs: post_checkout: - (git --no-pager log --pretty="tformat:%s" -1 | grep -vqF "[skip-rtd]") || exit 183 - git fetch --unshallow || true pre_install: - git update-index --assume-unchanged docs/conf.py pre_build: - python -c "import ceos_alos2" python: install: - requirements: docs/requirements.txt - method: pip path: . sphinx: configuration: docs/conf.py fail_on_warning: true xarray-ceos-alos2-2025.05.0/LICENSE000066400000000000000000000020711501510365500163060ustar00rootroot00000000000000MIT License Copyright (c) 2023, xarray-alos2 developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. xarray-ceos-alos2-2025.05.0/README.md000066400000000000000000000005431501510365500165620ustar00rootroot00000000000000# xarray-ceos-alos2 Read ALOS2 CEOS files into `xarray.DataTree` objects. ## Installation From PyPI ```sh pip install xarray-ceos-alos2 ``` From conda-forge ```sh conda install -c conda-forge xarray-ceos-alos2 ``` ## Usage ```python import ceos_alos2 tree = ceos_alos2.open_alos2(url, chunks={}, backend_options={"requests_per_chunk": 4096}) ``` xarray-ceos-alos2-2025.05.0/ceos_alos2/000077500000000000000000000000001501510365500173325ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/__init__.py000066400000000000000000000002621501510365500214430ustar00rootroot00000000000000from importlib.metadata import version from ceos_alos2.xarray import open_alos2 # noqa: F401 try: __version__ = version("alos2") except Exception: __version__ = "999" xarray-ceos-alos2-2025.05.0/ceos_alos2/array.py000066400000000000000000000123341501510365500210250ustar00rootroot00000000000000from dataclasses import dataclass, field from typing import Any import numpy as np from tlz.itertoolz import cons, first, get, groupby, partition_all, second from ceos_alos2.utils import parse_bytes raw_dtypes = { "C*8": np.dtype([("real", ">f4"), ("imag", ">f4")]), "IU2": np.dtype(">u2"), } def parse_data(content, type_code): dtype = raw_dtypes.get(type_code) if dtype is None: raise ValueError(f"unknown type code: {type_code}") raw = np.frombuffer(content, dtype) if type_code == "C*8": return raw["real"] + 1j * raw["imag"] return raw def normalize_chunksize(chunksize, dim_size): if chunksize in (None, -1) or chunksize > dim_size: return dim_size return chunksize def determine_nearest_chunksize(sizes, reference_size): diff = np.cumsum(sizes) - reference_size index = np.argmin(abs(diff)) return index + 1 def compute_chunk_ranges(byte_ranges, chunks): partitioned = partition_all(chunks, byte_ranges) return { chunk_number: (min(map(first, ranges_)), max(map(second, ranges_))) for chunk_number, ranges_ in enumerate(partitioned) } def to_offset_size(ranges): return { index: {"offset": start, "size": stop - start} for index, (start, stop) in ranges.items() } def compute_chunk_offsets(byte_ranges, chunks): ranges = compute_chunk_ranges(byte_ranges, chunks) return to_offset_size(ranges) def compute_selected_ranges(byte_ranges, indexer): n_rows = len(byte_ranges) if isinstance(indexer, int): indexer = [indexer] if isinstance(indexer, slice): selected_rows = range(n_rows)[indexer] else: selected_rows = indexer return list(get(list(selected_rows), list(enumerate(byte_ranges)))) def groupby_chunks(byte_ranges, chunksize): grouped = groupby(lambda it: it[0] // chunksize, byte_ranges) return {key: [value for _, value in ranges] for key, ranges in grouped.items()} def merge_chunk_info(selected, chunk_offsets): return [(chunk_offsets[index], ranges) for index, ranges in selected.items()] def relocate_ranges(chunk_info, ranges): offset = chunk_info["offset"] return chunk_info, [(min_ - offset, max_ - offset) for min_, max_ in ranges] def extract_ranges(content, ranges): return [content[start:stop] for start, stop in ranges] def read_chunk(f, offset, size): f.seek(offset) return f.read(size) @dataclass(order=False, unsafe_hash=True) class Array: """2d array from chunked data""" # TODO: decide whether having a cached file object or fs instance and url are better # file location / access fs: Any = field(repr=False) url: str = field(repr=True) # data positions byte_ranges: list[tuple[int, int]] = field(repr=False) # array information shape: tuple[int, int] = field(repr=True) dtype: str | np.dtype = field(repr=True) # convert raw bytes to data type_code: str = field(repr=False) # chunk sizes: chunks in (rows, cols) records_per_chunk: int | None = field(repr=True, default=None) chunk_offsets: list[tuple[int, int]] = field(repr=False, init=False) def __post_init__(self): sizes = np.array([stop - start for start, stop in self.byte_ranges]) if self.records_per_chunk is None: self.records_per_chunk = 1024 elif isinstance(self.records_per_chunk, str): if self.records_per_chunk == "auto": size = 100 * 2**20 else: size = parse_bytes(self.records_per_chunk) self.records_per_chunk = determine_nearest_chunksize(sizes, size) else: self.records_per_chunk = normalize_chunksize(self.records_per_chunk, self.shape[0]) self.chunk_offsets = compute_chunk_offsets(self.byte_ranges, self.records_per_chunk) def __eq__(self, other): if type(self) is not type(other): return False return ( self.url == other.url and self.fs == other.fs and self.byte_ranges == other.byte_ranges and self.shape == other.shape and self.dtype == other.dtype and self.records_per_chunk == other.records_per_chunk and self.type_code == other.type_code ) def __getitem__(self, indexers): selected_ranges = compute_selected_ranges(self.byte_ranges, indexers[0]) grouped = groupby_chunks(selected_ranges, chunksize=self.records_per_chunk) merged = merge_chunk_info(grouped, chunk_offsets=self.chunk_offsets) tasks = [relocate_ranges(info, ranges) for info, ranges in merged] with self.fs.open(self.url, mode="rb") as f: data_ = [] for chunk_info, ranges in tasks: chunk = read_chunk(f, **chunk_info) raw_bytes = extract_ranges(chunk, ranges) chunk_data = [parse_data(part, type_code=self.type_code) for part in raw_bytes] data_.extend(chunk_data) data = np.stack(data_, axis=0) new_indexers = tuple(cons(slice(None), indexers[1:])) return data[new_indexers] @property def ndim(self): return len(self.shape) @property def chunks(self): return (self.records_per_chunk, *self.shape[1:]) xarray-ceos-alos2-2025.05.0/ceos_alos2/common.py000066400000000000000000000004361501510365500211770ustar00rootroot00000000000000from construct import Int8ub, Int32ub, Struct record_preamble = Struct( "record_sequence_number" / Int32ub, "first_record_subtype" / Int8ub, "record_type" / Int8ub, "second_record_subtype" / Int8ub, "third_record_subtype" / Int8ub, "record_length" / Int32ub, ) xarray-ceos-alos2-2025.05.0/ceos_alos2/datatypes.py000066400000000000000000000062131501510365500217040ustar00rootroot00000000000000import datetime from construct import Adapter, Struct from construct import PaddedString as PaddedString_ class AsciiInteger(Adapter): def __init__(self, n_bytes): base = PaddedString_(n_bytes, "ascii") super().__init__(base) def _decode(self, obj, context, path): stripped = obj.strip() if not stripped: return -1 return int(stripped) def _encode(self, obj, context, path): raise NotImplementedError class AsciiFloat(Adapter): def __init__(self, n_bytes): base = PaddedString_(n_bytes, "ascii") super().__init__(base) def _decode(self, obj, context, path): stripped = obj.strip() if not stripped: stripped = "nan" return float(stripped) def _encode(self, obj, context, path): raise NotImplementedError class AsciiComplex(Adapter): def __init__(self, n_bytes): base = Struct( "real" / AsciiFloat(n_bytes // 2), "imaginary" / AsciiFloat(n_bytes // 2), ) super().__init__(base) def _decode(self, obj, context, path): return obj.real + 1j * obj.imaginary def _encode(self, obj, context, path): raise NotImplementedError class PaddedString(Adapter): def __init__(self, n_bytes): base = PaddedString_(n_bytes, "ascii") super().__init__(base) def _decode(self, obj, context, path): return obj.strip() def _encode(self, obj, context, path): raise NotImplementedError class Factor(Adapter): def __init__(self, obj, factor): super().__init__(obj) self.factor = factor def _decode(self, obj, context, path): return obj * self.factor def _encode(self, obj, context, path): raise NotImplementedError class Metadata(Adapter): def __init__(self, obj, **kwargs): super().__init__(obj) self.attrs = kwargs def _decode(self, obj, context, path): return (obj, self.attrs) def _encode(self, obj, context, path): raise NotImplementedError class StripNullBytes(Adapter): def _decode(self, obj, context, path): return obj.strip(b"\x00") def _encode(self, obj, context, path): raise NotImplementedError class DatetimeYdms(Adapter): def _decode(self, obj, context, path): base = datetime.datetime(obj["year"], 1, 1) timedelta = datetime.timedelta( days=obj["day_of_year"] - 1, milliseconds=obj["milliseconds"] ) return base + timedelta def _encode(self, obj, context, path): raise NotImplementedError class DatetimeYdus(Adapter): def __init__(self, base, reference_date): self.reference_date = reference_date super().__init__(base) def _decode(self, obj, context, path): reference_date = ( self.reference_date(context) if callable(self.reference_date) else self.reference_date ) truncated = datetime.datetime.combine(reference_date.date(), datetime.time.min) return truncated + datetime.timedelta(microseconds=obj) def _encode(self, obj, context, path): raise NotImplementedError xarray-ceos-alos2-2025.05.0/ceos_alos2/decoders.py000066400000000000000000000115261501510365500215010ustar00rootroot00000000000000import re import dateutil.parser from tlz.dicttoolz import merge from tlz.functoolz import curry from tlz.functoolz import identity as passthrough from ceos_alos2.dicttoolz import valsplit scene_id_re = re.compile( r"""(?x) (?P[A-Z0-9]{5}) (?P[0-9]{5}) (?P[0-9]{4}) -(?P[0-9]{6}) """ ) product_id_re = re.compile( r"""(?x) (?P[A-Z]{3}) (?P[LR]) (?P1\.0|1\.1|1\.5|3\.1) (?P[GR_]) (?P[UL_]) (?P[AD]) """ ) scan_info_re = re.compile( r"""(?x) (?P[BF]) (?P[0-9]) """ ) fname_re = re.compile( r"""(?x) (?P[A-Z]{3}) (-(?P[HV]{2}))? -(?P[A-Z0-9]{14}-[0-9]{6}) -(?P[A-Z0-9._]{10}) (-(?P[BF][0-9]))? """ ) observation_modes = { "SBS": "spotlight mode", "UBS": "ultra-fine mode single polarization", "UBD": "ultra-fine mode dual polarization", "HBS": "high-sensitive mode single polarization", "HBD": "high-sensitive mode dual polarization", "HBQ": "high-sensitive mode full (quad.) polarimetry", "FBS": "fine mode single polarization", "FBD": "fine mode dual polarization", "FBQ": "fine mode full (quad.) polarimetry", "WBS": "ScanSAR nominal 14MHz mode single polarization", "WBD": "ScanSAR nominal 14MHz mode dual polarization", "WWS": "ScanSAR nominal 28MHz mode single polarization", "WWD": "ScanSAR nominal 28MHz mode dual polarization", "VBS": "ScanSAR wide mode single polarization", "VBD": "ScanSAR wide mode dual polarization", } observation_directions = {"L": "left looking", "R": "right looking"} processing_levels = { "1.0": "level 1.0", "1.1": "level 1.1", "1.5": "level 1.5", "3.1": "level 3.1", } processing_options = {"G": "geo-code", "R": "geo-reference", "_": "not specified"} map_projections = {"U": "UTM", "P": "PS", "M": "MER", "L": "LCC", "_": "not specified"} orbit_directions = {"A": "ascending", "D": "descending"} processing_methods = {"F": "full aperture_method", "B": "SPECAN method"} resampling_methods = {"NN": "nearest-neighbor", "BL": "bilinear", "CC": "cubic convolution"} processing_facilities = { "SCMO": "spacecraft control mission operation system", "EICS": "earth intelligence collection and sharing system", } def lookup(mapping, code): value = mapping.get(code) if value is None: raise ValueError(f"invalid code {code!r}") return value translations = { "observation_mode": curry(lookup, observation_modes), "observation_direction": curry(lookup, observation_directions), "processing_level": curry(lookup, processing_levels), "processing_option": curry(lookup, processing_options), "map_projection": curry(lookup, map_projections), "orbit_direction": curry(lookup, orbit_directions), "date": curry(dateutil.parser.parse, yearfirst=True, dayfirst=False), "mission_name": passthrough, "orbit_accumulation": passthrough, "scene_frame": passthrough, "processing_method": curry(lookup, processing_methods), "scan_number": passthrough, } def decode_scene_id(scene_id): match = scene_id_re.match(scene_id) if match is None: raise ValueError(f"invalid scene id: {scene_id}") groups = match.groupdict() try: return {name: translations[name](value) for name, value in groups.items()} except ValueError as e: raise ValueError(f"invalid scene id: {scene_id}") from e def decode_product_id(product_id): match = product_id_re.fullmatch(product_id) if match is None: raise ValueError(f"invalid product id: {product_id}") groups = match.groupdict() try: return {name: translations[name](value) for name, value in groups.items()} except ValueError as e: raise ValueError(f"invalid product id: {product_id}") from e def decode_scan_info(scan_info): if scan_info is None: return {} match = scan_info_re.fullmatch(scan_info) if match is None: raise ValueError(f"invalid scan info: {scan_info}") groups = match.groupdict() return {name: translations[name](value) for name, value in groups.items()} def decode_filename(fname): match = fname_re.fullmatch(fname) if match is None: raise ValueError(f"invalid file name: {fname}") parts = match.groupdict() translators = { "filetype": passthrough, "polarization": passthrough, "scene_id": decode_scene_id, "product_id": decode_product_id, "scan_info": decode_scan_info, } mapping = {name: translators[name](value) for name, value in parts.items()} scalars, mappings = valsplit(lambda x: not isinstance(x, dict), mapping) return scalars | merge(*mappings.values()) xarray-ceos-alos2-2025.05.0/ceos_alos2/dicttoolz.py000066400000000000000000000035521501510365500217240ustar00rootroot00000000000000import copy from tlz.dicttoolz import assoc as assoc_ from tlz.dicttoolz import assoc_in, get_in, keyfilter from tlz.functoolz import identity as passthrough from tlz.itertoolz import concat, groupby from ceos_alos2.utils import unique sentinel = object() def itemsplit(predicate, d): groups = groupby(predicate, d.items()) first = dict(groups.get(True, ())) second = dict(groups.get(False, ())) return first, second def valsplit(predicate, d): wrapper = lambda item: predicate(item[1]) return itemsplit(wrapper, d) def keysplit(predicate, d): wrapper = lambda item: predicate(item[0]) return itemsplit(wrapper, d) def assoc(key, value, d): return assoc_(d, key, value) def dissoc(keys, d): return keyfilter(lambda k: k not in keys, d) def zip_default(*mappings, default=None): all_keys = unique(concat(map(list, mappings))) return {k: [m.get(k, default) for m in mappings] for k in all_keys} def apply_to_items(funcs, mapping, default=passthrough): return {k: funcs.get(k, default)(v) for k, v in mapping.items()} def copy_items(instructions, mapping): new = mapping for dest, source in instructions.items(): value = get_in(source, mapping, default=sentinel) if value is sentinel: continue new = assoc_in(new, list(dest), value) return new def move_items(instructions, mapping): copied = copy.deepcopy(copy_items(instructions, mapping)) for *head, tail in instructions.values(): subset = get_in(list(head), copied, default=sentinel) if subset is sentinel: continue subset.pop(tail, None) return copied def key_exists(key, mapping): if "." in key: key = key.split(".") elif not isinstance(key, list): key = [key] value = get_in(key, mapping, default=sentinel) return value is not sentinel xarray-ceos-alos2-2025.05.0/ceos_alos2/hierarchy.py000066400000000000000000000076121501510365500216700ustar00rootroot00000000000000import copy import posixpath from collections.abc import Mapping from dataclasses import dataclass from typing import Any import numpy as np from numpy.typing import ArrayLike from tlz.dicttoolz import valfilter from ceos_alos2.array import Array @dataclass(frozen=True) class Variable: dims: str | list[str] data: Array | ArrayLike attrs: dict[str, Any] def __post_init__(self): if isinstance(self.dims, str): # normalize, need the hack super().__setattr__("dims", [self.dims]) def __eq__(self, other): if not isinstance(other, Variable): return False if self.dims != other.dims: return False if type(self.data) is not type(other.data): return False if self.attrs != other.attrs: return False if isinstance(self.data, Array): return self.data == other.data else: return np.all(self.data == other.data) @property def ndim(self): return self.data.ndim @property def shape(self): return self.data.shape @property def dtype(self): return self.data.dtype @property def chunks(self): if not isinstance(self.data, Array): return {} return dict(zip(self.dims, self.data.chunks)) @property def sizes(self): return dict(zip(self.dims, self.data.shape)) @dataclass class Group(Mapping): path: str | None url: str data: dict[str, "Group | Variable"] attrs: dict[str, Any] def __post_init__(self): if self.path is None: self.path = "/" # or raise self.data = {name: self._adjust_item(name, value) for name, value in self.data.items()} def _adjust_item(self, name, value): new_value = copy.copy(value) if not isinstance(value, Group): return new_value new_value.path = posixpath.join(self.path, name) if new_value.url is None: new_value.url = self.url new_value.data = { name: new_value._adjust_item(name, item) for name, item in new_value.data.items() } return new_value def __getitem__(self, item): return self.data[item] def __setitem__(self, item, value): self.data[item] = self._adjust_item(item, value) @property def name(self): if self.path == "/" or "/" not in self.path: return self.path _, name = self.path.rsplit("/", 1) return name def __len__(self): return len(self.data.keys()) def __iter__(self): yield from self.data.keys() @property def groups(self): return valfilter(lambda el: isinstance(el, Group), self.data) @property def variables(self): return valfilter(lambda el: isinstance(el, Variable), self.data) def __eq__(self, other): if not isinstance(other, Group): return False if self.path != other.path: return False if self.url != other.url: return False if list(self.variables) != list(other.variables): # same variable names return False if list(self.groups) != list(other.groups): return False if self.attrs != other.attrs: return False for name, var in self.variables.items(): if var == other.data[name]: continue return False for name, group in self.groups.items(): if group == other.data[name]: continue return False return True def decouple(self): return Group(path=self.path, url=self.url, data=self.variables, attrs=self.attrs) @property def subtree(self): yield self.path, self.decouple() for item in self.data.values(): if isinstance(item, Group): yield from item.subtree xarray-ceos-alos2-2025.05.0/ceos_alos2/io.py000066400000000000000000000031041501510365500203110ustar00rootroot00000000000000import fsspec from tlz.functoolz import curry from ceos_alos2 import sar_image from ceos_alos2.hierarchy import Group from ceos_alos2.sar_leader import open_sar_leader from ceos_alos2.summary import open_summary from ceos_alos2.volume_directory import open_volume_directory def open(path, *, storage_options={}, create_cache=False, use_cache=True, records_per_chunk=1024): mapper = fsspec.get_mapper(path, **storage_options) # read summary summary = open_summary(mapper, "summary.txt") filenames = summary["product_information"]["data_files"].attrs # read volume directory volume_directory = open_volume_directory(mapper, filenames["volume_directory"]) # read sar leader sar_leader = open_sar_leader(mapper, filenames["sar_leader"]) # read actual imagery imagery_groups = list( map( curry( sar_image.open_image, mapper, records_per_chunk=records_per_chunk, create_cache=create_cache, use_cache=use_cache, ), filenames["sar_imagery"], ) ) imagery = Group( "/imagery", url=mapper.root, data={group.name: group for group in imagery_groups}, attrs={} ) # read sar trailer subgroups = {"summary": summary, "metadata": sar_leader, "imagery": imagery} attrs = { "reference_document": ( "https://www.eorc.jaxa.jp/ALOS-2/en/doc/fdata/PALSAR-2_xx_Format_CEOS_E_f.pdf" ) } return Group(path="/", data=subgroups, url=mapper.root, attrs=volume_directory.attrs | attrs) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/000077500000000000000000000000001501510365500212615ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/__init__.py000066400000000000000000000027051501510365500233760ustar00rootroot00000000000000from ceos_alos2.array import Array from ceos_alos2.decoders import decode_filename from ceos_alos2.hierarchy import Variable from ceos_alos2.sar_image import caching from ceos_alos2.sar_image.caching import CachingError from ceos_alos2.sar_image.io import read_metadata from ceos_alos2.sar_image.metadata import transform_metadata def filename_to_groupname(path): info = decode_filename(path) scan_number = f"scan{info['scan_number']}" if "scan_number" in info else None polarization = info.get("polarization") parts = [polarization, scan_number] return "_".join([_ for _ in parts if _]) def open_image(mapper, path, *, use_cache=True, create_cache=False, records_per_chunk=None): if use_cache: try: return caching.read_cache(mapper, path, records_per_chunk=records_per_chunk) except CachingError: pass from fsspec.implementations.dirfs import DirFileSystem fs = DirFileSystem(path=mapper.root, fs=mapper.fs) with fs.open(path, mode="rb") as f: header, metadata = read_metadata(f, records_per_chunk) group, array_metadata = transform_metadata(header, metadata) group["data"] = Variable( dims=["rows", "columns"], data=Array(fs=fs, url=path, records_per_chunk=records_per_chunk, **array_metadata), attrs={}, ) group.path = filename_to_groupname(path) if create_cache: caching.create_cache(mapper, path, group) return group xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/__main__.py000066400000000000000000000001211501510365500233450ustar00rootroot00000000000000from ceos_alos2.sar_image.cli import main if __name__ == "__main__": main() xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/caching/000077500000000000000000000000001501510365500226555ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/caching/__init__.py000066400000000000000000000023761501510365500247760ustar00rootroot00000000000000import json from ceos_alos2.sar_image.caching.decoders import decode_hierarchy, postprocess from ceos_alos2.sar_image.caching.encoders import encode_hierarchy, preprocess from ceos_alos2.sar_image.caching.path import ( local_cache_location, remote_cache_location, ) class CachingError(FileNotFoundError): pass def encode(obj): encoded = encode_hierarchy(obj) return json.dumps(preprocess(encoded)) def decode(cache, records_per_chunk): partially_decoded = json.loads(cache, object_hook=postprocess) return decode_hierarchy(partially_decoded, records_per_chunk=records_per_chunk) def read_cache(mapper, path, records_per_chunk): remote = remote_cache_location(mapper.root, path) local = local_cache_location(mapper.root, path) if local.is_file(): return decode(local.read_text(), records_per_chunk=records_per_chunk) if remote in mapper: return decode(mapper[remote].decode(), records_per_chunk=records_per_chunk) raise CachingError(f"no cache found for {path}") def create_cache(mapper, path, data): local = local_cache_location(mapper.root, path) # ensure the directory exists local.parent.mkdir(exist_ok=True, parents=True) encoded = encode(data) local.write_text(encoded) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/caching/decoders.py000066400000000000000000000042221501510365500250170ustar00rootroot00000000000000import fsspec import numpy as np from tlz.dicttoolz import valmap from tlz.functoolz import curry from ceos_alos2.array import Array from ceos_alos2.hierarchy import Group, Variable def postprocess(obj): if obj.get("__type__") == "tuple": return tuple(obj["data"]) return obj def decode_datetime(obj): encoding = obj["encoding"] reference = np.array(encoding["reference"], dtype=obj["dtype"]) offsets = np.array(obj["data"], dtype=f"timedelta64[{encoding['units']}]") return reference + offsets def decode_array(encoded, records_per_chunk): def default_decode(obj): return np.array(obj["data"], dtype=obj["dtype"]) if encoded.get("__type__") == "array": dtype = np.dtype(encoded["dtype"]) decoders = {"M": decode_datetime} decoder = decoders.get(dtype.kind, default_decode) return decoder(encoded) mapper = fsspec.get_mapper(encoded["root"]) from fsspec.implementations.dirfs import DirFileSystem fs = DirFileSystem(path=mapper.root, fs=mapper.fs) type_code = encoded["type_code"] url = encoded["url"] shape = encoded["shape"] dtype = encoded["dtype"] byte_ranges = encoded["byte_ranges"] return Array( fs=fs, url=url, byte_ranges=byte_ranges, shape=shape, dtype=dtype, type_code=type_code, records_per_chunk=records_per_chunk, ) def decode_variable(encoded, records_per_chunk): data = decode_array(encoded["data"], records_per_chunk=records_per_chunk) return Variable(dims=encoded["dims"], data=data, attrs=encoded["attrs"]) def decode_group(encoded, records_per_chunk): data = valmap(curry(decode_hierarchy, records_per_chunk=records_per_chunk), encoded["data"]) return Group(path=encoded["path"], url=encoded["url"], data=data, attrs=encoded["attrs"]) def decode_hierarchy(encoded, records_per_chunk): type_ = encoded.get("__type__") decoders = { "group": decode_group, "variable": decode_variable, } decoder = decoders.get(type_) if decoder is None: return encoded return decoder(encoded, records_per_chunk=records_per_chunk) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/caching/encoders.py000066400000000000000000000044361501510365500250400ustar00rootroot00000000000000import numpy as np from tlz.dicttoolz import valmap from ceos_alos2.array import Array from ceos_alos2.hierarchy import Group, Variable def encode_timedelta(obj): units, _ = np.datetime_data(obj.dtype) return obj.astype("int64").tolist(), {"units": units} def encode_datetime(obj): units, _ = np.datetime_data(obj.dtype) reference = obj[0] encoding = {"reference": str(reference), "units": units} encoded = (obj - reference).astype("int64").tolist() return encoded, encoding def encode_array(obj): if isinstance(obj, Array): return { "__type__": "backend_array", "root": obj.fs.path, "url": obj.url, "shape": obj.shape, "dtype": str(obj.dtype), "byte_ranges": obj.byte_ranges, "type_code": obj.type_code, } def default_encode(obj): return obj.tolist(), {} encoders = { "m": encode_timedelta, "M": encode_datetime, } encoder = encoders.get(obj.dtype.kind, default_encode) encoded, encoding = encoder(obj) return { "__type__": "array", "dtype": str(obj.dtype), "data": encoded, "encoding": encoding, } def encode_variable(var): encoded_data = encode_array(var.data) return { "__type__": "variable", "dims": var.dims, "data": encoded_data, "attrs": var.attrs, } def encode_group(group): def encode_entry(obj): if isinstance(obj, Group): return encode_group(obj) else: return encode_variable(obj) encoded_data = valmap(encode_entry, group.data) return { "__type__": "group", "url": group.url, "data": encoded_data, "path": group.path, "attrs": group.attrs, } def encode_hierarchy(obj): if isinstance(obj, Group): return encode_group(obj) elif isinstance(obj, Variable): return encode_variable(obj) else: return obj def preprocess(data): if isinstance(data, dict): return valmap(preprocess, data) elif isinstance(data, list): return list(map(preprocess, data)) elif isinstance(data, tuple): return {"__type__": "tuple", "data": list(map(preprocess, data))} else: return data xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/caching/path.py000066400000000000000000000007751501510365500241740ustar00rootroot00000000000000import hashlib import platformdirs project_name = "xarray-ceos-alos2" cache_root = platformdirs.user_cache_path(project_name) def hashsum(data, algorithm="sha256"): m = hashlib.new(algorithm) m.update(data.encode()) return m.hexdigest() def local_cache_location(remote_root, path): _, fname = f"/{path}".rsplit("/", 1) cache_name = f"{fname}.index" return cache_root / hashsum(remote_root) / cache_name def remote_cache_location(remote_root, path): return f"{path}.index" xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/cli.py000066400000000000000000000032251501510365500224040ustar00rootroot00000000000000import argparse import pathlib import sys import fsspec from ceos_alos2.sar_image import caching, open_image def create_cache(image_path, cache_root, records_per_chunk): if not image_path.is_file(): raise FileNotFoundError(f"Cannot find image file at given path: {image_path}") if cache_root is not None and not cache_root.is_dir(): raise OSError(f"Cannot find the target cache root: {cache_root}") elif cache_root is None: cache_root = image_path.parent uri = image_path.parent.as_uri() mapper = fsspec.get_mapper(uri) path = image_path.name group = open_image( mapper, path, use_cache=False, create_cache=False, records_per_chunk=records_per_chunk ) encoded = caching.encode(group) target = cache_root / f"{path}.index" target.write_text(encoded) def main(): parser = argparse.ArgumentParser() parser.add_argument( "--rpc", nargs="?", type=int, default=4096, help="records-per-chunk size used to create the cache files", ) parser.add_argument( "image_path", type=pathlib.Path, help="image path to create a cache file for", ) parser.add_argument( "cache_root", nargs="?", type=pathlib.Path, default=None, help=( "Root path to the new cache file. By default, it is created" " in the same directory as the image file." ), ) args = parser.parse_args() try: create_cache(args.image_path, args.cache_root, records_per_chunk=args.rpc) except OSError as e: print(e.args[0], file=sys.stderr) sys.exit(1) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/enums.py000066400000000000000000000015651501510365500227710ustar00rootroot00000000000000from construct import Adapter, Enum, Int8ub, Int16ub, Int32ub, Int64ub class Flag(Adapter): bases = { 1: Int8ub, 2: Int16ub, 4: Int32ub, 8: Int64ub, } def __init__(self, size): base = self.bases.get(size) if base is None: raise ValueError(f"unsupported size: {size}") super().__init__(base) def _decode(self, obj, context, path): return bool(obj) def _encode(self, obj, context, path): return int(obj) sar_channel_id = Enum(Int16ub, single_polarization=1, dual_polarization=2, full_polarization=4) sar_channel_code = Enum(Int16ub, L=0, S=1, C=2, X=3, KU=4, KA=5) pulse_polarization = Enum(Int16ub, horizontal=0, vertical=1) chirp_type_designator = Enum(Int16ub, linear_fm_chirp=0, phase_modulators=1) platform_position_parameters_update = Enum(Int32ub, repeat=0, update=1) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/file_descriptor.py000066400000000000000000000075531501510365500250220ustar00rootroot00000000000000from construct import Struct from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiInteger, PaddedString file_descriptor_record = Struct( "preamble" / record_preamble, "ascii_ebcdic_flag" / PaddedString(2), "blanks1" / PaddedString(2), "format_control_document_id" / PaddedString(12), "format_control_document_revision_level" / PaddedString(2), "file_design_descriptor_revision_letter" / PaddedString(2), "software_release_and_revision_number" / PaddedString(12), "file_number" / AsciiInteger(4), "file_id" / PaddedString(16), "record_sequence_and_location_type_flag" / PaddedString(4), "location_sequence_number" / AsciiInteger(8), "field_length_of_sequence_number" / AsciiInteger(4), "record_code_and_location_type_flag" / PaddedString(4), "record_code_location" / AsciiInteger(8), "record_code_field_length" / AsciiInteger(4), "record_length_and_location_type_flag" / PaddedString(4), "record_length_location" / AsciiInteger(8), "record_length_field_length" / AsciiInteger(4), "reserved1" / PaddedString(1), "reserved2" / PaddedString(1), "reserved3" / PaddedString(1), "reserved4" / PaddedString(1), "blanks6" / PaddedString(64), "number_of_sar_data_records" / AsciiInteger(6), "sar_data_record_length" / AsciiInteger(6), "reserved5" / PaddedString(24), "sample_group_data" / Struct( "bit_length_per_sample" / AsciiInteger(4), "number_of_samples_per_data_group" / AsciiInteger(4), "number_of_bytes_per_data_group" / AsciiInteger(4), "justification_and_order_of_samples_within_data_group" / PaddedString(4), ), "sar_related_data_in_the_record" / Struct( "number_of_sar_channels" / AsciiInteger(4), "number_of_lines_per_dataset" / AsciiInteger(8), "number_of_left_border_pixels_per_line" / AsciiInteger(4), "number_of_data_groups_per_line" / AsciiInteger(8), "number_of_right_border_pixels_per_line" / AsciiInteger(4), "number_of_top_border_lines" / AsciiInteger(4), "number_of_bottom_border_lines" / AsciiInteger(4), "interleaving_id" / PaddedString(4), ), "record_data_in_the_file" / Struct( "number_of_physical_records_per_line" / AsciiInteger(2), "number_of_physical_records_per_multichannel_line_in_this_file" / AsciiInteger(2), "number_of_bytes_of_prefix_data_per_record" / AsciiInteger(4), "number_of_bytes_of_sar_data_per_record" / AsciiInteger(8), "number_of_bytes_of_suffix_data_per_record" / AsciiInteger(4), "prefix_suffix_repeat_flag" / PaddedString(4), ), "prefix_suffix_data_locators" / Struct( "sample_data_line_number_locator" / PaddedString(8), "sar_channel_number_locator" / PaddedString(8), "time_of_sar_data_line_locator" / PaddedString(8), "left_fill_count_locator" / PaddedString(8), "right_fill_count_locator" / PaddedString(8), "pad_pixels_present_indicator" / PaddedString(4), "blanks" / PaddedString(28), "sar_data_line_quality_code_locator" / PaddedString(8), "calibration_information_field_locator" / PaddedString(8), "gain_values_field_locator" / PaddedString(8), "bias_values_field_locator" / PaddedString(8), "sar_data_format_type_indicator" / PaddedString(28), "sar_data_format_type_code" / PaddedString(4), "number_of_left_fill_bits_within_pixel" / AsciiInteger(4), "number_of_right_fill_bits_within_pixel" / AsciiInteger(4), "maximum_data_range_of_pixel" / AsciiInteger(8), "number_of_burst_data" / AsciiInteger(4), "number_of_lines_per_burst" / AsciiInteger(4), ), "scansar_burst_data_information" / Struct( "number_of_overlap_lines_with_adjacent_bursts" / AsciiInteger(4), "blanks" / PaddedString(260), ), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/io.py000066400000000000000000000043631501510365500222500ustar00rootroot00000000000000import itertools import math from tlz.itertoolz import concat from ceos_alos2.common import record_preamble from ceos_alos2.sar_image.file_descriptor import file_descriptor_record from ceos_alos2.sar_image.processed_data import processed_data_record from ceos_alos2.sar_image.signal_data import signal_data_record from ceos_alos2.utils import to_dict record_types = { 10: signal_data_record, 11: processed_data_record, } def parse_chunk(content, element_size): n_elements = len(content) // element_size if n_elements * element_size != len(content): raise ValueError( f"sizes mismatch: chunksize is {n_elements * element_size}" f" but got {len(content)} bytes" ) record_type = record_preamble.parse(content[:12]).record_type data_record = record_types.get(record_type) if data_record is None: raise ValueError(f"unknown record type code: {record_type}") parser = data_record[n_elements] return list(parser.parse(content)) def _adjust_offset(record, offset): record.record_start += offset record.data.start += offset record.data.stop += offset return record def adjust_offsets(records, offset): return [_adjust_offset(record, offset) for record in records] def read_file_descriptor(f): return file_descriptor_record.parse(f.read(720)) def read_metadata(f, records_per_chunk=1024): header = read_file_descriptor(f) n_records = header["number_of_sar_data_records"] record_size = header["sar_data_record_length"] n_chunks = math.ceil(n_records / records_per_chunk) chunksizes = [ ( records_per_chunk if records_per_chunk * (index + 1) <= n_records else n_records - records_per_chunk * index ) for index in range(n_chunks) ] chunk_offsets = [ offset * record_size + 720 for offset in itertools.accumulate(chunksizes, initial=0) ] raw_metadata = ( parse_chunk(f.read(chunksize * record_size), record_size) for chunksize in chunksizes ) adjusted = ( adjust_offsets(records, offset=offset) for records, offset in zip(raw_metadata, chunk_offsets) ) metadata = list(concat(adjusted)) return to_dict(header), to_dict(metadata) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/metadata.py000066400000000000000000000107601501510365500234170ustar00rootroot00000000000000import math import numpy as np from tlz.dicttoolz import keyfilter, merge_with, valfilter, valmap from tlz.functoolz import compose_left, curry, pipe from tlz.itertoolz import cons, first, second from ceos_alos2.dicttoolz import apply_to_items, dissoc, keysplit from ceos_alos2.transformers import as_group, remove_spares, separate_attrs from ceos_alos2.utils import remove_nesting_layer, rename, starcall def extract_format_type(header): return header["prefix_suffix_data_locators"]["sar_data_format_type_code"] def extract_shape(header): return ( header["sar_related_data_in_the_record"]["number_of_lines_per_dataset"], header["sar_related_data_in_the_record"]["number_of_data_groups_per_line"], ) def extract_attrs(header): # valid attrs: # - pixel range (level 1.5) # - burst data per file (level 1.1 specan) # - lines per burst (level 1.1 specan) # - overlap lines with adjacent bursts (level 1.1 specan) ignored = ["preamble"] known_attrs = { "interleaving_id", "maximum_data_range_of_pixel", "number_of_burst_data", "number_of_lines_per_burst", "number_of_overlap_lines_with_adjacent_bursts", } transformers = { "maximum_data_range_of_pixel": lambda v: [0, v] if not math.isnan(v) else [], "number_of_burst_data": lambda v: v if v != -1 else [], "number_of_lines_per_burst": lambda v: v if v != -1 else [], "number_of_overlap_lines_with_adjacent_bursts": lambda v: v if v != -1 else [], } translations = { "maximum_data_range_of_pixel": "valid_range", } return pipe( header, curry(dissoc, ignored), curry(remove_nesting_layer), curry(keyfilter, lambda k: k in known_attrs), curry(apply_to_items, transformers), curry(rename, translations=translations), curry(valfilter, lambda v: not isinstance(v, list) or v), ) def apply_overrides(dtype_overrides, mapping): def _apply(v, dtype): dims, data, attrs = v return dims, np.array(data, dtype=dtype), attrs return { k: v if k not in dtype_overrides else _apply(v, dtype_overrides[k]) for k, v in mapping.items() } def deduplicate_attrs(known, mapping): variables, attrs = keysplit(lambda k: k not in known, mapping) return variables | valmap(compose_left(second, first), attrs) def transform_line_metadata(metadata): ignored = [ "preamble", "record_start", "actual_count_of_left_fill_pixels", "actual_count_of_right_fill_pixels", "actual_count_of_data_pixels", "alos2_frame_number", "palsar_auxiliary_data", "data", ] translations = { "sar_image_data_line_number": "rows", } dtype_overrides = { "sensor_acquisition_date": "datetime64[ns]", "sensor_acquisition_date_microseconds": "datetime64[ns]", } known_attrs = { "sar_image_data_record_index", "sensor_parameters_update_flag", "scan_id", "sar_channel_code", "sar_channel_id", "onboard_range_compressed_flag", "chirp_type_designator", "platform_position_parameters_update_flag", "alos2_frame_number", "geographic_reference_parameter_update_flag", "transmitted_pulse_polarization", "received_pulse_polarization", } merged = pipe( metadata, curry(starcall, curry(merge_with, list)), curry(remove_spares), curry(dissoc, ignored), curry(valmap, compose_left(separate_attrs, curry(cons, "rows"), tuple)), curry(deduplicate_attrs, known_attrs), curry(apply_overrides, dtype_overrides), curry(rename, translations=translations), curry(as_group), ) return merged dtypes = { "C*8": np.dtype("complex64"), "IU2": np.dtype("uint16"), } def transform_metadata(header, metadata): byte_ranges = [(m["data"]["start"], m["data"]["stop"]) for m in metadata] type_code = extract_format_type(header) shape = extract_shape(header) dtype = dtypes.get(type_code) if dtype is None: raise ValueError(f"unknown type code: {type_code}") header_attrs = extract_attrs(header) group = transform_line_metadata(metadata) group.attrs |= header_attrs | {"coordinates": list(group.variables)} array_metadata = { "type_code": type_code, "shape": shape, "dtype": str(dtype), "byte_ranges": byte_ranges, } return group, array_metadata xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/processed_data.py000066400000000000000000000062061501510365500246170ustar00rootroot00000000000000from construct import Bytes, Computed, Int32ub, Seek, Struct, Tell, this from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import DatetimeYdms, Factor, Metadata, StripNullBytes from ceos_alos2.sar_image.enums import ( pulse_polarization, sar_channel_code, sar_channel_id, ) processed_data_record = Struct( "record_start" / Tell, "preamble" / record_preamble, "sar_image_data_line_number" / Int32ub, "sar_image_data_record_index" / Int32ub, "actual_count_of_left_fill_pixels" / Int32ub, "actual_count_of_data_pixels" / Int32ub, "actual_count_of_right_fill_pixels" / Int32ub, "sensor_parameters_update_flag" / Int32ub, "sensor_acquisition_date" / DatetimeYdms( Struct( "year" / Int32ub, "day_of_year" / Int32ub, "milliseconds" / Int32ub, ) ), "sar_channel_id" / sar_channel_id, "sar_channel_code" / sar_channel_code, "transmitted_pulse_polarization" / pulse_polarization, "received_pulse_polarization" / pulse_polarization, "prf" / Metadata(Int32ub, units="mHz"), "scan_id" / Int32ub, "slant_range_to_first_pixel" / Metadata(Int32ub, units="m"), "slant_range_to_mid_pixel" / Metadata(Int32ub, units="m"), "slant_range_to_last_pixel" / Metadata(Int32ub, units="m"), "doppler_centroid_value_at_first_pixel" / Metadata(Factor(Int32ub, 1e-3), units="Hz"), "doppler_centroid_value_at_mid_pixel" / Metadata(Factor(Int32ub, 1e-3), units="Hz"), "doppler_centroid_value_at_last_pixel" / Metadata(Factor(Int32ub, 1e-3), units="Hz"), "azimuth_fm_rate_of_first_pixel" / Metadata(Int32ub, units="Hz/ms"), "azimuth_fm_rate_of_mid_pixel" / Metadata(Int32ub, units="Hz/ms"), "azimuth_fm_rate_of_last_pixel" / Metadata(Int32ub, units="Hz/ms"), "look_angle_of_nadir" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "azimuth_squint_angle" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "blanks1" / StripNullBytes(Bytes(20)), "geographic_reference_parameter_update_flag" / Int32ub, "latitude_of_first_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "latitude_of_center_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "latitude_of_last_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "longitude_of_first_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "longitude_of_center_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "longitude_of_last_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "northing_of_first_pixel" / Metadata(Int32ub, units="m"), "blanks2" / StripNullBytes(Bytes(4)), "northing_of_last_pixel" / Metadata(Int32ub, units="m"), "easting_of_first_pixel" / Metadata(Int32ub, units="m"), "blanks3" / StripNullBytes(Bytes(4)), "easting_of_last_pixel" / Metadata(Int32ub, units="m"), "line_heading" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "blanks4" / StripNullBytes(Bytes(8)), "data" / Struct( "start" / Tell, "size" / Computed(this._.preamble.record_length - (this.start - this._.record_start)), "stop" / Seek(this._.record_start + this._.preamble.record_length), ), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_image/signal_data.py000066400000000000000000000103521501510365500241020ustar00rootroot00000000000000from construct import Bytes, Computed, Int32ub, Int64ub, Seek, Struct, Tell, this from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import ( DatetimeYdms, DatetimeYdus, Factor, Metadata, StripNullBytes, ) from ceos_alos2.sar_image.enums import ( Flag, chirp_type_designator, platform_position_parameters_update, pulse_polarization, sar_channel_code, sar_channel_id, ) signal_data_record = Struct( "record_start" / Tell, "preamble" / record_preamble, "sar_image_data_line_number" / Int32ub, "sar_image_data_record_index" / Int32ub, "actual_count_of_left_fill_pixels" / Int32ub, "actual_count_of_data_pixels" / Int32ub, "actual_count_of_right_fill_pixels" / Int32ub, "sensor_parameters_update_flag" / Int32ub, "sensor_acquisition_date" / DatetimeYdms( Struct( "year" / Int32ub, "day_of_year" / Int32ub, "milliseconds" / Int32ub, ) ), "sar_channel_id" / sar_channel_id, "sar_channel_code" / sar_channel_code, "transmitted_pulse_polarization" / pulse_polarization, "received_pulse_polarization" / pulse_polarization, "prf" / Metadata(Int32ub, units="mHz"), "scan_id" / Int32ub, "onboard_range_compressed_flag" / Flag(2), "chirp_type_designator" / chirp_type_designator, "chirp_length" / Metadata(Int32ub, units="ns"), "chirp_constant_coefficient" / Metadata(Int32ub, units="Hz"), "chirp_linear_coefficient" / Metadata(Int32ub, units="Hz/µs"), "chirp_quadratic_coefficient" / Metadata(Int32ub, units="Hz/µs^2"), "sensor_acquisition_date_microseconds" / DatetimeYdus(Int64ub, this.sensor_acquisition_date), "receiver_gain" / Metadata(Int32ub, units="dB"), "invalid_line_flag" / Flag(4), "elevation_angle_at_nadir_of_antenna" / Struct( "electronic" / Metadata(Int32ub, units="deg"), "mechanic" / Metadata(Int32ub, units="deg"), ), "antenna_squint_angle" / Struct( "electronic" / Metadata(Int32ub, units="deg"), "mechanic" / Metadata(Int32ub, units="deg"), ), "slant_range_to_first_data_sample" / Metadata(Int32ub, units="m"), "data_record_window_position" / Metadata(Int32ub, units="ns"), "blanks1" / Int32ub, "platform_position_parameters_update_flag" / platform_position_parameters_update, "platform_latitude" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "platform_longitude" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "platform_altitude" / Metadata(Int32ub, units="deg"), "platform_ground_speed" / Metadata(Int32ub, units="cm/s"), "platform_velocity" / Struct( "x" / Metadata(Int32ub, units="cm/s"), "y" / Metadata(Int32ub, units="cm/s"), "z" / Metadata(Int32ub, units="cm/s"), ), "platform_acceleration" / Struct( "x" / Metadata(Int32ub, units="cm/s^2"), "y" / Metadata(Int32ub, units="cm/s^2"), "z" / Metadata(Int32ub, units="cm/s^2"), ), "platform_track_angle" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "platform_true_track_angle" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "platform_attitude" / Struct( "pitch" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "roll" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "yaw" / Metadata(Factor(Int32ub, 1e-6), units="deg"), ), "latitude_of_first_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "latitude_of_center_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "latitude_of_last_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "longitude_of_first_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "longitude_of_center_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "longitude_of_last_pixel" / Metadata(Factor(Int32ub, 1e-6), units="deg"), "burst_number" / Int32ub, "line_number_in_this_burst" / Int32ub, "blanks2" / StripNullBytes(Bytes(60)), "alos2_frame_number" / Int32ub, "palsar_auxiliary_data" / StripNullBytes(Bytes(256)), "data" / Struct( "start" / Tell, "size" / Computed(this._.preamble.record_length - (this.start - this._.record_start)), "stop" / Seek(this._.record_start + this._.preamble.record_length), ), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/000077500000000000000000000000001501510365500214335ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/__init__.py000066400000000000000000000001031501510365500235360ustar00rootroot00000000000000from ceos_alos2.sar_leader.io import open_sar_leader # noqa: F401 xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/attitude.py000066400000000000000000000057531501510365500236420ustar00rootroot00000000000000import numpy as np from construct import Struct, this from tlz.dicttoolz import valmap from tlz.functoolz import curry, pipe from tlz.itertoolz import cons, get from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiFloat, AsciiInteger, Metadata, PaddedString from ceos_alos2.dicttoolz import apply_to_items, copy_items, dissoc from ceos_alos2.transformers import as_group, separate_attrs, transform_nested attitude_point = Struct( "time" / Struct( "day_of_year" / AsciiInteger(4), "millisecond_of_day" / AsciiInteger(8), ), "attitude" / Struct( "pitch_error" / AsciiInteger(4), "roll_error" / AsciiInteger(4), "yaw_error" / AsciiInteger(4), "pitch" / Metadata(AsciiFloat(14), units="deg"), "roll" / Metadata(AsciiFloat(14), units="deg"), "yaw" / Metadata(AsciiFloat(14), units="deg"), ), "rates" / Struct( "pitch_error" / AsciiInteger(4), "roll_error" / AsciiInteger(4), "yaw_error" / AsciiInteger(4), "pitch" / Metadata(AsciiFloat(14), units="deg/s"), "roll" / Metadata(AsciiFloat(14), units="deg/s"), "yaw" / Metadata(AsciiFloat(14), units="deg/s"), ), ) attitude_record = Struct( "preamble" / record_preamble, "number_of_points" / AsciiInteger(4), "data_points" / attitude_point[this.number_of_points], "blanks" / PaddedString(this.preamble.record_length - (12 + 4 + this.number_of_points * 120)), ) def transform_time(mapping): # no year information, so we have to convert to timedelta units = {"day_of_year": "D", "millisecond_of_day": "ms"} transformed = {k: np.asarray(v, dtype=f"timedelta64[{units[k]}]") for k, v in mapping.items()} return (transformed["day_of_year"] + transformed["millisecond_of_day"]).astype( "timedelta64[ns]" ) def prepend_dim(dim, var): if isinstance(var, dict): return valmap(curry(prepend_dim, dim), var) if not isinstance(var, tuple): var = (var, {}) return tuple(cons(dim, var)) def transform_section(mapping): transformers = { "pitch": separate_attrs, "roll": separate_attrs, "yaw": separate_attrs, "pitch_error": lambda data: list(map(bool, data)), "roll_error": lambda data: list(map(bool, data)), "yaw_error": lambda data: list(map(bool, data)), } return apply_to_items(transformers, mapping) def transform_attitude(mapping): transformers = { "time": transform_time, "attitude": transform_section, "rates": transform_section, } result = pipe( mapping, curry(get, "data_points"), curry(transform_nested), curry(apply_to_items, transformers), curry(prepend_dim, "points"), curry(copy_items, {("attitude", "time"): ["time"], ("rates", "time"): ["time"]}), curry(dissoc, ["time"]), curry(valmap, lambda x: (x, {"coordinates": ["time"]})), curry(as_group), ) return result xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/data_quality_summary.py000066400000000000000000000072631501510365500262530ustar00rootroot00000000000000from construct import Struct, this from tlz.dicttoolz import valmap from tlz.functoolz import compose_left, curry, pipe from tlz.itertoolz import cons, get from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiFloat, AsciiInteger, Metadata, PaddedString from ceos_alos2.dicttoolz import apply_to_items, dissoc from ceos_alos2.transformers import ( as_group, remove_spares, separate_attrs, transform_nested, ) calibration_uncertainty = Struct( "magnitude" / Metadata(AsciiFloat(16), units="dB"), "phase" / Metadata(AsciiFloat(16), units="deg"), ) misregistration_error = Struct( "along_track" / Metadata(AsciiFloat(16), units="m"), "across_track" / Metadata(AsciiFloat(16), units="m"), ) data_quality_summary_record = Struct( "preamble" / record_preamble, "record_number" / AsciiInteger(4), "sar_channel_id" / PaddedString(4), "date_of_the_last_calibration_update" / PaddedString(6), "number_of_channels" / AsciiInteger(4), "absolute_radiometric_data_quality" / Struct( "islr" / Metadata(AsciiFloat(16), units="dB"), "pslr" / Metadata(AsciiFloat(16), units="dB"), "azimuth_ambiguity_rate" / AsciiFloat(16), "range_ambiguity_rate" / AsciiFloat(16), "estimate_of_snr" / Metadata(AsciiFloat(16), units="dB"), "ber" / Metadata(AsciiFloat(16), units="dB"), "slant_range_resolution" / Metadata(AsciiFloat(16), units="m"), "azimuth_resolution" / Metadata(AsciiFloat(16), units="m"), "radiometric_resolution" / Metadata(AsciiFloat(16), units="dB"), "instantaneous_dynamic_range" / Metadata(AsciiFloat(16), units="dB"), "nominal_absolute_radiometric_calibration_uncertainty" / calibration_uncertainty, ), "relative_radiometric_quality" / Struct( # TODO: does that actually make sense? "nominal_relative_radiometric_calibration_uncertainty" / calibration_uncertainty[this._.number_of_channels], "blanks" / PaddedString(512 - this._.number_of_channels * 32), ), "absolute_geometric_quality" / Struct( "absolute_location_error" / Struct( "along_track" / Metadata(AsciiFloat(16), units="m"), "across_track" / Metadata(AsciiFloat(16), units="m"), ), "geometric_distortion_scale" / Struct( "line_direction" / AsciiFloat(16), "pixel_direction" / AsciiFloat(16), ), "geometric_distortion_skew" / AsciiFloat(16), "scene_orientation_error" / AsciiFloat(16), ), "relative_geometric_quality" / Struct( # TODO: does that actually make sense? "relative_misregistration_error" / misregistration_error[this._.number_of_channels], # TODO: 534 is 16 more than stated in the reference... is this on us or on JAXA? "blanks" / PaddedString(534 + (8 - this._.number_of_channels) * 32), ), ) def transform_relative(mapping, key): return pipe( mapping, curry(get, key), curry(transform_nested), curry(valmap, compose_left(separate_attrs, curry(cons, "channel"), tuple)), ) def transform_data_quality_summary(mapping): ignored = ["preamble", "record_number"] transformers = { "relative_radiometric_quality": curry( transform_relative, key="nominal_relative_radiometric_calibration_uncertainty" ), "relative_geometric_quality": curry( transform_relative, key="relative_misregistration_error" ), } result = pipe( mapping, curry(remove_spares), curry(dissoc, ignored), curry(apply_to_items, transformers), curry(as_group), ) return result xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/dataset_summary.py000066400000000000000000000262611501510365500252160ustar00rootroot00000000000000from construct import Enum, Struct from tlz.functoolz import curry, pipe from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import ( AsciiFloat, AsciiInteger, Factor, Metadata, PaddedString, ) from ceos_alos2.dicttoolz import apply_to_items, dissoc from ceos_alos2.transformers import as_group, normalize_datetime, remove_spares from ceos_alos2.utils import rename motion_compensation = Enum( AsciiInteger(2), no_compensation=0, on_board_compensation=1, in_processor_compensation=10, both=11, ) chirp_extraction_index = Enum(AsciiInteger(8), linear_up=0, linear_down=1, linear_up_and_down=2) flag = Enum(PaddedString(4), yes="YES", no="NO", on="ON", off="OFF") weighting_functions = Enum(PaddedString(32), rectangle="1") dataset_summary_record = Struct( "preamble" / record_preamble, "dataset_summary_records_sequence_number" / AsciiInteger(4), "sar_channel_id" / PaddedString(4), "scene_id" / PaddedString(32), "number_of_scene_reference" / PaddedString(16), "scene_center_time" / PaddedString(32), "spare1" / PaddedString(16), "geodetic_latitude" / Metadata(AsciiFloat(16), units="deg"), "geodetic_longitude" / Metadata(AsciiFloat(16), units="deg"), "processed_scene_center_true_heading" / Metadata(AsciiFloat(16), units="deg"), "ellipsoid_designator" / PaddedString(16), "ellipsoid_semimajor_axis" / Metadata(AsciiFloat(16), units="km"), "ellipsoid_semiminor_axis" / Metadata(AsciiFloat(16), units="km"), "earth_mass" / Metadata(Factor(AsciiFloat(16), 1e24), units="kg"), "gravitational_constant" / Metadata(Factor(AsciiFloat(16), 1e-14), units="m^3 / s^2"), "ellipsoid_j2_parameter" / AsciiFloat(16), "ellipsoid_j3_parameter" / AsciiFloat(16), "ellipsoid_j4_parameter" / AsciiFloat(16), "spare2" / PaddedString(16), "average_terrain_height_above_ellipsoid_at_scene_center" / AsciiFloat(16), "scene_center_line_number" / AsciiInteger(8), "scene_center_pixel_number" / AsciiInteger(8), "processing_scene_length" / Metadata(AsciiFloat(16), units="km"), "processing_scene_width" / Metadata(AsciiFloat(16), units="km"), "spare3" / PaddedString(16), "number_of_sar_channel" / AsciiInteger(4), "spare4" / PaddedString(4), "sensor_platform_mission_identifier" / PaddedString(16), "sensor_id_and_operation_mode" / PaddedString(32), "orbit_number_or_flight_line_indicator" / AsciiInteger(8), "sensor_platform_geodetic_latitude_at_nadir_corresponding_to_scene_center" / Metadata(AsciiFloat(8), units="deg"), "sensor_platform_geodetic_longitude_at_nadir_corresponding_to_scene_center" / Metadata(AsciiFloat(8), units="deg"), "sensor_platform_heading_at_nadir_corresponding_to_scene_center" / Metadata(AsciiFloat(8), units="deg"), "sensor_clock_angle_as_measured_relative_to_sensor_platform_flight_direction" / Metadata(AsciiFloat(8), units="deg"), "incidence_angle_at_scene_center" / Metadata(AsciiFloat(8), units="deg"), "spare5" / PaddedString(8), "nominal_radar_wavelength" / Metadata(AsciiFloat(16), units="m"), "motion_compensation_indicator" / motion_compensation, "range_pulse_code" / PaddedString(16), "range_pulse_amplitude_coefficients" / Struct( "coefficient_1" / AsciiFloat(16), "coefficient_2" / AsciiFloat(16), "coefficient_3" / AsciiFloat(16), "coefficient_4" / AsciiFloat(16), "coefficient_5" / AsciiFloat(16), ), "range_pulse_phase_coefficients" / Struct( "coefficient_1" / AsciiFloat(16), "coefficient_2" / AsciiFloat(16), "coefficient_3" / AsciiFloat(16), "coefficient_4" / AsciiFloat(16), "coefficient_5" / AsciiFloat(16), ), "down_linked_data_chirp_extraction_index" / AsciiInteger(8), "spare6" / PaddedString(8), "sampling_rate" / Metadata(AsciiFloat(16), units="MHz"), "range_gate" / Metadata(AsciiFloat(16), units="µs"), "range_pulse_width" / Metadata(AsciiFloat(16), units="µs"), "base_band_conversion_flag" / flag, "range_compression_flag" / flag, "receiver_gain_for_like_polarized_at_early_edge_at_the_start_of_the_image" / AsciiFloat(16), "receiver_gain_for_cross_polarized_at_early_edge_at_the_start_of_the_image" / AsciiFloat(16), "quantization_in_bits_per_channel" / AsciiInteger(8), "quantized_descriptor" / PaddedString(12), "dc_bias_for_I_component" / AsciiFloat(16), "dc_bias_for_Q_component" / AsciiFloat(16), "gain_imbalance_for_I_and_Q" / AsciiFloat(16), "spare7" / AsciiFloat(16), "spare8" / AsciiFloat(16), "electronic_boresight" / AsciiFloat(16), "mechanical_boresight" / AsciiFloat(16), "echo_tracker_status" / flag, "prf" / Metadata(AsciiFloat(16), units="mHz"), "two_way_antenna_beam_width_elevation" / Metadata(AsciiFloat(16), units="deg"), "two_way_antenna_beam_width_azimuth" / Metadata(AsciiFloat(16), units="deg"), "satellite_encoded_binary_time_code" / AsciiInteger(16), "satellite_clock_time" / PaddedString(32), "satellite_clock_increment" / Metadata(AsciiInteger(16), units="ns"), "processing_facility_id" / PaddedString(16), "processing_system_id" / PaddedString(8), "processing_version_id" / PaddedString(8), "processing_code_of_processing_facility" / PaddedString(16), "product_level_code" / PaddedString(16), "product_type_specifier" / PaddedString(32), "processing_algorithm_id" / PaddedString(32), "number_of_looks_in_azimuth" / AsciiFloat(16), "number_of_looks_in_range" / AsciiFloat(16), "bandwidth_per_look_in_azimuth" / Metadata(AsciiFloat(16), units="Hz"), "bandwidth_per_look_in_range" / Metadata(AsciiFloat(16), units="Hz"), "bandwidth_in_azimuth" / Metadata(AsciiFloat(16), units="Hz"), "bandwidth_in_range" / Metadata(AsciiFloat(16), units="kHz"), "weighting_function_in_azimuth" / weighting_functions, "weighting_function_in_range" / weighting_functions, "data_input_source" / PaddedString(16), "resolution_in_ground_range" / Metadata(AsciiFloat(16), units="m"), "resolution_in_azimuth" / Metadata(AsciiFloat(16), units="m"), "radiometric_bias" / AsciiFloat(16), "radiometric_gain" / AsciiFloat(16), "along_track_doppler_frequency_center" / Struct( "constant_term_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz"), "linear_coefficient_terms_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/px"), "quadratic_coefficient_terms_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/px^2"), ), "spare9" / PaddedString(16), "cross_track_doppler_frequency_center" / Struct( "constant_term_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz"), "linear_coefficient_terms_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/px"), "quadratic_coefficient_terms_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/px^2"), ), "time_direction_indicator_along_pixel_direction" / PaddedString(8), "time_direction_indicator_along_line_direction" / PaddedString(8), "along_track_doppler_frequency_rate" / Struct( "constant_terms_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/s"), "linear_coefficient_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/s/px"), "quadratic_coefficient_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/s/px^2"), ), "spare10" / PaddedString(16), "cross_track_doppler_frequency_rate" / Struct( "constant_terms_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/s"), "linear_coefficient_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/s/px"), "quadratic_coefficient_at_early_edge_of_the_image" / Metadata(AsciiFloat(16), units="Hz/s/px^2"), ), "spare11" / PaddedString(16), "line_content_indicator" / PaddedString(8), "clutter_lock_applied_flag" / flag, "auto_focusing_applied_flag" / flag, "line_spacing" / Metadata(AsciiFloat(16), units="m"), "pixel_spacing" / Metadata(AsciiFloat(16), units="m"), "processor_range_compression_designator" / PaddedString(16), "doppler_frequency_approximately_constant_coefficient_term" / Metadata(AsciiFloat(16), units="Hz"), "doppler_frequency_approximately_linear_coefficient_term" / Metadata(AsciiFloat(16), units="Hz/km"), "calibration_mode_data_location_flag" / AsciiInteger(4), "calibration_at_the_side_of_start" / Struct( "start_line_number" / AsciiInteger(8), "end_line_number" / AsciiInteger(8), ), "calibration_at_the_side_of_end" / Struct( "start_line_number" / AsciiInteger(8), "end_line_number" / AsciiInteger(8), ), "prf_switching_indicator" / AsciiInteger(4), "line_number_of_prf_switching" / AsciiInteger(8), "direction_of_a_beam_center_in_a_scene_center" / Metadata(AsciiFloat(16), units="deg"), "yaw_steering_mode_flag" / AsciiInteger(4), "parameter_table_number_of_automatically_setting" / AsciiInteger(4), "nominal_off_nadir_angle" / AsciiFloat(16), "antenna_beam_number" / AsciiInteger(4), "spare12" / PaddedString(28), "incidence_angle" / Metadata( Struct( "constant_term" / Metadata(AsciiFloat(20), units="rad"), "linear_term" / Metadata(AsciiFloat(20), units="rad/km"), "quadratic_term" / Metadata(AsciiFloat(20), units="rad/km^2"), "cubic_term" / Metadata(AsciiFloat(20), units="rad/km^3"), "fourth_term" / Metadata(AsciiFloat(20), units="rad/km^4"), "fifth_term" / Metadata(AsciiFloat(20), units="rad/km^5"), ), formula="θ = a0 + a1*R + a2*R^2 + a3*R^3 + a4*R^4 + a5*R^5", theta="incidence angle", r="slant range", ), "image_annotation_segment" / Struct( "number_of_annotation_points" / AsciiInteger(8), "spare" / PaddedString(8), "annotations" / Struct( "line_number_of_annotation_start" / AsciiInteger(8), "pixel_number_of_annotation_start" / AsciiInteger(8), "annotation_text" / PaddedString(16), )[64], "system_reserve" / PaddedString(26), ), ) def transform_dataset_summary(mapping): ignored = [ "preamble", "dataset_summary_records_sequence_number", "sar_channel_id", "number_of_scene_reference", "average_terrain_height_above_ellipsoid_at_scene_center", "processing_scene_length", "processing_scene_width", "range_pulse_phase_coefficients", "processing_code_of_processing_facility", "processing_algorithm_id", "radiometric_bias", "radiometric_gain", "time_direction_indicator_along_pixel_direction", "parameter_table_number_of_automatically_setting", "image_annotation_segment", ] transformers = { "scene_center_time": normalize_datetime, } translations = {} result = pipe( mapping, curry(remove_spares), curry(dissoc, ignored), curry(apply_to_items, transformers), curry(rename, translations=translations), curry(as_group), ) return result xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/facility_related_data.py000066400000000000000000000137721501510365500263140ustar00rootroot00000000000000from construct import Bytes, Enum, Struct, this from tlz.dicttoolz import valmap from tlz.functoolz import curry, pipe from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiFloat, AsciiInteger, Metadata, PaddedString from ceos_alos2.dicttoolz import apply_to_items, dissoc from ceos_alos2.transformers import as_group, remove_spares from ceos_alos2.utils import rename facility_related_data_record = Struct( "preamble" / record_preamble, "record_sequence_number" / AsciiInteger(4), "blanks" / PaddedString(50), "raw_file_data" / Bytes(this.preamble.record_length - 12 - 4 - 50), ) facility_related_data_5_record = Struct( "preamble" / record_preamble, "record_sequence_number" / AsciiInteger(4), "conversion_from_map_projection_to_pixel" / Metadata( Struct( "a" / AsciiFloat(20)[10], "b" / AsciiFloat(20)[10], ), formula=( "P = a0 + a1*φ + a2*λ + a3*φ*λ + a4*φ^2 + a5*λ^2 + a6*φ^2*λ + a7*φ*λ^2 + a8*φ^3 + a9*λ^3;" " L = b0 + b1*φ + b2*λ + b3*φ*λ + b4*φ^2 + b5*λ^2 + b6*φ^2*λ + b7*φ*λ^2 + b8*φ^3 + b9*λ^3" ), ), "calibration_mode_data_location_flag" / Enum( AsciiInteger(4), no_calibration=0, side_of_observation_start=1, side_of_observation_end=2, side_of_observation_start_and_end=3, ), "calibration_at_upper_image" / Struct( "start_line_number" / AsciiInteger(8), "end_line_number" / AsciiInteger(8), ), "calibration_at_bottom_image" / Struct( "start_line_number" / AsciiInteger(8), "end_line_number" / AsciiInteger(8), ), "prf_switching_flag" / AsciiInteger(4), "start_line_number_of_prf_switching" / AsciiInteger(8), "blanks1" / PaddedString(8), "number_of_loss_lines" / Struct( "level1.0" / AsciiInteger(8), "others" / AsciiInteger(8), ), "blanks2" / PaddedString(312), "system_reserve" / PaddedString(224), "conversion_from_pixel_to_geographic" / Metadata( Struct( "a" / AsciiFloat(20)[25], "b" / AsciiFloat(20)[25], "origin_pixel" / AsciiFloat(20), "origin_line" / AsciiFloat(20), ), formula=( ( "φ = a0*L^4*P^4 + a1*L^3*P^4 + a2*L^2*P^4 + a3*L*P^4 + a4*P^4" " + a5*L^4*P^3 + a6*L^3*P^3 + a7*L^2*P^3 + a8*L*P^3 + a9*P^3" " + a10*L^4*P^2 + a11*L^3*P^2 + a12*L^2*P^2 + a13*L*P^2 + a14*P^2" " + a15*L^4*P + a16*L^3*P + a17*L^2*P + a18*L*P + a19*P" " + a20*L^4 + a21*L^3 + a22*L^2 + a23*L + a24" ) + "; " + ( "λ = b0*L^4*P^4 + b1*L^3*P^4 + b2*L^2*P^4 + b3*L*P^4 + b4*P^4" " + b5*L^4*P^3 + b6*L^3*P^3 + b7*L^2*P^3 + b8*L*P^3 + b9*P^3" " + b10*L^4*P^2 + b11*L^3*P^2 + b12*L^2*P^2 + b13*L*P^2 + b14*P^2" " + b15*L^4*P + b16*L^3*P + b17*L^2*P + b18*L*P + b19*P" " + b20*L^4 + b21*L^3 + b22*L^2 + b23*L + b24" ) ), ), "conversion_from_geographic_to_pixel" / Metadata( Struct( "c" / AsciiFloat(20)[25], "d" / AsciiFloat(20)[25], "origin_latitude" / AsciiFloat(20), "origin_longitude" / AsciiFloat(20), ), formula=( ( "p = c0*Λ^4*Φ^4 + c1*Λ^3*Φ^4 + c2*Λ^2*Φ^4 + c3*Λ*Φ^4 + c4*Φ^4" " + c5*Λ^4*Φ^3 + c6*Λ^3*Φ^3 + c7*Λ^2*Φ^3 + c8*Λ*Φ^3 + c9*Φ^3" " + c10*Λ^4*Φ^2 + c11*Λ^3*Φ^2 + c12*Λ^2*Φ^2 + c13*Λ*Φ^2 + c14*Φ^2" " + c15*Λ^4*Φ + c16*Λ^3*Φ + c17*Λ^2*Φ + c18*Λ*Φ + c19*Φ" ) + "; " + ( "l = d0*Λ^4*Φ^4 + d1*Λ^3*Φ^4 + d2*Λ^2*Φ^4 + d3*Λ*Φ^4 + d4*Φ^4" " + d5*Λ^4*Φ^3 + d6*Λ^3*Φ^3 + d7*Λ^2*Φ^3 + d8*Λ*Φ^3 + d9*Φ^3" " + d10*Λ^4*Φ^2 + d11*Λ^3*Φ^2 + d12*Λ^2*Φ^2 + d13*Λ*Φ^2 + d14*Φ^2" " + d15*Λ^4*Φ + d16*Λ^3*Φ + d17*Λ^2*Φ + d18*Λ*Φ + d19*Φ" " + d20*Λ^4 + d21*Λ^3 + d22*Λ^2 + d23*Λ + d24" ) ), ), "blanks" / PaddedString(1896), ) def transform_auxiliary_file(mapping): ignored = ["preamble"] data_types = { 1: "dummy data", 2: "determined ephemeris", 3: "time error information", 4: "coordinate conversion information", } transformers = {"record_sequence_number": data_types.get} translations = {"record_sequence_number": "data_type"} return pipe( mapping, curry(remove_spares), curry(dissoc, ignored), curry(apply_to_items, transformers), curry(rename, translations=translations), ) def transform_group(mapping, dim): def attach_dim(value): if not isinstance(value, list): return (), value, {} return dim, value, {} mapping, attrs = mapping return valmap(attach_dim, mapping), attrs def transform_record5(mapping): ignored = ["preamble", "record_sequence_number", "system_reserve"] transformers = { "prf_switching_flag": bool, "conversion_from_map_projection_to_pixel": curry( transform_group, dim="mid_precision_coeffs" ), "conversion_from_pixel_to_geographic": curry(transform_group, dim="high_precision_coeffs"), "conversion_from_geographic_to_pixel": curry(transform_group, dim="high_precision_coeffs"), } translations = { "conversion_from_map_projection_to_pixel": "projected_to_image", "conversion_from_pixel_to_geographic": "image_to_geographic", "conversion_from_geographic_to_pixel": "geographic_to_image", "prf_switching_flag": "prf_switching", } return pipe( mapping, curry(remove_spares), curry(dissoc, ignored), curry(apply_to_items, transformers), curry(rename, translations=translations), curry(as_group), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/file_descriptor.py000066400000000000000000000057701501510365500251730ustar00rootroot00000000000000from construct import Struct from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiInteger, PaddedString record_types = { "dataset_summary": (18, 10, 18, 20), "map_projection_data": (18, 20, 18, 20), "platform_position_data": (18, 30, 18, 20), "attitude_data": (18, 40, 18, 20), "radiometric_data": (18, 50, 18, 20), "radiometric_compensation": (18, 51, 18, 20), "data_quality_summary": (18, 60, 18, 20), "data_histograms": (18, 70, 18, 20), "range_spectra": (18, 80, 18, 20), "digital_elevation_model_descriptor": (18, 90, 18, 20), "radar_parameter_data_update": (18, 100, 18, 20), "annotation_data": (18, 110, 18, 20), "detailed_processing_parameters": (18, 120, 18, 20), "calibration_data": (18, 130, 18, 20), "ground_control_points": (18, 140, 18, 20), "facility_related_data": (18, 200, 18, 20), } small_record_info = Struct( "number_of_records" / AsciiInteger(6), "record_length" / AsciiInteger(6), ) big_record_info = Struct( "number_of_records" / AsciiInteger(6), "record_length" / AsciiInteger(8), ) file_descriptor_record = Struct( "preamble" / record_preamble, "ascii_ebcdic_flag" / PaddedString(2), "blanks" / PaddedString(2), "format_control_document_id" / PaddedString(12), "format_control_document_revision_level" / PaddedString(2), "record_format_revision_level" / PaddedString(2), "software_release_and_revision_number" / PaddedString(12), "file_number" / AsciiInteger(4), "file_id" / PaddedString(16), "record_sequence_and_location_type_flag" / PaddedString(4), "sequence_number_of_location" / AsciiInteger(8), "field_length_of_sequence_number" / AsciiInteger(4), "record_code_and_location_type_flag" / PaddedString(4), "location_of_record_code" / AsciiInteger(8), "field_length_of_record_code" / AsciiInteger(4), "record_length_and_location_type_flag" / PaddedString(4), "location_of_record_length" / AsciiInteger(8), "field_length_of_record_length" / AsciiInteger(4), "blanks1" / PaddedString(68), "dataset_summary" / small_record_info, "map_projection" / small_record_info, "platform_position" / small_record_info, "attitude" / small_record_info, "radiometric_data" / small_record_info, "radiometric_compensation" / small_record_info, "data_quality_summary" / small_record_info, "data_histogram" / small_record_info, "range_spectra" / small_record_info, "dem_descriptor" / small_record_info, "radar_parameter_update" / small_record_info, "annotation_data" / small_record_info, "detail_processing" / small_record_info, "calibration" / small_record_info, "gcp" / small_record_info, "spare" / PaddedString(60), "facility_related_data_1" / big_record_info, "facility_related_data_2" / big_record_info, "facility_related_data_3" / big_record_info, "facility_related_data_4" / big_record_info, "facility_related_data_5" / big_record_info, "blanks2" / PaddedString(230), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/io.py000066400000000000000000000007271501510365500224220ustar00rootroot00000000000000from ceos_alos2.sar_leader.metadata import transform_metadata from ceos_alos2.sar_leader.structure import sar_leader_record from ceos_alos2.utils import to_dict def parse_data(data): return to_dict(sar_leader_record.parse(data)) def open_sar_leader(mapper, path): try: data = mapper[path] except KeyError as e: raise FileNotFoundError(f"Cannot open {path}") from e metadata = parse_data(data) return transform_metadata(metadata) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/map_projection.py000066400000000000000000000251471501510365500250270ustar00rootroot00000000000000import operator from construct import Struct from tlz.dicttoolz import merge_with, valmap from tlz.functoolz import curry, pipe from tlz.itertoolz import cons, get, remove from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiFloat, AsciiInteger, Metadata, PaddedString from ceos_alos2.dicttoolz import apply_to_items, dissoc from ceos_alos2.transformers import as_group, remove_spares from ceos_alos2.utils import rename projected_map_point = Struct( "northing" / Metadata(AsciiFloat(16), units="km"), "easting" / Metadata(AsciiFloat(16), units="km"), ) geographic_map_point = Struct( "latitude" / Metadata(AsciiFloat(16), units="deg"), "longitude" / Metadata(AsciiFloat(16), units="deg"), ) map_projection_record = Struct( "preamble" / record_preamble, "blanks" / PaddedString(16), "map_projection_general_information" / Struct( "map_projection_type" / PaddedString(32), "number_of_pixels_per_line" / AsciiInteger(16), "number_of_lines" / AsciiInteger(16), "inter_line_distance_in_output_scene" / Metadata(AsciiFloat(16), units="m"), "inter_pixel_distance_in_output_scene" / Metadata(AsciiFloat(16), units="m"), "angle_between_projection_aixs_from_true_north_at_processed_scene_center" / Metadata(AsciiFloat(16), units="deg"), "actual_platform_orbital_inclination" / Metadata(AsciiFloat(16), units="deg"), "actual_ascending_node" / Metadata(AsciiFloat(16), units="deg"), "distance_of_platform_at_input_scene_center_from_geocenter" / Metadata(AsciiFloat(16), units="m"), "geodetic_altitude_of_the_platform_relative_to_the_ellipsoid" / Metadata(AsciiFloat(16), units="m"), "actual_ground_speed_at_nadir_at_input_scene_center_time" / Metadata(AsciiFloat(16), units="m/s"), "platform_headings" / Metadata(AsciiFloat(16), units="deg"), ), "map_projection_ellipsoid_parameters" / Struct( "reference_ellipsoid" / PaddedString(32), "semimajor_axis" / Metadata(AsciiFloat(16), units="m"), "semiminor_axis" / Metadata(AsciiFloat(16), units="m"), "datum_shift_parameters" / Struct( "dx" / Metadata(AsciiFloat(16), units="m"), "dy" / Metadata(AsciiFloat(16), units="m"), "dz" / Metadata(AsciiFloat(16), units="m"), "rotation_angle_1" / Metadata(AsciiFloat(16), units="deg"), "rotation_angle_2" / Metadata(AsciiFloat(16), units="deg"), "rotation_angle_3" / Metadata(AsciiFloat(16), units="deg"), ), "scale_factor" / AsciiFloat(16), ), "map_projection_designator" / PaddedString(32), "utm_projection" / Struct( "type" / PaddedString(32), "zone_number" / PaddedString(4), "map_origin" / Struct( "false_easting" / Metadata(AsciiFloat(16), units="m"), "false_northing" / Metadata(AsciiFloat(16), units="m"), ), "center_of_projection" / Struct( "longitude" / Metadata(AsciiFloat(16), units="deg"), "latitude" / Metadata(AsciiFloat(16), units="deg"), ), "blanks1" / PaddedString(16), "blanks2" / PaddedString(16), "scale_factor" / AsciiFloat(16), ), "ups_projection" / Struct( "type" / PaddedString(32), "center_of_projection" / Struct( "longitude" / Metadata(AsciiFloat(16), units="deg"), "latitude" / Metadata(AsciiFloat(16), units="deg"), ), "scale_factor" / AsciiFloat(16), ), "national_system_projection" / Struct( "projection_descriptor" / PaddedString(32), "map_origin" / Struct( "false_easting" / Metadata(AsciiFloat(16), units="m"), "false_northing" / Metadata(AsciiFloat(16), units="m"), ), "center_of_projection" / Struct( "longitude" / Metadata(AsciiFloat(16), units="deg"), "latitude" / Metadata(AsciiFloat(16), units="deg"), ), "standard_parallel" / Struct( "phi1" / Metadata(AsciiFloat(16), units="deg"), "phi2" / Metadata(AsciiFloat(16), units="deg"), ), "standard_parallel2" / Struct( "param1" / Metadata(AsciiFloat(16), units="deg"), "param2" / Metadata(AsciiFloat(16), units="deg"), ), "central_meridian" / Struct( "param1" / Metadata(AsciiFloat(16), units="deg"), "param2" / Metadata(AsciiFloat(16), units="deg"), "param3" / Metadata(AsciiFloat(16), units="deg"), ), "blanks" / PaddedString(64), ), "corner_points" / Struct( "projected" / Struct( "top_left_corner" / projected_map_point, "top_right_corner" / projected_map_point, "bottom_right_corner" / projected_map_point, "bottom_left_corner" / projected_map_point, ), "geographic" / Struct( "top_left_corner" / geographic_map_point, "top_right_corner" / geographic_map_point, "bottom_right_corner" / geographic_map_point, "bottom_left_corner" / geographic_map_point, ), "terrain_heights_relative_to_ellipsoid" / Struct( "top_left_corner" / Metadata(AsciiFloat(16), units="deg"), "top_right_corner" / Metadata(AsciiFloat(16), units="deg"), "bottom_right_corner" / Metadata(AsciiFloat(16), units="deg"), "bottom_left_corner" / Metadata(AsciiFloat(16), units="deg"), ), ), "conversion_coefficients" / Struct( "map_projection_to_pixels" / Metadata( Struct( "A11" / AsciiFloat(20), "A12" / AsciiFloat(20), "A13" / AsciiFloat(20), "A14" / AsciiFloat(20), "A21" / AsciiFloat(20), "A22" / AsciiFloat(20), "A23" / AsciiFloat(20), "A24" / AsciiFloat(20), ), formula=( "E = A11 + A12 * R + A13 * C + A14 * R * C;" " N = A21 + A22 * R + A23 * C + A24 * R * C" ), E="easting", N="northing", R="row (1-based)", C="column (1-based)", ), "pixels_to_map_projection" / Metadata( Struct( "B11" / AsciiFloat(20), "B12" / AsciiFloat(20), "B13" / AsciiFloat(20), "B14" / AsciiFloat(20), "B21" / AsciiFloat(20), "B22" / AsciiFloat(20), "B23" / AsciiFloat(20), "B24" / AsciiFloat(20), ), formula=( "R = B11 + B12 * E + B13 * N + B14 * E * N;" " C = B21 + B22 * E + B23 * N + B24 * E * N" ), E="easting", N="northing", R="row (1-based)", C="column (1-based)", ), ), "blanks" / PaddedString(36), ) def filter_map_projection(mapping): all_projections = ["utm_projection", "ups_projection", "national_system_projection"] raw_designator = mapping.get("map_projection_designator") if raw_designator is None: return mapping designator, _ = raw_designator.lower().split("-", 1) sections = { "utm": "utm_projection", "ups": "ups_projection", "lcc": "national_system_projection", "mer": "national_system_projection", } to_keep = sections.get(designator) to_drop = list( cons("map_projection_designator", remove(lambda k: k == to_keep, all_projections)) ) return pipe( mapping, curry(dissoc, to_drop), curry(rename, translations={to_keep: "projection"}), ) def transform_general_info(mapping): translations = { "number_of_pixels_per_line": "n_columns", "number_of_lines": "n_rows", } return pipe( mapping, curry(rename, translations=translations), ) def transform_ellipsoid_parameters(mapping): # fixed to 0.0 ignored = ["datum_shift_parameters", "scale_factor"] return dissoc(ignored, mapping) def transform_projection(mapping): ignored = ["map_origin", "standard_parallel2", "central_meridian"] return dissoc(ignored, mapping) def transform_corner_points(mapping): coordinate = ["top_left", "top_right", "bottom_right", "bottom_left"] keys = [f"{v}_corner" for v in coordinate] def separate_attrs(data): values, metadata_ = zip(*data) metadata = metadata_[0] return ["corner"], list(values), metadata def combine_corners(mapping): items = get(keys, mapping) merged = merge_with(list, *items) processed = valmap(separate_attrs, merged) return processed ignored = ["terrain_heights_relative_to_ellipsoid"] transformers = { "projected": curry(operator.or_, {"corner": (["corner"], coordinate, {})}), "geographic": curry(operator.or_, {"corner": (["corner"], coordinate, {})}), } result = pipe( mapping, curry(dissoc, ignored), curry(valmap, combine_corners), curry(apply_to_items, transformers), ) return result def transform_conversion_coefficients(mapping): def transform_coeffs(entry): raw_data, attrs = entry names, coeffs = zip(*raw_data.items()) data = {"names": ("names", list(names), {}), "coefficients": ("names", list(coeffs), {})} return data, attrs translations = { "map_projection_to_pixels": "projected_to_image", "pixels_to_map_projection": "image_to_projected", } return pipe( mapping, curry(valmap, transform_coeffs), curry(rename, translations=translations), ) def transform_map_projection(mapping): ignored = ["preamble"] transformers = { "map_projection_general_information": transform_general_info, "map_projection_ellipsoid_parameters": transform_ellipsoid_parameters, "projection": transform_projection, "corner_points": transform_corner_points, "conversion_coefficients": transform_conversion_coefficients, } translations = { "map_projection_general_information": "general_information", "map_projection_ellipsoid_parameters": "ellipsoid_parameters", } result = pipe( mapping, curry(remove_spares), curry(dissoc, ignored), curry(filter_map_projection), curry(apply_to_items, transformers), curry(rename, translations=translations), curry(as_group), ) return result xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/metadata.py000066400000000000000000000045461501510365500235760ustar00rootroot00000000000000import numpy as np from tlz.dicttoolz import valfilter from tlz.functoolz import compose_left, curry, pipe from tlz.itertoolz import first from ceos_alos2.dicttoolz import apply_to_items, dissoc from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.sar_leader.attitude import transform_attitude from ceos_alos2.sar_leader.data_quality_summary import transform_data_quality_summary from ceos_alos2.sar_leader.dataset_summary import transform_dataset_summary from ceos_alos2.sar_leader.facility_related_data import transform_record5 from ceos_alos2.sar_leader.map_projection import transform_map_projection from ceos_alos2.sar_leader.platform_position import transform_platform_position from ceos_alos2.sar_leader.radiometric_data import transform_radiometric_data from ceos_alos2.utils import rename def fix_attitude_time(group): if "platform_position" not in group or "attitude" not in group: return group reference_year = group["platform_position"].attrs["datetime_of_first_point"][:4] reference_date = np.array(f"{reference_year}-01-01", dtype="datetime64[ns]") for subgroup in group["attitude"].groups.values(): time = subgroup.data["time"] new_data = reference_date + time.data subgroup.data["time"] = Variable(time.dims, new_data, time.attrs) return group def transform_metadata(mapping): ignored = [ "file_descriptor", "facility_related_data_1", "facility_related_data_2", "facility_related_data_3", "facility_related_data_4", ] transformers = { "dataset_summary": transform_dataset_summary, "map_projection": compose_left(first, transform_map_projection), "platform_position": transform_platform_position, "attitude": transform_attitude, "radiometric_data": transform_radiometric_data, "data_quality_summary": transform_data_quality_summary, "facility_related_data_5": transform_record5, } translations = { "facility_related_data_5": "transformations", } postprocessors = [fix_attitude_time] groups = pipe( mapping, curry(dissoc, ignored), curry(valfilter, bool), curry(apply_to_items, transformers), curry(rename, translations=translations), compose_left(*postprocessors), ) return Group(path=None, url=None, data=groups, attrs={}) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/platform_position.py000066400000000000000000000101631501510365500255560ustar00rootroot00000000000000import datetime as dt from construct import Enum, Struct from tlz.dicttoolz import merge_with, valmap from tlz.functoolz import compose_left, curry, pipe from tlz.itertoolz import cons from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiFloat, AsciiInteger, Metadata, PaddedString from ceos_alos2.dicttoolz import apply_to_items, dissoc, move_items from ceos_alos2.transformers import as_group, remove_spares, separate_attrs from ceos_alos2.utils import rename, starcall orbital_elements_designator = Enum( PaddedString(32), preliminary="0", decision="1", high_precision="2" ) orbit_point = Struct( "position" / Struct( "x" / Metadata(AsciiFloat(22), units="m"), "y" / Metadata(AsciiFloat(22), units="m"), "z" / Metadata(AsciiFloat(22), units="m"), ), "velocity" / Struct( "x" / Metadata(AsciiFloat(22), units="m/s"), "y" / Metadata(AsciiFloat(22), units="m/s"), "z" / Metadata(AsciiFloat(22), units="m/s"), ), ) platform_position_record = Struct( "preamble" / record_preamble, "orbital_elements_designator" / orbital_elements_designator, "orbital_elements" / Struct( "position" / Struct( "x" / Metadata(AsciiFloat(16), units="m"), "y" / Metadata(AsciiFloat(16), units="m"), "z" / Metadata(AsciiFloat(16), units="m"), ), "velocity" / Struct( "x" / Metadata(AsciiFloat(16), units="m/s"), "y" / Metadata(AsciiFloat(16), units="m/s"), "z" / Metadata(AsciiFloat(16), units="m/s"), ), ), "number_of_data_points" / AsciiInteger(4), "datetime_of_first_point" / Struct( "date" / PaddedString(12), "day_of_year" / AsciiInteger(4), "seconds_of_day" / AsciiFloat(22), ), "time_interval_between_data_points" / Metadata(AsciiFloat(22), units="s"), "reference_coordinate_system" / PaddedString(64), "greenwich_mean_hour_angle" / Metadata(AsciiFloat(22), units="deg"), "nominal_error" / Struct( "position" / Struct( "along_track" / Metadata(AsciiFloat(16), units="m"), "across_track" / Metadata(AsciiFloat(16), units="m"), "radial" / Metadata(AsciiFloat(16), units="m"), ), "velocity" / Struct( "along_track" / Metadata(AsciiFloat(16), units="m/s"), "across_track" / Metadata(AsciiFloat(16), units="m/s"), "radial" / Metadata(AsciiFloat(16), units="m/s"), ), ), "positions" / orbit_point[28], "blanks1" / PaddedString(18), "occurrence_flag_of_a_leap_second" / AsciiInteger(1), "blanks2" / PaddedString(579), ) def transform_composite_datetime(mapping): date_str = "-".join(mapping["date"].split()) date = dt.datetime.strptime(date_str, "%Y-%m-%d") timedelta = dt.timedelta(seconds=mapping["seconds_of_day"]) datetime = date + timedelta return datetime.isoformat() def transform_positions(elements): result = pipe( elements, curry(starcall, curry(merge_with, list)), curry( valmap, compose_left( curry(starcall, curry(merge_with, list)), curry(valmap, compose_left(separate_attrs, curry(cons, ["positions"]), tuple)), ), ), ) return result def transform_platform_position(mapping): ignored = ["preamble", "number_of_data_points", "greenwich_mean_hour_angle"] transformers = { "datetime_of_first_point": transform_composite_datetime, "positions": transform_positions, "occurrence_flag_of_a_leap_second": bool, } translations = { "occurrence_flag_of_a_leap_second": "leap_second", "time_interval_between_data_points": "sampling_frequency", } result = pipe( mapping, curry(dissoc, ignored), curry(remove_spares), curry(apply_to_items, transformers), curry(rename, translations=translations), curry(move_items, {("orbital_elements", "type"): ["orbital_elements_designator"]}), curry(as_group), ) return result xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/radiometric_data.py000066400000000000000000000055401501510365500253040ustar00rootroot00000000000000from construct import Struct from tlz.dicttoolz import valmap from tlz.functoolz import curry, pipe from tlz.itertoolz import partition from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import ( AsciiComplex, AsciiFloat, AsciiInteger, Metadata, PaddedString, ) from ceos_alos2.dicttoolz import apply_to_items, assoc, dissoc from ceos_alos2.transformers import as_group, remove_spares radiometric_data_record = Struct( "preamble" / record_preamble, "radiometric_data_records_sequence_number" / AsciiInteger(4), "number_of_radiometric_fields" / AsciiInteger(4), "calibration_factor" / Metadata( AsciiFloat(16), formula=( "σ⁰=10*log_10 + CF - 32.0;" " σ⁰(level1.5/level3.1)=10*log_10 + CF" ), I="level 1.1 real pixel value", Q="level 1.1 imaginary pixel value", DN="level 1.5/3.1 pixel value", ), "distortion_matrix" / Metadata( Struct( "transmission" / Struct( "dt11" / AsciiComplex(32), "dt12" / AsciiComplex(32), "dt21" / AsciiComplex(32), "dt22" / AsciiComplex(32), ), "reception" / Struct( "dr11" / AsciiComplex(32), "dr12" / AsciiComplex(32), "dr21" / AsciiComplex(32), "dr22" / AsciiComplex(32), ), ), formula="Z = A*1/r*exp(-4πr/λ) * RST + N", Z="measurement matrix", A="amplitude", r="slant range", S="true scattering matrix", N="noise component", R="reception distortion matrix", T="transmission distortion matrix", ), "blanks" / PaddedString(9568), ) def transform_matrices(mapping): def transform_matrix(mapping): values = mapping.values() matrix = list(map(list, partition(2, values))) dims = ["i", "j"] return (dims, matrix, {}) if isinstance(mapping, tuple): mapping, attrs = mapping else: attrs = {} var_i = ("i", ["horizontal", "vertical"], {"long_name": "reception polarization"}) var_j = ("j", ["horizontal", "vertical"], {"long_name": "transmission polarization"}) matrices = pipe( mapping, curry(valmap, transform_matrix), curry(assoc, "i", var_i), curry(assoc, "j", var_j), ) return matrices, attrs def transform_radiometric_data(mapping): ignored = [ "preamble", "radiometric_data_records_sequence_number", "number_of_radiometric_fields", ] transformers = { "distortion_matrix": transform_matrices, } return pipe( mapping, curry(dissoc, ignored), curry(remove_spares), curry(apply_to_items, transformers), curry(as_group), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_leader/structure.py000066400000000000000000000026031501510365500240460ustar00rootroot00000000000000from construct import Struct, this from ceos_alos2.sar_leader.attitude import attitude_record from ceos_alos2.sar_leader.data_quality_summary import data_quality_summary_record from ceos_alos2.sar_leader.dataset_summary import dataset_summary_record from ceos_alos2.sar_leader.facility_related_data import ( facility_related_data_5_record, facility_related_data_record, ) from ceos_alos2.sar_leader.file_descriptor import file_descriptor_record from ceos_alos2.sar_leader.map_projection import map_projection_record from ceos_alos2.sar_leader.platform_position import platform_position_record from ceos_alos2.sar_leader.radiometric_data import radiometric_data_record sar_leader_record = Struct( "file_descriptor" / file_descriptor_record, "dataset_summary" / dataset_summary_record, "map_projection" / map_projection_record[this.file_descriptor.map_projection.number_of_records], "platform_position" / platform_position_record, "attitude" / attitude_record, "radiometric_data" / radiometric_data_record, "data_quality_summary" / data_quality_summary_record, "facility_related_data_1" / facility_related_data_record, "facility_related_data_2" / facility_related_data_record, "facility_related_data_3" / facility_related_data_record, "facility_related_data_4" / facility_related_data_record, "facility_related_data_5" / facility_related_data_5_record, ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_trailer/000077500000000000000000000000001501510365500216415ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_trailer/__init__.py000066400000000000000000000016271501510365500237600ustar00rootroot00000000000000import itertools from ceos_alos2.sar_trailer.file_descriptor import file_descriptor_record from ceos_alos2.sar_trailer.image_data import parse_image_data def read_sar_trailer(f): header = file_descriptor_record.parse(f.read(720)) data = f.read() data_sizes = [record["record_length"] for record in header.low_resolution_image_sizes] offsets = list(itertools.accumulate(data_sizes, initial=0)) ranges = list(zip(offsets, offsets[1:])) shapes = [ (record["number_of_pixels"], record["number_of_lines"]) for record in header.low_resolution_image_sizes ] n_bytes = [ record["number_of_bytes_per_one_sample"] for record in header.low_resolution_image_sizes ] low_res_images = [ parse_image_data(data[start:stop], shape, n_bytes_) for (start, stop), shape, n_bytes_ in zip(ranges, shapes, n_bytes) ] return header, low_res_images xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_trailer/file_descriptor.py000066400000000000000000000052721501510365500253760ustar00rootroot00000000000000from construct import Struct, this from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiInteger, PaddedString small_record_info = Struct( "number_of_records" / AsciiInteger(6), "record_length" / AsciiInteger(6), ) big_record_info = Struct( "number_of_records" / AsciiInteger(6), "record_length" / AsciiInteger(8), ) low_res_image_size = Struct( "record_length" / AsciiInteger(8), "number_of_pixels" / AsciiInteger(6), "number_of_lines" / AsciiInteger(6), "number_of_bytes_per_one_sample" / AsciiInteger(6), ) file_descriptor_record = Struct( "preamble" / record_preamble, "ascii_ebcdic_code" / PaddedString(2), "blanks1" / PaddedString(2), "format_control_document_id" / PaddedString(12), "format_control_document_revision_number" / PaddedString(2), "record_format_revision_level" / PaddedString(2), "software_release_and_revision_number" / PaddedString(12), "file_number" / AsciiInteger(4), "file_id" / PaddedString(16), "record_sequence_and_location_type_flag" / PaddedString(4), "sequence_number_of_location" / AsciiInteger(8), "field_length_of_sequence_number" / AsciiInteger(4), "record_code_and_location_type_flag" / PaddedString(4), "location_of_record_code" / AsciiInteger(8), "field_length_of_record_code" / AsciiInteger(4), "record_length_and_location_type_flag" / PaddedString(4), "location_of_record_length" / AsciiInteger(8), "field_length_of_record_length" / AsciiInteger(4), "blanks1" / PaddedString(68), "dataset_summary" / small_record_info, "map_projection" / small_record_info, "platform_position" / small_record_info, "attitude" / small_record_info, "radiometric_data" / small_record_info, "radiometric_compensation" / small_record_info, "data_quality_summary" / small_record_info, "data_histogram" / small_record_info, "range_spectra" / small_record_info, "dem_descriptor" / small_record_info, "radar_parameter_update" / small_record_info, "annotation_data" / small_record_info, "detail_processing" / small_record_info, "calibration" / small_record_info, "gcp" / small_record_info, "spare" / PaddedString(60), "facility_related_data_1" / big_record_info, "facility_related_data_2" / big_record_info, "facility_related_data_3" / big_record_info, "facility_related_data_4" / big_record_info, "facility_related_data_5" / big_record_info, "number_of_low_resolution_images" / AsciiInteger(6), "low_resolution_image_sizes" / low_res_image_size[this.number_of_low_resolution_images], "blanks" / PaddedString(720 - 522 - this.number_of_low_resolution_images * low_res_image_size.sizeof()), ) xarray-ceos-alos2-2025.05.0/ceos_alos2/sar_trailer/image_data.py000066400000000000000000000002421501510365500242640ustar00rootroot00000000000000import numpy as np def parse_image_data(content, shape, n_bytes): dtype = np.dtype(f">i{n_bytes}") return np.frombuffer(content, dtype).reshape(shape) xarray-ceos-alos2-2025.05.0/ceos_alos2/summary.py000066400000000000000000000157371501510365500214160ustar00rootroot00000000000000import re from tlz.dicttoolz import dissoc, keyfilter, keymap, merge, valmap from tlz.functoolz import compose_left, curry, pipe from tlz.functoolz import identity as passthrough from tlz.itertoolz import first, get, groupby, second from ceos_alos2 import decoders from ceos_alos2.dicttoolz import apply_to_items from ceos_alos2.hierarchy import Group from ceos_alos2.utils import remove_nesting_layer, rename try: ExceptionGroup except NameError: # pragma: no cover from exceptiongroup import ExceptionGroup # pragma: no cover entry_re = re.compile(r'(?P
[A-Za-z]{3})_(?P.*?)="(?P.*?)"') section_names = { "odi": "ordering_information", "scs": "scene_specification", "pds": "product_specification", "img": "image_information", "pdi": "product_information", "ach": "autocheck", "rad": "result_information", "lbi": "label_information", } def parse_line(line): match = entry_re.fullmatch(line) if match is None: raise ValueError("invalid line") return match.groupdict() def with_lineno(e, lineno): message = e.args[0] e.args = (f"line {lineno:02d}: {message}",) + e.args[1:] return e def parse_summary(content): lines = content.splitlines() entries = [] errors = {} for lineno, line in enumerate(lines): try: parsed = parse_line(line) entries.append(parsed) except ValueError as e: errors[lineno] = e if errors: new_errors = [with_lineno(error, lineno) for lineno, error in errors.items()] raise ExceptionGroup("failed to parse the summary", new_errors) merged = pipe( entries, curry(groupby, curry(get, "section")), curry( valmap, compose_left(curry(map, lambda x: {x["keyword"]: x["value"]}), merge), ), ) return keymap(str.lower, merged) def categorize_filenames(mapping): filenames = list(mapping.values()) volume_directory, leader, *imagery, trailer = filenames return { "volume_directory": volume_directory, "sar_leader": leader, "sar_imagery": imagery, "sar_trailer": trailer, } def reformat_date(s): return f"{s[:4]}-{s[4:6]}-{s[6:]}" def to_isoformat(s): date, time = s.split() return f"{reformat_date(date)}T{time}" def transform_ordering_info(section): # TODO: figure out what this means or what it would be used for return Group(path=None, url=None, data={}, attrs=section) def transform_scene_spec(section): transformers = { "SceneID": compose_left( decoders.decode_scene_id, curry( apply_to_items, { "date": lambda d: d.isoformat().split("T")[0], "scene_frame": int, "orbit_accumulation": int, }, ), ), "SceneShift": int, } attrs = remove_nesting_layer(apply_to_items(transformers, section)) return Group(path=None, url=None, data={}, attrs=attrs) def transform_product_spec(section): transformers = { "ProductID": decoders.decode_product_id, "ResamplingMethod": curry(decoders.lookup, decoders.resampling_methods), "UTM_ZoneNo": int, "MapDirection": passthrough, "OrbitDataPrecision": passthrough, "AttitudeDataPrecision": passthrough, } attrs = remove_nesting_layer(apply_to_items(transformers, section, default=float)) return Group(path=None, url=None, data={}, attrs=attrs) def transform_image_info(section): def determine_type(key): if "DateTime" in key: return "datetime" else: return "float" transformers = { "datetime": to_isoformat, "float": float, } attrs = {k: transformers[determine_type(k)](v) for k, v in section.items()} return Group(path=None, url=None, data={}, attrs=attrs) def transform_product_info(section): def categorize_key(item): key, _ = item if "ProductFileName" in key: return "data_files" elif key.startswith(("NoOfPixels", "NoOfLines")): return "shapes" else: return "other" def transform_file_info(mapping): filenames = keyfilter(lambda k: not k.startswith("Cnt"), mapping) categorized = categorize_filenames(filenames) return Group(path="data_files", url=None, data={}, attrs=categorized) def transform_shape(mapping): split_keys = keymap(lambda k: tuple(k.split("_")), mapping) grouped = groupby(lambda it: second(it[0]), split_keys.items()) shapes = valmap( compose_left( dict, curry(keymap, first), curry(get, ["NoOfPixels", "NoOfLines"]), curry(map, int), tuple, ), grouped, ) return Group(path="shapes", url=None, data={}, attrs=shapes) def transform_other(mapping): transformers = { "ProductFormat": passthrough, "BitPixel": int, "ProductDataSize": float, } return apply_to_items(transformers, mapping) categorized = valmap(dict, groupby(categorize_key, section.items())) transformers = { "data_files": transform_file_info, "shapes": transform_shape, "other": transform_other, } groups = apply_to_items(transformers, categorized) return Group( path="product_info", url=None, data=dissoc(groups, "other"), attrs=groups.get("other", {}) ) def transform_autocheck(section): attrs = valmap(lambda s: s or "N/A", section) return Group(path=None, url=None, data={}, attrs=attrs) def transform_result_info(section): return Group(path=None, url=None, data={}, attrs=section) def transform_label_info(section): transformers = { "ObservationDate": reformat_date, "ProcessFacility": curry(decoders.lookup, decoders.processing_facilities), } attrs = apply_to_items(transformers, section) return Group(path=None, url=None, data={}, attrs=attrs) def transform_summary(summary): transformers = { "odi": transform_ordering_info, "scs": transform_scene_spec, "pds": transform_product_spec, "img": transform_image_info, "pdi": transform_product_info, "ach": transform_autocheck, "rad": transform_result_info, "lbi": transform_label_info, } return pipe( summary, curry(apply_to_items, transformers), curry(rename, translations=section_names), curry(Group, "summary", None, attrs={}), ) def open_summary(mapper, path): try: bytes_ = mapper[path] except KeyError as e: raise OSError( f"Cannot find the summary file (`{path}`)." f" Make sure the dataset at {mapper.root} is complete and in the JAXA CEOS format." ) from e raw_summary = parse_summary(bytes_.decode()) return transform_summary(raw_summary) xarray-ceos-alos2-2025.05.0/ceos_alos2/testing.py000066400000000000000000000215111501510365500213610ustar00rootroot00000000000000import textwrap from itertools import zip_longest import numpy as np from tlz.dicttoolz import merge_with, valfilter, valmap from tlz.functoolz import curry, pipe from tlz.itertoolz import cons, groupby from ceos_alos2.array import Array from ceos_alos2.dicttoolz import valsplit, zip_default from ceos_alos2.hierarchy import Group, Variable newline = "\n" def dict_overlap(a, b): def status(k): if k not in a: return "missing_left" elif k not in b: return "missing_right" else: return "common" all_keys = list(a | b) g = groupby(status, all_keys) missing_left = g.get("missing_left", []) common = g.get("common", []) missing_right = g.get("missing_right", []) return missing_left, common, missing_right def format_item(x): dtype = x.dtype if dtype.kind in {"m", "M"}: return str(x) return repr(x.item()) def format_array(arr): if isinstance(arr, Array): url = f"{arr.fs.fs.protocol}://" + arr.fs.sep.join([arr.fs.path, arr.url]) lines = [ f"Array(shape={arr.shape}, dtype={arr.dtype}, rpc={arr.records_per_chunk})", f" url: {url}", ] return newline.join(lines) flattened = np.reshape(arr, (-1,)) if flattened.size < 8: return f"{flattened.dtype} " + " ".join(format_item(x) for x in flattened) else: head = flattened[:3] tail = flattened[-2:] return ( f"{flattened.dtype} " + " ".join(format_item(x) for x in head) + " ... " + " ".join(format_item(x) for x in tail) ) def format_variable(var): base_string = f"({', '.join(var.dims)}) {format_array(var.data)}" attrs = [f" {k}: {v}" for k, v in var.attrs.items()] return newline.join(cons(base_string, attrs)) def format_inline(value): if isinstance(value, Variable): return format_variable(value) else: return str(value) def diff_mapping_missing(keys, side): lines = [f"Missing {side}:"] + [f" - {k}" for k in keys] return newline.join(lines) def diff_mapping_not_equal(left, right, name): merged = valfilter(lambda v: len(v) == 2, merge_with(list, left, right)) lines = [] for k, (vl, vr) in merged.items(): if vl == vr: continue lines.append(f" L {k} {format_inline(vl)}") lines.append(f" R {k} {format_inline(vr)}") if not lines: return None formatted_lines = textwrap.indent(newline.join(lines), " ") return newline.join([f"Differing {name}:", formatted_lines]) def diff_mapping(a, b, name): missing_left, common, missing_right = dict_overlap(a, b) sections = [] if missing_left: sections.append(diff_mapping_missing(missing_left, "left")) if missing_right: sections.append(diff_mapping_missing(missing_right, "right")) if common: sections.append(diff_mapping_not_equal(a, b, name=name.lower())) formatted_sections = textwrap.indent(newline.join(filter(None, sections)), " ") return newline.join([f"{name.title()}:", formatted_sections]) def diff_scalar(a, b, name): return textwrap.dedent( f"""\ Differing {name.title()}: L {a} R {b} """.rstrip() ) def compare_data(a, b): if type(a) is not type(b): return False if isinstance(a, Array): return a == b else: return a.shape == b.shape and np.all(a == b) def diff_array(a, b): if not isinstance(a, Array): lines = [ f" L {format_array(a)}", f" R {format_array(b)}", ] return newline.join(lines) sections = [] if a.fs != b.fs: lines = ["Differing filesystem:"] # fs.protocol is always `dir`, so we have to check the wrapped fs if a.fs.fs.protocol != b.fs.fs.protocol: lines.append(f" L protocol {a.fs.fs.protocol}") lines.append(f" R protocol {b.fs.fs.protocol}") if a.fs.path != b.fs.path: lines.append(f" L path {a.fs.path}") lines.append(f" R path {b.fs.path}") sections.append(newline.join(lines)) if a.url != b.url: lines = [ "Differing urls:", f" L url {a.url}", f" R url {b.url}", ] sections.append(newline.join(lines)) if a.byte_ranges != b.byte_ranges: lines = ["Differing byte ranges:"] for index, (range_a, range_b) in enumerate(zip_longest(a.byte_ranges, b.byte_ranges)): if range_a == range_b: continue lines.append(f" L line {index + 1} {range_a}") lines.append(f" R line {index + 1} {range_b}") sections.append(newline.join(lines)) if a.shape != b.shape: lines = [ "Differing shapes:", f" {a.shape} != {b.shape}", ] sections.append(newline.join(lines)) if a.dtype != b.dtype: lines = [ "Differing dtypes:", f" {a.dtype} != {b.dtype}", ] sections.append(newline.join(lines)) if a.type_code != b.type_code: lines = [ "Differing type code:", f" L type_code {a.type_code}", f" R type_code {b.type_code}", ] sections.append(newline.join(lines)) if a.records_per_chunk != b.records_per_chunk: lines = [ "Differing chunksizes:", f" L records_per_chunk {a.records_per_chunk}", f" R records_per_chunk {b.records_per_chunk}", ] sections.append(newline.join(lines)) return newline.join(sections) def diff_data(a, b, name): if type(a) is not type(b): lines = [ f"Differing {name.lower()} types:", f" L {type(a)}", f" R {type(b)}", ] return newline.join(lines) diff = diff_array(a, b) return newline.join([f"Differing {name.lower()}:", textwrap.indent(diff, " ")]) def format_sizes(sizes): return "(" + ", ".join(f"{k}: {s}" for k, s in sizes.items()) + ")" def diff_variable(a, b): sections = [] if a.dims != b.dims: lines = ["Differing dimensions:", f" {format_sizes(a.sizes)} != {format_sizes(b.sizes)}"] sections.append(newline.join(lines)) if not compare_data(a.data, b.data): sections.append(diff_data(a.data, b.data, name="Data")) if a.attrs != b.attrs: sections.append(diff_mapping(a.attrs, b.attrs, name="Attributes")) diff = newline.join(sections) return newline.join( ["Left and right Variable objects are not equal", textwrap.indent(diff, " ")] ) def diff_group(a, b): sections = [] if a.path != b.path: sections.append(diff_scalar(a.path, b.path, name="Path")) if a.url != b.url: sections.append(diff_scalar(a.url, b.url, name="URL")) if a.variables != b.variables: sections.append(diff_mapping(a.variables, b.variables, name="Variables")) if a.attrs != b.attrs: sections.append(diff_mapping(a.attrs, b.attrs, name="Attributes")) return newline.join(sections) def diff_tree(a, b): tree_a = dict(a.subtree) tree_b = dict(b.subtree) sections = [] missing, common = pipe( zip_default(tree_a, tree_b, default=None), curry(valmap, curry(map, lambda g: g.decouple() if g is not None else None)), curry(valmap, list), curry(valsplit, lambda groups: any(g is None for g in groups)), ) if missing: lines = ["Differing tree structure:"] missing_left, missing_right = map(list, valsplit(lambda x: x[0] is None, missing)) if missing_left: lines.append(" Missing left:") lines.extend(f" - {k}" for k in missing_left) if missing_right: lines.append(" Missing right:") lines.extend(f" - {k}" for k in missing_right) sections.append(newline.join(lines)) if common: lines = [] for path, (left, right) in common.items(): if left == right: continue lines.append(f" Group {path}:") lines.append(textwrap.indent(diff_group(left, right), " ")) if lines: sections.append(newline.join(cons("Differing groups:", lines))) diff = newline.join(sections) return newline.join(["Left and right Group objects are not equal", textwrap.indent(diff, " ")]) def assert_identical(a, b): __tracebackhide__ = True # compare types assert type(a) is type(b), f"types mismatch: {type(a)} != {type(b)}" if not isinstance(a, (Group, Variable, Array)): raise TypeError("can only compare Group and Variable and Array objects") if isinstance(a, Group): assert a == b, diff_tree(a, b) elif isinstance(a, Variable): assert a == b, diff_variable(a, b) else: assert a == b, diff_array(a, b) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/000077500000000000000000000000001501510365500204745ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/__init__.py000066400000000000000000000000001501510365500225730ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_array.py000066400000000000000000000443561501510365500232370ustar00rootroot00000000000000import io import fsspec import numpy as np import pytest from fsspec.implementations.dirfs import DirFileSystem from ceos_alos2 import array @pytest.mark.parametrize( ["sizes", "reference_size", "expected"], ( pytest.param([5, 5, 5], 8, 2, id="all-equal"), pytest.param([4, 9, 2], 2, 1, id="unbalanced"), pytest.param([7, 6, 8], 19, 3, id="balanced"), ), ) def test_determine_nearest_chunksize(sizes, reference_size, expected): actual = array.determine_nearest_chunksize(sizes, reference_size) assert actual == expected @pytest.mark.parametrize( ["ranges", "n_chunks", "expected"], ( pytest.param( [(0, 3), (3, 6), (6, 9), (9, 12), (12, 15), (15, 18)], 1, {0: (0, 3), 1: (3, 6), 2: (6, 9), 3: (9, 12), 4: (12, 15), 5: (15, 18)}, ), pytest.param( [(0, 3), (3, 6), (6, 9), (9, 12), (12, 15), (15, 18)], 2, {0: (0, 6), 1: (6, 12), 2: (12, 18)}, ), pytest.param( [(0, 3), (3, 6), (6, 9), (9, 12), (12, 15), (15, 18)], 3, {0: (0, 9), 1: (9, 18)} ), ), ) def test_compute_chunk_ranges(ranges, n_chunks, expected): actual = array.compute_chunk_ranges(ranges, n_chunks) assert actual == expected @pytest.mark.parametrize( ["ranges", "expected"], ( pytest.param( {0: (0, 3), 1: (3, 9), 2: (9, 16)}, {0: {"offset": 0, "size": 3}, 1: {"offset": 3, "size": 6}, 2: {"offset": 9, "size": 7}}, ), pytest.param( {0: (0, 3), 1: (17, 18), 2: (31, 100)}, { 0: {"offset": 0, "size": 3}, 1: {"offset": 17, "size": 1}, 2: {"offset": 31, "size": 69}, }, ), ), ) def test_to_offset_size(ranges, expected): actual = array.to_offset_size(ranges) assert actual == expected @pytest.mark.parametrize( ["indexer", "expected"], ( pytest.param(2, [(2, (16, 19))], id="scalar-positive"), pytest.param(-1, [(3, (22, 25))], id="scalar-negative"), pytest.param([0, 2], [(0, (0, 3)), (2, (16, 19))], id="list-positive"), pytest.param([0, -1], [(0, (0, 3)), (3, (22, 25))], id="list-negative"), pytest.param(slice(0, 1), [(0, (0, 3))], id="slice-positive_start-positive_stop-no_step"), pytest.param( slice(2, None), [(2, (16, 19)), (3, (22, 25))], id="slice-positive_start-no_stop-no_step", ), pytest.param( slice(None, 2), [(0, (0, 3)), (1, (5, 8))], id="slice-no_start-postive_stop-no_step" ), pytest.param( slice(None, None, 2), [(0, (0, 3)), (2, (16, 19))], id="slice-no_start-no_stop-positive_step", ), pytest.param( slice(-1, None, -2), [(3, (22, 25)), (1, (5, 8))], id="slice-negative_start-no_stop-negative_step", ), ), ) def test_compute_selected_ranges(indexer, expected): byte_ranges = [(0, 3), (5, 8), (16, 19), (22, 25)] actual = array.compute_selected_ranges(byte_ranges, indexer) assert actual == expected @pytest.mark.parametrize( ["chunksize", "expected"], ( pytest.param(2, {0: [(0, 3), (3, 6)], 1: [(6, 9), (9, 12)], 2: [(12, 15), (15, 18)]}), pytest.param(3, {0: [(0, 3), (3, 6), (6, 9)], 1: [(9, 12), (12, 15), (15, 18)]}), ), ) def test_groupby_chunks(chunksize, expected): byte_ranges = [(0, 3), (3, 6), (6, 9), (9, 12), (12, 15), (15, 18)] actual = array.groupby_chunks(list(enumerate(byte_ranges)), chunksize) assert actual == expected @pytest.mark.parametrize( ["selected", "expected"], ( pytest.param( {0: [(0, 5), (6, 9)], 2: [(26, 28), (28, 30)]}, [((0, 12), [(0, 5), (6, 9)]), ((26, 5), [(26, 28), (28, 30)])], ), pytest.param( {1: [(17, 18), (19, 23)], 3: [(38, 41), (41, 45), (46, 48)]}, [((17, 8), [(17, 18), (19, 23)]), ((37, 10), [(38, 41), (41, 45), (46, 48)])], ), ), ) def test_merge_chunk_info(selected, expected): chunk_offsets = {0: (0, 12), 1: (17, 8), 2: (26, 5), 3: (37, 10)} actual = array.merge_chunk_info(selected, chunk_offsets) assert actual == expected @pytest.mark.parametrize( ["chunk_info", "expected"], ( pytest.param( {"offset": 10, "size": 200}, [(30, 33), (33, 36), (36, 39), (39, 42), (42, 45), (45, 48)], ), pytest.param( {"offset": 15, "size": 200}, [(25, 28), (28, 31), (31, 34), (34, 37), (37, 40), (40, 43)], ), ), ) def test_relocate_ranges(chunk_info, expected): byte_ranges = [(40, 43), (43, 46), (46, 49), (49, 52), (52, 55), (55, 58)] actual = array.relocate_ranges(chunk_info, byte_ranges) assert actual == (chunk_info, expected) def test_extract_ranges(): data = b"buwaox94ks" ranges = [(1, 3), (2, 4), (3, 4), (7, 10)] expected = [b"uw", b"wa", b"a", b"4ks"] actual = array.extract_ranges(data, ranges) assert actual == expected @pytest.mark.parametrize( ["offset", "size"], ( pytest.param(0, 10), pytest.param(50, 100), pytest.param(29, 1), pytest.param(240, 0), ), ) def test_read_chunk(offset, size): data = bytes(range(256)) f = io.BytesIO(data) actual = array.read_chunk(f, offset, size) expected = data[offset : offset + size] assert actual == expected @pytest.mark.parametrize( ["content", "type_code", "expected"], ( pytest.param(b"\x00", "F*8", ValueError("unknown type code"), id="wrong_type_code"), pytest.param( b"\x00\x01", "IU2", np.array([1], dtype="int16"), id="unsigned_int", ), pytest.param( b"\x00\x00\x00\x00\x00\x00\x00\x00", "C*8", np.array([0 + 0j], dtype="complex64"), id="complex", ), ), ) def test_parse_data(content, type_code, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): array.parse_data(content, type_code) return actual = array.parse_data(content, type_code) np.testing.assert_allclose(actual, expected) class TestArray: @pytest.mark.parametrize("shape", ((2, 10), (4, 10), (2, 20), (4, 20))) @pytest.mark.parametrize("dtype", ("uint16", "complex64")) @pytest.mark.parametrize("chunksize", (None, "auto", -1, 2, "80B")) def test_init(self, shape, dtype, chunksize): fs = DirFileSystem(fs=fsspec.filesystem("memory"), path="/") url = "image-file" byte_ranges = [(1, 40), (40, 80), (80, 120), (120, 160)] byte_ranges_ = byte_ranges[: shape[0]] arr = array.Array( fs=fs, url=url, byte_ranges=byte_ranges_, shape=shape, dtype=dtype, records_per_chunk=chunksize, type_code="IU2", ) assert arr.url == url assert arr.byte_ranges == byte_ranges_ assert arr.shape == shape assert arr.dtype == dtype @pytest.mark.parametrize( ["other", "expected"], ( pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), True, id="all_equal", ), pytest.param(1, False, id="mismatching_types"), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("file"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), False, id="fs-protocol", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/b"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), False, id="fs-path", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file2", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), False, id="url", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 20), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), False, id="byte_ranges", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 2), dtype="uint16", records_per_chunk=2, type_code="IU2", ), False, id="shape", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint8", records_per_chunk=2, type_code="IU2", ), False, id="dtype", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=4, type_code="IU2", ), False, id="records_per_chunk", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="C*8", ), False, id="type_code", ), ), ) def test_eq(self, other, expected): fs = DirFileSystem(fs=fsspec.filesystem("memory"), path="/a") byte_ranges = [(0, 40), (40, 80), (80, 120), (120, 160)] arr = array.Array( fs=fs, url="image-file1", byte_ranges=byte_ranges, shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ) actual = arr == other assert actual == expected @pytest.mark.parametrize( ["arr", "expected"], ( pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4,), dtype="uint16", records_per_chunk=2, type_code="IU2", ), 1, id="1D", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), 2, id="2D", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), 3, id="3D", ), ), ) def test_ndim(self, arr, expected): assert arr.ndim == expected @pytest.mark.parametrize( ["arr", "expected"], ( pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4,), dtype="uint16", records_per_chunk=4, type_code="IU2", ), (4,), id="1D-4", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=1, type_code="IU2", ), (1, 3), id="2D-1", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=2, type_code="IU2", ), (2, 3), id="2D-2", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3), dtype="uint16", records_per_chunk=4, type_code="IU2", ), (4, 3), id="2D-4", ), pytest.param( array.Array( fs=DirFileSystem(fs=fsspec.filesystem("memory"), path="/a"), url="image-file1", byte_ranges=[(0, 40), (40, 80), (80, 120), (120, 160)], shape=(4, 3, 3), dtype="uint16", records_per_chunk=4, type_code="IU2", ), (4, 3, 3), id="3D-4", ), ), ) def test_chunks(self, arr, expected): assert arr.chunks == expected @pytest.mark.parametrize("records_per_chunk", [1, 2, 3, 4, 5]) @pytest.mark.parametrize( "indexer_0", ( slice(None), slice(2, None), slice(None, 2), slice(-2, None), slice(None, -2), slice(None, None, 2), slice(None, 4, 2), slice(2, None, 2), slice(2, 4, 2), slice(-1, None, -1), ), ) @pytest.mark.parametrize( "indexer_1", ( slice(None), slice(2, None), slice(None, 2), slice(-2, None), slice(None, -2), slice(None, None, 2), slice(None, 4, 2), slice(2, None, 2), slice(2, 4, 2), slice(-1, None, -1), ), ) def test_getitem(self, indexer_0, indexer_1, records_per_chunk): fs = DirFileSystem(fs=fsspec.filesystem("memory"), path="/") url = "image-file" data = np.arange(100, dtype="uint16").reshape(5, 20) type_code = "IU2" encoded_ = data.astype(">u2").tobytes(order="C") chunksize = data.shape[1] * data.dtype.itemsize chunks = [ encoded_[index * chunksize : (index + 1) * chunksize] for index in range(data.shape[0]) ] metadata_size = 20 gap = b"\x00" * metadata_size encoded = b"".join(gap + chunk for chunk in chunks) byte_ranges = [ ( index * chunksize + metadata_size * (index + 1), (index + 1) * chunksize + metadata_size * (index + 1), ) for index in range(data.shape[0]) ] shape = data.shape dtype = data.dtype with fs.open(url, mode="wb") as f: f.write(encoded) arr = array.Array( fs=fs, url=url, byte_ranges=byte_ranges, shape=shape, dtype=dtype, type_code=type_code, records_per_chunk=records_per_chunk, ) indexers = (indexer_0, indexer_1) actual = arr[indexers] expected = data[indexers] np.testing.assert_equal(actual, expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_caching.py000066400000000000000000001012061501510365500235010ustar00rootroot00000000000000import json from pathlib import Path import fsspec import numpy as np import pytest from ceos_alos2.array import Array from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.sar_image import caching from ceos_alos2.sar_image.caching.path import project_name from ceos_alos2.testing import assert_identical from ceos_alos2.tests.utils import create_dummy_array def test_hashsum(): # no need to verify the external library extensively data = "ddeeaaddbbeeeeff" expected = "b8be84665c5cd09ec19677ce9714bcd987422de886ac2e8432a3e2311b5f0cde" actual = caching.path.hashsum(data) assert actual == expected @pytest.mark.parametrize( "cache_dir", [ Path("/path/to/cache1") / project_name, Path("/path/to/cache2") / project_name, ], ) @pytest.mark.parametrize( ["remote_root", "path", "expected"], ( pytest.param( "http://127.0.0.1/path/to/data", "image1", Path("c9db4f27e586452c6517524752dc472863ee42230ba98e83a346b8da94a33235/image1.index"), ), pytest.param( "s3://bucket/path/to/data", "image1", Path("04391cfcf37045b78e7b4793392821b5b4c84591edfcb475954130eb34b87366/image1.index"), ), pytest.param( "file:///path/to/data", "image1", Path("9506f2b2ddfa8498bc4c1d3cc50d02ee5f799f6716710ff4dd31a9f6e41eac45/image1.index"), ), pytest.param( "/path/to/data", "image2", Path("7b405676e8ed8556a3f4f98f4dc5b6df940f3a5ce48674046eebda551e335b37/image2.index"), ), ), ) def test_local_cache_location(monkeypatch, cache_dir, remote_root, path, expected): monkeypatch.setattr(caching.path, "cache_root", cache_dir) actual = caching.path.local_cache_location(remote_root, path) assert cache_dir in actual.parents assert actual.relative_to(cache_dir) == expected @pytest.mark.parametrize( "remote_root", ("http://127.0.0.1/path/to/data", "s3://bucket/path/to/data") ) @pytest.mark.parametrize( ["path", "expected"], ( ("image1", "image1.index"), ("image7", "image7.index"), ), ) def test_remote_cache_location(remote_root, path, expected): actual = caching.path.remote_cache_location(remote_root, path) assert actual == expected class TestEncoders: @pytest.mark.parametrize( ["arr", "expected_data", "expected_attrs"], ( pytest.param( np.array([0, 10, 20, 30], dtype="timedelta64[ms]"), np.array([0, 10, 20, 30]), {"units": "ms"}, ), pytest.param( np.array([0, 1, 7, 12, 13], dtype="timedelta64[s]"), np.array([0, 1, 7, 12, 13]), {"units": "s"}, ), ), ) def test_encode_timedelta(self, arr, expected_data, expected_attrs): actual_data, actual_attrs = caching.encoders.encode_timedelta(arr) np.testing.assert_equal(actual_data, expected_data) assert actual_attrs == expected_attrs @pytest.mark.parametrize( ["arr", "expected_data", "expected_attrs"], ( pytest.param( np.array(["2019-01-01 00:01:00", "2019-01-02 00:02:00"], dtype="datetime64[ms]"), [0, 86460000], {"units": "ms", "reference": "2019-01-01T00:01:00.000"}, ), pytest.param( np.array(["2019-01-01 00:00:00"], dtype="datetime64[ns]"), np.array([0]), {"units": "ns", "reference": "2019-01-01T00:00:00.000000000"}, ), pytest.param( np.array(["2019-01-01 00:00:00", "2020-01-01 00:00:00"], dtype="datetime64[s]"), [0, 31536000], {"units": "s", "reference": "2019-01-01T00:00:00"}, ), ), ) def test_encode_datetime(self, arr, expected_data, expected_attrs): actual_data, actual_attrs = caching.encoders.encode_datetime(arr) assert actual_data == expected_data assert actual_attrs == expected_attrs @pytest.mark.parametrize( ["arr", "expected"], ( pytest.param( np.array([0, 1, 2], dtype="int32"), {"__type__": "array", "dtype": "int32", "data": [0, 1, 2], "encoding": {}}, id="int32-array", ), pytest.param( np.array([0.0, 1.0, 2.0], dtype="float16"), {"__type__": "array", "dtype": "float16", "data": [0.0, 1.0, 2.0], "encoding": {}}, id="float16-array", ), pytest.param( np.array([0, 1, 2], dtype="timedelta64[s]"), { "__type__": "array", "dtype": "timedelta64[s]", "data": [0, 1, 2], "encoding": {"units": "s"}, }, id="timedelta64-array", ), pytest.param( np.array( ["2019-01-01", "2020-01-01", "2021-01-01", "2022-01-01"], dtype="datetime64[ns]" ), { "__type__": "array", "dtype": "datetime64[ns]", "data": [0, 31536000000000000, 63158400000000000, 94694400000000000], "encoding": {"units": "ns", "reference": "2019-01-01T00:00:00.000000000"}, }, id="datetime64-array", ), pytest.param( create_dummy_array(shape=(4, 3), dtype="int16"), { "__type__": "backend_array", "root": "/path/to", "url": "file", "shape": (4, 3), "dtype": "int16", "byte_ranges": [(5, 10), (15, 20), (25, 30), (35, 40)], "type_code": "IU2", }, id="int16-backend_array", ), ), ) def test_encode_array(self, arr, expected): actual = caching.encoders.encode_array(arr) assert actual == expected @pytest.mark.parametrize( ["var", "expected"], ( pytest.param( Variable("x", np.array([1, 2], dtype="int32"), {}), { "__type__": "variable", "dims": ["x"], "data": {"__type__": "array", "data": [1, 2], "dtype": "int32", "encoding": {}}, "attrs": {}, }, id="1d-array-no_attrs", ), pytest.param( Variable(["x", "y"], np.array([[1, 2], [2, 3], [3, 4]], dtype="int32"), {}), { "__type__": "variable", "dims": ["x", "y"], "data": { "__type__": "array", "data": [[1, 2], [2, 3], [3, 4]], "dtype": "int32", "encoding": {}, }, "attrs": {}, }, id="2d-array-no_attrs", ), pytest.param( Variable("x", create_dummy_array(shape=(4,), dtype="int8"), {}), { "__type__": "variable", "dims": ["x"], "data": { "__type__": "backend_array", "root": "/path/to", "url": "file", "shape": (4,), "dtype": "int8", "byte_ranges": [(5, 10), (15, 20), (25, 30), (35, 40)], "type_code": "IU2", }, "attrs": {}, }, id="1d-backend_array-no_attrs", ), pytest.param( Variable("x", np.array([1, 2], dtype="int32"), {"a": "a", "b": 1, "c": 1.5}), { "__type__": "variable", "dims": ["x"], "data": {"__type__": "array", "data": [1, 2], "dtype": "int32", "encoding": {}}, "attrs": {"a": "a", "b": 1, "c": 1.5}, }, id="1d-array-attrs", ), ), ) def test_encode_variable(self, var, expected): actual = caching.encoders.encode_variable(var) assert actual == expected @pytest.mark.parametrize( ["group", "expected"], ( pytest.param( Group(path="path", url="abc", data={}, attrs={"abc": "def"}), { "__type__": "group", "path": "path", "url": "abc", "data": {}, "attrs": {"abc": "def"}, }, id="no_variables-no_subgroups", ), pytest.param( Group( path=None, url=None, data={"v": Variable("x", np.array([1, 2], dtype="int8"), {})}, attrs={}, ), { "__type__": "group", "path": "/", "url": None, "data": { "v": { "__type__": "variable", "dims": ["x"], "data": { "__type__": "array", "data": [1, 2], "dtype": "int8", "encoding": {}, }, "attrs": {}, } }, "attrs": {}, }, id="variables-no_subgroups", ), pytest.param( Group( path=None, url=None, data={"g": Group(path=None, url=None, data={}, attrs={"n": "g"})}, attrs={}, ), { "__type__": "group", "path": "/", "url": None, "data": { "g": { "__type__": "group", "path": "/g", "url": None, "data": {}, "attrs": {"n": "g"}, } }, "attrs": {}, }, id="no_variables-subgroups", ), ), ) def test_encode_group(self, group, expected): actual = caching.encoders.encode_group(group) assert actual == expected @pytest.mark.parametrize( ["obj", "expected"], ( pytest.param( Group(path=None, url=None, data={}, attrs={"a": 1}), {"__type__": "group", "path": "/", "url": None, "data": {}, "attrs": {"a": 1}}, id="group", ), pytest.param( Variable("x", np.array([1, 2], dtype="int64"), {}), { "__type__": "variable", "dims": ["x"], "data": {"__type__": "array", "data": [1, 2], "dtype": "int64", "encoding": {}}, "attrs": {}, }, id="variable", ), pytest.param(1, 1, id="other"), ), ) def test_encode_hierarchy(self, obj, expected): actual = caching.encoders.encode_hierarchy(obj) assert actual == expected @pytest.mark.parametrize( ["data", "expected"], ( pytest.param(1, 1, id="other"), pytest.param((2, 3), {"__type__": "tuple", "data": [2, 3]}, id="tuple"), pytest.param([2, 3], [2, 3], id="list"), pytest.param({"a": 1, "b": 2}, {"a": 1, "b": 2}, id="dict"), pytest.param( ({"a": 1}, 2), {"__type__": "tuple", "data": [{"a": 1}, 2]}, id="nested_tuple" ), pytest.param( [(2, 3), (3, 4)], [{"__type__": "tuple", "data": [2, 3]}, {"__type__": "tuple", "data": [3, 4]}], id="nested_list", ), pytest.param( {"a": (2, 3), "b": [{"c": 1}]}, {"a": {"__type__": "tuple", "data": [2, 3]}, "b": [{"c": 1}]}, id="nested_dict", ), ), ) def test_preprocess(self, data, expected): actual = caching.encoders.preprocess(data) assert actual == expected class TestDecoders: @pytest.mark.parametrize( ["data", "expected"], ( pytest.param({}, {}, id="empty"), pytest.param({"a": 1}, {"a": 1}, id="default"), pytest.param({"__type__": "tuple", "data": [2, 3]}, (2, 3), id="tuple"), pytest.param( {"__type__": "array", "data": [1, 2], "dtype": "int8", "encoding": {}}, {"__type__": "array", "data": [1, 2], "dtype": "int8", "encoding": {}}, id="array", ), ), ) def test_postprocess(self, data, expected): actual = caching.decoders.postprocess(data) assert actual == expected @pytest.mark.parametrize( ["data", "expected"], ( pytest.param( { "__type__": "array", "data": [0, 3600], "dtype": "datetime64[s]", "encoding": {"units": "s", "reference": "2020-01-01T00:00:00"}, }, np.array(["2020-01-01 00:00:00", "2020-01-01 01:00:00"], dtype="datetime64[s]"), id="datetime-s", ), pytest.param( { "__type__": "array", "data": [0, 60000], "dtype": "datetime64[ms]", "encoding": {"units": "ms", "reference": "2021-01-01T00:00:00"}, }, np.array(["2021-01-01 00:00:00", "2021-01-01 00:01:00"], dtype="datetime64[ms]"), id="datetime-ms", ), ), ) def test_decode_datetime(self, data, expected): actual = caching.decoders.decode_datetime(data) np.testing.assert_equal(actual, expected) @pytest.mark.parametrize( ["data", "records_per_chunk", "expected"], ( pytest.param( {"__type__": "array", "dtype": "int8", "data": [1, 2], "encoding": {}}, 2, np.array([1, 2], dtype="int8"), id="array-int8", ), pytest.param( { "__type__": "array", "dtype": "timedelta64[s]", "data": [1, 2], "encoding": {"units": "s"}, }, 2, np.array([1, 2], dtype="timedelta64[s]"), id="array-timedelta64", ), pytest.param( { "__type__": "array", "dtype": "datetime64[s]", "data": [0, 120000], "encoding": {"units": "ms", "reference": "1997-05-27T00:00:00.000"}, }, 2, np.array(["1997-05-27 00:00:00", "1997-05-27 00:02:00"], dtype="datetime64[s]"), id="array-datetime64", ), pytest.param( { "__type__": "backend_array", "root": "memory:///path/to", "url": "file", "shape": (4, 3), "dtype": "int16", "byte_ranges": [(5, 10), (15, 20), (25, 30), (35, 40)], "type_code": "IU2", }, 1, create_dummy_array(shape=(4, 3), dtype="int16", records_per_chunk=1), id="backend_array-int16", ), ), ) def test_decode_array(self, data, records_per_chunk, expected): actual = caching.decoders.decode_array(data, records_per_chunk) if isinstance(expected, Array): assert_identical(actual, expected) else: np.testing.assert_equal(actual, expected) @pytest.mark.parametrize( ["data", "rpc", "expected"], ( pytest.param( { "__type__": "variable", "dims": ["x"], "data": {"__type__": "array", "dtype": "int8", "data": [1, 2], "encoding": {}}, "attrs": {}, }, 2, Variable("x", np.array([1, 2], dtype="int8"), attrs={}), id="array2-no_attrs", ), pytest.param( { "__type__": "variable", "dims": ["x"], "data": { "__type__": "array", "dtype": "float16", "data": [1.5, 2.5], "encoding": {}, }, "attrs": {}, }, 3, Variable("x", np.array([1.5, 2.5], dtype="float16"), attrs={}), id="array2-no_attrs", ), pytest.param( { "__type__": "variable", "dims": ["x"], "data": {"__type__": "array", "dtype": "int8", "data": [1, 2], "encoding": {}}, "attrs": {"a": 1}, }, 1, Variable("x", np.array([1, 2], dtype="int8"), attrs={"a": 1}), id="array1-attrs", ), pytest.param( { "__type__": "variable", "dims": ["x", "y"], "data": { "__type__": "backend_array", "root": "memory:///path/to", "url": "file", "shape": (4, 3), "dtype": "complex64", "byte_ranges": [(5, 10), (15, 20), (25, 30), (35, 40)], "type_code": "C*8", }, "attrs": {}, }, 2, Variable( ["x", "y"], create_dummy_array( shape=(4, 3), dtype="complex64", records_per_chunk=2, type_code="C*8" ), attrs={}, ), id="backend_array1-no_attrs", ), ), ) def test_decode_variable(self, data, rpc, expected): actual = caching.decoders.decode_variable(data, records_per_chunk=rpc) assert_identical(actual, expected) @pytest.mark.parametrize( ["data", "rpc", "expected"], ( pytest.param( { "__type__": "group", "path": "path", "url": "abc", "data": {}, "attrs": {"abc": "def"}, }, 2, Group(path="path", url="abc", data={}, attrs={"abc": "def"}), id="no_variables-no_subgroups", ), pytest.param( { "__type__": "group", "path": "/", "url": None, "data": { "v": { "__type__": "variable", "dims": ["x"], "data": { "__type__": "array", "data": [1, 2], "dtype": "int8", "encoding": {}, }, "attrs": {}, } }, "attrs": {}, }, 2, Group( path=None, url=None, data={"v": Variable("x", np.array([1, 2], dtype="int8"), {})}, attrs={}, ), id="variables-no_subgroups", ), pytest.param( { "__type__": "group", "path": "/", "url": None, "data": { "v": { "__type__": "variable", "dims": ["x"], "data": { "__type__": "backend_array", "root": "memory:///path/to", "url": "file", "shape": (4, 3), "dtype": "complex64", "byte_ranges": [(5, 10), (15, 20), (25, 30), (35, 40)], "type_code": "C*8", }, "attrs": {}, } }, "attrs": {}, }, 4, Group( path=None, url=None, data={ "v": Variable( "x", create_dummy_array( shape=(4, 3), dtype="complex64", type_code="C*8", records_per_chunk=4, ), {}, ) }, attrs={}, ), id="variables-backend_array-no_subgroups", ), pytest.param( { "__type__": "group", "path": "/", "url": None, "data": { "g": { "__type__": "group", "path": "/g", "url": None, "data": {}, "attrs": {"n": "g"}, } }, "attrs": {}, }, 2, Group( path=None, url=None, data={"g": Group(path=None, url=None, data={}, attrs={"n": "g"})}, attrs={}, ), id="no_variables-subgroups", ), ), ) def test_decode_group(self, data, rpc, expected): actual = caching.decoders.decode_group(data, records_per_chunk=rpc) assert_identical(actual, expected) @pytest.mark.parametrize( ["obj", "rpc", "expected"], ( pytest.param( {"__type__": "group", "path": "/", "url": None, "data": {}, "attrs": {"a": 1}}, 2, Group(path=None, url=None, data={}, attrs={"a": 1}), id="group", ), pytest.param( { "__type__": "variable", "dims": ["x"], "data": {"__type__": "array", "data": [1, 2], "dtype": "int64", "encoding": {}}, "attrs": {}, }, 3, Variable("x", np.array([1, 2], dtype="int64"), {}), id="variable", ), pytest.param( { "__type__": "variable", "dims": ["x", "y"], "data": { "__type__": "backend_array", "root": "memory:///path/to", "url": "file", "shape": (4, 3), "dtype": "complex64", "byte_ranges": [(5, 10), (15, 20), (25, 30), (35, 40)], "type_code": "C*8", }, "attrs": {}, }, 4, Variable( ["x", "y"], create_dummy_array( shape=(4, 3), dtype="complex64", records_per_chunk=4, type_code="C*8" ), attrs={}, ), id="variable-backend_array", ), pytest.param({"a": 1}, 1, {"a": 1}, id="other"), ), ) def test_decode_hierarchy(self, obj, rpc, expected): actual = caching.decoders.decode_hierarchy(obj, records_per_chunk=rpc) if not isinstance(expected, (Variable, Group)): assert actual == expected else: assert_identical(actual, expected) class TestHighLevel: @pytest.mark.parametrize( ["obj", "expected"], ( pytest.param( Group( path=None, url=None, data={"v": Variable("x", np.array([1, 2], dtype="int16"), attrs={})}, attrs={"a": (1, 2)}, ), " ".join( [ '{"__type__": "group", "url": null, "data": {"v":', '{"__type__": "variable", "dims": ["x"], "data":', '{"__type__": "array", "dtype": "int16", "data": [1, 2], "encoding": {}},', '"attrs": {}}}, "path": "/",', '"attrs": {"a": {"__type__": "tuple", "data": [1, 2]}}}', ] ), id="variables", ), pytest.param( Group( path=None, url="s3://bucket/data", data={"g": Group(path=None, url=None, data={}, attrs={"g": 1})}, attrs={}, ), " ".join( [ '{"__type__": "group", "url": "s3://bucket/data", "data": {"g":', '{"__type__": "group", "url": "s3://bucket/data", "data": {},', '"path": "/g", "attrs": {"g": 1}}},', '"path": "/", "attrs": {}}', ] ), id="subgroups", ), ), ) def test_encode(self, obj, expected): actual = caching.encode(obj) assert actual == expected @pytest.mark.parametrize( ["data", "rpc", "expected"], ( pytest.param( " ".join( [ '{"__type__": "group", "url": null, "data": {"v":', '{"__type__": "variable", "dims": ["x"], "data":', '{"__type__": "array", "dtype": "int16", "data": [1, 2], "encoding": {}},', '"attrs": {}}}, "path": "/",', '"attrs": {"a": {"__type__": "tuple", "data": [1, 2]}}}', ] ), 2, Group( path=None, url=None, data={"v": Variable("x", np.array([1, 2], dtype="int16"), attrs={})}, attrs={"a": (1, 2)}, ), id="variable", ), pytest.param( " ".join( [ '{"__type__": "group", "url": "s3://bucket/data", "data": {"g":', '{"__type__": "group", "url": "s3://bucket/data", "data": {},', '"path": "/g", "attrs": {"g": 1}}},', '"path": "/", "attrs": {}}', ] ), 1, Group( path=None, url="s3://bucket/data", data={"g": Group(path=None, url=None, data={}, attrs={"g": 1})}, attrs={}, ), id="subgroup", ), ), ) def test_decode(self, data, rpc, expected): actual = caching.decode(data, records_per_chunk=rpc) assert_identical(actual, expected) @pytest.mark.parametrize( ["path", "rpc", "expected"], ( pytest.param( "not-a-cache", 2, caching.CachingError("no cache found for .+"), id="not-a-cache" ), pytest.param( "image1", 3, Group( path=None, url=None, data={ "v": Variable( ["x", "y"], create_dummy_array( shape=(4, 3), dtype="complex64", type_code="C*8", records_per_chunk=3, ), {}, ) }, attrs={}, ), id="remote_cache", ), pytest.param( "image2", 4, Group( path=None, url=None, data={ "v": Variable( ["x", "y"], create_dummy_array( shape=(4, 3), dtype="complex64", type_code="C*8", records_per_chunk=4, ), {}, ) }, attrs={}, ), id="local_cache", ), ), ) def test_read_cache(self, monkeypatch, path, rpc, expected): mapper = fsspec.get_mapper("memory://cache") data = json.dumps( { "__type__": "group", "url": None, "data": { "v": { "__type__": "variable", "dims": ["x", "y"], "data": { "__type__": "backend_array", "root": "memory:///path/to", "url": "file", "shape": {"__type__": "tuple", "data": [4, 3]}, "dtype": "complex64", "byte_ranges": [ {"__type__": "tuple", "data": [5, 10]}, {"__type__": "tuple", "data": [15, 20]}, {"__type__": "tuple", "data": [25, 30]}, {"__type__": "tuple", "data": [35, 40]}, ], "type_code": "C*8", }, "attrs": {}, } }, "path": "/", "attrs": {}, } ) mapper["image1.index"] = data.encode() def fake_is_file(self): return self.name == "image2.index" def fake_read_text(self): if self.name == "image2.index": return data else: raise OSError monkeypatch.setattr(Path, "is_file", fake_is_file) monkeypatch.setattr(Path, "read_text", fake_read_text) if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): caching.read_cache(mapper, path, records_per_chunk=rpc) return actual = caching.read_cache(mapper, path, records_per_chunk=rpc) assert_identical(actual, expected) def test_create_cache(self, monkeypatch): monkeypatch.setattr(Path, "mkdir", lambda *args, **kwargs: None) parameters = [] def recorder(*args): nonlocal parameters parameters.append(args) monkeypatch.setattr(Path, "write_text", recorder) mapper = fsspec.get_mapper("memory://") path = "image" data = Group(path="/", url="s3://bucket/data", data={}, attrs={}) expected = ( '{"__type__": "group", "url": "s3://bucket/data",' ' "data": {}, "path": "/", "attrs": {}}' ) caching.create_cache(mapper, path, data) actual_path, actual_data = parameters[0] assert actual_path.name == f"{path}.index" assert actual_data == expected xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_datatypes.py000066400000000000000000000131261501510365500241060ustar00rootroot00000000000000import datetime import numpy as np import pytest from construct import Bytes, Int8ub, Int32ub, Int64ub, Struct from ceos_alos2 import datatypes @pytest.mark.parametrize( ["data", "n_bytes", "expected"], ( pytest.param(b"15", 2, 15, id="2bytes-no_padding"), pytest.param(b"3989", 4, 3989, id="4bytes-no_padding"), pytest.param(b" 16", 4, 16, id="4bytes-padding"), pytest.param(b" ", 4, -1, id="4bytes-all_padding"), ), ) def test_ascii_integer(data, n_bytes, expected): parser = datatypes.AsciiInteger(n_bytes) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "n_bytes", "expected"], ( pytest.param(b"1558.423", 8, 1558.423, id="8bytes-no_padding"), pytest.param(b" 165.820", 8, 165.820, id="8bytes-padding"), pytest.param(b" ", 8, float("nan"), id="8bytes-all_padding"), pytest.param(b"162436598487.832", 16, 162436598487.832, id="16bytes-no_padding"), pytest.param(b" 6598487.832", 16, 6598487.832, id="16bytes-padding"), ), ) def test_ascii_float(data, n_bytes, expected): parser = datatypes.AsciiFloat(n_bytes) actual = parser.parse(data) np.testing.assert_equal(actual, expected) with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "n_bytes", "expected"], ( pytest.param(b"1.558.42", 8, 1.55 + 8.42j, id="8bytes-no_padding"), pytest.param(b" ", 8, float("nan") + 1j * float("nan"), id="8bytes-all_padding"), pytest.param(b"162.3659487.8321", 16, 162.3659 + 487.8321j, id="16bytes-no_padding"), pytest.param(b" 62.3659 87.8321", 16, 62.3659 + 87.8321j, id="16bytes-padding"), ), ) def test_ascii_complex(data, n_bytes, expected): parser = datatypes.AsciiComplex(n_bytes) actual = parser.parse(data) np.testing.assert_equal(actual, expected) with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "n_bytes", "expected"], ( pytest.param(b"ALOS", 4, "ALOS", id="4bytes-no_padding"), pytest.param(b"abc ", 4, "abc", id="4bytes-padding"), ), ) def test_padded_string(data, n_bytes, expected): parser = datatypes.PaddedString(n_bytes) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "factor", "expected"], ( pytest.param(b"\x32", 1e-2, 0.5, id="negative_factor"), pytest.param(b"\x32", 1e3, 50000, id="positive_factor"), ), ) def test_factor(data, factor, expected): base = Int8ub parser = datatypes.Factor(base, factor=factor) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "expected"], ( pytest.param( b"\x00\x00\x07\xc6\x00\x00\x01\x0e\x03\x19\xf2f", datetime.datetime(1990, 9, 27, 14, 27, 12, 102000), ), pytest.param( b"\x00\x00\x08\x0b\x00\x00\x00\x01\x00\x00\x00\x00", datetime.datetime(2059, 1, 1), ), ), ) def test_datetime_ydms(data, expected): base = Struct( "year" / Int32ub, "day_of_year" / Int32ub, "milliseconds" / Int32ub, ) parser = datatypes.DatetimeYdms(base) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "expected"], ( pytest.param( b"\x00\x00\x00\x00\x00\x00\x00\x00", datetime.datetime(2019, 1, 1), id="offset_zero", ), pytest.param( b"\x00\x00\x00\tx\x0f\xb1@", datetime.datetime(2019, 1, 1, 11, 17, 49), id="full_seconds", ), ), ) def test_datetime_ydus(data, expected): reference_date = datetime.datetime(2019, 1, 1, 21, 37, 52, 107000) parser = datatypes.DatetimeYdus(Int64ub, reference_date) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["metadata"], ( pytest.param({"units": "m"}), pytest.param({"scale": 10, "units": "us"}), ), ) def test_metadata(metadata): data = b"\x32" base = Int8ub expected = (50, metadata) parser = datatypes.Metadata(base, **metadata) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) @pytest.mark.parametrize( ["data", "expected"], ( pytest.param(b"\x01", b"\x01", id="no_padding"), pytest.param(b"\x00\x01", b"\x01", id="left_padding-1"), pytest.param(b"\x00\x00\x01", b"\x01", id="left_padding-2"), pytest.param(b"\x01\x00", b"\x01", id="right_padding-1"), pytest.param(b"\x01\x00\x00", b"\x01", id="right_padding-2"), pytest.param(b"\x00\x01\x00", b"\x01", id="both_padding-1"), pytest.param(b"\x00\x00\x01\x00\x00", b"\x01", id="both_padding-1"), pytest.param(b"\x01\x00\x01", b"\x01\x00\x01", id="mid"), ), ) def test_strip_null_bytes(data, expected): base = Bytes(len(data)) parser = datatypes.StripNullBytes(base) actual = parser.parse(data) assert actual == expected with pytest.raises(NotImplementedError): parser.build(expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_decoders.py000066400000000000000000000147611501510365500237060ustar00rootroot00000000000000import datetime import pytest from ceos_alos2 import decoders @pytest.mark.parametrize( ["scene_id", "expected"], ( pytest.param( "ALOS2225333200-180726", { "mission_name": "ALOS2", "orbit_accumulation": "22533", "scene_frame": "3200", "date": datetime.datetime(2018, 7, 26), }, id="valid_id", ), pytest.param( "ALOS2xxxxx3200-180726", ValueError("invalid scene id:"), id="invalid_id-invalid_orbit_accumulation", ), pytest.param( "ALOS2225333200-a87433", ValueError("invalid scene id"), id="invalid_id-invalid_date_chars", ), pytest.param( "ALOS2225333200-987433", ValueError("invalid scene id"), id="invalid_id-invalid_date", ), ), ) def test_decode_scene_id(scene_id, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): decoders.decode_scene_id(scene_id) return actual = decoders.decode_scene_id(scene_id) assert actual == expected @pytest.mark.parametrize( ["product_id", "expected"], ( pytest.param( "WWDR1.1__D", { "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.1", "processing_option": "not specified", "map_projection": "not specified", "orbit_direction": "descending", }, id="valid_id-l11rd", ), pytest.param( "WWDL1.1__A", { "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "left looking", "processing_level": "level 1.1", "processing_option": "not specified", "map_projection": "not specified", "orbit_direction": "ascending", }, id="valid_id-l11la", ), pytest.param( "WWDR1.5RUA", { "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.5", "processing_option": "geo-reference", "map_projection": "UTM", "orbit_direction": "ascending", }, id="valid_id-l15rd", ), pytest.param( "WWDR1.6__A", ValueError("invalid product id"), id="invalid_id-invalid_level", ), pytest.param( "WRDR1.1__A", ValueError("invalid product id"), id="invalid_id-wrong_observation_mode", ), ), ) def test_decode_product_id(product_id, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): decoders.decode_product_id(product_id) return actual = decoders.decode_product_id(product_id) assert actual == expected @pytest.mark.parametrize( ["scan_info", "expected"], ( pytest.param( "B4", {"processing_method": "SPECAN method", "scan_number": "4"}, id="valid_code", ), pytest.param("Ac", ValueError("invalid scan info"), id="invalid_code"), pytest.param(None, {}, id="empty"), ), ) def test_decode_scan_info(scan_info, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): decoders.decode_scan_info(scan_info) return actual = decoders.decode_scan_info(scan_info) assert actual == expected @pytest.mark.parametrize( ["path", "expected"], ( pytest.param( "IMG-HV-ALOS2225333100-180726-WWDR1.1__D-B3", { "filetype": "IMG", "polarization": "HV", "mission_name": "ALOS2", "orbit_accumulation": "22533", "scene_frame": "3100", "date": datetime.datetime(2018, 7, 26, 0, 0, 0), "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.1", "processing_option": "not specified", "map_projection": "not specified", "orbit_direction": "descending", "processing_method": "SPECAN method", "scan_number": "3", }, ), pytest.param( "TRL-ALOS2225333100-180726-WWDR1.1__D", { "filetype": "TRL", "polarization": None, "mission_name": "ALOS2", "orbit_accumulation": "22533", "scene_frame": "3100", "date": datetime.datetime(2018, 7, 26, 0, 0, 0), "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.1", "processing_option": "not specified", "map_projection": "not specified", "orbit_direction": "descending", }, ), pytest.param( "LED-ALOS2290760600-191011-WWDR1.5RUA", { "filetype": "LED", "polarization": None, "mission_name": "ALOS2", "orbit_accumulation": "29076", "scene_frame": "0600", "date": datetime.datetime(2019, 10, 11, 0, 0, 0), "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.5", "processing_option": "geo-reference", "map_projection": "UTM", "orbit_direction": "ascending", }, ), pytest.param("summary.txt", ValueError("invalid file name")), ), ) def test_decode_filename(path, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): decoders.decode_filename(path) return actual = decoders.decode_filename(path) assert actual == expected xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_dicttoolz.py000066400000000000000000000106501501510365500241220ustar00rootroot00000000000000import copy import pytest from tlz.functoolz import identity from ceos_alos2 import dicttoolz @pytest.mark.parametrize( ["data", "expected"], ( pytest.param( {1: 0, 2: 1, 3: 2}, ({3: 2}, {1: 0, 2: 1}), ), pytest.param( {1: 0, 3: 0, 2: 1}, ({}, {1: 0, 3: 0, 2: 1}), ), ), ) def test_itemsplit(data, expected): predicate = lambda item: item[0] % 2 == 1 and item[1] != 0 actual = dicttoolz.itemsplit(predicate, data) assert actual == expected def test_valsplit(): data = {0: 1, 1: 0, 2: 2} actual = dicttoolz.valsplit(lambda v: v != 0, data) expected = ({0: 1, 2: 2}, {1: 0}) assert actual == expected def test_keysplit(): data = {0: 1, 1: 0, 2: 2} actual = dicttoolz.keysplit(lambda k: k % 2 == 0, data) expected = ({0: 1, 2: 2}, {1: 0}) assert actual == expected @pytest.mark.parametrize("default", [False, None, object()]) @pytest.mark.parametrize( "mappings", ( [{}, {}], [{"a": 1}, {}], [{}, {"a": 1}], [{"a": 1}, {"a": 1}], [{"a": 1}, {"b": 1}], ), ) def test_zip_default(mappings, default): zipped = dicttoolz.zip_default(*mappings, default=default) assert all( key not in mappings[index] for key, seq in zipped.items() for index, value in enumerate(seq) if value is default ) @pytest.mark.parametrize("keys", (list("ce"), list("af"))) def test_dissoc(keys): mapping = {"a": 1, "b": 2, "c": 3, "e": 4, "f": 5} actual = dicttoolz.dissoc(keys, mapping) expected = {k: v for k, v in mapping.items() if k not in keys} assert actual == expected @pytest.mark.parametrize( ["key", "value", "expected"], ( pytest.param("b", "abc", {"a": 1, "b": "abc"}), pytest.param("c", 2, {"a": 1, "c": 2}), ), ) def test_assoc(key, value, expected): mapping = {"a": 1} actual = dicttoolz.assoc(key, value, mapping) assert actual == expected @pytest.mark.parametrize( ["funcs", "default", "expected"], ( pytest.param({"a": int, "b": str, "c": float}, identity, {"a": 1, "b": "6.4", "c": 4.0}), pytest.param({"a": int, "c": float}, identity, {"a": 1, "b": 6.4, "c": 4.0}), pytest.param({"b": str}, lambda x: x * 2, {"a": "11", "b": "6.4", "c": 8}), ), ) def test_apply_to_items(funcs, default, expected): mapping = {"a": "1", "b": 6.4, "c": 4} actual = dicttoolz.apply_to_items(funcs, mapping, default=default) assert actual == expected @pytest.mark.parametrize( ["instructions", "expected"], ( pytest.param({("b", "d"): ["a"]}, {"a": 1, "b": {"c": 2, "d": 1}}, id="multiple_dest"), pytest.param({("d",): ["b", "c"]}, {"a": 1, "b": {"c": 2}, "d": 2}, id="multiple_src"), pytest.param({("d",): ["e"]}, {"a": 1, "b": {"c": 2}}, id="missing"), pytest.param({("d",): ["e", "f"]}, {"a": 1, "b": {"c": 2}}, id="missing_multiple"), ), ) def test_copy_items(instructions, expected): mapping = {"a": 1, "b": {"c": 2}} copied = copy.deepcopy(mapping) actual = dicttoolz.copy_items(instructions, mapping) assert mapping == copied assert actual == expected @pytest.mark.parametrize( ["instructions", "expected"], ( pytest.param({("b", "d"): ["a"]}, {"b": {"c": 2, "d": 1}}, id="multiple_dest"), pytest.param({("d",): ["b", "c"]}, {"a": 1, "b": {}, "d": 2}, id="multiple_src"), pytest.param({("d",): ["e"]}, {"a": 1, "b": {"c": 2}}, id="missing"), pytest.param({("d",): ["e", "f"]}, {"a": 1, "b": {"c": 2}}, id="missing_multiple"), ), ) def test_move_items(instructions, expected): mapping = {"a": 1, "b": {"c": 2}} copied = copy.deepcopy(mapping) actual = dicttoolz.move_items(instructions, mapping) assert mapping == copied assert actual == expected @pytest.mark.parametrize( ["key", "expected"], ( pytest.param("a", True, id="flat-existing"), pytest.param("z", False, id="flat-missing"), pytest.param("b.c", True, id="nested_dot-existing"), pytest.param("a.b", False, id="nested_dot-missing"), pytest.param(["b", "d", "e"], True, id="nested_list-existing"), pytest.param(["a", "b"], False, id="nested_list-missing"), ), ) def test_key_exists(key, expected): mapping = {"a": 1, "b": {"c": 2, "d": {"e": 4}}} actual = dicttoolz.key_exists(key, mapping) assert actual == expected xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_hierarchy.py000066400000000000000000000427031501510365500240710ustar00rootroot00000000000000import posixpath import numpy as np import pytest from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.tests.utils import create_dummy_array class TestVariable: @pytest.mark.parametrize( ["dims", "data"], ( pytest.param("x", np.arange(5), id="str-1d"), pytest.param(["x"], np.arange(5), id="list-1d"), pytest.param(["x", "y"], np.arange(6).reshape(3, 2), id="list-2d"), ), ) @pytest.mark.parametrize( "attrs", ( {}, {"a": 1}, ), ) def test_init(self, dims, data, attrs): var = Variable(dims=dims, data=data, attrs=attrs) if isinstance(dims, str): dims = [dims] assert var.dims == dims assert type(var.data) is type(data) and np.all(var.data == data) assert var.attrs == attrs @pytest.mark.parametrize( ["dims", "data", "expected"], ( pytest.param(["x"], np.arange(5), 1, id="1d"), pytest.param(["x", "y"], np.arange(6).reshape(3, 2), 2, id="2d"), ), ) def test_ndim(self, dims, data, expected): var = Variable(dims=dims, data=data, attrs={}) actual = var.ndim assert actual == expected @pytest.mark.parametrize( ["dims", "data", "expected"], ( pytest.param(["x"], np.arange(5), (5,), id="1d"), pytest.param(["x", "y"], np.arange(6).reshape(3, 2), (3, 2), id="2d"), ), ) def test_shape(self, dims, data, expected): var = Variable(dims=dims, data=data, attrs={}) actual = var.shape assert actual == expected @pytest.mark.parametrize( ["dims", "data", "expected"], ( pytest.param(["x"], np.arange(5, dtype="int8"), "int8", id="int"), pytest.param(["x"], np.arange(5, dtype="float16"), "float16", id="int"), pytest.param(["x"], np.arange(5, dtype="complex64"), "complex64", id="int"), ), ) def test_dtype(self, dims, data, expected): var = Variable(dims=dims, data=data, attrs={}) actual = var.dtype assert actual == expected @pytest.mark.parametrize( ["data", "expected"], ( pytest.param(np.arange(12).reshape(3, 4), {}, id="numpy"), pytest.param(create_dummy_array(shape=(4, 3)), {"rows": 2, "cols": 3}, id="array"), ), ) def test_chunks(self, data, expected): dims = ["rows", "cols"] var = Variable(dims=dims, data=data, attrs={}) actual = var.chunks assert actual == expected @pytest.mark.parametrize( ["dims", "data", "expected"], ( pytest.param(["x"], np.arange(5), {"x": 5}, id="1d"), pytest.param(["x", "y"], np.arange(6).reshape(3, 2), {"x": 3, "y": 2}, id="2d"), pytest.param( ["y", "x"], np.arange(6).reshape(3, 2), {"y": 3, "x": 2}, id="2d-switched" ), ), ) def test_sizes(self, dims, data, expected): var = Variable(dims=dims, data=data, attrs={}) actual = var.sizes assert actual == expected @pytest.mark.parametrize( ["a", "b", "expected"], ( pytest.param( Variable(dims="x", data=np.array([1]), attrs={}), [1], False, id="type_mismatch", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={}), Variable(dims="x", data=np.array([1]), attrs={}), True, id="identical-without_attrs", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={}), Variable(dims="y", data=np.array([1]), attrs={}), False, id="different_dims-without_attrs", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={}), Variable(dims="x", data=np.array([2]), attrs={}), False, id="different_data-without_attrs", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={}), Variable(dims="x", data=create_dummy_array(shape=(1,)), attrs={}), False, id="different_data-without_attrs", ), pytest.param( Variable(dims="x", data=create_dummy_array(shape=(1,)), attrs={}), Variable(dims="x", data=create_dummy_array(shape=(1,)), attrs={}), True, id="identical_array-without_attrs", ), pytest.param( Variable(dims="x", data=create_dummy_array(shape=(1,)), attrs={}), Variable(dims="x", data=create_dummy_array(shape=(2,)), attrs={}), False, id="different_array-without_attrs", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={"a": 1}), Variable(dims="x", data=np.array([1]), attrs={"a": 1}), True, id="identical-identical_attrs", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={"a": 1}), Variable(dims="x", data=np.array([1]), attrs={"a": 2}), False, id="identical-different_attrs", ), pytest.param( Variable(dims="x", data=np.array([1]), attrs={"a": 1}), Variable(dims="x", data=np.array([1]), attrs={"a": 1, "b": 1}), False, id="identical-mismatching_attrs", ), ), ) def test_equal(self, a, b, expected): actual = a == b assert actual == expected class TestGroup: @pytest.mark.parametrize("attrs", [{"a": 1}, {"a": 1, "b": 2}, {"b": 3, "c": 4}]) @pytest.mark.parametrize("data", [{"a": Variable("x", [1], {})}]) @pytest.mark.parametrize("url", (None, "file:///a", "memory:///a")) @pytest.mark.parametrize("path", (None, "/", "/a/b")) def test_init_flat(self, path, url, data, attrs): group = Group(path=path, url=url, data=data, attrs=attrs) if path is None: path = "/" assert group.path == path assert group.url == url assert group.data == data assert group.attrs == attrs @pytest.mark.parametrize( ["structure"], ( pytest.param( { "a": {"path": "/a", "url": "file:///a", "data": {}, "attrs": {"a": 1}}, "b": {"path": "/b", "url": "file:///b", "data": {}, "attrs": {"b": 1}}, } ), pytest.param( { "a": {"path": None, "url": "file:///a", "data": {}, "attrs": {"a": 1}}, "b": {"path": None, "url": "file:///b", "data": {}, "attrs": {"b": 1}}, } ), pytest.param( { "a": {"path": "/a", "url": None, "data": {}, "attrs": {"a": 1}}, "b": {"path": "/b", "url": None, "data": {}, "attrs": {"b": 1}}, } ), ), ) @pytest.mark.parametrize("url", [None, "file:///r", "memory:///r"]) @pytest.mark.parametrize("path", [None, "/", "/abc"]) def test_init_nested(self, path, url, structure): subgroups = {name: Group(**kwargs) for name, kwargs in structure.items()} group = Group(path=path, url=url, data=subgroups, attrs={}) if path is None: path = "/" assert group.path == path assert group.url == url assert all( subgroup.path == posixpath.join(group.path, name) for name, subgroup in group.data.items() ) assert all( subgroup.url == (structure[name]["url"] or group.url) for name, subgroup in group.data.items() ) @pytest.mark.parametrize( ["first", "second", "expected"], ( pytest.param( Group(path=None, url=None, data={}, attrs={}), Group(path=None, url=None, data={}, attrs={}), True, id="all_equal", ), pytest.param( Group(path=None, url=None, data={}, attrs={}), 1, False, id="mismatching_types", ), pytest.param( Group(path="a", url=None, data={}, attrs={}), Group(path="b", url=None, data={}, attrs={}), False, id="path", ), pytest.param( Group(path=None, url="a", data={}, attrs={}), Group(path=None, url="b", data={}, attrs={}), False, id="url", ), pytest.param( Group(path=None, url=None, data={}, attrs={"a": 1}), Group(path=None, url=None, data={}, attrs={"a": 2}), False, id="attrs", ), pytest.param( Group(path=None, url=None, data={"a": Variable("x", [], {})}, attrs={}), Group(path=None, url=None, data={}, attrs={}), False, id="variables_mismatching", ), pytest.param( Group(path=None, url=None, data={"a": Group(None, None, {}, {})}, attrs={}), Group(path=None, url=None, data={}, attrs={}), False, id="groups_mismatching", ), pytest.param( Group(path=None, url=None, data={"a": Variable("x", [], {})}, attrs={}), Group(path=None, url=None, data={"a": Variable("y", [], {})}, attrs={}), False, id="unequal_variables", ), pytest.param( Group(path=None, url=None, data={"a": Variable("x", [], {})}, attrs={}), Group(path=None, url=None, data={"a": Variable("x", [], {})}, attrs={}), True, id="equal_variables", ), pytest.param( Group( path=None, url=None, data={"a": Group(None, None, {}, {"a": 1})}, attrs={}, ), Group( path=None, url=None, data={"a": Group(None, None, {}, {"a": 2})}, attrs={}, ), False, id="unequal_groups", ), pytest.param( Group( path=None, url=None, data={"a": Group(None, None, {}, {"a": 1})}, attrs={}, ), Group( path=None, url=None, data={"a": Group(None, None, {}, {"a": 1})}, attrs={}, ), True, id="equal_variables", ), ), ) def test_equal(self, first, second, expected): actual = first == second assert actual == expected @pytest.mark.parametrize("key", ["b", "c", "e"]) def test_getitem(self, key): subgroups = { "a": Group(path=None, url=None, data={}, attrs={"a": 1}), "b": Group(path=None, url=None, data={}, attrs={"b": 1}), "c": Group(path=None, url=None, data={}, attrs={"c": 1}), "d": Group(path=None, url=None, data={}, attrs={"d": 1}), } group = Group(path=None, url=None, data=subgroups, attrs={}) if key not in subgroups: with pytest.raises(KeyError): group[key] return actual = group[key] expected = subgroups[key] expected.path = f"/{key}" assert actual == expected @pytest.mark.parametrize( "item", [ Variable("x", [], {}), Group(None, "abc", {}, {}), ], ) def test_setitem(self, item): group = Group(None, None, {}, {}) group["a"] = item if isinstance(item, Group): item.path = "/a" assert group.data["a"] == item @pytest.mark.parametrize( ["group", "expected"], ( pytest.param(Group(None, None, {}, {}), 0, id="default"), pytest.param( Group(None, None, {"a": Variable("x", [], {})}, {}), 1, id="single_variable" ), pytest.param( Group(None, None, {"b": Group(None, None, {}, {})}, {}), 1, id="single_group" ), pytest.param( Group(None, None, {"a": Variable("x", [], {}), "b": Group(None, None, {}, {})}, {}), 2, id="mixed", ), ), ) def test_len(self, group, expected): actual = len(group) assert actual == expected @pytest.mark.parametrize( ["group", "expected"], ( pytest.param(Group(None, None, {}, {}), [], id="default"), pytest.param( Group(None, None, {"a": Variable("x", [], {})}, {}), ["a"], id="single_variable" ), pytest.param( Group(None, None, {"b": Group(None, None, {}, {})}, {}), ["b"], id="single_group" ), pytest.param( Group(None, None, {"a": Variable("x", [], {}), "b": Group(None, None, {}, {})}, {}), ["a", "b"], id="mixed", ), ), ) def test_iter(self, group, expected): actual = list(group) assert actual == expected @pytest.mark.parametrize( ["group", "expected"], ( pytest.param(Group(None, None, {}, {}), "/", id="default"), pytest.param(Group("/", None, {}, {}), "/", id="root"), pytest.param(Group("abc", None, {}, {}), "abc", id="name"), pytest.param(Group("a/b", None, {}, {}), "b", id="relative"), pytest.param(Group("/a/abc", None, {}, {}), "abc", id="absolute"), ), ) def test_name(self, group, expected): actual = group.name assert actual == expected @pytest.mark.parametrize( "group", ( pytest.param(Group(None, None, {}, {}), id="default"), pytest.param(Group(None, None, {"a": Variable("x", [], {})}, {}), id="single_variable"), pytest.param( Group(None, None, {"b": Group(None, None, {}, {})}, {}), id="single_group" ), pytest.param( Group(None, None, {"a": Variable("x", [], {}), "b": Group(None, None, {}, {})}, {}), id="mixed", ), ), ) def test_groups(self, group): actual = group.groups assert all(isinstance(item, Group) for item in actual.values()) @pytest.mark.parametrize( "group", ( pytest.param(Group(None, None, {}, {}), id="default"), pytest.param(Group(None, None, {"a": Variable("x", [], {})}, {}), id="single_variable"), pytest.param( Group(None, None, {"b": Group(None, None, {}, {})}, {}), id="single_group" ), pytest.param( Group(None, None, {"a": Variable("x", [], {}), "b": Group(None, None, {}, {})}, {}), id="mixed", ), ), ) def test_variables(self, group): actual = group.variables assert all(isinstance(item, Variable) for item in actual.values()) def test_decouple(self): subgroups = { "a": Group("a", None, {}, {}), "b": Group("b", None, {}, {}), } variables = { "v1": Variable(["x"], [], {}), "v2": Variable(["y"], [], {}), } group = Group("a", None, data=subgroups | variables, attrs={}) actual = group.decouple() assert actual.groups == {} assert actual.variables == variables assert actual.path == group.path assert actual.url == group.url assert actual.attrs == group.attrs def test_subtree(self): group = Group( "/", None, { "a": Group( "a", None, { "aa": Group("aa", None, {}, {"n": "aa"}), "ab": Group("ab", None, {}, {"n": "ab"}), }, {"n": "a"}, ), "b": Group( "b", None, { "ba": Group("ba", None, {}, {"n": "ba"}), "bb": Group("bb", None, {}, {"n": "bb"}), }, {"n": "b"}, ), "v": Variable("x", [], {}), }, {"n": ""}, ) expected = { "/": Group("/", None, {"v": Variable("x", [], {})}, {"n": ""}), "/a": Group("/a", None, {}, {"n": "a"}), "/a/aa": Group("/a/aa", None, {}, {"n": "aa"}), "/a/ab": Group("/a/ab", None, {}, {"n": "ab"}), "/b": Group("/b", None, {}, {"n": "b"}), "/b/ba": Group("/b/ba", None, {}, {"n": "ba"}), "/b/bb": Group("/b/bb", None, {}, {"n": "bb"}), } actual = dict(group.subtree) assert actual == expected xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_sar_image.py000066400000000000000000000455061501510365500240460ustar00rootroot00000000000000import datetime as dt from dataclasses import dataclass import fsspec import numpy as np import pytest from construct import Int8ub, Int16ub, Seek, Struct, Tell, this from ceos_alos2 import sar_image from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.sar_image import enums, io, metadata from ceos_alos2.testing import assert_identical @dataclass class Data: start: int stop: int @dataclass class Record: record_start: int data: Data dummy_record_types = { 10: Struct("preamble" / io.record_preamble, "a" / Int8ub, "b" / Int8ub, "c" / Int16ub), 11: Struct("preamble" / io.record_preamble, "x" / Int8ub, "y" / Int8ub), } class TestEnums: @pytest.mark.parametrize( ["size", "expected"], ( (1, enums.Int8ub), (2, enums.Int16ub), (4, enums.Int32ub), (8, enums.Int64ub), (3, ValueError("unsupported size")), ), ) def test_flag_init(self, size, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): enums.Flag(size) return flag = enums.Flag(size) assert flag.subcon is expected @pytest.mark.parametrize( ["size", "data", "expected"], ( (1, b"\x00", False), (1, b"\x0f", True), (2, b"\x00\x00", False), ), ) def test_flag_decode(self, size, data, expected): flag = enums.Flag(size) actual = flag.parse(data) assert actual == expected @pytest.mark.parametrize( ["size", "data", "expected"], ( (1, False, b"\x00"), (1, True, b"\x01"), (2, False, b"\x00\x00"), (2, True, b"\x00\x01"), ), ) def test_flag_encode(self, size, data, expected): flag = enums.Flag(size) actual = flag.build(data) assert actual == expected class TestMetadata: @pytest.mark.parametrize( ["header", "expected"], ( ({"prefix_suffix_data_locators": {"sar_data_format_type_code": "IU2"}}, "IU2"), ({"prefix_suffix_data_locators": {"sar_data_format_type_code": "C*8"}}, "C*8"), ), ) def test_extract_format_type(self, header, expected): actual = metadata.extract_format_type(header) assert actual == expected @pytest.mark.parametrize( ["header", "expected"], ( ( { "sar_related_data_in_the_record": { "number_of_lines_per_dataset": 3, "number_of_data_groups_per_line": 2, } }, (3, 2), ), ( { "sar_related_data_in_the_record": { "number_of_lines_per_dataset": 6, "number_of_data_groups_per_line": 4, } }, (6, 4), ), ), ) def test_extract_shape(self, header, expected): actual = metadata.extract_shape(header) assert actual == expected @pytest.mark.parametrize( ["header", "expected"], ( pytest.param({"preamble": {}}, {}, id="preamble"), pytest.param( { "interleaving_id": "BSQ", "number_of_burst_data": 5, "number_of_lines_per_burst": 1, "number_of_overlap_lines_with_adjacent_bursts": 3, }, { "interleaving_id": "BSQ", "number_of_burst_data": 5, "number_of_lines_per_burst": 1, "number_of_overlap_lines_with_adjacent_bursts": 3, }, id="known_attrs", ), pytest.param( {"maximum_data_range_of_pixel": 27}, {"valid_range": [0, 27]}, id="transformed1" ), pytest.param({"maximum_data_range_of_pixel": float("nan")}, {}, id="transformed2"), ), ) def test_extract_attrs(self, header, expected): actual = metadata.extract_attrs(header) assert actual == expected @pytest.mark.parametrize( "overrides", ( {"a": "int8"}, {"b": "float16"}, ), ) def test_apply_overrides(self, overrides): mapping = {"a": ("x", [1, 2], {}), "b": ("y", [1.0, 2.1], {})} applied = metadata.apply_overrides(overrides, mapping) actual = {k: v[1].dtype for k, v in applied.items() if hasattr(v[1], "dtype")} assert actual == overrides @pytest.mark.parametrize( ["known", "expected"], ( (["b"], {"a": 1, "c": ("y", [2, 2], {}), "b": 1}), (["c"], {"a": 1, "b": ("x", [1, 1], {}), "c": 2}), (["b", "c"], {"a": 1, "b": 1, "c": 2}), ), ) def test_deduplicate_attrs(self, known, expected): mapping = {"a": 1, "b": ("x", [1, 1], {}), "c": ("y", [2, 2], {})} actual = metadata.deduplicate_attrs(known, mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( [ { "preamble": {}, "record_start": 1, "actual_count_of_left_fill_pixels": 0, "actual_count_of_right_fill_pixels": 0, "actual_count_of_data_pixels": 0, "palsar_auxiliary_data": b"", "blanks2": "", "data": {}, } ], Group(path=None, url=None, data={}, attrs={}), id="ignored", ), pytest.param( [{"a": (1, {"units": "m"})}, {"a": (2, {"units": "m"})}], Group( path=None, url=None, data={"a": Variable("rows", [1, 2], {"units": "m"})}, attrs={}, ), id="variables_transformed", ), pytest.param( [{"scan_id": 1}, {"scan_id": 1}], Group(path=None, url=None, data={}, attrs={"scan_id": 1}), id="deduplicated_attrs", ), pytest.param( [ {"sensor_acquisition_date": dt.datetime(2020, 10, 1, 12, 37, 42, 451000)}, {"sensor_acquisition_date": dt.datetime(2020, 10, 2, 12, 37, 42, 451000)}, ], Group( path=None, url=None, data={ "sensor_acquisition_date": Variable( "rows", np.array( ["2020-10-01 12:37:42.451", "2020-10-02 12:37:42.451"], dtype="datetime64[ns]", ), {}, ) }, attrs={}, ), id="dtype_overrides", ), pytest.param( [{"sar_image_data_line_number": 1}, {"sar_image_data_line_number": 2}], Group(path=None, url=None, data={"rows": Variable("rows", [1, 2], {})}, attrs={}), id="renamed", ), ), ) def test_transform_line_metadata(self, mapping, expected): actual = metadata.transform_line_metadata(mapping) assert_identical(actual, expected) @pytest.mark.parametrize( ["header", "mapping", "expected_group", "expected_attrs"], ( pytest.param( { "prefix_suffix_data_locators": {"sar_data_format_type_code": "IU2"}, "sar_related_data_in_the_record": { "number_of_lines_per_dataset": 2, "number_of_data_groups_per_line": 4, }, }, [{"data": {"start": 1, "stop": 5}}, {"data": {"start": 6, "stop": 10}}], Group(path=None, url=None, data={}, attrs={"coordinates": []}), { "byte_ranges": [(1, 5), (6, 10)], "type_code": "IU2", "shape": (2, 4), "dtype": "uint16", }, id="array_metadata1", ), pytest.param( { "prefix_suffix_data_locators": {"sar_data_format_type_code": "C*8"}, "sar_related_data_in_the_record": { "number_of_lines_per_dataset": 6, "number_of_data_groups_per_line": 3, }, }, [{"data": {"start": 5, "stop": 21}}, {"data": {"start": 25, "stop": 41}}], Group(path=None, url=None, data={}, attrs={"coordinates": []}), { "byte_ranges": [(5, 21), (25, 41)], "type_code": "C*8", "shape": (6, 3), "dtype": "complex64", }, id="array_metadata2", ), pytest.param( { "prefix_suffix_data_locators": {"sar_data_format_type_code": "F*4"}, "sar_related_data_in_the_record": { "number_of_lines_per_dataset": 6, "number_of_data_groups_per_line": 3, }, }, [], ValueError("unknown type code"), {}, id="array_metadata3", ), pytest.param( { "prefix_suffix_data_locators": {"sar_data_format_type_code": "C*8"}, "sar_related_data_in_the_record": { "number_of_lines_per_dataset": 6, "number_of_data_groups_per_line": 3, }, }, [ { "scan_id": 1, "sar_image_data_line_number": 1, "data": {"start": 5, "stop": 21}, }, { "scan_id": 1, "sar_image_data_line_number": 2, "data": {"start": 25, "stop": 41}, }, ], Group( path=None, url=None, data={"rows": Variable("rows", [1, 2], {})}, attrs={"coordinates": ["rows"], "scan_id": 1}, ), { "byte_ranges": [(5, 21), (25, 41)], "type_code": "C*8", "shape": (6, 3), "dtype": "complex64", }, id="line_metadata", ), ), ) def test_transform_metadata(self, header, mapping, expected_group, expected_attrs): if isinstance(expected_group, Exception): exc = expected_group with pytest.raises(type(exc), match=exc.args[0]): metadata.transform_metadata(header, mapping) return actual_group, actual_attrs = metadata.transform_metadata(header, mapping) assert actual_attrs == expected_attrs assert_identical(actual_group, expected_group) class TestIO: @pytest.mark.parametrize( ["content", "element_size", "expected"], ( pytest.param(b"\x00\x00\x00", 2, ValueError("sizes mismatch"), id="wrong_element_size"), pytest.param( b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 2, ValueError("unknown record type code"), id="unkown_record_type", ), pytest.param( ( b"\x00\x00\x00\x01\x00\x0a\x00\x00\x00\x00\x00\x10\x02\x03\x00\x1f" + b"\x00\x00\x00\x02\x00\x0a\x00\x00\x00\x00\x00\x10\x04\x05\x00\x2f" ), 16, [ { "preamble": { "record_sequence_number": 1, "first_record_subtype": 0, "record_type": 10, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 16, }, "a": 2, "b": 3, "c": 31, }, { "preamble": { "record_sequence_number": 2, "first_record_subtype": 0, "record_type": 10, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 16, }, "a": 4, "b": 5, "c": 47, }, ], id="signal-2elem", ), pytest.param( ( b"\x00\x00\x00\x01\x00\x0b\x00\x00\x00\x00\x00\x0e\x03\x04" + b"\x00\x00\x00\x02\x00\x0b\x00\x00\x00\x00\x00\x0e\x04\x05" ), 14, [ { "preamble": { "record_sequence_number": 1, "first_record_subtype": 0, "record_type": 11, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 14, }, "x": 3, "y": 4, }, { "preamble": { "record_sequence_number": 2, "first_record_subtype": 0, "record_type": 11, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 14, }, "x": 4, "y": 5, }, ], id="processed-2elem", ), ), ) def test_parse_chunk(self, monkeypatch, content, element_size, expected): from ceos_alos2.utils import to_dict monkeypatch.setattr(io, "record_types", dummy_record_types) if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): io.parse_chunk(content, element_size) return actual = to_dict(io.parse_chunk(content, element_size)) assert actual == expected @pytest.mark.parametrize( ["records", "offset", "expected"], ( pytest.param( [Record(1, Data(4, 6)), Record(6, Data(9, 11))], 12, [Record(13, Data(16, 18)), Record(18, Data(21, 23))], ), pytest.param( [Record(3, Data(5, 9)), Record(9, Data(11, 15)), Record(15, Data(17, 21))], 3, [Record(6, Data(8, 12)), Record(12, Data(14, 18)), Record(18, Data(20, 24))], ), ), ) def test_adjust_offsets(self, records, offset, expected): actual = io.adjust_offsets(records, offset) assert actual == expected @pytest.mark.parametrize("rpc", [1, 2]) def test_read_metadata(self, monkeypatch, rpc): dummy_header = {"number_of_sar_data_records": 3, "sar_data_record_length": 17} content = ( b"\x03\x0e" + b"\x00\x00\x00\x01\x00\x0b\x00\x00\x00\x00\x00\x11\x03\x00\x00\x00\x00" + b"\x00\x00\x00\x02\x00\x0b\x00\x00\x00\x00\x00\x11\x04\x00\x00\x00\x00" + b"\x00\x00\x00\x03\x00\x0b\x00\x00\x00\x00\x00\x11\x05\x00\x00\x00\x00" ) print(len(content)) dummy_record_types = { 11: Struct( "preamble" / io.record_preamble, "record_start" / Tell, "a" / Int8ub, "data" / Struct("start" / Tell, "stop" / Seek(this.start + 4)), ), } expected = [ { "preamble": { "record_sequence_number": 1, "first_record_subtype": 0, "record_type": 11, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 17, }, "record_start": 732, "a": 3, "data": {"start": 733, "stop": 737}, }, { "preamble": { "record_sequence_number": 2, "first_record_subtype": 0, "record_type": 11, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 17, }, "record_start": 749, "a": 4, "data": {"start": 750, "stop": 754}, }, { "preamble": { "record_sequence_number": 3, "first_record_subtype": 0, "record_type": 11, "second_record_subtype": 0, "third_record_subtype": 0, "record_length": 17, }, "record_start": 766, "a": 5, "data": {"start": 767, "stop": 771}, }, ] mapper = fsspec.get_mapper("memory://") mapper["path"] = content def dummy_read_file_descriptor(f): f.read(2) return dummy_header monkeypatch.setattr(io, "read_file_descriptor", dummy_read_file_descriptor) monkeypatch.setattr(io, "record_types", dummy_record_types) with mapper.fs.open("path", mode="rb") as f: header, metadata_ = io.read_metadata(f, records_per_chunk=rpc) assert header == dummy_header assert metadata_ == expected class TestInit: @pytest.mark.parametrize( ["path", "expected"], ( ("IMG-HH-ALOS2225333100-180726-WWDR1.1__D-B3", "HH_scan3"), ("IMG-HV-ALOS2290760600-191011-WWDR1.5RUA", "HV"), ), ) def test_filename_to_groupname(self, path, expected): actual = sar_image.filename_to_groupname(path) assert actual == expected xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_sar_leader.py000066400000000000000000001726071501510365500242230ustar00rootroot00000000000000import fsspec import numpy as np import pytest from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.sar_leader import ( attitude, data_quality_summary, dataset_summary, facility_related_data, io, map_projection, metadata, platform_position, radiometric_data, ) from ceos_alos2.testing import assert_identical @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "preamble": "", "spare1": "", "dataset_summary_records_sequence_number": "", "sar_channel_id": "", "number_of_scene_reference": "", "average_terrain_height_above_ellipsoid_at_scene_center": "", "processing_scene_length": 1, "processing_scene_width": 2, "range_pulse_phase_coefficients": {}, "processing_code_of_processing_facility": "", "processing_algorithm_id": "", "radiometric_bias": 0, "radiometric_gain": "", "time_direction_indicator_along_pixel_direction": "", "spare72": "", "parameter_table_number_of_automatically_setting": 0, "image_annotation_segment": {}, "spare_width": 0, }, Group(path=None, url=None, data={}, attrs={"spare_width": 0}), id="ignored", ), pytest.param( {"scene_center_time": "2020101117213774"}, Group( path=None, url=None, data={}, attrs={"scene_center_time": "2020-10-11T17:21:37.740000"}, ), id="transformers", ), pytest.param( { "scene_id": "abc", "geodetic_latitude": (61.6, {"units": "deg"}), "range_pulse_amplitude_coefficients": {"a0": 0, "a1": 1}, "incidence_angle": ( {"a0": (0, {"units": "rad"}), "a1": (0.5, {"units": "rad/km"})}, {"formula": "def"}, ), }, Group( path=None, url=None, data={ "geodetic_latitude": Variable((), 61.6, {"units": "deg"}), "range_pulse_amplitude_coefficients": Group( path=None, url=None, data={}, attrs={"a0": 0, "a1": 1} ), "incidence_angle": Group( path=None, url=None, data={ "a0": Variable((), 0, {"units": "rad"}), "a1": Variable((), 0.5, {"units": "rad/km"}), }, attrs={"formula": "def"}, ), }, attrs={"scene_id": "abc"}, ), id="groups", ), ), ) def test_transform_dataset_summary(mapping, expected): actual = dataset_summary.transform_dataset_summary(mapping) assert_identical(actual, expected) class TestMapProjection: @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param({"a": 1}, {"a": 1}, id="no_projection"), pytest.param( { "map_projection_designator": "UTM-PROJECTION", "utm_projection": {"type": "UNIVERSAL TRANSVERSE MERCATOR"}, "ups_projection": {"type": ""}, "national_system_projection": {"projection_descriptor": ""}, }, {"projection": {"type": "UNIVERSAL TRANSVERSE MERCATOR"}}, id="utm", ), pytest.param( { "map_projection_designator": "UPS-PROJECTION", "utm_projection": {"zone": ""}, "ups_projection": {"type": "UNIVERSAL POLAR STEREOGRAPHIC"}, "national_system_projection": {"projection_descriptor": ""}, }, {"projection": {"type": "UNIVERSAL POLAR STEREOGRAPHIC"}}, id="ups", ), pytest.param( { "map_projection_designator": "LCC-PROJECTION", "utm_projection": {"zone": ""}, "ups_projection": {"type": ""}, "national_system_projection": { "projection_descriptor": "LAMBERT-CONFORMAL CONIC" }, }, {"projection": {"projection_descriptor": "LAMBERT-CONFORMAL CONIC"}}, id="lcc", ), pytest.param( { "map_projection_designator": "MER-PROJECTION", "utm_projection": {"zone": ""}, "ups_projection": {"type": ""}, "national_system_projection": {"projection_descriptor": "MERCATOR"}, }, {"projection": {"projection_descriptor": "MERCATOR"}}, id="mer", ), ), ) def test_filter_map_projection(self, mapping, expected): actual = map_projection.filter_map_projection(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"map_projection_type": "GEOREFERENCE"}, {"map_projection_type": "GEOREFERENCE"}, id="no_rename", ), pytest.param( {"number_of_pixels_per_line": 2, "number_of_lines": 4}, {"n_columns": 2, "n_rows": 4}, id="renames", ), ), ) def test_transform_general_info(self, mapping, expected): actual = map_projection.transform_general_info(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param({"datum_shift_parameters": {"a": 1, "b": 2}}, {}, id="ignored1"), pytest.param( {"reference_ellipsoid": "GRS80", "scale_factor": 1.1}, {"reference_ellipsoid": "GRS80"}, id="ignored2", ), ), ) def test_transform_ellipsoid_parameters(self, mapping, expected): actual = map_projection.transform_ellipsoid_parameters(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "map_origin": {"a": 1, "b": 2}, "standard_parallel": {"phi1": 1, "phi2": 2}, "standard_parallel2": {"param1": 1, "param2": 2}, "central_meridian": {"param1": 0, "param2": 1, "param3": 2}, }, {"standard_parallel": {"phi1": 1, "phi2": 2}}, id="ignored", ), ), ) def test_transform_projection(self, mapping, expected): actual = map_projection.transform_projection(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "projected": { "top_left_corner": { "northing": (7.5, {"units": "m"}), "easting": (10.0, {"units": "m"}), }, "top_right_corner": { "northing": (7.5, {"units": "m"}), "easting": (12.0, {"units": "m"}), }, "bottom_right_corner": { "northing": (6.5, {"units": "m"}), "easting": (12.0, {"units": "m"}), }, "bottom_left_corner": { "northing": (6.5, {"units": "m"}), "easting": (10.0, {"units": "m"}), }, } }, { "projected": { "corner": ( ["corner"], ["top_left", "top_right", "bottom_right", "bottom_left"], {}, ), "northing": (["corner"], [7.5, 7.5, 6.5, 6.5], {"units": "m"}), "easting": (["corner"], [10.0, 12.0, 12.0, 10.0], {"units": "m"}), } }, id="projected", ), pytest.param( { "geographic": { "top_left_corner": { "latitude": (7.5, {"units": "deg"}), "longitude": (10.0, {"units": "deg"}), }, "top_right_corner": { "latitude": (7.5, {"units": "deg"}), "longitude": (12.0, {"units": "deg"}), }, "bottom_right_corner": { "latitude": (6.5, {"units": "deg"}), "longitude": (12.0, {"units": "deg"}), }, "bottom_left_corner": { "latitude": (6.5, {"units": "deg"}), "longitude": (10.0, {"units": "deg"}), }, } }, { "geographic": { "corner": ( ["corner"], ["top_left", "top_right", "bottom_right", "bottom_left"], {}, ), "latitude": (["corner"], [7.5, 7.5, 6.5, 6.5], {"units": "deg"}), "longitude": (["corner"], [10.0, 12.0, 12.0, 10.0], {"units": "deg"}), } }, id="geographic", ), pytest.param( { "terrain_heights_relative_to_ellipsoid": { "top_left_corner": (1.2, {"units": "m"}), "top_right_corner": (1.3, {"units": "m"}), "bottom_right_corner": (1.1, {"units": "m"}), "bottom_left_corner": (1.0, {"units": "m"}), } }, {}, id="ignored", ), ), ) def test_transform_corner_points(self, mapping, expected): actual = map_projection.transform_corner_points(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"a": ({"a1": 1, "a2": 2}, {"abc": "def"})}, { "a": ( { "names": ("names", ["a1", "a2"], {}), "coefficients": ("names", [1, 2], {}), }, {"abc": "def"}, ) }, id="reordering", ), pytest.param( { "map_projection_to_pixels": ({"a": 1}, {}), "pixels_to_map_projection": ({"a": 1}, {}), }, { "projected_to_image": ( {"names": ("names", ["a"], {}), "coefficients": ("names", [1], {})}, {}, ), "image_to_projected": ( {"names": ("names", ["a"], {}), "coefficients": ("names", [1], {})}, {}, ), }, id="renaming", ), ), ) def test_transform_conversion_coefficients(self, mapping, expected): actual = map_projection.transform_conversion_coefficients(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"spare1": "", "blanks10": "", "a": {}}, Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), id="spares", ), pytest.param( {"preamble": {}}, Group(path=None, url=None, data={}, attrs={}), id="ignored" ), pytest.param( { "map_projection_designator": "UTM-PROJECTION", "utm_projection": {"type": "UNIVERSAL TRANSVERSE MERCATOR"}, "ups_projection": {"type": ""}, "national_system_projection": {"map_projection_description": ""}, }, Group( path=None, url=None, data={ "projection": Group( path=None, url=None, data={}, attrs={"type": "UNIVERSAL TRANSVERSE MERCATOR"}, ) }, attrs={}, ), id="filtered", ), pytest.param( {"conversion_coefficients": {"a": ({"a1": 1, "a2": 2}, {"abc": "def"})}}, Group( path=None, url=None, data={ "conversion_coefficients": Group( path=None, url=None, data={ "a": Group( path=None, url=None, data={ "names": Variable("names", ["a1", "a2"], {}), "coefficients": Variable("names", [1, 2], {}), }, attrs={"abc": "def"}, ) }, attrs={}, ) }, attrs={}, ), id="transformed", ), pytest.param( { "map_projection_general_information": {}, "map_projection_ellipsoid_parameters": {}, }, Group( path=None, url=None, data={ "general_information": Group(path=None, url=None, data={}, attrs={}), "ellipsoid_parameters": Group(path=None, url=None, data={}, attrs={}), }, attrs={}, ), id="renamed", ), ), ) def test_transform_map_projection(self, mapping, expected): actual = map_projection.transform_map_projection(mapping) assert_identical(actual, expected) class TestPlatformPositions: @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param({"date": "2011 07 16", "seconds_of_day": 0}, "2011-07-16T00:00:00"), pytest.param({"date": "2021 12 31", "seconds_of_day": 81431}, "2021-12-31T22:37:11"), ), ) def test_composite_datetime(self, mapping, expected): actual = platform_position.transform_composite_datetime(mapping) assert actual == expected @pytest.mark.parametrize( ["elements", "expected"], ( pytest.param( [ { "position": { "x": (1, {"units": "m"}), "y": (1, {"units": "m"}), "z": (1, {"units": "m"}), } }, { "position": { "x": (2, {"units": "m"}), "y": (2, {"units": "m"}), "z": (2, {"units": "m"}), } }, { "position": { "x": (3, {"units": "m"}), "y": (3, {"units": "m"}), "z": (3, {"units": "m"}), } }, ], { "position": { "x": (["positions"], [1, 2, 3], {"units": "m"}), "y": (["positions"], [1, 2, 3], {"units": "m"}), "z": (["positions"], [1, 2, 3], {"units": "m"}), } }, id="position", ), pytest.param( [ { "velocity": { "x": (2, {"units": "m/s"}), "y": (3, {"units": "m/s"}), "z": (1, {"units": "m/s"}), } }, { "velocity": { "x": (3, {"units": "m/s"}), "y": (2, {"units": "m/s"}), "z": (2, {"units": "m/s"}), } }, { "velocity": { "x": (4, {"units": "m/s"}), "y": (3, {"units": "m/s"}), "z": (1, {"units": "m/s"}), } }, ], { "velocity": { "x": (["positions"], [2, 3, 4], {"units": "m/s"}), "y": (["positions"], [3, 2, 3], {"units": "m/s"}), "z": (["positions"], [1, 2, 1], {"units": "m/s"}), } }, id="velocity", ), ), ) def test_transform_positions(self, elements, expected): actual = platform_position.transform_positions(elements) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"preamble": {}, "number_of_data_points": 28, "greenwich_mean_hour_angle": 0}, Group(path=None, url=None, data={}, attrs={}), id="ignored", ), pytest.param( {"spare10": ""}, Group(path=None, url=None, data={}, attrs={}), id="spares" ), pytest.param( { "datetime_of_first_point": { "date": "2011 07 16", "day_of_year": "197", "seconds_of_day": 0, }, "occurrence_flag_of_a_leap_second": 0, }, Group( path=None, url=None, data={}, attrs={"datetime_of_first_point": "2011-07-16T00:00:00", "leap_second": False}, ), id="transformed1", ), pytest.param( { "positions": [ { "position": { "x": (1, {"units": "m"}), "y": (1, {"units": "m"}), "z": (1, {"units": "m"}), }, "velocity": { "x": (4, {"units": "m/s"}), "y": (3, {"units": "m/s"}), "z": (6, {"units": "m/s"}), }, } ] }, Group( path=None, url=None, data={ "positions": Group( path=None, url=None, data={ "position": Group( path=None, url=None, data={ "x": Variable(["positions"], [1], {"units": "m"}), "y": Variable(["positions"], [1], {"units": "m"}), "z": Variable(["positions"], [1], {"units": "m"}), }, attrs={}, ), "velocity": Group( path=None, url=None, data={ "x": Variable(["positions"], [4], {"units": "m/s"}), "y": Variable(["positions"], [3], {"units": "m/s"}), "z": Variable(["positions"], [6], {"units": "m/s"}), }, attrs={}, ), }, attrs={}, ) }, attrs={}, ), id="transformed2", ), pytest.param( { "occurrence_flag_of_a_leap_second": 1, "time_interval_between_data_points": (60.0, {"units": "s"}), }, Group( path=None, url=None, data={"sampling_frequency": Variable((), 60.0, {"units": "s"})}, attrs={"leap_second": True}, ), id="renamed", ), pytest.param( {"orbital_elements_designator": "high_precision"}, Group( path=None, url=None, data={ "orbital_elements": Group( path=None, url=None, data={}, attrs={"type": "high_precision"} ) }, attrs={}, ), id="moved", ), ), ) def test_transform_platform_position(self, mapping, expected): actual = platform_position.transform_platform_position(mapping) assert_identical(actual, expected) class TestAttitude: @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"day_of_year": [0, 1], "millisecond_of_day": [548, 749]}, np.array([548000000, 86400749000000], dtype="timedelta64[ns]"), ), pytest.param( {"day_of_year": [9], "millisecond_of_day": [698]}, np.array([777600698000000], dtype="timedelta64[ns]"), ), ), ) def test_transform_time(self, mapping, expected): actual = attitude.transform_time(mapping) np.testing.assert_equal(actual, expected) @pytest.mark.parametrize( ["dim", "var", "expected"], ( pytest.param("x", 1, ("x", 1, {})), pytest.param("x", (1, {"b": 1}), ("x", 1, {"b": 1})), pytest.param("x", {"a": 1}, {"a": ("x", 1, {})}), ), ) def test_prepend_dim(self, dim, var, expected): actual = attitude.prepend_dim(dim, var) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "roll": [(1, {"units": "deg"}), (2, {"units": "deg"})], "pitch": [(2, {"units": "deg"}), (3, {"units": "deg"})], "yaw": [(3, {"units": "deg"}), (4, {"units": "deg"})], }, { "roll": ([1, 2], {"units": "deg"}), "pitch": ([2, 3], {"units": "deg"}), "yaw": ([3, 4], {"units": "deg"}), }, ), pytest.param( {"roll_error": [0, 1], "pitch_error": [1, 0], "yaw_error": [1, 1]}, { "roll_error": [False, True], "pitch_error": [True, False], "yaw_error": [True, True], }, ), ), ) def test_transform_section(self, mapping, expected): actual = attitude.transform_section(mapping) assert actual == expected @pytest.mark.parametrize( ["raw_points", "expected"], ( pytest.param( [ {"a": {"b": 1, "c": 2}, "d": {"e": 3}}, {"a": {"b": 2, "c": 3}, "d": {"e": 4}}, {"a": {"b": 3, "c": 4}, "d": {"e": 5}}, ], Group( path=None, url=None, data={ "a": Group( path=None, url=None, data={ "b": Variable(["points"], [1, 2, 3], {}), "c": Variable(["points"], [2, 3, 4], {}), }, attrs={"coordinates": ["time"]}, ), "d": Group( path=None, url=None, data={"e": Variable(["points"], [3, 4, 5], {})}, attrs={"coordinates": ["time"]}, ), }, attrs={}, ), id="transformed", ), pytest.param( [ {"time": {"day_of_year": 0, "millisecond_of_day": 10}}, {"time": {"day_of_year": 0, "millisecond_of_day": 11}}, {"time": {"day_of_year": 0, "millisecond_of_day": 12}}, {"time": {"day_of_year": 0, "millisecond_of_day": 13}}, ], Group( path=None, url=None, data={ "attitude": Group( path=None, url=None, data={ "time": Variable( ["points"], np.array( [10000000, 11000000, 12000000, 13000000], dtype="timedelta64[ns]", ), {}, ) }, attrs={"coordinates": ["time"]}, ), "rates": Group( path=None, url=None, data={ "time": Variable( ["points"], np.array( [10000000, 11000000, 12000000, 13000000], dtype="timedelta64[ns]", ), {}, ) }, attrs={"coordinates": ["time"]}, ), }, attrs={}, ), id="time", ), ), ) def test_transform_attitude(self, raw_points, expected): mapping = {"data_points": raw_points} actual = attitude.transform_attitude(mapping) assert_identical(actual, expected) class TestRadiometricData: @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"a": {"a": 0, "b": 1, "c": 2, "d": 3}}, ( { "a": (["i", "j"], [[0, 1], [2, 3]], {}), "i": ( "i", ["horizontal", "vertical"], {"long_name": "reception polarization"}, ), "j": ( "j", ["horizontal", "vertical"], {"long_name": "transmission polarization"}, ), }, {}, ), ), pytest.param( ({"a": {"a": 0, "b": 1, "c": 2, "d": 3}}, {"a": "def"}), ( { "a": (["i", "j"], [[0, 1], [2, 3]], {}), "i": ( "i", ["horizontal", "vertical"], {"long_name": "reception polarization"}, ), "j": ( "j", ["horizontal", "vertical"], {"long_name": "transmission polarization"}, ), }, {"a": "def"}, ), ), pytest.param( { "a": {"a": 1j, "b": 2j, "c": 3j, "d": 4j}, "b": {"f": 0j, "e": 1j, "d": 2j, "c": 3j}, }, ( { "a": (["i", "j"], [[1j, 2j], [3j, 4j]], {}), "b": (["i", "j"], [[0j, 1j], [2j, 3j]], {}), "i": ( "i", ["horizontal", "vertical"], {"long_name": "reception polarization"}, ), "j": ( "j", ["horizontal", "vertical"], {"long_name": "transmission polarization"}, ), }, {}, ), ), ), ) def test_transform_matrices(self, mapping, expected): actual = radiometric_data.transform_matrices(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "preamble": "", "radiometric_data_records_sequence_number": 0, "number_of_radiometric_fields": 1, "blanks": "", }, Group(path=None, url=None, data={}, attrs={}), id="ignored", ), pytest.param( {"calibration_factor": (-10.0, {"formula": "abc"})}, Group( path=None, url=None, data={"calibration_factor": Variable((), -10.0, {"formula": "abc"})}, attrs={}, ), id="calibration_factor", ), pytest.param( { "distortion_matrix": ( { "a": {"a": 1, "b": 2, "c": 3, "d": 4}, "b": {"f": 0, "e": 1, "d": 2, "c": 3}, }, {"formula": "def"}, ) }, Group( path=None, url=None, data={ "distortion_matrix": Group( path=None, url=None, data={ "a": Variable(["i", "j"], [[1, 2], [3, 4]], {}), "b": Variable(["i", "j"], [[0, 1], [2, 3]], {}), "i": Variable( ["i"], ["horizontal", "vertical"], {"long_name": "reception polarization"}, ), "j": Variable( ["j"], ["horizontal", "vertical"], {"long_name": "transmission polarization"}, ), }, attrs={"formula": "def"}, ) }, attrs={}, ), id="distortion_matrix", ), ), ) def test_transform_radiometric_data(self, mapping, expected): actual = radiometric_data.transform_radiometric_data(mapping) assert_identical(actual, expected) class TestDataQualitySummary: @pytest.mark.parametrize( ["mapping", "key", "expected"], ( pytest.param( { "a": [ {"b": (1, {"u": "v"}), "c": (-1, {"u": "v"})}, {"b": (2, {"u": "v"}), "c": (-2, {"u": "v"})}, ] }, "a", {"b": ("channel", [1, 2], {"u": "v"}), "c": ("channel", [-1, -2], {"u": "v"})}, ), pytest.param( {"f1": [{"r": 1, "o": (-1, {"a": "e"})}, {"r": 2, "o": (-2, {"a": "e"})}]}, "f1", {"r": ("channel", [1, 2], {}), "o": ("channel", [-1, -2], {"a": "e"})}, ), ), ) def test_transform_relative(self, mapping, key, expected): actual = data_quality_summary.transform_relative(mapping, key) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"a": {"spare1": ""}, "blanks": ""}, Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), id="spares", ), pytest.param( {"preamble": {}, "record_number": 1}, Group(path=None, url=None, data={}, attrs={}), id="ignored", ), pytest.param( { "relative_radiometric_quality": { "nominal_relative_radiometric_calibration_uncertainty": [{"a": 1}, {"a": 2}] }, "relative_geometric_quality": { "relative_misregistration_error": [ {"b": (-1, {"b": 1})}, {"b": (-2, {"b": 1})}, {"b": (-3, {"b": 1})}, ] }, }, Group( path=None, url=None, data={ "relative_radiometric_quality": Group( path=None, url=None, data={"a": Variable("channel", [1, 2], {})}, attrs={}, ), "relative_geometric_quality": Group( path=None, url=None, data={"b": Variable("channel", [-1, -2, -3], {"b": 1})}, attrs={}, ), }, attrs={}, ), id="transformed", ), ), ) def test_transform_data_quality_summary(self, mapping, expected): actual = data_quality_summary.transform_data_quality_summary(mapping) assert_identical(actual, expected) class TestFacilityRelatedData: @pytest.mark.parametrize( ["mapping", "dim", "expected"], ( pytest.param( ({"a": [1, 2]}, {"u": "v"}), "dim", ({"a": ("dim", [1, 2], {})}, {"u": "v"}), id="array", ), pytest.param(({"b": 1.0}, {}), "dim", ({"b": ((), 1.0, {})}, {}), id="scalar"), ), ) def test_transform_group(self, mapping, dim, expected): actual = facility_related_data.transform_group(mapping, dim) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "preamble": {}, "spare11": "", "blanks4": "", "record_sequence_number": 1, "system_reserve": "", }, Group(path=None, url=None, data={}, attrs={}), id="ignored", ), pytest.param( {"prf_switching_flag": 0}, Group(path=None, url=None, data={}, attrs={"prf_switching": False}), id="transformed1", ), pytest.param( { "conversion_from_map_projection_to_pixel": ( {"a": [1, 2], "b": [3, 4]}, {"a": "b"}, ) }, Group( path=None, url=None, data={ "projected_to_image": Group( path=None, url=None, data={ "a": Variable("mid_precision_coeffs", [1, 2], {}), "b": Variable("mid_precision_coeffs", [3, 4], {}), }, attrs={"a": "b"}, ) }, attrs={}, ), id="transformed2", ), pytest.param( {"conversion_from_pixel_to_geographic": ({"a": [1, 2], "b": 1.0}, {"d": "e"})}, Group( path=None, url=None, data={ "image_to_geographic": Group( path=None, url=None, data={ "a": Variable("high_precision_coeffs", [1, 2], {}), "b": Variable((), 1.0, {}), }, attrs={"d": "e"}, ) }, attrs={}, ), id="transformed3", ), pytest.param( {"conversion_from_geographic_to_pixel": ({"c": [1, 2], "d": 1.0}, {"f": "e"})}, Group( path=None, url=None, data={ "geographic_to_image": Group( path=None, url=None, data={ "c": Variable("high_precision_coeffs", [1, 2], {}), "d": Variable((), 1.0, {}), }, attrs={"f": "e"}, ) }, attrs={}, ), id="transformed4", ), ), ) def test_transform_record5(self, mapping, expected): actual = facility_related_data.transform_record5(mapping) assert_identical(actual, expected) class TestMetadata: @pytest.mark.parametrize( ["group", "expected"], ( pytest.param( Group( path=None, url=None, data={"attitude": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), Group( path=None, url=None, data={"attitude": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), id="without_platform_position", ), pytest.param( Group( path=None, url=None, data={"platform_position": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), Group( path=None, url=None, data={"platform_position": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), id="without_attitude", ), pytest.param( Group(path=None, url=None, data={}, attrs={}), Group(path=None, url=None, data={}, attrs={}), id="without_either", ), pytest.param( Group( path=None, url=None, data={ "platform_position": Group( path=None, url=None, data={}, attrs={"datetime_of_first_point": "2012-11-10T01:31:54"}, ), "attitude": Group( path=None, url=None, data={ "positions": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [3600000000000, 7200000000000], dtype="timedelta64[ns]", ), {}, ) }, attrs={}, ) }, attrs={}, ), }, attrs={}, ), Group( path=None, url=None, data={ "platform_position": Group( path=None, url=None, data={}, attrs={"datetime_of_first_point": "2012-11-10T01:31:54"}, ), "attitude": Group( path=None, url=None, data={ "positions": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [ "2012-01-01T01:00:00.000000000", "2012-01-01T02:00:00.000000000", ], dtype="datetime64[ns]", ), {}, ) }, attrs={}, ) }, attrs={}, ), }, attrs={}, ), id="fixed1", ), pytest.param( Group( path=None, url=None, data={ "platform_position": Group( path=None, url=None, data={}, attrs={"datetime_of_first_point": "1986-05-24T16:52:01"}, ), "attitude": Group( path=None, url=None, data={ "positions": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [19329831000000000, 20467200000000000], dtype="timedelta64[ns]", ), {}, ) }, attrs={}, ), "velocity": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [19329831000000000, 20467200000000000], dtype="timedelta64[ns]", ), {}, ) }, attrs={}, ), }, attrs={}, ), }, attrs={}, ), Group( path=None, url=None, data={ "platform_position": Group( path=None, url=None, data={}, attrs={"datetime_of_first_point": "1986-05-24T16:52:01"}, ), "attitude": Group( path=None, url=None, data={ "positions": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [ "1986-08-12T17:23:51.000000000", "1986-08-25T21:20:00.000000000", ], dtype="datetime64[ns]", ), {}, ) }, attrs={}, ), "velocity": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [ "1986-08-12T17:23:51.000000000", "1986-08-25T21:20:00.000000000", ], dtype="datetime64[ns]", ), {}, ) }, attrs={}, ), }, attrs={}, ), }, attrs={}, ), id="fixed2", ), ), ) def test_fix_attitude_time(self, group, expected): actual = metadata.fix_attitude_time(group) assert_identical(actual, expected) @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "file_descriptor": {"a": ""}, "facility_related_data_1": {}, "facility_related_data_2": {}, "facility_related_data_3": {}, "facility_related_data_4": {}, }, Group(path=None, url=None, data={}, attrs={}), id="ignored", ), pytest.param( {"map_projection": []}, Group(path=None, url=None, data={}, attrs={}), id="empty-items", ), pytest.param( {"dataset_summary": {"scene_center_time": "2020101117213774"}}, Group( path=None, url=None, data={ "dataset_summary": Group( path=None, url=None, data={}, attrs={"scene_center_time": "2020-10-11T17:21:37.740000"}, ) }, attrs={}, ), id="transformed-dataset_summary", ), pytest.param( {"map_projection": [{"map_projection_general_information": {}}]}, Group( path=None, url=None, data={ "map_projection": Group( path=None, url=None, data={ "general_information": Group(path=None, url=None, data={}, attrs={}) }, attrs={}, ) }, attrs={}, ), id="transformed-map_projection", ), pytest.param( {"platform_position": {"occurrence_flag_of_a_leap_second": True}}, Group( path=None, url=None, data={ "platform_position": Group( path=None, url=None, data={}, attrs={"leap_second": True} ) }, attrs={}, ), id="transformed-platform_position", ), pytest.param( { "attitude": { "data_points": [ {"a": {"b": 1, "c": 2}}, {"a": {"b": 2, "c": 3}}, {"a": {"b": 3, "c": 4}}, ] } }, Group( path=None, url=None, data={ "attitude": Group( path=None, url=None, data={ "a": Group( path=None, url=None, data={ "b": Variable(["points"], [1, 2, 3], {}), "c": Variable(["points"], [2, 3, 4], {}), }, attrs={"coordinates": ["time"]}, ) }, attrs={}, ) }, attrs={}, ), id="transformed-attitude", ), pytest.param( {"radiometric_data": {"calibration_factor": (-10.0, {"formula": "abc"})}}, Group( path=None, url=None, data={ "radiometric_data": Group( path=None, url=None, data={"calibration_factor": Variable((), -10.0, {"formula": "abc"})}, attrs={}, ) }, attrs={}, ), id="transformed-radiometric_data", ), pytest.param( {"data_quality_summary": {"number_of_channels": 8, "record_number": 1}}, Group( path=None, url=None, data={ "data_quality_summary": Group( path=None, url=None, data={}, attrs={"number_of_channels": 8} ) }, attrs={}, ), id="transformed-data_quality_summary", ), pytest.param( {"facility_related_data_5": {"prf_switching_flag": 1}}, Group( path=None, url=None, data={ "transformations": Group( path=None, url=None, data={}, attrs={"prf_switching": True} ) }, attrs={}, ), id="transformed-record5", ), pytest.param( { "radiometric_data": {"calibration_factor": (-10.0, {"formula": "abc"})}, "facility_related_data_5": {"prf_switching_flag": 1}, }, Group( path=None, url=None, data={ "radiometric_data": Group( path=None, url=None, data={"calibration_factor": Variable((), -10.0, {"formula": "abc"})}, attrs={}, ), "transformations": Group( path=None, url=None, data={}, attrs={"prf_switching": True} ), }, attrs={}, ), id="transformed-multiple", ), pytest.param( {"facility_related_data_5": {"prf_switching_flag": 1}}, Group( path=None, url=None, data={ "transformations": Group( path=None, url=None, data={}, attrs={"prf_switching": True} ) }, attrs={}, ), id="renamed", ), pytest.param( { "attitude": { "data_points": [ {"time": {"day_of_year": 0, "millisecond_of_day": 10}}, {"time": {"day_of_year": 0, "millisecond_of_day": 11}}, {"time": {"day_of_year": 0, "millisecond_of_day": 12}}, {"time": {"day_of_year": 0, "millisecond_of_day": 13}}, ] }, "platform_position": { "datetime_of_first_point": { "date": "2011 07 16", "day_of_year": "197", "seconds_of_day": 0, } }, }, Group( path=None, url=None, data={ "attitude": Group( path=None, url=None, data={ "attitude": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [ "2011-01-01T00:00:00.010", "2011-01-01T00:00:00.011", "2011-01-01T00:00:00.012", "2011-01-01T00:00:00.013", ], dtype="datetime64[ns]", ), {}, ) }, attrs={"coordinates": ["time"]}, ), "rates": Group( path=None, url=None, data={ "time": Variable( "points", np.array( [ "2011-01-01T00:00:00.010", "2011-01-01T00:00:00.011", "2011-01-01T00:00:00.012", "2011-01-01T00:00:00.013", ], dtype="datetime64[ns]", ), {}, ) }, attrs={"coordinates": ["time"]}, ), }, attrs={}, ), "platform_position": Group( path=None, url=None, data={}, attrs={"datetime_of_first_point": "2011-07-16T00:00:00"}, ), }, attrs={}, ), id="postprocessed", ), ), ) def test_transform_metadata(self, mapping, expected): actual = metadata.transform_metadata(mapping) assert_identical(actual, expected) @pytest.mark.parametrize( ["path", "expected"], ( pytest.param("led1", FileNotFoundError("Cannot open .+"), id="not-existing"), pytest.param( "led2", Group( path=None, url=None, data={ "transformations": Group( path=None, url=None, data={}, attrs={"prf_switching": False} ) }, attrs={}, ), id="existing", ), ), ) def test_open_sar_leader(monkeypatch, path, expected): binary = b"\x01\x03" mapping = {"facility_related_data_5": {"prf_switching_flag": 0}} recorded_binary = [] def fake_parse_data(data): recorded_binary.append(data) return mapping monkeypatch.setattr(io, "parse_data", fake_parse_data) mapper = fsspec.get_mapper("memory://") mapper["led2"] = binary if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): io.open_sar_leader(mapper, path) return actual = io.open_sar_leader(mapper, path) assert_identical(actual, expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_summary.py000066400000000000000000000533601501510365500236110ustar00rootroot00000000000000import fsspec import pytest from ceos_alos2 import summary from ceos_alos2.hierarchy import Group from ceos_alos2.testing import assert_identical try: ExceptionGroup except NameError: # pragma: no cover from exceptiongroup import ExceptionGroup def compare_exceptions(e1, e2): return type(e1) is type(e2) and e1.args == e2.args @pytest.mark.parametrize( ["line", "expected"], ( pytest.param( 'Scs_SceneShift="0"', {"section": "Scs", "keyword": "SceneShift", "value": "0"}, id="valid_line1", ), pytest.param( 'Pds_ProductID="WWDR1.1__D"', {"section": "Pds", "keyword": "ProductID", "value": "WWDR1.1__D"}, id="valid_line2", ), pytest.param('Scs_SceneShift"0"', ValueError("invalid line"), id="invalid_line1"), pytest.param( 'PdsProductID="WWDR1.1__D"', ValueError("invalid line"), id="invalid_line2", ), ), ) def test_parse_line(line, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): summary.parse_line(line) return actual = summary.parse_line(line) assert actual == expected @pytest.mark.parametrize( ["content", "expected"], ( pytest.param( 'Scs_SceneShift="0"\nPds_ProductID="WWDR1.1__D"', {"scs": {"SceneShift": "0"}, "pds": {"ProductID": "WWDR1.1__D"}}, id="valid_lines", ), pytest.param( 'Scs_SceneShift"0"\nPdsProductID="WWDR1.1__D"', ExceptionGroup( "failed to parse the summary", [ ValueError("line 00: invalid line"), ValueError("line 01: invalid line"), ], ), id="invalid_lines", ), ), ) def test_parse_summary(content, expected): if isinstance(expected, Exception): with pytest.raises(type(expected)) as e: summary.parse_summary(content) assert e.value.message == expected.message assert all( compare_exceptions(e1, e2) for e1, e2 in zip(e.value.exceptions, expected.exceptions) ) return actual = summary.parse_summary(content) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"1": "vd", "2": "l", "3": "im1", "4": "im2", "5": "tr"}, { "volume_directory": "vd", "sar_leader": "l", "sar_imagery": ["im1", "im2"], "sar_trailer": "tr", }, id="normal", ), pytest.param( {"1": "vd", "2": "l", "3": "im1", "4": "im2", "5": "im3", "6": "im4", "7": "tr"}, { "volume_directory": "vd", "sar_leader": "l", "sar_imagery": ["im1", "im2", "im3", "im4"], "sar_trailer": "tr", }, id="long", ), pytest.param({"1": "vd", "2": "l"}, ValueError(""), id="short"), ), ) def test_categorize_filenames(mapping, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): summary.categorize_filenames(mapping) return actual = summary.categorize_filenames(mapping) assert actual == expected @pytest.mark.parametrize( ["date", "expected"], ( pytest.param("20190109", "2019-01-09"), pytest.param("19971231", "1997-12-31"), ), ) def test_reformat_date(date, expected): actual = summary.reformat_date(date) assert actual == expected @pytest.mark.parametrize( ["date", "expected"], ( pytest.param("20190109 02:37:01.764", "2019-01-09T02:37:01.764"), pytest.param("19971231 12:29:46.182", "1997-12-31T12:29:46.182"), ), ) def test_to_isoformat(date, expected): actual = summary.to_isoformat(date) assert actual == expected @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( {"SceneId": "abc", "SiteDateTime": "def"}, Group(path=None, url=None, data={}, attrs={"SceneId": "abc", "SiteDateTime": "def"}), ), pytest.param( {"SceneId": "ab", "SiteDateTime": "cd"}, Group(path=None, url=None, data={}, attrs={"SceneId": "ab", "SiteDateTime": "cd"}), ), ), ) def test_transform_ordering_info(section, expected): actual = summary.transform_ordering_info(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( {"SceneID": "ALOS2290760600-191011", "SceneShift": "0"}, Group( path=None, url=None, data={}, attrs={ "mission_name": "ALOS2", "orbit_accumulation": 29076, "scene_frame": 600, "date": "2019-10-11", "SceneShift": 0, }, ), id="scene1", ), pytest.param( {"SceneID": "ALOS2225333200-180726", "SceneShift": "1"}, Group( path=None, url=None, data={}, attrs={ "mission_name": "ALOS2", "orbit_accumulation": 22533, "scene_frame": 3200, "date": "2018-07-26", "SceneShift": 1, }, ), id="scene2", ), ), ) def test_transform_scene_spec(section, expected): actual = summary.transform_scene_spec(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( {"ProductID": "WWDR1.1__D"}, Group( path=None, url=None, data={}, attrs={ "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.1", "processing_option": "not specified", "map_projection": "not specified", "orbit_direction": "descending", }, ), id="product_id1", ), pytest.param( {"ProductID": "WWDR1.5RUA"}, Group( path=None, url=None, data={}, attrs={ "observation_mode": "ScanSAR nominal 28MHz mode dual polarization", "observation_direction": "right looking", "processing_level": "level 1.5", "processing_option": "geo-reference", "map_projection": "UTM", "orbit_direction": "ascending", }, ), id="product_id2", ), pytest.param( {"PixelSpacing": "25.000000"}, Group(path=None, url=None, data={}, attrs={"PixelSpacing": 25.0}), id="floats", ), pytest.param( { "ResamplingMethod": "NN", "UTM_ZoneNo": "53", "OrbitDataPrecision": "Precision", "AttitudeDataPrecision": "Onboard", "MapDirection": "MapNorth", }, Group( path=None, url=None, data={}, attrs={ "ResamplingMethod": "nearest-neighbor", "UTM_ZoneNo": 53, "OrbitDataPrecision": "Precision", "AttitudeDataPrecision": "Onboard", "MapDirection": "MapNorth", }, ), id="other_metadata", ), ), ) def test_transform_product_spec(section, expected): actual = summary.transform_product_spec(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( { "SceneCenterDateTime": "20191011 14:43:15.525", "SceneStartDateTime": "20191011 14:42:49.525", "SceneEndDateTime": "20191011 14:43:41.524", }, Group( path=None, url=None, data={}, attrs={ "SceneCenterDateTime": "2019-10-11T14:43:15.525", "SceneStartDateTime": "2019-10-11T14:42:49.525", "SceneEndDateTime": "2019-10-11T14:43:41.524", }, ), id="datetime", ), pytest.param( { "ImageSceneCenterLatitude": "30.385", "ImageSceneCenterLongitude": "137.504", "OffNadirAngle": "21.3", }, Group( path=None, url=None, data={}, attrs={ "ImageSceneCenterLatitude": 30.385, "ImageSceneCenterLongitude": 137.504, "OffNadirAngle": 21.3, }, ), id="floats", ), ), ) def test_transform_image_info(section, expected): actual = summary.transform_image_info(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( { "CntOfL15ProductFileName": "5", "L15ProductFileName01": "a", "L15ProductFileName02": "b", "L15ProductFileName03": "c", "L15ProductFileName04": "d", "L15ProductFileName05": "e", }, Group( path="product_info", url=None, data={ "data_files": Group( path="data_files", url=None, data={}, attrs={ "volume_directory": "a", "sar_leader": "b", "sar_imagery": ["c", "d"], "sar_trailer": "e", }, ) }, attrs={}, ), id="data_files", ), pytest.param( { "NoOfPixels_1": " 9196", "NoOfPixels_2": " 8722", "NoOfLines_1": "60568", "NoOfLines_2": "75710", }, Group( path="product_info", url=None, data={ "shapes": Group( path="shapes", url=None, data={}, attrs={"1": (9196, 60568), "2": (8722, 75710)}, ) }, attrs={}, ), id="shapes", ), pytest.param( {"ProductDataSize": "798.2", "ProductFormat": "CEOS", "BitPixel": "16"}, Group( path="product_info", url=None, data={}, attrs={"ProductDataSize": 798.2, "ProductFormat": "CEOS", "BitPixel": 16}, ), id="other_metadata", ), ), ) def test_transform_product_info(section, expected): actual = summary.transform_product_info(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( {"TimeCheck": "GOOD", "AttitudeCheck": "POOR"}, Group( path=None, url=None, data={}, attrs={"TimeCheck": "GOOD", "AttitudeCheck": "POOR"} ), id="available", ), pytest.param( {"AbsoluteNavigationTime": "", "PRF_Check": ""}, Group( path=None, url=None, data={}, attrs={"AbsoluteNavigationTime": "N/A", "PRF_Check": "N/A"}, ), id="missing", ), ), ) def test_transform_autocheck(section, expected): actual = summary.transform_autocheck(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( {"PracticeResultCode": "GOOD"}, Group(path=None, url=None, data={}, attrs={"PracticeResultCode": "GOOD"}), id="good", ), pytest.param( {"PracticeResultCode": "FAIR"}, Group(path=None, url=None, data={}, attrs={"PracticeResultCode": "FAIR"}), id="fair", ), ), ) def test_transform_result_info(section, expected): actual = summary.transform_result_info(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["section", "expected"], ( pytest.param( {"ObservationDate": "20191011"}, Group(path=None, url=None, data={}, attrs={"ObservationDate": "2019-10-11"}), id="datetime1", ), pytest.param( {"ObservationDate": "20180726"}, Group(path=None, url=None, data={}, attrs={"ObservationDate": "2018-07-26"}), id="datetime2", ), pytest.param( {"ProcessFacility": "SCMO"}, Group( path=None, url=None, data={}, attrs={"ProcessFacility": "spacecraft control mission operation system"}, ), id="facility1", ), pytest.param( {"ProcessFacility": "EICS"}, Group( path=None, url=None, data={}, attrs={"ProcessFacility": "earth intelligence collection and sharing system"}, ), id="facility2", ), pytest.param( {"Satellite": "ALOS2", "Sensor": "SAR", "ProcessLevel": "1.1"}, Group( path=None, url=None, data={}, attrs={"Satellite": "ALOS2", "Sensor": "SAR", "ProcessLevel": "1.1"}, ), id="other_metadata", ), ), ) def test_transform_label_info(section, expected): actual = summary.transform_label_info(section) assert_identical(actual, expected) @pytest.mark.parametrize( ["sections", "expected"], ( pytest.param( {"odi": {"SceneId": "abc", "SiteDateTime": "def"}}, Group( path="summary", url=None, data={ "ordering_information": Group( None, None, {}, {"SceneId": "abc", "SiteDateTime": "def"} ) }, attrs={}, ), id="ordering_info", ), pytest.param( {"scs": {"SceneID": "ALOS2290760600-191011", "SceneShift": "0"}}, Group( path="summary", url=None, data={ "scene_specification": Group( path=None, url=None, data={}, attrs={ "mission_name": "ALOS2", "orbit_accumulation": 29076, "scene_frame": 600, "date": "2019-10-11", "SceneShift": 0, }, ) }, attrs={}, ), id="scene_spec", ), pytest.param( {"pds": {"PixelSpacing": "25.000000"}}, Group( path="summary", url=None, data={ "product_specification": Group( path=None, url=None, data={}, attrs={"PixelSpacing": 25.0} ) }, attrs={}, ), id="product_spec", ), pytest.param( {"img": {"OffNadirAngle": "21.3"}}, Group( path="summary", url=None, data={ "image_information": Group( path=None, url=None, data={}, attrs={"OffNadirAngle": 21.3} ) }, attrs={}, ), id="image_info", ), pytest.param( {"pdi": {"BitPixel": "16"}}, Group( path="summary", url=None, data={ "product_information": Group( path=None, url=None, data={}, attrs={"BitPixel": 16} ) }, attrs={}, ), id="product_info", ), pytest.param( {"ach": {"AbsoluteNavigationTime": "", "PRF_Check": ""}}, Group( path="summary", url=None, data={ "autocheck": Group( path=None, url=None, data={}, attrs={"AbsoluteNavigationTime": "N/A", "PRF_Check": "N/A"}, ) }, attrs={}, ), id="autocheck", ), pytest.param( {"rad": {"PracticeResultCode": "FAIR"}}, Group( path="summary", url=None, data={ "result_information": Group( path=None, url=None, data={}, attrs={"PracticeResultCode": "FAIR"} ) }, attrs={}, ), id="result_info", ), pytest.param( {"lbi": {"Satellite": "ALOS2", "Sensor": "SAR", "ProcessLevel": "1.1"}}, Group( path="summary", url=None, data={ "label_information": Group( path=None, url=None, data={}, attrs={"Satellite": "ALOS2", "Sensor": "SAR", "ProcessLevel": "1.1"}, ) }, attrs={}, ), id="label_info", ), pytest.param( { "ach": {"AbsoluteNavigationTime": "", "PRF_Check": ""}, "rad": {"PracticeResultCode": "FAIR"}, }, Group( path="summary", url=None, data={ "autocheck": Group( path=None, url=None, data={}, attrs={"AbsoluteNavigationTime": "N/A", "PRF_Check": "N/A"}, ), "result_information": Group( path=None, url=None, data={}, attrs={"PracticeResultCode": "FAIR"} ), }, attrs={}, ), id="multiple", ), ), ) def test_transform_summary(sections, expected): actual = summary.transform_summary(sections) assert_identical(actual, expected) @pytest.mark.parametrize( ["path", "expected"], ( pytest.param("something.txt", OSError("Cannot find the summary file (.+)")), pytest.param( "summary.txt", Group( path="summary", url=None, data={ "ordering_information": Group( path=None, url=None, data={}, attrs={"SceneId": "SARD000000276461-00043-005-000"}, ), "scene_specification": Group( path=None, url=None, data={}, attrs={"SceneShift": 0} ), "product_specification": Group( path=None, url=None, data={}, attrs={"ResamplingMethod": "nearest-neighbor"} ), "image_information": Group( path=None, url=None, data={}, attrs={"OffNadirAngle": 21.3} ), "product_information": Group( path=None, url=None, data={}, attrs={"ProductDataSize": 798.2} ), "autocheck": Group(path=None, url=None, data={}, attrs={"PRF_Check": "N/A"}), "result_information": Group( path=None, url=None, data={}, attrs={"PracticeResultCode": "GOOD"} ), "label_information": Group( path=None, url=None, data={}, attrs={"Sensor": "SAR"} ), }, attrs={}, ), ), ), ) def test_open_summary(path, expected): data = "\n".join( [ 'Odi_SceneId="SARD000000276461-00043-005-000"', 'Scs_SceneShift="0"', 'Pds_ResamplingMethod="NN"', 'Img_OffNadirAngle="21.3"', 'Pdi_ProductDataSize="798.2"', 'Ach_PRF_Check=""', 'Rad_PracticeResultCode="GOOD"', 'Lbi_Sensor="SAR"', ] ) mapper = fsspec.get_mapper("memory://") mapper["summary.txt"] = data.encode() if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): summary.open_summary(mapper, path) return actual = summary.open_summary(mapper, path) assert_identical(actual, expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_testing.py000066400000000000000000000453301501510365500235670ustar00rootroot00000000000000import numpy as np import pytest from ceos_alos2 import testing from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.tests.utils import create_dummy_array @pytest.mark.parametrize("b", ({"a": 1}, {"c": 1}, {"a": 1, "c": 1})) @pytest.mark.parametrize("a", ({"a": 1}, {"b": 1}, {"a": 1, "b": 1})) def test_dict_overlap(a, b): missing_left, common, missing_right = testing.dict_overlap(a, b) assert set(a) == set(missing_right) | set(common) assert set(b) == set(missing_left) | set(common) assert all(k in a and k in b for k in common) assert all(k not in a and k in b for k in missing_left) assert all(k in a and k not in b for k in missing_right) @pytest.mark.parametrize( ["item", "expected"], ( (np.int16(1), "1"), (np.float16(3.5), "3.5"), (np.complex64(1.5 + 1.5j), "(1.5+1.5j)"), (np.str_("abc"), "'abc'"), (np.datetime64("2011-04-27 00:00:00.0", "ms"), "2011-04-27T00:00:00.000"), (np.timedelta64(201, "s"), "201 seconds"), ), ) def test_format_item(item, expected): actual = testing.format_item(item) assert actual == expected @pytest.mark.parametrize( ["arr", "expected"], ( (np.array([0, 1], dtype="int8"), "int8 0 1"), (np.arange(10, dtype="int32"), "int32 0 1 2 ... 8 9"), ( create_dummy_array(shape=(4, 3)), "\n".join( ["Array(shape=(4, 3), dtype=int16, rpc=2)", " url: memory:///path/to/file"] ), ), ), ) def test_format_array(arr, expected): actual = testing.format_array(arr) assert actual == expected @pytest.mark.parametrize( ["var", "expected"], ( (Variable("x", np.array([0, 1], dtype="int8"), {}), "(x) int8 0 1"), (Variable(["x"], np.array([0, 1], dtype="int16"), {}), "(x) int16 0 1"), ( Variable(["x", "y"], np.array([[0, 1], [2, 3]], dtype="int32"), {}), "(x, y) int32 0 1 2 3", ), ( Variable("x", np.array([0, 1], dtype="int8"), {"a": 1}), "\n".join(["(x) int8 0 1", " a: 1"]), ), ( Variable("x", np.array([0, 1], dtype="int64"), {"a": 1, "b": "b"}), "\n".join(["(x) int64 0 1", " a: 1", " b: b"]), ), ), ) def test_format_variable(var, expected): actual = testing.format_variable(var) assert actual == expected @pytest.mark.parametrize("keys", (["ab"], ["ab", "cd"])) @pytest.mark.parametrize("side", ("left", "right")) def test_diff_mapping_missing(keys, side): diff = testing.diff_mapping_missing(keys, side) assert side in diff assert all(f"- {key}" in diff for key in keys) @pytest.mark.parametrize("name", ("attributes", "variables", "groups")) @pytest.mark.parametrize( ["left", "right", "unequal"], ( pytest.param({"a": 1, "b": 2}, {"a": 1, "b": 3}, ["b"]), pytest.param({"a": 2, "b": 2}, {"a": 1, "b": 3}, ["a", "b"]), pytest.param({"c": 1, "e": 5}, {"c": 2, "e": 2}, ["c", "e"]), pytest.param({"c": Variable("x", 1, {})}, {"c": Variable("y", 1, {})}, ["c"]), ), ) def test_diff_mapping_not_equal(left, right, unequal, name): actual = testing.diff_mapping_not_equal(left, right, name=name) assert actual.startswith(f"Differing {name}") assert all(f"L {k} " in actual and f"R {k} " in actual for k in unequal) @pytest.mark.parametrize("name", ("Attributes", "Variables", "Groups")) @pytest.mark.parametrize( ["left", "right", "missing_left", "common_unequal", "missing_right"], ( pytest.param({"b": 2}, {"b": 2, "c": 3}, True, False, False, id="missing_left"), pytest.param({"a": 1, "b": 2}, {"b": 2}, False, False, True, id="missing_right"), pytest.param({"b": 2}, {"b": 3}, False, True, False, id="unequal_common"), pytest.param({"b": 2}, {"b": 2}, False, False, False, id="all_equal"), pytest.param( {"b": 2}, {"b": 3, "c": 3}, True, True, False, id="unequal_common-missing_left" ), pytest.param( {"a": 1, "b": 2}, {"b": 3}, False, True, True, id="unequal_common-missing_right" ), pytest.param({"a": 1}, {"c": 3}, True, False, True, id="disjoint"), pytest.param({"a": 1, "b": 2}, {"b": 3, "c": 3}, True, True, True, id="all_different"), ), ) def test_diff_mapping(left, right, missing_left, common_unequal, missing_right, name): actual = testing.diff_mapping(left, right, name=name) assert actual.startswith(name) assert not missing_left or "Missing left" in actual assert not common_unequal or f"Differing {name.lower()}" in actual assert not missing_right or "Missing right" in actual @pytest.mark.parametrize("name", ["name1", "Name2"]) @pytest.mark.parametrize(["left", "right"], [(1, 2), ("a", "b")]) def test_diff_scalar(left, right, name): actual = testing.diff_scalar(left, right, name=name) assert name.title() in actual assert f"L {left}" in actual assert f"R {right}" in actual @pytest.mark.parametrize( ["left", "right", "expected"], ( pytest.param( np.array([1], dtype="int32"), create_dummy_array(), False, id="different_types" ), pytest.param( np.array([1, 2], dtype="int64"), np.array([1, 2], dtype="int64"), True, id="numpy-equal" ), pytest.param( np.array([1], dtype="int32"), np.array([2, 2], dtype="int32"), False, id="numpy-different_shapes", ), pytest.param( np.array([1], dtype="int32"), np.array([2], dtype="float32"), False, id="numpy-different_values", ), pytest.param( create_dummy_array(dtype="int32"), create_dummy_array(dtype="float64"), False, id="array-different_dtypes", ), pytest.param(create_dummy_array(), create_dummy_array(), True, id="array-equal"), ), ) def test_compare_data(left, right, expected): actual = testing.compare_data(left, right) assert actual == expected @pytest.mark.parametrize( ["left", "right", "expected"], ( pytest.param( np.array([1], dtype="int32"), np.array([2, 3], dtype="int8"), "\n".join([" L int32 1", " R int8 2 3"]), id="numpy", ), pytest.param( create_dummy_array(protocol="http"), create_dummy_array(protocol="memory"), "\n".join( [ "Differing filesystem:", " L protocol http", " R protocol memory", ] ), id="array-fs-protocol", ), pytest.param( create_dummy_array(path="/path/to1"), create_dummy_array(path="/path/to2"), "\n".join( [ "Differing filesystem:", " L path /path/to1", " R path /path/to2", ] ), id="array-fs-path", ), pytest.param( create_dummy_array(url="file1"), create_dummy_array(url="file2"), "\n".join( [ "Differing urls:", " L url file1", " R url file2", ] ), id="array-url", ), pytest.param( create_dummy_array(byte_ranges=[(0, 1), (2, 3), (3, 4), (4, 5)]), create_dummy_array(byte_ranges=[(0, 2), (2, 3), (3, 4), (4, 5)]), "\n".join( [ "Differing byte ranges:", " L line 1 (0, 1)", " R line 1 (0, 2)", ] ), id="array-byte_ranges", ), pytest.param( create_dummy_array(shape=(4, 3), byte_ranges=[]), create_dummy_array(shape=(6, 3), byte_ranges=[]), "\n".join( [ "Differing shapes:", " (4, 3) != (6, 3)", ] ), id="array-shape", ), pytest.param( create_dummy_array(dtype="int8"), create_dummy_array(dtype="int16"), "\n".join( [ "Differing dtypes:", " int8 != int16", ] ), id="array-dtype", ), pytest.param( create_dummy_array(type_code="IU2"), create_dummy_array(type_code="C*8"), "\n".join( [ "Differing type code:", " L type_code IU2", " R type_code C*8", ] ), id="array-type_code", ), pytest.param( create_dummy_array(records_per_chunk=2), create_dummy_array(records_per_chunk=1), "\n".join( [ "Differing chunksizes:", " L records_per_chunk 2", " R records_per_chunk 1", ] ), id="array-rpc", ), ), ) def test_diff_array(left, right, expected): actual = testing.diff_array(left, right) assert actual == expected @pytest.mark.parametrize( ["left", "right", "name", "expected"], ( pytest.param( np.array([1, 2], dtype="int8"), create_dummy_array(), "Data1", "\n".join( [ "Differing data1 types:", " L ", " R ", ] ), id="differing_types", ), pytest.param( np.array([1, 2], dtype="int8"), np.array([2, 3], dtype="int16"), "Data2", "\n".join( [ "Differing data2:", " L int8 1 2", " R int16 2 3", ] ), id="differing_data", ), ), ) def test_diff_data(left, right, name, expected): actual = testing.diff_data(left, right, name) assert actual == expected @pytest.mark.parametrize( ["sizes", "expected"], ( ({"a": 2, "b": 3}, "(a: 2, b: 3)"), ({"dim0": 5, "dim1": 4, "dim2": 3}, "(dim0: 5, dim1: 4, dim2: 3)"), ), ) def test_format_sizes(sizes, expected): actual = testing.format_sizes(sizes) assert actual == expected @pytest.mark.parametrize( ["left", "right", "expected"], ( pytest.param( Variable("x", np.array([1], dtype="int8"), {}), Variable("y", np.array([1], dtype="int8"), {}), "\n".join( [ "Left and right Variable objects are not equal", " Differing dimensions:", " (x: 1) != (y: 1)", ] ), id="dims", ), pytest.param( Variable("x", np.array([1, 2], dtype="int8"), {}), Variable("x", np.array([2, 3], dtype="int16"), {}), "\n".join( [ "Left and right Variable objects are not equal", " Differing data:", " L int8 1 2", " R int16 2 3", ] ), id="data", ), pytest.param( Variable("x", np.array([1], dtype="int16"), {"a": 1}), Variable("x", np.array([1], dtype="int16"), {"a": 2}), "\n".join( [ "Left and right Variable objects are not equal", " Attributes:", " Differing attributes:", " L a 1", " R a 2", ] ), id="attrs", ), ), ) def test_diff_variable(left, right, expected): actual = testing.diff_variable(left, right) assert actual == expected @pytest.mark.parametrize( ["left", "right", "expected"], ( pytest.param( Group(path="/a", url=None, data={}, attrs={}), Group(path="/b", url=None, data={}, attrs={}), "\n".join( [ "Differing Path:", "L /a", "R /b", ] ), id="path", ), pytest.param( Group(path=None, url="memory://a", data={}, attrs={}), Group(path=None, url="memory://b", data={}, attrs={}), "\n".join( [ "Differing Url:", "L memory://a", "R memory://b", ] ), id="url", ), pytest.param( Group( path=None, url=None, data={"a": Variable("x", np.array([1], dtype="int8"), {})}, attrs={}, ), Group( path=None, url=None, data={"a": Variable("y", np.array([1], dtype="int8"), {})}, attrs={}, ), "\n".join( [ "Variables:", " Differing variables:", " L a (x) int8 1", " R a (y) int8 1", ] ), id="variables", ), pytest.param( Group(path=None, url=None, data={}, attrs={"a": 1}), Group(path=None, url=None, data={}, attrs={"a": 2}), "\n".join( [ "Attributes:", " Differing attributes:", " L a 1", " R a 2", ] ), id="attrs", ), ), ) def test_diff_group(left, right, expected): actual = testing.diff_group(left, right) assert actual == expected @pytest.mark.parametrize( ["left", "right", "expected"], ( pytest.param( Group(path="/a", url=None, data={}, attrs={}), Group(path="/b", url=None, data={}, attrs={}), "\n".join( [ "Left and right Group objects are not equal", " Differing tree structure:", " Missing left:", " - /b", " Missing right:", " - /a", ] ), id="zero_level-disjoint", ), pytest.param( Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), Group(path=None, url=None, data={}, attrs={}), "\n".join( [ "Left and right Group objects are not equal", " Differing tree structure:", " Missing right:", " - /a", ] ), id="one_level-missing_right", ), pytest.param( Group(path=None, url=None, data={}, attrs={}), Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), "\n".join( [ "Left and right Group objects are not equal", " Differing tree structure:", " Missing left:", " - /a", ] ), id="one_level-missing_left", ), pytest.param( Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), Group( path=None, url=None, data={"b": Group(path=None, url=None, data={}, attrs={})}, attrs={}, ), "\n".join( [ "Left and right Group objects are not equal", " Differing tree structure:", " Missing left:", " - /b", " Missing right:", " - /a", ] ), id="one_level-disjoint", ), pytest.param( Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={"a": 1})}, attrs={}, ), Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={"a": 2})}, attrs={}, ), "\n".join( [ "Left and right Group objects are not equal", " Differing groups:", " Group /a:", " Attributes:", " Differing attributes:", " L a 1", " R a 2", ] ), id="one_level-common_differing", ), ), ) def test_diff_tree(left, right, expected): actual = testing.diff_tree(left, right) assert actual == expected @pytest.mark.parametrize( ["left", "right", "expected"], ( pytest.param(1, 1.0, AssertionError("types mismatch"), id="mismatching_types"), pytest.param( 1, 2, TypeError("can only compare Group and Variable and Array objects"), id="unsupported_types", ), pytest.param( Group(path=None, url="memory://a", data={}, attrs={}), Group(path=None, url="memory://b", data={}, attrs={}), AssertionError("Differing Url"), id="groups", ), pytest.param( Variable("x", np.array([1], dtype="int8"), {}), Variable("y", np.array([1], dtype="int8"), {}), AssertionError(".+Differing dimensions"), id="variables", ), pytest.param( create_dummy_array(dtype="int8"), create_dummy_array(dtype="int16"), AssertionError("Differing dtypes"), id="arrays", ), ), ) def test_assert_identical(left, right, expected): if expected is not None: with pytest.raises(type(expected), match=expected.args[0]): testing.assert_identical(left, right) return testing.assert_identical(left, right) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_transformers.py000066400000000000000000000107101501510365500246310ustar00rootroot00000000000000import pytest from ceos_alos2 import transformers from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.testing import assert_identical @pytest.mark.parametrize( ["s", "expected"], ( ("1990012012563297", "1990-01-20T12:56:32.970000"), ("2001112923595915", "2001-11-29T23:59:59.150000"), ), ) def test_parse_datetime(s, expected): actual = transformers.normalize_datetime(s) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param({"spare1": "", "spare2": ""}, {}, id="spares"), pytest.param({"blanks1": "", "blanks20": "", "blanks": ""}, {}, id="blanks"), pytest.param( {"spare_values": "", "blank_page": ""}, {"spare_values": "", "blank_page": ""}, id="false_positives", ), pytest.param({"a": {"b": {"blanks": ""}}}, {"a": {"b": {}}}, id="nested-dict"), pytest.param({"a": [{"b": {"blanks": ""}}]}, {"a": [{"b": {}}]}, id="nested-list"), ), ) def test_remove_spares(mapping, expected): actual = transformers.remove_spares(mapping) assert actual == expected @pytest.mark.parametrize( ["value", "expected"], ( pytest.param(("", ((), [])), "variable", id="variable"), pytest.param(("", ["abc"]), "variable", id="variable-array"), pytest.param(("", {}), "group", id="group"), pytest.param(("", ({}, {})), "group", id="group_with_attrs"), pytest.param(("", "abc"), "attribute", id="attribute"), ), ) def test_item_type(value, expected): actual = transformers.item_type(value) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( [{"a": {"b": 1, "c": 2}}, {"a": {"b": 2, "c": 3}}, {"a": {"b": 3, "c": 4}}], {"a": {"b": [1, 2, 3], "c": [2, 3, 4]}}, ), pytest.param( [ {"a": {"b": 1, "c": 2}, "d": {"e": 3}}, {"a": {"b": 2, "c": 3}, "d": {"e": 4}}, {"a": {"b": 3, "c": 4}, "d": {"e": 5}}, ], {"a": {"b": [1, 2, 3], "c": [2, 3, 4]}, "d": {"e": [3, 4, 5]}}, ), pytest.param([{"a": 1}, {"a": 2}, {"a": 3}], {"a": [1, 2, 3]}), ), ) def test_transform_nested(mapping, expected): actual = transformers.transform_nested(mapping) assert actual == expected @pytest.mark.parametrize( ["value", "expected"], ( pytest.param( [(1, {"abc": "def"}), (2, {"abc": "def"}), (3, {"abc": "def"})], ([1, 2, 3], {"abc": "def"}), ), pytest.param( [(6, {"cba": "fed"}), (2, {"cba": "fed"}), (3, {"cba": "fed"})], ([6, 2, 3], {"cba": "fed"}), ), pytest.param( [6, 2, 3], ([6, 2, 3], {}), ), ), ) def test_separate_attrs(value, expected): actual = transformers.separate_attrs(value) assert actual == expected @pytest.mark.parametrize( ["value", "expected"], ( pytest.param((1, {"a": 1}), Variable((), 1, {"a": 1})), pytest.param(("d1", [1, 2], {"b": 3}), Variable("d1", [1, 2], {"b": 3})), ), ) def test_as_variable(value, expected): actual = transformers.as_variable(value) assert_identical(actual, expected) @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( ({}, {"a": 1}), Group(path=None, url=None, data={}, attrs={"a": 1}), id="group_attrs" ), pytest.param( {"a": (1, {})}, Group(path=None, url=None, data={"a": Variable((), 1, {})}, attrs={}), id="variables", ), pytest.param( {"a": ({}, {"b": 2})}, Group( path=None, url=None, data={"a": Group(path=None, url=None, data={}, attrs={"b": 2})}, attrs={}, ), id="subgroups", ), pytest.param( {"a": ({"c": ("d", [1, 2], {})}, {"b": 2})}, Group( path=None, url=None, data={ "a": Group( path=None, url=None, data={"c": Variable(["d"], [1, 2], {})}, attrs={"b": 2} ) }, attrs={}, ), id="subgroups-variables", ), ), ) def test_as_group(mapping, expected): actual = transformers.as_group(mapping) assert_identical(actual, expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_utils.py000066400000000000000000000047431501510365500232550ustar00rootroot00000000000000import datetime import pytest from construct import EnumIntegerString from construct.lib.containers import ListContainer from ceos_alos2 import utils @pytest.mark.parametrize( ["data", "expected"], ( ("aaceajde", ["a", "c", "e", "j", "d"]), ("baebdwea", ["b", "a", "e", "d", "w"]), ), ) def test_unique(data, expected): actual = utils.unique(data) assert actual == expected @pytest.mark.parametrize( ["f", "args", "kwargs", "expected"], ( pytest.param(lambda x, y: x + y, [1, 2], {}, 3, id="args"), pytest.param(lambda x, y: x + y, (2,), {"y": 2}, 4, id="args+kwargs"), pytest.param(lambda x, y: x + y, (), {"x": 2, "y": 3}, 5, id="kwargs"), ), ) def test_starcall(f, args, kwargs, expected): actual = utils.starcall(f, args, **kwargs) assert actual == expected def test_to_dict(): container = { "_io": None, "a": { "aa": EnumIntegerString.new(1, "value"), "ab": 1, "ac": "ac", "ad": b"ad", "ae": 1j, "af": datetime.datetime(1999, 1, 1, 0, 0, 0), }, "b": ListContainer( [ {"ba1": 1}, {"ba2": 1}, ] ), "c": (1, 2, 3), "d": [1, 2, 3], } expected = { "a": { "aa": "value", "ab": 1, "ac": "ac", "ad": b"ad", "ae": 1j, "af": datetime.datetime(1999, 1, 1, 0, 0, 0), }, "b": [ {"ba1": 1}, {"ba2": 1}, ], "c": (1, 2, 3), "d": [1, 2, 3], } actual = utils.to_dict(container) assert actual == expected @pytest.mark.parametrize( ["data", "expected"], ( ("100", 100), ("100 MB", 100000000), ("100M", 100000000), ("5kB", 5000), ("5.4 kB", 5400), ("1kiB", 1024), ("1Mi", 2**20), ("1e6", 1000000), ("1e6 kB", 1000000000), ("MB", 1000000), (123, 123), (".5GB", 500000000), ("123 def", ValueError("Could not interpret '.*' as a byte unit")), ("abc# GB", ValueError("Could not interpret '.*' as a number")), ), ) def test_parse_bytes(data, expected): if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): utils.parse_bytes(data) return actual = utils.parse_bytes(data) assert actual == expected xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_volume_directory.py000066400000000000000000000157061501510365500255110ustar00rootroot00000000000000import fsspec import pytest from ceos_alos2.hierarchy import Group from ceos_alos2.testing import assert_identical from ceos_alos2.volume_directory import io, metadata class TestMetadata: @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "preamble": {"a": 1}, "ascii_ebcdic_flag": "A", "blanks": "", "spare": "", "local_use_segment": "", "total_number_of_physical_volumes_in_logical_volume": 1, "physical_volume_sequence_number_of_the_first_tape": 1, "physical_volume_sequence_number_of_the_last_tape": 1, "physical_volume_sequence_number_of_the_current_tape": 1, "file_number_in_the_logical_volume": 2, "logical_volume_within_a_volume_set": "a", "logical_volume_number_within_physical_volume": 4, "number_of_file_pointer_records": 4, "number_of_text_records_in_volume_directory": 1, }, {}, id="ignored1", ), pytest.param( {"number_of_file_pointer_records": 4, "volume_set_id": "abc"}, {"volume_set_id": "abc"}, id="ignored2", ), pytest.param( { "superstructure_format_control_document_id": "a", "superstructure_format_control_document_revision_level": "a", "superstructure_record_format_revision_level": "a", "software_release_and_revision_level": "001.001", "logical_volume_generation_country": "a", "logical_volume_generating_agency": "a", "logical_volume_generating_facility": "a", }, { "control_document_id": "a", "control_document_revision_level": "a", "record_format_revision_level": "a", "software_version": "001.001", "creation_country": "a", "creation_agency": "a", "creation_facility": "a", }, id="translations1", ), pytest.param( {"logical_volume_creation_datetime": "2020101117233798"}, {"creation_datetime": "2020-10-11T17:23:37.980000"}, id="translations2", ), pytest.param( {"creation_datetime": "2020101117233798"}, {"creation_datetime": "2020-10-11T17:23:37.980000"}, id="postprocessing", ), ), ) def test_transform_volume_descriptor(self, mapping, expected): actual = metadata.transform_volume_descriptor(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( {"preamble": {}, "ascii_ebcdic_flag": "a", "blanks": "", "physical_tape_id": 1}, {}, id="ignored1", ), pytest.param( {"blanks": "", "product_id": "PRODUCT:WWDR1.5RUA"}, {"product_id": "PRODUCT:WWDR1.5RUA"}, id="ignored2", ), pytest.param( {"product_id": "b", "location_and_datetime_of_product_creation": "a"}, {"product_id": "b", "product_creation": "a"}, id="translations", ), ), ) def test_transform_text(self, mapping, expected): actual = metadata.transform_text(mapping) assert actual == expected @pytest.mark.parametrize( ["mapping", "expected"], ( pytest.param( { "volume_descriptor": {"a": 1}, "file_descriptors": [{"b": 2}, {"c": 3}], "text_record": {"d": 4}, }, Group(path=None, url=None, data={}, attrs={"a": 1, "d": 4}), id="ignored", ), pytest.param( { "volume_descriptor": { "preamble": "a", "logical_volume_generation_country": "a", }, "text_record": {"blanks": "", "location_and_datetime_of_product_creation": "b"}, }, Group( path=None, url=None, data={}, attrs={"creation_country": "a", "product_creation": "b"}, ), id="transformers", ), pytest.param( {"volume_descriptor": {"a": 1, "b": 2}, "text_record": {"c": 3, "d": 4}}, Group(path=None, url=None, data={}, attrs={"a": 1, "b": 2, "c": 3, "d": 4}), id="flattened", ), ), ) def test_transform_record(self, mapping, expected): actual = metadata.transform_record(mapping) assert_identical(actual, expected) class TestHighLevel: @pytest.mark.skip(reason="need actual data") def test_parse_data(self): data = b"" actual = io.parse_data(data) # no need to verify the result, just make sure we get something usable assert isinstance(actual, dict) assert list(actual) == ["volume_descriptor", "file_descriptors", "text_record"] @pytest.mark.parametrize( ["path", "expected"], ( pytest.param("vol1", FileNotFoundError("Cannot open .+"), id="not-existing"), pytest.param( "vol2", Group( path=None, url=None, data={}, attrs={"creation_agency": "b", "product_creation": "c"}, ), id="existing", ), ), ) def test_open_volume_directory(self, monkeypatch, path, expected): binary = b"\x01\x02" recorded_binary = [] mapping = { "volume_descriptor": {"preamble": "a", "logical_volume_generating_agency": "b"}, "file_descriptors": [], "text_record": { "physical_tape_id": 2, "location_and_datetime_of_product_creation": "c", }, } def fake_parse_data(data): recorded_binary.append(data) return mapping monkeypatch.setattr(io, "parse_data", fake_parse_data) mapper = fsspec.get_mapper("memory://") mapper["vol2"] = binary if isinstance(expected, Exception): with pytest.raises(type(expected), match=expected.args[0]): io.open_volume_directory(mapper, path) return actual = io.open_volume_directory(mapper, path) assert_identical(actual, expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/test_xarray.py000066400000000000000000000146111501510365500234160ustar00rootroot00000000000000import numpy as np import pytest import xarray as xr from xarray.core.indexing import BasicIndexer, VectorizedIndexer from ceos_alos2 import xarray from ceos_alos2.hierarchy import Group, Variable from ceos_alos2.tests.utils import create_dummy_array class TestLazilyIndexedWrapper: def test_init(self): lock = xarray.SerializableLock() data = create_dummy_array() wrapper = xarray.LazilyIndexedWrapper(data, lock) assert wrapper.shape == data.shape assert wrapper.dtype == data.dtype assert wrapper.array is data assert wrapper.lock is lock @pytest.mark.parametrize( "indexer", ( pytest.param(BasicIndexer((0, 1))), pytest.param(VectorizedIndexer((slice(None), slice(None)))), ), ) def test_getitem(self, indexer): array = np.arange(6).reshape(2, 3) expected = array[indexer.tuple] lock = xarray.SerializableLock() wrapped = xarray.LazilyIndexedWrapper(array, lock) actual = wrapped[indexer] np.testing.assert_equal(actual, expected) @pytest.mark.parametrize( ["var", "expected"], ( pytest.param(Variable("x", np.array([1], dtype="int8"), {}), {}, id="numpy-no_chunks"), pytest.param( Variable("x", create_dummy_array(shape=(4,), records_per_chunk=2), {}), {"preferred_chunksizes": {"x": 2}}, id="array-chunks1d", ), pytest.param( Variable(["a", "b"], create_dummy_array(shape=(4, 3), records_per_chunk=1), {}), {"preferred_chunksizes": {"a": 1, "b": 3}}, id="array-chunks2d", ), ), ) def test_extract_encoding(var, expected): actual = xarray.extract_encoding(var) assert actual == expected @pytest.mark.parametrize( ["ds", "expected"], ( (xr.Dataset({"a": 1, "b": 2}, attrs={}), xr.Dataset({"a": 1, "b": 2}, attrs={})), ( xr.Dataset({"a": 1, "b": 2}, attrs={"coordinates": ["a"]}), xr.Dataset({"b": 2}, coords={"a": 1}, attrs={}), ), ( xr.Dataset({"a": 1, "b": 2}, attrs={"coordinates": ["b"]}), xr.Dataset({"a": 1}, coords={"b": 2}, attrs={}), ), ( xr.Dataset({"a": 1, "b": 2}, attrs={"coordinates": ["a", "b"]}), xr.Dataset({}, coords={"a": 1, "b": 2}, attrs={}), ), ), ) def test_decode_coords(ds, expected): actual = xarray.decode_coords(ds) xr.testing.assert_identical(actual, expected) @pytest.mark.parametrize( ["var", "expected"], ( pytest.param(Variable("x", np.array([1, 2], dtype="int8"), {"a": 1}), True, id="in_memory"), pytest.param(Variable(["x", "y"], create_dummy_array(), {"b": 3}), False, id="lazy"), ), ) def test_to_variable(var, expected): actual = xarray.to_variable(var) assert actual._in_memory == expected assert tuple(var.dims) == actual.dims assert var.attrs == actual.attrs @pytest.mark.parametrize("chunks", [None, {}, {"x": 1, "y": 2}]) @pytest.mark.parametrize( ["group", "expected"], ( pytest.param( Group(path=None, url=None, data={}, attrs={"a": 1, "b": 2, "c": 3}), xr.Dataset(attrs={"a": 1, "b": 2, "c": 3}), id="attrs", ), pytest.param( Group( path=None, url=None, data={ "a": Variable("x", np.array([1, 2, 3], dtype="int8"), {"a": 1}), "b": Variable(["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"}), }, attrs={}, ), xr.Dataset( { "a": ("x", np.array([1, 2, 3], dtype="int8"), {"a": 1}), "b": (["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"}), } ), id="variables", ), pytest.param( Group( path=None, url=None, data={ "c": Variable("x", np.array([1, 2, 3], dtype="int8"), {"a": 1}), "d": Variable(["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"}), }, attrs={"coordinates": ["d"]}, ), xr.Dataset( {"c": ("x", np.array([1, 2, 3], dtype="int8"), {"a": 1})}, coords={"d": (["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"})}, ), id="coords", ), ), ) def test_to_dataset(group, chunks, expected): actual = xarray.to_dataset(group, chunks=chunks) xr.testing.assert_identical(actual, expected) @pytest.mark.parametrize("chunks", [None, {}, {"x": 1, "y": 2}]) @pytest.mark.parametrize( ["group", "expected"], ( pytest.param( Group( path=None, url=None, data={ "c": Variable("x", np.array([1, 2, 3], dtype="int8"), {"a": 1}), "d": Variable(["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"}), }, attrs={"coordinates": ["d"]}, ), xr.DataTree.from_dict( { "/": xr.Dataset( {"c": ("x", [1, 2, 3], {"a": 1})}, coords={"d": (["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"})}, ) } ), id="flat", ), pytest.param( Group( path=None, url=None, data={ "c": Variable("x", np.array([1, 2, 3], dtype="int8"), {"a": 1}), "d": Group( path=None, url=None, data={"e": Variable(["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"})}, attrs={}, ), }, attrs={}, ), xr.DataTree.from_dict( { "/": xr.Dataset({"c": ("x", [1, 2, 3], {"a": 1})}), "d": xr.Dataset({"e": (["x", "y"], np.arange(12).reshape(3, 4), {"b": "abc"})}), } ), id="nested", ), ), ) def test_to_datatree(group, chunks, expected): actual = xarray.to_datatree(group, chunks=chunks) xr.testing.assert_identical(actual, expected) xarray-ceos-alos2-2025.05.0/ceos_alos2/tests/utils.py000066400000000000000000000012211501510365500222020ustar00rootroot00000000000000import fsspec from ceos_alos2.array import Array def create_dummy_array( *, protocol="memory", byte_ranges=None, path="/path/to", url="file", shape=(4, 3), dtype="int16", records_per_chunk=2, type_code="IU2", ): if byte_ranges is None: byte_ranges = [(x * 10 + 5, (x + 1) * 10) for x in range(shape[0])] fs = fsspec.filesystem(protocol) dirfs = fsspec.filesystem("dir", path=path, fs=fs) return Array( fs=dirfs, url=url, byte_ranges=byte_ranges, shape=shape, dtype=dtype, type_code=type_code, records_per_chunk=records_per_chunk, ) xarray-ceos-alos2-2025.05.0/ceos_alos2/transformers.py000066400000000000000000000045301501510365500224330ustar00rootroot00000000000000import datetime as dt from tlz.dicttoolz import keyfilter, merge_with, valmap from tlz.functoolz import curry, pipe from tlz.itertoolz import groupby, second from ceos_alos2.hierarchy import Group, Variable def normalize_datetime(string): return dt.datetime.strptime(string, "%Y%m%d%H%M%S%f").isoformat() def remove_spares(mapping): def predicate(k): if not k.startswith(("spare", "blanks")): return True k_ = k.removeprefix("spare").removeprefix("blanks") return k_ and not k_.isdigit() def _recursive(value): if isinstance(value, list): return list(map(remove_spares, value)) elif isinstance(value, dict): filtered = keyfilter(predicate, value) return valmap(_recursive, filtered) else: return value return _recursive(mapping) def item_type(item): value = second(item) if (isinstance(value, tuple) and not isinstance(value[0], dict)) or isinstance(value, list): return "variable" elif isinstance(value, dict) or (isinstance(value, tuple) and isinstance(value[0], dict)): return "group" else: return "attribute" def transform_nested(mapping): def _transform(value): if not isinstance(value, list) or not value or not isinstance(value[0], dict): return value return merge_with(list, *value) return pipe( mapping, curry(_transform), curry(valmap, _transform), ) def separate_attrs(data): if not isinstance(data, list) or not data or not isinstance(data[0], tuple): return data, {} values, metadata_ = zip(*data) metadata = metadata_[0] return list(values), metadata def as_variable(value): if len(value) == 2: data, attrs = value dims = () else: dims, data, attrs = value return Variable(dims, data, attrs) def as_group(mapping): if isinstance(mapping, tuple): mapping, additional_attrs = mapping else: additional_attrs = {} grouped = valmap(dict, dict(groupby(item_type, mapping.items()))) attrs = grouped.get("attribute", {}) variables = valmap(as_variable, grouped.get("variable", {})) groups = valmap(as_group, grouped.get("group", {})) return Group(path=None, url=None, data=variables | groups, attrs=attrs | additional_attrs) xarray-ceos-alos2-2025.05.0/ceos_alos2/utils.py000066400000000000000000000060641501510365500210520ustar00rootroot00000000000000import datetime from construct import EnumIntegerString from construct.lib.containers import ListContainer from tlz.dicttoolz import keymap def unique(seq): return list(dict.fromkeys(seq)) def starcall(f, args, **kwargs): return f(*args, **kwargs) def to_dict(container): if isinstance(container, EnumIntegerString): return str(container) if isinstance(container, (int, float, str, bytes, complex, datetime.datetime)): return container elif isinstance(container, (list, tuple)): if isinstance(container, ListContainer): type_ = list else: type_ = type(container) return type_(to_dict(elem) for elem in container) return {name: to_dict(section) for name, section in container.items() if name != "_io"} def rename(mapping, translations): return keymap(lambda k: translations.get(k, k), mapping) def remove_nesting_layer(mapping): def _remove(mapping): for key, value in mapping.items(): if not isinstance(value, dict): yield key, value continue yield from value.items() return dict(_remove(mapping)) # vendored from `dask.utils.parse_bytes` # https://github.com/dask/dask/blob/a68bbc814306c51177407d32067ee5a8aaa22181/dask/utils.py#L1455-L1528 byte_sizes = { "kB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12, "PB": 10**15, "KiB": 2**10, "MiB": 2**20, "GiB": 2**30, "TiB": 2**40, "PiB": 2**50, "B": 1, "": 1, } byte_sizes = {k.lower(): v for k, v in byte_sizes.items()} byte_sizes.update({k[0]: v for k, v in byte_sizes.items() if k and "i" not in k}) byte_sizes.update({k[:-1]: v for k, v in byte_sizes.items() if k and "i" in k}) def parse_bytes(s: float | str) -> int: """Parse byte string to numbers >>> from dask.utils import parse_bytes >>> parse_bytes("100") 100 >>> parse_bytes("100 MB") 100000000 >>> parse_bytes("100M") 100000000 >>> parse_bytes("5kB") 5000 >>> parse_bytes("5.4 kB") 5400 >>> parse_bytes("1kiB") 1024 >>> parse_bytes("1e6") 1000000 >>> parse_bytes("1e6 kB") 1000000000 >>> parse_bytes("MB") 1000000 >>> parse_bytes(123) 123 >>> parse_bytes("5 foos") Traceback (most recent call last): ... ValueError: Could not interpret 'foos' as a byte unit """ if isinstance(s, (int, float)): return int(s) s = s.replace(" ", "") if not any(char.isdigit() for char in s): s = "1" + s # this will never run until the end for i in range(len(s) - 1, -1, -1): if not s[i].isalpha(): break index = i + 1 prefix = s[:index] suffix = s[index:] try: n = float(prefix) except ValueError as e: raise ValueError(f"Could not interpret '{prefix}' as a number") from e try: multiplier = byte_sizes[suffix.lower()] except KeyError as e: raise ValueError(f"Could not interpret '{suffix}' as a byte unit") from e result = n * multiplier return int(result) xarray-ceos-alos2-2025.05.0/ceos_alos2/volume_directory/000077500000000000000000000000001501510365500227255ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ceos_alos2/volume_directory/__init__.py000066400000000000000000000001171501510365500250350ustar00rootroot00000000000000from ceos_alos2.volume_directory.io import open_volume_directory # noqa: F401 xarray-ceos-alos2-2025.05.0/ceos_alos2/volume_directory/io.py000066400000000000000000000007611501510365500237120ustar00rootroot00000000000000from ceos_alos2.utils import to_dict from ceos_alos2.volume_directory.metadata import transform_record from ceos_alos2.volume_directory.structure import volume_directory_record def parse_data(data): return to_dict(volume_directory_record.parse(data)) def open_volume_directory(mapper, path): try: data = mapper[path] except KeyError as e: raise FileNotFoundError(f"Cannot open {path}") from e metadata = parse_data(data) return transform_record(metadata) xarray-ceos-alos2-2025.05.0/ceos_alos2/volume_directory/metadata.py000066400000000000000000000050071501510365500250610ustar00rootroot00000000000000from tlz.functoolz import curry, pipe from ceos_alos2.dicttoolz import apply_to_items, dissoc from ceos_alos2.hierarchy import Group from ceos_alos2.transformers import normalize_datetime from ceos_alos2.utils import remove_nesting_layer, rename def transform_volume_descriptor(mapping): ignored = [ "preamble", "ascii_ebcdic_flag", "blanks", "spare", "local_use_segment", "total_number_of_physical_volumes_in_logical_volume", "physical_volume_sequence_number_of_the_first_tape", "physical_volume_sequence_number_of_the_last_tape", "physical_volume_sequence_number_of_the_current_tape", "file_number_in_the_logical_volume", "logical_volume_within_a_volume_set", "logical_volume_number_within_physical_volume", "number_of_file_pointer_records", "number_of_text_records_in_volume_directory", ] translations = { "superstructure_format_control_document_id": "control_document_id", "superstructure_format_control_document_revision_level": "control_document_revision_level", "superstructure_record_format_revision_level": "record_format_revision_level", "software_release_and_revision_level": "software_version", "logical_volume_creation_datetime": "creation_datetime", "logical_volume_generation_country": "creation_country", "logical_volume_generating_agency": "creation_agency", "logical_volume_generating_facility": "creation_facility", } postprocessors = { "creation_datetime": normalize_datetime, } return pipe( mapping, curry(dissoc, ignored), curry(rename, translations=translations), curry(apply_to_items, postprocessors), ) def transform_text(mapping): ignored = ["preamble", "ascii_ebcdic_flag", "blanks", "physical_tape_id"] translations = { "location_and_datetime_of_product_creation": "product_creation", } transformed = pipe( mapping, curry(dissoc, ignored), curry(rename, translations=translations), ) return transformed def transform_record(mapping): ignored = ["file_descriptors"] transformers = { "volume_descriptor": transform_volume_descriptor, "text_record": transform_text, } transformed = pipe( mapping, curry(dissoc, ignored), curry(apply_to_items, transformers), curry(remove_nesting_layer), ) return Group(path=None, url=None, data={}, attrs=transformed) xarray-ceos-alos2-2025.05.0/ceos_alos2/volume_directory/structure.py000066400000000000000000000066251501510365500253500ustar00rootroot00000000000000from construct import Struct, this from ceos_alos2.common import record_preamble from ceos_alos2.datatypes import AsciiInteger, PaddedString volume_descriptor = Struct( "preamble" / record_preamble, "ascii_ebcdic_flag" / PaddedString(2), "blanks" / PaddedString(2), "superstructure_format_control_document_id" / PaddedString(12), "superstructure_format_control_document_revision_level" / PaddedString(2), "superstructure_record_format_revision_level" / PaddedString(2), "software_release_and_revision_level" / PaddedString(12), "physical_volume_id" / PaddedString(16), "logical_volume_id" / PaddedString(16), "volume_set_id" / PaddedString(16), "total_number_of_physical_volumes_in_logical_volume" / AsciiInteger(2), "physical_volume_sequence_number_of_the_first_tape" / AsciiInteger(2), "physical_volume_sequence_number_of_the_last_tape" / AsciiInteger(2), "physical_volume_sequence_number_of_the_current_tape" / AsciiInteger(2), "file_number_in_the_logical_volume" / AsciiInteger(4), "logical_volume_within_a_volume_set" / AsciiInteger(4), "logical_volume_number_within_physical_volume" / AsciiInteger(4), "logical_volume_creation_datetime" / PaddedString(16), # merged two entries "logical_volume_generation_country" / PaddedString(12), "logical_volume_generating_agency" / PaddedString(8), "logical_volume_generating_facility" / PaddedString(12), "number_of_file_pointer_records" / AsciiInteger(4), "number_of_text_records_in_volume_directory" / AsciiInteger(4), "spare" / PaddedString(92), "local_use_segment" / PaddedString(100), ) file_descriptor = Struct( "preamble" / record_preamble, "ascii_ebcdic_flag" / PaddedString(2), "blanks" / PaddedString(2), "referenced_file_number" / AsciiInteger(4), "referenced_file_name_id" / PaddedString(16), "referenced_file_class" / PaddedString(28), "referenced_file_class_code" / PaddedString(4), "referenced_file_data_type" / PaddedString(28), "referenced_file_data_type_code" / PaddedString(4), "number_of_records_in_referenced_file" / AsciiInteger(8), "length_of_the_first_record_in_referenced_file" / AsciiInteger(8), "maximum_record_length_in_referenced_file" / AsciiInteger(8), "referenced_file_record_length_type" / PaddedString(12), "referenced_file_record_length_type_code" / PaddedString(4), "number_of_the_physical_volume_set_containing_the_first_record_of_the_file" / AsciiInteger(2), "number_of_the_physical_volume_set_containing_the_last_record_of_the_file" / AsciiInteger(2), "record_number_of_the_first_record_appearing_on_this_physical_volume" / AsciiInteger(8), "record_number_of_the_last_record_appearing_on_this_physical_volume" / AsciiInteger(8), "spare" / PaddedString(100), "local_use_segment" / PaddedString(100), ) text_record = Struct( "preamble" / record_preamble, "ascii_ebcdic_flag" / PaddedString(2), "blanks" / PaddedString(2), "product_id" / PaddedString(40), "location_and_datetime_of_product_creation" / PaddedString(60), "physical_tape_id" / PaddedString(40), "scene_id" / PaddedString(40), "scene_location_id" / PaddedString(40), "blanks" / PaddedString(124), ) volume_directory_record = Struct( "volume_descriptor" / volume_descriptor, "file_descriptors" / file_descriptor[this.volume_descriptor.number_of_file_pointer_records], "text_record" / text_record, ) xarray-ceos-alos2-2025.05.0/ceos_alos2/xarray.py000066400000000000000000000072131501510365500212150ustar00rootroot00000000000000import numpy as np import numpy.typing import xarray as xr from xarray.backends import BackendArray from xarray.backends.locks import SerializableLock from xarray.core import indexing from ceos_alos2 import io from ceos_alos2.array import Array class LazilyIndexedWrapper(BackendArray): def __init__(self, array, lock): self.array = array self.lock = lock self.shape = array.shape self.dtype = array.dtype def __getitem__(self, key: indexing.ExplicitIndexer) -> np.typing.ArrayLike: return indexing.explicit_indexing_adapter( key, self.shape, indexing.IndexingSupport.BASIC, self._raw_indexing_method, ) def _raw_indexing_method(self, key: tuple) -> np.typing.ArrayLike: with self.lock: return self.array[key] def extract_encoding(var): chunks = var.chunks if all(c is None for c in chunks.values()): return {} normalized_chunks = { dim: chunksize if chunksize not in (None, -1) else var.sizes[dim] for dim, chunksize in chunks.items() } return {"preferred_chunksizes": normalized_chunks} def to_variable(var): # only need a read lock, we don't support writing # TODO: do we even need the lock? if isinstance(var.data, Array): lock = SerializableLock() data = indexing.LazilyIndexedArray(LazilyIndexedWrapper(var.data, lock)) else: data = var.data return xr.Variable(var.dims, data, var.attrs, encoding=extract_encoding(var)) def decode_coords(ds): coords = ds.attrs.pop("coordinates", []) return ds.set_coords(coords) def to_dataset(group, chunks=None): variables = {name: to_variable(var) for name, var in group.variables.items()} ds = xr.Dataset(variables, attrs=group.attrs).pipe(decode_coords) if chunks is None: return ds filtered_chunks = {dim: size for dim, size in chunks.items() if dim in ds.dims} return ds.chunk(filtered_chunks) def to_datatree(group, chunks=None): mapping = {"/": to_dataset(group, chunks=chunks)} | { path: to_dataset(subgroup, chunks=chunks) for path, subgroup in group.subtree } return xr.DataTree.from_dict(mapping) def open_alos2(path, chunks=None, backend_options={}): """Open CEOS ALOS2 datasets Parameters ---------- path : str Path or URL to the dataset. chunks : int, dict, "auto" or None, optional If chunks is provided, it is used to load the new dataset into dask arrays. ``chunks=-1`` loads the dataset with dask using a single chunk for all arrays. ``chunks={}`` loads the dataset with dask using engine preferred chunks if exposed by the backend, otherwise with a single chunk for all arrays. ``chunks='auto'`` will use dask ``auto`` chunking taking into account the engine preferred chunks. See dask chunking for more details. backend_options : dict, optional Additional keyword arguments passed on to the low-level open function: - 'storage_options': Additional arguments for `fsspec.get_mapper` - 'use_cache': Make use of image cache files, if they exist. Default: True - 'create_cache': Create a local cache file after reading the image metadata. Default: False - 'records_per_chunk': The image metadata is stored line by line. In order to avoid sending potentially thousands of requests, read this many lines at once. Default: 1024 Returns ------- tree : xarray.DataTree The newly created datatree. """ root = io.open(path, **backend_options) return to_datatree(root, chunks=chunks) xarray-ceos-alos2-2025.05.0/ci/000077500000000000000000000000001501510365500156745ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ci/install-upstream-dev.sh000066400000000000000000000011641501510365500223120ustar00rootroot00000000000000#!/usr/bin/env bash if command -v micromamba >/dev/null; then conda=micromamba elif command -v mamba >/dev/null; then conda=mamba else conda=conda fi conda remove -y --force cytoolz numpy xarray construct toolz fsspec python-dateutil pandas python -m pip install \ -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \ --no-deps \ --pre \ --upgrade \ numpy \ pandas \ xarray python -m pip install --upgrade \ git+https://github.com/construct/construct \ git+https://github.com/pytoolz/toolz \ git+https://github.com/fsspec/filesystem_spec \ git+https://github.com/dateutil/dateutil xarray-ceos-alos2-2025.05.0/ci/requirements/000077500000000000000000000000001501510365500204175ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/ci/requirements/docs.yaml000066400000000000000000000002001501510365500222230ustar00rootroot00000000000000name: xarray-alos2-docs channels: - conda-forge dependencies: - python=3.11 - sphinx>=4 - sphinx_book_theme - ipython xarray-ceos-alos2-2025.05.0/ci/requirements/environment.yaml000066400000000000000000000004401501510365500236450ustar00rootroot00000000000000name: xarray-alos2-tests channels: - conda-forge dependencies: - ipython - pre-commit - pytest - pytest-reportlog - pytest-cov - numpy - toolz - cytoolz - python-dateutil - construct - fsspec - xarray>=2024.10.0 - dask - distributed - aiohttp - requests xarray-ceos-alos2-2025.05.0/docs/000077500000000000000000000000001501510365500162315ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/docs/api.rst000066400000000000000000000001401501510365500175270ustar00rootroot00000000000000API === .. currentmodule:: ceos_alos2 .. autosummary:: :toctree: generated/ open_alos2 xarray-ceos-alos2-2025.05.0/docs/changelog.md000066400000000000000000000007571501510365500205130ustar00rootroot00000000000000# Changelog ## 2025.05.0 (26 May 2025) - support `python=3.13` ({pull}`99`) - don't decode facility-related data records 1-4 ({pull}`98`) ## 2024.10.0 (30 Oct 2024) - migrate to {py:class}`xarray.DataTree` ({pull}`81`) - support `python=3.12` ({pull}`82`) ## 2023.08.2 (28 Aug 2023) - explicitly add `platformdirs` to the dependencies ({pull}`57`) ## 2023.08.1 (28 Aug 2023) - add documentation ({pull}`51`) - expose the script ({pull}`53`) ## 2023.08.0 (25 Aug 2023) Initial release. xarray-ceos-alos2-2025.05.0/docs/conf.py000066400000000000000000000050601501510365500175310ustar00rootroot00000000000000import datetime as dt import subprocess # -- System information ------------------------------------------------------ subprocess.run(["pip", "list"]) # -- Project information ----------------------------------------------------- project = "xarray-ceos-alos2" author = f"{project} developers" initial_year = "2023" year = dt.datetime.now().year copyright = f"{initial_year}-{year}, {author}" # The root toctree document. root_doc = "index" # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "myst_parser", "sphinx.ext.extlinks", "sphinx.ext.intersphinx", "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.napoleon", "IPython.sphinxext.ipython_directive", "IPython.sphinxext.ipython_console_highlighting", ] extlinks = { "issue": ("https://github.com/umr-lops/xarray-ceos-alos2/issues/%s", "GH%s"), "pull": ("https://github.com/umr-lops/xarray-ceos-alos2/pull/%s", "PR%s"), } # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "directory"] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "sphinx_book_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ["_static"] # -- Options for the intersphinx extension ----------------------------------- intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "sphinx": ("https://www.sphinx-doc.org/en/stable/", None), "xarray": ("https://docs.xarray.dev/en/latest/", None), } # -- Options for the autosummary extension ----------------------------------- autosummary_generate = True autodoc_typehints = "none" napoleon_use_param = False napoleon_use_rtype = True napoleon_preprocess_types = True napoleon_type_aliases = {} # -- Options for the myst-parser extension ----------------------------------- myst_enable_extensions = ["colon_fence"] xarray-ceos-alos2-2025.05.0/docs/index.md000066400000000000000000000006001501510365500176560ustar00rootroot00000000000000# xarray-ceos-alos2 `xarray-ceos-alos2` allows reading CEOS ALOS2 datasets (for now, level 1.1, 1.5, and 3.1 only) into {py:class}`xarray.DataTree` objects. **Contents**: - {doc}`Installing ` - {doc}`Changelog ` - {doc}`Usage ` - {doc}`API ` :::{toctree} :caption: Contents :maxdepth: 1 :hidden: installing.md changelog.md usage.md api.rst ::: xarray-ceos-alos2-2025.05.0/docs/installing.md000066400000000000000000000003501501510365500207150ustar00rootroot00000000000000# Installing 1. Install from [PyPI](https://pypi.org): ```sh pip install xarray-ceos-alos2 ``` 2. Install from [conda-forge](https://conda-forge.org) ```sh conda activate conda install -c conda-forge xarray-ceos-alos2 ``` xarray-ceos-alos2-2025.05.0/docs/requirements.txt000066400000000000000000000002111501510365500215070ustar00rootroot00000000000000sphinx>=6 sphinx_book_theme>=0.3 myst-parser ipython numpy toolz cytoolz python-dateutil construct fsspec xarray>=2024.10.0 platformdirs xarray-ceos-alos2-2025.05.0/docs/usage.md000066400000000000000000000063051501510365500176630ustar00rootroot00000000000000# Usage ## Opening datasets Datasets of levels 1.1, 1.5, or 3.1 can be opened using: ```python import ceos_alos2 url = "..." tree = ceos_alos2.open_alos2(url, chunks={}) ``` ## Backend options Additional parameters can be set using the `backend_options` parameter. The valid options are: - `storage_options`: additional parameters passed on to the appropriate `fsspec` filesystem - `records_per_chunk`: request size when fetching image data (see {ref}`request-size`) - `use_cache`: use cache files instead of parsing the image files (see {ref}`caching`) - `create_cache`: create cache files (see {ref}`caching`) ## Access optimizations (request-size)= ### Request size The image data of CEOS ALOS2 datasets is subdivided into records that represent the lines (rows) of the image. Each record begins with metadata, followed by the actual line data. However, this is not optimized for the typical data access pattern: when opening the dataset, all the metadata is read, and once computations on the image are performed the image data is accessed separately. Thus, each record is requested at least twice, once when collecting the metadata and at least once when computing the image data. Additionally, requesting the records one-by-one means that tens of thousands of requests (or syscalls in case of a local file system) have to performed, which can take a very long time. Using the `records_per_chunk` setting, a number of records can be grouped and requested together. This allows reducing the number of requests by a lot, which in turn makes data access much faster. :::{tip} Always specify the request size explicitly ::: For example: ```python tree = ceos_alos2.open_alos2(url, chunks={}, backend_options={"records_per_chunk": 4096}) ``` (caching)= ### Caching Even though adjusting the request size can decrease access times, reading and parsing the image metadata still takes a lot of time (in the case of level 1.1 ScanSAR this can take more than 10 minutes). To avoid that, it is possible to save the metadata in special cache files, allowing to open datasets (i.e. reading the file metadata) in a matter of seconds. The `use_cache` parameter controls whether or not these cache files are used, which can be stored either alongside the image file (i.e. a "remote cache file") or in a local directory (`$user_cache_dir/xarray-ceos-alos2//.index`, where `$user_cache_dir` depends on the OS). If both exist the remote cache file is preferred. Cache files can be created either by enabling the `create_cache` flag or by running the `ceos-alos2-create-cache` executable. Using `create_cache`: ```python tree = ceos_alos2.open_alos2( url, chunks={}, backend_options={"records_per_chunk": 4096, "create_cache": True, "use_cache": False}, ) ``` This will open the dataset with a request size of 4096 records and write a local cache file for _each image_. Using `ceos-alos2-create-cache`: ```sh ceos-alos2-create-cache --rpc 4096 # or, with a explict target path ceos-alos2-create-cache --rpc 4096 ``` This will open a _single_ image with a request size of 4096 records and create a cache file, either in the specified target path, or adjacent to the image file. xarray-ceos-alos2-2025.05.0/licenses/000077500000000000000000000000001501510365500171065ustar00rootroot00000000000000xarray-ceos-alos2-2025.05.0/licenses/DASK000066400000000000000000000027731501510365500175640ustar00rootroot00000000000000BSD 3-Clause License Copyright (c) 2014, Anaconda, Inc. and contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. xarray-ceos-alos2-2025.05.0/pyproject.toml000066400000000000000000000051261501510365500202210ustar00rootroot00000000000000[project] name = "xarray-ceos-alos2" requires-python = ">= 3.10" license = { text = "MIT" } description = "xarray reader for advanced land observing satellite 2 (ALOS2) CEOS files" readme = "README.md" dependencies = [ "toolz", "python-dateutil", "xarray>=2024.10.0", "numpy", "construct>=2.10", "fsspec", "platformdirs", "exceptiongroup; python_version < '3.11'", ] keywords = [ "xarray", "earth-observation", "remote-sensing", "satellite-imagery", "ceos", "alos2", "sar", "synthetic-aperture-radar", ] classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering", ] dynamic = ["version"] [project.urls] homepage = "https://xarray-ceos-alos2.readthedocs.io" documentation = "https://xarray-ceos-alos2.readthedocs.io" repository = "https://github.com/umr-lops/xarray-ceos-alos2" changelog = "https://xarray-ceos-alos2.readthedocs.io/en/latest/changelog.html" [project.scripts] ceos-alos2-create-cache = "ceos_alos2.sar_image.cli:main" [build-system] requires = ["setuptools>=64.0", "setuptools-scm"] build-backend = "setuptools.build_meta" [tool.setuptools] packages = ["ceos_alos2"] [tool.setuptools_scm] fallback_version = "999" [tool.black] line-length = 100 [tool.ruff] target-version = "py310" builtins = ["ellipsis"] exclude = [".git", ".eggs", "build", "dist", "__pycache__"] line-length = 100 [tool.ruff.lint] ignore = [ "E402", # module level import not at top of file "E501", # line too long - let black worry about that "E731", # do not assign a lambda expression, use a def "UP038", # type union instead of tuple for isinstance etc ] select = [ "F", # Pyflakes "E", # Pycodestyle "I", # isort "UP", # Pyupgrade "TID", # flake8-tidy-imports "W", ] extend-safe-fixes = [ "TID252", # absolute imports "UP031", # percent string interpolation ] fixable = ["I", "TID252", "UP"] [tool.ruff.lint.isort] known-first-party = ["ceos_alos2"] known-third-party = ["xarray", "toolz", "construct"] [tool.ruff.lint.flake8-tidy-imports] # Disallow all relative imports. ban-relative-imports = "all" [tool.coverage.run] source = ["ceos_alos2"] branch = true [tool.coverage.report] show_missing = true exclude_lines = ["pragma: no cover", "if TYPE_CHECKING"]