pax_global_header00006660000000000000000000000064147464114040014517gustar00rootroot0000000000000052 comment=d836ca86738431f3797c6ce0323eec210d815c10 itemadapter-0.11.0/000077500000000000000000000000001474641140400140755ustar00rootroot00000000000000itemadapter-0.11.0/.bandit.yml000066400000000000000000000000301474641140400161300ustar00rootroot00000000000000exclude_dirs: ['tests'] itemadapter-0.11.0/.bumpversion.cfg000066400000000000000000000002111474641140400171770ustar00rootroot00000000000000[bumpversion] current_version = 0.11.0 commit = True tag = True [bumpversion:file:itemadapter/__init__.py] [bumpversion:file:setup.py] itemadapter-0.11.0/.editorconfig000066400000000000000000000002651474641140400165550ustar00rootroot00000000000000[*] trim_trailing_whitespace = true insert_final_newline = true indent_style = space [Makefile] indent_style = tab [*.py] indent_size = 4 charset = utf-8 [*.yml] indent_size = 2 itemadapter-0.11.0/.git-blame-ignore-revs000066400000000000000000000001231474641140400201710ustar00rootroot00000000000000# applying pre-commit hooks to the project 106a4e0af9e9ac07defef3c9a781d2fe0ac4640fitemadapter-0.11.0/.github/000077500000000000000000000000001474641140400154355ustar00rootroot00000000000000itemadapter-0.11.0/.github/workflows/000077500000000000000000000000001474641140400174725ustar00rootroot00000000000000itemadapter-0.11.0/.github/workflows/checks.yml000066400000000000000000000016031474641140400214550ustar00rootroot00000000000000name: Checks on: [push, pull_request] jobs: checks: runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - python-version: 3.13 env: TOXENV: typing - python-version: 3.13 env: TOXENV: docs - python-version: 3.13 env: TOXENV: twinecheck - python-version: 3.13 env: TOXENV: pylint steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Run check env: ${{ matrix.env }} run: | pip install -U pip pip install -U tox tox pre-commit: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.1 itemadapter-0.11.0/.github/workflows/publish.yml000066400000000000000000000007411474641140400216650ustar00rootroot00000000000000name: Publish on: release: types: [published] jobs: publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python 3.13 uses: actions/setup-python@v5 with: python-version: 3.13 - name: Build run: | pip install --upgrade build twine python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@v1.8.14 with: password: ${{ secrets.PYPI_TOKEN }} itemadapter-0.11.0/.github/workflows/tests.yml000066400000000000000000000043601474641140400213620ustar00rootroot00000000000000name: Tests on: [push, pull_request] jobs: tests-ubuntu: name: "Test: ${{ matrix.python-version }}, Ubuntu" runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - python-version: "3.9" env: TOXENV: min-attrs - python-version: "3.9" env: TOXENV: min-pydantic - python-version: "3.9" env: TOXENV: min-scrapy - python-version: "3.9" env: TOXENV: min-extra - python-version: "3.9" env: TOXENV: py - python-version: "3.10" env: TOXENV: py - python-version: "pypy3.10" env: TOXENV: py - python-version: "3.11" env: TOXENV: py - python-version: "3.12" env: TOXENV: py - python-version: "3.13" env: TOXENV: py - python-version: "3.13" env: TOXENV: attrs - python-version: "3.13" env: TOXENV: pydantic1 - python-version: "3.13" env: TOXENV: pydantic - python-version: "3.13" env: TOXENV: scrapy - python-version: "3.13" env: TOXENV: extra - python-version: "3.13" env: TOXENV: extra-pydantic1 steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install tox run: pip install tox - name: Run tests env: ${{ matrix.env }} run: tox - name: Upload coverage report uses: codecov/codecov-action@v5 tests-other-os: name: "Test: py39, ${{ matrix.os }}" runs-on: "${{ matrix.os }}" strategy: matrix: os: [macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install tox run: pip install tox - name: Run tests run: tox -e py - name: Upload coverage report uses: codecov/codecov-action@v5 itemadapter-0.11.0/.gitignore000066400000000000000000000001361474641140400160650ustar00rootroot00000000000000*.pyc .~lock* .DS_Store .mypy_cache/ *.egg-info/ .tox/ .coverage htmlcov/ coverage.xml /dist/ itemadapter-0.11.0/.isort.cfg000066400000000000000000000000321474641140400157670ustar00rootroot00000000000000[settings] profile = blackitemadapter-0.11.0/.pre-commit-config.yaml000066400000000000000000000007141474641140400203600ustar00rootroot00000000000000repos: - repo: https://github.com/PyCQA/bandit rev: 1.8.2 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/psf/black.git rev: 24.10.0 hooks: - id: black - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: - id: isort - repo: https://github.com/asottile/pyupgrade rev: v3.19.1 hooks: - id: pyupgrade args: [--py39-plus] itemadapter-0.11.0/Changelog.md000066400000000000000000000115621474641140400163130ustar00rootroot00000000000000# Changelog ### 0.11.0 (2025-01-29) Removed functions deprecated in 0.5.0: * `itemadapter.utils.is_attrs_instance()` * `itemadapter.utils.is_dataclass_instance()` * `itemadapter.utils.is_pydantic_instance()` * `itemadapter.utils.is_scrapy_item()` ([#93](https://github.com/scrapy/itemadapter/pull/93)). Added support for Pydantic 2 ([#91](https://github.com/scrapy/itemadapter/pull/91)). Added `__all__` to the top-level module to improve type checks ([#90](https://github.com/scrapy/itemadapter/pull/90)). Improved `pre-commit` and CI configuration ([#91](https://github.com/scrapy/itemadapter/pull/91), [#92](https://github.com/scrapy/itemadapter/pull/92)). ### 0.10.0 (2024-11-29) Dropped Python 3.8 support, added official Python 3.13 and PyPy 3.10 support ([#79](https://github.com/scrapy/itemadapter/pull/79), [#87](https://github.com/scrapy/itemadapter/pull/87)). Fixed the typing check when run with Scrapy 2.12.0+ ([#88](https://github.com/scrapy/itemadapter/pull/88)). Fixed `MANIFEST.in` that was missing some files ([#84](https://github.com/scrapy/itemadapter/pull/84)). Enabled `pre-commit` ([#85](https://github.com/scrapy/itemadapter/pull/85)). ### 0.9.0 (2024-05-07) Dropped Python 3.7 support, added official Python 3.12 support ([#75](https://github.com/scrapy/itemadapter/pull/75), [#77](https://github.com/scrapy/itemadapter/pull/77)). Updated the documentation and the type hint about `ItemAdapter.ADAPTER_CLASSES` to say that subclasses can use any iterable, not just `collections.deque` ([#74](https://github.com/scrapy/itemadapter/pull/74)). Documented that `Pydantic >= 2` is not supported yet ([#73](https://github.com/scrapy/itemadapter/pull/73)). Updated CI configuration ([#77](https://github.com/scrapy/itemadapter/pull/77), [#80](https://github.com/scrapy/itemadapter/pull/80)). ### 0.8.0 (2023-03-30) Dropped Python 3.6 support, and made Python 3.11 support official ([#65](https://github.com/scrapy/itemadapter/pull/65), [#66](https://github.com/scrapy/itemadapter/pull/66), [#69](https://github.com/scrapy/itemadapter/pull/69)). It is now possible to declare custom `ItemAdapter` subclasses with their own `ADAPTER_CLASSES` attribute, allowing to support different item types in different parts of the same code base ([#68](https://github.com/scrapy/itemadapter/pull/68)). Improved type hint support ([#67](https://github.com/scrapy/itemadapter/pull/67)). ### 0.7.0 (2022-08-02) ItemAdapter.get_field_names_from_class ([#64](https://github.com/scrapy/itemadapter/pull/64)) ### 0.6.0 (2022-05-12) Slight performance improvement ([#62](https://github.com/scrapy/itemadapter/pull/62)) ### 0.5.0 (2022-03-18) Improve performance by removing imports inside functions ([#60](https://github.com/scrapy/itemadapter/pull/60)) ### 0.4.0 (2021-08-26) Added `ItemAdapter.is_item_class` and `ItemAdapter.get_field_meta_from_class` ([#54](https://github.com/scrapy/itemadapter/pull/54)) ### 0.3.0 (2021-07-15) Added built-in support for `pydantic` models ([#53](https://github.com/scrapy/itemadapter/pull/53)) ### 0.2.0 (2020-11-06) Adapter interface: added the ability to support arbitrary types, by implementing a MutableMapping-based interface. By way of this change, now any type can be used as a Scrapy item. ### 0.1.1 (2020-09-28) Dropped support for Python 3.5 (#38). The new `get_field_meta_from_class` function offers the same functionality as `ItemAdapter.get_field_meta` but for an item class, as opposed to an item object (#34, #35). `ItemAdapter.__repr__` no longer raises exceptions caused by the underlying item (#31, #41). Minor improvement to the release process (#37), and cleanup of test warnings (#40). ### 0.1.0 (2020-06-10) Added `ItemAdapter.asdict`, which allows converting an item and all of its nested items into `dict` objects (#27, #29). Improved `ItemAdapter` performance by reducing time complexity for lookups and traversals for dataclass and attrs items (#28). ### 0.0.8 (2020-05-22) `ItemAdapter.field_names` now returns a `KeysView` instead of a `list`. Minor CI and test changes. ### 0.0.7 (2020-05-22) `ItemAdapter.get_field_meta` now returns an empty `MappingProxyType` object for items without metadata support, instead of raising `TypeError`. Improved the README and some docstrings. Provided full test coverage, and refactored CI configuration, test configuration and tests themselves. ### 0.0.6 (2020-05-09) Added support for Scrapy’s `BaseItem`. Refactored and extended tests. Code style and documentation fixes. ### 0.0.5 (2020-04-28) Removed support for `MutableMapping`. ### 0.0.4 (2020-04-28) Removed metadata support for arbitrary mutable mappings. ### 0.0.3 (2020-04-27) Rebuild for the Python Package Index. ### 0.0.2 (2020-04-27) Split the implementation into several files for better code organization, and without an impact on the existing API import paths. Also improved the README. ### 0.0.1 (2020-04-25) Initial release. itemadapter-0.11.0/LICENSE000066400000000000000000000026651474641140400151130ustar00rootroot00000000000000Copyright 2020 Eugenio Lacuesta Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. itemadapter-0.11.0/MANIFEST.in000066400000000000000000000001341474641140400156310ustar00rootroot00000000000000include Changelog.md include pytest.ini include tox.ini recursive-include tests *.py *.txt itemadapter-0.11.0/README.md000066400000000000000000000417711474641140400153660ustar00rootroot00000000000000# itemadapter [![version](https://img.shields.io/pypi/v/itemadapter.svg)](https://pypi.python.org/pypi/itemadapter) [![pyversions](https://img.shields.io/pypi/pyversions/itemadapter.svg)](https://pypi.python.org/pypi/itemadapter) [![actions](https://github.com/scrapy/itemadapter/workflows/Tests/badge.svg)](https://github.com/scrapy/itemadapter/actions) [![codecov](https://codecov.io/gh/scrapy/itemadapter/branch/master/graph/badge.svg)](https://codecov.io/gh/scrapy/itemadapter) The `ItemAdapter` class is a wrapper for data container objects, providing a common interface to handle objects of different types in an uniform manner, regardless of their underlying implementation. Currently supported types are: * [`scrapy.item.Item`](https://docs.scrapy.org/en/latest/topics/items.html#scrapy.item.Item) * [`dict`](https://docs.python.org/3/library/stdtypes.html#dict) * [`dataclass`](https://docs.python.org/3/library/dataclasses.html)-based classes * [`attrs`](https://www.attrs.org)-based classes * [`pydantic`](https://pydantic-docs.helpmanual.io/)-based classes Additionally, interaction with arbitrary types is supported, by implementing a pre-defined interface (see [extending `itemadapter`](#extending-itemadapter)). --- ## Requirements * Python 3.9+, either the CPython implementation (default) or the PyPy implementation * [`scrapy`](https://scrapy.org/) 2.2+: optional, needed to interact with `scrapy` items * [`attrs`](https://pypi.org/project/attrs/) 18.1.0+: optional, needed to interact with `attrs`-based items * [`pydantic`](https://pypi.org/project/pydantic/) 1.8+: optional, needed to interact with `pydantic`-based items --- ## Installation `itemadapter` is available on [`PyPI`](https://pypi.python.org/pypi/itemadapter), it can be installed with `pip`: ``` pip install itemadapter ``` For `attrs`, `pydantic` and `scrapy` support, install the corresponding extra to ensure that a supported version of the corresponding dependencies is installed. For example: ``` pip install itemadapter[scrapy] ``` Mind that you can install multiple extras as needed. For example: ``` pip install itemadapter[attrs,pydantic,scrapy] ``` --- ## License `itemadapter` is distributed under a [BSD-3](https://opensource.org/licenses/BSD-3-Clause) license. --- ## Basic usage The following is a simple example using a `dataclass` object. Consider the following type definition: ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class InventoryItem: ... name: str ... price: float ... stock: int >>> ``` An `ItemAdapter` object can be treated much like a dictionary: ```python >>> obj = InventoryItem(name='foo', price=20.5, stock=10) >>> ItemAdapter.is_item(obj) True >>> adapter = ItemAdapter(obj) >>> len(adapter) 3 >>> adapter["name"] 'foo' >>> adapter.get("price") 20.5 >>> ``` The wrapped object is modified in-place: ```python >>> adapter["name"] = "bar" >>> adapter.update({"price": 12.7, "stock": 9}) >>> adapter.item InventoryItem(name='bar', price=12.7, stock=9) >>> adapter.item is obj True >>> ``` ### Converting to dict The `ItemAdapter` class provides the `asdict` method, which converts nested items recursively. Consider the following example: ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class Price: ... value: int ... currency: str >>> @dataclass ... class Product: ... name: str ... price: Price >>> ``` ```python >>> item = Product("Stuff", Price(42, "UYU")) >>> adapter = ItemAdapter(item) >>> adapter.asdict() {'name': 'Stuff', 'price': {'value': 42, 'currency': 'UYU'}} >>> ``` Note that just passing an adapter object to the `dict` built-in also works, but it doesn't traverse the object recursively converting nested items: ```python >>> dict(adapter) {'name': 'Stuff', 'price': Price(value=42, currency='UYU')} >>> ``` --- ## API reference ### Built-in adapters The following adapters are included by default: * `itemadapter.adapter.ScrapyItemAdapter`: handles `Scrapy` items * `itemadapter.adapter.DictAdapter`: handles `Python` dictionaries * `itemadapter.adapter.DataclassAdapter`: handles `dataclass` objects * `itemadapter.adapter.AttrsAdapter`: handles `attrs` objects * `itemadapter.adapter.PydanticAdapter`: handles `pydantic` objects ### class `itemadapter.adapter.ItemAdapter(item: Any)` This is the main entrypoint for the package. Tipically, user code wraps an item using this class, and proceeds to handle it with the provided interface. `ItemAdapter` implements the [`MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping) interface, providing a `dict`-like API to manipulate data for the object it wraps (which is modified in-place). **Attributes** #### class attribute `ADAPTER_CLASSES: Iterable` Stores the currently registered adapter classes. The order in which the adapters are registered is important. When an `ItemAdapter` object is created for a specific item, the registered adapters are traversed in order and the first adapter class to return `True` for the `is_item` class method is used for all subsequent operations. The default order is the one defined in the [built-in adapters](#built-in-adapters) section. The default implementation uses a [`collections.deque`](https://docs.python.org/3/library/collections.html#collections.deque) to support efficient addition/deletion of adapters classes to both ends, but if you are deriving a subclass (see the section on [extending itemadapter](#extending-itemadapter) for additional information), any other iterable (e.g. `list`, `tuple`) will work. **Methods** #### class method `is_item(item: Any) -> bool` Return `True` if any of the registed adapters can handle the item (i.e. if any of them returns `True` for its `is_item` method with `item` as argument), `False` otherwise. #### class method `is_item_class(item_class: type) -> bool` Return `True` if any of the registered adapters can handle the item class (i.e. if any of them returns `True` for its `is_item_class` method with `item_class` as argument), `False` otherwise. #### class method `get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType` Return a [`types.MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which is a read-only mapping with metadata about the given field. If the item class does not support field metadata, or there is no metadata for the given field, an empty object is returned. The returned value is taken from the following sources, depending on the item type: * [`scrapy.item.Field`](https://docs.scrapy.org/en/latest/topics/items.html#item-fields) for `scrapy.item.Item`s * [`dataclasses.field.metadata`](https://docs.python.org/3/library/dataclasses.html#dataclasses.field) for `dataclass`-based items * [`attr.Attribute.metadata`](https://www.attrs.org/en/stable/examples.html#metadata) for `attrs`-based items * [`pydantic.fields.FieldInfo`](https://pydantic-docs.helpmanual.io/usage/schema/#field-customisation) for `pydantic`-based items #### class method `get_field_names_from_class(item_class: type) -> Optional[list[str]]` Return a list with the names of all the fields defined for the item class. If an item class doesn't support defining fields upfront, None is returned. #### `get_field_meta(field_name: str) -> MappingProxyType` Return metadata for the given field, if available. Unless overriden in a custom adapter class, by default this method calls the adapter's `get_field_meta_from_class` method, passing the wrapped item's class. #### `field_names() -> collections.abc.KeysView` Return a [keys view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) with the names of all the defined fields for the item. #### `asdict() -> dict` Return a `dict` object with the contents of the adapter. This works slightly different than calling `dict(adapter)`, because it's applied recursively to nested items (if there are any). ### function `itemadapter.utils.is_item(obj: Any) -> bool` Return `True` if the given object belongs to (at least) one of the supported types, `False` otherwise. This is an alias, using the `itemadapter.adapter.ItemAdapter.is_item` class method is encouraged for better performance. ### function `itemadapter.utils.get_field_meta_from_class(item_class: type, field_name: str) -> types.MappingProxyType` Alias for `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` --- ## Metadata support `scrapy.item.Item`, `dataclass`, `attrs`, and `pydantic` objects allow the definition of arbitrary field metadata. This can be accessed through a [`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) object, which can be retrieved from an item instance with `itemadapter.adapter.ItemAdapter.get_field_meta`, or from an item class with the `itemadapter.adapter.ItemAdapter.get_field_meta_from_class` method (or its alias `itemadapter.utils.get_field_meta_from_class`). The source of the data depends on the underlying type (see the docs for `ItemAdapter.get_field_meta_from_class`). #### `scrapy.item.Item` objects ```python >>> from scrapy.item import Item, Field >>> from itemadapter import ItemAdapter >>> class InventoryItem(Item): ... name = Field(serializer=str) ... value = Field(serializer=int, limit=100) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'serializer': }) >>> adapter.get_field_meta("value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` #### `dataclass` objects ```python >>> from dataclasses import dataclass, field >>> @dataclass ... class InventoryItem: ... name: str = field(metadata={"serializer": str}) ... value: int = field(metadata={"serializer": int, "limit": 100}) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'serializer': }) >>> adapter.get_field_meta("value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` #### `attrs` objects ```python >>> import attr >>> @attr.s ... class InventoryItem: ... name = attr.ib(metadata={"serializer": str}) ... value = attr.ib(metadata={"serializer": int, "limit": 100}) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'serializer': }) >>> adapter.get_field_meta("value") mappingproxy({'serializer': , 'limit': 100}) >>> ``` #### `pydantic` objects ```python >>> from pydantic import BaseModel, Field >>> class InventoryItem(BaseModel): ... name: str = Field(serializer=str) ... value: int = Field(serializer=int, limit=100) ... >>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) >>> adapter.get_field_meta("name") mappingproxy({'default': PydanticUndefined, 'json_schema_extra': {'serializer': }, 'repr': True}) >>> adapter.get_field_meta("value") mappingproxy({'default': PydanticUndefined, 'json_schema_extra': {'serializer': , 'limit': 100}, 'repr': True}) >>> ``` --- ## Extending `itemadapter` This package allows to handle arbitrary item classes, by implementing an adapter interface: _class `itemadapter.adapter.AdapterInterface(item: Any)`_ Abstract Base Class for adapters. An adapter that handles a specific type of item must inherit from this class and implement the abstract methods defined on it. `AdapterInterface` inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), so all methods from the `MutableMapping` interface must be implemented as well. * _class method `is_item_class(cls, item_class: type) -> bool`_ Return `True` if the adapter can handle the given item class, `False` otherwise. Abstract (mandatory). * _class method `is_item(cls, item: Any) -> bool`_ Return `True` if the adapter can handle the given item, `False` otherwise. The default implementation calls `cls.is_item_class(item.__class__)`. * _class method `get_field_meta_from_class(cls, item_class: type) -> bool`_ Return metadata for the given item class and field name, if available. By default, this method returns an empty `MappingProxyType` object. Please supply your own method definition if you want to handle field metadata based on custom logic. See the [section on metadata support](#metadata-support) for additional information. * _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ Return metadata for the given field name, if available. It's usually not necessary to override this method, since the `itemadapter.adapter.AdapterInterface` base class provides a default implementation that calls `ItemAdapter.get_field_meta_from_class` with the wrapped item's class as argument. See the [section on metadata support](#metadata-support) for additional information. * _method `field_names(self) -> collections.abc.KeysView`_: Return a [dynamic view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) of the item's field names. By default, this method returns the result of calling `keys()` on the current adapter, i.e., its return value depends on the implementation of the methods from the `MutableMapping` interface (more specifically, it depends on the return value of `__iter__`). You might want to override this method if you want a way to get all fields for an item, whether or not they are populated. For instance, Scrapy uses this method to define column names when exporting items to CSV. ### Registering an adapter Add your custom adapter class to the `itemadapter.adapter.ItemAdapter.ADAPTER_CLASSES` class attribute in order to handle custom item classes. **Example** ``` pip install zyte-common-items ``` ```python >>> from itemadapter.adapter import ItemAdapter >>> from zyte_common_items import Item, ZyteItemAdapter >>> >>> ItemAdapter.ADAPTER_CLASSES.appendleft(ZyteItemAdapter) >>> item = Item() >>> adapter = ItemAdapter(item) >>> adapter >>> ``` ### Multiple adapter classes If you need to have different handlers and/or priorities for different cases you can subclass the `ItemAdapter` class and set the `ADAPTER_CLASSES` attribute as needed: **Example** ```python >>> from itemadapter.adapter import ( ... ItemAdapter, ... AttrsAdapter, ... DataclassAdapter, ... DictAdapter, ... PydanticAdapter, ... ScrapyItemAdapter, ... ) >>> from scrapy.item import Item, Field >>> >>> class BuiltinTypesItemAdapter(ItemAdapter): ... ADAPTER_CLASSES = [DictAdapter, DataclassAdapter] ... >>> class ThirdPartyTypesItemAdapter(ItemAdapter): ... ADAPTER_CLASSES = [AttrsAdapter, PydanticAdapter, ScrapyItemAdapter] ... >>> class ScrapyItem(Item): ... foo = Field() ... >>> BuiltinTypesItemAdapter.is_item(dict()) True >>> ThirdPartyTypesItemAdapter.is_item(dict()) False >>> BuiltinTypesItemAdapter.is_item(ScrapyItem(foo="bar")) False >>> ThirdPartyTypesItemAdapter.is_item(ScrapyItem(foo="bar")) True >>> ``` --- ## More examples ### `scrapy.item.Item` objects ```python >>> from scrapy.item import Item, Field >>> from itemadapter import ItemAdapter >>> class InventoryItem(Item): ... name = Field() ... price = Field() ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item {'name': 'bar', 'price': 5} >>> ``` ### `dict` ```python >>> from itemadapter import ItemAdapter >>> item = dict(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item {'name': 'bar', 'price': 5} >>> ``` ### `dataclass` objects ```python >>> from dataclasses import dataclass >>> from itemadapter import ItemAdapter >>> @dataclass ... class InventoryItem: ... name: str ... price: int ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item InventoryItem(name='bar', price=5) >>> ``` ### `attrs` objects ```python >>> import attr >>> from itemadapter import ItemAdapter >>> @attr.s ... class InventoryItem: ... name = attr.ib() ... price = attr.ib() ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item InventoryItem(name='bar', price=5) >>> ``` ### `pydantic` objects ```python >>> from pydantic import BaseModel >>> from itemadapter import ItemAdapter >>> class InventoryItem(BaseModel): ... name: str ... price: int ... >>> item = InventoryItem(name="foo", price=10) >>> adapter = ItemAdapter(item) >>> adapter.item is item True >>> adapter["name"] 'foo' >>> adapter["name"] = "bar" >>> adapter["price"] = 5 >>> item InventoryItem(name='bar', price=5) >>> ``` ## Changelog See the [full changelog](Changelog.md) itemadapter-0.11.0/itemadapter/000077500000000000000000000000001474641140400163745ustar00rootroot00000000000000itemadapter-0.11.0/itemadapter/__init__.py000066400000000000000000000003011474641140400204770ustar00rootroot00000000000000from .adapter import ItemAdapter from .utils import get_field_meta_from_class, is_item __version__ = "0.11.0" __all__ = [ "ItemAdapter", "get_field_meta_from_class", "is_item", ] itemadapter-0.11.0/itemadapter/_imports.py000066400000000000000000000022761474641140400206110ustar00rootroot00000000000000from __future__ import annotations from typing import Any # attempt the following imports only once, # to be imported from itemadapter's submodules _scrapy_item_classes: tuple scrapy: Any try: import scrapy except ImportError: _scrapy_item_classes = () scrapy = None else: try: # handle deprecated base classes _base_item_cls = getattr( scrapy.item, "_BaseItem", scrapy.item.BaseItem, ) except AttributeError: _scrapy_item_classes = (scrapy.item.Item,) else: _scrapy_item_classes = (scrapy.item.Item, _base_item_cls) attr: Any try: import attr # pylint: disable=W0611 (unused-import) except ImportError: attr = None pydantic_v1: Any = None pydantic: Any = None try: import pydantic except ImportError: # No pydantic pass else: try: import pydantic.v1 as pydantic_v1 # pylint: disable=W0611 (unused-import) except ImportError: # Pydantic <1.10.17 pydantic_v1 = pydantic pydantic = None else: # Pydantic 1.10.17+ if not hasattr(pydantic.BaseModel, "model_fields"): # Pydantic <2 pydantic_v1 = pydantic pydantic = None itemadapter-0.11.0/itemadapter/adapter.py000066400000000000000000000346121474641140400203740ustar00rootroot00000000000000from __future__ import annotations import dataclasses from abc import ABCMeta, abstractmethod from collections import deque from collections.abc import Iterable, Iterator, KeysView, MutableMapping from types import MappingProxyType from typing import Any from itemadapter._imports import _scrapy_item_classes, attr from itemadapter.utils import ( _get_pydantic_model_metadata, _get_pydantic_v1_model_metadata, _is_attrs_class, _is_pydantic_model, _is_pydantic_v1_model, ) __all__ = [ "AdapterInterface", "AttrsAdapter", "DataclassAdapter", "DictAdapter", "ItemAdapter", "PydanticAdapter", "ScrapyItemAdapter", ] class AdapterInterface(MutableMapping, metaclass=ABCMeta): """Abstract Base Class for adapters. An adapter that handles a specific type of item should inherit from this class and implement the abstract methods defined here, plus the abtract methods inherited from the MutableMapping base class. """ def __init__(self, item: Any) -> None: self.item = item @classmethod @abstractmethod def is_item_class(cls, item_class: type) -> bool: """Return True if the adapter can handle the given item class, False otherwise.""" raise NotImplementedError() @classmethod def is_item(cls, item: Any) -> bool: """Return True if the adapter can handle the given item, False otherwise.""" return cls.is_item_class(item.__class__) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType({}) @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: """Return a list of fields defined for ``item_class``. If a class doesn't support fields, None is returned.""" return None def get_field_meta(self, field_name: str) -> MappingProxyType: """Return metadata for the given field name, if available.""" return self.get_field_meta_from_class(self.item.__class__, field_name) def field_names(self) -> KeysView: """Return a dynamic view of the item's field names.""" return self.keys() # type: ignore[return-value] class _MixinAttrsDataclassAdapter: _fields_dict: dict item: Any def get_field_meta(self, field_name: str) -> MappingProxyType: return self._fields_dict[field_name].metadata def field_names(self) -> KeysView: return KeysView(self._fields_dict) def __getitem__(self, field_name: str) -> Any: if field_name in self._fields_dict: return getattr(self.item, field_name) raise KeyError(field_name) def __setitem__(self, field_name: str, value: Any) -> None: if field_name in self._fields_dict: setattr(self.item, field_name, value) else: raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") def __delitem__(self, field_name: str) -> None: if field_name in self._fields_dict: try: if hasattr(self.item, field_name): delattr(self.item, field_name) else: raise AttributeError except AttributeError: raise KeyError(field_name) else: raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") def __iter__(self) -> Iterator: return iter(attr for attr in self._fields_dict if hasattr(self.item, attr)) def __len__(self) -> int: return len(list(iter(self))) class AttrsAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: super().__init__(item) if attr is None: raise RuntimeError("attr module is not available") # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals self._fields_dict = attr.fields_dict(self.item.__class__) @classmethod def is_item(cls, item: Any) -> bool: return _is_attrs_class(item) and not isinstance(item, type) @classmethod def is_item_class(cls, item_class: type) -> bool: return _is_attrs_class(item_class) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: if attr is None: raise RuntimeError("attr module is not available") try: return attr.fields_dict(item_class)[field_name].metadata # type: ignore except KeyError: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: if attr is None: raise RuntimeError("attr module is not available") return [a.name for a in attr.fields(item_class)] class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): def __init__(self, item: Any) -> None: super().__init__(item) # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals self._fields_dict = {field.name: field for field in dataclasses.fields(self.item)} @classmethod def is_item(cls, item: Any) -> bool: return dataclasses.is_dataclass(item) and not isinstance(item, type) @classmethod def is_item_class(cls, item_class: type) -> bool: return dataclasses.is_dataclass(item_class) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: for field in dataclasses.fields(item_class): if field.name == field_name: return field.metadata # type: ignore raise KeyError(f"{item_class.__name__} does not support field: {field_name}") @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: return [a.name for a in dataclasses.fields(item_class)] class PydanticAdapter(AdapterInterface): item: Any @classmethod def is_item_class(cls, item_class: type) -> bool: return _is_pydantic_model(item_class) or _is_pydantic_v1_model(item_class) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: try: try: return _get_pydantic_model_metadata(item_class, field_name) except AttributeError: return _get_pydantic_v1_model_metadata(item_class, field_name) except KeyError: raise KeyError(f"{item_class.__name__} does not support field: {field_name}") @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: try: return list(item_class.model_fields.keys()) # type: ignore[attr-defined] except AttributeError: return list(item_class.__fields__.keys()) # type: ignore[attr-defined] def field_names(self) -> KeysView: try: return KeysView(self.item.model_fields) except AttributeError: return KeysView(self.item.__fields__) def __getitem__(self, field_name: str) -> Any: try: self.item.model_fields except AttributeError: if field_name in self.item.__fields__: return getattr(self.item, field_name) else: if field_name in self.item.model_fields: return getattr(self.item, field_name) raise KeyError(field_name) def __setitem__(self, field_name: str, value: Any) -> None: try: self.item.model_fields except AttributeError: if field_name in self.item.__fields__: setattr(self.item, field_name, value) return else: if field_name in self.item.model_fields: setattr(self.item, field_name, value) return raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") def __delitem__(self, field_name: str) -> None: try: self.item.model_fields except AttributeError: if field_name in self.item.__fields__: try: if hasattr(self.item, field_name): delattr(self.item, field_name) return raise AttributeError except AttributeError: raise KeyError(field_name) else: if field_name in self.item.model_fields: try: if hasattr(self.item, field_name): delattr(self.item, field_name) return raise AttributeError except AttributeError: raise KeyError(field_name) raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") def __iter__(self) -> Iterator: try: return iter(attr for attr in self.item.model_fields if hasattr(self.item, attr)) except AttributeError: return iter(attr for attr in self.item.__fields__ if hasattr(self.item, attr)) def __len__(self) -> int: return len(list(iter(self))) class _MixinDictScrapyItemAdapter: _fields_dict: dict item: Any def __getitem__(self, field_name: str) -> Any: return self.item[field_name] def __setitem__(self, field_name: str, value: Any) -> None: self.item[field_name] = value def __delitem__(self, field_name: str) -> None: del self.item[field_name] def __iter__(self) -> Iterator: return iter(self.item) def __len__(self) -> int: return len(self.item) class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): @classmethod def is_item(cls, item: Any) -> bool: return isinstance(item, dict) @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, dict) def field_names(self) -> KeysView: return KeysView(self.item) class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): @classmethod def is_item(cls, item: Any) -> bool: return isinstance(item, _scrapy_item_classes) @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, _scrapy_item_classes) @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class.fields[field_name]) # type: ignore[attr-defined] @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: return list(item_class.fields.keys()) # type: ignore[attr-defined] def field_names(self) -> KeysView: return KeysView(self.item.fields) class ItemAdapter(MutableMapping): """Wrapper class to interact with data container objects. It provides a common interface to extract and set data without having to take the object's type into account. """ ADAPTER_CLASSES: Iterable[type[AdapterInterface]] = deque( [ ScrapyItemAdapter, DictAdapter, DataclassAdapter, AttrsAdapter, PydanticAdapter, ] ) def __init__(self, item: Any) -> None: for cls in self.ADAPTER_CLASSES: if cls.is_item(item): self.adapter = cls(item) break else: raise TypeError(f"No adapter found for objects of type: {type(item)} ({item})") @classmethod def is_item(cls, item: Any) -> bool: for adapter_class in cls.ADAPTER_CLASSES: if adapter_class.is_item(item): return True return False @classmethod def is_item_class(cls, item_class: type) -> bool: for adapter_class in cls.ADAPTER_CLASSES: if adapter_class.is_item_class(item_class): return True return False @classmethod def _get_adapter_class(cls, item_class: type) -> type[AdapterInterface]: for adapter_class in cls.ADAPTER_CLASSES: if adapter_class.is_item_class(item_class): return adapter_class raise TypeError(f"{item_class} is not a valid item class") @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: adapter_class = cls._get_adapter_class(item_class) return adapter_class.get_field_meta_from_class(item_class, field_name) @classmethod def get_field_names_from_class(cls, item_class: type) -> list[str] | None: adapter_class = cls._get_adapter_class(item_class) return adapter_class.get_field_names_from_class(item_class) @property def item(self) -> Any: return self.adapter.item def __repr__(self) -> str: values = ", ".join([f"{key}={value!r}" for key, value in self.items()]) return f"<{self.__class__.__name__} for {self.item.__class__.__name__}({values})>" def __getitem__(self, field_name: str) -> Any: return self.adapter.__getitem__(field_name) def __setitem__(self, field_name: str, value: Any) -> None: self.adapter.__setitem__(field_name, value) def __delitem__(self, field_name: str) -> None: self.adapter.__delitem__(field_name) def __iter__(self) -> Iterator: return self.adapter.__iter__() def __len__(self) -> int: return self.adapter.__len__() def get_field_meta(self, field_name: str) -> MappingProxyType: """Return metadata for the given field name.""" return self.adapter.get_field_meta(field_name) def field_names(self) -> KeysView: """Return read-only key view with the names of all the defined fields for the item.""" return self.adapter.field_names() def asdict(self) -> dict: """Return a dict object with the contents of the adapter. This works slightly different than calling `dict(adapter)`: it's applied recursively to nested items (if there are any). """ return {key: self._asdict(value) for key, value in self.items()} @classmethod def _asdict(cls, obj: Any) -> Any: if isinstance(obj, dict): return {key: cls._asdict(value) for key, value in obj.items()} if isinstance(obj, (list, set, tuple)): return obj.__class__(cls._asdict(x) for x in obj) if isinstance(obj, cls): return obj.asdict() if cls.is_item(obj): return cls(obj).asdict() return obj itemadapter-0.11.0/itemadapter/py.typed000066400000000000000000000000001474641140400200610ustar00rootroot00000000000000itemadapter-0.11.0/itemadapter/utils.py000066400000000000000000000071741474641140400201170ustar00rootroot00000000000000from __future__ import annotations from types import MappingProxyType from typing import Any from itemadapter._imports import attr, pydantic, pydantic_v1 __all__ = ["is_item", "get_field_meta_from_class"] def _is_attrs_class(obj: Any) -> bool: if attr is None: return False return attr.has(obj) def _is_pydantic_model(obj: Any) -> bool: if pydantic is None: return False return issubclass(obj, pydantic.BaseModel) def _is_pydantic_v1_model(obj: Any) -> bool: if pydantic_v1 is None: return False return issubclass(obj, pydantic_v1.BaseModel) def _get_pydantic_model_metadata(item_model: Any, field_name: str) -> MappingProxyType: metadata = {} field = item_model.model_fields[field_name] for attribute in [ "default", "default_factory", "alias", "alias_priority", "validation_alias", "serialization_alias", "title", "field_title_generator", "description", "examples", "exclude", "discriminator", "deprecated", "json_schema_extra", "frozen", "validate_default", "repr", "init", "init_var", "kw_only", "pattern", "strict", "coerce_numbers_to_str", "gt", "ge", "lt", "le", "multiple_of", "allow_inf_nan", "max_digits", "decimal_places", "min_length", "max_length", "union_mode", "fail_fast", ]: if hasattr(field, attribute) and (value := getattr(field, attribute)) is not None: metadata[attribute] = value return MappingProxyType(metadata) def _get_pydantic_v1_model_metadata(item_model: Any, field_name: str) -> MappingProxyType: metadata = {} field = item_model.__fields__[field_name].field_info for attribute in [ "alias", "title", "description", "const", "gt", "ge", "lt", "le", "multiple_of", "min_items", "max_items", "min_length", "max_length", "regex", ]: value = getattr(field, attribute) if value is not None: metadata[attribute] = value if not field.allow_mutation: metadata["allow_mutation"] = field.allow_mutation metadata.update(field.extra) return MappingProxyType(metadata) def is_item(obj: Any) -> bool: """Return True if the given object belongs to one of the supported types, False otherwise. Alias for ItemAdapter.is_item """ from itemadapter.adapter import ItemAdapter return ItemAdapter.is_item(obj) def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType: """Return a read-only mapping with metadata for the given field name, within the given item class. If there is no metadata for the field, or the item class does not support field metadata, an empty object is returned. Field metadata is taken from different sources, depending on the item type: * scrapy.item.Item: corresponding scrapy.item.Field object * dataclass items: "metadata" attribute for the corresponding field * attrs items: "metadata" attribute for the corresponding field * pydantic models: corresponding pydantic.field.FieldInfo/ModelField object The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view of the original mapping, which gets automatically updated if the original mapping changes. """ from itemadapter.adapter import ItemAdapter return ItemAdapter.get_field_meta_from_class(item_class, field_name) itemadapter-0.11.0/pylintrc000066400000000000000000000010701474641140400156620ustar00rootroot00000000000000[MESSAGES CONTROL] enable=useless-suppression disable= cyclic-import, duplicate-code, import-error, import-outside-toplevel, invalid-name, missing-class-docstring, missing-function-docstring, missing-module-docstring, not-callable, pointless-statement, protected-access, raise-missing-from, too-few-public-methods, too-many-return-statements, unused-argument, [FORMAT] expected-line-ending-format=LF max-line-length=99 [IMPORTS] allow-any-import-level=attr,dataclasses,scrapy itemadapter-0.11.0/pyproject.toml000066400000000000000000000000361474641140400170100ustar00rootroot00000000000000[tool.black] line-length = 99 itemadapter-0.11.0/pytest.ini000066400000000000000000000000621474641140400161240ustar00rootroot00000000000000[pytest] filterwarnings = ignore:.*BaseItem.* itemadapter-0.11.0/setup.cfg000066400000000000000000000000361474641140400157150ustar00rootroot00000000000000[flake8] max-line-length = 99 itemadapter-0.11.0/setup.py000066400000000000000000000030231474641140400156050ustar00rootroot00000000000000import setuptools with open("README.md") as fh: long_description = fh.read() setuptools.setup( name="itemadapter", version="0.11.0", license="BSD", description="Common interface for data container classes", long_description=long_description, long_description_content_type="text/markdown", author="Eugenio Lacuesta", author_email="eugenio.lacuesta@gmail.com", url="https://github.com/scrapy/itemadapter", packages=["itemadapter"], package_data={ "itemadapter": ["py.typed"], }, include_package_data=True, python_requires=">=3.9", extras_require={ "attrs": ["attrs>=18.1.0"], "pydantic": ["pydantic>=1.8"], "scrapy": ["scrapy>=2.2"], }, classifiers=[ "Development Status :: 3 - Alpha", "License :: OSI Approved :: BSD License", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Framework :: Scrapy", "Intended Audience :: Developers", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], ) itemadapter-0.11.0/tests/000077500000000000000000000000001474641140400152375ustar00rootroot00000000000000itemadapter-0.11.0/tests/__init__.py000066400000000000000000000135061474641140400173550ustar00rootroot00000000000000import importlib import sys from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass, field from typing import Callable, Optional from itemadapter import ItemAdapter from itemadapter._imports import pydantic, pydantic_v1 def make_mock_import(block_name: str) -> Callable: def mock_import(name: str, *args, **kwargs): """Prevent importing a specific module, let everything else pass.""" if name.split(".")[0] == block_name: raise ImportError(name) return importlib.__import__(name, *args, **kwargs) return mock_import @contextmanager def clear_itemadapter_imports() -> Generator[None, None, None]: backup = {} for key in sys.modules.copy(): if key.startswith("itemadapter"): backup[key] = sys.modules.pop(key) try: yield finally: sys.modules.update(backup) @dataclass class DataClassItem: name: str = field(default_factory=lambda: None, metadata={"serializer": str}) value: int = field(default_factory=lambda: None, metadata={"serializer": int}) @dataclass class DataClassItemNested: nested: DataClassItem adapter: ItemAdapter dict_: dict list_: list set_: set tuple_: tuple int_: int @dataclass(init=False) class DataClassWithoutInit: name: str = field(metadata={"serializer": str}) value: int = field(metadata={"serializer": int}) @dataclass class DataClassItemSubclassed(DataClassItem): subclassed: bool = True @dataclass class DataClassItemEmpty: pass try: import attr except ImportError: AttrsItem = None AttrsItemNested = None AttrsItemWithoutInit = None AttrsItemSubclassed = None AttrsItemEmpty = None else: @attr.s class AttrsItem: name = attr.ib(default=None, metadata={"serializer": str}) value = attr.ib(default=None, metadata={"serializer": int}) @attr.s class AttrsItemNested: nested = attr.ib(type=AttrsItem) adapter = attr.ib(type=ItemAdapter) dict_ = attr.ib(type=dict) list_ = attr.ib(type=list) set_ = attr.ib(type=set) tuple_ = attr.ib(type=tuple) int_ = attr.ib(type=int) @attr.s(init=False) class AttrsItemWithoutInit: name = attr.ib(default=None, metadata={"serializer": str}) value = attr.ib(default=None, metadata={"serializer": int}) @attr.s(init=False) class AttrsItemSubclassed(AttrsItem): subclassed = attr.ib(default=True, type=bool) @attr.s class AttrsItemEmpty: pass if pydantic_v1 is None: PydanticV1Model = None PydanticV1SpecialCasesModel = None PydanticV1ModelNested = None PydanticV1ModelSubclassed = None PydanticV1ModelEmpty = None else: class PydanticV1Model(pydantic_v1.BaseModel): name: Optional[str] = pydantic_v1.Field( default_factory=lambda: None, serializer=str, ) value: Optional[int] = pydantic_v1.Field( default_factory=lambda: None, serializer=int, ) class PydanticV1SpecialCasesModel(pydantic_v1.BaseModel): special_cases: Optional[int] = pydantic_v1.Field( default_factory=lambda: None, alias="special_cases", allow_mutation=False, ) class Config: validate_assignment = True class PydanticV1ModelNested(pydantic_v1.BaseModel): nested: PydanticV1Model adapter: ItemAdapter dict_: dict list_: list set_: set tuple_: tuple int_: int class Config: arbitrary_types_allowed = True class PydanticV1ModelSubclassed(PydanticV1Model): subclassed: bool = pydantic_v1.Field( default_factory=lambda: True, ) class PydanticV1ModelEmpty(pydantic_v1.BaseModel): pass if pydantic is None: PydanticModel = None PydanticSpecialCasesModel = None PydanticModelNested = None PydanticModelSubclassed = None PydanticModelEmpty = None else: class PydanticModel(pydantic.BaseModel): name: Optional[str] = pydantic.Field( default_factory=lambda: None, serializer=str, ) value: Optional[int] = pydantic.Field( default_factory=lambda: None, serializer=int, ) class PydanticSpecialCasesModel(pydantic.BaseModel): special_cases: Optional[int] = pydantic.Field( default_factory=lambda: None, alias="special_cases", allow_mutation=False, ) class Config: validate_assignment = True class PydanticModelNested(pydantic.BaseModel): nested: PydanticModel adapter: ItemAdapter dict_: dict list_: list set_: set tuple_: tuple int_: int class Config: arbitrary_types_allowed = True class PydanticModelSubclassed(PydanticModel): subclassed: bool = pydantic.Field( default_factory=lambda: True, ) class PydanticModelEmpty(pydantic.BaseModel): pass try: from scrapy.item import Field from scrapy.item import Item as ScrapyItem except ImportError: ScrapyItem = None ScrapySubclassedItem = None ScrapySubclassedItemNested = None ScrapySubclassedItemSubclassed = None ScrapySubclassedItemEmpty = None else: class ScrapySubclassedItem(ScrapyItem): name = Field(serializer=str) value = Field(serializer=int) class ScrapySubclassedItemNested(ScrapyItem): nested = Field() adapter = Field() dict_ = Field() list_ = Field() set_ = Field() tuple_ = Field() int_ = Field() class ScrapySubclassedItemSubclassed(ScrapySubclassedItem): subclassed = Field() class ScrapySubclassedItemEmpty(ScrapyItem): pass itemadapter-0.11.0/tests/requirements.txt000066400000000000000000000000351474641140400205210ustar00rootroot00000000000000pytest>=5.4 pytest-cov>=2.8 itemadapter-0.11.0/tests/test_adapter.py000066400000000000000000000244201474641140400202720ustar00rootroot00000000000000import unittest from collections.abc import KeysView from types import MappingProxyType from itemadapter.adapter import ItemAdapter from tests import ( AttrsItem, AttrsItemEmpty, AttrsItemNested, AttrsItemSubclassed, AttrsItemWithoutInit, DataClassItem, DataClassItemEmpty, DataClassItemNested, DataClassItemSubclassed, DataClassWithoutInit, PydanticV1Model, PydanticV1ModelEmpty, PydanticV1ModelNested, PydanticV1ModelSubclassed, ScrapySubclassedItem, ScrapySubclassedItemEmpty, ScrapySubclassedItemNested, ScrapySubclassedItemSubclassed, ) class ItemAdapterReprTestCase(unittest.TestCase): def test_repr_dict(self): item = {"name": "asdf", "value": 1234} adapter = ItemAdapter(item) self.assertEqual(repr(adapter), "") @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_repr_scrapy_item(self): item = ScrapySubclassedItem(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual( repr(adapter), "" ) @unittest.skipIf(not DataClassItem, "dataclasses module is not available") def test_repr_dataclass(self): item = DataClassItem(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual( repr(adapter), "", ) @unittest.skipIf(not DataClassWithoutInit, "dataclasses module is not available") def test_repr_dataclass_init_false(self): item = DataClassWithoutInit() adapter = ItemAdapter(item) self.assertEqual(repr(adapter), "") adapter["name"] = "set after init" self.assertEqual( repr(adapter), "" ) @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_repr_attrs(self): item = AttrsItem(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual( repr(adapter), "", ) @unittest.skipIf(not AttrsItemWithoutInit, "attrs module is not available") def test_repr_attrs_init_false(self): item = AttrsItemWithoutInit() adapter = ItemAdapter(item) self.assertEqual(repr(adapter), "") adapter["name"] = "set after init" self.assertEqual( repr(adapter), "" ) @unittest.skipIf(not PydanticV1Model, "pydantic module is not available") def test_repr_pydantic(self): item = PydanticV1Model(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual( repr(adapter), "", ) class ItemAdapterInitError(unittest.TestCase): def test_non_item(self): with self.assertRaises(TypeError): ItemAdapter(ScrapySubclassedItem) with self.assertRaises(TypeError): ItemAdapter(dict) with self.assertRaises(TypeError): ItemAdapter(1234) class BaseTestMixin: item_class = None item_class_nested = None def setUp(self): if self.item_class is None: raise unittest.SkipTest() def test_get_set_value(self): item = self.item_class() adapter = ItemAdapter(item) self.assertEqual(adapter.get("name"), None) self.assertEqual(adapter.get("value"), None) adapter["name"] = "asdf" adapter["value"] = 1234 self.assertEqual(adapter.get("name"), "asdf") self.assertEqual(adapter.get("value"), 1234) self.assertEqual(adapter["name"], "asdf") self.assertEqual(adapter["value"], 1234) item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual(adapter.get("name"), "asdf") self.assertEqual(adapter.get("value"), 1234) self.assertEqual(adapter["name"], "asdf") self.assertEqual(adapter["value"], 1234) def test_get_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with self.assertRaises(KeyError): adapter["undefined_field"] def test_as_dict(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual({"name": "asdf", "value": 1234}, dict(adapter)) def test_as_dict_nested(self): item = self.item_class_nested( nested=self.item_class(name="asdf", value=1234), adapter=ItemAdapter({"foo": "bar", "nested_list": [1, 2, 3, 4, 5]}), dict_={"foo": "bar", "answer": 42, "nested_dict": {"a": "b"}}, list_=[1, 2, 3], set_={1, 2, 3}, tuple_=(1, 2, 3), int_=123, ) adapter = ItemAdapter(item) self.assertEqual( adapter.asdict(), { "nested": {"name": "asdf", "value": 1234}, "adapter": {"foo": "bar", "nested_list": [1, 2, 3, 4, 5]}, "dict_": {"foo": "bar", "answer": 42, "nested_dict": {"a": "b"}}, "list_": [1, 2, 3], "set_": {1, 2, 3}, "tuple_": (1, 2, 3), "int_": 123, }, ) def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertIsInstance(adapter.field_names(), KeysView) self.assertEqual(sorted(adapter.field_names()), ["name", "value"]) class NonDictTestMixin(BaseTestMixin): item_class_subclassed = None item_class_empty = None def test_set_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with self.assertRaises(KeyError): adapter["undefined_field"] = "some value" def test_metadata_common(self): adapter = ItemAdapter(self.item_class()) self.assertIsInstance(adapter.get_field_meta("name"), MappingProxyType) self.assertIsInstance(adapter.get_field_meta("value"), MappingProxyType) with self.assertRaises(KeyError): adapter.get_field_meta("undefined_field") def test_get_field_meta_defined_fields(self): adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({"serializer": str})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({"serializer": int})) def test_delitem_len_iter(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual(len(adapter), 2) self.assertEqual(sorted(list(iter(adapter))), ["name", "value"]) del adapter["name"] self.assertEqual(len(adapter), 1) self.assertEqual(sorted(list(iter(adapter))), ["value"]) del adapter["value"] self.assertEqual(len(adapter), 0) self.assertEqual(sorted(list(iter(adapter))), []) with self.assertRaises(KeyError): del adapter["name"] with self.assertRaises(KeyError): del adapter["value"] with self.assertRaises(KeyError): del adapter["undefined_field"] def test_field_names_from_class(self): field_names = ItemAdapter.get_field_names_from_class(self.item_class) assert isinstance(field_names, list) self.assertEqual(sorted(field_names), ["name", "value"]) def test_field_names_from_class_nested(self): field_names = ItemAdapter.get_field_names_from_class(self.item_class_subclassed) assert isinstance(field_names, list) self.assertEqual(sorted(field_names), ["name", "subclassed", "value"]) def test_field_names_from_class_empty(self): field_names = ItemAdapter.get_field_names_from_class(self.item_class_empty) assert isinstance(field_names, list) self.assertEqual(field_names, []) class DictTestCase(unittest.TestCase, BaseTestMixin): item_class = dict item_class_nested = dict def test_get_value_keyerror_item_dict(self): """Instantiate without default values.""" adapter = ItemAdapter(self.item_class()) with self.assertRaises(KeyError): adapter["name"] def test_empty_metadata(self): adapter = ItemAdapter(self.item_class(name="foo", value=5)) for field_name in ("name", "value", "undefined_field"): self.assertEqual(adapter.get_field_meta(field_name), MappingProxyType({})) def test_field_names_updated(self): item = self.item_class(name="asdf") field_names = ItemAdapter(item).field_names() self.assertEqual(sorted(field_names), ["name"]) item["value"] = 1234 self.assertEqual(sorted(field_names), ["name", "value"]) def test_field_names_from_class(self): assert ItemAdapter.get_field_names_from_class(dict) is None class ScrapySubclassedItemTestCase(NonDictTestMixin, unittest.TestCase): item_class = ScrapySubclassedItem item_class_nested = ScrapySubclassedItemNested item_class_subclassed = ScrapySubclassedItemSubclassed item_class_empty = ScrapySubclassedItemEmpty def test_get_value_keyerror_item_dict(self): """Instantiate without default values.""" adapter = ItemAdapter(self.item_class()) with self.assertRaises(KeyError): adapter["name"] class PydanticV1ModelTestCase(NonDictTestMixin, unittest.TestCase): item_class = PydanticV1Model item_class_nested = PydanticV1ModelNested item_class_subclassed = PydanticV1ModelSubclassed item_class_empty = PydanticV1ModelEmpty class DataClassItemTestCase(NonDictTestMixin, unittest.TestCase): item_class = DataClassItem item_class_nested = DataClassItemNested item_class_subclassed = DataClassItemSubclassed item_class_empty = DataClassItemEmpty class AttrsItemTestCase(NonDictTestMixin, unittest.TestCase): item_class = AttrsItem item_class_nested = AttrsItemNested item_class_subclassed = AttrsItemSubclassed item_class_empty = AttrsItemEmpty itemadapter-0.11.0/tests/test_adapter_attrs.py000066400000000000000000000073441474641140400215150ustar00rootroot00000000000000import unittest from types import MappingProxyType from unittest import mock from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticModel, PydanticV1Model, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, ) class AttrsTestCase(unittest.TestCase): def test_false(self): from itemadapter.adapter import AttrsAdapter self.assertFalse(AttrsAdapter.is_item(int)) self.assertFalse(AttrsAdapter.is_item(sum)) self.assertFalse(AttrsAdapter.is_item(1234)) self.assertFalse(AttrsAdapter.is_item(object())) self.assertFalse(AttrsAdapter.is_item(DataClassItem())) self.assertFalse(AttrsAdapter.is_item("a string")) self.assertFalse(AttrsAdapter.is_item(b"some bytes")) self.assertFalse(AttrsAdapter.is_item({"a": "dict"})) self.assertFalse(AttrsAdapter.is_item(["a", "list"])) self.assertFalse(AttrsAdapter.is_item(("a", "tuple"))) self.assertFalse(AttrsAdapter.is_item({"a", "set"})) self.assertFalse(AttrsAdapter.is_item(AttrsItem)) if PydanticModel is not None: self.assertFalse(AttrsAdapter.is_item(PydanticModel())) if PydanticV1Model is not None: self.assertFalse(AttrsAdapter.is_item(PydanticV1Model())) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(AttrsAdapter.is_item(ScrapyItem())) self.assertFalse(AttrsAdapter.is_item(ScrapySubclassedItem())) @unittest.skipIf(not AttrsItem, "attrs module is not available") @mock.patch("builtins.__import__", make_mock_import("attr")) def test_module_import_error(self): with clear_itemadapter_imports(): from itemadapter.adapter import AttrsAdapter self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) with self.assertRaises(RuntimeError, msg="attr module is not available"): AttrsAdapter(AttrsItem(name="asdf", value=1234)) with self.assertRaises(RuntimeError, msg="attr module is not available"): AttrsAdapter.get_field_meta_from_class(AttrsItem, "name") with self.assertRaises(RuntimeError, msg="attr module is not available"): AttrsAdapter.get_field_names_from_class(AttrsItem) with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"): get_field_meta_from_class(AttrsItem, "name") @unittest.skipIf(not AttrsItem, "attrs module is not available") @mock.patch("itemadapter.utils.attr", None) def test_module_not_available(self): from itemadapter.adapter import AttrsAdapter self.assertFalse(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="AttrsItem is not a valid item class"): get_field_meta_from_class(AttrsItem, "name") @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_true(self): from itemadapter.adapter import AttrsAdapter self.assertTrue(AttrsAdapter.is_item(AttrsItem())) self.assertTrue(AttrsAdapter.is_item(AttrsItem(name="asdf", value=1234))) # field metadata self.assertEqual( get_field_meta_from_class(AttrsItem, "name"), MappingProxyType({"serializer": str}) ) self.assertEqual( get_field_meta_from_class(AttrsItem, "value"), MappingProxyType({"serializer": int}) ) with self.assertRaises(KeyError, msg="AttrsItem does not support field: non_existent"): get_field_meta_from_class(AttrsItem, "non_existent") itemadapter-0.11.0/tests/test_adapter_dataclasses.py000066400000000000000000000047331474641140400226460ustar00rootroot00000000000000from types import MappingProxyType from unittest import TestCase from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticModel, PydanticV1Model, ScrapyItem, ScrapySubclassedItem, ) class DataclassTestCase(TestCase): def test_false(self): from itemadapter.adapter import DataclassAdapter self.assertFalse(DataclassAdapter.is_item(int)) self.assertFalse(DataclassAdapter.is_item(sum)) self.assertFalse(DataclassAdapter.is_item(1234)) self.assertFalse(DataclassAdapter.is_item(object())) self.assertFalse(DataclassAdapter.is_item("a string")) self.assertFalse(DataclassAdapter.is_item(b"some bytes")) self.assertFalse(DataclassAdapter.is_item({"a": "dict"})) self.assertFalse(DataclassAdapter.is_item(["a", "list"])) self.assertFalse(DataclassAdapter.is_item(("a", "tuple"))) self.assertFalse(DataclassAdapter.is_item({"a", "set"})) self.assertFalse(DataclassAdapter.is_item(DataClassItem)) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(DataclassAdapter.is_item(AttrsItem())) if PydanticModel is not None: self.assertFalse(DataclassAdapter.is_item(PydanticModel())) if PydanticV1Model is not None: self.assertFalse(DataclassAdapter.is_item(PydanticV1Model())) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(DataclassAdapter.is_item(ScrapyItem())) self.assertFalse(DataclassAdapter.is_item(ScrapySubclassedItem())) def test_true(self): from itemadapter.adapter import DataclassAdapter self.assertTrue(DataclassAdapter.is_item(DataClassItem())) self.assertTrue(DataclassAdapter.is_item(DataClassItem(name="asdf", value=1234))) # field metadata self.assertEqual( get_field_meta_from_class(DataClassItem, "name"), MappingProxyType({"serializer": str}) ) self.assertEqual( get_field_meta_from_class(DataClassItem, "value"), MappingProxyType({"serializer": int}), ) with self.assertRaises(KeyError, msg="DataClassItem does not support field: non_existent"): get_field_meta_from_class(DataClassItem, "non_existent") itemadapter-0.11.0/tests/test_adapter_pydantic.py000066400000000000000000000114751474641140400221730ustar00rootroot00000000000000import unittest from types import MappingProxyType from unittest import mock from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticModel, PydanticSpecialCasesModel, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, ) class PydanticTestCase(unittest.TestCase): def test_false(self): from itemadapter.adapter import PydanticAdapter self.assertFalse(PydanticAdapter.is_item(int)) self.assertFalse(PydanticAdapter.is_item(sum)) self.assertFalse(PydanticAdapter.is_item(1234)) self.assertFalse(PydanticAdapter.is_item(object())) self.assertFalse(PydanticAdapter.is_item(DataClassItem())) self.assertFalse(PydanticAdapter.is_item("a string")) self.assertFalse(PydanticAdapter.is_item(b"some bytes")) self.assertFalse(PydanticAdapter.is_item({"a": "dict"})) self.assertFalse(PydanticAdapter.is_item(["a", "list"])) self.assertFalse(PydanticAdapter.is_item(("a", "tuple"))) self.assertFalse(PydanticAdapter.is_item({"a", "set"})) self.assertFalse(PydanticAdapter.is_item(PydanticModel)) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(PydanticAdapter.is_item(AttrsItem())) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(PydanticAdapter.is_item(ScrapyItem())) self.assertFalse(PydanticAdapter.is_item(ScrapySubclassedItem())) @unittest.skipIf(not PydanticModel, "pydantic <2 module is not available") @mock.patch("builtins.__import__", make_mock_import("pydantic")) def test_module_import_error(self): with clear_itemadapter_imports(): from itemadapter.adapter import PydanticAdapter self.assertFalse(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="PydanticModel is not a valid item class"): get_field_meta_from_class(PydanticModel, "name") @unittest.skipIf(not PydanticModel, "pydantic module is not available") @mock.patch("itemadapter.utils.pydantic", None) @mock.patch("itemadapter.utils.pydantic_v1", None) def test_module_not_available(self): from itemadapter.adapter import PydanticAdapter self.assertFalse(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="PydanticModel is not a valid item class"): get_field_meta_from_class(PydanticModel, "name") @unittest.skipIf(not PydanticModel, "pydantic module is not available") def test_true(self): from pydantic_core import PydanticUndefined from itemadapter.adapter import PydanticAdapter self.assertTrue(PydanticAdapter.is_item(PydanticModel())) self.assertTrue(PydanticAdapter.is_item(PydanticModel(name="asdf", value=1234))) # field metadata mapping_proxy_type = get_field_meta_from_class(PydanticModel, "name") self.assertEqual( mapping_proxy_type, MappingProxyType( { "default": PydanticUndefined, "default_factory": mapping_proxy_type["default_factory"], "json_schema_extra": {"serializer": str}, "repr": True, } ), ) mapping_proxy_type = get_field_meta_from_class(PydanticModel, "value") self.assertEqual( get_field_meta_from_class(PydanticModel, "value"), MappingProxyType( { "default": PydanticUndefined, "default_factory": mapping_proxy_type["default_factory"], "json_schema_extra": {"serializer": int}, "repr": True, } ), ) mapping_proxy_type = get_field_meta_from_class(PydanticSpecialCasesModel, "special_cases") self.assertEqual( mapping_proxy_type, MappingProxyType( { "default": PydanticUndefined, "default_factory": mapping_proxy_type["default_factory"], "alias": "special_cases", "alias_priority": 2, "validation_alias": "special_cases", "serialization_alias": "special_cases", "frozen": True, "repr": True, } ), ) with self.assertRaises(KeyError, msg="PydanticModel does not support field: non_existent"): get_field_meta_from_class(PydanticModel, "non_existent") itemadapter-0.11.0/tests/test_adapter_pydantic_v1.py000066400000000000000000000074111474641140400225740ustar00rootroot00000000000000import unittest from types import MappingProxyType from unittest import mock from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticV1Model, PydanticV1SpecialCasesModel, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, ) class PydanticTestCase(unittest.TestCase): def test_false(self): from itemadapter.adapter import PydanticAdapter self.assertFalse(PydanticAdapter.is_item(int)) self.assertFalse(PydanticAdapter.is_item(sum)) self.assertFalse(PydanticAdapter.is_item(1234)) self.assertFalse(PydanticAdapter.is_item(object())) self.assertFalse(PydanticAdapter.is_item(DataClassItem())) self.assertFalse(PydanticAdapter.is_item("a string")) self.assertFalse(PydanticAdapter.is_item(b"some bytes")) self.assertFalse(PydanticAdapter.is_item({"a": "dict"})) self.assertFalse(PydanticAdapter.is_item(["a", "list"])) self.assertFalse(PydanticAdapter.is_item(("a", "tuple"))) self.assertFalse(PydanticAdapter.is_item({"a", "set"})) self.assertFalse(PydanticAdapter.is_item(PydanticV1Model)) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(PydanticAdapter.is_item(AttrsItem())) try: import scrapy # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(PydanticAdapter.is_item(ScrapyItem())) self.assertFalse(PydanticAdapter.is_item(ScrapySubclassedItem())) @unittest.skipIf(not PydanticV1Model, "pydantic <2 module is not available") @mock.patch("builtins.__import__", make_mock_import("pydantic")) def test_module_import_error(self): with clear_itemadapter_imports(): from itemadapter.adapter import PydanticAdapter self.assertFalse(PydanticAdapter.is_item(PydanticV1Model(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="PydanticV1Model is not a valid item class"): get_field_meta_from_class(PydanticV1Model, "name") @unittest.skipIf(not PydanticV1Model, "pydantic module is not available") @mock.patch("itemadapter.utils.pydantic", None) @mock.patch("itemadapter.utils.pydantic_v1", None) def test_module_not_available(self): from itemadapter.adapter import PydanticAdapter self.assertFalse(PydanticAdapter.is_item(PydanticV1Model(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="PydanticV1Model is not a valid item class"): get_field_meta_from_class(PydanticV1Model, "name") @unittest.skipIf(not PydanticV1Model, "pydantic module is not available") def test_true(self): from itemadapter.adapter import PydanticAdapter self.assertTrue(PydanticAdapter.is_item(PydanticV1Model())) self.assertTrue(PydanticAdapter.is_item(PydanticV1Model(name="asdf", value=1234))) # field metadata self.assertEqual( get_field_meta_from_class(PydanticV1Model, "name"), MappingProxyType({"serializer": str}), ) self.assertEqual( get_field_meta_from_class(PydanticV1Model, "value"), MappingProxyType({"serializer": int}), ) self.assertEqual( get_field_meta_from_class(PydanticV1SpecialCasesModel, "special_cases"), MappingProxyType({"alias": "special_cases", "allow_mutation": False}), ) with self.assertRaises( KeyError, msg="PydanticV1Model does not support field: non_existent" ): get_field_meta_from_class(PydanticV1Model, "non_existent") itemadapter-0.11.0/tests/test_adapter_scrapy.py000066400000000000000000000125761474641140400216640ustar00rootroot00000000000000import unittest from types import MappingProxyType from unittest import mock from itemadapter.utils import get_field_meta_from_class from tests import ( AttrsItem, DataClassItem, PydanticModel, PydanticV1Model, ScrapyItem, ScrapySubclassedItem, clear_itemadapter_imports, make_mock_import, ) class ScrapyItemTestCase(unittest.TestCase): def test_false(self): from itemadapter.adapter import ScrapyItemAdapter self.assertFalse(ScrapyItemAdapter.is_item(int)) self.assertFalse(ScrapyItemAdapter.is_item(sum)) self.assertFalse(ScrapyItemAdapter.is_item(1234)) self.assertFalse(ScrapyItemAdapter.is_item(object())) self.assertFalse(ScrapyItemAdapter.is_item(DataClassItem())) self.assertFalse(ScrapyItemAdapter.is_item("a string")) self.assertFalse(ScrapyItemAdapter.is_item(b"some bytes")) self.assertFalse(ScrapyItemAdapter.is_item({"a": "dict"})) self.assertFalse(ScrapyItemAdapter.is_item(["a", "list"])) self.assertFalse(ScrapyItemAdapter.is_item(("a", "tuple"))) self.assertFalse(ScrapyItemAdapter.is_item({"a", "set"})) self.assertFalse(ScrapyItemAdapter.is_item(ScrapySubclassedItem)) try: import attrs # noqa: F401 # pylint: disable=unused-import except ImportError: pass else: self.assertFalse(ScrapyItemAdapter.is_item(AttrsItem())) if PydanticModel is not None: self.assertFalse(ScrapyItemAdapter.is_item(PydanticModel())) if PydanticV1Model is not None: self.assertFalse(ScrapyItemAdapter.is_item(PydanticV1Model())) @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") @mock.patch("builtins.__import__", make_mock_import("scrapy")) def test_module_import_error(self): with clear_itemadapter_imports(): from itemadapter.adapter import ScrapyItemAdapter self.assertFalse( ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234)) ) with self.assertRaises( TypeError, msg="ScrapySubclassedItem is not a valid item class" ): get_field_meta_from_class(ScrapySubclassedItem, "name") @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") @mock.patch("itemadapter.adapter._scrapy_item_classes", ()) def test_module_not_available(self): from itemadapter.adapter import ScrapyItemAdapter self.assertFalse(ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234))) with self.assertRaises(TypeError, msg="ScrapySubclassedItem is not a valid item class"): get_field_meta_from_class(ScrapySubclassedItem, "name") @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_true(self): from itemadapter.adapter import ScrapyItemAdapter self.assertTrue(ScrapyItemAdapter.is_item(ScrapyItem())) self.assertTrue(ScrapyItemAdapter.is_item(ScrapySubclassedItem())) self.assertTrue(ScrapyItemAdapter.is_item(ScrapySubclassedItem(name="asdf", value=1234))) # field metadata self.assertEqual( get_field_meta_from_class(ScrapySubclassedItem, "name"), MappingProxyType({"serializer": str}), ) self.assertEqual( get_field_meta_from_class(ScrapySubclassedItem, "value"), MappingProxyType({"serializer": int}), ) try: import scrapy except ImportError: scrapy = None class ScrapyDeprecatedBaseItemTestCase(unittest.TestCase): """Tests for deprecated classes. These will go away once the upstream classes are removed.""" @unittest.skipIf( scrapy is None or not hasattr(scrapy.item, "_BaseItem"), "scrapy.item._BaseItem not available", ) def test_deprecated_underscore_baseitem(self): from itemadapter.adapter import ScrapyItemAdapter class SubClassed_BaseItem(scrapy.item._BaseItem): pass self.assertTrue(ScrapyItemAdapter.is_item(scrapy.item._BaseItem())) self.assertTrue(ScrapyItemAdapter.is_item(SubClassed_BaseItem())) @unittest.skipIf( scrapy is None or not hasattr(scrapy.item, "BaseItem"), "scrapy.item.BaseItem not available", ) def test_deprecated_baseitem(self): from itemadapter.adapter import ScrapyItemAdapter class SubClassedBaseItem(scrapy.item.BaseItem): pass self.assertTrue(ScrapyItemAdapter.is_item(scrapy.item.BaseItem())) self.assertTrue(ScrapyItemAdapter.is_item(SubClassedBaseItem())) @unittest.skipIf(scrapy is None, "scrapy module is not available") def test_removed_baseitem(self): """Mock the scrapy.item module so it does not contain the deprecated _BaseItem class.""" from itemadapter.adapter import ScrapyItemAdapter class MockItemModule: Item = ScrapyItem with mock.patch("scrapy.item", MockItemModule): self.assertFalse(ScrapyItemAdapter.is_item({})) self.assertEqual( get_field_meta_from_class(ScrapySubclassedItem, "name"), MappingProxyType({"serializer": str}), ) self.assertEqual( get_field_meta_from_class(ScrapySubclassedItem, "value"), MappingProxyType({"serializer": int}), ) itemadapter-0.11.0/tests/test_interface.py000066400000000000000000000174661474641140400206260ustar00rootroot00000000000000from __future__ import annotations import unittest from collections.abc import Iterator, KeysView from types import MappingProxyType from typing import Any from unittest import mock from itemadapter.adapter import AdapterInterface, ItemAdapter class AdapterInterfaceTest(unittest.TestCase): @mock.patch.multiple(AdapterInterface, __abstractmethods__=set()) def test_interface_class_methods(self): with self.assertRaises(NotImplementedError): AdapterInterface.is_item(object()) with self.assertRaises(NotImplementedError): AdapterInterface.is_item_class(object) class FakeItemClass: _fields = { "name": {"serializer": str}, "value": {"serializer": int}, } def __init__(self, **kwargs) -> None: self._values = {**kwargs} class BaseFakeItemAdapter(AdapterInterface): """An adapter that only implements the required methods.""" @classmethod def is_item_class(cls, item_class: type) -> bool: return issubclass(item_class, FakeItemClass) def __getitem__(self, field_name: str) -> Any: if field_name in self.item._fields: return self.item._values[field_name] raise KeyError(field_name) def __setitem__(self, field_name: str, value: Any) -> None: if field_name in self.item._fields: self.item._values[field_name] = value else: raise KeyError(field_name) def __delitem__(self, field_name: str) -> None: if field_name in self.item._fields and field_name in self.item._values: del self.item._values[field_name] else: raise KeyError(field_name) def __iter__(self) -> Iterator: return iter(self.item._values) def __len__(self) -> int: return len(self.item._values) class FieldNamesFakeItemAdapter(BaseFakeItemAdapter): """An adapter that also implements the field_names method.""" def field_names(self) -> KeysView: return KeysView({key.upper(): value for key, value in self.item._fields.items()}) class MetadataFakeItemAdapter(BaseFakeItemAdapter): """An adapter that also implements metadata-related methods.""" @classmethod def get_field_meta_from_class(cls, item_class: type, field_name: str) -> MappingProxyType: return MappingProxyType(item_class._fields.get(field_name) or {}) class BaseFakeItemAdapterTest(unittest.TestCase): item_class = FakeItemClass adapter_class = BaseFakeItemAdapter def setUp(self): ItemAdapter.ADAPTER_CLASSES.appendleft(self.adapter_class) def tearDown(self): ItemAdapter.ADAPTER_CLASSES.popleft() def test_repr(self): item = self.item_class() adapter = ItemAdapter(item) self.assertEqual(repr(adapter), "") adapter["name"] = "asdf" adapter["value"] = 1234 self.assertEqual(repr(adapter), "") def test_get_set_value(self): item = self.item_class() adapter = ItemAdapter(item) self.assertEqual(adapter.get("name"), None) self.assertEqual(adapter.get("value"), None) adapter["name"] = "asdf" adapter["value"] = 1234 self.assertEqual(adapter.get("name"), "asdf") self.assertEqual(adapter.get("value"), 1234) self.assertEqual(adapter["name"], "asdf") self.assertEqual(adapter["value"], 1234) def test_get_set_value_init(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual(adapter.get("name"), "asdf") self.assertEqual(adapter.get("value"), 1234) self.assertEqual(adapter["name"], "asdf") self.assertEqual(adapter["value"], 1234) def test_get_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with self.assertRaises(KeyError): adapter["_undefined_"] def test_as_dict(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual({"name": "asdf", "value": 1234}, dict(adapter)) def test_set_value_keyerror(self): item = self.item_class() adapter = ItemAdapter(item) with self.assertRaises(KeyError): adapter["_undefined_"] = "some value" def test_delitem_len_iter(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual(len(adapter), 2) self.assertEqual(sorted(list(iter(adapter))), ["name", "value"]) del adapter["name"] self.assertEqual(len(adapter), 1) self.assertEqual(sorted(list(iter(adapter))), ["value"]) del adapter["value"] self.assertEqual(len(adapter), 0) self.assertEqual(sorted(list(iter(adapter))), []) with self.assertRaises(KeyError): del adapter["name"] with self.assertRaises(KeyError): del adapter["value"] with self.assertRaises(KeyError): del adapter["_undefined_"] def test_get_value_keyerror_item_dict(self): """Instantiate without default values.""" adapter = ItemAdapter(self.item_class()) with self.assertRaises(KeyError): adapter["name"] def test_get_field_meta(self): """Metadata is always empty for the default implementation.""" adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("_undefined_"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({})) def test_get_field_meta_from_class(self): """Metadata is always empty for the default implementation.""" self.assertEqual( ItemAdapter.get_field_meta_from_class(self.item_class, "_undefined_"), MappingProxyType({}), ) self.assertEqual( ItemAdapter.get_field_meta_from_class(self.item_class, "name"), MappingProxyType({}) ) self.assertEqual( ItemAdapter.get_field_meta_from_class(self.item_class, "value"), MappingProxyType({}) ) def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertIsInstance(adapter.field_names(), KeysView) self.assertEqual(sorted(adapter.field_names()), ["name", "value"]) class MetadataFakeItemAdapterTest(BaseFakeItemAdapterTest): item_class = FakeItemClass adapter_class = MetadataFakeItemAdapter def test_get_field_meta(self): adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("_undefined_"), MappingProxyType({})) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({"serializer": str})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({"serializer": int})) def test_get_field_meta_from_class(self): self.assertEqual( ItemAdapter.get_field_meta_from_class(self.item_class, "_undefined_"), MappingProxyType({}), ) self.assertEqual( ItemAdapter.get_field_meta_from_class(self.item_class, "name"), MappingProxyType({"serializer": str}), ) self.assertEqual( ItemAdapter.get_field_meta_from_class(self.item_class, "value"), MappingProxyType({"serializer": int}), ) class FieldNamesFakeItemAdapterTest(BaseFakeItemAdapterTest): item_class = FakeItemClass adapter_class = FieldNamesFakeItemAdapter def test_field_names(self): item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertIsInstance(adapter.field_names(), KeysView) self.assertEqual(sorted(adapter.field_names()), ["NAME", "VALUE"]) itemadapter-0.11.0/tests/test_itemadapter.py000066400000000000000000000010121474641140400211410ustar00rootroot00000000000000import unittest from itemadapter.adapter import DictAdapter, ItemAdapter class DictOnlyItemAdapter(ItemAdapter): ADAPTER_CLASSES = [DictAdapter] class ItemAdapterTestCase(unittest.TestCase): def test_repr(self): adapter = ItemAdapter({"foo": "bar"}) self.assertEqual(repr(adapter), "") def test_repr_subclass(self): adapter = DictOnlyItemAdapter({"foo": "bar"}) self.assertEqual(repr(adapter), "") itemadapter-0.11.0/tests/test_utils.py000066400000000000000000000060301474641140400200070ustar00rootroot00000000000000import unittest from types import MappingProxyType from itemadapter import ItemAdapter from itemadapter.utils import get_field_meta_from_class, is_item from tests import ( AttrsItem, DataClassItem, PydanticV1Model, ScrapyItem, ScrapySubclassedItem, ) class FieldMetaFromClassTestCase(unittest.TestCase): def test_invalid_item_class(self): with self.assertRaises(TypeError, msg="1 is not a valid item class"): get_field_meta_from_class(1, "field") with self.assertRaises(TypeError, msg="list is not a valid item class"): get_field_meta_from_class(list, "field") def test_empty_meta_for_dict(self): class DictSubclass(dict): pass self.assertEqual(get_field_meta_from_class(DictSubclass, "name"), MappingProxyType({})) self.assertEqual(get_field_meta_from_class(dict, "name"), MappingProxyType({})) class ItemLikeTestCase(unittest.TestCase): def test_false(self): self.assertFalse(is_item(int)) self.assertFalse(is_item(sum)) self.assertFalse(is_item(1234)) self.assertFalse(is_item(object())) self.assertFalse(is_item("a string")) self.assertFalse(is_item(b"some bytes")) self.assertFalse(is_item(["a", "list"])) self.assertFalse(is_item(("a", "tuple"))) self.assertFalse(is_item({"a", "set"})) self.assertFalse(is_item(dict)) self.assertFalse(is_item(ScrapyItem)) self.assertFalse(is_item(DataClassItem)) self.assertFalse(is_item(ScrapySubclassedItem)) self.assertFalse(is_item(AttrsItem)) self.assertFalse(is_item(PydanticV1Model)) self.assertFalse(ItemAdapter.is_item_class(list)) self.assertFalse(ItemAdapter.is_item_class(int)) self.assertFalse(ItemAdapter.is_item_class(tuple)) def test_true_dict(self): self.assertTrue(is_item({"a": "dict"})) self.assertTrue(ItemAdapter.is_item_class(dict)) @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_true_scrapy(self): self.assertTrue(is_item(ScrapyItem())) self.assertTrue(is_item(ScrapySubclassedItem(name="asdf", value=1234))) self.assertTrue(ItemAdapter.is_item_class(ScrapyItem)) self.assertTrue(ItemAdapter.is_item_class(ScrapySubclassedItem)) @unittest.skipIf(not DataClassItem, "dataclasses module is not available") def test_true_dataclass(self): self.assertTrue(is_item(DataClassItem(name="asdf", value=1234))) self.assertTrue(ItemAdapter.is_item_class(DataClassItem)) @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_true_attrs(self): self.assertTrue(is_item(AttrsItem(name="asdf", value=1234))) self.assertTrue(ItemAdapter.is_item_class(AttrsItem)) @unittest.skipIf(not PydanticV1Model, "pydantic module is not available") def test_true_pydantic(self): self.assertTrue(is_item(PydanticV1Model(name="asdf", value=1234))) self.assertTrue(ItemAdapter.is_item_class(PydanticV1Model)) itemadapter-0.11.0/tox.ini000066400000000000000000000030551474641140400154130ustar00rootroot00000000000000[tox] envlist = min-attrs,min-pydantic,min-scrapy,min-extra,py39,py310,py311,py312,py313,attrs,pydantic1,pydantic,scrapy,extra,extra-pydantic1,pre-commit,typing,docs,twinecheck,pylint [testenv] basepython = min-attrs,min-pydantic,min-scrapy,min-extra: python3.9 deps = -rtests/requirements.txt min-attrs,min-extra: attrs==18.1.0 min-pydantic,min-extra: pydantic==1.8 min-scrapy,min-extra: scrapy==2.2 pydantic1,extra-pydantic1: pydantic<2 extras = min-attrs,attrs,min-extra,extra,extra-pydantic1: attrs min-pydantic,pydantic1,pydantic,min-extra,extra,extra-pydantic1: pydantic min-scrapy,scrapy,min-extra,extra,extra-pydantic1: scrapy commands = pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml {posargs: itemadapter tests} [testenv:docs] deps = {[testenv]deps} zyte-common-items extras = attrs pydantic scrapy commands = pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml --doctest-glob=README.md {posargs:README.md} [testenv:typing] basepython = python3 deps = mypy==1.14.1 attrs pydantic scrapy commands = mypy {posargs:itemadapter} [testenv:pylint] deps = pylint==3.3.4 commands = pylint {posargs:itemadapter tests} [testenv:twinecheck] basepython = python3 deps = twine==6.1.0 build==1.2.2.post1 commands = python -m build --sdist twine check dist/* [testenv:pre-commit] deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure skip_install = true