From bcc9d30167de8d229afa18dc27b8a1597d2e5836 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 18 Sep 2024 21:51:04 +0500 Subject: [PATCH] Add support for customAttributesOptions. --- scrapy_zyte_api/_annotations.py | 24 +++++++++++++++++++--- scrapy_zyte_api/providers.py | 26 +++++++----------------- tests/test_providers.py | 36 +++++++++++++++++---------------- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/scrapy_zyte_api/_annotations.py b/scrapy_zyte_api/_annotations.py index 20336b59..2bad453a 100644 --- a/scrapy_zyte_api/_annotations.py +++ b/scrapy_zyte_api/_annotations.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Iterable, List, Optional, TypedDict +from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Tuple, TypedDict class ExtractFrom(str, Enum): @@ -56,7 +56,7 @@ class _ActionResult(TypedDict, total=False): error: Optional[str] -def make_hashable(obj): +def make_hashable(obj: Any) -> Any: if isinstance(obj, (tuple, list)): return tuple((make_hashable(e) for e in obj)) @@ -66,7 +66,25 @@ def make_hashable(obj): return obj -def actions(value: Iterable[Action]): +def _from_hashable(obj: Any) -> Any: + if isinstance(obj, tuple): + return [_from_hashable(o) for o in obj] + + if isinstance(obj, frozenset): + return {_from_hashable(k): _from_hashable(v) for k, v in obj} + + return obj + + +def actions(value: Iterable[Action]) -> Tuple[Any, ...]: """Convert an iterable of :class:`~scrapy_zyte_api.Action` dicts into a hashable value.""" # both lists and dicts are not hashable and we need dep types to be hashable return tuple(make_hashable(action) for action in value) + + +def custom_attrs( + input: Dict[str, Any], options: Optional[Dict[str, Any]] = None +) -> Tuple[FrozenSet[Any], Optional[FrozenSet[Any]]]: + input_wrapped = make_hashable(input) + options_wrapped = make_hashable(options) if options else None + return input_wrapped, options_wrapped diff --git a/scrapy_zyte_api/providers.py b/scrapy_zyte_api/providers.py index 977a226b..23c37869 100644 --- a/scrapy_zyte_api/providers.py +++ b/scrapy_zyte_api/providers.py @@ -38,7 +38,7 @@ from zyte_common_items.fields import is_auto_field from scrapy_zyte_api import Actions, ExtractFrom, Geolocation, Screenshot -from scrapy_zyte_api._annotations import _ActionResult +from scrapy_zyte_api._annotations import _ActionResult, _from_hashable from scrapy_zyte_api.responses import ZyteAPITextResponse try: @@ -180,27 +180,15 @@ async def __call__( # noqa: C901 ) zyte_api_meta["actions"] = [] for action in cls.__metadata__[0]: # type: ignore[attr-defined] - zyte_api_meta["actions"].append( - { - k: ( - dict(v) - if isinstance(v, frozenset) - else list(v) if isinstance(v, tuple) else v - ) - for k, v in action - } - ) + zyte_api_meta["actions"].append(_from_hashable(action)) continue if cls_stripped in {CustomAttributes, CustomAttributesValues}: - zyte_api_meta["customAttributes"] = { - k: ( - dict(v) - if isinstance(v, frozenset) - else list(v) if isinstance(v, tuple) else v + custom_attrs_input, custom_attrs_options = cls.__metadata__[0] # type: ignore[attr-defined] + zyte_api_meta["customAttributes"] = _from_hashable(custom_attrs_input) + if custom_attrs_options: + zyte_api_meta["customAttributesOptions"] = _from_hashable( + custom_attrs_options ) - for k, v in cls.__metadata__[0] # type: ignore[attr-defined] - } - continue kw = _ITEM_KEYWORDS.get(cls_stripped) if not kw: diff --git a/tests/test_providers.py b/tests/test_providers.py index c3d851e0..10bef8cf 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -3,7 +3,7 @@ import pytest -from scrapy_zyte_api._annotations import make_hashable +from scrapy_zyte_api._annotations import custom_attrs pytest.importorskip("scrapy_poet") @@ -403,25 +403,32 @@ def parse_(self, response: DummyResponse, page: GeoProductPage): # type: ignore assert "Geolocation dependencies must be annotated" in caplog.text +custom_attrs_input = { + "attr1": {"type": "string", "description": "descr1"}, + "attr2": {"type": "number", "description": "descr2"}, +} + + @pytest.mark.skipif( sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9" ) +@pytest.mark.parametrize( + "annotation", + [ + custom_attrs(custom_attrs_input), + custom_attrs(custom_attrs_input, None), + custom_attrs(custom_attrs_input, {}), + custom_attrs(custom_attrs_input, {"foo": "bar"}), + ], +) @ensureDeferred -async def test_provider_custom_attrs(mockserver): +async def test_provider_custom_attrs(mockserver, annotation): from typing import Annotated @attrs.define class CustomAttrsPage(BasePage): product: Product - custom_attrs: Annotated[ - CustomAttributes, - make_hashable( - { - "attr1": {"type": "string", "description": "descr1"}, - "attr2": {"type": "number", "description": "descr2"}, - } - ), - ] + custom_attrs: Annotated[CustomAttributes, annotation] class CustomAttrsZyteAPISpider(ZyteAPISpider): def parse_(self, response: DummyResponse, page: CustomAttrsPage): # type: ignore[override] @@ -468,12 +475,7 @@ class CustomAttrsPage(BasePage): product: Product custom_attrs: Annotated[ CustomAttributesValues, - make_hashable( - { - "attr1": {"type": "string", "description": "descr1"}, - "attr2": {"type": "number", "description": "descr2"}, - } - ), + custom_attrs(custom_attrs_input), ] class CustomAttrsZyteAPISpider(ZyteAPISpider):