From 71f2d46e62350dd4a67fbec91a7b57615fb2a959 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 4 Feb 2022 13:36:53 +0100 Subject: [PATCH 1/3] Update to the new plugins system in core --- README.md | 2 +- {{ cookiecutter.project_slug }}/setup.cfg | 24 +++++++++---------- .../strategies/download.py | 6 ++--- .../strategies/filter.py | 2 -- .../strategies/mapping.py | 3 --- .../strategies/parse.py | 11 ++++----- .../strategies/resource.py | 5 ++-- .../strategies/transformation.py | 4 +--- 8 files changed, 23 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index dbb7544..e23e9b7 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Furthermore, the repository will contain **dummy/demo** strategies for each of t - Mapping (`mappingType`) - Parse (`mediaType`) - Resource (`accessService`) -- Transformation (`transformation_type`) +- Transformation (`transformationType`) These should be **updated** to **your specific needs** according to the strategy or strategies you intend to implement. The ones that are not used should be deleted. diff --git a/{{ cookiecutter.project_slug }}/setup.cfg b/{{ cookiecutter.project_slug }}/setup.cfg index e61cafd..78fa7e6 100644 --- a/{{ cookiecutter.project_slug }}/setup.cfg +++ b/{{ cookiecutter.project_slug }}/setup.cfg @@ -1,13 +1,13 @@ [options.entry_points] -oteapi.download_strategy = - {{ cookiecutter.package_name }}.demo = {{ cookiecutter.package_name }}.strategies.download -oteapi.filter_strategy = - {{ cookiecutter.package_name }}.demo = {{ cookiecutter.package_name }}.strategies.filter -oteapi.mapping_strategy = - {{ cookiecutter.package_name }}.demo = {{ cookiecutter.package_name }}.strategies.mapping -oteapi.resource_strategy = - {{ cookiecutter.package_name }}.demo = {{ cookiecutter.package_name }}.strategies.resource -oteapi.transformation_strategy = - {{ cookiecutter.package_name }}.demo = {{ cookiecutter.package_name }}.strategies.transformation -oteapi.parse_strategy = - {{ cookiecutter.package_name }}.demo = {{ cookiecutter.package_name }}.strategies.parse +oteapi.download = + {{ cookiecutter.package_name }}.fileDEMO = {{ cookiecutter.package_name }}.strategies.download:DemoFileStrategy +oteapi.filter = + {{ cookiecutter.package_name }}.filter/DEMO = {{ cookiecutter.package_name }}.strategies.filter:DemoFilter +oteapi.mapping = + {{ cookiecutter.package_name }}.mapping/DEMO = {{ cookiecutter.package_name }}.strategies.mapping:DemoMappingStrategy +oteapi.parse = + {{ cookiecutter.package_name }}.text/jsonDEMO = {{ cookiecutter.package_name }}.strategies.parse:DemoJSONDataParseStrategy +oteapi.resource = + {{ cookiecutter.package_name }}.DEMO-access-service = {{ cookiecutter.package_name }}.strategies.resource:DemoResourceStrategy +oteapi.transformation = + {{ cookiecutter.package_name }}.script/DEMO = {{ cookiecutter.package_name }}.strategies.transformation:DummyTransformationStrategy diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py index 8562f8a..b3ae24d 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py @@ -4,8 +4,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional -from oteapi.datacache.datacache import DataCache -from oteapi.plugins.factories import StrategyFactory +from oteapi.datacache import DataCache from pydantic import BaseModel, Extra, Field if TYPE_CHECKING: @@ -33,11 +32,10 @@ class FileConfig(BaseModel): @dataclass -@StrategyFactory.register(("scheme", "fileDEMO")) class DemoFileStrategy: """Strategy for retrieving data via local file.""" - resource_config: "ResourceConfig" + download_config: "ResourceConfig" def initialize( self, session: "Optional[Dict[str, Any]]" = None diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/filter.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/filter.py index d15c476..1e031c1 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/filter.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/filter.py @@ -3,7 +3,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, List -from oteapi.plugins.factories import StrategyFactory from pydantic import BaseModel, Field if TYPE_CHECKING: @@ -19,7 +18,6 @@ class DemoDataModel(BaseModel): @dataclass -@StrategyFactory.register(("filterType", "filter/DEMO")) class DemoFilter: """Filter Strategy.""" diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/mapping.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/mapping.py index 70e94ab..53c68f8 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/mapping.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/mapping.py @@ -3,8 +3,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from oteapi.plugins.factories import StrategyFactory - if TYPE_CHECKING: from typing import Any, Dict, Optional @@ -12,7 +10,6 @@ @dataclass -@StrategyFactory.register(("mappingType", "mapping/DEMO")) class DemoMappingStrategy: """Mapping Strategy.""" diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py index d35ca26..d477eaa 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py @@ -4,8 +4,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from oteapi.datacache.datacache import DataCache -from oteapi.plugins.factories import StrategyFactory, create_download_strategy +from oteapi.datacache import DataCache +from oteapi.plugins import create_strategy if TYPE_CHECKING: from typing import Any, Dict, Optional @@ -14,11 +14,10 @@ @dataclass -@StrategyFactory.register(("mediaType", "text/jsonDEMO")) class DemoJSONDataParseStrategy: """Parse Strategy.""" - resource_config: "ResourceConfig" + parse_config: "ResourceConfig" def initialize( self, session: "Optional[Dict[str, Any]]" = None @@ -38,7 +37,7 @@ def initialize( """ return {} - def parse(self, session: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]": + def get(self, session: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]": """Execute the strategy. This method will be called through the strategy-specific endpoint of the @@ -52,7 +51,7 @@ def parse(self, session: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]": dictionary context. """ - downloader = create_download_strategy(self.resource_config) + downloader = create_strategy("download", self.resource_config) output = downloader.get() cache = DataCache(self.resource_config.configuration) content = cache.get(output["key"]) diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/resource.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/resource.py index 635d7b0..58a3aec 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/resource.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/resource.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from oteapi.plugins.factories import StrategyFactory, create_download_strategy +from oteapi.plugins import create_strategy if TYPE_CHECKING: from typing import Any, Dict, Optional @@ -12,7 +12,6 @@ @dataclass -@StrategyFactory.register(("accessService", "DEMO-access-service")) class DemoResourceStrategy: """Resource Strategy.""" @@ -51,6 +50,6 @@ def get(self, session: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]": """ # Example of the plugin using the download strategy to fetch the data - download_strategy = create_download_strategy(self.resource_config) + download_strategy = create_strategy("download", self.resource_config) read_output = download_strategy.get(session) return {"output": read_output} diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/transformation.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/transformation.py index af3f388..2311c53 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/transformation.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/transformation.py @@ -4,8 +4,7 @@ from datetime import datetime from typing import TYPE_CHECKING -from oteapi.models.transformationconfig import TransformationStatus -from oteapi.plugins.factories import StrategyFactory +from oteapi.models import TransformationStatus if TYPE_CHECKING: from typing import Any, Dict, Optional @@ -14,7 +13,6 @@ @dataclass -@StrategyFactory.register(("transformation_type", "script/DEMO")) class DummyTransformationStrategy: """Transformation Strategy.""" From 26407ef1bb0ea21fc46c2fd392a99d19eb7b1203 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 4 Feb 2022 13:47:52 +0100 Subject: [PATCH 2/3] Fix test in generated project Use the new oteapi.plugins API for loading plugin strategies and update to use `oteapi-core` v0.0.5. --- {{ cookiecutter.project_slug }}/requirements.txt | 2 +- {{ cookiecutter.project_slug }}/tests/conftest.py | 6 +++--- {{ cookiecutter.project_slug }}/tests/test_parse.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/{{ cookiecutter.project_slug }}/requirements.txt b/{{ cookiecutter.project_slug }}/requirements.txt index 658abb6..b741552 100644 --- a/{{ cookiecutter.project_slug }}/requirements.txt +++ b/{{ cookiecutter.project_slug }}/requirements.txt @@ -1 +1 @@ -oteapi-core==0.0.4 +oteapi-core==0.0.5 diff --git a/{{ cookiecutter.project_slug }}/tests/conftest.py b/{{ cookiecutter.project_slug }}/tests/conftest.py index dac72c2..fa4aec1 100644 --- a/{{ cookiecutter.project_slug }}/tests/conftest.py +++ b/{{ cookiecutter.project_slug }}/tests/conftest.py @@ -4,7 +4,7 @@ @pytest.fixture(scope="session", autouse=True) def load_plugins() -> None: - """Load plugins.""" - from oteapi.plugins.plugins import load_plugins + """Load pip installed plugin strategies.""" + from oteapi.plugins.factories import load_strategies - load_plugins() + load_strategies() diff --git a/{{ cookiecutter.project_slug }}/tests/test_parse.py b/{{ cookiecutter.project_slug }}/tests/test_parse.py index e1c39c1..3ce853f 100644 --- a/{{ cookiecutter.project_slug }}/tests/test_parse.py +++ b/{{ cookiecutter.project_slug }}/tests/test_parse.py @@ -21,6 +21,6 @@ def test_json(): mediaType="text/jsonDEMO", ) parser = DemoJSONDataParseStrategy(config) - json = parser.parse() + json = parser.get() assert json == data From 37c64361e38f204bfed8e227cf4848730dffda52 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 4 Feb 2022 15:10:30 +0100 Subject: [PATCH 3/3] Use correct config attribute name Co-authored-by: Kristine Wiik --- .../strategies/download.py | 10 +++++----- .../strategies/parse.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py index b3ae24d..9b0874e 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/download.py @@ -70,20 +70,20 @@ def get(self, session: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]": """ if ( - self.resource_config.downloadUrl is None - or self.resource_config.downloadUrl.scheme != "file" + self.download_config.downloadUrl is None + or self.download_config.downloadUrl.scheme != "file" ): raise ValueError( "Expected 'downloadUrl' to have scheme 'file' in the configuration." ) - filename = Path(self.resource_config.downloadUrl.host).resolve() + filename = Path(self.download_config.downloadUrl.host).resolve() - cache = DataCache(self.resource_config.configuration) + cache = DataCache(self.download_config.configuration) if cache.config.accessKey and cache.config.accessKey in cache: key = cache.config.accessKey else: config = FileConfig( - **self.resource_config.configuration, extra=Extra.ignore + **self.download_config.configuration, extra=Extra.ignore ) key = cache.add( filename.read_text(encoding=config.encoding) diff --git a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py index d477eaa..d2eb688 100644 --- a/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py +++ b/{{ cookiecutter.project_slug }}/{{ cookiecutter.package_name }}/strategies/parse.py @@ -51,9 +51,9 @@ def get(self, session: "Optional[Dict[str, Any]]" = None) -> "Dict[str, Any]": dictionary context. """ - downloader = create_strategy("download", self.resource_config) + downloader = create_strategy("download", self.parse_config) output = downloader.get() - cache = DataCache(self.resource_config.configuration) + cache = DataCache(self.parse_config.configuration) content = cache.get(output["key"]) if isinstance(content, dict):