From 65d9a566aefdb696f95cc8b1f3130b00093a2cc7 Mon Sep 17 00:00:00 2001 From: Johannes Kanig Date: Wed, 8 Dec 2021 18:16:41 +0900 Subject: [PATCH] Add commandline option for location of rfi files - command line option for rfi files - new argument of Parser and Integration for rfi files location - move all rfi file handling to Integration class - move testing accordingly - new test for different location of rfi file For #713 --- rflx/cli.py | 18 +++++++-- rflx/integration.py | 44 +++++++++++++++++----- rflx/specification/parser.py | 31 ++++------------ tests/unit/integration_test.py | 49 ++++++++++++++++++++++++- tests/unit/specification/parser_test.py | 33 ----------------- 5 files changed, 103 insertions(+), 72 deletions(-) diff --git a/rflx/cli.py b/rflx/cli.py index e44343440..75e87eb49 100644 --- a/rflx/cli.py +++ b/rflx/cli.py @@ -6,7 +6,7 @@ from collections import defaultdict from multiprocessing import cpu_count from pathlib import Path -from typing import Dict, List, Sequence, Tuple, Union +from typing import Dict, List, Optional, Sequence, Tuple, Union import librflxlang from pkg_resources import get_distribution @@ -85,6 +85,9 @@ def main(argv: List[str]) -> Union[int, str]: help="ignore checksum aspects during code generation", action="store_true", ) + parser_generate.add_argument( + "--integration-files-dir", help="directory for the .rfi files", type=Path + ) parser_generate.add_argument( "files", metavar="SPECIFICATION_FILE", type=Path, nargs="*", help="specification file" ) @@ -249,7 +252,9 @@ def generate(args: argparse.Namespace) -> None: if not args.output_directory.is_dir(): fail(f'directory not found: "{args.output_directory}"', Subsystem.CLI) - model, integration = parse(args.files, args.no_verification, args.workers) + model, integration = parse( + args.files, args.no_verification, args.workers, args.integration_files_dir + ) generator = Generator( model, @@ -267,9 +272,14 @@ def generate(args: argparse.Namespace) -> None: def parse( - files: Sequence[Path], skip_verification: bool = False, workers: int = 1 + files: Sequence[Path], + skip_verification: bool = False, + workers: int = 1, + integration_files_dir: Optional[Path] = None, ) -> Tuple[Model, Integration]: - parser = Parser(skip_verification, cached=True, workers=workers) + parser = Parser( + skip_verification, cached=True, workers=workers, integration_files_dir=integration_files_dir + ) error = RecordFluxError() present_files = [] diff --git a/rflx/integration.py b/rflx/integration.py index bf07d2eb9..d572d522f 100644 --- a/rflx/integration.py +++ b/rflx/integration.py @@ -3,6 +3,8 @@ from pydantic import BaseModel, Extra, Field, ValidationError from pydantic.types import ConstrainedInt +from ruamel.yaml.error import MarkedYAMLError +from ruamel.yaml.main import YAML from rflx.error import Location, RecordFluxError, Severity, Subsystem from rflx.identifier import ID @@ -33,16 +35,32 @@ class Integration: def defaultsize(self) -> int: return 4096 - def __init__(self) -> None: + def __init__(self, integration_files_dir: Optional[Path] = None) -> None: self._packages: Dict[str, IntegrationFile] = {} - - def add_integration_file(self, filename: Path, file: object, error: RecordFluxError) -> None: - try: - self._packages[filename.stem] = IntegrationFile.parse_obj(file) - except ValidationError as e: - error.extend( - [(f"{e}", Subsystem.PARSER, Severity.ERROR, self._to_location(filename.stem))] - ) + self._integration_files_dir = integration_files_dir + + def load_integration_file(self, spec_file: Path, error: RecordFluxError) -> None: + integration_file = ( + spec_file.with_suffix(".rfi") + if self._integration_files_dir is None + else self._integration_files_dir / (spec_file.stem + ".rfi") + ) + if integration_file.exists(): + yaml = YAML() + try: + content = yaml.load(integration_file) + except MarkedYAMLError as e: + location = Location( + start=( + (0, 0) + if e.problem_mark is None + else (e.problem_mark.line + 1, e.problem_mark.column + 1) + ), + source=integration_file, + ) + error.extend([(str(e), Subsystem.PARSER, Severity.ERROR, location)]) + return + self._add_integration_object(integration_file, content, error) def validate(self, model: Model, error: RecordFluxError) -> None: for package, integration_file in self._packages.items(): @@ -69,6 +87,14 @@ def validate(self, model: Model, error: RecordFluxError) -> None: self._validate_globals(package, integration, session, error) self._validate_states(package, integration, session, error) + def _add_integration_object(self, filename: Path, file: object, error: RecordFluxError) -> None: + try: + self._packages[filename.stem] = IntegrationFile.parse_obj(file) + except ValidationError as e: + error.extend( + [(f"{e}", Subsystem.PARSER, Severity.ERROR, self._to_location(filename.stem))] + ) + def get_size(self, session: ID, variable: ID, state: Optional[ID]) -> int: """ Return the requested buffer size for a variable of a given session and state. diff --git a/rflx/specification/parser.py b/rflx/specification/parser.py index e013eea8b..1619ba79a 100644 --- a/rflx/specification/parser.py +++ b/rflx/specification/parser.py @@ -8,8 +8,6 @@ from typing import Callable, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type, Union import librflxlang as lang -from ruamel.yaml.error import MarkedYAMLError -from ruamel.yaml.main import YAML from rflx import expression as expr, model from rflx.error import Location, RecordFluxError, Severity, Subsystem, fail, warn @@ -1424,7 +1422,11 @@ class SpecificationNode: class Parser: def __init__( - self, skip_verification: bool = False, cached: bool = False, workers: int = 1 + self, + skip_verification: bool = False, + cached: bool = False, + workers: int = 1, + integration_files_dir: Optional[Path] = None, ) -> None: if skip_verification: warn("model verification skipped", Subsystem.MODEL) @@ -1436,7 +1438,7 @@ def __init__( *model.INTERNAL_TYPES.values(), ] self.__sessions: List[model.Session] = [] - self.__integration: Integration = Integration() + self.__integration: Integration = Integration(integration_files_dir) self.__cache = Cache(not skip_verification and cached) def __convert_unit( @@ -1514,8 +1516,7 @@ def __parse_specfile(self, filename: Path, transitions: List[ID] = None) -> Reco unit = lang.AnalysisContext().get_from_file(str(filename)) if diagnostics_to_error(unit.diagnostics, error, filename): return error - integration_file = filename.with_suffix(".rfi") - self._load_integration_file(integration_file, error) + self.__integration.load_integration_file(filename, error) if unit.root: assert isinstance(unit.root, lang.Specification) self.__convert_unit(error, unit.root, filename, transitions) @@ -1582,24 +1583,6 @@ def create_model(self) -> model.Model: def get_integration(self) -> Integration: return self.__integration - def _load_integration_file(self, integration_file: Path, error: RecordFluxError) -> None: - if integration_file.exists(): - yaml = YAML() - try: - content = yaml.load(integration_file) - except MarkedYAMLError as e: - location = Location( - start=( - (0, 0) - if e.problem_mark is None - else (e.problem_mark.line + 1, e.problem_mark.column + 1) - ), - source=integration_file, - ) - error.extend([(str(e), Subsystem.PARSER, Severity.ERROR, location)]) - return - self.__integration.add_integration_file(integration_file, content, error) - @property def specifications(self) -> Dict[str, lang.Specification]: return { diff --git a/tests/unit/integration_test.py b/tests/unit/integration_test.py index a6e9f16aa..e6c1c66b9 100644 --- a/tests/unit/integration_test.py +++ b/tests/unit/integration_test.py @@ -1,5 +1,6 @@ import re from pathlib import Path +from typing import Sequence import pytest from ruamel.yaml.main import YAML @@ -110,7 +111,8 @@ def test_rfi_add_integration(rfi_content: str, match_error: str) -> None: error = RecordFluxError() integration = Integration() with pytest.raises(RecordFluxError, match=regex): - integration.add_integration_file(Path("test.rfi"), content, error) + # pylint: disable = protected-access + integration._add_integration_object(Path("test.rfi"), content, error) error.propagate() @@ -124,7 +126,8 @@ def test_rfi_get_size() -> None: } } error = RecordFluxError() - integration.add_integration_file(Path("p.rfi"), session_object, error) + # pylint: disable = protected-access + integration._add_integration_object(Path("p.rfi"), session_object, error) error.propagate() assert integration.get_size(ID("P::S"), ID("x"), ID("S")) == 1024 assert integration.get_size(ID("P::S"), ID("x"), ID("S")) == 1024 @@ -132,3 +135,45 @@ def test_rfi_get_size() -> None: assert integration.get_size(ID("P::S2"), ID("x"), None) == 4096 assert integration.get_size(ID("P::S"), ID("y"), None) == 2048 assert integration.get_size(ID("P::S"), ID("y"), ID("S")) == 8192 + + +@pytest.mark.parametrize( + "content, error_msg, line, column", + [ + ('"', ["while scanning a quoted scalar", "unexpected end of stream"], 1, 2), + ("Session: 1, Session : 1", ["mapping values are not allowed here"], 1, 21), + ( + "Session: 1\nSession : 1", + ["while constructing a mapping", 'found duplicate key "Session" with value "1"'], + 2, + 1, + ), + ], +) +def test_load_integration_file( + tmp_path: Path, content: str, error_msg: Sequence[str], line: int, column: int +) -> None: + test_rfi = tmp_path / "test.rfi" + test_rfi.write_text(content) + integration = Integration() + error = RecordFluxError() + regex = fr"^{test_rfi}:{line}:{column}: parser: error: " + for elt in error_msg: + regex += elt + regex += fr'.*in "{test_rfi}", line [0-9]+, column [0-9]+.*' + regex += "$" + compiled_regex = re.compile(regex, re.DOTALL) + with pytest.raises(RecordFluxError, match=compiled_regex): + integration.load_integration_file(test_rfi, error) + error.propagate() + + +def test_load_integration_path(tmp_path: Path) -> None: + subfolder = tmp_path / "sub" + subfolder.mkdir() + test_rfi = subfolder / "test.rfi" + test_rfi.write_text("{ Session: { Session : { Buffer_Size : {} }}}") + integration = Integration(integration_files_dir=subfolder) + error = RecordFluxError() + integration.load_integration_file(tmp_path / "test.rflx", error) + error.propagate() diff --git a/tests/unit/specification/parser_test.py b/tests/unit/specification/parser_test.py index 54c81c9a6..ded56ae05 100644 --- a/tests/unit/specification/parser_test.py +++ b/tests/unit/specification/parser_test.py @@ -1,6 +1,5 @@ # pylint: disable=too-many-lines -import re from itertools import zip_longest from pathlib import Path from typing import Any, Dict, Sequence @@ -2803,35 +2802,3 @@ def test_parse_reserved_word_as_channel_name() -> None: end Test; """ ) - - -@pytest.mark.parametrize( - "content, error_msg, line, column", - [ - ('"', ["while scanning a quoted scalar", "unexpected end of stream"], 1, 2), - ("Session: 1, Session : 1", ["mapping values are not allowed here"], 1, 21), - ( - "Session: 1\nSession : 1", - ["while constructing a mapping", 'found duplicate key "Session" with value "1"'], - 2, - 1, - ), - ], -) -def test_load_integration_file( - tmp_path: Path, content: str, error_msg: Sequence[str], line: int, column: int -) -> None: - test_rfi = tmp_path / "test.rfi" - test_rfi.write_text(content) - p = parser.Parser() - error = RecordFluxError() - regex = fr"^{test_rfi}:{line}:{column}: parser: error: " - for elt in error_msg: - regex += elt - regex += fr'.*in "{test_rfi}", line [0-9]+, column [0-9]+.*' - regex += "$" - compiled_regex = re.compile(regex, re.DOTALL) - with pytest.raises(RecordFluxError, match=compiled_regex): - # pylint: disable = protected-access - p._load_integration_file(test_rfi, error) - error.propagate()