Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement the official gherkin parser #698

Open
wants to merge 25 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
1a2baff
First commit for using the official gherkin parser (trying to maintai…
jsa34 Sep 5, 2024
65c06e4
Improve docstrings in parser.py
jsa34 Sep 5, 2024
abe5e79
Improve docstrings in parser.py
jsa34 Sep 5, 2024
240ac6d
Fix issues and create a FeatureParser class to consolidate parsing logic
jsa34 Sep 5, 2024
e7b5326
Forgot to go back and implement the templated bool
jsa34 Sep 6, 2024
2f3e029
Remove unused import
jsa34 Sep 6, 2024
cc9b37f
Move Gherkin parsing to pydantic models for easier future reference o…
jsa34 Sep 6, 2024
4e17ccb
Move the calculating of given/when/then to pydantic models, as well a…
jsa34 Sep 6, 2024
57b9e55
Fix silly mistakes
jsa34 Sep 6, 2024
ff1a926
Fix type hints for py3.8
jsa34 Sep 6, 2024
21afdb1
Response to feedback
jsa34 Sep 8, 2024
fec8270
Another grammar fix
jsa34 Sep 8, 2024
6676692
Use dataclasses and not attr
jsa34 Sep 8, 2024
becfed2
Response to feedback
jsa34 Sep 12, 2024
2c8455b
Response to feedback
jsa34 Sep 12, 2024
c3008c1
Couple of tidy ups
jsa34 Sep 12, 2024
9c12dbf
Forgot to fix background in steps and revert test that was skipped
jsa34 Sep 12, 2024
93a11ae
Fix import (Python < 3.11 compat)
youtux Sep 14, 2024
fee0eb9
Remove default to None
youtux Sep 14, 2024
7cbfc47
Revert string literals to their original form
youtux Sep 15, 2024
2f3acbd
Response to feedback and make mypy happy.
jsa34 Sep 15, 2024
9b45269
Merge remote-tracking branch 'origin/gherkin-official-parser' into gh…
jsa34 Sep 15, 2024
a3a5195
Do not fail the CI job if we can't upload to Codecov.
youtux Sep 16, 2024
656c7ce
Ignore py3.13 failures for now
youtux Sep 16, 2024
d11096f
Fix matching result in case there are warnings
youtux Sep 16, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
508 changes: 184 additions & 324 deletions poetry.lock

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ pytest = ">=6.2.0"
typing-extensions = "*"
packaging = "*"
gherkin-official = "^29.0.0"
pydantic = "^2.9.0"

[tool.poetry.group.dev.dependencies]
tox = ">=4.11.3"
Expand Down
238 changes: 177 additions & 61 deletions src/pytest_bdd/gherkin_parser.py
Original file line number Diff line number Diff line change
@@ -1,56 +1,94 @@
import linecache
import textwrap
from pathlib import Path
from typing import List, Optional
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

from gherkin.errors import CompositeParserException
from gherkin.parser import Parser
from gherkin.token_scanner import TokenScanner
from pydantic import BaseModel, field_validator, model_validator

from . import exceptions
from .types import STEP_TYPES


class Location(BaseModel):
@dataclass
class Location:
column: int
line: int

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Location":
jsa34 marked this conversation as resolved.
Show resolved Hide resolved
return cls(column=data["column"], line=data["line"])

class Comment(BaseModel):

@dataclass
class Comment:
location: Location
text: str

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Comment":
return cls(location=Location.from_dict(data["location"]), text=data["text"])


class Cell(BaseModel):
@dataclass
class Cell:
location: Location
value: str

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Cell":
return cls(location=Location.from_dict(data["location"]), value=_convert_to_raw_string(data["value"]))


class Row(BaseModel):
@dataclass
class Row:
id: str
location: Location
cells: List[Cell]

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Row":
return cls(
id=data["id"],
location=Location.from_dict(data["location"]),
cells=[Cell.from_dict(cell) for cell in data["cells"]],
)

class DataTable(BaseModel):
name: Optional[str] = None

@dataclass
class DataTable:
location: Location
name: Optional[str] = None
tableHeader: Optional[Row] = None
tableBody: Optional[List[Row]] = None
tableBody: Optional[List[Row]] = field(default_factory=list)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DataTable":
return cls(
location=Location.from_dict(data["location"]),
name=data.get("name"),
tableHeader=Row.from_dict(data["tableHeader"]) if data.get("tableHeader") else None,
tableBody=[Row.from_dict(row) for row in data.get("tableBody", [])],
)

class DocString(BaseModel):

@dataclass
class DocString:
content: str
delimiter: str
location: Location

@field_validator("content", mode="before")
def dedent_content(cls, value: str) -> str:
return textwrap.dedent(value)
def __post_init__(self):
self.content = textwrap.dedent(self.content)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DocString":
return cls(content=data["content"], delimiter=data["delimiter"], location=Location.from_dict(data["location"]))


class Step(BaseModel):
@dataclass
class Step:
id: str
keyword: str
keywordType: str
Expand All @@ -59,43 +97,71 @@ class Step(BaseModel):
dataTable: Optional[DataTable] = None
docString: Optional[DocString] = None

@field_validator("keyword", mode="before")
def normalize_keyword(cls, value: str) -> str:
return value.lower().strip()
def __post_init__(self):
self.keyword = self.keyword.lower().strip()

@property
def given_when_then(self) -> str:
jsa34 marked this conversation as resolved.
Show resolved Hide resolved
return self._gwt
return getattr(self, "_gwt", "")

@given_when_then.setter
def given_when_then(self, gwt: str) -> None:
self._gwt = gwt


class Tag(BaseModel):
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Step":
return cls(
id=data["id"],
keyword=data["keyword"],
keywordType=data["keywordType"],
location=Location.from_dict(data["location"]),
text=data["text"],
dataTable=DataTable.from_dict(data["dataTable"]) if data.get("dataTable") else None,
docString=DocString.from_dict(data["docString"]) if data.get("docString") else None,
)


@dataclass
class Tag:
id: str
location: Location
name: str

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Tag":
return cls(id=data["id"], location=Location.from_dict(data["location"]), name=data["name"])

class Scenario(BaseModel):

@dataclass
class Scenario:
id: str
keyword: str
location: Location
name: str
description: str
steps: List[Step]
tags: List[Tag]
examples: Optional[List[DataTable]] = None

@model_validator(mode="after")
def process_steps(cls, instance):
steps = instance.steps
instance.steps = _compute_given_when_then(steps)
return instance


class Rule(BaseModel):
examples: Optional[List[DataTable]] = field(default_factory=list)

def __post_init__(self):
self.steps = _compute_given_when_then(self.steps)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Scenario":
return cls(
id=data["id"],
keyword=data["keyword"],
location=Location.from_dict(data["location"]),
name=data["name"],
description=data["description"],
steps=[Step.from_dict(step) for step in data["steps"]],
tags=[Tag.from_dict(tag) for tag in data["tags"]],
examples=[DataTable.from_dict(example) for example in data.get("examples", [])],
)


@dataclass
class Rule:
id: str
keyword: str
location: Location
Expand All @@ -104,41 +170,91 @@ class Rule(BaseModel):
tags: List[Tag]
children: List[Scenario]


class Background(BaseModel):
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Rule":
return cls(
id=data["id"],
keyword=data["keyword"],
location=Location.from_dict(data["location"]),
name=data["name"],
description=data["description"],
tags=[Tag.from_dict(tag) for tag in data["tags"]],
children=[Scenario.from_dict(child) for child in data["children"]],
)


@dataclass
class Background:
id: str
keyword: str
location: Location
name: str
description: str
steps: List[Step]

@model_validator(mode="after")
def process_steps(cls, instance):
steps = instance.steps
instance.steps = _compute_given_when_then(steps)
return instance
def __post_init__(self):
self.steps = _compute_given_when_then(self.steps)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Background":
return cls(
id=data["id"],
keyword=data["keyword"],
location=Location.from_dict(data["location"]),
name=data["name"],
description=data["description"],
steps=[Step.from_dict(step) for step in data["steps"]],
)

class Child(BaseModel):

@dataclass
class Child:
background: Optional[Background] = None
rule: Optional[Rule] = None
scenario: Optional[Scenario] = None

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Child":
return cls(
background=Background.from_dict(data["background"]) if data.get("background") else None,
rule=Rule.from_dict(data["rule"]) if data.get("rule") else None,
scenario=Scenario.from_dict(data["scenario"]) if data.get("scenario") else None,
)


class Feature(BaseModel):
@dataclass
class Feature:
keyword: str
location: Location
tags: List[Tag]
name: str
description: str
children: List[Child]

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Feature":
return cls(
keyword=data["keyword"],
location=Location.from_dict(data["location"]),
tags=[Tag.from_dict(tag) for tag in data["tags"]],
name=data["name"],
description=data["description"],
children=[Child.from_dict(child) for child in data["children"]],
)


class GherkinDocument(BaseModel):
@dataclass
class GherkinDocument:
feature: Feature
comments: List[Comment]

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "GherkinDocument":
return cls(
feature=Feature.from_dict(data["feature"]),
comments=[Comment.from_dict(comment) for comment in data["comments"]],
)


def _compute_given_when_then(steps: List[Step]) -> List[Step]:
last_gwt = None
Expand All @@ -149,22 +265,22 @@ def _compute_given_when_then(steps: List[Step]) -> List[Step]:
return steps


class GherkinParser:
def __init__(self, abs_filename: str = None, encoding: str = "utf-8"):
self.abs_filename = Path(abs_filename)
self.encoding = encoding

with open(self.abs_filename, encoding=self.encoding) as f:
self.feature_file_text = f.read()
try:
self.gherkin_data = Parser().parse(TokenScanner(self.feature_file_text))
except CompositeParserException as e:
raise exceptions.FeatureError(
e.args[0],
e.errors[0].location["line"],
linecache.getline(str(self.abs_filename), e.errors[0].location["line"]).rstrip("\n"),
self.abs_filename,
) from e

def to_gherkin_document(self) -> GherkinDocument:
return GherkinDocument(**self.gherkin_data)
def _convert_to_raw_string(normal_string: str) -> str:
return normal_string.replace("\\", "\\\\")


def get_gherkin_document(abs_filename: str = None, encoding: str = "utf-8") -> GherkinDocument:
with open(abs_filename, encoding=encoding) as f:
feature_file_text = f.read()

try:
gherkin_data = Parser().parse(TokenScanner(feature_file_text))
jsa34 marked this conversation as resolved.
Show resolved Hide resolved
except CompositeParserException as e:
raise exceptions.FeatureError(
e.args[0],
e.errors[0].location["line"],
linecache.getline(abs_filename, e.errors[0].location["line"]).rstrip("\n"),
abs_filename,
) from e

return GherkinDocument.from_dict(gherkin_data)
5 changes: 3 additions & 2 deletions src/pytest_bdd/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@

from .gherkin_parser import Background as GherkinBackground
from .gherkin_parser import Feature as GherkinFeature
from .gherkin_parser import GherkinDocument, GherkinParser
from .gherkin_parser import GherkinDocument
from .gherkin_parser import Scenario as GherkinScenario
from .gherkin_parser import Step as GherkinStep
from .gherkin_parser import Tag as GherkinTag
from .gherkin_parser import get_gherkin_document
from .types import GIVEN, THEN, WHEN

STEP_PARAM_RE = re.compile(r"<(.+?)>")
Expand Down Expand Up @@ -422,7 +423,7 @@ def _parse_feature_file(self) -> GherkinDocument:
Returns:
Dict: A Gherkin document representation of the feature file.
"""
return GherkinParser(self.abs_filename, self.encoding).to_gherkin_document()
return get_gherkin_document(self.abs_filename, self.encoding)

def parse(self):
gherkin_doc: GherkinDocument = self._parse_feature_file()
Expand Down
2 changes: 1 addition & 1 deletion src/pytest_bdd/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ def _scenario() -> None:

for test_name in get_python_name_generator(scenario_name):
if test_name not in caller_locals:
# found an unique test name
# found a unique test name
caller_locals[test_name] = _scenario
break
found = True
Expand Down
2 changes: 1 addition & 1 deletion src/pytest_bdd/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def step(
:return: Decorator function for the step.

Example:
>>> @step("there is an wallet", target_fixture="wallet")
>>> @step("there is a wallet", target_fixture="wallet")
>>> def _() -> dict[str, int]:
>>> return {"eur": 0, "usd": 0}

Expand Down
4 changes: 2 additions & 2 deletions tests/feature/test_outline.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,6 @@ def _(string):
r"bork |",
r"bork||bork",
r"|",
"bork \\",
"bork \\|",
r"bork \\",
r"bork \\|",
]
Loading
Loading