chore[ci]: enable python3.12 tests (#3860)

add a python3.12 job in the CI.

also:
- update dependencies
- update tests for new dependencies
- refactor the grammar tests to use less filtering
- python3.12 deprecates the "n" field on constants; remove it

.github/workflows/test.yml

@@ -26,6 +26,9 @@ jobs:
     - name: Install Dependencies
       run: pip install .[lint]
 
+    - name: Debug dependencies
+      run: pip freeze
+
     - name: Run Black
       run: black --check -C --force-exclude=vyper/version.py ./vyper ./tests ./setup.py
 
@@ -93,9 +96,10 @@ jobs:
             opt-mode: gas
             debug: false
             evm-version: shanghai
-
-          # TODO 3.12 doesn't work yet, investigate - may be hypothesis issue
-          #- python-version: ["3.12", "312"]
+          - python-version: ["3.12", "312"]
+            opt-mode: gas
+            debug: false
+            evm-version: shanghai
 
     name: py${{ matrix.python-version[1] }}-opt-${{ matrix.opt-mode }}${{ matrix.debug && '-debug' || '' }}-${{ matrix.evm-version }}
 
@@ -114,6 +118,9 @@ jobs:
     - name: Install dependencies
       run: pip install .[test]
 
+    - name: Debug dependencies
+      run: pip freeze
+
     - name: Run tests
       run: |
         pytest \

setup.py

@@ -8,7 +8,7 @@
 
 extras_require = {
     "test": [
-        "pytest>=6.2.5,<7.0",
+        "pytest>=8.0,<9.0",
         "pytest-cov>=2.10,<3.0",
         "pytest-instafail>=0.4,<1.0",
         "pytest-xdist>=2.5,<3.0",
@@ -19,8 +19,9 @@
         "web3==6.0.0",
         "tox>=3.15,<4.0",
         "lark==1.1.9",
-        "hypothesis[lark]>=5.37.1,<6.0",
-        "eth-stdlib==0.2.6",
+        "hypothesis[lark]>=6.0,<7.0",
+        "eth-stdlib==0.2.7",
+        "setuptools",
     ],
     "lint": [
         "black==23.12.0",
@@ -115,6 +116,7 @@ def _global_version(version):
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     package_data={"vyper.ast": ["grammar.lark"]},
     data_files=[("", [hash_file_rel_path])],

tests/functional/builtins/codegen/test_slice.py

@@ -2,7 +2,8 @@
 import pytest
 from hypothesis import given, settings
 
-from vyper.compiler.settings import OptimizationLevel
+from vyper.compiler import compile_code
+from vyper.compiler.settings import OptimizationLevel, Settings
 from vyper.exceptions import ArgumentException, TypeMismatch
 
 _fun_bytes32_bounds = [(0, 32), (3, 29), (27, 5), (0, 5), (5, 3), (30, 2)]
@@ -32,6 +33,12 @@ def slice_tower_test(inp1: Bytes[50]) -> Bytes[50]:
 _bytes_1024 = st.binary(min_size=0, max_size=1024)
 
 
+def _fail_contract(code, opt_level, exceptions):
+    settings = Settings(optimize=opt_level)
+    with pytest.raises(exceptions):
+        compile_code(code, settings)
+
+
 @pytest.mark.parametrize("use_literal_start", (True, False))
 @pytest.mark.parametrize("use_literal_length", (True, False))
 @pytest.mark.parametrize("opt_level", list(OptimizationLevel))
@@ -40,7 +47,6 @@ def slice_tower_test(inp1: Bytes[50]) -> Bytes[50]:
 @pytest.mark.fuzzing
 def test_slice_immutable(
     get_contract,
-    assert_compile_failed,
     tx_failed,
     opt_level,
     bytesdata,
@@ -76,7 +82,8 @@ def _get_contract():
         or (use_literal_start and start > length_bound)
         or (use_literal_length and length == 0)
     ):
-        assert_compile_failed(lambda: _get_contract(), ArgumentException)
+        _fail_contract(code, opt_level, ArgumentException)
+
     elif start + length > len(bytesdata) or (len(bytesdata) > length_bound):
         # deploy fail
         with tx_failed():
@@ -95,7 +102,6 @@ def _get_contract():
 @pytest.mark.fuzzing
 def test_slice_bytes_fuzz(
     get_contract,
-    assert_compile_failed,
     tx_failed,
     opt_level,
     location,
@@ -173,7 +179,8 @@ def _get_contract():
     )
 
     if compile_time_oob or slice_output_too_large:
-        assert_compile_failed(lambda: _get_contract(), (ArgumentException, TypeMismatch))
+        _fail_contract(code, opt_level, (ArgumentException, TypeMismatch))
+
     elif location == "code" and len(bytesdata) > length_bound:
         # deploy fail
         with tx_failed():

tests/functional/builtins/folding/test_powmod.py

@@ -4,7 +4,7 @@
 
 from tests.utils import parse_and_fold
 
-st_uint256 = st.integers(min_value=0, max_value=2**256)
+st_uint256 = st.integers(min_value=0, max_value=(2**256 - 1))
 
 
 @pytest.mark.fuzzing

tests/functional/codegen/types/test_bytes_zero_padding.py

@@ -25,7 +25,7 @@ def get_count(counter: uint256) -> Bytes[24]:
 
 
 @pytest.mark.fuzzing
-@hypothesis.given(value=hypothesis.strategies.integers(min_value=0, max_value=2**64))
+@hypothesis.given(value=hypothesis.strategies.integers(min_value=0, max_value=(2**64 - 1)))
 def test_zero_pad_range(little_endian_contract, value):
     actual_bytes = value.to_bytes(8, byteorder="little")
     contract_bytes = little_endian_contract.get_count(value)

tests/functional/grammar/test_grammar.py

@@ -37,36 +37,31 @@ def test_basic_grammar_empty():
     assert len(tree.children) == 0
 
 
-def utf8_encodable(terminal: str) -> bool:
-    try:
-        if "\x00" not in terminal and "\\ " not in terminal and "\x0c" not in terminal:
-            terminal.encode("utf-8-sig")
-            return True
-        else:
-            return False
-    except UnicodeEncodeError:  # pragma: no cover
-        # Very rarely, a "." in some terminal regex will generate a surrogate
-        # character that cannot be encoded as UTF-8.  We apply this filter to
-        # ensure it doesn't happen at runtime, but don't worry about coverage.
-        return False
+def fix_terminal(terminal: str) -> bool:
+    # these throw exceptions in the grammar
+    for bad in ("\x00", "\\ ", "\x0c"):
+        terminal = terminal.replace(bad, " ")
+    return terminal
+
+
+ALLOWED_CHARS = st.characters(codec="utf-8", min_codepoint=1)
 
 
 # With help from hyposmith
 # https://github.com/Zac-HD/hypothesmith/blob/master/src/hypothesmith/syntactic.py
 class GrammarStrategy(LarkStrategy):
     def __init__(self, grammar, start, explicit_strategies):
-        super().__init__(grammar, start, explicit_strategies)
+        super().__init__(grammar, start, explicit_strategies, alphabet=ALLOWED_CHARS)
         self.terminal_strategies = {
-            k: v.map(lambda s: s.replace("\0", "")).filter(utf8_encodable)
-            for k, v in self.terminal_strategies.items()  # type: ignore
+            k: v.map(fix_terminal) for k, v in self.terminal_strategies.items()  # type: ignore
         }
 
     def draw_symbol(self, data, symbol, draw_state):  # type: ignore
-        count = len(draw_state.result)
+        count = len(draw_state)
         super().draw_symbol(data, symbol, draw_state)
         try:
             compile(
-                source="".join(draw_state.result[count:])
+                source="".join(draw_state[count:])
                 .replace("contract", "class")
                 .replace("struct", "class"),  # HACK: Python ast.parse
                 filename="<string>",
@@ -102,10 +97,11 @@ def has_no_docstrings(c):
 
 
 @pytest.mark.fuzzing
-@given(code=from_grammar().filter(lambda c: utf8_encodable(c)))
-@hypothesis.settings(max_examples=500, suppress_health_check=[HealthCheck.too_slow])
+@given(code=from_grammar())
+@hypothesis.settings(
+    max_examples=500, suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much]
+)
 def test_grammar_bruteforce(code):
-    if utf8_encodable(code):
-        _, _, _, reformatted_code = pre_parse(code + "\n")
-        tree = parse_to_ast(reformatted_code)
-        assert isinstance(tree, Module)
+    _, _, _, reformatted_code = pre_parse(code + "\n")
+    tree = parse_to_ast(reformatted_code)
+    assert isinstance(tree, Module)

tests/functional/syntax/test_structs.py

@@ -589,9 +589,9 @@ def foo():
     with warnings.catch_warnings(record=True) as w:
         assert compiler.compile_code(code) is not None
 
-        expected = "Instantiating a struct using a dictionary is deprecated "
-        expected += "as of v0.4.0 and will be disallowed in a future release. "
-        expected += "Use kwargs instead e.g. Foo(a=1, b=2)"
+    expected = "Instantiating a struct using a dictionary is deprecated "
+    expected += "as of v0.4.0 and will be disallowed in a future release. "
+    expected += "Use kwargs instead e.g. Foo(a=1, b=2)"
 
-        assert len(w) == 1
-        assert str(w[0].message).startswith(expected)
+    assert len(w) == 1, [s.message for s in w]
+    assert str(w[0].message).startswith(expected)

vyper/ast/nodes.py

@@ -774,7 +774,6 @@ def is_literal_value(self):
 class Num(Constant):
     # inherited class for all numeric constant node types
     __slots__ = ()
-    _translated_fields = {"n": "value"}
 
     @property
     def n(self):
@@ -843,7 +842,6 @@ class Hex(Constant):
     """
 
     __slots__ = ()
-    _translated_fields = {"n": "value"}
 
     def validate(self):
         if "_" in self.value:

vyper/ast/parse.py

@@ -435,7 +435,7 @@ def visit_Num(self, node):
                     node.col_offset,
                 )
             node.ast_type = "Hex"
-            node.n = value
+            node.value = value
 
         elif value.lower()[:2] == "0b":
             node.ast_type = "Bytes"
@@ -449,15 +449,15 @@ def visit_Num(self, node):
                 )
             node.value = int(value, 2).to_bytes(len(value) // 8, "big")
 
-        elif isinstance(node.n, float):
+        elif isinstance(node.value, float):
             node.ast_type = "Decimal"
-            node.n = Decimal(value)
+            node.value = Decimal(value)
 
-        elif isinstance(node.n, int):
+        elif isinstance(node.value, int):
             node.ast_type = "Int"
 
-        else:
-            raise CompilerPanic(f"Unexpected type for Constant value: {type(node.n).__name__}")
+        else:  # pragma: nocover
+            raise CompilerPanic(f"Unexpected type for Constant value: {type(node.value).__name__}")
 
         return node