build: update modernisers for python312 (#478)

* build: update modernisers for python312 support

edx_repo_tools/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = '0.8.2'
+__version__ = '0.8.3'

edx_repo_tools/codemods/python312/__init__.py

@@ -0,0 +1,2 @@
+from .tox_modernizer import ConfigReader, ToxModernizer
+from .gh_actions_modernizer import GithubCIModernizer

edx_repo_tools/codemods/python312/gh_actions_modernizer.py

@@ -0,0 +1,86 @@
+"""
+Github Actions CI Modernizer to add Python 3.12 and drop Django 3.2 testing
+"""
+from copy import deepcopy
+import click
+from edx_repo_tools.utils import YamlLoader
+
+TO_BE_REMOVED_PYTHON = ['3.5', '3.6', '3.7']
+ALLOWED_PYTHON_VERSIONS = ['3.8', '3.12']
+
+ALLOWED_DJANGO_VERSIONS = ['4.2', 'django42']
+DJANGO_ENV_TO_ADD = ['django42']
+DJANGO_ENV_TO_REMOVE = ['django32', 'django40', 'django41']
+
+
+class GithubCIModernizer(YamlLoader):
+    def __init__(self, file_path):
+        super().__init__(file_path)
+
+    def _update_python_and_django_in_matrix(self):
+        django_versions = list()
+        python_versions = list()
+        matrix_elements = dict()
+
+
+        for section_key in self.elements['jobs']:
+            matrix_elements = deepcopy(self.elements['jobs'][section_key]['strategy']['matrix'])
+
+            for key, value in matrix_elements.items():
+                if key == 'django-version':
+                    for dj_version in DJANGO_ENV_TO_ADD:
+                        if dj_version not in value:
+                            value.append(dj_version)
+                    django_versions = list(filter(lambda version: version in ALLOWED_DJANGO_VERSIONS, value))
+                    if django_versions:
+                        self.elements['jobs'][section_key]['strategy']['matrix'][key] = django_versions
+
+                if key in ['tox', 'toxenv', 'tox-env']:
+                    for dj_env in DJANGO_ENV_TO_ADD:
+                        if dj_env not in value:
+                            value.append(dj_env)
+                    tox_envs = list(filter(lambda version: version not in DJANGO_ENV_TO_REMOVE, value))
+                    if tox_envs:
+                        self.elements['jobs'][section_key]['strategy']['matrix'][key] = tox_envs
+
+                if key == 'python-version':
+                    for version in ALLOWED_PYTHON_VERSIONS:
+                        if version not in value:
+                            value.append(version)
+                    python_versions = list(filter(lambda version: version not in TO_BE_REMOVED_PYTHON, value))
+                    if python_versions:
+                        self.elements['jobs'][section_key]['strategy']['matrix'][key] = python_versions
+                    else:
+                        del self.elements['jobs'][section_key]['strategy']['matrix'][key]
+
+                elif key in ['include', 'exclude']:
+                    allowed_python_vers = list()
+                    for item in value:
+                        if item['python-version'] not in TO_BE_REMOVED_PYTHON:
+                            allowed_python_vers.append(item)
+
+                    if len(allowed_python_vers):
+                        self.elements['jobs'][section_key]['strategy']['matrix'][key] = allowed_python_vers
+                    else:
+                        del self.elements['jobs'][section_key]['strategy']['matrix'][key]
+
+
+    def _update_github_actions(self):
+        self._update_python_and_django_in_matrix()
+
+    def modernize(self):
+        self._update_github_actions()
+        self.update_yml_file()
+
+
+@click.command()
+@click.option(
+    '--path', default='.github/workflows/ci.yml',
+    help="Path to default CI workflow file")
+def main(path):
+    modernizer = GithubCIModernizer(path)
+    modernizer.modernize()
+
+
+if __name__ == '__main__':
+    main()

edx_repo_tools/codemods/python312/tox_modernizer.py

@@ -0,0 +1,118 @@
+import io
+import re
+from configparser import ConfigParser, NoSectionError
+
+import click
+
+TOX_SECTION = "tox"
+ENVLIST = "envlist"
+TEST_ENV_SECTION = "testenv"
+TEST_ENV_DEPS = "deps"
+PYTHON_SUBSTITUTE = "py{38, 312}"
+DJANGO_SUBSTITUTE = "django{42}"
+
+DJANGO_42_DEPENDENCY = "django42: Django>=4.2,<4.3\n"
+NEW_DJANGO_DEPENDENCIES = DJANGO_42_DEPENDENCY
+
+SECTIONS = [TOX_SECTION, TEST_ENV_SECTION]
+
+PYTHON_PATTERN = "(py{.*?}-?|py[0-9]+,|py[0-9]+-)"
+
+DJANGO_PATTERN = "(django[0-9]+,|django[0-9]+\n|django{.*}\n|django{.*?}|django[0-9]+-|django{.*}-)"
+
+DJANGO_DEPENDENCY_PATTERN = "([^\n]*django[0-9]+:.*\n?)"
+
+
+class ConfigReader:
+    def __init__(self, file_path=None, config_dict=None):
+        self.config_dict = config_dict
+        self.file_path = file_path
+
+    def get_modernizer(self):
+        config_parser = ConfigParser()
+        if self.config_dict is not None:
+            config_parser.read_dict(self.config_dict)
+        else:
+            config_parser.read(self.file_path)
+        return ToxModernizer(config_parser, self.file_path)
+
+
+class ToxModernizer:
+    def __init__(self, config_parser, file_path):
+        self.file_path = file_path
+        self.config_parser = config_parser
+        self._validate_tox_config_sections()
+
+    def _validate_tox_config_sections(self):
+        if not self.config_parser.sections():
+            raise NoSectionError("Bad Config. No sections found.")
+
+        if all(section not in SECTIONS for section in self.config_parser.sections()):
+            raise NoSectionError("File doesn't contain required sections")
+
+    def _update_env_list(self):
+        tox_section = self.config_parser[TOX_SECTION]
+        env_list = tox_section[ENVLIST]
+
+        env_list = ToxModernizer._replace_runners(PYTHON_PATTERN, PYTHON_SUBSTITUTE, env_list)
+        env_list = ToxModernizer._replace_runners(DJANGO_PATTERN, DJANGO_SUBSTITUTE, env_list)
+        self.config_parser[TOX_SECTION][ENVLIST] = env_list
+
+    @staticmethod
+    def _replace_runners(pattern, substitute, env_list):
+        matches = re.findall(pattern, env_list)
+        if not matches:
+            return env_list
+        substitute = ToxModernizer._get_runner_substitute(matches, substitute)
+        return ToxModernizer._replace_matches(pattern, substitute, env_list, matches)
+
+    @staticmethod
+    def _replace_matches(pattern, substitute, target, matches):
+        if not matches:
+            return target
+        occurrences_to_replace = len(matches) - 1
+        if occurrences_to_replace > 0:
+            target = re.sub(pattern, '', target, occurrences_to_replace)
+        target = re.sub(pattern, substitute, target)
+        return target
+
+    @staticmethod
+    def _get_runner_substitute(matches, substitute):
+        last_match = matches[-1]
+        has_other_runners = last_match.endswith('-')
+        return substitute + "-" if has_other_runners else substitute
+
+    def _replace_django_versions(self):
+        test_environment = self.config_parser[TEST_ENV_SECTION]
+        dependencies = test_environment[TEST_ENV_DEPS]
+        matches = re.findall(DJANGO_DEPENDENCY_PATTERN, dependencies)
+        dependencies = self._replace_matches(DJANGO_DEPENDENCY_PATTERN, NEW_DJANGO_DEPENDENCIES, dependencies, matches)
+
+        self.config_parser[TEST_ENV_SECTION][TEST_ENV_DEPS] = dependencies
+
+    def _update_config_file(self):
+        # ConfigParser insists on using tabs for output. We want spaces.
+        with io.StringIO() as configw:
+            self.config_parser.write(configw)
+            new_ini = configw.getvalue()
+        new_ini = new_ini.replace("\t", "    ")
+        with open(self.file_path, 'w') as configfile:
+            configfile.write(new_ini)
+
+    def modernize(self):
+        self._update_env_list()
+        self._replace_django_versions()
+        self._update_config_file()
+
+
+@click.command()
+@click.option(
+    '--path', default='tox.ini',
+    help="Path to target tox config file")
+def main(path):
+    modernizer = ConfigReader(path).get_modernizer()
+    modernizer.modernize()
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -113,6 +113,8 @@ def is_requirement(line):
             'modernize_tox_django42 = edx_repo_tools.codemods.django42.tox_moderniser_django42:main',
             'modernize_github_actions_django42 = edx_repo_tools.codemods.django42.github_actions_modernizer_django42:main',
             'remove_providing_args = edx_repo_tools.codemods.django42.remove_providing_args_arg:main',
+            'python312_gh_actions_modernizer = edx_repo_tools.codemods.python312.gh_actions_modernizer:main',
+            'python312_tox_modernizer = edx_repo_tools.codemods.python312.tox_modernizer:main',
         ],
     },
     package_data={

tests/sample_files/sample_ci_file.yml

@@ -14,8 +14,12 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
         python-version: ['3.5', '3.8']
-        django-version: ['3.2', '4.0']
-        toxenv: ['django32', 'quality', 'docs', 'pii_check']
+        django-version: [django42]
+        toxenv:
+        - 'quality'
+        - 'docs'
+        - 'pii_check'
+        - django42
     steps:
     - uses: actions/checkout@v2
     - name: setup python

tests/sample_files/sample_ci_file_2.yml

@@ -13,19 +13,19 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python-version: ['3.5', '3.6', '3.7', '3.8']
+        python-version: ['3.5', '3.6', '3.7', '3.8', '3.12']
         django-version: ['3.2']
         include:
-          - python-version: "3.5"
-            toxenv: 'quality'
-            ubuntu: 20.04
+        - python-version: "3.5"
+          toxenv: 'quality'
+          ubuntu: 20.04
         exclude:
-          - python-version: "3.5"
-            toxenv: 'quality'
-          - python-version: "3.7"
-            toxenv: 'docs'
-          - python-version: "3.8"
-            toxenv: 'pii_check'
+        - python-version: "3.5"
+          toxenv: 'quality'
+        - python-version: "3.7"
+          toxenv: 'docs'
+        - python-version: "3.8"
+          toxenv: 'pii_check'
     steps:
     - uses: actions/checkout@v2
     - name: setup python
@@ -45,8 +45,8 @@ jobs:
       run: tox
 
     - name: Run Coverage
-      if: matrix.python-version == '3.8' && matrix.toxenv=='django32'
-      uses: codecov/codecov-action@v1
+      if: matrix.python-version == '3.8' && matrix.toxenv=='django42'
+      uses: codecov/codecov-action@v2
       with:
         flags: unittests
         fail_ci_if_error: true

tests/sample_tox_config.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist =
-        py{27,35,36,37}-django{111,20,21,22}-drf{39,310,latest},
+        py{38}-django{32,40}-drf{39,310,latest},
         docs,
         quality,
         version_check,
@@ -9,10 +9,8 @@ envlist =
 
 [testenv]
 deps =
-    django111: Django>=1.11,<2.0
-    django20: Django>=2.0,<2.1
-    django21: Django>=2.1,<2.2
-    django22: Django>=2.2,<2.3
+    django32: Django>=3.2,<3.3
+    django40: Django>=4.0,<4.1
     drf39: djangorestframework<3.10.0
     drf310: djangorestframework<3.11.0
     drflatest: djangorestframework

tests/sample_tox_config_2.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist =
-        py27,py35,py36,py37-django111,django20,django21,django22-drf39,drf310,drflatest,
+        py37,py38-django32,django40-drf39,drf310,drflatest,
         docs,
         quality,
         version_check,
@@ -9,10 +9,8 @@ envlist =
 
 [testenv]
 deps =
-    django111: Django>=1.11,<2.0
-    django20: Django>=2.0,<2.1
-    django21: Django>=2.1,<2.2
-    django22: Django>=2.2,<2.3
+    django32: Django>=3.2,<3.3
+    django40: Django>=4.0,<4.1
     drf39: djangorestframework<3.10.0
     drf310: djangorestframework<3.11.0
     drflatest: djangorestframework

tests/test_actions_modernizer.py

@@ -6,7 +6,7 @@
 import uuid
 from unittest import TestCase
 
-from edx_repo_tools.codemods.django3 import GithubCIModernizer
+from edx_repo_tools.codemods.python312 import GithubCIModernizer
 from edx_repo_tools.utils import YamlLoader
 
 
@@ -37,22 +37,16 @@ def test_python_matrix_items(self):
         python_versions = ci_elements['jobs']['run_tests']['strategy']['matrix']['python-version']
 
         self.assertIsInstance(python_versions, list)
-        self.assertNotIn('3.5', python_versions)
+        self.assertIn('3.8', python_versions)
+        self.assertIn('3.12', python_versions)
 
     def test_python_matrix_items_build_tag(self):
         ci_elements = TestGithubActionsModernizer._get_updated_yaml_elements(self.test_file3)
         python_versions = ci_elements['jobs']['build']['strategy']['matrix']['python-version']
 
         self.assertIsInstance(python_versions, list)
-        self.assertNotIn('3.5', python_versions)
-
-    def test_include_exclude_list(self):
-        ci_elements = TestGithubActionsModernizer._get_updated_yaml_elements(self.test_file2)
-        include_list = ci_elements['jobs']['run_tests']['strategy']['matrix'].get('include', {})
-        exclude_list = ci_elements['jobs']['run_tests']['strategy']['matrix'].get('exclude', {})
-
-        for item in list(include_list) + list(exclude_list):
-            self.assertNotEqual(item['python-version'], '3.5')
+        self.assertIn('3.8', python_versions)
+        self.assertIn('3.12', python_versions)
 
     def tearDown(self):
         os.remove(self.test_file1)