Merge pull request #467 from salman2013/salman/update-btr-release-script

Update BTR release scripts to look at catalog-info.yaml for release info.#439

edx_repo_tools/data.py

@@ -6,7 +6,6 @@
 
 logging.basicConfig()
 LOGGER = logging.getLogger(__name__)
-OPEN_EDX_YAML = 'openedx.yaml'
 
 
 def iter_nonforks(hub, orgs):
@@ -28,9 +27,9 @@ def iter_nonforks(hub, orgs):
                 yield repo
 
 
-def iter_openedx_yaml(hub, orgs, branches=None):
+def iter_openedx_yaml(file_name, hub, orgs, branches=None):
     """
-    Yield the data from all openedx.yaml files found in repositories in ``orgs``
+    Yield the data from all catalog-info.yaml or openedx.yaml files found in repositories in ``orgs``
     on any of ``branches``.
 
     Arguments:
@@ -45,19 +44,20 @@ def iter_openedx_yaml(hub, orgs, branches=None):
         Repositories (:class:`~github3.Repository)
 
     """
+
     for repo in iter_nonforks(hub, orgs):
         for branch in (branches or [repo.default_branch]):
             try:
-                contents = repo.file_contents(OPEN_EDX_YAML, ref=branch)
+                contents = repo.file_contents(file_name, ref=branch)
             except NotFoundError:
                 contents = None
 
             if contents is not None:
-                LOGGER.debug("Found openedx.yaml at %s:%s", repo.full_name, branch)
+                LOGGER.debug("Found %s at %s:%s", file_name, repo.full_name, branch)
                 try:
                     data = yaml.safe_load(contents.decoded)
                 except Exception as exc:
-                    LOGGER.error("Couldn't parse openedx.yaml from %s:%s, skipping repo", repo.full_name, branch, exc_info=True)
+                    LOGGER.error("Couldn't parse %s from %s:%s, skipping repo", file_name, repo.full_name, branch, exc_info=True)
                 else:
                     if data is not None:
                         yield repo, data

edx_repo_tools/oep2/explode_repos_yaml.py

@@ -27,7 +27,7 @@
 )
 def implode(hub, org, branch):
     """
-    Implode all openedx.yaml files, and print the results as formatted output.
+    Implode all catalog-info.yaml or openedx.yaml files, and print the results as formatted output.
     """
     data = {
         repo.full_name: openedx_yaml

edx_repo_tools/release/tag_release.py

@@ -52,6 +52,57 @@ def nice_tqdm(iterable, desc):
     return tqdm(iterable, desc=desc.ljust(27))
 
 
+def filter_repos(openedx_repo, catalog_repo):
+    """
+    Return the subset of the repos with catalog-info.yaml file if they have 'openedx.org/release' section otherwise
+    with openedx.yaml file. 
+
+    Arguments:
+        openedx_repo (list of dictionaries): list of repos with data of openedx.yaml with `openedx-release` section.
+        catalog_repo (list of dictionaries): list of repos with data of catalog.yaml with 'openedx.org/release' section.  
+    """
+    result_dict = {}
+    for repo_key, openedx_data in openedx_repo.items():
+        if repo_key in catalog_repo:
+            result_dict[repo_key] = catalog_repo[repo_key]
+        else:
+            result_dict[repo_key] = openedx_data 
+
+    return result_dict        
+
+
+def openedx_repos_with_catalog_info(hub, orgs=None, branches=None):
+    """
+    Return a subset of the repos with catalog-info.yaml files: the repos
+    with an annotation `openedx.org/release`.
+
+    Arguments:
+        hub (:class:`~github3.GitHub`): an authenticated GitHub instance.
+        orgs (list of str): The GitHub organizations to scan. Defaults to
+            OPENEDX_ORGS.
+        branches (list of str): The branches to scan in all repos in the selected
+                  orgs, defaulting to the repo's default branch.
+
+    Returns:
+        A dict from :class:`~github3.Repository` objects to openedx.yaml data for all of the
+        repos with an ``openedx-release`` key specified.
+
+    """
+    orgs = orgs or OPENEDX_ORGS
+    repos = {}
+    for repo, data in tqdm(iter_openedx_yaml('catalog-info.yaml', hub, orgs=orgs, branches=branches), desc='Find repos'):
+
+        if 'metadata' in data:
+            annotations = data['metadata'].get('annotations')
+            if annotations:
+                # Check if 'openedx.org/release' is present in annotations
+                if 'openedx.org/release' in annotations:
+                    repo = repo.refresh()
+                    repos[repo] = data
+
+    return repos
+
+
 def openedx_release_repos(hub, orgs=None, branches=None):
     """
     Return a subset of the repos with openedx.yaml files: the repos
@@ -71,9 +122,8 @@ def openedx_release_repos(hub, orgs=None, branches=None):
     """
     orgs = orgs or OPENEDX_ORGS
     repos = {}
-
-    for repo, data in tqdm(iter_openedx_yaml(hub, orgs=orgs, branches=branches), desc='Find repos'):
-        if data.get('openedx-release'):
+    for repo, data in tqdm(iter_openedx_yaml('openedx.yaml', hub, orgs=orgs, branches=branches), desc='Find repos'):
+        if data.get('openedx-release'):            
             repo = repo.refresh()
             repos[repo] = data
 
@@ -180,7 +230,10 @@ def override_repo_refs(repos, override_ref=None, overrides=None):
         for repo, repo_data in repos.items():
             local_override = overrides.get(repo.full_name, override_ref)
             if local_override:
-                repo_data["openedx-release"]["ref"] = local_override
+                if "metadata" in repo_data:
+                    repo_data["metadata"]["annotations"]["openedx.org/release"] = local_override
+                elif "openedx-release" in repo_data:
+                    repo_data["openedx-release"]["ref"] = local_override    
     return repos
 
 
@@ -219,11 +272,19 @@ def commit_ref_info(repos, skip_invalid=False):
         }
 
     """
-
     ref_info = {}
     for repo, repo_data in nice_tqdm(repos.items(), desc='Find commits'):
         # are we specifying a ref?
-        ref = repo_data["openedx-release"].get("ref")
+
+        if 'metadata' in repo_data:
+            annotations = repo_data['metadata'].get('annotations', {})
+            ref = annotations.get('openedx.org/release')
+
+        # Check if 'openedx-release' is present in repo_data and get the ref
+        # This check will be remove once we will just support catalog_info.yaml and remove openedx.yaml
+        elif 'openedx-release' in repo_data:
+            ref = repo_data['openedx-release'].get('ref')
+
         if ref:
             try:
                 ref_info[repo] = get_latest_commit_for_ref(repo, ref)
@@ -742,19 +803,21 @@ def main(hub, ref, use_tag, override_ref, overrides, interactive, quiet,
                 for r in json.load(f)
             }
     else:
-        repos = openedx_release_repos(hub, orgs, branches)
+        repos = filter_repos(openedx_release_repos(hub, orgs, branches), openedx_repos_with_catalog_info(hub, orgs, branches))
+
         if output_repos:
             with open(output_repos, "w") as f:
                 dumped = [{"repo": repo.as_dict(), "data": data} for repo, data in repos.items()]
                 json.dump(dumped, f, indent=2, sort_keys=True)
     if not repos:
-        raise ValueError("No repos marked for openedx-release in their openedx.yaml files!")
+        raise ValueError("No repos marked for openedx-release neither in openedx.yaml nor catalog_info.yaml files!")
 
     if included_repos:
         repos = include_only_repos(repos, included_repos)
-    repos = trim_skipped_repos(repos, skip_repos)
-    repos = trim_dependent_repos(repos)
-    repos = trim_indecisive_repos(repos)
+    if 'openedx-release' in repos:    
+        repos = trim_skipped_repos(repos, skip_repos)
+        repos = trim_dependent_repos(repos)
+        repos = trim_indecisive_repos(repos)
     repos = override_repo_refs(
         repos,
         override_ref=override_ref,

tests/test_tag_release.py

@@ -88,6 +88,13 @@ def expected_repos():
                 'ref': 'master',
             }
         },
+        mock_repository('edx/edx-package'): {
+            'metadata': {
+                'annotations': {
+                    'openedx.org/release': 'master',
+                }
+            }
+        },
         mock_repository('edx/XBlock'): {
             'openedx-release': {
                 'parent-repo': 'edx/edx-platform',
@@ -122,6 +129,14 @@ def expected_commits():
             'ref': 'master',
             'message': 'commit message for configuration master commit',
         },
+        mock_repository('edx/edx-package'): {
+            'committer': {'name': 'Dev 2'},
+            'author': {'name': 'Dev 2'},
+            'sha': '12345deadbeef',
+            'ref_type': 'branch',
+            'ref': 'master',
+            'message': 'commit message for configuration master commit',
+        },
         mock_repository('edx/XBlock'): {
             'committer': {'name': 'Dev 1'},
             'author': {'name': 'Dev 1'},
@@ -159,6 +174,7 @@ def test_get_ref_for_repos_not_exist():
     repos = [
         mock_repository('edx/edx-platform'),
         mock_repository('edx/configuration'),
+        mock_repository('edx/edx-package'),
         mock_repository('edx/XBlock'),
     ]
     for repo in repos:
@@ -172,16 +188,20 @@ def test_commit_ref_info(expected_repos):
 
     edx_edx_platform = find_repo(expected_repos, 'edx/edx-platform')
     edx_configuration = find_repo(expected_repos, 'edx/configuration')
+    edx_edx_package = find_repo(expected_repos, 'edx/edx-package')
     edx_edx_platform.branch.assert_called_once_with('release')
     assert not edx_edx_platform.ref.called
     edx_configuration.branch.assert_called_once_with('master')
     assert not edx_configuration.ref.called
+    edx_edx_package.branch.assert_called_once_with('master')
+    assert not edx_edx_package.ref.called
 
     # The XBlock repo shouldn't have been examined at all.
     assert not find_repo(expected_repos, 'edx/XBlock').branch.called
 
     edx_platform_commit = edx_edx_platform.git_commit().refresh()
     configuration_commit = edx_configuration.git_commit().refresh()
+    edx_edx_package_commit = edx_edx_package.git_commit().refresh()
 
     expected_commits = {
         edx_edx_platform: {
@@ -200,6 +220,14 @@ def test_commit_ref_info(expected_repos):
             'ref': 'master',
             'message': configuration_commit.message,
         },
+        edx_edx_package: {
+            'committer': edx_edx_package_commit.committer,
+            'author': edx_edx_package_commit.author,
+            'sha': edx_edx_package_commit.sha,
+            'ref_type': 'branch',
+            'ref': 'master',
+            'message': edx_edx_package_commit.message,
+        },
     }
 
     assert result == expected_commits
@@ -214,17 +242,20 @@ def test_overrides_global_ref(expected_repos):
     result = override_repo_refs(expected_repos, override_ref="abcdef")
     assert find_repo_data(result, "edx/edx-platform")["openedx-release"]["ref"] == "abcdef"
     assert find_repo_data(result, "edx/configuration")["openedx-release"]["ref"] == "abcdef"
+    assert find_repo_data(result, "edx/edx-package")["metadata"]["annotations"]["openedx.org/release"] == "abcdef"
 
 
 def test_overrides_dict(expected_repos):
     overrides = {
         "edx/edx-platform": "xyz",
         "edx/configuration": "refs/branch/no-way",
         "edx/does-not-exist": "does-not-matter",
+        "edx/edx-package": "release"
     }
     result = override_repo_refs(expected_repos, overrides=overrides)
     assert find_repo_data(result, "edx/edx-platform")["openedx-release"]["ref"] == "xyz"
     assert find_repo_data(result, "edx/configuration")["openedx-release"]["ref"] == "refs/branch/no-way"
+    assert find_repo_data(result, "edx/edx-package")["metadata"]["annotations"]["openedx.org/release"] == "release"
 
 
 def test_overrides_global_ref_and_dict(expected_repos):
@@ -240,6 +271,7 @@ def test_overrides_global_ref_and_dict(expected_repos):
     )
     assert find_repo_data(result, "edx/edx-platform")["openedx-release"]["ref"] == "xyz"
     assert find_repo_data(result, "edx/configuration")["openedx-release"]["ref"] == "fakie-mcfakerson"
+    assert find_repo_data(result, "edx/edx-package")["metadata"]["annotations"]["openedx.org/release"] == "fakie-mcfakerson"
 
 
 def test_create_happy_path(expected_commits):