Skip to content

Commit

Permalink
Add parser and downloader factory for the upgrade case (#427)
Browse files Browse the repository at this point in the history
When we're doing template upgrades, we'll have a canonical location and
a location type, and we'll need a `Downloader` to get the new template
contents. This code does just that: `f(location_type, location) ->
Downloader`
  • Loading branch information
drevell committed Feb 14, 2024
1 parent c92334d commit 23068d1
Show file tree
Hide file tree
Showing 9 changed files with 505 additions and 154 deletions.
2 changes: 1 addition & 1 deletion templates/common/git/git_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ func TestHeadTags(t *testing.T) {
{
name: "git_repo_in_subdir",
dir: "mysubdir",
files: abctestutil.WithGitRepoAt("mysubdir/",
files: abctestutil.WithGitRepoAt("mysubdir",
map[string]string{
"mysubdir/.git/refs/tags/v1.2.3": abctestutil.MinimalGitHeadSHA,
}),
Expand Down
121 changes: 121 additions & 0 deletions templates/common/templatesource/git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright 2024 The Authors (see AUTHORS file)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package templatesource

import (
"context"
"sort"

"github.com/Masterminds/semver/v3"

"github.com/abcxyz/abc/templates/common/git"
"github.com/abcxyz/pkg/logging"
)

const (
LocTypeLocalGit = "local_git"
LocTypeRemoteGit = "remote_git"
)

// gitCanonicalVersion examines a template directory and tries to determine the
// "best" template version by looking at .git. The "best" template version is
// defined as (in decreasing order of precedence):
//
// - tags in decreasing order of semver (recent releases first)
// - other non-semver tags in reverse alphabetical order
// - the HEAD SHA
//
// It returns false if:
//
// - the given directory is not in a git workspace
// - the git workspace is not clean (uncommitted changes) (for testing, you
// can provide allowDirty=true to override this)
//
// It returns error only if something weird happened when running git commands.
// The returned string is always empty if the boolean is false.
func gitCanonicalVersion(ctx context.Context, dir string, allowDirty bool) (string, bool, error) {
logger := logging.FromContext(ctx).With("logger", "CanonicalVersion")

_, ok, err := git.Workspace(ctx, dir)
if err != nil {
return "", false, err //nolint:wrapcheck
}
if !ok {
return "", false, nil
}

if !allowDirty {
ok, err = git.IsClean(ctx, dir)
if err != nil {
return "", false, err //nolint:wrapcheck
}
if !ok {
logger.WarnContext(ctx, "omitting template git version from manifest because the workspace is dirty",
"source_git_workspace", dir)
return "", false, nil
}
}

tag, ok, err := bestHeadTag(ctx, dir)
if err != nil {
return "", false, err
}
if ok {
return tag, true, nil
}

sha, err := git.CurrentSHA(ctx, dir)
if err != nil {
return "", false, err //nolint:wrapcheck
}
return sha, true, nil
}

// bestHeadTag returns the tag that points to the current HEAD SHA. If there are
// multiple such tags, the precedence order is:
// - tags in decreasing order of semver (recent releases first)
// - other non-semver tags in reverse alphabetical order
//
// Returns false if there are no tags pointing to HEAD.
func bestHeadTag(ctx context.Context, dir string) (string, bool, error) {
tags, err := git.HeadTags(ctx, dir)
if err != nil {
return "", false, err //nolint:wrapcheck
}

var nonSemverTags []string
var semverTags []*semver.Version
for _, tag := range tags {
semverTag, err := git.ParseSemverTag(tag)
if err != nil {
nonSemverTags = append(nonSemverTags, tag)
} else {
semverTags = append(semverTags, semverTag)
}
}

if len(semverTags) > 0 {
sort.Sort(sort.Reverse(semver.Collection(semverTags)))
// The "v" was trimmed off during parsing. Add it back.
return "v" + semverTags[0].Original(), true, nil
}

if len(nonSemverTags) > 0 {
sort.Sort(sort.Reverse(sort.StringSlice(nonSemverTags)))
return nonSemverTags[0], true, nil
}

return "", false, nil
}
8 changes: 4 additions & 4 deletions templates/common/templatesource/localsource_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,8 @@ func TestLocalDownloader_Download(t *testing.T) {
name: "dest_dir_in_different_git_workspace",
srcDir: "src/dir1",
destDir: "dst/dir1",
initialContents: abctestutil.WithGitRepoAt("src/",
abctestutil.WithGitRepoAt("dst/",
initialContents: abctestutil.WithGitRepoAt("src",
abctestutil.WithGitRepoAt("dst",
map[string]string{
"src/dir1/spec.yaml": "file1 contents",
"src/dir1/file1.txt": "file1 contents",
Expand All @@ -178,7 +178,7 @@ func TestLocalDownloader_Download(t *testing.T) {
name: "source_in_git_but_dest_is_not",
srcDir: "src/dir1",
destDir: "dst",
initialContents: abctestutil.WithGitRepoAt("src/",
initialContents: abctestutil.WithGitRepoAt("src",
map[string]string{
"src/dir1/spec.yaml": "file1 contents",
"src/dir1/file1.txt": "file1 contents",
Expand All @@ -200,7 +200,7 @@ func TestLocalDownloader_Download(t *testing.T) {
name: "dest_in_git_but_src_is_not",
srcDir: "src",
destDir: "dst",
initialContents: abctestutil.WithGitRepoAt("dst/",
initialContents: abctestutil.WithGitRepoAt("dst",
map[string]string{
"src/spec.yaml": "file1 contents",
"src/file1.txt": "file1 contents",
Expand Down
107 changes: 71 additions & 36 deletions templates/common/templatesource/remote_git.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/abcxyz/abc/templates/common/git"
"github.com/abcxyz/abc/templates/common/tempdir"
"github.com/abcxyz/pkg/logging"
"github.com/abcxyz/pkg/sets"
)

var _ sourceParser = (*remoteGitSourceParser)(nil)
Expand All @@ -47,10 +48,6 @@ type remoteGitSourceParser struct {
// "${groupname}" to refer to the values captured by the groups of the regex
// above.

// Example: `https://${host}/${org}/${repo}.git`
httpsRemoteExpansion string
// Example: `git@${host}:${org}/${repo}.git`
sshRemoteExpansion string
// Example: `${subdir}`
subdirExpansion string
// Example: `${version}`
Expand All @@ -64,55 +61,64 @@ type remoteGitSourceParser struct {
}

func (g *remoteGitSourceParser) sourceParse(ctx context.Context, params *ParseSourceParams) (Downloader, bool, error) {
logger := logging.FromContext(ctx).With("logger", "remoteGitSourceParser.sourceParse")
return newRemoteGitDownloader(&newRemoteGitDownloaderParams{
re: g.re,
input: params.Source,
gitProtocol: params.GitProtocol,
defaultVersion: g.defaultVersion,
})
}

// newRemoteGitDownloaderParams contains the parameters to
// newRemoteGitDownloader.
type newRemoteGitDownloaderParams struct {
// defaultVersion is the template version (e.g. "latest", "v1.2.3") that
// will be used if the "re" regular expression either doesn't have a
// matching group named "version", or
defaultVersion string
gitProtocol string
input string
re *regexp.Regexp
}

match := g.re.FindStringSubmatchIndex(params.Source)
// newRemoteGitDownloader is basically a fancy constructor for
// remoteGitDownloader. It returns false if the provided input doesn't match the
// provided regex.
func newRemoteGitDownloader(p *newRemoteGitDownloaderParams) (Downloader, bool, error) {
match := p.re.FindStringSubmatchIndex(p.input)
if match == nil {
// It's not an error if this regex match fails, it just means that src
// isn't formatted as the kind of template source that we're looking
// for. It's probably something else, like a local directory name, and
// the caller should continue and try a different sourceParser.
return nil, false, nil
}

var remote string
switch params.GitProtocol {
case "https", "":
remote = string(g.re.ExpandString(nil, g.httpsRemoteExpansion, params.Source, match))
case "ssh":
remote = string(g.re.ExpandString(nil, g.sshRemoteExpansion, params.Source, match))
default:
return nil, false, fmt.Errorf("protocol %q isn't usable with a template sourced from a remote git repo", params.GitProtocol)
}

if g.warning != "" {
logger.WarnContext(ctx, g.warning)
remote, err := gitRemote(p.re, match, p.input, p.gitProtocol)
if err != nil {
return nil, false, err
}

version := string(g.re.ExpandString(nil, g.versionExpansion, params.Source, match))
version := string(p.re.ExpandString(nil, "${version}", p.input, match))
if version == "" {
version = g.defaultVersion
version = p.defaultVersion
}

canonicalSource := string(g.re.ExpandString(nil, "${host}/${org}/${repo}", params.Source, match))
if subdir := string(g.re.ExpandString(nil, "${subdir}", params.Source, match)); subdir != "" {
canonicalSource := string(p.re.ExpandString(nil, "${host}/${org}/${repo}", p.input, match))
if subdir := string(p.re.ExpandString(nil, "${subdir}", p.input, match)); subdir != "" {
canonicalSource += "/" + subdir
}

out := &remoteGitDownloader{
remote: remote,
subdir: string(g.re.ExpandString(nil, g.subdirExpansion, params.Source, match)),
version: version,
subdir := string(p.re.ExpandString(nil, "${subdir}", p.input, match))

return &remoteGitDownloader{
canonicalSource: canonicalSource,
cloner: &realCloner{},
remote: remote,
subdir: subdir,
tagser: &realTagser{},
canonicalSource: canonicalSource,
}

return out, true, nil
version: version,
}, true, nil
}

// remoteGitDownloader implements templateSource for templates hosted in a remote git
// repo, regardless of which git hosting service it uses.
// remoteGitDownloader implements templateSource for templates hosted in a
// remote git repo, regardless of which git hosting service it uses.
type remoteGitDownloader struct {
// An HTTPS or SSH connection string understood by "git clone".
remote string
Expand Down Expand Up @@ -337,3 +343,32 @@ type realTagser struct{}
func (r *realTagser) Tags(ctx context.Context, remote string) ([]string, error) {
return git.RemoteTags(ctx, remote) //nolint:wrapcheck
}

// gitRemote returns a git remote string (see "man git-remote") for the given
// remote git repo.
//
// The host, org, and repo name are provided by the given regex match. The
// "match" parameter must be the result of calling re.FindStringSubmatchIndex(),
// and must not be nil. reInput must be the string passed to
// re.FindStringSubmatchIndex(), this allows us to extract the matched host,
// org, and repo names that were match by the regex.
//
// The given regex must have matching groups (i.e. P<foo>) named "host", "org",
// and "repo".
func gitRemote(re *regexp.Regexp, match []int, reInput, gitProtocol string) (string, error) {
// Sanity check that the regular expression has the necessary named subgroups.
wantSubexps := []string{"host", "org", "repo"}
missingSubexps := sets.Subtract(wantSubexps, re.SubexpNames())
if len(missingSubexps) > 0 {
return "", fmt.Errorf("internal error: regexp expansion didn't have a named subgroup for: %v", missingSubexps)
}

switch gitProtocol {
case "https", "":
return string(re.ExpandString(nil, "https://${host}/${org}/${repo}.git", reInput, match)), nil
case "ssh":
return string(re.ExpandString(nil, "git@${host}:${org}/${repo}.git", reInput, match)), nil
default:
return "", fmt.Errorf("protocol %q isn't usable with a template sourced from a remote git repo", gitProtocol)
}
}
Loading

0 comments on commit 23068d1

Please sign in to comment.