Skip to content

Commit

Permalink
Add git-optimized-clones feature gate
Browse files Browse the repository at this point in the history
OptimizedGitClones decreases resource utilization for GitRepository
reconciliations. It supports both go-git and libgit2 implementations
when cloning repositories using branches or tags.

This is an opt-out feature, which can be disabled by starting the
controller with the argument '--feature-gates=OptimizedGitClones=false'.

Signed-off-by: Paulo Gomes <paulo.gomes@weave.works>
  • Loading branch information
Paulo Gomes committed May 11, 2022
1 parent 90ef278 commit 262efc0
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 8 deletions.
9 changes: 6 additions & 3 deletions controllers/gitrepository_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (

sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
serror "github.com/fluxcd/source-controller/internal/error"
"github.com/fluxcd/source-controller/internal/features"
sreconcile "github.com/fluxcd/source-controller/internal/reconcile"
"github.com/fluxcd/source-controller/internal/reconcile/summarize"
"github.com/fluxcd/source-controller/internal/util"
Expand Down Expand Up @@ -413,8 +414,10 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
checkoutOpts.SemVer = ref.SemVer
}

if artifact := obj.GetArtifact(); artifact != nil {
checkoutOpts.LastRevision = artifact.Revision
if oc, _ := features.Enabled(features.OptimizedGitClones); oc {
if artifact := obj.GetArtifact(); artifact != nil {
checkoutOpts.LastRevision = artifact.Revision
}
}

checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(ctx,
Expand Down Expand Up @@ -464,7 +467,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
var v git.NoChangesError
if errors.As(err, &v) {
return sreconcile.ResultSuccess,
&serror.Waiting{Err: v, Reason: v.Message}
&serror.Waiting{Err: v, Reason: v.Message, RequeueAfter: obj.GetRequeueAfter()}
}

e := &serror.Event{
Expand Down
16 changes: 16 additions & 0 deletions docs/spec/v1beta2/gitrepositories.md
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,22 @@ transport being handled by the controller, instead of `libgit2`.
This may lead to an increased number of timeout messages in the logs, however
it will fix the bug in which Git operations make the controllers hang indefinitely.

#### Optimized Git clones

Optimized Git clones decreases resource utilization for GitRepository
reconciliations. It supports both `go-git` and `libgit2` implementations
when cloning repositories using branches or tags.

When enabled, avoids full clone operations by first checking whether
the last revision is still the same at the target repository,
and if that is so, skips the reconciliation.

This feature is enabled by default. It can be disabled by starting the
controller with the argument `--feature-gates=OptimizedGitClones=false`.

NB: GitRepository objects configured for SemVer or Commit clones are
not affected by this functionality.

#### Proxy support

When a proxy is configured in the source-controller Pod through the appropriate
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ require (
github.com/fluxcd/pkg/gitutil v0.1.0
github.com/fluxcd/pkg/helmtestserver v0.7.2
github.com/fluxcd/pkg/lockedfile v0.1.0
github.com/fluxcd/pkg/runtime v0.14.2
github.com/fluxcd/pkg/runtime v0.15.1
github.com/fluxcd/pkg/ssh v0.3.3
github.com/fluxcd/pkg/testserver v0.2.0
github.com/fluxcd/pkg/untar v0.1.0
Expand Down Expand Up @@ -185,7 +185,7 @@ require (
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/cobra v1.4.0 // indirect
github.com/stretchr/testify v1.7.1 // indirect
github.com/xanzy/ssh-agent v0.3.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
Expand Down
7 changes: 4 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -364,8 +364,8 @@ github.com/fluxcd/pkg/helmtestserver v0.7.2/go.mod h1:WtUXBrfpJdwK54LX1Tqd8PpLJY
github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
github.com/fluxcd/pkg/runtime v0.13.0-rc.6/go.mod h1:4oKUO19TeudXrnCRnxCfMSS7EQTYpYlgfXwlQuDJ/Eg=
github.com/fluxcd/pkg/runtime v0.14.2 h1:ktyUjcX4pHoC8DRoBmhEP6eMHbmR6+/MYoARe4YulZY=
github.com/fluxcd/pkg/runtime v0.14.2/go.mod h1:NZr3PRK7xX2M1bl0LdtugvQyWkOmu2NcW3NrZH6U0is=
github.com/fluxcd/pkg/runtime v0.15.1 h1:PKooYqlZM+KLhnNz10sQnBH0AHllS40PIDHtiRH/BGU=
github.com/fluxcd/pkg/runtime v0.15.1/go.mod h1:TPAoOEgUFG60FXBA4ID41uaPldxuXCEI4jt3qfd5i5Q=
github.com/fluxcd/pkg/ssh v0.3.3 h1:/tc7W7LO1VoVUI5jB+p9ZHCA+iQaXTkaSCDZJsxcZ9k=
github.com/fluxcd/pkg/ssh v0.3.3/go.mod h1:+bKhuv0/pJy3HZwkK54Shz68sNv1uf5aI6wtPaEHaYk=
github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
Expand Down Expand Up @@ -990,8 +990,9 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0=
github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
Expand Down
54 changes: 54 additions & 0 deletions internal/features/features.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package features sets the feature gates that
// source-controller supports, and their default
// states.
package features

import feathelper "github.com/fluxcd/pkg/runtime/features"

const (
// OptimizedGitClones decreases resource utilization for GitRepository
// reconciliations. It supports both go-git and libgit2 implementations
// when cloning repositories using branches or tags.
//
// When enabled, avoids full clone operations by first checking whether
// the last revision is still the same at the target repository,
// and if that is so, skips the reconciliation.
OptimizedGitClones = "OptimizedGitClones"
)

var features = map[string]bool{
// OptimizedGitClones
// opt-out from v0.25
OptimizedGitClones: true,
}

// DefaultFeatureGates contains a list of all supported feature gates and
// their default values.
func FeatureGates() map[string]bool {
return features
}

// Enabled verifies whether the feature is enabled or not.
//
// This is only a wrapper around the Enabled func in
// pkg/runtime/features, so callers won't need to import
// both packages for checking whether a feature is enabled.
func Enabled(feature string) (bool, error) {
return feathelper.Enabled(feature)
}
12 changes: 12 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ import (
"github.com/fluxcd/pkg/runtime/client"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
feathelper "github.com/fluxcd/pkg/runtime/features"
"github.com/fluxcd/pkg/runtime/leaderelection"
"github.com/fluxcd/pkg/runtime/logger"
"github.com/fluxcd/pkg/runtime/pprof"
"github.com/fluxcd/pkg/runtime/probes"
"github.com/fluxcd/source-controller/internal/features"

sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/controllers"
Expand Down Expand Up @@ -88,6 +90,7 @@ func main() {
logOptions logger.Options
leaderElectionOptions leaderelection.Options
rateLimiterOptions helper.RateLimiterOptions
featureGates feathelper.FeatureGates
helmCacheMaxSize int
helmCacheTTL string
helmCachePurgeInterval string
Expand Down Expand Up @@ -136,11 +139,20 @@ func main() {
logOptions.BindFlags(flag.CommandLine)
leaderElectionOptions.BindFlags(flag.CommandLine)
rateLimiterOptions.BindFlags(flag.CommandLine)
featureGates.BindFlags(flag.CommandLine)

flag.Parse()

ctrl.SetLogger(logger.NewLogger(logOptions))

err := featureGates.WithLogger(setupLog).
SupportedFeatures(features.FeatureGates())

if err != nil {
setupLog.Error(err, "unable to load feature gates")
os.Exit(1)
}

// Set upper bound file size limits Helm
helm.MaxIndexSize = helmIndexLimit
helm.MaxChartSize = helmChartLimit
Expand Down

0 comments on commit 262efc0

Please sign in to comment.