Merge pull request #606 from pjbgf/managed-transport-libgit130

Experimental managed transport for libgit2 operations

controllers/gitrepository_controller.go

@@ -50,6 +50,7 @@ import (
 	"github.com/fluxcd/source-controller/internal/reconcile/summarize"
 	"github.com/fluxcd/source-controller/internal/util"
 	"github.com/fluxcd/source-controller/pkg/git"
+	"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
 	"github.com/fluxcd/source-controller/pkg/git/strategy"
 	"github.com/fluxcd/source-controller/pkg/sourceignore"
 )
@@ -369,10 +370,37 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
 		return sreconcile.ResultEmpty, e
 	}
 
+	repositoryURL := obj.Spec.URL
+	// managed GIT transport only affects the libgit2 implementation
+	if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
+		// At present only HTTP connections have the ability to define remote options.
+		// Although this can be easily extended by ensuring that the fake URL below uses the
+		// target ssh scheme, and the libgit2/managed/ssh.go pulls that information accordingly.
+		//
+		// This is due to the fact the key libgit2 remote callbacks do not take place for HTTP
+		// whilst most still work for SSH.
+		if strings.HasPrefix(repositoryURL, "http") {
+			// Due to the lack of the callback feature, a fake target URL is created to allow
+			// for the smart sub transport be able to pick the options specific for this
+			// GitRepository object.
+			// The URL should use unique information that do not collide in a multi tenant
+			// deployment.
+			repositoryURL = fmt.Sprintf("http://%s/%s/%d", obj.Name, obj.UID, obj.Generation)
+			managed.AddTransportOptions(repositoryURL,
+				managed.TransportOptions{
+					TargetURL: obj.Spec.URL,
+					CABundle:  authOpts.CAFile,
+				})
+
+			// We remove the options from memory, to avoid accumulating unused options over time.
+			defer managed.RemoveTransportOptions(repositoryURL)
+		}
+	}
+
 	// Checkout HEAD of reference in object
 	gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
 	defer cancel()
-	c, err := checkoutStrategy.Checkout(gitCtx, dir, obj.Spec.URL, authOpts)
+	c, err := checkoutStrategy.Checkout(gitCtx, dir, repositoryURL, authOpts)
 	if err != nil {
 		e := &serror.Event{
 			Err:    fmt.Errorf("failed to checkout and determine revision: %w", err),

hack/ci/e2e.sh

@@ -139,3 +139,19 @@ echo "Run large Git repo tests"
 kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/git/large-repo.yaml"
 kubectl -n source-system wait gitrepository/large-repo-go-git --for=condition=ready --timeout=2m15s
 kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
+
+
+# Test experimental libgit2 transport. Any tests against the default transport must
+# either run before this, or patch the deployment again to disable this, as once enabled
+# only the managed transport will be used.
+kubectl -n source-system patch deployment source-controller \
+    --patch '{"spec": {"template": {"spec": {"containers": [{"name": "manager","env": [{"name": "EXPERIMENTAL_GIT_TRANSPORT", "value": "true"}]}]}}}}'
+
+# wait until the patch took effect and the new source-controller is running
+sleep 20s
+
+kubectl -n source-system wait --for=condition=ready --timeout=1m -l app=source-controller pod
+
+echo "Re-run large libgit2 repo test with managed transport"
+kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
+kubectl -n source-system exec deploy/source-controller -- printenv | grep EXPERIMENTAL_GIT_TRANSPORT=true

main.go

@@ -45,6 +45,7 @@ import (
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	"github.com/fluxcd/source-controller/controllers"
 	"github.com/fluxcd/source-controller/internal/helm"
+	"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -226,6 +227,10 @@ func main() {
 		startFileServer(storage.BasePath, storageAddr, setupLog)
 	}()
 
+	if managed.Enabled() {
+		managed.InitManagedTransport()
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

pkg/git/libgit2/checkout.go

@@ -31,6 +31,7 @@ import (
 	"github.com/fluxcd/pkg/version"
 
 	"github.com/fluxcd/source-controller/pkg/git"
+	"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
 )
 
 // CheckoutStrategyForOptions returns the git.CheckoutStrategy for the given
@@ -72,7 +73,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
 		CheckoutBranch: c.Branch,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
+		return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
 	}
 	defer repo.Free()
 	head, err := repo.Head()
@@ -101,7 +102,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
 		},
 	})
 	if err != nil {
-		return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
+		return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
 	}
 	defer repo.Free()
 	cc, err := checkoutDetachedDwim(repo, c.Tag)
@@ -125,7 +126,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *g
 		},
 	})
 	if err != nil {
-		return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
+		return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
 	}
 	defer repo.Free()
 	oid, err := git2go.NewOid(c.Commit)
@@ -157,7 +158,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
 		},
 	})
 	if err != nil {
-		return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
+		return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
 	}
 	defer repo.Free()
 

pkg/git/libgit2/managed/flag.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package managed
+
+import (
+	"os"
+	"strings"
+)
+
+// Enabled defines whether the use of Managed Transport should be enabled.
+// This is only affects git operations that uses libgit2 implementation.
+//
+// True is returned when the environment variable `EXPERIMENTAL_GIT_TRANSPORT`
+// is detected with the value of `true` or `1`.
+func Enabled() bool {
+	if v, ok := os.LookupEnv("EXPERIMENTAL_GIT_TRANSPORT"); ok {
+		return strings.ToLower(v) == "true" || v == "1"
+	}
+	return false
+}