libgit2: fixes issues when using Managed Transport for BitBucket and Azure DevOps repositories

controllers/gitrepository_controller.go

@@ -419,21 +419,28 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
 	}
 
 	repositoryURL := obj.Spec.URL
-	// managed GIT transport only affects the libgit2 implementation
-	if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
+	// Managed Transport only affects the libgit2 implementation.
+	if obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
+		// Ensures the correct protocol is being used, taking into
+		// account opt-in/opt-out and auto upgrade settings.
+		repositoryURL = managed.EnsureProtocol(repositoryURL)
+
 		// At present only HTTP connections have the ability to define remote options.
 		// Although this can be easily extended by ensuring that the fake URL below uses the
 		// target ssh scheme, and the libgit2/managed/ssh.go pulls that information accordingly.
 		//
 		// This is due to the fact the key libgit2 remote callbacks do not take place for HTTP
 		// whilst most still work for SSH.
-		if strings.HasPrefix(repositoryURL, "http") {
+		if strings.HasPrefix(repositoryURL, managed.HTTPManagedProtocol) ||
+			strings.HasPrefix(repositoryURL, managed.HTTPSManagedProtocol) {
 			// Due to the lack of the callback feature, a fake target URL is created to allow
 			// for the smart sub transport be able to pick the options specific for this
 			// GitRepository object.
 			// The URL should use unique information that do not collide in a multi tenant
 			// deployment.
-			repositoryURL = fmt.Sprintf("http://%s/%s/%d", obj.Name, obj.UID, obj.Generation)
+			repositoryURL = fmt.Sprintf("%s://%s/%s/%d",
+				managed.HTTPManagedProtocol, obj.Name, obj.UID, obj.Generation)
+
 			managed.AddTransportOptions(repositoryURL,
 				managed.TransportOptions{
 					TargetURL: obj.Spec.URL,

main.go

@@ -268,9 +268,8 @@ func main() {
 		startFileServer(storage.BasePath, storageAddr, setupLog)
 	}()
 
-	if managed.Enabled() {
-		managed.InitManagedTransport(ctrl.Log.WithName("managed-transport"))
-	}
+	// registers three new Git protocols: http+managed, https+managed and ssh+managed.
+	managed.InitManagedTransport(ctrl.Log.WithName("managed-transport"))
 
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {

pkg/git/libgit2/managed/const.go

@@ -24,4 +24,65 @@ const (
 	// PathMaxLength represents the max length for the path element
 	// when cloning Git repositories via SSH.
 	PathMaxLength = 4096
+
+	// HTTPProtocol represents the HTTP protocol.
+	HTTPProtocol string = "http"
+
+	// HTTPSProtocol represents the HTTPS protocol.
+	HTTPSProtocol string = "https"
+
+	// SSHProtocol represents the SSH protocol.
+	SSHProtocol string = "ssh"
+
+	// SSHGitProtocol represents the ssh+git protocol.
+	SSHGitProtocol string = "ssh+git"
+
+	// GitSSHProtocol represents the git+ssh protocol.
+	GitSSHProtocol string = "git+ssh"
+
+	// HTTPManagedProtocol represents the HTTP managed protocol.
+	// It can be used to opt-in HTTP URLs over Managed Transport.
+	HTTPManagedProtocol string = "http+managed"
+
+	// HTTPUnmanagedProtocol represents the HTTP unmanaged protocol.
+	// It can be used to opt-out HTTP URLs over Managed Transport.
+	HTTPUnmanagedProtocol string = "http+unmanaged"
+
+	// HTTPSManagedProtocol represents the HTTPS managed protocol.
+	// It can be used to opt-in HTTPS URLs over Managed Transport.
+	HTTPSManagedProtocol string = "https+managed"
+
+	// HTTPSUnmanagedProtocol represents the HTTPS unmanaged protocol.
+	// It can be used to opt-out HTTPS URLs over Managed Transport.
+	HTTPSUnmanagedProtocol string = "https+unmanaged"
+
+	// SSHManagedProtocol represents the SSH managed protocol.
+	// It can be used to opt-in SSH URLs over Managed Transport.
+	SSHManagedProtocol string = "ssh+managed"
+
+	// SSHUnmanagedProtocol represents the SSH unmanaged protocol.
+	// It can be used to opt-out SSH URLs over Managed Transport.
+	SSHUnmanagedProtocol string = "ssh+unmanaged"
+)
+
+var (
+	// denySSHAutoUpgradeDomains is a list of domains that cannot be
+	// supported in managed transport via SSH.
+	denySSHAutoUpgradeDomains = []string{
+		// DevOps requires the Git protocol capabilities (e.g. multi_ack
+		// and multi_ack_detailed) that are not fully supported by libgit2/git2go
+		// in managed transport mode.
+		"dev.azure.com",
+	}
+
+	// denyConcurrentConnections is a list of servers (<DOMAIN>:<PORT>)
+	// that should use single-use connections.
+	//
+	// Some servers do not support concurrent connections
+	// (e.g. public bitbucket.org accounts) and may struggle with
+	// multiple sessions within the same connection. Avoid such problems
+	// by closing the connection as soon as they are no longer needed.
+	denyConcurrentConnections = []string{
+		"bitbucket.org:22",
+	}
 )

pkg/git/libgit2/managed/flag.go

@@ -21,14 +21,25 @@ import (
 	"strings"
 )
 
-// Enabled defines whether the use of Managed Transport should be enabled.
-// This is only affects git operations that uses libgit2 implementation.
+var autoUpgradeEnabled bool = false
+
+// Enabled defines whether all SSH and HTTP connections should be automatically
+// upgraded to Managed Transport.
+// This only affects git operations that uses libgit2 implementation.
+//
+// A per connection opt-in/opt-out can be achieved by using the specific protocols:
+// - opt-in:  ssh+managed://git@server:22/repo-path
+// - opt-out: ssh+unmanaged://git@server:22/repo-path
+//
+// If a connection opted-in for Managed Transport, it won't be affected by the value
+// of Enabled.
 //
 // True is returned when the environment variable `EXPERIMENTAL_GIT_TRANSPORT`
 // is detected with the value of `true` or `1`.
 func Enabled() bool {
 	if v, ok := os.LookupEnv("EXPERIMENTAL_GIT_TRANSPORT"); ok {
-		return strings.ToLower(v) == "true" || v == "1"
+		autoUpgradeEnabled = strings.ToLower(v) == "true" || v == "1"
 	}
-	return false
+
+	return autoUpgradeEnabled
 }

pkg/git/libgit2/managed/http.go

@@ -62,7 +62,7 @@ import (
 // HTTP(S) transport that doesn't rely on any lower-level libraries
 // such as OpenSSL.
 func registerManagedHTTP() error {
-	for _, protocol := range []string{"http", "https"} {
+	for _, protocol := range []string{HTTPManagedProtocol, HTTPSManagedProtocol} {
 		_, err := git2go.NewRegisteredSmartTransport(protocol, true, httpSmartSubtransportFactory)
 		if err != nil {
 			return fmt.Errorf("failed to register transport for %q: %v", protocol, err)
@@ -127,7 +127,7 @@ func (t *httpSmartSubtransport) Action(targetUrl string, action git2go.SmartServ
 
 		// golang will change POST to GET in case of redirects.
 		if len(via) >= 0 && req.Method != via[0].Method {
-			if via[0].URL.Scheme == "https" && req.URL.Scheme == "http" {
+			if via[0].URL.Scheme == HTTPSProtocol && req.URL.Scheme == HTTPProtocol {
 				return fmt.Errorf("downgrade from https to http is not allowed: from %q to %q", via[0].URL.String(), req.URL.String())
 			}
 			if via[0].URL.Host != req.URL.Host {

pkg/git/libgit2/managed/managed_test.go

@@ -263,9 +263,8 @@ func TestManagedTransport_E2E(t *testing.T) {
 
 	// Test HTTP transport
 
-	// Use a fake-url and force it to be overriden by the smart transport.
-	// This was the way found to ensure that the built-in transport was not used.
-	httpAddress := "http://fake-url"
+	// Use a fake-url over managed protocol, and force the URL to be overriden by the smart transport.
+	httpAddress := "http+managed://fake-url"
 	AddTransportOptions(httpAddress, TransportOptions{
 		TargetURL: server.HTTPAddress() + "/" + repoPath,
 	})
@@ -293,6 +292,12 @@ func TestManagedTransport_E2E(t *testing.T) {
 
 	// Test SSH transport
 	sshAddress := server.SSHAddress() + "/" + repoPath
+
+	// Force auto upgrade by enabling it and ensuring protocol
+	autoUpgradeEnabled = true
+	sshAddress = EnsureProtocol(sshAddress)
+	g.Expect(sshAddress).To(HavePrefix(SSHManagedProtocol))
+
 	repo, err = git2go.Clone(sshAddress, tmpDir2, &git2go.CloneOptions{
 		FetchOptions: git2go.FetchOptions{
 			RemoteCallbacks: git2go.RemoteCallbacks{

pkg/git/libgit2/managed/protocol.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package managed
+
+import (
+	"net/url"
+)
+
+// EnsureProtocol returns targetURL with the correct SSH or HTTP protocol.
+//
+// It takes into account whether targetURL includes opt-in protocols
+// ssh+unmanaged or ssh+managed. For automatic upgrade, it must be
+// enabled at controller level and the target domain must not be in
+// the deny list.
+//
+// If targetURL is not a valid URL, itself is returned.
+func EnsureProtocol(targetURL string) string {
+	u, err := url.Parse(targetURL)
+	if err != nil {
+		return targetURL
+	}
+
+	// opt-in unmanaged SSH, change scheme to ssh://
+	if u.Scheme == SSHUnmanagedProtocol {
+		u.Scheme = SSHProtocol
+		return u.String()
+	}
+	// opt-in managed SSH
+	if u.Scheme == SSHManagedProtocol {
+		return u.String()
+	}
+	// ignore other SSH protocols (i.e. ssh+git / git+ssh)
+	if u.Scheme == SSHGitProtocol || u.Scheme == GitSSHProtocol {
+		return u.String()
+	}
+	// auto upgrade to managed transport
+	if u.Scheme == SSHProtocol && upgradeToManagedSSH(u.Hostname()) {
+		u.Scheme = SSHManagedProtocol
+		return u.String()
+	}
+
+	// opt-in unmanaged HTTP, change scheme to http://
+	if u.Scheme == HTTPUnmanagedProtocol {
+		u.Scheme = HTTPProtocol
+		return u.String()
+	}
+	// opt-in unmanaged HTTPS, change scheme to https://
+	if u.Scheme == HTTPSUnmanagedProtocol {
+		u.Scheme = HTTPSProtocol
+		return u.String()
+	}
+	// opt-in managed HTTP
+	if u.Scheme == HTTPManagedProtocol {
+		return u.String()
+	}
+	// opt-in managed HTTPS
+	if u.Scheme == HTTPSManagedProtocol {
+		return u.String()
+	}
+	// auto upgrade to managed transport
+	if Enabled() && u.Scheme == HTTPProtocol {
+		u.Scheme = HTTPManagedProtocol
+		return u.String()
+	}
+	if Enabled() && u.Scheme == HTTPSProtocol {
+		u.Scheme = HTTPSManagedProtocol
+		return u.String()
+	}
+
+	return u.String()
+}
+
+func upgradeToManagedSSH(hostName string) bool {
+	return Enabled() && !HasAnySuffix(denySSHAutoUpgradeDomains, hostName)
+}