fix(repo-server): excess git requests, resolveReferencedSources and r…

…unManifestGenAsync not using cache (Issue #14725) (#16410) * fix(repo-server): excess git requests part 1, resolveReferencedSources and runManifestGenAsync Signed-off-by: nromriell <[email protected]> * fix: remove unnecessary settings instantiation Signed-off-by: nromriell <[email protected]> --------- Signed-off-by: nromriell <[email protected]>
argoproj · Nov 30, 2023 · bfbe4c3 · bfbe4c3
1 parent 30f68e1
commit bfbe4c3
Show file tree

Hide file tree

Showing 4 changed files with 488 additions and 108 deletions.
diff --git a/reposerver/cache/mocks/reposervercache.go b/reposerver/cache/mocks/reposervercache.go
@@ -0,0 +1,71 @@
+package mocks
+
+import (
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+	cacheutil "github.com/argoproj/argo-cd/v2/util/cache"
+	cacheutilmocks "github.com/argoproj/argo-cd/v2/util/cache/mocks"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockCacheType int
+
+const (
+	MockCacheTypeRedis MockCacheType = iota
+	MockCacheTypeInMem
+)
+
+type MockRepoCache struct {
+	mock.Mock
+	RedisClient       *cacheutilmocks.MockCacheClient
+	StopRedisCallback func()
+}
+
+type MockCacheOptions struct {
+	RepoCacheExpiration     time.Duration
+	RevisionCacheExpiration time.Duration
+	ReadDelay               time.Duration
+	WriteDelay              time.Duration
+}
+
+type CacheCallCounts struct {
+	ExternalSets    int
+	ExternalGets    int
+	ExternalDeletes int
+}
+
+// Checks that the cache was called the expected number of times
+func (mockCache *MockRepoCache) AssertCacheCalledTimes(t *testing.T, calls *CacheCallCounts) {
+	mockCache.RedisClient.AssertNumberOfCalls(t, "Get", calls.ExternalGets)
+	mockCache.RedisClient.AssertNumberOfCalls(t, "Set", calls.ExternalSets)
+	mockCache.RedisClient.AssertNumberOfCalls(t, "Delete", calls.ExternalDeletes)
+}
+
+func (mockCache *MockRepoCache) ConfigureDefaultCallbacks() {
+	mockCache.RedisClient.On("Get", mock.Anything, mock.Anything).Return(nil)
+	mockCache.RedisClient.On("Set", mock.Anything).Return(nil)
+	mockCache.RedisClient.On("Delete", mock.Anything).Return(nil)
+}
+
+func NewInMemoryRedis() (*redis.Client, func()) {
+	cacheutil.NewInMemoryCache(5 * time.Second)
+	mr, err := miniredis.Run()
+	if err != nil {
+		panic(err)
+	}
+	return redis.NewClient(&redis.Options{Addr: mr.Addr()}), mr.Close
+}
+
+func NewMockRepoCache(cacheOpts *MockCacheOptions) *MockRepoCache {
+	redisClient, stopRedis := NewInMemoryRedis()
+	redisCacheClient := &cacheutilmocks.MockCacheClient{
+		ReadDelay:  cacheOpts.ReadDelay,
+		WriteDelay: cacheOpts.WriteDelay,
+		BaseCache:  cacheutil.NewRedisCache(redisClient, cacheOpts.RepoCacheExpiration, cacheutil.RedisCompressionNone)}
+	newMockCache := &MockRepoCache{RedisClient: redisCacheClient, StopRedisCallback: stopRedis}
+	newMockCache.ConfigureDefaultCallbacks()
+	return newMockCache
+}
diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go
@@ -300,6 +300,7 @@ func (s *Service) runRepoOperation(
 	var gitClient git.Client
 	var helmClient helm.Client
 	var err error
+	gitClientOpts := git.WithCache(s.cache, !settings.noRevisionCache && !settings.noCache)
 	revision = textutils.FirstNonEmpty(revision, source.TargetRevision)
 	unresolvedRevision := revision
 	if source.IsHelm() {
@@ -308,13 +309,13 @@ func (s *Service) runRepoOperation(
 			return err
 		}
 	} else {
-		gitClient, revision, err = s.newClientResolveRevision(repo, revision, git.WithCache(s.cache, !settings.noRevisionCache && !settings.noCache))
+		gitClient, revision, err = s.newClientResolveRevision(repo, revision, gitClientOpts)
 		if err != nil {
 			return err
 		}
 	}
 
-	repoRefs, err := resolveReferencedSources(hasMultipleSources, source.Helm, refSources, s.newClientResolveRevision)
+	repoRefs, err := resolveReferencedSources(hasMultipleSources, source.Helm, refSources, s.newClientResolveRevision, gitClientOpts)
 	if err != nil {
 		return err
 	}
@@ -463,7 +464,7 @@ type gitClientGetter func(repo *v1alpha1.Repository, revision string, opts ...gi
 //
 // Much of this logic is duplicated in runManifestGenAsync. If making changes here, check whether runManifestGenAsync
 // should be updated.
-func resolveReferencedSources(hasMultipleSources bool, source *v1alpha1.ApplicationSourceHelm, refSources map[string]*v1alpha1.RefTarget, newClientResolveRevision gitClientGetter) (map[string]string, error) {
+func resolveReferencedSources(hasMultipleSources bool, source *v1alpha1.ApplicationSourceHelm, refSources map[string]*v1alpha1.RefTarget, newClientResolveRevision gitClientGetter, gitClientOpts git.ClientOpts) (map[string]string, error) {
 	repoRefs := make(map[string]string)
 	if !hasMultipleSources || source == nil {
 		return repoRefs, nil
@@ -490,7 +491,7 @@ func resolveReferencedSources(hasMultipleSources bool, source *v1alpha1.Applicat
 			normalizedRepoURL := git.NormalizeGitURL(refSourceMapping.Repo.Repo)
 			_, ok = repoRefs[normalizedRepoURL]
 			if !ok {
-				_, referencedCommitSHA, err := newClientResolveRevision(&refSourceMapping.Repo, refSourceMapping.TargetRevision)
+				_, referencedCommitSHA, err := newClientResolveRevision(&refSourceMapping.Repo, refSourceMapping.TargetRevision, gitClientOpts)
 				if err != nil {
 					log.Errorf("Failed to get git client for repo %s: %v", refSourceMapping.Repo.Repo, err)
 					return nil, fmt.Errorf("failed to get git client for repo %s", refSourceMapping.Repo.Repo)
@@ -728,7 +729,7 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA,
 								return
 							}
 						} else {
-							gitClient, referencedCommitSHA, err := s.newClientResolveRevision(&refSourceMapping.Repo, refSourceMapping.TargetRevision)
+							gitClient, referencedCommitSHA, err := s.newClientResolveRevision(&refSourceMapping.Repo, refSourceMapping.TargetRevision, git.WithCache(s.cache, !q.NoRevisionCache && !q.NoCache))
 							if err != nil {
 								log.Errorf("Failed to get git client for repo %s: %v", refSourceMapping.Repo.Repo, err)
 								ch.errCh <- fmt.Errorf("failed to get git client for repo %s", refSourceMapping.Repo.Repo)