feat: add option to persist rule index

cmd/gazelle/fix-update.go

@@ -55,6 +55,9 @@ type updateConfig struct {
 	patchBuffer    bytes.Buffer
 	print0         bool
 	profile        profiler
+
+	// EXPERIMENTAL: caching of the rule index across runs
+	ruleIndexFile string
 }
 
 type emitFunc func(c *config.Config, f *rule.File) error
@@ -94,6 +97,7 @@ func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *conf
 	fs.StringVar(&ucr.memProfile, "memprofile", "", "write memory profile to `file`")
 	fs.Var(&gzflag.MultiFlag{Values: &ucr.knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)")
 	fs.StringVar(&ucr.repoConfigPath, "repo_config", "", "file where Gazelle should load repository configuration. Defaults to WORKSPACE.")
+	fs.StringVar(&uc.ruleIndexFile, "indexdb", "", "EXPERIMENTAL: file to cache the rule index")
 }
 
 func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
@@ -110,6 +114,9 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro
 	if uc.patchPath != "" && !filepath.IsAbs(uc.patchPath) {
 		uc.patchPath = filepath.Join(c.WorkDir, uc.patchPath)
 	}
+	if uc.ruleIndexFile != "" && !filepath.IsAbs(uc.ruleIndexFile) {
+		uc.ruleIndexFile = filepath.Join(c.WorkDir, uc.ruleIndexFile)
+	}
 	p, err := newProfiler(ucr.cpuProfile, ucr.memProfile)
 	if err != nil {
 		return err
@@ -311,15 +318,38 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 		}
 	}()
 
+	walkMode := uc.walkMode
+
+	updateRels := walk.NewUpdateFilter(c.RepoRoot, uc.dirs, uc.walkMode)
+	preindexed := false
+
+	// Load the rule index file if it exists.
+	if c.IndexLibraries && uc.ruleIndexFile != "" {
+		// Do not load index entries from directories that are being updated.
+		indexLoaded, err := ruleIndex.LoadIndex(uc.ruleIndexFile, updateRels.ShouldReIndex)
+		if err != nil {
+			log.Printf("Failed to load index file %s: %v", uc.ruleIndexFile, err)
+		} else if indexLoaded {
+			// Drop "visit all" since indexing has been loaded from disk.
+			if walkMode == walk.VisitAllUpdateSubdirsMode {
+				walkMode = walk.UpdateSubdirsMode
+			} else if walkMode == walk.VisitAllUpdateDirsMode {
+				walkMode = walk.UpdateDirsMode
+			}
+
+			preindexed = true
+		}
+	}
+
 	var errorsFromWalk []error
-	walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
+	walk.Walk(c, cexts, uc.dirs, walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
 		// If this file is ignored or if Gazelle was not asked to update this
 		// directory, just index the build file and move on.
 		if !update {
-			if c.IndexLibraries && f != nil {
-				for _, repl := range c.KindMap {
-					mrslv.MappedKind(rel, repl)
-				}
+			for _, repl := range c.KindMap {
+				mrslv.MappedKind(rel, repl)
+			}
+			if c.IndexLibraries && f != nil && (!preindexed || updateRels.ShouldReIndex(rel)) {
 				for _, r := range f.Rules {
 					ruleIndex.AddRule(c, r, f)
 				}
@@ -446,7 +476,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 		})
 
 		// Add library rules to the dependency resolution table.
-		if c.IndexLibraries {
+		if c.IndexLibraries && (!preindexed || updateRels.ShouldReIndex(rel)) {
 			for _, r := range f.Rules {
 				ruleIndex.AddRule(c, r, f)
 			}
@@ -475,6 +505,14 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
 	// Finish building the index for dependency resolution.
 	ruleIndex.Finish()
 
+	// Persist the index for future runs.
+	if c.IndexLibraries && uc.ruleIndexFile != "" {
+		err := ruleIndex.SaveIndex(uc.ruleIndexFile)
+		if err != nil {
+			fmt.Printf("Failed to save index file %s: %v", uc.ruleIndexFile, err)
+		}
+	}
+
 	// Resolve dependencies.
 	rc, cleanupRc := repo.NewRemoteCache(uc.repos)
 	defer func() {

resolve/index.go

@@ -16,7 +16,10 @@ limitations under the License.
 package resolve
 
 import (
+	"encoding/json"
+	"io"
 	"log"
+	"os"
 
 	"github.com/bazelbuild/bazel-gazelle/config"
 	"github.com/bazelbuild/bazel-gazelle/label"
@@ -145,6 +148,66 @@ func NewRuleIndex(mrslv func(ruleKind, pkgRel string) Resolver, exts ...interfac
 	}
 }
 
+func (ix *RuleIndex) LoadIndex(indexDbPath string, isPkgExcluded func(string) bool) (bool, error) {
+	if ix.indexed {
+		log.Fatal("LoadIndex called after Finish")
+	}
+
+	indexDbFile, err := os.Open(indexDbPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	defer indexDbFile.Close()
+
+	indexDbContent, err := io.ReadAll(indexDbFile)
+	if err != nil {
+		return false, err
+	}
+
+	// TODO: read & verify index version
+
+	var rules []*ruleRecord
+
+	err = json.Unmarshal(indexDbContent, &rules)
+	if err != nil {
+		return false, err
+	}
+
+	ix.rules = make([]*ruleRecord, 0, len(rules))
+	for _, r := range rules {
+		if !isPkgExcluded(r.Label.Pkg) {
+			ix.rules = append(ix.rules, r)
+		}
+	}
+
+	return true, nil
+}
+
+func (ix *RuleIndex) SaveIndex(indexDbPath string) error {
+	// TODO: write index version
+
+	indexDbContent, err := json.Marshal(ix.rules)
+	if err != nil {
+		return err
+	}
+
+	indexDbFile, err := os.OpenFile(indexDbPath, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return err
+	}
+	defer indexDbFile.Close()
+
+	_, err = indexDbFile.Write(indexDbContent)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // AddRule adds a rule r to the index. The rule will only be indexed if there
 // is a known resolver for the rule's kind and Resolver.Imports returns a
 // non-nil slice.

walk/walk.go

@@ -115,7 +115,7 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode,
 		}
 	}
 
-	updateRels := buildUpdateRelMap(c.RepoRoot, dirs)
+	updateRels := NewUpdateFilter(c.RepoRoot, dirs, mode)
 
 	var visit func(*config.Config, string, string, bool)
 	visit = func(c *config.Config, dir, rel string, updateParent bool) {
@@ -162,32 +162,38 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode,
 			}
 		}
 
-		shouldUpdate := shouldUpdate(rel, mode, updateParent, updateRels)
+		shouldUpdate := updateRels.shouldUpdate(rel, updateParent)
 		for _, sub := range subdirs {
-			if subRel := path.Join(rel, sub); shouldVisit(subRel, mode, shouldUpdate, updateRels) {
+			if subRel := path.Join(rel, sub); updateRels.shouldVisit(subRel, shouldUpdate) {
 				visit(c, filepath.Join(dir, sub), subRel, shouldUpdate)
 			}
 		}
 
 		update := !haveError && !wc.ignore && shouldUpdate
-		if shouldCall(rel, mode, updateParent, updateRels) {
+		if updateRels.shouldCall(rel, updateParent) {
 			genFiles := findGenFiles(wc, f)
 			wf(dir, rel, c, update, f, subdirs, regularFiles, genFiles)
 		}
 	}
 	visit(c, c.RepoRoot, "", false)
 }
 
-// buildUpdateRelMap builds a table of prefixes, used to determine which
+// An UpdateFilter tracks which directories need to be updated
+type UpdateFilter struct {
+	mode Mode
+
+	// map from slash-separated paths relative to the
+	// root directory ("" for the root itself) to a boolean indicating whether
+	// the directory should be updated.
+	updateRels map[string]bool
+}
+
+// NewUpdateFilter builds a table of prefixes, used to determine which
 // directories to update and visit.
 //
 // root and dirs must be absolute, canonical file paths. Each entry in dirs
 // must be a subdirectory of root. The caller is responsible for checking this.
-//
-// buildUpdateRelMap returns a map from slash-separated paths relative to the
-// root directory ("" for the root itself) to a boolean indicating whether
-// the directory should be updated.
-func buildUpdateRelMap(root string, dirs []string) map[string]bool {
+func NewUpdateFilter(root string, dirs []string, mode Mode) *UpdateFilter {
 	relMap := make(map[string]bool)
 	for _, dir := range dirs {
 		rel, _ := filepath.Rel(root, dir)
@@ -210,42 +216,60 @@ func buildUpdateRelMap(root string, dirs []string) map[string]bool {
 			i = next + 1
 		}
 	}
-	return relMap
+	return &UpdateFilter{mode, relMap}
+}
+
+func (u *UpdateFilter) ShouldReIndex(rel string) bool {
+	if rel == "." {
+		rel = ""
+	}
+
+	if should, found := u.updateRels[rel]; found {
+		return should
+	}
+
+	if rel != "" && (u.mode == UpdateSubdirsMode || u.mode == VisitAllUpdateSubdirsMode) {
+		u.updateRels[rel] = u.ShouldReIndex(path.Dir(rel))
+	} else {
+		u.updateRels[rel] = false
+	}
+
+	return u.updateRels[rel]
 }
 
 // shouldCall returns true if Walk should call the callback in the
 // directory rel.
-func shouldCall(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
-	switch mode {
+func (u *UpdateFilter) shouldCall(rel string, updateParent bool) bool {
+	switch u.mode {
 	case VisitAllUpdateSubdirsMode, VisitAllUpdateDirsMode:
 		return true
 	case UpdateSubdirsMode:
-		return updateParent || updateRels[rel]
+		return updateParent || u.updateRels[rel]
 	default: // UpdateDirsMode
-		return updateRels[rel]
+		return u.updateRels[rel]
 	}
 }
 
 // shouldUpdate returns true if Walk should pass true to the callback's update
 // parameter in the directory rel. This indicates the build file should be
 // updated.
-func shouldUpdate(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
-	if (mode == VisitAllUpdateSubdirsMode || mode == UpdateSubdirsMode) && updateParent {
+func (u *UpdateFilter) shouldUpdate(rel string, updateParent bool) bool {
+	if (u.mode == VisitAllUpdateSubdirsMode || u.mode == UpdateSubdirsMode) && updateParent {
 		return true
 	}
-	return updateRels[rel]
+	return u.updateRels[rel]
 }
 
 // shouldVisit returns true if Walk should visit the subdirectory rel.
-func shouldVisit(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool {
-	switch mode {
+func (u *UpdateFilter) shouldVisit(rel string, updateParent bool) bool {
+	switch u.mode {
 	case VisitAllUpdateSubdirsMode, VisitAllUpdateDirsMode:
 		return true
 	case UpdateSubdirsMode:
-		_, ok := updateRels[rel]
+		_, ok := u.updateRels[rel]
 		return ok || updateParent
 	default: // UpdateDirsMode
-		_, ok := updateRels[rel]
+		_, ok := u.updateRels[rel]
 		return ok
 	}
 }