From aa019da1428d3fc6c61740b7639c02fea8e7ce92 Mon Sep 17 00:00:00 2001 From: Mehul Kar Date: Fri, 9 Sep 2022 12:28:33 -0700 Subject: [PATCH] Include package.json when user specifies inputs (#1832) We want to include each workspace's package.json as an input when the user has configured a smaller set of inputs, because the package.json defines the actual task turbo will execute for that workspace (via scripts). If that task definition changes, the cache is invalid. We only apply this inclusion when inputs are defined, because if inputs are not defined, all git-tracked files are included in the hash, which will also include the package.json file. --- cli/internal/hashing/package_deps_hash.go | 24 +++++++++++--- .../hashing/package_deps_hash_test.go | 31 +++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/cli/internal/hashing/package_deps_hash.go b/cli/internal/hashing/package_deps_hash.go index 03d97e6f85afe..21ad409cad80f 100644 --- a/cli/internal/hashing/package_deps_hash.go +++ b/cli/internal/hashing/package_deps_hash.go @@ -31,25 +31,41 @@ func GetPackageDeps(rootPath turbopath.AbsolutePath, p *PackageDepsOptions) (map pkgPath := rootPath.Join(p.PackagePath.ToStringDuringMigration()) // Add all the checked in hashes. var result map[turbopath.AnchoredUnixPath]string - if len(p.InputPatterns) == 0 { + + // make a copy of the inputPatterns array, becase we may be appending to it later. + calculatedInputs := make([]string, len(p.InputPatterns)) + copy(calculatedInputs, p.InputPatterns) + + if len(calculatedInputs) == 0 { gitLsTreeOutput, err := gitLsTree(pkgPath) if err != nil { return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", p.PackagePath, err) } result = gitLsTreeOutput } else { - absoluteFilesToHash, err := globby.GlobFiles(pkgPath.ToStringDuringMigration(), p.InputPatterns, nil) + + // Add in package.json to input patterns because if the `scripts` in + // the package.json change (i.e. the tasks that turbo executes), we want + // a cache miss, since any existing cache could be invalid. + // Note this package.json will be resolved relative to the pkgPath. + calculatedInputs = append(calculatedInputs, "package.json") + + absoluteFilesToHash, err := globby.GlobFiles(pkgPath.ToStringDuringMigration(), calculatedInputs, nil) if err != nil { - return nil, errors.Wrapf(err, "failed to resolve input globs %v", p.InputPatterns) + return nil, errors.Wrapf(err, "failed to resolve input globs %v", calculatedInputs) } + filesToHash := make([]turbopath.AnchoredSystemPath, len(absoluteFilesToHash)) for i, rawPath := range absoluteFilesToHash { relativePathString, err := pkgPath.RelativePathString(rawPath) + if err != nil { return nil, errors.Wrapf(err, "not relative to package: %v", rawPath) } + filesToHash[i] = turbopath.AnchoredSystemPathFromUpstream(relativePathString) } + hashes, err := gitHashObject(turbopath.AbsoluteSystemPathFromUpstream(pkgPath.ToStringDuringMigration()), filesToHash) if err != nil { return nil, errors.Wrap(err, "failed hashing resolved inputs globs") @@ -59,7 +75,7 @@ func GetPackageDeps(rootPath turbopath.AbsolutePath, p *PackageDepsOptions) (map // Update the checked in hashes with the current repo status // The paths returned from this call are anchored at the package directory - gitStatusOutput, err := gitStatus(pkgPath, p.InputPatterns) + gitStatusOutput, err := gitStatus(pkgPath, calculatedInputs) if err != nil { return nil, fmt.Errorf("Could not get git hashes from git status: %v", err) } diff --git a/cli/internal/hashing/package_deps_hash_test.go b/cli/internal/hashing/package_deps_hash_test.go index 3aa7ac6217386..2eaedf570e649 100644 --- a/cli/internal/hashing/package_deps_hash_test.go +++ b/cli/internal/hashing/package_deps_hash_test.go @@ -239,24 +239,43 @@ func TestGetPackageDeps(t *testing.T) { repoRoot := fs.AbsolutePathFromUpstream(t.TempDir()) myPkgDir := repoRoot.Join("my-pkg") + + // create the dir first + err := myPkgDir.MkdirAll() + assert.NilError(t, err, "CreateDir") + + // create file 1 committedFilePath := myPkgDir.Join("committed-file") - err := committedFilePath.EnsureDir() - assert.NilError(t, err, "EnsureDir") err = committedFilePath.WriteFile([]byte("committed bytes"), 0644) assert.NilError(t, err, "WriteFile") + + // create file 2 deletedFilePath := myPkgDir.Join("deleted-file") err = deletedFilePath.WriteFile([]byte("delete-me"), 0644) assert.NilError(t, err, "WriteFile") + + // create file 3 nestedPath := myPkgDir.Join("dir", "nested-file") assert.NilError(t, nestedPath.EnsureDir(), "EnsureDir") assert.NilError(t, nestedPath.WriteFile([]byte("nested"), 0644), "WriteFile") + + // create a package.json + packageJSONPath := myPkgDir.Join("package.json") + err = packageJSONPath.WriteFile([]byte("{}"), 0644) + assert.NilError(t, err, "WriteFile") + + // set up git repo and commit all requireGitCmd(t, repoRoot, "init", ".") requireGitCmd(t, repoRoot, "config", "--local", "user.name", "test") requireGitCmd(t, repoRoot, "config", "--local", "user.email", "test@example.com") requireGitCmd(t, repoRoot, "add", ".") requireGitCmd(t, repoRoot, "commit", "-m", "foo") + + // remove a file err = deletedFilePath.Remove() assert.NilError(t, err, "Remove") + + // create another untracked file in git uncommittedFilePath := myPkgDir.Join("uncommitted-file") err = uncommittedFilePath.WriteFile([]byte("uncommitted bytes"), 0644) assert.NilError(t, err, "WriteFile") @@ -265,6 +284,7 @@ func TestGetPackageDeps(t *testing.T) { opts *PackageDepsOptions expected map[turbopath.AnchoredUnixPath]string }{ + // base case. when inputs aren't specified, all files hashes are computed { opts: &PackageDepsOptions{ PackagePath: "my-pkg", @@ -272,18 +292,22 @@ func TestGetPackageDeps(t *testing.T) { expected: map[turbopath.AnchoredUnixPath]string{ "committed-file": "3a29e62ea9ba15c4a4009d1f605d391cdd262033", "uncommitted-file": "4e56ad89387e6379e4e91ddfe9872cf6a72c9976", + "package.json": "9e26dfeeb6e641a33dae4961196235bdb965b21b", "dir/nested-file": "bfe53d766e64d78f80050b73cd1c88095bc70abb", }, }, + // with inputs, only the specified inputs are hashed { opts: &PackageDepsOptions{ PackagePath: "my-pkg", InputPatterns: []string{"uncommitted-file"}, }, expected: map[turbopath.AnchoredUnixPath]string{ + "package.json": "9e26dfeeb6e641a33dae4961196235bdb965b21b", "uncommitted-file": "4e56ad89387e6379e4e91ddfe9872cf6a72c9976", }, }, + // inputs with glob pattern also works { opts: &PackageDepsOptions{ PackagePath: "my-pkg", @@ -292,9 +316,11 @@ func TestGetPackageDeps(t *testing.T) { expected: map[turbopath.AnchoredUnixPath]string{ "committed-file": "3a29e62ea9ba15c4a4009d1f605d391cdd262033", "uncommitted-file": "4e56ad89387e6379e4e91ddfe9872cf6a72c9976", + "package.json": "9e26dfeeb6e641a33dae4961196235bdb965b21b", "dir/nested-file": "bfe53d766e64d78f80050b73cd1c88095bc70abb", }, }, + // inputs with another glob pattern works { opts: &PackageDepsOptions{ PackagePath: "my-pkg", @@ -302,6 +328,7 @@ func TestGetPackageDeps(t *testing.T) { }, expected: map[turbopath.AnchoredUnixPath]string{ "committed-file": "3a29e62ea9ba15c4a4009d1f605d391cdd262033", + "package.json": "9e26dfeeb6e641a33dae4961196235bdb965b21b", "uncommitted-file": "4e56ad89387e6379e4e91ddfe9872cf6a72c9976", }, },