From 9f8c8330391e98e828d17c2588d2fb6f2d1cc156 Mon Sep 17 00:00:00 2001 From: cubicroot Date: Fri, 13 Sep 2024 14:18:15 +0200 Subject: [PATCH 1/3] merge coverprofiles instead of overwritting them --- gotesplit.go | 3 ++- run.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/gotesplit.go b/gotesplit.go index f9ad530..38ab8f4 100644 --- a/gotesplit.go +++ b/gotesplit.go @@ -40,6 +40,7 @@ Options: total := fs.Uint("total", 1, "total number of test splits (CIRCLE_NODE_TOTAL is used if set)") index := fs.Uint("index", 0, "zero-based index number of test splits (CIRCLE_NODE_INDEX is used if set)") junitDir := fs.String("junit-dir", "", "directory to store test result in JUnit format") + coverageDir := fs.String("coverprofile-dir", ".cover", "directory to store granular coverprofiles") fs.VisitAll(func(f *flag.Flag) { if f.Name == "index" || f.Name == "total" { if s := os.Getenv("CIRCLE_NODE_" + strings.ToUpper(f.Name)); s != "" { @@ -57,7 +58,7 @@ Options: return rnr.run(ctx, argv[1:], outStream, errStream) } } - return run(ctx, *total, *index, *junitDir, argv, outStream, errStream) + return run(ctx, *total, *index, *junitDir, *coverageDir, argv, outStream, errStream) } func getTestListsFromPkgs(pkgs []string, tags string, withRace bool) ([]testList, error) { diff --git a/run.go b/run.go index dec9614..85b26e5 100644 --- a/run.go +++ b/run.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/jstemmer/go-junit-report/v2/gtr" @@ -20,7 +21,7 @@ import ( "golang.org/x/sync/errgroup" ) -func run(ctx context.Context, total, idx uint, junitDir string, argv []string, outStream io.Writer, errStream io.Writer) error { +func run(_ context.Context, total, idx uint, junitDir string, coverageDir string, argv []string, outStream io.Writer, errStream io.Writer) error { if idx >= total { return fmt.Errorf("`index` should be the range from 0 to `total`-1, but: %d (total:%d)", idx, total) } @@ -111,6 +112,25 @@ func run(ctx context.Context, total, idx uint, junitDir string, argv []string, o } } + coverprofileOut := "" + for i := range testOpts { + if strings.HasPrefix(testOpts[i], "-coverprofile=") { + coverprofileOut = strings.TrimPrefix(testOpts[i], "-coverprofile=") + + if i == len(testOpts)-1 { + testOpts = testOpts[:i] + } else { + testOpts = append(testOpts[:i], testOpts[i+1:]...) + } + break + } + } + if coverprofileOut != "" { + if err := os.MkdirAll(coverageDir, 0755); err != nil { + return err + } + } + var testArgsList [][]string if len(allPkgs) > 0 { @@ -126,6 +146,10 @@ func run(ctx context.Context, total, idx uint, junitDir string, argv []string, o } for i, args := range testArgsList { + if coverprofileOut != "" { + args = append(args, fmt.Sprintf("-coverprofile=%s/coverprofile_%d", coverageDir, i)) + } + report := goTest(args, outStream, errStream, junitDir) if err2 := report.err; err2 != nil { err = err2 @@ -147,7 +171,46 @@ func run(ctx context.Context, total, idx uint, junitDir string, argv []string, o } } } - return err + if err != nil { + return err + } + + if coverprofileOut != "" { + err = mergeCoverprofiles(coverageDir, coverprofileOut) + if err != nil { + return err + } + + err = os.RemoveAll(coverageDir) + if err != nil { + return err + } + } + + return nil +} + +func mergeCoverprofiles(dir string, coverprofileOut string) error { + files, err := os.ReadDir(dir) + if err != nil { + return err + } + + modeRegex := regexp.MustCompile(`^mode: [a-zA-Z]+\n`) + mergedContent := []byte{} + for i, file := range files { + content, err := os.ReadFile(dir + "/" + file.Name()) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", file.Name(), err) + } + + if i != 0 { + content = modeRegex.ReplaceAll(content, []byte{}) + } + mergedContent = append(mergedContent, content...) + } + + return os.WriteFile(coverprofileOut, mergedContent, 0755) } type junitReport struct { From b197cbf55a934b093c674af152dddff27b27a15b Mon Sep 17 00:00:00 2001 From: cubicroot Date: Sun, 15 Sep 2024 08:41:03 +0200 Subject: [PATCH 2/3] improve variable naming & add coments --- gotesplit.go | 4 ++-- run.go | 28 ++++++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/gotesplit.go b/gotesplit.go index 38ab8f4..c7fa85a 100644 --- a/gotesplit.go +++ b/gotesplit.go @@ -40,7 +40,7 @@ Options: total := fs.Uint("total", 1, "total number of test splits (CIRCLE_NODE_TOTAL is used if set)") index := fs.Uint("index", 0, "zero-based index number of test splits (CIRCLE_NODE_INDEX is used if set)") junitDir := fs.String("junit-dir", "", "directory to store test result in JUnit format") - coverageDir := fs.String("coverprofile-dir", ".cover", "directory to store granular coverprofiles") + coverprofileDir := fs.String("coverprofile-dir", ".cover", "temporary directory for collecting coverprofile") fs.VisitAll(func(f *flag.Flag) { if f.Name == "index" || f.Name == "total" { if s := os.Getenv("CIRCLE_NODE_" + strings.ToUpper(f.Name)); s != "" { @@ -58,7 +58,7 @@ Options: return rnr.run(ctx, argv[1:], outStream, errStream) } } - return run(ctx, *total, *index, *junitDir, *coverageDir, argv, outStream, errStream) + return run(ctx, *total, *index, *junitDir, *coverprofileDir, argv, outStream, errStream) } func getTestListsFromPkgs(pkgs []string, tags string, withRace bool) ([]testList, error) { diff --git a/run.go b/run.go index 85b26e5..5d9df98 100644 --- a/run.go +++ b/run.go @@ -21,7 +21,7 @@ import ( "golang.org/x/sync/errgroup" ) -func run(_ context.Context, total, idx uint, junitDir string, coverageDir string, argv []string, outStream io.Writer, errStream io.Writer) error { +func run(_ context.Context, total, idx uint, junitDir string, coverprofilesDir string, argv []string, outStream io.Writer, errStream io.Writer) error { if idx >= total { return fmt.Errorf("`index` should be the range from 0 to `total`-1, but: %d (total:%d)", idx, total) } @@ -112,10 +112,12 @@ func run(_ context.Context, total, idx uint, junitDir string, coverageDir string } } - coverprofileOut := "" + // Check if coverprofile flag is set. If so remove it from args and replace it + // later with separate coverprofile files for each `go test` run. + coverprofileFile := "" for i := range testOpts { if strings.HasPrefix(testOpts[i], "-coverprofile=") { - coverprofileOut = strings.TrimPrefix(testOpts[i], "-coverprofile=") + coverprofileFile = strings.TrimPrefix(testOpts[i], "-coverprofile=") if i == len(testOpts)-1 { testOpts = testOpts[:i] @@ -125,8 +127,9 @@ func run(_ context.Context, total, idx uint, junitDir string, coverageDir string break } } - if coverprofileOut != "" { - if err := os.MkdirAll(coverageDir, 0755); err != nil { + if coverprofileFile != "" { + // Make temporary directory to store single coverprofile files in. + if err := os.MkdirAll(coverprofilesDir, 0755); err != nil { return err } } @@ -146,8 +149,9 @@ func run(_ context.Context, total, idx uint, junitDir string, coverageDir string } for i, args := range testArgsList { - if coverprofileOut != "" { - args = append(args, fmt.Sprintf("-coverprofile=%s/coverprofile_%d", coverageDir, i)) + if coverprofileFile != "" { + // Write coverprofiles to temp folder. + args = append(args, fmt.Sprintf("-coverprofile=%s/coverprofile_%d", coverprofilesDir, i)) } report := goTest(args, outStream, errStream, junitDir) @@ -175,13 +179,15 @@ func run(_ context.Context, total, idx uint, junitDir string, coverageDir string return err } - if coverprofileOut != "" { - err = mergeCoverprofiles(coverageDir, coverprofileOut) + if coverprofileFile != "" { + // Merge single coverprofiles to one file. + err = mergeCoverprofiles(coverprofilesDir, coverprofileFile) if err != nil { return err } - err = os.RemoveAll(coverageDir) + // Remove temp directory. + err = os.RemoveAll(coverprofilesDir) if err != nil { return err } @@ -205,6 +211,8 @@ func mergeCoverprofiles(dir string, coverprofileOut string) error { } if i != 0 { + // Cover mode is set in first line, remove it from all the following + // files. content = modeRegex.ReplaceAll(content, []byte{}) } mergedContent = append(mergedContent, content...) From 4289468a624ca87335857123c4f161da6c93ed17 Mon Sep 17 00:00:00 2001 From: cubicroot Date: Sun, 22 Sep 2024 17:50:29 +0200 Subject: [PATCH 3/3] update to go 1.23 and bump dependencies --- go.mod | 6 +++--- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 5d5ac9e..2dcd4aa 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/Songmu/gotesplit -go 1.21 +go 1.23 require ( - github.com/jstemmer/go-junit-report/v2 v2.0.0 - golang.org/x/sync v0.3.0 + github.com/jstemmer/go-junit-report/v2 v2.1.0 + golang.org/x/sync v0.8.0 ) diff --git a/go.sum b/go.sum index eeee71c..b0a9a0f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/jstemmer/go-junit-report/v2 v2.0.0 h1:bMZNO9B16VFn07tKyi4YJFIbZtVmJaa5Xakv9dcwK58= -github.com/jstemmer/go-junit-report/v2 v2.0.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= +github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=