Skip to content

Commit

Permalink
Optimize Dockerized integTest with GOCACHE (elastic#17957)
Browse files Browse the repository at this point in the history
Create a reusable GOCACHE located at $repoRoot/docker-gocache to speed up building the Linux
system test binaries and running tests.

This can yield proportionally large speed-up when doing selective testing as you often do in the development cycle.
For example when testing a single module in x-pack/filebeat:

```
NOSE_TESTMATCH=misp time mage -v pythonIntegTest
      199.62 real        20.86 user        35.55 sys

NOSE_TESTMATCH=misp time mage -v pythonIntegTest
      104.97 real        21.44 user        35.55 sys
```

I also compared running mage integTest for x-pack/filebeat without a cache then with a cache:

```
Go Unit Test: 2m35s
Go Test Compile: 30.2s
real	12m2.626s
user	0m21.141s
sys	0m35.212s
```

```
Go Unit Test: 47s
Go Test Compile: 36s
real	9m16.326s
user	0m21.697s
sys	0m38.908s
```
  • Loading branch information
andrewkroh authored May 2, 2020
1 parent 522ffd5 commit f39d869
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 4 deletions.
4 changes: 3 additions & 1 deletion dev-tools/mage/crossbuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,9 @@ func DockerChown(path string) {
func chownPaths(uid, gid int, path string) error {
start := time.Now()
numFixed := 0
defer log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
defer func() {
log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
}()

return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
if err != nil {
Expand Down
5 changes: 5 additions & 0 deletions dev-tools/mage/gotest.go
Original file line number Diff line number Diff line change
Expand Up @@ -456,5 +456,10 @@ func BuildSystemTestGoBinary(binArgs TestBinaryArgs) error {
if len(binArgs.InputFiles) > 0 {
args = append(args, binArgs.InputFiles...)
}

start := time.Now()
defer func() {
log.Printf("BuildSystemTestGoBinary (go %v) took %v.", strings.Join(args, " "), time.Since(start))
}()
return sh.RunV("go", args...)
}
5 changes: 4 additions & 1 deletion dev-tools/mage/integtest_docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,17 +90,20 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri
if err != nil {
return err
}
dockerRepoRoot := filepath.Join("/go/src", repo.CanonicalRootImportPath)
dockerGoCache := filepath.Join(dockerRepoRoot, "build/docker-gocache")
magePath := filepath.Join("/go/src", repo.CanonicalRootImportPath, repo.SubDir, "build/mage-linux-amd64")

// Execute the inside of docker-compose.
args := []string{"-p", dockerComposeProjectName(), "run",
"-e", "DOCKER_COMPOSE_PROJECT_NAME=" + dockerComposeProjectName(),
// Disable strict.perms because we moust host dirs inside containers
// Disable strict.perms because we mount host dirs inside containers
// and the UID/GID won't meet the strict requirements.
"-e", "BEAT_STRICT_PERMS=false",
// compose.EnsureUp needs to know the environment type.
"-e", "STACK_ENVIRONMENT=" + StackEnvironment,
"-e", "TESTING_ENVIRONMENT=" + StackEnvironment,
"-e", "GOCACHE=" + dockerGoCache,
}
args, err = addUidGidEnvArgs(args)
if err != nil {
Expand Down
1 change: 0 additions & 1 deletion libbeat/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ services:
depends_on:
- proxy_dep
environment:
- LIBBEAT_PATH=/go/src/github.com/elastic/beats/libbeat
- BEAT_STRICT_PERMS=false
- REDIS_HOST=redis
- REDIS_PORT=6379
Expand Down
2 changes: 1 addition & 1 deletion packetbeat/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.13.7
FROM golang:1.13.10

RUN \
apt-get update \
Expand Down

0 comments on commit f39d869

Please sign in to comment.