runservice: build and use multiple toolboxes per architecture

Dockerfile

@@ -2,15 +2,15 @@
 ####### Build the backend
 #######
 
-# Base build image
+# base build image
 FROM golang:1.11 AS build_base
 
 WORKDIR /agola
 
 # use go modules
 ENV GO111MODULE=on
 
-# Only copy go.mod and go.sum
+# only copy go.mod and go.sum
 COPY go.mod .
 COPY go.sum .
 
@@ -19,10 +19,10 @@ RUN go mod download
 # This image builds the weavaite server
 FROM build_base AS server_builder
 
-# Copy all the source
+# copy all the source
 COPY . .
 
-# Copy the agola-web dist
+# copy the agola-web dist
 COPY --from=agola-web /agola-web/dist/ /agola-web/dist/
 
 RUN make WEBBUNDLE=1 WEBDISTPATH=/agola-web/dist
@@ -35,8 +35,8 @@ FROM debian:stable AS agola
 
 WORKDIR /
 
-# Finally we copy the statically compiled Go binary.
-COPY --from=server_builder /agola/bin/agola /agola/bin/agola-toolbox /bin/
+# copy to agola binaries
+COPY --from=server_builder /agola/bin/agola /agola/bin/agola-toolbox-* /bin/
 
 ENTRYPOINT ["/bin/agola"]
 
@@ -54,7 +54,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy the example config
+# copy the example config
 COPY examples/agolademo/config.yml .
 
 ENTRYPOINT ["/bin/agola"]

Makefile

@@ -29,40 +29,43 @@ AGOLA_DEPS = $(AGOLA_WEBBUNDLE_DEPS)
 AGOLA_TAGS += $(AGOLA_WEBBUNDLE_TAGS)
 endif
 
+TOOLBOX_OSES=linux
+TOOLBOX_ARCHS=amd64 arm64
+
 .PHONY: all
 all: build
 
 .PHONY: build
-build: bin/agola bin/agola-toolbox bin/agola-git-hook
+build: agola agola-toolbox agola-git-hook
 
 .PHONY: test
-test: tools/bin/gocovmerge
+test: gocovmerge
 	@scripts/test.sh
 
 # don't use existing file names and track go sources, let's do this to the go tool
-.PHONY: bin/agola
-bin/agola: $(AGOLA_DEPS)
+.PHONY: agola
+agola: $(AGOLA_DEPS)
 	GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola $(REPO_PATH)/cmd/agola
 
 # toolbox MUST be statically compiled so it can be used in any image for that arch
-# TODO(sgotti) cross compile to multiple archs
-.PHONY: bin/agola-toolbox
-bin/agola-toolbox: 
-	CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-toolbox $(REPO_PATH)/cmd/toolbox
+.PHONY: agola-toolbox
+agola-toolbox:
+	$(foreach GOOS, $(TOOLBOX_OSES),\
+	$(foreach GOARCH, $(TOOLBOX_ARCHS), $(shell GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-toolbox-$(GOOS)-$(GOARCH) $(REPO_PATH)/cmd/toolbox)))
 
-.PHONY: tools/bin/go-bindata
-tools/bin/go-bindata:
+.PHONY: go-bindata
+go-bindata:
 	GOBIN=$(PROJDIR)/tools/bin go install github.com/go-bindata/go-bindata/go-bindata
 
-.PHONY: bin/agola-git-hook
-bin/agola-git-hook:
+.PHONY: agola-git-hook
+agola-git-hook:
 	CGO_ENABLED=0 GO111MODULE=on go build $(if $(AGOLA_TAGS),-tags "$(AGOLA_TAGS)") -ldflags $(LD_FLAGS) -o $(PROJDIR)/bin/agola-git-hook $(REPO_PATH)/cmd/agola-git-hook
 
-.PHONY: tools/bin/gocovmerge
-tools/bin/gocovmerge:
+.PHONY: gocovmerge
+gocovmerge:
 	GOBIN=$(PROJDIR)/tools/bin go install github.com/wadey/gocovmerge
 
-webbundle/bindata.go: tools/bin/go-bindata $(WEBDISTPATH)
+webbundle/bindata.go: go-bindata $(WEBDISTPATH)
 	./tools/bin/go-bindata -o webbundle/bindata.go -tags webbundle -pkg webbundle -prefix "$(WEBDISTPATH)" -nocompress=true "$(WEBDISTPATH)/..."
 
 .PHONY: docker-agola

examples/agolademo/config.yml

@@ -43,7 +43,8 @@ runservice:
 
 executor:
   dataDir: /tmp/agola/executor
-  toolboxPath: ./bin/agola-toolbox
+  # The directory containing the toolbox compiled for the various supported architectures
+  toolboxPath: ./bin
   runserviceURL: "http://localhost:4000"
   web:
     listenAddress: ":4001"

examples/kubernetes/distributed/agola.yml

@@ -121,7 +121,8 @@ data:
 
     executor:
       dataDir: /mnt/agola/local/executor
-      toolboxPath: ./bin/agola-toolbox
+      # The directory containing the toolbox compiled for the various supported architectures
+      toolboxPath: ./bin
       runserviceURL: "http://agola-runservice:4000"
       web:
         listenAddress: ":4001"

examples/kubernetes/simple/agola.yml

@@ -110,7 +110,8 @@ data:
 
     executor:
       dataDir: /mnt/agola/local/executor
-      toolboxPath: ./bin/agola-toolbox
+      # The directory containing the toolbox compiled for the various supported architectures
+      toolboxPath: ./bin
       runserviceURL: "http://agola-internal:4000"
       web:
         listenAddress: ":4001"

internal/services/executor/driver/docker.go

@@ -49,19 +49,22 @@ type DockerDriver struct {
 	initVolumeHostDir string
 	toolboxPath       string
 	executorID        string
+	arch              common.Arch
 }
 
 func NewDockerDriver(logger *zap.Logger, executorID, initVolumeHostDir, toolboxPath string) (*DockerDriver, error) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
 		return nil, err
 	}
+
 	return &DockerDriver{
 		logger:            logger,
 		client:            cli,
 		initVolumeHostDir: initVolumeHostDir,
 		toolboxPath:       toolboxPath,
 		executorID:        executorID,
+		arch:              common.ArchFromString(runtime.GOARCH),
 	}, nil
 }
 
@@ -98,10 +101,15 @@ func (d *DockerDriver) CopyToolbox(ctx context.Context) error {
 		return err
 	}
 
-	srcInfo, err := archive.CopyInfoSourcePath(d.toolboxPath, false)
+	toolboxExecPath, err := toolboxExecPath(d.toolboxPath, d.arch)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get toolbox path for arch %q", d.arch)
+	}
+	srcInfo, err := archive.CopyInfoSourcePath(toolboxExecPath, false)
 	if err != nil {
 		return err
 	}
+	srcInfo.RebaseName = "agola-toolbox"
 
 	srcArchive, err := archive.TarResource(srcInfo)
 	if err != nil {
@@ -126,7 +134,7 @@ func (d *DockerDriver) CopyToolbox(ctx context.Context) error {
 
 func (d *DockerDriver) Archs(ctx context.Context) ([]common.Arch, error) {
 	// since we are using the local docker driver we can return our go arch information
-	return []common.Arch{common.ArchFromString(runtime.GOARCH)}, nil
+	return []common.Arch{d.arch}, nil
 }
 
 func (d *DockerDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Writer) (Pod, error) {

internal/services/executor/driver/driver.go

@@ -19,13 +19,18 @@ package driver
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 
 	"github.com/sorintlab/agola/internal/common"
 	"github.com/sorintlab/agola/internal/services/executor/registry"
 )
 
 const (
+	toolboxPrefix = "agola-toolbox"
+
 	labelPrefix = "agola.io/"
 
 	agolaLabelKey   = labelPrefix + "agola"
@@ -102,3 +107,12 @@ type ExecConfig struct {
 	Stderr     io.Writer
 	Tty        bool
 }
+
+func toolboxExecPath(toolboxDir string, arch common.Arch) (string, error) {
+	toolboxPath := filepath.Join(toolboxDir, fmt.Sprintf("%s-linux-%s", toolboxPrefix, arch))
+	_, err := os.Stat(toolboxPath)
+	if err != nil {
+		return "", err
+	}
+	return toolboxPath, nil
+}

internal/services/executor/driver/k8s.go

@@ -18,6 +18,7 @@
 package driver
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -422,23 +423,69 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri
 
 	fmt.Fprintf(out, "init container ready\n")
 
-	srcInfo, err := archive.CopyInfoSourcePath(d.toolboxPath, false)
+	coreclient, err := corev1client.NewForConfig(d.restconfig)
 	if err != nil {
 		return nil, err
 	}
 
-	srcArchive, err := archive.TarResource(srcInfo)
+	// get the pod arch
+	req := coreclient.RESTClient().
+		Post().
+		Namespace(pod.Namespace).
+		Resource("pods").
+		Name(pod.Name).
+		SubResource("exec").
+		VersionedParams(&corev1.PodExecOptions{
+			Container: "initcontainer",
+			Command:   []string{"uname", "-m"},
+			Stdout:    true,
+			Stderr:    true,
+			TTY:       false,
+		}, scheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL())
+	}
+
+	stdout := bytes.Buffer{}
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: out,
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to execute command on initcontainer")
+	}
+	osArch := strings.TrimSpace(stdout.String())
+
+	var arch common.Arch
+	switch osArch {
+	case "x86_64":
+		arch = common.ArchAMD64
+	case "aarch64":
+		arch = common.ArchARM64
+	default:
+		return nil, errors.Errorf("unsupported pod arch %q", osArch)
+	}
+
+	// copy the toolbox for the pod arch
+	toolboxExecPath, err := toolboxExecPath(d.toolboxPath, arch)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get toolbox path for arch %q", arch)
+	}
+	srcInfo, err := archive.CopyInfoSourcePath(toolboxExecPath, false)
 	if err != nil {
 		return nil, err
 	}
-	defer srcArchive.Close()
+	srcInfo.RebaseName = "agola-toolbox"
 
-	coreclient, err := corev1client.NewForConfig(d.restconfig)
+	srcArchive, err := archive.TarResource(srcInfo)
 	if err != nil {
 		return nil, err
 	}
+	defer srcArchive.Close()
 
-	req := coreclient.RESTClient().
+	req = coreclient.RESTClient().
 		Post().
 		Namespace(pod.Namespace).
 		Resource("pods").
@@ -453,11 +500,12 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri
 			TTY:       false,
 		}, scheme.ParameterCodec)
 
-	exec, err := remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL())
+	exec, err = remotecommand.NewSPDYExecutor(d.restconfig, "POST", req.URL())
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to generate k8s client spdy executor for url %q, method: POST", req.URL())
 	}
 
+	fmt.Fprintf(out, "extracting toolbox\n")
 	err = exec.Stream(remotecommand.StreamOptions{
 		Stdin:  srcArchive,
 		Stdout: out,
@@ -466,6 +514,7 @@ func (d *K8sDriver) NewPod(ctx context.Context, podConfig *PodConfig, out io.Wri
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to execute command on initcontainer")
 	}
+	fmt.Fprintf(out, "extracting toolbox done\n")
 
 	req = coreclient.RESTClient().
 		Post().

internal/services/executor/executor.go

@@ -28,7 +28,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -1247,14 +1246,7 @@ func NewExecutor(c *config.Executor) (*Executor, error) {
 	var err error
 	c.ToolboxPath, err = filepath.Abs(c.ToolboxPath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "cannot find \"agola-toolbox\" absolute path")
-	}
-	if c.ToolboxPath == "" {
-		path, err := exec.LookPath("agola-toolbox")
-		if err != nil {
-			return nil, errors.Errorf("cannot find \"agola-toolbox\" binaries in PATH, agola-toolbox path must be explicitly provided")
-		}
-		c.ToolboxPath = path
+		return nil, errors.Wrapf(err, "cannot determine \"agola-toolbox\" absolute path")
 	}
 
 	e := &Executor{