From 15af756d228e5b6dbb96d3ed8f165c4f47ee56e3 Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Fri, 10 May 2024 15:38:08 -0700 Subject: [PATCH] Temporarily remove support for gcedisk option The integrity fs setup script currently has some bugs when supporting attached disks. --- launcher/container_runner.go | 27 +++-------- launcher/spec/launch_spec.go | 86 +++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/launcher/container_runner.go b/launcher/container_runner.go index 7d7437fc..6306faaf 100644 --- a/launcher/container_runner.go +++ b/launcher/container_runner.go @@ -73,16 +73,6 @@ const ( defaultRefreshJitter = 0.1 ) -var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} - // NewRunner returns a runner. func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.Token, launchSpec spec.LaunchSpec, mdsClient *metadata.Client, tpm io.ReadWriteCloser, logger *log.Logger, serialConsole *os.File) (*ContainerRunner, error) { image, err := initImage(ctx, cdClient, launchSpec, token) @@ -148,12 +138,11 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To Soft: nofile, }} - randSuffix := randSeq(8) container, err = cdClient.NewContainer( ctx, - containerID+randSuffix, + containerID, containerd.WithImage(image), - containerd.WithNewSnapshot(snapshotID+randSuffix, image), + containerd.WithNewSnapshot(snapshotID, image), containerd.WithNewSpec( oci.WithImageConfigArgs(image, launchSpec.Cmd), oci.WithEnv(envs), @@ -506,9 +495,9 @@ func (r *ContainerRunner) Run(ctx context.Context) error { return fmt.Errorf("failed to measure CEL events: %v", err) } - // if err := r.fetchAndWriteToken(ctx); err != nil { - // return fmt.Errorf("failed to fetch and write OIDC token: %v", err) - // } + if err := r.fetchAndWriteToken(ctx); err != nil { + return fmt.Errorf("failed to fetch and write OIDC token: %v", err) + } r.logger.Printf("EnableTestFeatureForImage is set to %v\n", r.launchSpec.Experiments.EnableTestFeatureForImage) // create and start the TEE server behind the experiment @@ -561,8 +550,6 @@ func (r *ContainerRunner) Run(ctx context.Context) error { return fmt.Errorf("unknown logging redirect location: %v", r.launchSpec.LogRedirect) } - w := io.MultiWriter(os.Stdout, r.serialConsole) - streamOpt = cio.WithStreams(nil, w, w) task, err := r.container.NewTask(ctx, cio.NewCreator(streamOpt)) if err != nil { return &RetryableError{err} @@ -576,7 +563,6 @@ func (r *ContainerRunner) Run(ctx context.Context) error { r.logger.Println("workload task started") if err := task.Start(ctx); err != nil { - r.logger.Println(err) return &RetryableError{err} } status := <-exitStatusC @@ -586,9 +572,6 @@ func (r *ContainerRunner) Run(ctx context.Context) error { return err } - r.logger.Println(status.Error()) - r.logger.Println(status.ExitCode()) - r.logger.Println(status.ExitTime()) if code != 0 { r.logger.Println("workload task ended and returned non-zero") return &WorkloadError{code} diff --git a/launcher/spec/launch_spec.go b/launcher/spec/launch_spec.go index 48cf51a9..9e72abea 100644 --- a/launcher/spec/launch_spec.go +++ b/launcher/spec/launch_spec.go @@ -5,9 +5,15 @@ package spec import ( "encoding/json" "fmt" + "log" "os" + "os/exec" + "path" + "path/filepath" + "regexp" "strconv" "strings" + "syscall" "cloud.google.com/go/compute/metadata" "github.com/google/go-tpm-tools/cel" @@ -85,10 +91,10 @@ const ( mountDestinationKey = "destination" mountTypeTmpfs = "tmpfs" mountTypeBind = "bind" - mountTypeGCEDisk = "gcedisk" + // https://cloud.google.com/compute/docs/disks/set-persistent-device-name-in-linux-vm + mountTypeGCEDisk = "gcedisk" ) -// https://cloud.google.com/compute/docs/disks/set-persistent-device-name-in-linux-vm var errImageRefNotSpecified = fmt.Errorf("%s is not specified in the custom metadata", imageRefKey) // EnvVar represent a single environment variable key/value pair. @@ -288,16 +294,24 @@ func processMount(singleMount string) (specs.Mount, error) { } return specs.Mount{Type: mountTypeTmpfs, Destination: mntDst}, nil case mountTypeGCEDisk: - mntSrc, okSrc := mntConfig[mountSourceKey] - mntDst, okDst := mntConfig[mountDestinationKey] - if !(okSrc && okDst) { - return specs.Mount{}, fmt.Errorf("found bad bind mount config %v, expect exact keys [%v, %v, %v]", mountOpts, mountTypeKey, mountSourceKey, mountDestinationKey) - } - // TODO: check valid sources with prefix /dev/disk/by-id/google-*. - // overwrite the source with integrity fs .sh - return specs.Mount{Type: mountTypeBind, Source: mntSrc, Destination: mntDst, Options: []string{"rbind", "rw"}}, nil + return specs.Mount{}, fmt.Errorf("found unknown or unspecified mount type: %v, expect one of types [%v]", mountOpts, mountTypeTmpfs) + /* + underlyingSrc, okSrc := mntConfig[mountSourceKey] + mntDst, okDst := mntConfig[mountDestinationKey] + if !(okSrc && okDst) { + return specs.Mount{}, fmt.Errorf("found bad gcedisk mount config %v, expect exact keys [%v, %v, %v]", mountOpts, mountTypeKey, mountSourceKey, mountDestinationKey) + } + + log.Println(underlyingSrc, mntDst) + log.Println("underlyingSrc, mntDst") + mntSrc, err := processGCEDiskMount(underlyingSrc) + if err != nil { + return specs.Mount{}, fmt.Errorf("failed to create integrity FS from %v: %v", underlyingSrc, err) + } + return specs.Mount{Type: mountTypeBind, Source: mntSrc, Destination: mntDst, Options: []string{"rbind", "rw"}}, nil + */ default: - return specs.Mount{}, fmt.Errorf("found unknown or unspecified mount type: %v, expect one of types [%v, %v]", mountOpts, mountTypeTmpfs, mountTypeGCEDisk) + return specs.Mount{}, fmt.Errorf("found unknown or unspecified mount type: %v, expect one of types [%v]", mountOpts, mountTypeTmpfs) } } @@ -330,3 +344,53 @@ func readCmdline() (string, error) { } return string(kernelCmd), nil } + +// processGCEDiskMount takes a GCE attached disk mount type, creates an +// integrity file-system on the disk, and then mounts the file-system. +// It returns the mount path on success. +func processGCEDiskMount(attachedDiskName string) (string, error) { + diskNamePattern := "^[a-zA-Z0-9-]*$" + validNameRegexp := regexp.MustCompile(diskNamePattern) + if !validNameRegexp.MatchString(attachedDiskName) { + return "", fmt.Errorf("got malformed disk name: must match regexp \"%v\"", diskNamePattern) + } + diskSymlink := "/dev/disk/by-id/google-" + attachedDiskName + diskInfo, err := os.Stat(diskSymlink) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("could not find specified disk at %v", diskSymlink) + } + return "", fmt.Errorf("failed to stat file %v: %w", diskSymlink, err) + } + if diskInfo.Mode().Type() != os.ModeDevice { + return "", fmt.Errorf("found a non-device file %v of type: %v", diskSymlink, diskInfo.Mode()) + } + + underlyingDev, err := filepath.EvalSymlinks(diskSymlink) + if err != nil { + return "", fmt.Errorf("failed to eval symlink for %v", diskSymlink) + } + log.Println(underlyingDev) + cmd, err := exec.Command("/usr/bin/cgpt", "create", underlyingDev).CombinedOutput() + log.Printf("cmd: %v", string(cmd)) + if err != nil { + return "", fmt.Errorf("failed to setup integrity fs: %v, output: %v", err, cmd) + } + + cmd, err = exec.Command("/var/lib/google/setup-integrity-fs-v2.sh", "m", underlyingDev, attachedDiskName+"2").CombinedOutput() + log.Printf("cmd: %v", string(cmd)) + if err != nil { + return "", fmt.Errorf("failed to setup integrity fs: %v", err) + } + + mntTgt := path.Join("/mnt", attachedDiskName) + if err := os.Mkdir(mntTgt, 0777); err != nil { + return "", fmt.Errorf("failed to make mount point for %v: %w", attachedDiskName, err) + } + mntSrc := path.Join("/dev/mapper", attachedDiskName) + if err := syscall.Mount(mntSrc, mntTgt, "ext4", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { + return "", fmt.Errorf("failed to mount on attached disk %v ext4 fs: %w", attachedDiskName, err) + } + + return mntTgt, nil +}