From 6da762415fcbcd727167344fdb226e0b8868dd60 Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Tue, 7 May 2024 19:07:04 -0700 Subject: [PATCH 1/3] Add configurable /dev/shm size for CS workloads CS operators can use tee-dev-shm-size to increase the size of the /dev/shm mount for the workload. --- launcher/container_runner.go | 1 + launcher/spec/launch_spec.go | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/launcher/container_runner.go b/launcher/container_runner.go index f08e83ce..1c894420 100644 --- a/launcher/container_runner.go +++ b/launcher/container_runner.go @@ -156,6 +156,7 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To oci.WithHostNamespace(specs.NetworkNamespace), oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}), withRlimits(rlimits), + oci.WithDevShmSize(launchSpec.DevShmSize), ), ) if err != nil { diff --git a/launcher/spec/launch_spec.go b/launcher/spec/launch_spec.go index c3714ca7..037ed775 100644 --- a/launcher/spec/launch_spec.go +++ b/launcher/spec/launch_spec.go @@ -15,6 +15,8 @@ import ( "github.com/google/go-tpm-tools/verifier/util" ) +const MaxInt64 = int(^uint64(0) >> 1) + // RestartPolicy is the enum for the container restart policy. type RestartPolicy string @@ -68,6 +70,7 @@ const ( attestationServiceAddrKey = "tee-attestation-service-endpoint" logRedirectKey = "tee-container-log-redirect" memoryMonitoringEnable = "tee-monitoring-memory-enable" + devShmSizeKey = "tee-dev-shm-size" ) const ( @@ -98,6 +101,7 @@ type LaunchSpec struct { Hardened bool MemoryMonitoringEnabled bool LogRedirect LogRedirectLocation + DevShmSize int64 Experiments experiments.Experiments } @@ -164,6 +168,23 @@ func (s *LaunchSpec) UnmarshalJSON(b []byte) error { s.AttestationServiceAddr = unmarshaledMap[attestationServiceAddrKey] + if val, ok := unmarshaledMap[devShmSizeKey]; ok && val != "" { + size, err := strconv.ParseUint(val, 10, 64) + if err != nil { + return fmt.Errorf("failed to convert %v into uint64, got: %v", devShmSizeKey, val) + } + freeMem, err := getLinuxFreeMem() + if err != nil { + return err + } + if size > freeMem { + return fmt.Errorf("got a /dev/shm size (%v) larger than free memory (%v)", size, freeMem) + } + if size > uint64(MaxInt64) { + return fmt.Errorf("got a size greater than max int64: %v", val) + } + s.DevShmSize = int64(size) + } return nil } @@ -210,6 +231,28 @@ func isHardened(kernelCmd string) bool { return false } +func getLinuxFreeMem() (uint64, error) { + meminfo, err := os.ReadFile("/proc/meminfo") + if err != nil { + return 0, fmt.Errorf("failed to read /proc/meminfo: %w", err) + } + for _, memtype := range strings.Split(string(meminfo), "\n") { + if !strings.Contains(memtype, "MemFree") { + continue + } + split := strings.Fields(memtype) + if len(split) != 3 { + return 0, fmt.Errorf("found invalid MemInfo entry: got: %v, expected format: MemFree: kB", memtype) + } + freeMem, err := strconv.ParseUint(split[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to convert MemFree to uint64: %v", memtype) + } + return freeMem, nil + } + return 0, fmt.Errorf("failed to find MemFree in /proc/meminfo: %v", string(meminfo)) +} + func readCmdline() (string, error) { kernelCmd, err := os.ReadFile("/proc/cmdline") if err != nil { From d0f5bcd8680a6f08e98ef46385d6dc7ff3132e39 Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Wed, 8 May 2024 23:12:55 -0700 Subject: [PATCH 2/3] [launcher] Add tmpfs mount and launch policy Add tmpfs mount support, with destination and size options. Add launchermount package and LaunchPolicy for AllowedMountDestinations. launchermount has a new Mount interface and a TmpfsMount type. LaunchPolicy now supports tee.launch_policy.allow_mount_destination, which specifies allowed parent filepaths to mount on via a PATH-like string. --- go.work.sum | 16 +- launcher/container_runner.go | 6 +- .../image/testworkloads/mounts/Dockerfile | 10 + .../testworkloads/mounts/print_mounts.sh | 9 + launcher/internal/launchermount/mount.go | 30 +++ launcher/internal/launchermount/tmpfs.go | 80 +++++++ launcher/internal/launchermount/tmpfs_test.go | 226 ++++++++++++++++++ launcher/spec/launch_policy.go | 62 ++++- launcher/spec/launch_policy_test.go | 106 ++++++++ launcher/spec/launch_spec.go | 118 +++++++-- launcher/spec/launch_spec_test.go | 129 +++++++++- 11 files changed, 760 insertions(+), 32 deletions(-) create mode 100644 launcher/image/testworkloads/mounts/Dockerfile create mode 100755 launcher/image/testworkloads/mounts/print_mounts.sh create mode 100644 launcher/internal/launchermount/mount.go create mode 100644 launcher/internal/launchermount/tmpfs.go create mode 100644 launcher/internal/launchermount/tmpfs_test.go diff --git a/go.work.sum b/go.work.sum index eb0c4c86..5b36a8f9 100644 --- a/go.work.sum +++ b/go.work.sum @@ -286,6 +286,7 @@ cloud.google.com/go/gkemulticloud v1.0.1/go.mod h1:AcrGoin6VLKT/fwZEYuqvVominLri cloud.google.com/go/gkemulticloud v1.1.1/go.mod h1:C+a4vcHlWeEIf45IB5FFR5XGjTeYhF83+AYIpTy4i2Q= cloud.google.com/go/gkemulticloud v1.1.3/go.mod h1:4WzfPnsOfdCIj6weekE5FIGCaeQKZ1HzGNUVZ1PpIxw= cloud.google.com/go/gkemulticloud v1.2.2/go.mod h1:VMsMYDKpUVYNrhese31TVJMVXPLEtFT/AnIarqlcwVo= +cloud.google.com/go/grafeas v0.3.6/go.mod h1:to6ECAPgRO2xeqD8ISXHc70nObJuaKZThreQOjeOH3o= cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= cloud.google.com/go/gsuiteaddons v1.6.2/go.mod h1:K65m9XSgs8hTF3X9nNTPi8IQueljSdYo9F+Mi+s4MyU= cloud.google.com/go/gsuiteaddons v1.6.5/go.mod h1:Lo4P2IvO8uZ9W+RaC6s1JVxo42vgy+TX5a6hfBZ0ubs= @@ -297,7 +298,6 @@ cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCta cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= -cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/iap v1.9.1/go.mod h1:SIAkY7cGMLohLSdBR25BuIxO+I4fXJiL06IBL7cy/5Q= cloud.google.com/go/iap v1.9.4/go.mod h1:vO4mSq0xNf/Pu6E5paORLASBwEmphXEjgCFg7aeNu1w= @@ -332,16 +332,12 @@ cloud.google.com/go/logging v1.4.2 h1:Mu2Q75VBDQlW1HlBMjTX4X84UFR73G1TiLlRYc/b7t cloud.google.com/go/logging v1.4.2/go.mod h1:jco9QZSx8HiVVqLJReq7z7bVdj0P1Jb9PDFs63T+axo= cloud.google.com/go/logging v1.8.1 h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU= cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= -cloud.google.com/go/logging v1.10.0 h1:f+ZXMqyrSJ5vZ5pE/zr0xC8y/M9BLNzQeLBwfeZ+wY4= -cloud.google.com/go/logging v1.10.0/go.mod h1:EHOwcxlltJrYGqMGfghSet736KR3hX1MAj614mrMk9I= cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= cloud.google.com/go/longrunning v0.5.2 h1:u+oFqfEwwU7F9dIELigxbe0XVnBAo9wqMuQLA50CZ5k= cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= -cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= -cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= cloud.google.com/go/managedidentities v1.6.2/go.mod h1:5c2VG66eCa0WIq6IylRk3TBW83l161zkFvCj28X7jn8= cloud.google.com/go/managedidentities v1.6.5/go.mod h1:fkFI2PwwyRQbjLxlm5bQ8SjtObFMW3ChBGNqaMcgZjI= cloud.google.com/go/managedidentities v1.6.7/go.mod h1:UzslJgHnc6luoyx2JV19cTCi2Fni/7UtlcLeSYRzTV8= @@ -560,6 +556,7 @@ github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3 github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= @@ -572,6 +569,7 @@ github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4Rq github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= @@ -686,6 +684,7 @@ github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -712,6 +711,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4Zs github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/intel/goresctrl v0.3.0/go.mod h1:fdz3mD85cmP9sHD8JUlrNWAxvwM86CrbmVXltEKd7zk= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= @@ -731,6 +731,7 @@ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbq github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -768,6 +769,7 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -789,6 +791,7 @@ github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZ github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= @@ -814,6 +817,8 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= @@ -967,7 +972,6 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= diff --git a/launcher/container_runner.go b/launcher/container_runner.go index 1c894420..6775a61c 100644 --- a/launcher/container_runner.go +++ b/launcher/container_runner.go @@ -82,7 +82,11 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To return nil, err } - mounts := make([]specs.Mount, 0) + mounts := make([]specs.Mount, 0, len(launchSpec.Mounts)+1) + + for _, lsMnt := range launchSpec.Mounts { + mounts = append(mounts, lsMnt.SpecsMount()) + } mounts = appendTokenMounts(mounts) envs, err := formatEnvVars(launchSpec.Envs) diff --git a/launcher/image/testworkloads/mounts/Dockerfile b/launcher/image/testworkloads/mounts/Dockerfile new file mode 100644 index 00000000..2f5ca00a --- /dev/null +++ b/launcher/image/testworkloads/mounts/Dockerfile @@ -0,0 +1,10 @@ +# From current directory: +# gcloud builds submit --tag us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/mounts_workload:latest +FROM alpine + +COPY print_mounts.sh / + +LABEL "tee.launch_policy.log_redirect"="always" + +ENTRYPOINT ["/print_mounts.sh"] + diff --git a/launcher/image/testworkloads/mounts/print_mounts.sh b/launcher/image/testworkloads/mounts/print_mounts.sh new file mode 100755 index 00000000..3b3c75ad --- /dev/null +++ b/launcher/image/testworkloads/mounts/print_mounts.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +df -h + +ls -lathr / + +ls -lathr /my-new-disk + +mkdir /my-new-disk/sldifj \ No newline at end of file diff --git a/launcher/internal/launchermount/mount.go b/launcher/internal/launchermount/mount.go new file mode 100644 index 00000000..6c87c428 --- /dev/null +++ b/launcher/internal/launchermount/mount.go @@ -0,0 +1,30 @@ +// Package launchermount defines mount types for the launcher workload. +package launchermount + +import "github.com/opencontainers/runtime-spec/specs-go" + +// Key-value constants for mount configurations. +// Keys are used to specify the specific mount configuration. +// For example, TypeKey is used to specify the type of mount. +// Consts not suffixed with Key are constant values for given mount configs. +const ( + TypeKey = "type" + SourceKey = "source" + DestinationKey = "destination" + SizeKey = "size" + TypeTmpfs = "tmpfs" +) + +var ( + // AllMountKeys are all possible mount configuration key names. + AllMountKeys = []string{TypeKey, SourceKey, DestinationKey, SizeKey} +) + +// Mount is the interface to implement for a new container launcher mount type. +type Mount interface { + // SpecsMount converts the Mount type to an OCI spec Mount. + SpecsMount() specs.Mount + // The absolute path mount point for this mount in the container. + // Stored as Destination in specs.Mount. + Mountpoint() string +} diff --git a/launcher/internal/launchermount/tmpfs.go b/launcher/internal/launchermount/tmpfs.go new file mode 100644 index 00000000..26dc54ae --- /dev/null +++ b/launcher/internal/launchermount/tmpfs.go @@ -0,0 +1,80 @@ +package launchermount + +import ( + "errors" + "fmt" + "path/filepath" + "strconv" + + "github.com/opencontainers/runtime-spec/specs-go" +) + +var errTmpfsMustHaveDest = errors.New("mount type \"tmpfs\" must have destination specified") + +// TmpfsMount creates a launcher mount type backed by tmpfs, with an optional +// size. If size is not specified, it is 50% of memory. +// Example input: `type=tmpfs,source=tmpfs,destination=/tmpmount` +// `type=tmpfs,source=tmpfs,destination=/sizedtmpmount,size=123345` +type TmpfsMount struct { + // If the path is relative, it will be interpreted as relative to "/". + Destination string + // Size in bytes. No support for k, m, g suffixes. + Size uint64 +} + +// CreateTmpfsMount takes a map of tmpfs options, with keys defined in the spec package. +// Typically, this is called when creating a LaunchSpec and should not be used +// in other settings. +func CreateTmpfsMount(mountMap map[string]string) (TmpfsMount, error) { + if val := mountMap[TypeKey]; val != TypeTmpfs { + return TmpfsMount{}, fmt.Errorf("received wrong mount type %v, expected %v", val, TypeTmpfs) + } + delete(mountMap, TypeKey) + + if val := mountMap[SourceKey]; val != TypeTmpfs { + return TmpfsMount{}, fmt.Errorf("received wrong mount source %v, expected %v", val, TypeTmpfs) + } + delete(mountMap, SourceKey) + + dst := mountMap[DestinationKey] + if dst == "" { + return TmpfsMount{}, errTmpfsMustHaveDest + } + if !filepath.IsAbs(dst) { + dst = filepath.Join("/", dst) + } + delete(mountMap, DestinationKey) + mnt := TmpfsMount{Destination: dst} + + szStr, ok := mountMap[SizeKey] + if ok { + sz, err := strconv.ParseUint(szStr, 10, 64) + if err != nil { + return TmpfsMount{}, fmt.Errorf("failed to convert size option \"%v\" to uint64: %v", szStr, err) + } + mnt.Size = sz + delete(mountMap, SizeKey) + } + + if len(mountMap) != 0 { + return TmpfsMount{}, fmt.Errorf("received unknown mount options for tmpfs mount: %+v", mountMap) + } + return mnt, nil +} + +// SpecsMount returns the OCI runtime spec Mount for the given TmpfsMount. +func (tm TmpfsMount) SpecsMount() specs.Mount { + specsMnt := specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: tm.Destination, + Options: []string{"nosuid", "noexec", "nodev"}} + if tm.Size != 0 { + specsMnt.Options = append(specsMnt.Options, fmt.Sprintf("size=%s", strconv.FormatUint(tm.Size, 10))) + } + return specsMnt +} + +// Mountpoint gives the place in the container where the tmpfs is mounted. +func (tm TmpfsMount) Mountpoint() string { + return tm.Destination +} diff --git a/launcher/internal/launchermount/tmpfs_test.go b/launcher/internal/launchermount/tmpfs_test.go new file mode 100644 index 00000000..b1f60638 --- /dev/null +++ b/launcher/internal/launchermount/tmpfs_test.go @@ -0,0 +1,226 @@ +package launchermount + +import ( + "regexp" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestCreateTmpfsMountAndSpecsMount(t *testing.T) { + var testCases = []struct { + testName string + mountMap map[string]string + expectedTmpfs TmpfsMount + expectedSpecsMount specs.Mount + }{ + { + "Basic Tmpfs Mount", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "/d", + }, + TmpfsMount{Destination: "/d"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/d", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + { + "Tmpfs Mount with Size", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "/my dest", + "size": "21342314", + }, + TmpfsMount{Destination: "/my dest", Size: 21342314}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/my dest", + Options: []string{"nosuid", "noexec", "nodev", "size=21342314"}, + }, + }, + { + "Tmpfs Mount with Relative Dst", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "my dest", + "size": "21342314", + }, + TmpfsMount{Destination: "/my dest", Size: 21342314}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/my dest", + Options: []string{"nosuid", "noexec", "nodev", "size=21342314"}, + }, + }, + { + "Tmpfs Mount with Relative Dst More Complex Filepath", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "grandparent dir/parentDir/my dest", + }, + TmpfsMount{Destination: "/grandparent dir/parentDir/my dest"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/grandparent dir/parentDir/my dest", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + { + "Tmpfs Mount with Dst Internal Rel Parent", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "grandparent dir/parentDir/../../my dest", + }, + TmpfsMount{Destination: "/my dest"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/my dest", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + { + "Tmpfs Mount with Relative Dst Internal Cwd", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "grandparent dir/parentDir/.././my dest", + }, + TmpfsMount{Destination: "/grandparent dir/my dest"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/grandparent dir/my dest", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + { + "Tmpfs Mount with Malformed Relative Dst", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "grandparent dir/parentDir/.../.../my dest", + }, + TmpfsMount{Destination: "/grandparent dir/parentDir/.../.../my dest"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/grandparent dir/parentDir/.../.../my dest", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + { + "Tmpfs Mount with Parent Relative Dst", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "../my dest", + }, + TmpfsMount{Destination: "/my dest"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/my dest", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + { + "Tmpfs Mount with Grandparent Relative Dst", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "../../my dest", + }, + TmpfsMount{Destination: "/my dest"}, + specs.Mount{Type: TypeTmpfs, + Source: TypeTmpfs, + Destination: "/my dest", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + }, + } + for _, testcase := range testCases { + t.Run(testcase.testName, func(t *testing.T) { + mnt, err := CreateTmpfsMount(testcase.mountMap) + if err != nil { + t.Errorf("got non-nil error %v, want nil error", err) + } + if diff := cmp.Diff(mnt, testcase.expectedTmpfs); diff != "" { + t.Errorf("got %v, want %v:\ndiff: %v", mnt, testcase.expectedTmpfs, diff) + } + spMnt := mnt.SpecsMount() + if diff := cmp.Diff(spMnt, testcase.expectedSpecsMount); diff != "" { + t.Errorf("got %v, want %v:\ndiff: %v", spMnt, testcase.expectedSpecsMount, diff) + } + }) + } +} + +func TestCreateTmpfsMountFail(t *testing.T) { + var testCases = []struct { + testName string + mountMap map[string]string + wantErr string + }{ + { + "Bad Mount Type", + map[string]string{ + "type": "tfs", + }, + "received wrong mount type", + }, + { + "Bad Mount Src", + map[string]string{ + "type": "tmpfs", + "source": "tfffffs", + }, + "received wrong mount source", + }, + { + "No Dest", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + }, + errTmpfsMustHaveDest.Error(), + }, + { + "Bad Size", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "dst", + "size": "notanum", + }, + "failed to convert size option", + }, + { + "Unknown Opts", + map[string]string{ + "type": "tmpfs", + "source": "tmpfs", + "destination": "dst", + "size": "111", + "rw": "true", + }, + "received unknown mount options for tmpfs mount", + }, + } + for _, testcase := range testCases { + t.Run(testcase.testName, func(t *testing.T) { + if _, err := CreateTmpfsMount(testcase.mountMap); err == nil { + t.Errorf("got nil error, want non-nil error \"%v\"", testcase.wantErr) + } else { + if match, _ := regexp.MatchString(testcase.wantErr, err.Error()); !match { + t.Errorf("got error \"%v\", but expected \"%v\"", err, testcase.wantErr) + } + } + }) + } +} diff --git a/launcher/spec/launch_policy.go b/launcher/spec/launch_policy.go index 901a6db5..71467eef 100644 --- a/launcher/spec/launch_policy.go +++ b/launcher/spec/launch_policy.go @@ -1,7 +1,9 @@ package spec import ( + "errors" "fmt" + "path/filepath" "strconv" "strings" ) @@ -9,10 +11,11 @@ import ( // LaunchPolicy contains policies on starting the container. // The policy comes from the labels of the image. type LaunchPolicy struct { - AllowedEnvOverride []string - AllowedCmdOverride bool - AllowedLogRedirect policy - AllowedMemoryMonitoring policy + AllowedEnvOverride []string + AllowedCmdOverride bool + AllowedLogRedirect policy + AllowedMemoryMonitoring policy + AllowedMountDestinations []string } type policy int @@ -58,6 +61,11 @@ const ( cmdOverride = "tee.launch_policy.allow_cmd_override" logRedirect = "tee.launch_policy.log_redirect" memoryMonitoring = "tee.launch_policy.monitoring_memory_allow" + // Values look like a PATH list, with ':' as a separator. + // Empty paths will be ignored and relative paths will be interpreted as + // relative to "/". + // Paths will be cleaned using filepath.Clean. + mountDestinations = "tee.launch_policy.allow_mount_destinations" ) // GetLaunchPolicy takes in a map[string] string which should come from image labels, @@ -97,6 +105,18 @@ func GetLaunchPolicy(imageLabels map[string]string) (LaunchPolicy, error) { } } + if v, ok := imageLabels[mountDestinations]; ok { + + paths := filepath.SplitList(v) + for _, path := range paths { + // Strip out empty path name. + if path != "" { + path = filepath.Clean(path) + launchPolicy.AllowedMountDestinations = append(launchPolicy.AllowedMountDestinations, path) + } + } + } + return launchPolicy, nil } @@ -128,9 +148,43 @@ func (p LaunchPolicy) Verify(ls LaunchSpec) error { return fmt.Errorf("memory monitoring only allowed on debug environment by image") } + var err error + for _, mnt := range ls.Mounts { + err = errors.Join(err, p.verifyMountDestination(mnt.Mountpoint())) + } + if err != nil { + return fmt.Errorf("destination mount points are not allowed: %v", err) + } + return nil } +// verifyMountDestination assumes AllowedMountDestinations contains +// `filepath.Clean`ed paths. +func (p LaunchPolicy) verifyMountDestination(dstPath string) error { + if !filepath.IsAbs(dstPath) { + return fmt.Errorf("received a non-absolute destination path: %v", dstPath) + } + dstPath = filepath.Clean(dstPath) + for _, allowDst := range p.AllowedMountDestinations { + if !filepath.IsAbs(allowDst) { + return fmt.Errorf("received a non-absolute allowed destination path: %v", allowDst) + } + rel, err := filepath.Rel(allowDst, dstPath) + if err != nil { + return err + } + + // If dest is not the parent dir relative to the allowed mountpoint + // or dest is not relative from the allowed's parent directory, then + // dest must be a child (or the exact same directory). + if rel != ".." && !strings.HasPrefix(rel, "../") { + return nil + } + } + return fmt.Errorf("destination mount point \"%v\" is invalid: policy only allows mounts in the following paths: %v", dstPath, p.AllowedMountDestinations) +} + func contains(strs []string, target string) bool { for _, s := range strs { if s == target { diff --git a/launcher/spec/launch_policy_test.go b/launcher/spec/launch_policy_test.go index c967e09a..dc938272 100644 --- a/launcher/spec/launch_policy_test.go +++ b/launcher/spec/launch_policy_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-tpm-tools/launcher/internal/launchermount" ) func TestLaunchPolicy(t *testing.T) { @@ -525,6 +526,111 @@ func TestVerify(t *testing.T) { }, false, }, + { + "allowed mount dest", + LaunchPolicy{ + AllowedMountDestinations: []string{"/a"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/a/b"}, + }, + }, + false, + }, + { + "allowed mount dest same dir", + LaunchPolicy{ + AllowedMountDestinations: []string{"/a"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/a"}, + }, + }, + false, + }, + { + "allowed mount dest multiple", + LaunchPolicy{ + AllowedMountDestinations: []string{"/a", "/b", "/c/d"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/a"}, + launchermount.TmpfsMount{Destination: "/b"}, + launchermount.TmpfsMount{Destination: "/c/d"}, + launchermount.TmpfsMount{Destination: "/a/b"}, + launchermount.TmpfsMount{Destination: "/a/b/c"}, + launchermount.TmpfsMount{Destination: "/c/d/e"}, + launchermount.TmpfsMount{Destination: "/c/d/f"}, + launchermount.TmpfsMount{Destination: "/c/d/e/f/g/../b"}, + launchermount.TmpfsMount{Destination: "/c/d/e/f/./../b"}, + launchermount.TmpfsMount{Destination: "/c/d/e/f/./../../b"}, + }, + }, + false, + }, + { + "mount dest relative", + LaunchPolicy{ + AllowedMountDestinations: []string{"/b"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/a/../b"}, + }, + }, + false, + }, + { + "mount dest not abs", + LaunchPolicy{ + AllowedMountDestinations: []string{"/as"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "asd"}, + }, + }, + true, + }, + { + "allowed mount dest not abs", + LaunchPolicy{ + AllowedMountDestinations: []string{"as"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/asd"}, + }, + }, + true, + }, + { + "mount dest prefix but not subdir", + LaunchPolicy{ + AllowedMountDestinations: []string{"/a"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/abcd"}, + }, + }, + true, + }, + { + "mount dest parent of allowed", + LaunchPolicy{ + AllowedMountDestinations: []string{"/a/b"}, + }, + LaunchSpec{ + Mounts: []launchermount.Mount{ + launchermount.TmpfsMount{Destination: "/a"}, + }, + }, + true, + }, } for _, testCase := range testCases { t.Run(testCase.testName, func(t *testing.T) { diff --git a/launcher/spec/launch_spec.go b/launcher/spec/launch_spec.go index 037ed775..3e35c626 100644 --- a/launcher/spec/launch_spec.go +++ b/launcher/spec/launch_spec.go @@ -5,17 +5,22 @@ package spec import ( "context" "encoding/json" + "errors" "fmt" "os" "strconv" "strings" "cloud.google.com/go/compute/metadata" + + "github.com/google/go-tpm-tools/cel" "github.com/google/go-tpm-tools/launcher/internal/experiments" + "github.com/google/go-tpm-tools/launcher/internal/launchermount" "github.com/google/go-tpm-tools/verifier/util" ) -const MaxInt64 = int(^uint64(0) >> 1) +// MaxInt64 is the maximum value of a signed int64. +const MaxInt64 = 9223372036854775807 // RestartPolicy is the enum for the container restart policy. type RestartPolicy string @@ -70,7 +75,8 @@ const ( attestationServiceAddrKey = "tee-attestation-service-endpoint" logRedirectKey = "tee-container-log-redirect" memoryMonitoringEnable = "tee-monitoring-memory-enable" - devShmSizeKey = "tee-dev-shm-size" + devShmSizeKey = "tee-dev-shm-size-kb" + mountKey = "tee-mount" ) const ( @@ -101,8 +107,10 @@ type LaunchSpec struct { Hardened bool MemoryMonitoringEnabled bool LogRedirect LogRedirectLocation - DevShmSize int64 - Experiments experiments.Experiments + Mounts []launchermount.Mount + // DevShmSize is specified in kiB. + DevShmSize int64 + Experiments experiments.Experiments } // UnmarshalJSON unmarshals an instance attributes list in JSON format from the metadata @@ -119,7 +127,7 @@ func (s *LaunchSpec) UnmarshalJSON(b []byte) error { } s.RestartPolicy = RestartPolicy(unmarshaledMap[restartPolicyKey]) - // set the default restart policy to "Never" for now + // Set the default restart policy to "Never" for now. if s.RestartPolicy == "" { s.RestartPolicy = Never } @@ -143,14 +151,14 @@ func (s *LaunchSpec) UnmarshalJSON(b []byte) error { } } - // populate cmd override + // Populate cmd override. if val, ok := unmarshaledMap[cmdKey]; ok && val != "" { if err := json.Unmarshal([]byte(val), &s.Cmd); err != nil { return err } } - // populate all env vars + // Populate all env vars. for k, v := range unmarshaledMap { if strings.HasPrefix(k, envKeyPrefix) { s.Envs = append(s.Envs, EnvVar{strings.TrimPrefix(k, envKeyPrefix), v}) @@ -168,23 +176,29 @@ func (s *LaunchSpec) UnmarshalJSON(b []byte) error { s.AttestationServiceAddr = unmarshaledMap[attestationServiceAddrKey] + // Populate /dev/shm size override. if val, ok := unmarshaledMap[devShmSizeKey]; ok && val != "" { size, err := strconv.ParseUint(val, 10, 64) if err != nil { return fmt.Errorf("failed to convert %v into uint64, got: %v", devShmSizeKey, val) } - freeMem, err := getLinuxFreeMem() - if err != nil { - return err - } - if size > freeMem { - return fmt.Errorf("got a /dev/shm size (%v) larger than free memory (%v)", size, freeMem) - } - if size > uint64(MaxInt64) { - return fmt.Errorf("got a size greater than max int64: %v", val) - } s.DevShmSize = int64(size) } + + // Populate mount override. + // https://cloud.google.com/compute/docs/disks/set-persistent-device-name-in-linux-vm + // https://cloud.google.com/compute/docs/disks/add-local-ssd + if val, ok := unmarshaledMap[mountKey]; ok && val != "" { + mounts := strings.Split(val, ";") + for _, mount := range mounts { + specMnt, err := processMount(mount) + if err != nil { + return err + } + s.Mounts = append(s.Mounts, specMnt) + } + } + return nil } @@ -203,6 +217,20 @@ func GetLaunchSpec(ctx context.Context, client *metadata.Client) (LaunchSpec, er return LaunchSpec{}, err } + var errs []error + for _, mnt := range spec.Mounts { + if err := validateMount(mnt); err != nil { + errs = append(errs, err) + } + } + if len(errs) != 0 { + return LaunchSpec{}, fmt.Errorf("failed to validate mounts: %v", errors.Join(errs...)) + } + + if err := validateMemorySizeKb(uint64(spec.DevShmSize)); err != nil { + return LaunchSpec{}, fmt.Errorf("failed to validate /dev/shm size: %v", err) + } + spec.ProjectID, err = client.ProjectIDWithContext(ctx) if err != nil { return LaunchSpec{}, fmt.Errorf("failed to retrieve projectID from MDS: %v", err) @@ -231,6 +259,59 @@ func isHardened(kernelCmd string) bool { return false } +func processMount(singleMount string) (launchermount.Mount, error) { + mntConfig := make(map[string]string) + var mntType string + mountOpts := strings.Split(singleMount, ",") + for _, mountOpt := range mountOpts { + name, val, err := cel.ParseEnvVar(mountOpt) + if err != nil { + return nil, fmt.Errorf("failed to parse mount option: %w", err) + } + switch name { + case launchermount.TypeKey: + mntType = val + case launchermount.SourceKey: + case launchermount.DestinationKey: + case launchermount.SizeKey: + default: + return nil, fmt.Errorf("found unknown mount option: %v, expect keys of %v", mountOpt, launchermount.AllMountKeys) + } + mntConfig[name] = val + } + + switch mntType { + case launchermount.TypeTmpfs: + return launchermount.CreateTmpfsMount(mntConfig) + default: + return nil, fmt.Errorf("found unknown or unspecified mount type: %v, expect one of types [%v]", mountOpts, launchermount.TypeTmpfs) + } +} + +func validateMount(mnt launchermount.Mount) error { + switch v := mnt.(type) { + case launchermount.TmpfsMount: + return validateMemorySizeKb(v.Size / 1024) + default: + return fmt.Errorf("got unknown mount type: %T", v) + } +} + +// Ensures that system free memory is larger than the specified memory size. +func validateMemorySizeKb(memSize uint64) error { + freeMem, err := getLinuxFreeMem() + if err != nil { + return fmt.Errorf("failed to get free memory: %v", err) + } + if memSize > freeMem { + return fmt.Errorf("got a /dev/shm size (%v) larger than free memory (%v) kB", memSize, freeMem) + } + if memSize > MaxInt64 { + return fmt.Errorf("got a size greater than max int64: %v", memSize) + } + return nil +} + func getLinuxFreeMem() (uint64, error) { meminfo, err := os.ReadFile("/proc/meminfo") if err != nil { @@ -244,6 +325,9 @@ func getLinuxFreeMem() (uint64, error) { if len(split) != 3 { return 0, fmt.Errorf("found invalid MemInfo entry: got: %v, expected format: MemFree: kB", memtype) } + if split[2] != "kB" { + return 0, fmt.Errorf("found invalid MemInfo entry: got: %v, expected format: MemFree: kB", memtype) + } freeMem, err := strconv.ParseUint(split[1], 10, 64) if err != nil { return 0, fmt.Errorf("failed to convert MemFree to uint64: %v", memtype) diff --git a/launcher/spec/launch_spec_test.go b/launcher/spec/launch_spec_test.go index f3df6608..73405ba1 100644 --- a/launcher/spec/launch_spec_test.go +++ b/launcher/spec/launch_spec_test.go @@ -1,9 +1,11 @@ package spec import ( + "regexp" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-tpm-tools/launcher/internal/launchermount" ) func TestLaunchSpecUnmarshalJSONHappyCases(t *testing.T) { @@ -21,7 +23,9 @@ func TestLaunchSpecUnmarshalJSONHappyCases(t *testing.T) { "tee-restart-policy":"Always", "tee-impersonate-service-accounts":"sv1@developer.gserviceaccount.com,sv2@developer.gserviceaccount.com", "tee-container-log-redirect":"true", - "tee-monitoring-memory-enable":"true" + "tee-monitoring-memory-enable":"true", + "tee-dev-shm-size-kb":"234234", + "tee-mount":"type=tmpfs,source=tmpfs,destination=/tmpmount;type=tmpfs,source=tmpfs,destination=/sized,size=222" }`, }, { @@ -36,7 +40,9 @@ func TestLaunchSpecUnmarshalJSONHappyCases(t *testing.T) { "tee-restart-policy":"Always", "tee-impersonate-service-accounts":"sv1@developer.gserviceaccount.com,sv2@developer.gserviceaccount.com", "tee-container-log-redirect":"true", - "tee-monitoring-memory-enable":"TRUE" + "tee-monitoring-memory-enable":"TRUE", + "tee-dev-shm-size-kb":"234234", + "tee-mount":"type=tmpfs,source=tmpfs,destination=/tmpmount;type=tmpfs,source=tmpfs,destination=/sized,size=222" }`, }, } @@ -50,6 +56,9 @@ func TestLaunchSpecUnmarshalJSONHappyCases(t *testing.T) { ImpersonateServiceAccounts: []string{"sv1@developer.gserviceaccount.com", "sv2@developer.gserviceaccount.com"}, LogRedirect: Everywhere, MemoryMonitoringEnabled: true, + DevShmSize: 234234, + Mounts: []launchermount.Mount{launchermount.TmpfsMount{Destination: "/tmpmount", Size: 0}, + launchermount.TmpfsMount{Destination: "/sized", Size: 222}}, } for _, testcase := range testCases { @@ -123,7 +132,8 @@ func TestLaunchSpecUnmarshalJSONWithDefaultValue(t *testing.T) { "tee-signed-image-repos":"", "tee-container-log-redirect":"", "tee-restart-policy":"", - "tee-monitoring-memory-enable":"" + "tee-monitoring-memory-enable":"", + "tee-mount":"" }` spec := &LaunchSpec{} @@ -152,6 +162,117 @@ func TestLaunchSpecUnmarshalJSONWithoutImageReference(t *testing.T) { spec := &LaunchSpec{} if err := spec.UnmarshalJSON([]byte(mdsJSON)); err == nil || err != errImageRefNotSpecified { - t.Fatalf("got %v error, but expected %v error", err, errImageRefNotSpecified) + t.Errorf("got %v error, but expected %v error", err, errImageRefNotSpecified) + } +} + +func TestLaunchSpecUnmarshalJSONWithTmpfsMounts(t *testing.T) { + var testCases = []struct { + testName string + mdsJSON string + wantDst string + wantSz uint64 + }{ + { + "Empty Mounts", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"" + }`, + "", + 0, + }, + { + "Tmpfs", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source=tmpfs,destination=/tmpmount" + }`, + "/tmpmount", + 0, + }, + { + "Tmpfs Sized", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source=tmpfs,destination=/tmpmount,size=78987" + }`, + "/tmpmount", + 78987, + }, + } + for _, testcase := range testCases { + t.Run(testcase.testName, func(t *testing.T) { + spec := &LaunchSpec{} + if err := spec.UnmarshalJSON([]byte(testcase.mdsJSON)); err != nil { + t.Errorf("got %v error, but expected nil error", err) + } + }) + } +} + +func TestLaunchSpecUnmarshalJSONWithBadMounts(t *testing.T) { + var testCases = []struct { + testName string + mdsJSON string + errMatch string + }{ + { + "Unknown Type", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=hallo" + }`, + "found unknown or unspecified mount type", + }, + { + "Not k=v", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source" + }`, + "failed to parse mount option", + }, + { + "Unknown Option", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source=tmpfs,destination=/tmpmount,size=123,foo=bar" + }`, + "found unknown mount option", + }, + { + "Tmpfs Bad Source", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source=src,destination=/tmpmount" + }`, + "received wrong mount source", + }, + { + "Tmpfs No Destination", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source=tmpfs" + }`, + "mount type \"tmpfs\" must have destination specified", + }, + { + "Tmpfs Size Not Int", + `{ + "tee-image-reference":"docker.io/library/hello-world:latest", + "tee-mount":"type=tmpfs,source=tmpfs,destination=/tmpmount,size=foo" + }`, + "failed to convert size option", + }, + } + for _, testcase := range testCases { + t.Run(testcase.testName, func(t *testing.T) { + spec := &LaunchSpec{} + err := spec.UnmarshalJSON([]byte(testcase.mdsJSON)) + if match, _ := regexp.MatchString(testcase.errMatch, err.Error()); !match { + t.Errorf("got %v error, but expected %v error", err, testcase.errMatch) + } + }) } } From af7d53e6ee5cee342f2ac08727cb94e409d6b2d6 Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Mon, 24 Jun 2024 08:56:20 -0700 Subject: [PATCH 3/3] Add mounts Cloud Build test This tests the allowed mount points for tmpfs mounts and /dev/shm size. --- launcher/cloudbuild.yaml | 16 +++ launcher/image/test/test_log_redirection.yaml | 2 +- launcher/image/test/test_mounts.yaml | 112 ++++++++++++++++++ .../image/testworkloads/mounts/Dockerfile | 1 + .../testworkloads/mounts/print_mounts.sh | 6 +- 5 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 launcher/image/test/test_mounts.yaml diff --git a/launcher/cloudbuild.yaml b/launcher/cloudbuild.yaml index 5be77fdf..ed9b2378 100644 --- a/launcher/cloudbuild.yaml +++ b/launcher/cloudbuild.yaml @@ -242,6 +242,22 @@ steps: gcloud builds submit --config=test_oda_with_signed_container.yaml --region us-west1 \ --substitutions _IMAGE_NAME=${OUTPUT_IMAGE_PREFIX}-hardened-${OUTPUT_IMAGE_SUFFIX},_IMAGE_PROJECT=${PROJECT_ID} exit +- name: 'gcr.io/cloud-builders/gcloud' + id: MountTests + waitFor: ['HardenedImageBuild'] + env: + - 'OUTPUT_IMAGE_PREFIX=$_OUTPUT_IMAGE_PREFIX' + - 'OUTPUT_IMAGE_SUFFIX=$_OUTPUT_IMAGE_SUFFIX' + - 'PROJECT_ID=$PROJECT_ID' + script: | + #!/usr/bin/env bash + cd launcher/image/test + dev_shm_size_kb=$(shuf -i 70000-256000 -n 1) + tmpfs_size_kb=$(shuf -i 256-256000 -n 1) + echo "running mount tests on ${OUTPUT_IMAGE_PREFIX}-hardened-${OUTPUT_IMAGE_SUFFIX}" + gcloud builds submit --config=test_mounts.yaml --region us-west1 \ + --substitutions _IMAGE_NAME=${OUTPUT_IMAGE_PREFIX}-hardened-${OUTPUT_IMAGE_SUFFIX},_IMAGE_PROJECT=${PROJECT_ID} + exit options: pool: name: 'projects/confidential-space-images-dev/locations/us-west1/workerPools/cs-image-build-vpc' diff --git a/launcher/image/test/test_log_redirection.yaml b/launcher/image/test/test_log_redirection.yaml index 64ad5a97..9fea6ba5 100644 --- a/launcher/image/test/test_log_redirection.yaml +++ b/launcher/image/test/test_log_redirection.yaml @@ -3,7 +3,7 @@ substitutions: '_IMAGE_PROJECT': '' '_CLEANUP': 'true' '_VM_NAME_PREFIX': 'cs-logredirect-test' - '_ZONE': 'us-central1-a' + '_ZONE': 'us-west1-a' '_WORKLOAD_IMAGE': 'us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/basic_test:latest' steps: diff --git a/launcher/image/test/test_mounts.yaml b/launcher/image/test/test_mounts.yaml new file mode 100644 index 00000000..d933b99b --- /dev/null +++ b/launcher/image/test/test_mounts.yaml @@ -0,0 +1,112 @@ +substitutions: + '_IMAGE_NAME': '' + '_IMAGE_PROJECT': '' + '_CLEANUP': 'true' + '_VM_NAME_PREFIX': 'cs-mounts-test' + '_ZONE': 'us-west1-a' + '_WORKLOAD_IMAGE': 'us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/mounts_workload:latest' + '_DEV_SHM_SIZE_KB': '128000' + '_TMPFS_SIZE_KB': '222' +steps: +- name: 'gcr.io/cloud-builders/gcloud' + id: CreateVMWithMountsAllowed + entrypoint: 'bash' + env: + - 'BUILD_ID=$BUILD_ID' + - '_DEV_SHM_SIZE_KB=$_DEV_SHM_SIZE_KB' + - '_TMPFS_SIZE_KB=$_TMPFS_SIZE_KB' + args: ['create_vm.sh','-i', '${_IMAGE_NAME}', + '-p', '${_IMAGE_PROJECT}', + '-m', '^~^tee-image-reference=${_WORKLOAD_IMAGE}~tee-container-log-redirect=true~tee-mount=type=tmpfs,source=tmpfs,destination=/tmp/sized,size=${_TMPFS_SIZE_KB}000~tee-dev-shm-size-kb=${_DEV_SHM_SIZE_KB}', + '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}-allowed', + '-z', '${_ZONE}', + ] +- name: 'gcr.io/cloud-builders/gcloud' + id: CreateVMWithMountsDenied + entrypoint: 'bash' + env: + - 'BUILD_ID=$BUILD_ID' + args: ['create_vm.sh','-i', '${_IMAGE_NAME}', + '-p', '${_IMAGE_PROJECT}', + '-m', '^~^tee-image-reference=${_WORKLOAD_IMAGE}~tee-container-log-redirect=true~tee-mount=type=tmpfs,source=tmpfs,destination=/disallowed', + '-n', '${_VM_NAME_PREFIX}-${BUILD_ID}-denied', + '-z', '${_ZONE}', + ] +- name: 'gcr.io/cloud-builders/gcloud' + id: CheckMountsAllowed + env: + - '_VM_NAME_PREFIX=$_VM_NAME_PREFIX' + - 'BUILD_ID=$BUILD_ID' + - '_ZONE=$_ZONE' + - '_DEV_SHM_SIZE_KB=$_DEV_SHM_SIZE_KB' + - '_TMPFS_SIZE_KB=$_TMPFS_SIZE_KB' + script: | + #!/bin/bash + set -euo pipefail + source util/read_serial.sh + + sleep 45 + SERIAL_OUTPUT=$(read_serial ${_VM_NAME_PREFIX}-${BUILD_ID}-allowed ${_ZONE}) + echo $SERIAL_OUTPUT + if echo $SERIAL_OUTPUT | grep -q "tmpfs.*${_TMPFS_SIZE_KB}.*/tmp/sized" + then + echo "- Mount launch policy verified for sized tmpfs" + else + echo "FAILED: Mount launch policy verification for sized tmpfs" + echo 'TEST FAILED' > /workspace/status.txt + echo $SERIAL_OUTPUT + fi + if echo $SERIAL_OUTPUT | grep -q "shm.*${_DEV_SHM_SIZE_KB}.*/dev/shm" + then + echo "- Mount launch policy verified for /dev/shm size" + else + echo "FAILED: Mount launch policy verification for /dev/shm size" + echo 'TEST FAILED' > /workspace/status.txt + echo $SERIAL_OUTPUT + fi + +- name: 'gcr.io/cloud-builders/gcloud' + id: CheckMountsDenied + env: + - '_VM_NAME_PREFIX=$_VM_NAME_PREFIX' + - 'BUILD_ID=$BUILD_ID' + - '_ZONE=$_ZONE' + script: | + #!/bin/bash + set -euo pipefail + source util/read_serial.sh + + sleep 45 + SERIAL_OUTPUT=$(read_serial ${_VM_NAME_PREFIX}-${BUILD_ID}-denied ${_ZONE}) + if echo $SERIAL_OUTPUT | grep -q "policy only allows mounts in the following paths" + then + echo "- Mount launch policy verified for disallowed mounts" + else + echo "FAILED: Mount launch policy verification for disallowed mounts" + echo 'TEST FAILED' > /workspace/status.txt + echo $SERIAL_OUTPUT + fi + + waitFor: ['CreateVMWithMountsDenied'] +- name: 'gcr.io/cloud-builders/gcloud' + id: CleanUpVMWithMountsAllowed + entrypoint: 'bash' + env: + - 'CLEANUP=$_CLEANUP' + args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-allowed', '${_ZONE}'] + waitFor: ['CheckMountsAllowed'] +- name: 'gcr.io/cloud-builders/gcloud' + id: CleanUpVMWithMountsDenied + entrypoint: 'bash' + env: + - 'CLEANUP=$_CLEANUP' + args: ['cleanup.sh', '${_VM_NAME_PREFIX}-${BUILD_ID}-denied', '${_ZONE}'] + waitFor: ['CheckMountsDenied'] + +# Must come after cleanup. +- name: 'gcr.io/cloud-builders/gcloud' + id: CheckFailure + entrypoint: 'bash' + env: + - 'BUILD_ID=$BUILD_ID' + args: ['check_failure.sh'] diff --git a/launcher/image/testworkloads/mounts/Dockerfile b/launcher/image/testworkloads/mounts/Dockerfile index 2f5ca00a..3192d02b 100644 --- a/launcher/image/testworkloads/mounts/Dockerfile +++ b/launcher/image/testworkloads/mounts/Dockerfile @@ -5,6 +5,7 @@ FROM alpine COPY print_mounts.sh / LABEL "tee.launch_policy.log_redirect"="always" +LABEL "tee.launch_policy.allow_mount_destinations"="/run/tmp:/var/tmp:/tmp" ENTRYPOINT ["/print_mounts.sh"] diff --git a/launcher/image/testworkloads/mounts/print_mounts.sh b/launcher/image/testworkloads/mounts/print_mounts.sh index 3b3c75ad..d1315bb8 100755 --- a/launcher/image/testworkloads/mounts/print_mounts.sh +++ b/launcher/image/testworkloads/mounts/print_mounts.sh @@ -1,9 +1,7 @@ #!/bin/sh df -h +df ls -lathr / - -ls -lathr /my-new-disk - -mkdir /my-new-disk/sldifj \ No newline at end of file +ls -lathr /tmp