From 963ff16b4090e2ddfb3ce12323b942257b712ac3 Mon Sep 17 00:00:00 2001 From: ncordon Date: Wed, 23 Oct 2019 11:14:45 +0200 Subject: [PATCH 1/4] Bumps libcontainer version The Init: true flag is needed due to changes introduced from 1.0.0rc5 version of libcontainer to 1.0.0rc6. If we do not provide it, containers cannot find their entrypoint Signed-off-by: ncordon --- daemon/driver.go | 1 + go.mod | 8 +++++--- go.sum | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/daemon/driver.go b/daemon/driver.go index 125b5b9..514557c 100644 --- a/daemon/driver.go +++ b/daemon/driver.go @@ -74,6 +74,7 @@ func NewDriverInstance(r *runtime.Runtime, lang string, i runtime.DriverImage, o Env: o.Env, Stdout: os.Stdout, Stderr: os.Stderr, + Init: true, } tmp := filepath.Join(r.Root, fmt.Sprintf(TmpPathPattern, id)) diff --git a/go.mod b/go.mod index 459e446..b4fa5ab 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/beorn7/perks v1.0.0 // indirect github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5 github.com/cenkalti/backoff v2.1.1+incompatible + github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b // indirect github.com/containerd/console v0.0.0-20180220200639-2748ece16665 // indirect github.com/containers/image v0.0.0-20170921085858-33dad2514b7d github.com/containers/storage v0.0.0-20180220103138-477e551dd493 // indirect @@ -34,9 +35,9 @@ require ( github.com/oklog/ulid v1.3.1 github.com/olekukonko/tablewriter v0.0.0-20170925234030-a7a4c189eb47 github.com/opencontainers/image-spec v1.0.1 - github.com/opencontainers/runc v1.0.0-rc5 + github.com/opencontainers/runc v1.0.0-rc9 github.com/opencontainers/runtime-spec v0.0.0-20170712232009-96de01bbb42c // indirect - github.com/opencontainers/selinux v0.0.0-20170621221121-4a2974bf1ee9 // indirect + github.com/opencontainers/selinux v1.3.0 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/ostreedev/ostree-go v0.0.0-20170727130318-80ab7dbb8986 // indirect github.com/pkg/errors v0.8.1 @@ -48,9 +49,10 @@ require ( github.com/src-d/enry/v2 v2.0.0 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.3.0 - github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect + github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.0.0+incompatible // indirect + github.com/urfave/cli v1.22.1 // indirect github.com/vishvananda/netlink v0.0.0-20170710030406-8c5a115d793b // indirect github.com/vishvananda/netns v0.0.0-20170707011535-86bef332bfc3 // indirect golang.org/x/net v0.0.0-20190628185345-da137c7871d7 diff --git a/go.sum b/go.sum index 3811051..57319c4 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5 h1:osZyZB7J4kE1 github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b h1:T4nWG1TXIxeor8mAu5bFguPJgSIGhZqv/f0z55KCrJM= +github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -47,6 +49,8 @@ github.com/coreos/go-systemd v0.0.0-20170609144627-24036eb3df68 h1:Z80xKolFQLHEJ github.com/coreos/go-systemd v0.0.0-20170609144627-24036eb3df68/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20170420172956-8dbaa491b063 h1:xjjDTql0P7yB/ZNeKvozGwVM0mmX+tPndHwCssWMH+g= github.com/coreos/pkg v0.0.0-20170420172956-8dbaa491b063/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cyphar/filepath-securejoin v0.2.1 h1:5DPkzz/0MwUpvR4fxASKzgApeq2OMFY5FfYtrX28Coo= github.com/cyphar/filepath-securejoin v0.2.1/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -160,10 +164,16 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v1.0.0-rc5 h1:rYjdzMDXVly2Av0RLs3nf/iVkaWh2UrDhuTdTT2KggQ= github.com/opencontainers/runc v1.0.0-rc5/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc6 h1:7AoN22rYxxkmsJS48wFaziH/n0OvrZVqL/TglgHKbKQ= +github.com/opencontainers/runc v1.0.0-rc6/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.0.0-20170712232009-96de01bbb42c h1:2nC8isMK+o8MEaxkQx5hGoRd1vIbPn4BodOzawUaaZA= github.com/opencontainers/runtime-spec v0.0.0-20170712232009-96de01bbb42c/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v0.0.0-20170621221121-4a2974bf1ee9 h1:vVmQZ2IaaEe1MiuvZQbcydTbnlTG0OnZO5/4j7VZv0A= github.com/opencontainers/selinux v0.0.0-20170621221121-4a2974bf1ee9/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= +github.com/opencontainers/selinux v1.3.0 h1:xsI95WzPZu5exzA6JzkLSfdr/DilzOhCJOqGe5TgR0g= +github.com/opencontainers/selinux v1.3.0/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -192,8 +202,12 @@ github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0 h1:c8R11WC8m7KNM github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/seccomp/libseccomp-golang v0.0.0-20170625204121-f6ec81daf48e h1:9O+ppf4lSP+JT6Ym0zIPyy/Ql1lEcU6PAs861rvrzIM= github.com/seccomp/libseccomp-golang v0.0.0-20170625204121-f6ec81daf48e/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -216,6 +230,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/toqueteos/trie v1.0.0 h1:8i6pXxNUXNRAqP246iibb7w/pSFquNTQ+uNfriG7vlk= github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= @@ -230,6 +246,8 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v0.0.0-20170710030406-8c5a115d793b h1:V/rhZov2dVBDmfHI+k2DwFxHlvF8UXWbXST0AaaMl4Q= github.com/vishvananda/netlink v0.0.0-20170710030406-8c5a115d793b/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netns v0.0.0-20170707011535-86bef332bfc3 h1:NcYCJC+LbOrfvuf/uHeM/kxh6vOmiuInC4GAWRdc+P0= From 814dc32f2867d310a1aceb7fafe0f8edf19953eb Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Sat, 3 Mar 2018 15:27:49 +0100 Subject: [PATCH 2/4] run containers in rootless mode Signed-off-by: Denys Smirnov --- runtime/container_test.go | 2 +- runtime/runtime.go | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/runtime/container_test.go b/runtime/container_test.go index 9375fa2..f40ac72 100644 --- a/runtime/container_test.go +++ b/runtime/container_test.go @@ -114,7 +114,7 @@ func (s *ContainerSuite) TestContainer_StartWait() { err = c.Wait() require.NoError(err) - require.Equal("bin\ndev\netc\nhome\nopt\nproc\nroot\nsys\ntmp\nusr\nvar\n", out.String()) + require.Equal("bin\ndev\netc\nhome\nopt\nproc\nroot\ntmp\nusr\nvar\n", out.String()) } func (s *ContainerSuite) TestContainer_StartWaitExit1() { diff --git a/runtime/runtime.go b/runtime/runtime.go index f13d871..c120b58 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -105,12 +105,20 @@ func ContainerConfigFactory(containerID string) *configs.Config { defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV return &configs.Config{ + Rootless: true, Namespaces: configs.Namespaces([]configs.Namespace{ {Type: configs.NEWNS}, {Type: configs.NEWUTS}, {Type: configs.NEWIPC}, {Type: configs.NEWPID}, + {Type: configs.NEWUSER}, }), + UidMappings: []configs.IDMap{ + {ContainerID: 0, HostID: os.Getuid(), Size: 1}, + }, + GidMappings: []configs.IDMap{ + {ContainerID: 0, HostID: os.Getgid(), Size: 1}, + }, Cgroups: &configs.Cgroup{ Name: containerID, Parent: "system", @@ -148,7 +156,7 @@ func ContainerConfigFactory(containerID string) *configs.Config { Destination: "/dev/pts", Device: "devpts", Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, - Data: "newinstance,ptmxmode=0666,mode=0620,gid=5", + Data: "newinstance,ptmxmode=0666,mode=0620", }, { Source: "mqueue", @@ -156,12 +164,12 @@ func ContainerConfigFactory(containerID string) *configs.Config { Device: "mqueue", Flags: defaultMountFlags, }, - { - Source: "sysfs", - Destination: "/sys", - Device: "sysfs", - Flags: defaultMountFlags | syscall.MS_RDONLY, - }, + //{ + // Source: "sysfs", + // Destination: "/sys", + // Device: "sysfs", + // Flags: defaultMountFlags | syscall.MS_RDONLY, + //}, { Source: "/etc/localtime", Destination: "/etc/localtime", From 924c14eb9e31bffb0162759fab588982470447e0 Mon Sep 17 00:00:00 2001 From: ncordon Date: Wed, 23 Oct 2019 12:46:17 +0200 Subject: [PATCH 3/4] Updates rootless params and adds new Init: true requirement Init: true is needed for the first process spawning in a container Signed-off-by: ncordon --- runtime/container_test.go | 7 +++++++ runtime/runtime.go | 11 +++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/runtime/container_test.go b/runtime/container_test.go index f40ac72..54c8296 100644 --- a/runtime/container_test.go +++ b/runtime/container_test.go @@ -53,6 +53,7 @@ func (s *ContainerSuite) TestContainer_Run() { p := &Process{ Args: []string{"/bin/ls"}, Stdout: os.Stdout, + Init: true, } c, err := s.Runtime.Container("run", s.Image, p, nil) @@ -67,6 +68,7 @@ func (s *ContainerSuite) TestContainer_StartStopStart() { p := &Process{ Args: []string{"/bin/sleep", "5m"}, Stdout: os.Stdout, + Init: true, } c, err := s.Runtime.Container("1", s.Image, p, nil) @@ -82,6 +84,7 @@ func (s *ContainerSuite) TestContainer_StartStopStart() { p = &Process{ Args: []string{"/bin/sleep", "5m"}, Stdout: os.Stdout, + Init: true, } c, err = s.Runtime.Container("2", s.Image, p, nil) @@ -103,6 +106,7 @@ func (s *ContainerSuite) TestContainer_StartWait() { p := &Process{ Args: []string{"/bin/ls"}, Stdout: out, + Init: true, } c, err := s.Runtime.Container("wait", s.Image, p, nil) @@ -125,6 +129,7 @@ func (s *ContainerSuite) TestContainer_StartWaitExit1() { p := &Process{ Args: []string{"/bin/false"}, Stdout: out, + Init: true, } c, err := s.Runtime.Container("wait-exit", s.Image, p, nil) @@ -147,6 +152,7 @@ func (s *ContainerSuite) TestContainer_StartFailure() { p := &Process{ Args: []string{"/bin/non-existent"}, Stdout: out, + Init: true, } c, err := s.Runtime.Container("start-failure", s.Image, p, nil) @@ -164,6 +170,7 @@ func (s *ContainerSuite) TestContainer_Env() { p := &Process{ Args: []string{"/bin/env"}, Stdout: out, + Init: true, } c, err := s.Runtime.Container("env", s.Image, p, nil) diff --git a/runtime/runtime.go b/runtime/runtime.go index c120b58..6e1f54a 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -48,7 +48,7 @@ func (r *Runtime) Init() error { var err error r.f, err = libcontainer.New( filepath.Join(r.Root, containersPath), - libcontainer.Cgroupfs, + libcontainer.RootlessCgroupfs, ) return err @@ -105,7 +105,8 @@ func ContainerConfigFactory(containerID string) *configs.Config { defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV return &configs.Config{ - Rootless: true, + RootlessEUID: true, + RootlessCgroups: true, Namespaces: configs.Namespaces([]configs.Namespace{ {Type: configs.NEWNS}, {Type: configs.NEWUTS}, @@ -164,12 +165,6 @@ func ContainerConfigFactory(containerID string) *configs.Config { Device: "mqueue", Flags: defaultMountFlags, }, - //{ - // Source: "sysfs", - // Destination: "/sys", - // Device: "sysfs", - // Flags: defaultMountFlags | syscall.MS_RDONLY, - //}, { Source: "/etc/localtime", Destination: "/etc/localtime", From 65dc49129fe250805af449b2f3f9bc85be2e679d Mon Sep 17 00:00:00 2001 From: ncordon Date: Thu, 24 Oct 2019 16:17:56 +0200 Subject: [PATCH 4/4] Updates documentation and kills containers with SIGKILL Also reindents files with gofmt. If we killed the process with SIGTERM instead of SIGKILL in rootless mode, the containers ignored our order. Also added rootless.md file to explain the rootless configuration for running bblfshd Signed-off-by: ncordon --- README.md | 43 +- bblfshd-seccomp.json | 804 ++++++++++++++++++++++++++++++++++++++ daemon/driver.go | 2 +- rootless.md | 69 ++++ runtime/container.go | 9 +- runtime/container_test.go | 14 +- 6 files changed, 923 insertions(+), 18 deletions(-) create mode 100644 bblfshd-seccomp.json create mode 100644 rootless.md diff --git a/README.md b/README.md index fe1000a..7f417b3 100644 --- a/README.md +++ b/README.md @@ -18,26 +18,57 @@ This project is now part of [source{d} Engine](https://sourced.tech/engine), which provides the simplest way to get started with a single command. Visit [sourced.tech/engine](https://sourced.tech/engine) for more information. +#### Rootless mode + The recommended way to run *bblfshd* by itself is using Docker: ```sh -docker run -d --name bblfshd --privileged -p 9432:9432 -v /var/lib/bblfshd:/var/lib/bblfshd bblfsh/bblfshd +docker run -d --name bblfshd \ + -p 9432:9432 \ + -v /var/lib/bblfshd:/var/lib/bblfshd \ + -v /proc:/newproc \ + --security-opt seccomp=./bblfshd-seccomp.json \ + bblfsh/bblfshd ``` On macOS, use this command instead to use a Docker volume: ```sh -docker run -d --name bblfshd --privileged -p 9432:9432 -v bblfsh-storage:/var/lib/bblfshd bblfsh/bblfshd +docker run -d --name bblfshd \ + -p 9432:9432 \ + -v bblfsh-storage:/var/lib/bblfshd bblfsh/bblfshd \ + -v /proc:/newproc \ + --security-opt seccomp=./bblfshd-seccomp.json \ + bblfsh/bblfshd ``` -The container should be executed with the `--privileged` flag since *bblfshd* is -based on [container technology](https://github.com/opencontainers/runc/tree/master/libcontainer) -and interacts with the kernel at a low level. *bblfshd* exposes a gRPC server at -the port `9432` by default which is used by the [clients](https://github.com/search?q=topic%3Aclient+org%3Abblfsh&type=Repositories) +To understand the flags `-v /proc:/newproc` and `--security-opt seccomp=./bblfshd-seccomp.json`, +where [`bblfshd-seccomp.json`](./bblfshd-seccomp.json) is a file present in this repo, and check +further requirements, please refer to [rootless.md](./rootless.md). `bblfshd` is based on +[container technology](https://github.com/opencontainers/runc/tree/master/libcontainer) +and interacts with the kernel at a low level. It exposes a gRPC server at the port `9432` by default +which is used by the [clients](https://github.com/search?q=topic%3Aclient+org%3Abblfsh&type=Repositories) to interact with the server. Also, we mount the path `/var/lib/bblfshd/` where all the driver images and container instances will be stored. +#### Privileged mode + +We advise against it, but if you prefer to run `bblfshd` in `privileged` mode to skip configuration steps of +[rootless.md](rootless.md), you could do, in Linux: + +```sh +docker run -d --name bblfshd --privileged -p 9432:9432 -v /var/lib/bblfshd:/var/lib/bblfshd bblfsh/bblfshd +``` + +or macOs: + +```sh +docker run -d --name bblfshd --privileged -p 9432:9432 -v bblfsh-storage:/var/lib/bblfshd bblfsh/bblfshd +``` + +#### Install drivers + Now you need to install the driver images into the daemon, you can install the official images just running the command: diff --git a/bblfshd-seccomp.json b/bblfshd-seccomp.json new file mode 100644 index 0000000..ead2ad8 --- /dev/null +++ b/bblfshd-seccomp.json @@ -0,0 +1,804 @@ +{ + "defaultAction": "SCMP_ACT_ERRNO", + "archMap": [ + { + "architecture": "SCMP_ARCH_X86_64", + "subArchitectures": [ + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" + ] + }, + { + "architecture": "SCMP_ARCH_AARCH64", + "subArchitectures": [ + "SCMP_ARCH_ARM" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64" + ] + }, + { + "architecture": "SCMP_ARCH_S390X", + "subArchitectures": [ + "SCMP_ARCH_S390" + ] + } + ], + "syscalls": [ + { + "names": [ + "accept", + "accept4", + "access", + "adjtimex", + "alarm", + "bind", + "brk", + "capget", + "capset", + "chdir", + "chmod", + "chown", + "chown32", + "clock_getres", + "clock_gettime", + "clock_nanosleep", + "close", + "connect", + "copy_file_range", + "creat", + "dup", + "dup2", + "dup3", + "epoll_create", + "epoll_create1", + "epoll_ctl", + "epoll_ctl_old", + "epoll_pwait", + "epoll_wait", + "epoll_wait_old", + "eventfd", + "eventfd2", + "execve", + "execveat", + "exit", + "exit_group", + "faccessat", + "fadvise64", + "fadvise64_64", + "fallocate", + "fanotify_mark", + "fchdir", + "fchmod", + "fchmodat", + "fchown", + "fchown32", + "fchownat", + "fcntl", + "fcntl64", + "fdatasync", + "fgetxattr", + "flistxattr", + "flock", + "fork", + "fremovexattr", + "fsetxattr", + "fstat", + "fstat64", + "fstatat64", + "fstatfs", + "fstatfs64", + "fsync", + "ftruncate", + "ftruncate64", + "futex", + "futimesat", + "getcpu", + "getcwd", + "getdents", + "getdents64", + "getegid", + "getegid32", + "geteuid", + "geteuid32", + "getgid", + "getgid32", + "getgroups", + "getgroups32", + "getitimer", + "getpeername", + "getpgid", + "getpgrp", + "getpid", + "getppid", + "getpriority", + "getrandom", + "getresgid", + "getresgid32", + "getresuid", + "getresuid32", + "getrlimit", + "get_robust_list", + "getrusage", + "getsid", + "getsockname", + "getsockopt", + "get_thread_area", + "gettid", + "gettimeofday", + "getuid", + "getuid32", + "getxattr", + "inotify_add_watch", + "inotify_init", + "inotify_init1", + "inotify_rm_watch", + "io_cancel", + "ioctl", + "io_destroy", + "io_getevents", + "io_pgetevents", + "ioprio_get", + "ioprio_set", + "io_setup", + "io_submit", + "io_uring_enter", + "io_uring_register", + "io_uring_setup", + "ipc", + "kill", + "lchown", + "lchown32", + "lgetxattr", + "link", + "linkat", + "listen", + "listxattr", + "llistxattr", + "_llseek", + "lremovexattr", + "lseek", + "lsetxattr", + "lstat", + "lstat64", + "madvise", + "memfd_create", + "mincore", + "mkdir", + "mkdirat", + "mknod", + "mknodat", + "mlock", + "mlock2", + "mlockall", + "mmap", + "mmap2", + "mprotect", + "mq_getsetattr", + "mq_notify", + "mq_open", + "mq_timedreceive", + "mq_timedsend", + "mq_unlink", + "mremap", + "msgctl", + "msgget", + "msgrcv", + "msgsnd", + "msync", + "munlock", + "munlockall", + "munmap", + "nanosleep", + "newfstatat", + "_newselect", + "open", + "openat", + "pause", + "pipe", + "pipe2", + "poll", + "ppoll", + "prctl", + "pread64", + "preadv", + "preadv2", + "prlimit64", + "pselect6", + "pwrite64", + "pwritev", + "pwritev2", + "read", + "readahead", + "readlink", + "readlinkat", + "readv", + "recv", + "recvfrom", + "recvmmsg", + "recvmsg", + "remap_file_pages", + "removexattr", + "rename", + "renameat", + "renameat2", + "restart_syscall", + "rmdir", + "rt_sigaction", + "rt_sigpending", + "rt_sigprocmask", + "rt_sigqueueinfo", + "rt_sigreturn", + "rt_sigsuspend", + "rt_sigtimedwait", + "rt_tgsigqueueinfo", + "sched_getaffinity", + "sched_getattr", + "sched_getparam", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_getscheduler", + "sched_rr_get_interval", + "sched_setaffinity", + "sched_setattr", + "sched_setparam", + "sched_setscheduler", + "sched_yield", + "seccomp", + "select", + "semctl", + "semget", + "semop", + "semtimedop", + "send", + "sendfile", + "sendfile64", + "sendmmsg", + "sendmsg", + "sendto", + "setfsgid", + "setfsgid32", + "setfsuid", + "setfsuid32", + "setgid", + "setgid32", + "setgroups", + "setgroups32", + "setitimer", + "setpgid", + "setpriority", + "setregid", + "setregid32", + "setresgid", + "setresgid32", + "setresuid", + "setresuid32", + "setreuid", + "setreuid32", + "setrlimit", + "set_robust_list", + "setsid", + "setsockopt", + "set_thread_area", + "set_tid_address", + "setuid", + "setuid32", + "setxattr", + "shmat", + "shmctl", + "shmdt", + "shmget", + "shutdown", + "sigaltstack", + "signalfd", + "signalfd4", + "sigprocmask", + "sigreturn", + "socket", + "socketcall", + "socketpair", + "splice", + "stat", + "stat64", + "statfs", + "statfs64", + "statx", + "symlink", + "symlinkat", + "sync", + "sync_file_range", + "syncfs", + "sysinfo", + "tee", + "tgkill", + "time", + "timer_create", + "timer_delete", + "timerfd_create", + "timerfd_gettime", + "timerfd_settime", + "timer_getoverrun", + "timer_gettime", + "timer_settime", + "times", + "tkill", + "truncate", + "truncate64", + "ugetrlimit", + "umask", + "uname", + "unlink", + "unlinkat", + "utime", + "utimensat", + "utimes", + "vfork", + "vmsplice", + "wait4", + "waitid", + "waitpid", + "write", + "writev", + "mount", + "unshare", + "pivot_root", + "keyctl", + "umount2", + "sethostname" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "ptrace" + ], + "action": "SCMP_ACT_ALLOW", + "args": null, + "comment": "", + "includes": { + "minKernel": "4.8" + }, + "excludes": {} + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 0, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 8, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 131072, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 131080, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 4294967295, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "sync_file_range2" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "ppc64le" + ] + }, + "excludes": {} + }, + { + "names": [ + "arm_fadvise64_64", + "arm_sync_file_range", + "sync_file_range2", + "breakpoint", + "cacheflush", + "set_tls" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "arm", + "arm64" + ] + }, + "excludes": {} + }, + { + "names": [ + "arch_prctl" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "amd64", + "x32" + ] + }, + "excludes": {} + }, + { + "names": [ + "modify_ldt" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "amd64", + "x32", + "x86" + ] + }, + "excludes": {} + }, + { + "names": [ + "s390_pci_mmio_read", + "s390_pci_mmio_write", + "s390_runtime_instr" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "s390", + "s390x" + ] + }, + "excludes": {} + }, + { + "names": [ + "open_by_handle_at" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_DAC_READ_SEARCH" + ] + }, + "excludes": {} + }, + { + "names": [ + "bpf", + "clone", + "fanotify_init", + "lookup_dcookie", + "mount", + "name_to_handle_at", + "perf_event_open", + "quotactl", + "setdomainname", + "sethostname", + "setns", + "syslog", + "umount", + "umount2", + "unshare" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + }, + "excludes": {} + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 2114060288, + "valueTwo": 0, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ], + "arches": [ + "s390", + "s390x" + ] + } + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 1, + "value": 2114060288, + "valueTwo": 0, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "s390 parameter ordering for clone is different", + "includes": { + "arches": [ + "s390", + "s390x" + ] + }, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + } + }, + { + "names": [ + "reboot" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_BOOT" + ] + }, + "excludes": {} + }, + { + "names": [ + "chroot" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_CHROOT" + ] + }, + "excludes": {} + }, + { + "names": [ + "delete_module", + "init_module", + "finit_module", + "query_module" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_MODULE" + ] + }, + "excludes": {} + }, + { + "names": [ + "acct" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_PACCT" + ] + }, + "excludes": {} + }, + { + "names": [ + "kcmp", + "process_vm_readv", + "process_vm_writev", + "ptrace" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_PTRACE" + ] + }, + "excludes": {} + }, + { + "names": [ + "iopl", + "ioperm" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_RAWIO" + ] + }, + "excludes": {} + }, + { + "names": [ + "settimeofday", + "stime", + "clock_settime" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_TIME" + ] + }, + "excludes": {} + }, + { + "names": [ + "vhangup" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_TTY_CONFIG" + ] + }, + "excludes": {} + }, + { + "names": [ + "get_mempolicy", + "mbind", + "set_mempolicy" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_NICE" + ] + }, + "excludes": {} + }, + { + "names": [ + "syslog" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYSLOG" + ] + }, + "excludes": {} + } + ] +} diff --git a/daemon/driver.go b/daemon/driver.go index 514557c..003ab4d 100644 --- a/daemon/driver.go +++ b/daemon/driver.go @@ -74,7 +74,7 @@ func NewDriverInstance(r *runtime.Runtime, lang string, i runtime.DriverImage, o Env: o.Env, Stdout: os.Stdout, Stderr: os.Stderr, - Init: true, + Init: true, } tmp := filepath.Join(r.Root, fmt.Sprintf(TmpPathPattern, id)) diff --git a/rootless.md b/rootless.md new file mode 100644 index 0000000..86b5348 --- /dev/null +++ b/rootless.md @@ -0,0 +1,69 @@ +# Rootless + +## Requirements + +Being able to run the rootless containers may require `sudo` access in some OSs to enable unprivileged containers +support (Arch or Debian for example), as documented, for example, in the +[buildah documentation](https://wiki.archlinux.org/index.php/Buildah#Enable_support_to_build_unprivileged_containers) +or in the [usernetes documentation](https://github.com/rootless-containers/usernetes#distribution-specific-hint): + +```sh +# Only for the current session +sudo sysctl kernel.unprivileged_userns_clone=1 +``` + +```sh +# Enable the permission permanently +echo "kernel.unprivileged_userns_clone=1" >> /etc/sysctl.conf +sudo sysctl -p +``` + +## Run bblfshd in non-privileged mode + +As documented in the [Docker docs](https://docs.docker.com/engine/security/seccomp/), +the default security profile disables commands such as `unshare`, `mount` or `sethostname` inside containers +(which are needed for example to spawn containers inside `bblfshd` and also to give a `Hostname` to each +container to be an identified driver). Also, as documented in +[libcontainer#1658](https://github.com/opencontainers/runc/issues/1658), +there is a known bug with rootless containers inside another non-root container and the `/proc` mount / masking. +Adding a volume `-v /proc:/newproc` would solve that problem. + +Therefore to run `bblfshd` in non privileged mode, this would suffice: + + +```sh +docker run --name bblfshd \ + -p 9432:9432 \ + -v /var/lib/bblfshd:/var/lib/bblfshd \ + -v /proc:/newproc \ + --security-opt seccomp=unconfined \ + bblfshd +``` + +A better (and recommended) confinement configuration, would be: + +```sh +docker run --name bblfshd \ + -p 9432:9432 \ + -v /var/lib/bblfshd:/var/lib/bblfshd \ + -v /proc:/newproc \ + --security-opt seccomp=./bblfshd-seccomp.json \ + bblfshd +``` + +[`./bblfshd-seccomp.json`](./bblfshd-seccomp.json) file is a modification of +[`default.json`](https://github.com/moby/moby/blob/master/profiles/seccomp/default.json) from Docker which allows +the following syscalls inside `bblfshd` container: `mount, unshare, pivot_root, keyctl, umount2, sethostname`. + +## Known bugs + +Running `bblfshd` in rootless mode, you may see, in `bblfshd` logs, warning messages such as: + +``` +level=warning msg="no such directory for freezer.state" +``` + +They do not have further repercussions and are due to the inability of a rootless container to manage `cgroups` +for containers created inside them. + + diff --git a/runtime/container.go b/runtime/container.go index ef74ea3..1d78d42 100644 --- a/runtime/container.go +++ b/runtime/container.go @@ -5,7 +5,7 @@ package runtime import ( "os" "syscall" - + "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -86,15 +86,16 @@ func (c *container) Run() error { } func (c *container) Stop() error { - if err := c.process.Signal(syscall.SIGTERM); err != nil { + // Running bblfshd as a rootless container requires to use + // SIGKILL instead of SIGTERM or SIGINT to kill the process. + // Otherwise it ignores the order + if err := c.process.Signal(syscall.SIGKILL); err != nil { return err } - // kills all the remaining processes if err := c.Signal(syscall.SIGKILL); err != nil { return err } - return c.Destroy() } diff --git a/runtime/container_test.go b/runtime/container_test.go index 54c8296..ba7681f 100644 --- a/runtime/container_test.go +++ b/runtime/container_test.go @@ -53,7 +53,7 @@ func (s *ContainerSuite) TestContainer_Run() { p := &Process{ Args: []string{"/bin/ls"}, Stdout: os.Stdout, - Init: true, + Init: true, } c, err := s.Runtime.Container("run", s.Image, p, nil) @@ -68,7 +68,7 @@ func (s *ContainerSuite) TestContainer_StartStopStart() { p := &Process{ Args: []string{"/bin/sleep", "5m"}, Stdout: os.Stdout, - Init: true, + Init: true, } c, err := s.Runtime.Container("1", s.Image, p, nil) @@ -84,7 +84,7 @@ func (s *ContainerSuite) TestContainer_StartStopStart() { p = &Process{ Args: []string{"/bin/sleep", "5m"}, Stdout: os.Stdout, - Init: true, + Init: true, } c, err = s.Runtime.Container("2", s.Image, p, nil) @@ -106,7 +106,7 @@ func (s *ContainerSuite) TestContainer_StartWait() { p := &Process{ Args: []string{"/bin/ls"}, Stdout: out, - Init: true, + Init: true, } c, err := s.Runtime.Container("wait", s.Image, p, nil) @@ -129,7 +129,7 @@ func (s *ContainerSuite) TestContainer_StartWaitExit1() { p := &Process{ Args: []string{"/bin/false"}, Stdout: out, - Init: true, + Init: true, } c, err := s.Runtime.Container("wait-exit", s.Image, p, nil) @@ -152,7 +152,7 @@ func (s *ContainerSuite) TestContainer_StartFailure() { p := &Process{ Args: []string{"/bin/non-existent"}, Stdout: out, - Init: true, + Init: true, } c, err := s.Runtime.Container("start-failure", s.Image, p, nil) @@ -170,7 +170,7 @@ func (s *ContainerSuite) TestContainer_Env() { p := &Process{ Args: []string{"/bin/env"}, Stdout: out, - Init: true, + Init: true, } c, err := s.Runtime.Container("env", s.Image, p, nil)