Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Investigating guestagent on FreeBSD, for containerd with runj #1509

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ config GUESTAGENT_OS_LINUX
Build lima-guestagent for "Linux" OS
default y

config GUESTAGENT_OS_FREEBSD
bool "guestagent OS: FreeBSD"
help
Build lima-guestagent for "FreeBSD" OS
default n

config GUESTAGENT_ARCH_X8664
bool "guestagent Arch: x86_64"
help
Expand Down
29 changes: 29 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,20 @@ GUESTAGENT += \
_output/share/lima/lima-guestagent.Linux-riscv64
endif
endif
ifeq ($(CONFIG_GUESTAGENT_OS_FREEBSD),y)
ifeq ($(CONFIG_GUESTAGENT_ARCH_X8664),y)
GUESTAGENT += \
_output/share/lima/lima-guestagent.FreeBSD-x86_64
endif
ifeq ($(CONFIG_GUESTAGENT_ARCH_AARCH64),y)
GUESTAGENT += \
_output/share/lima/lima-guestagent.FreeBSD-aarch64
endif
ifeq ($(CONFIG_GUESTAGENT_ARCH_RISCV64),y)
GUESTAGENT += \
_output/share/lima/lima-guestagent.FreeBSD-riscv64
endif
endif

.PHONY: binaries
binaries: clean \
Expand Down Expand Up @@ -188,6 +202,21 @@ _output/share/lima/lima-guestagent.Linux-riscv64:
GOOS=linux GOARCH=riscv64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
chmod 644 $@

.PHONY: _output/share/lima/lima-guestagent.FreeBSD-x86_64
_output/share/lima/lima-guestagent.FreeBSD-x86_64:
GOOS=freebsd GOARCH=amd64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
chmod 644 $@

.PHONY: _output/share/lima/lima-guestagent.FreeBSD-aarch64
_output/share/lima/lima-guestagent.FreeBSD-aarch64:
GOOS=freebsd GOARCH=arm64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
chmod 644 $@

.PHONY: _output/share/lima/lima-guestagent.FreeBSD-riscv64
_output/share/lima/lima-guestagent.FreeBSD-riscv64:
GOOS=freebsd GOARCH=riscv64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
chmod 644 $@

.PHONY: manpages
manpages: _output/bin/limactl$(exe)
@mkdir -p _output/share/man/man1
Expand Down
79 changes: 79 additions & 0 deletions cmd/lima-guestagent/daemon_freebsd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package main

// Same as linux, but without vsock

import (
"errors"
"net"
"net/http"
"os"
"time"

"github.com/gorilla/mux"
"github.com/lima-vm/lima/pkg/guestagent"
"github.com/lima-vm/lima/pkg/guestagent/api/server"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

func newDaemonCommand() *cobra.Command {
daemonCommand := &cobra.Command{
Use: "daemon",
Short: "run the daemon",
RunE: daemonAction,
}
daemonCommand.Flags().Duration("tick", 3*time.Second, "tick for polling events")
return daemonCommand
}

func daemonAction(cmd *cobra.Command, _ []string) error {
socket := "/run/lima-guestagent.sock"
{ // runtime.GOOS == "freebsd"
socket = "/var" + socket
}
tick, err := cmd.Flags().GetDuration("tick")
if err != nil {
return err
}
if tick == 0 {
return errors.New("tick must be specified")
}
if os.Geteuid() != 0 {
return errors.New("must run as the root")
}
logrus.Infof("event tick: %v", tick)

newTicker := func() (<-chan time.Time, func()) {
ticker := time.NewTicker(tick)
return ticker.C, ticker.Stop
}

agent, err := guestagent.New(newTicker, tick*20)
if err != nil {
return err
}
backend := &server.Backend{
Agent: agent,
}
r := mux.NewRouter()
server.AddRoutes(r, backend)
srv := &http.Server{Handler: r}
err = os.RemoveAll(socket)
if err != nil {
return err
}

var l net.Listener
{
socketL, err := net.Listen("unix", socket)
if err != nil {
return err
}
if err := os.Chmod(socket, 0777); err != nil {
return err
}
l = socketL
logrus.Infof("serving the guest agent on %q", socket)
}
return srv.Serve(l)
}
afbjorklund marked this conversation as resolved.
Show resolved Hide resolved
37 changes: 37 additions & 0 deletions cmd/lima-guestagent/main_freebsd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package main

// Same as linux, but without InstallSystemdCommand

import (
"strings"

"github.com/lima-vm/lima/pkg/version"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

func main() {
if err := newApp().Execute(); err != nil {
logrus.Fatal(err)
}
}

func newApp() *cobra.Command {
afbjorklund marked this conversation as resolved.
Show resolved Hide resolved
var rootCmd = &cobra.Command{
Use: "lima-guestagent",
Short: "Do not launch manually",
Version: strings.TrimPrefix(version.Version, "v"),
}
rootCmd.PersistentFlags().Bool("debug", false, "debug mode")
rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
debug, _ := cmd.Flags().GetBool("debug")
if debug {
logrus.SetLevel(logrus.DebugLevel)
}
return nil
}
rootCmd.AddCommand(
newDaemonCommand(),
)
return rootCmd
}
1 change: 1 addition & 0 deletions config.mk
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
CONFIG_GUESTAGENT_OS_LINUX=y
CONFIG_GUESTAGENT_OS_FREEBSD=n
CONFIG_GUESTAGENT_ARCH_X8664=y
CONFIG_GUESTAGENT_ARCH_AARCH64=y
CONFIG_GUESTAGENT_ARCH_ARMV7L=y
Expand Down
1 change: 1 addition & 0 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ Optional feature enablers:
- [`vmnet`](./vmnet.yaml): ⭐enable [`vmnet.framework`](../docs/network.md)
- [`experimental/9p`](./experimental/9p.yaml): [experimental] use 9p mount type
- [`experimental/virtiofs-linux`](./experimental/9p.yaml): [experimental] use virtiofs mount type for Linux
- [`experimental/freebsd.yaml`](./experimental/freebsd.yaml): [experimental] FreeBSD
- [`experimental/armv7l`](./experimental/armv7l.yaml): [experimental] ARMv7
- [`experimental/riscv64`](./experimental/riscv64.yaml): [experimental] RISC-V
- [`experimental/net-user-v2`](./experimental/net-user-v2.yaml): [experimental] user-v2 network
Expand Down
2 changes: 1 addition & 1 deletion examples/default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# 🟢 Builtin default: "qemu"
vmType: null

# OS: "Linux".
# OS: "Linux", "FreeBSD".
# 🟢 Builtin default: "Linux"
os: null

Expand Down
28 changes: 28 additions & 0 deletions examples/experimental/freebsd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
os: "FreeBSD"

images:
- location: https://download.freebsd.org/ftp/snapshots/VM-IMAGES/14.0-CURRENT/amd64/Latest/FreeBSD-14.0-CURRENT-amd64.qcow2.xz
arch: "x86_64"
- location: https://download.freebsd.org/ftp/snapshots/VM-IMAGES/14.0-CURRENT/aarch64/Latest/FreeBSD-14.0-CURRENT-arm64-aarch64.qcow2.xz
arch: "aarch64"
- location: https://download.freebsd.org/ftp/snapshots/VM-IMAGES/14.0-CURRENT/riscv64/Latest/FreeBSD-14.0-CURRENT-riscv-riscv64.qcow2.xz
arch: "riscv64"
kernel:
Copy link
Member Author

@afbjorklund afbjorklund Aug 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this u-boot image was mostly a hack to get past the validate (for amd64/arm64), but it is the real image:

https://wiki.freebsd.org/riscv/QEMU

We might want to import it to https://github.com/lima-vm/u-boot-qemu-mirror/ proper, so it can be used here.

The image does seem to be booting.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why can't we use Debian's u-boot binary for FreeBSD?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that should be fine as well, just haven't tested it.

# Extracted from https://pkg.freebsd.org/FreeBSD:13:aarch64/latest/All/u-boot-qemu-riscv64-2023.01.pkg (GPL-2.0)
location: "file://./usr/local/share/u-boot/u-boot-qemu-riscv64/u-boot.bin"
digest: "sha256:a7959ef65183bd4847b0fa1d2a298392148271ef62aeaa7d0ebd03399a0b1cc0"

video:
display: "default"

mounts:
- location: "~"
- location: "/tmp/lima"
writable: true

additionalDisks:
- "zfs"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ZFS disk is actually only needed if wanting to run containers, it will boot fine without it.


containerd:
system: false
user: false
140 changes: 140 additions & 0 deletions pkg/guestagent/guestagent_freebsd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
package guestagent

import (
"context"
"reflect"
"time"

"github.com/lima-vm/lima/pkg/guestagent/api"
"github.com/lima-vm/lima/pkg/guestagent/sockstat"
"github.com/sirupsen/logrus"
)

func New(newTicker func() (<-chan time.Time, func()), _ time.Duration) (Agent, error) {
a := &agent{
newTicker: newTicker,
}

return a, nil
}

type agent struct {
// Ticker is like time.Ticker.
newTicker func() (<-chan time.Time, func())
}

type eventState struct {
ports []api.IPPort
}

func comparePorts(old, neww []api.IPPort) (added, removed []api.IPPort) {
mRaw := make(map[string]api.IPPort, len(old))
mStillExist := make(map[string]bool, len(old))

for _, f := range old {
k := f.String()
mRaw[k] = f
mStillExist[k] = false
}
for _, f := range neww {
k := f.String()
if _, ok := mRaw[k]; !ok {
added = append(added, f)
}
mStillExist[k] = true
}

for k, stillExist := range mStillExist {
if !stillExist {
if x, ok := mRaw[k]; ok {
removed = append(removed, x)
}
}
}
return
}

func (a *agent) collectEvent(ctx context.Context, st eventState) (api.Event, eventState) {
var (
ev api.Event
err error
)
newSt := st
newSt.ports, err = a.LocalPorts(ctx)
if err != nil {
ev.Errors = append(ev.Errors, err.Error())
ev.Time = time.Now()
return ev, newSt
}
ev.LocalPortsAdded, ev.LocalPortsRemoved = comparePorts(st.ports, newSt.ports)
ev.Time = time.Now()
return ev, newSt
}

func isEventEmpty(ev api.Event) bool {
var empty api.Event
// ignore ev.Time
copied := ev
copied.Time = time.Time{}
return reflect.DeepEqual(empty, copied)
}

func (a *agent) Events(ctx context.Context, ch chan api.Event) {
defer close(ch)
tickerCh, tickerClose := a.newTicker()
defer tickerClose()
var st eventState
for {
var ev api.Event
ev, st = a.collectEvent(ctx, st)
if !isEventEmpty(ev) {
ch <- ev
}
select {
case <-ctx.Done():
return
case _, ok := <-tickerCh:
if !ok {
return
}
logrus.Debug("tick!")
}
}
}

func (a *agent) LocalPorts(_ context.Context) ([]api.IPPort, error) {
var res []api.IPPort
tcpParsed, err := sockstat.ParseOutput()
if err != nil {
return res, err
}

for _, f := range tcpParsed {
switch f.Kind {
case sockstat.TCP4, sockstat.TCP6:
default:
continue
}
if f.State == sockstat.Listen {
res = append(res,
api.IPPort{
IP: f.IP,
Port: int(f.Port),
})
}
}

return res, nil
}

func (a *agent) Info(ctx context.Context) (*api.Info, error) {
var (
info api.Info
err error
)
info.LocalPorts, err = a.LocalPorts(ctx)
if err != nil {
return nil, err
}
return &info, nil
}
Loading
Loading