Support for configurable network address for user-v2

Signed-off-by: Balaji Vijayakumar <[email protected]>

cmd/limactl/usernet.go

@@ -22,6 +22,7 @@ func newUsernetCommand() *cobra.Command {
 	hostagentCommand.Flags().StringP("endpoint", "e", "", "exposes usernet api(s) on this endpoint")
 	hostagentCommand.Flags().String("listen-qemu", "", "listen for qemu connections")
 	hostagentCommand.Flags().String("listen", "", "listen on a Unix socket and receive Bess-compatible FDs as SCM_RIGHTS messages")
+	hostagentCommand.Flags().String("subnet", "192.168.5.0/24", "sets subnet value for the usernet network")
 	hostagentCommand.Flags().Int("mtu", 1500, "mtu")
 	return hostagentCommand
 }
@@ -52,6 +53,10 @@ func usernetAction(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
+	subnet, err := cmd.Flags().GetString("subnet")
+	if err != nil {
+		return err
+	}
 
 	mtu, err := cmd.Flags().GetInt("mtu")
 	if err != nil {
@@ -67,5 +72,6 @@ func usernetAction(cmd *cobra.Command, _ []string) error {
 		Endpoint:   endpoint,
 		QemuSocket: qemuSocket,
 		FdSocket:   fdSocket,
+		Subnet:     subnet,
 	})
 }

examples/experimental/net-user-v2.yaml

@@ -6,6 +6,11 @@ images:
 - location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img"
   arch: "aarch64"
 
+hostResolver:
+  # Marking enabled false will make use of system dns / inbuilt dns resolver of usernet for user-v2 network
+  enabled: false
+  hosts:
+    host.docker.internal: host.lima.internal
 mounts:
 - location: "~"
 - location: "/tmp/lima"

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Code-Hex/vz/v3 v3.0.6
 	github.com/alessio/shellescape v1.4.1
+	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e
 	github.com/cheggaaa/pb/v3 v3.1.2
 	github.com/containerd/containerd v1.7.2
@@ -54,7 +55,6 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/a8m/envsubst v1.4.2 // indirect
 	github.com/alecthomas/participle/v2 v2.0.0 // indirect
-	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect

hack/test-example.sh

@@ -133,7 +133,8 @@ INFO "Testing proxy settings are imported"
 got=$(limactl shell "$NAME" env | grep FTP_PROXY)
 # Expected: FTP_PROXY is set in addition to ftp_proxy, localhost is replaced
 # by the gateway address, and the value is set immediately without a restart
-expected="FTP_PROXY=http://192.168.5.2:2121"
+gatewayIp=$(limactl shell "$NAME" ip route show 0.0.0.0/0 dev eth0 | cut -d\  -f3)
+expected="FTP_PROXY=http://${gatewayIp}:2121"
 INFO "FTP_PROXY: expected=${expected} got=${got}"
 if [ "$got" != "$expected" ]; then
 	ERROR "proxy environment variable not set to correct value"
@@ -335,10 +336,10 @@ if [[ -n ${CHECKS["user-v2"]} ]]; then
 	secondvm="$NAME-1"
 	"${LIMACTL_CREATE[@]}" "$FILE" --name "$secondvm"
 	limactl start "$secondvm"
-	guestNewip="$(limactl shell "$secondvm" ip -4 -j addr show dev eth0 | jq -r '.[0].addr_info[0].local')"
-	INFO "IP of $secondvm is $guestNewip"
+	secondvmDNS="lima-$secondvm.internal"
+	INFO "DNS of $secondvm is $secondvmDNS"
 	set -x
-	if ! limactl shell "$NAME" ping -c 1 "$guestNewip"; then
+	if ! limactl shell "$NAME" ping -c 1 "$secondvmDNS"; then
 		ERROR "Failed to do vm->vm communication via user-v2"
 		INFO "Stopping \"$secondvm\""
 		limactl stop "$secondvm"

pkg/cidata/cidata.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/lima-vm/lima/pkg/networks/usernet"
+
 	"github.com/lima-vm/lima/pkg/networks"
 
 	"github.com/docker/go-units"
@@ -35,7 +37,7 @@ var netLookupIP = func(host string) []net.IP {
 	return ips
 }
 
-func setupEnv(y *limayaml.LimaYAML) (map[string]string, error) {
+func setupEnv(y *limayaml.LimaYAML, args TemplateArgs) (map[string]string, error) {
 	// Start with the proxy variables from the system settings.
 	env, err := osutil.ProxySettings()
 	if err != nil {
@@ -74,7 +76,7 @@ func setupEnv(y *limayaml.LimaYAML) (map[string]string, error) {
 
 			for _, ip := range netLookupIP(u.Hostname()) {
 				if ip.IsLoopback() {
-					newHost := networks.SlirpGateway
+					newHost := args.SlirpGateway
 					if u.Port() != "" {
 						newHost = net.JoinHostPort(newHost, u.Port())
 					}
@@ -123,13 +125,35 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 		UID:            uid,
 		Containerd:     Containerd{System: *y.Containerd.System, User: *y.Containerd.User},
 		SlirpNICName:   networks.SlirpNICName,
-		SlirpGateway:   networks.SlirpGateway,
-		SlirpDNS:       networks.SlirpDNS,
-		SlirpIPAddress: networks.SlirpIPAddress,
 		RosettaEnabled: *y.Rosetta.Enabled,
 		RosettaBinFmt:  *y.Rosetta.BinFmt,
 	}
 
+	firstUsernetIndex := limayaml.FirstUsernetIndex(y)
+	var subnet net.IP
+
+	if firstUsernetIndex != -1 {
+		usernetName := y.Networks[firstUsernetIndex].Lima
+		subnet, err = usernet.Subnet(usernetName)
+		if err != nil {
+			return err
+		}
+		args.SlirpGateway = usernet.GatewayIP(subnet)
+		args.SlirpDNS = usernet.GatewayIP(subnet)
+	} else {
+		subnet, err = usernet.ParseSubnet(networks.SlirpNetwork)
+		if err != nil {
+			return err
+		}
+		args.SlirpGateway = usernet.GatewayIP(subnet)
+		if *y.VMType == limayaml.VZ {
+			args.SlirpDNS = usernet.GatewayIP(subnet)
+		} else {
+			args.SlirpDNS = usernet.DNSIP(subnet)
+		}
+		args.SlirpIPAddress = networks.SlirpIPAddress
+	}
+
 	// change instance id on every boot so network config will be processed again
 	args.IID = fmt.Sprintf("iid-%d", time.Now().Unix())
 
@@ -209,24 +233,24 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 		})
 	}
 
-	slirpMACAddress := limayaml.MACAddress(instDir)
-	args.Networks = append(args.Networks, Network{MACAddress: slirpMACAddress, Interface: networks.SlirpNICName})
-	firstUsernetIndex := limayaml.FirstUsernetIndex(y)
+	args.Networks = append(args.Networks, Network{MACAddress: limayaml.MACAddress(instDir), Interface: networks.SlirpNICName})
 	for i, nw := range y.Networks {
 		if i == firstUsernetIndex {
 			continue
 		}
 		args.Networks = append(args.Networks, Network{MACAddress: nw.MACAddress, Interface: nw.Interface})
 	}
 
-	args.Env, err = setupEnv(y)
+	args.Env, err = setupEnv(y, args)
 	if err != nil {
 		return err
 	}
 	if *y.HostResolver.Enabled {
 		args.UDPDNSLocalPort = udpDNSLocalPort
 		args.TCPDNSLocalPort = tcpDNSLocalPort
-		args.DNSAddresses = append(args.DNSAddresses, networks.SlirpDNS)
+		args.DNSAddresses = append(args.DNSAddresses, args.SlirpDNS)
+	} else if firstUsernetIndex != -1 || *y.VMType == limayaml.VZ {
+		args.DNSAddresses = append(args.DNSAddresses, args.SlirpDNS)
 	} else if len(y.DNS) > 0 {
 		for _, addr := range y.DNS {
 			args.DNSAddresses = append(args.DNSAddresses, addr.String())

pkg/cidata/cidata_test.go

@@ -48,7 +48,8 @@ func TestSetupEnv(t *testing.T) {
 		t.Run(httpProxy.Host, func(t *testing.T) {
 			envKey := "http_proxy"
 			envValue := httpProxy.String()
-			envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}})
+			templateArgs := TemplateArgs{SlirpGateway: networks.SlirpGateway}
+			envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}}, templateArgs)
 			assert.NilError(t, err)
 			assert.Equal(t, envs[envKey], strings.ReplaceAll(envValue, httpProxy.Hostname(), networks.SlirpGateway))
 		})
@@ -58,7 +59,8 @@ func TestSetupEnv(t *testing.T) {
 func TestSetupInvalidEnv(t *testing.T) {
 	envKey := "http_proxy"
 	envValue := "://localhost:8080"
-	envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}})
+	templateArgs := TemplateArgs{SlirpGateway: networks.SlirpGateway}
+	envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}}, templateArgs)
 	assert.NilError(t, err)
 	assert.Equal(t, envs[envKey], envValue)
 }

pkg/cidata/template.go

@@ -50,19 +50,20 @@ type Disk struct {
 	Device string
 }
 type TemplateArgs struct {
-	Name                            string // instance name
-	IID                             string // instance id
-	User                            string // user name
-	UID                             int
-	SSHPubKeys                      []string
-	Mounts                          []Mount
-	MountType                       string
-	Disks                           []Disk
-	Containerd                      Containerd
-	Networks                        []Network
-	SlirpNICName                    string
-	SlirpGateway                    string
-	SlirpDNS                        string
+	Name         string // instance name
+	IID          string // instance id
+	User         string // user name
+	UID          int
+	SSHPubKeys   []string
+	Mounts       []Mount
+	MountType    string
+	Disks        []Disk
+	Containerd   Containerd
+	Networks     []Network
+	SlirpNICName string
+	SlirpGateway string
+	SlirpDNS     string
+	//Deprecated
 	SlirpIPAddress                  string
 	UDPDNSLocalPort                 int
 	TCPDNSLocalPort                 int

pkg/networks/config_test.go

@@ -15,7 +15,7 @@ func TestFillDefault(t *testing.T) {
 	userNet := newYaml.Networks[ModeUserV2]
 	assert.Equal(t, userNet.Mode, ModeUserV2)
 	assert.Equal(t, userNet.Interface, "")
-	assert.DeepEqual(t, userNet.NetMask, net.IP{})
-	assert.DeepEqual(t, userNet.Gateway, net.IP{})
+	assert.DeepEqual(t, userNet.NetMask, net.ParseIP("255.255.255.0"))
+	assert.DeepEqual(t, userNet.Gateway, net.ParseIP("192.168.104.1"))
 	assert.DeepEqual(t, userNet.DHCPEnd, net.IP{})
 }

pkg/networks/const.go

@@ -5,6 +5,5 @@ const (
 	// CIDR is intentionally hardcoded to 192.168.5.0/24, as each of QEMU has its own independent slirp network.
 	SlirpNetwork   = "192.168.5.0/24"
 	SlirpGateway   = "192.168.5.2"
-	SlirpDNS       = "192.168.5.3"
 	SlirpIPAddress = "192.168.5.15"
 )

pkg/networks/networks.TEMPLATE.yaml

@@ -24,6 +24,8 @@ group: everyone
 networks:
   user-v2:
     mode: user-v2
+    gateway: 192.168.104.1
+    netmask: 255.255.255.0
     # user-v2 network is experimental network mode which supports all functionalities of default usernet network and also allows vm -> vm communication.
     # Doesn't support configuration of custom gateway; hardcoded to 192.168.5.0/24
   shared:

pkg/networks/usernet/client.go

@@ -9,6 +9,9 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/lima-vm/lima/pkg/driver"
+	"github.com/lima-vm/lima/pkg/limayaml"
+
 	gvproxyclient "github.com/containers/gvisor-tap-vsock/pkg/client"
 	"github.com/containers/gvisor-tap-vsock/pkg/types"
 )
@@ -19,6 +22,23 @@ type Client struct {
 	client   *http.Client
 	delegate *gvproxyclient.Client
 	base     string
+	subnet   net.IP
+}
+
+func (c *Client) ConfigureDriver(driver *driver.BaseDriver) error {
+	macAddress := limayaml.MACAddress(driver.Instance.Dir)
+	ipAddress, err := c.ResolveIPAddress(macAddress)
+	if err != nil {
+		return err
+	}
+	err = c.ResolveAndForwardSSH(ipAddress, driver.SSHLocalPort)
+	if err != nil {
+		return err
+	}
+	var hosts = driver.Yaml.HostResolver.Hosts
+	hosts[fmt.Sprintf("lima-%s.internal", driver.Instance.Name)] = ipAddress
+	err = c.AddDNSHosts(hosts)
+	return err
 }
 
 func (c *Client) UnExposeSSH(sshPort int) error {
@@ -28,30 +48,46 @@ func (c *Client) UnExposeSSH(sshPort int) error {
 	})
 }
 
-func (c *Client) ResolveAndForwardSSH(vmMacAddr string, sshPort int) error {
+func (c *Client) AddDNSHosts(hosts map[string]string) error {
+	hosts["host.lima.internal"] = GatewayIP(c.subnet)
+	zones := extractZones(hosts)
+	for _, zone := range zones {
+		err := c.delegate.AddDNS(&zone)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Client) ResolveAndForwardSSH(ipAddr string, sshPort int) error {
+	err := c.delegate.Expose(&types.ExposeRequest{
+		Local:    fmt.Sprintf("127.0.0.1:%d", sshPort),
+		Remote:   fmt.Sprintf("%s:22", ipAddr),
+		Protocol: "tcp",
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *Client) ResolveIPAddress(vmMacAddr string) (string, error) {
 	timeout := time.After(2 * time.Minute)
 	ticker := time.NewTicker(500 * time.Millisecond)
 	for {
 		select {
 		case <-timeout:
-			return errors.New("usernet unable to resolve IP for SSH forwarding")
+			return "", errors.New("usernet unable to resolve IP for SSH forwarding")
 		case <-ticker.C:
 			leases, err := c.leases()
 			if err != nil {
-				return err
+				return "", err
 			}
 
 			for ipAddr, leaseAddr := range leases {
 				if vmMacAddr == leaseAddr {
-					err = c.delegate.Expose(&types.ExposeRequest{
-						Local:    fmt.Sprintf("127.0.0.1:%d", sshPort),
-						Remote:   fmt.Sprintf("%s:22", ipAddr),
-						Protocol: "tcp",
-					})
-					if err != nil {
-						return err
-					}
-					return nil
+					return ipAddr, nil
 				}
 			}
 		}
@@ -75,11 +111,11 @@ func (c *Client) leases() (map[string]string, error) {
 	return leases, nil
 }
 
-func NewClient(endpointSock string) *Client {
-	return create(endpointSock, "http://lima")
+func NewClient(endpointSock string, subnet net.IP) *Client {
+	return create(endpointSock, subnet, "http://lima")
 }
 
-func create(sock string, base string) *Client {
+func create(sock string, subnet net.IP, base string) *Client {
 	client := &http.Client{
 		Transport: &http.Transport{
 			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
@@ -92,5 +128,6 @@ func create(sock string, base string) *Client {
 		client:   client,
 		delegate: delegate,
 		base:     base,
+		subnet:   subnet,
 	}
 }