Support for configurable network address for user-v2

cmd/limactl/usernet.go

@@ -22,6 +22,7 @@ func newUsernetCommand() *cobra.Command {
 	hostagentCommand.Flags().StringP("endpoint", "e", "", "exposes usernet api(s) on this endpoint")
 	hostagentCommand.Flags().String("listen-qemu", "", "listen for qemu connections")
 	hostagentCommand.Flags().String("listen", "", "listen on a Unix socket and receive Bess-compatible FDs as SCM_RIGHTS messages")
+	hostagentCommand.Flags().String("subnet", "192.168.5.0/24", "sets subnet value for the usernet network")
 	hostagentCommand.Flags().Int("mtu", 1500, "mtu")
 	return hostagentCommand
 }
@@ -52,6 +53,10 @@ func usernetAction(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
+	subnet, err := cmd.Flags().GetString("subnet")
+	if err != nil {
+		return err
+	}
 
 	mtu, err := cmd.Flags().GetInt("mtu")
 	if err != nil {
@@ -67,5 +72,6 @@ func usernetAction(cmd *cobra.Command, _ []string) error {
 		Endpoint:   endpoint,
 		QemuSocket: qemuSocket,
 		FdSocket:   fdSocket,
+		Subnet:     subnet,
 	})
 }

examples/experimental/net-user-v2.yaml

@@ -6,6 +6,9 @@ images:
 - location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img"
   arch: "aarch64"
 
+hostResolver:
+  hosts:
+    host.docker.internal: host.lima.internal
 mounts:
 - location: "~"
 - location: "/tmp/lima"

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Code-Hex/vz/v3 v3.0.6
 	github.com/alessio/shellescape v1.4.2
+	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e
 	github.com/cheggaaa/pb/v3 v3.1.4
 	github.com/containerd/containerd v1.7.3
@@ -55,7 +56,6 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/a8m/envsubst v1.4.2 // indirect
 	github.com/alecthomas/participle/v2 v2.0.0 // indirect
-	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect

hack/test-templates.sh

@@ -129,7 +129,8 @@ INFO "Testing proxy settings are imported"
 got=$(limactl shell "$NAME" env | grep FTP_PROXY)
 # Expected: FTP_PROXY is set in addition to ftp_proxy, localhost is replaced
 # by the gateway address, and the value is set immediately without a restart
-expected="FTP_PROXY=http://192.168.5.2:2121"
+gatewayIp=$(limactl shell "$NAME" ip route show 0.0.0.0/0 dev eth0 | cut -d\  -f3)
+expected="FTP_PROXY=http://${gatewayIp}:2121"
 INFO "FTP_PROXY: expected=${expected} got=${got}"
 if [ "$got" != "$expected" ]; then
 	ERROR "proxy environment variable not set to correct value"
@@ -331,10 +332,10 @@ if [[ -n ${CHECKS["user-v2"]} ]]; then
 	secondvm="$NAME-1"
 	"${LIMACTL_CREATE[@]}" "$FILE" --name "$secondvm"
 	limactl start "$secondvm"
-	guestNewip="$(limactl shell "$secondvm" ip -4 -j addr show dev eth0 | jq -r '.[0].addr_info[0].local')"
-	INFO "IP of $secondvm is $guestNewip"
+	secondvmDNS="lima-$secondvm.internal"
+	INFO "DNS of $secondvm is $secondvmDNS"
 	set -x
-	if ! limactl shell "$NAME" ping -c 1 "$guestNewip"; then
+	if ! limactl shell "$NAME" ping -c 1 "$secondvmDNS"; then
 		ERROR "Failed to do vm->vm communication via user-v2"
 		INFO "Stopping \"$secondvm\""
 		limactl stop "$secondvm"

pkg/cidata/cidata.TEMPLATE.d/boot/09-host-dns-setup.sh

@@ -18,8 +18,12 @@ if command -v iptables >/dev/null 2>&1; then
 	# Remove old rules
 	iptables --table nat --flush ${chain}
 	# Add rules for the existing ip:port
-	iptables --table nat --append "${chain}" --destination "${LIMA_CIDATA_SLIRP_DNS}" --protocol udp --dport 53 --jump DNAT \
-		--to-destination "${LIMA_CIDATA_SLIRP_GATEWAY}:${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}"
-	iptables --table nat --append "${chain}" --destination "${LIMA_CIDATA_SLIRP_DNS}" --protocol tcp --dport 53 --jump DNAT \
-		--to-destination "${LIMA_CIDATA_SLIRP_GATEWAY}:${LIMA_CIDATA_TCP_DNS_LOCAL_PORT}"
+	if [ -n "${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}" ] && [ "${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}" -ne 0 ]; then
+		iptables --table nat --append "${chain}" --destination "${LIMA_CIDATA_SLIRP_DNS}" --protocol udp --dport 53 --jump DNAT \
+			--to-destination "${LIMA_CIDATA_SLIRP_GATEWAY}:${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}"
+	fi
+	if [ -n "${LIMA_CIDATA_TCP_DNS_LOCAL_PORT}" ] && [ "${LIMA_CIDATA_TCP_DNS_LOCAL_PORT}" -ne 0 ]; then
+		iptables --table nat --append "${chain}" --destination "${LIMA_CIDATA_SLIRP_DNS}" --protocol tcp --dport 53 --jump DNAT \
+			--to-destination "${LIMA_CIDATA_SLIRP_GATEWAY}:${LIMA_CIDATA_TCP_DNS_LOCAL_PORT}"
+	fi
 fi

pkg/cidata/cidata.go

@@ -12,12 +12,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/lima-vm/lima/pkg/networks"
-
 	"github.com/docker/go-units"
 	"github.com/lima-vm/lima/pkg/iso9660util"
 	"github.com/lima-vm/lima/pkg/limayaml"
 	"github.com/lima-vm/lima/pkg/localpathutil"
+	"github.com/lima-vm/lima/pkg/networks"
+	"github.com/lima-vm/lima/pkg/networks/usernet"
 	"github.com/lima-vm/lima/pkg/osutil"
 	"github.com/lima-vm/lima/pkg/sshutil"
 	"github.com/lima-vm/lima/pkg/store/filenames"
@@ -35,7 +35,7 @@ var netLookupIP = func(host string) []net.IP {
 	return ips
 }
 
-func setupEnv(y *limayaml.LimaYAML) (map[string]string, error) {
+func setupEnv(y *limayaml.LimaYAML, args TemplateArgs) (map[string]string, error) {
 	// Start with the proxy variables from the system settings.
 	env, err := osutil.ProxySettings()
 	if err != nil {
@@ -74,7 +74,7 @@ func setupEnv(y *limayaml.LimaYAML) (map[string]string, error) {
 
 			for _, ip := range netLookupIP(u.Hostname()) {
 				if ip.IsLoopback() {
-					newHost := networks.SlirpGateway
+					newHost := args.SlirpGateway
 					if u.Port() != "" {
 						newHost = net.JoinHostPort(newHost, u.Port())
 					}
@@ -124,11 +124,34 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 		GuestInstallPrefix: *y.GuestInstallPrefix,
 		Containerd:         Containerd{System: *y.Containerd.System, User: *y.Containerd.User},
 		SlirpNICName:       networks.SlirpNICName,
-		SlirpGateway:       networks.SlirpGateway,
-		SlirpDNS:           networks.SlirpDNS,
-		SlirpIPAddress:     networks.SlirpIPAddress,
-		RosettaEnabled:     *y.Rosetta.Enabled,
-		RosettaBinFmt:      *y.Rosetta.BinFmt,
+
+		RosettaEnabled: *y.Rosetta.Enabled,
+		RosettaBinFmt:  *y.Rosetta.BinFmt,
+	}
+
+	firstUsernetIndex := limayaml.FirstUsernetIndex(y)
+	var subnet net.IP
+
+	if firstUsernetIndex != -1 {
+		usernetName := y.Networks[firstUsernetIndex].Lima
+		subnet, err = usernet.Subnet(usernetName)
+		if err != nil {
+			return err
+		}
+		args.SlirpGateway = usernet.GatewayIP(subnet)
+		args.SlirpDNS = usernet.GatewayIP(subnet)
+	} else {
+		subnet, err = usernet.ParseSubnet(networks.SlirpNetwork)
+		if err != nil {
+			return err
+		}
+		args.SlirpGateway = usernet.GatewayIP(subnet)
+		if *y.VMType == limayaml.VZ {
+			args.SlirpDNS = usernet.GatewayIP(subnet)
+		} else {
+			args.SlirpDNS = usernet.DNSIP(subnet)
+		}
+		args.SlirpIPAddress = networks.SlirpIPAddress
 	}
 
 	// change instance id on every boot so network config will be processed again
@@ -221,24 +244,24 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 		})
 	}
 
-	slirpMACAddress := limayaml.MACAddress(instDir)
-	args.Networks = append(args.Networks, Network{MACAddress: slirpMACAddress, Interface: networks.SlirpNICName})
-	firstUsernetIndex := limayaml.FirstUsernetIndex(y)
+	args.Networks = append(args.Networks, Network{MACAddress: limayaml.MACAddress(instDir), Interface: networks.SlirpNICName})
 	for i, nw := range y.Networks {
 		if i == firstUsernetIndex {
 			continue
 		}
 		args.Networks = append(args.Networks, Network{MACAddress: nw.MACAddress, Interface: nw.Interface})
 	}
 
-	args.Env, err = setupEnv(y)
+	args.Env, err = setupEnv(y, args)
 	if err != nil {
 		return err
 	}
-	if *y.HostResolver.Enabled {
+	if firstUsernetIndex != -1 || *y.VMType == limayaml.VZ {
+		args.DNSAddresses = append(args.DNSAddresses, args.SlirpDNS)
+	} else if *y.HostResolver.Enabled {
 		args.UDPDNSLocalPort = udpDNSLocalPort
 		args.TCPDNSLocalPort = tcpDNSLocalPort
-		args.DNSAddresses = append(args.DNSAddresses, networks.SlirpDNS)
+		args.DNSAddresses = append(args.DNSAddresses, args.SlirpDNS)
 	} else if len(y.DNS) > 0 {
 		for _, addr := range y.DNS {
 			args.DNSAddresses = append(args.DNSAddresses, addr.String())

pkg/cidata/cidata_test.go

@@ -48,7 +48,8 @@ func TestSetupEnv(t *testing.T) {
 		t.Run(httpProxy.Host, func(t *testing.T) {
 			envKey := "http_proxy"
 			envValue := httpProxy.String()
-			envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}})
+			templateArgs := TemplateArgs{SlirpGateway: networks.SlirpGateway}
+			envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}}, templateArgs)
 			assert.NilError(t, err)
 			assert.Equal(t, envs[envKey], strings.ReplaceAll(envValue, httpProxy.Hostname(), networks.SlirpGateway))
 		})
@@ -58,7 +59,8 @@ func TestSetupEnv(t *testing.T) {
 func TestSetupInvalidEnv(t *testing.T) {
 	envKey := "http_proxy"
 	envValue := "://localhost:8080"
-	envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}})
+	templateArgs := TemplateArgs{SlirpGateway: networks.SlirpGateway}
+	envs, err := setupEnv(&limayaml.LimaYAML{PropagateProxyEnv: pointer.Bool(false), Env: map[string]string{envKey: envValue}}, templateArgs)
 	assert.NilError(t, err)
 	assert.Equal(t, envs[envKey], envValue)
 }

pkg/hostagent/hostagent.go

@@ -267,7 +267,8 @@ func (a *HostAgent) Run(ctx context.Context) error {
 		a.emitEvent(ctx, exitingEv)
 	}()
 
-	if *a.y.HostResolver.Enabled {
+	firstUsernetIndex := limayaml.FirstUsernetIndex(a.y)
+	if firstUsernetIndex == -1 && *a.y.HostResolver.Enabled {
 		hosts := a.y.HostResolver.Hosts
 		hosts["host.lima.internal"] = networks.SlirpGateway
 		hosts[fmt.Sprintf("lima-%s", a.instName)] = networks.SlirpIPAddress

pkg/networks/config_test.go

@@ -15,7 +15,7 @@ func TestFillDefault(t *testing.T) {
 	userNet := newYaml.Networks[ModeUserV2]
 	assert.Equal(t, userNet.Mode, ModeUserV2)
 	assert.Equal(t, userNet.Interface, "")
-	assert.DeepEqual(t, userNet.NetMask, net.IP{})
-	assert.DeepEqual(t, userNet.Gateway, net.IP{})
+	assert.DeepEqual(t, userNet.NetMask, net.ParseIP("255.255.255.0"))
+	assert.DeepEqual(t, userNet.Gateway, net.ParseIP("192.168.104.1"))
 	assert.DeepEqual(t, userNet.DHCPEnd, net.IP{})
 }

pkg/networks/const.go

@@ -5,6 +5,5 @@ const (
 	// CIDR is intentionally hardcoded to 192.168.5.0/24, as each of QEMU has its own independent slirp network.
 	SlirpNetwork   = "192.168.5.0/24"
 	SlirpGateway   = "192.168.5.2"
-	SlirpDNS       = "192.168.5.3"
 	SlirpIPAddress = "192.168.5.15"
 )

pkg/networks/networks.TEMPLATE.yaml

@@ -24,6 +24,8 @@ group: everyone
 networks:
   user-v2:
     mode: user-v2
+    gateway: 192.168.104.1
+    netmask: 255.255.255.0
     # user-v2 network is experimental network mode which supports all functionalities of default usernet network and also allows vm -> vm communication.
     # Doesn't support configuration of custom gateway; hardcoded to 192.168.5.0/24
   shared:

pkg/networks/usernet/client.go

@@ -11,6 +11,9 @@ import (
 
 	gvproxyclient "github.com/containers/gvisor-tap-vsock/pkg/client"
 	"github.com/containers/gvisor-tap-vsock/pkg/types"
+	"github.com/lima-vm/lima/pkg/driver"
+	"github.com/lima-vm/lima/pkg/limayaml"
+	"github.com/lima-vm/lima/pkg/networks/usernet/dnshosts"
 )
 
 type Client struct {
@@ -19,6 +22,23 @@ type Client struct {
 	client   *http.Client
 	delegate *gvproxyclient.Client
 	base     string
+	subnet   net.IP
+}
+
+func (c *Client) ConfigureDriver(driver *driver.BaseDriver) error {
+	macAddress := limayaml.MACAddress(driver.Instance.Dir)
+	ipAddress, err := c.ResolveIPAddress(macAddress)
+	if err != nil {
+		return err
+	}
+	err = c.ResolveAndForwardSSH(ipAddress, driver.SSHLocalPort)
+	if err != nil {
+		return err
+	}
+	var hosts = driver.Yaml.HostResolver.Hosts
+	hosts[fmt.Sprintf("lima-%s.internal", driver.Instance.Name)] = ipAddress
+	err = c.AddDNSHosts(hosts)
+	return err
 }
 
 func (c *Client) UnExposeSSH(sshPort int) error {
@@ -28,30 +48,46 @@ func (c *Client) UnExposeSSH(sshPort int) error {
 	})
 }
 
-func (c *Client) ResolveAndForwardSSH(vmMacAddr string, sshPort int) error {
+func (c *Client) AddDNSHosts(hosts map[string]string) error {
+	hosts["host.lima.internal"] = GatewayIP(c.subnet)
+	zones := dnshosts.ExtractZones(hosts)
+	for _, zone := range zones {
+		err := c.delegate.AddDNS(&zone)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Client) ResolveAndForwardSSH(ipAddr string, sshPort int) error {
+	err := c.delegate.Expose(&types.ExposeRequest{
+		Local:    fmt.Sprintf("127.0.0.1:%d", sshPort),
+		Remote:   fmt.Sprintf("%s:22", ipAddr),
+		Protocol: "tcp",
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *Client) ResolveIPAddress(vmMacAddr string) (string, error) {
 	timeout := time.After(2 * time.Minute)
 	ticker := time.NewTicker(500 * time.Millisecond)
 	for {
 		select {
 		case <-timeout:
-			return errors.New("usernet unable to resolve IP for SSH forwarding")
+			return "", errors.New("usernet unable to resolve IP for SSH forwarding")
 		case <-ticker.C:
 			leases, err := c.leases()
 			if err != nil {
-				return err
+				return "", err
 			}
 
 			for ipAddr, leaseAddr := range leases {
 				if vmMacAddr == leaseAddr {
-					err = c.delegate.Expose(&types.ExposeRequest{
-						Local:    fmt.Sprintf("127.0.0.1:%d", sshPort),
-						Remote:   fmt.Sprintf("%s:22", ipAddr),
-						Protocol: "tcp",
-					})
-					if err != nil {
-						return err
-					}
-					return nil
+					return ipAddr, nil
 				}
 			}
 		}
@@ -75,11 +111,11 @@ func (c *Client) leases() (map[string]string, error) {
 	return leases, nil
 }
 
-func NewClient(endpointSock string) *Client {
-	return create(endpointSock, "http://lima")
+func NewClient(endpointSock string, subnet net.IP) *Client {
+	return create(endpointSock, subnet, "http://lima")
 }
 
-func create(sock string, base string) *Client {
+func create(sock string, subnet net.IP, base string) *Client {
 	client := &http.Client{
 		Transport: &http.Transport{
 			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
@@ -92,5 +128,6 @@ func create(sock string, base string) *Client {
 		client:   client,
 		delegate: delegate,
 		base:     base,
+		subnet:   subnet,
 	}
 }