feat: add --non-masquerade-cidrs flag to talosctl cluster create

Allow skipping NAT for the given destinations from a cluster network. This option makes it possible to form an etcd cluster from clusters in different networks created by running `talosctl cluster create` command multiple times using different CIDRs: they simply should have the CIDR of the other clusters passed with `--non-masquerade-cidrs`.

Signed-off-by: Utku Ozdemir <[email protected]>

cmd/talosctl/cmd/mgmt/cluster/create.go

@@ -77,6 +77,7 @@ const (
 	networkIPv6Flag               = "ipv6"
 	networkMTUFlag                = "mtu"
 	networkCIDRFlag               = "cidr"
+	networkNoMasqueradeCIDRsFlag  = "no-masquerade-cidrs"
 	nameserversFlag               = "nameservers"
 	clusterDiskSizeFlag           = "disk"
 	clusterDiskPreallocateFlag    = "disk-preallocate"
@@ -115,6 +116,7 @@ var (
 	extraUEFISearchPaths       []string
 	configDebug                bool
 	networkCIDR                string
+	networkNoMasqueradeCIDRs   []string
 	networkMTU                 int
 	networkIPv4                bool
 	networkIPv6                bool
@@ -345,6 +347,19 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {
 		}
 	}
 
+	noMasqueradeCIDRs := make([]netip.Prefix, 0, len(networkNoMasqueradeCIDRs))
+
+	for _, cidr := range networkNoMasqueradeCIDRs {
+		var parsedCIDR netip.Prefix
+
+		parsedCIDR, err = netip.ParsePrefix(cidr)
+		if err != nil {
+			return fmt.Errorf("error parsing non-masquerade CIDR %q: %w", cidr, err)
+		}
+
+		noMasqueradeCIDRs = append(noMasqueradeCIDRs, parsedCIDR)
+	}
+
 	// Parse nameservers
 	nameserverIPs := make([]netip.Addr, len(nameservers))
 
@@ -386,6 +401,7 @@ func create(ctx context.Context, flags *pflag.FlagSet) error {
 		Network: provision.NetworkRequest{
 			Name:              clusterName,
 			CIDRs:             cidrs,
+			NoMasqueradeCIDRs: noMasqueradeCIDRs,
 			GatewayAddrs:      gatewayIPs,
 			MTU:               networkMTU,
 			Nameservers:       nameserverIPs,
@@ -1115,6 +1131,7 @@ func init() {
 	createCmd.Flags().BoolVar(&configDebug, configDebugFlag, false, "enable debug in Talos config to send service logs to the console")
 	createCmd.Flags().IntVar(&networkMTU, networkMTUFlag, 1500, "MTU of the cluster network")
 	createCmd.Flags().StringVar(&networkCIDR, networkCIDRFlag, "10.5.0.0/24", "CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way)")
+	createCmd.Flags().StringSliceVar(&networkNoMasqueradeCIDRs, networkNoMasqueradeCIDRsFlag, []string{}, "list of CIDRs to exclude from NAT (QEMU provisioner only)")
 	createCmd.Flags().BoolVar(&networkIPv4, networkIPv4Flag, true, "enable IPv4 network in the cluster")
 	createCmd.Flags().BoolVar(&networkIPv6, networkIPv6Flag, false, "enable IPv6 network in the cluster (QEMU provisioner only)")
 	createCmd.Flags().StringVar(&wireguardCIDR, "wireguard-cidr", "", "CIDR of the wireguard network")

hack/start-registry-proxies.sh

@@ -23,3 +23,8 @@ docker run -d -p 5004:5000 \
     -e REGISTRY_PROXY_REMOTEURL=https://ghcr.io \
     --restart always \
     --name registry-ghcr.io registry:2
+
+docker run -d -p 5006:5000 \
+    -e REGISTRY_PROXY_REMOTEURL=https://factory.talos.dev \
+    --restart always \
+    --name registry-factory.talos.dev registry:2

pkg/provision/providers/qemu/create.go

@@ -122,10 +122,11 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque
 	state.ClusterInfo = provision.ClusterInfo{
 		ClusterName: request.Name,
 		Network: provision.NetworkInfo{
-			Name:         request.Network.Name,
-			CIDRs:        request.Network.CIDRs,
-			GatewayAddrs: request.Network.GatewayAddrs,
-			MTU:          request.Network.MTU,
+			Name:              request.Network.Name,
+			CIDRs:             request.Network.CIDRs,
+			NoMasqueradeCIDRs: request.Network.NoMasqueradeCIDRs,
+			GatewayAddrs:      request.Network.GatewayAddrs,
+			MTU:               request.Network.MTU,
 		},
 		Nodes:              nodeInfo,
 		ExtraNodes:         pxeNodeInfo,

pkg/provision/providers/qemu/launch.go

@@ -61,15 +61,16 @@ type LaunchConfig struct {
 	Config string
 
 	// Network
-	BridgeName    string
-	NetworkConfig *libcni.NetworkConfigList
-	CNI           provision.CNIConfig
-	IPs           []netip.Addr
-	CIDRs         []netip.Prefix
-	Hostname      string
-	GatewayAddrs  []netip.Addr
-	MTU           int
-	Nameservers   []netip.Addr
+	BridgeName        string
+	NetworkConfig     *libcni.NetworkConfigList
+	CNI               provision.CNIConfig
+	IPs               []netip.Addr
+	CIDRs             []netip.Prefix
+	NoMasqueradeCIDRs []netip.Prefix
+	Hostname          string
+	GatewayAddrs      []netip.Addr
+	MTU               int
+	Nameservers       []netip.Addr
 
 	// PXE
 	TFTPServer       string
@@ -226,6 +227,12 @@ func withCNI(ctx context.Context, config *LaunchConfig, f func(config *LaunchCon
 		return fmt.Errorf("failed to insert iptables rule to allow broadcast traffic: %w", err)
 	}
 
+	for _, cidr := range config.NoMasqueradeCIDRs {
+		if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", cidr.String(), "-j", "ACCEPT"); err != nil {
+			return fmt.Errorf("failed to insert iptables rule to allow non-masquerade traffic to cidr %q: %w", cidr.String(), err)
+		}
+	}
+
 	config.tapName = tapIface.Name
 	config.vmMAC = vmIface.Mac
 	config.ns = ns

pkg/provision/providers/qemu/node.go

@@ -133,6 +133,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe
 		NetworkConfig:     state.VMCNIConfig,
 		CNI:               clusterReq.Network.CNI,
 		CIDRs:             clusterReq.Network.CIDRs,
+		NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs,
 		IPs:               nodeReq.IPs,
 		GatewayAddrs:      clusterReq.Network.GatewayAddrs,
 		MTU:               clusterReq.Network.MTU,

pkg/provision/request.go

@@ -58,11 +58,12 @@ type CNIConfig struct {
 
 // NetworkRequest describes cluster network.
 type NetworkRequest struct {
-	Name         string
-	CIDRs        []netip.Prefix
-	GatewayAddrs []netip.Addr
-	MTU          int
-	Nameservers  []netip.Addr
+	Name              string
+	CIDRs             []netip.Prefix
+	NoMasqueradeCIDRs []netip.Prefix
+	GatewayAddrs      []netip.Addr
+	MTU               int
+	Nameservers       []netip.Addr
 
 	LoadBalancerPorts []int
 

pkg/provision/result.go

@@ -38,10 +38,11 @@ type ClusterInfo struct {
 
 // NetworkInfo describes cluster network.
 type NetworkInfo struct {
-	Name         string
-	CIDRs        []netip.Prefix
-	GatewayAddrs []netip.Addr
-	MTU          int
+	Name              string
+	CIDRs             []netip.Prefix
+	GatewayAddrs      []netip.Addr
+	MTU               int
+	NoMasqueradeCIDRs []netip.Prefix
 }
 
 // NodeInfo describes a node.

website/content/v1.8/reference/cli.md

@@ -140,6 +140,7 @@ talosctl cluster create [flags]
       --memory-workers int                       the limit on memory usage in MB (each worker/VM) (default 2048)
       --mtu int                                  MTU of the cluster network (default 1500)
       --nameservers strings                      list of nameservers to use (default [8.8.8.8,1.1.1.1,2001:4860:4860::8888,2606:4700:4700::1111])
+      --no-masquerade-cidrs strings              list of CIDRs to exclude from NAT (QEMU provisioner only)
       --registry-insecure-skip-verify strings    list of registry hostnames to skip TLS verification for
       --registry-mirror strings                  list of registry mirrors to use in format: <registry host>=<mirror URL>
       --skip-boot-phase-finished-check           skip waiting for node to finish boot phase