Merge pull request #1145 from stgraber/ovn

Fix OVN interconnect ECMP handling

internal/server/db/cluster/schema.go

@@ -334,7 +334,6 @@ CREATE TABLE "networks_peers_config" (
     UNIQUE (network_peer_id, key),
     FOREIGN KEY (network_peer_id) REFERENCES "networks_peers" (id) ON DELETE CASCADE
 );
-CREATE UNIQUE INDEX networks_peers_unique_network_id_target_network_integration_id ON "networks_peers" (network_id, target_network_integration_id);
 CREATE UNIQUE INDEX networks_unique_network_id_node_id_key ON "networks_config" (network_id, IFNULL(node_id, -1), key);
 CREATE TABLE "networks_zones" (
     id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
@@ -657,5 +656,5 @@ CREATE TABLE "warnings" (
 );
 CREATE UNIQUE INDEX warnings_unique_node_id_project_id_entity_type_code_entity_id_type_code ON warnings(IFNULL(node_id, -1), IFNULL(project_id, -1), entity_type_code, entity_id, type_code);
 
-INSERT INTO schema (version, updated_at) VALUES (74, strftime("%s"))
+INSERT INTO schema (version, updated_at) VALUES (75, strftime("%s"))
 `

internal/server/db/cluster/update.go

@@ -111,6 +111,18 @@ var updates = map[int]schema.Update{
 	72: updateFromV71,
 	73: updateFromV72,
 	74: updateFromV73,
+	75: updateFromV74,
+}
+
+// updateFromV74 removes the index preventing the same integration to be used multiple times.
+func updateFromV74(ctx context.Context, tx *sql.Tx) error {
+	q := `DROP INDEX IF EXISTS networks_peers_unique_network_id_target_network_integration_id;`
+	_, err := tx.Exec(q)
+	if err != nil {
+		return fmt.Errorf("Failed dropping network peer index: %w", err)
+	}
+
+	return nil
 }
 
 // updateFromV73 adds a config table to cluster groups.

internal/server/network/driver_ovn.go

@@ -2,10 +2,8 @@ package network
 
 import (
 	"context"
-	"encoding/binary"
 	"fmt"
 	"math/big"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -5723,27 +5721,17 @@ func (n *ovn) remotePeerCreate(peer api.NetworkPeersPost) error {
 		bridgeMTU = 1500
 	}
 
-	// EUI64 for IPv6.
-	ipv6, err := eui64.ParseMAC(net.ParseIP("fd80::"), routerMAC)
+	// Get peering addresses.
+	ipv4Net, ipv6Net, err := icnb.CreateTransitSwitchAllocation(ctx, string(tsName), azName)
 	if err != nil {
 		return err
 	}
 
-	ipv6Net := net.IPNet{IP: ipv6, Mask: net.CIDRMask(64, 128)}
-
-	// Random IPv4.
-	buf := make([]byte, 4)
-	binary.LittleEndian.PutUint32(buf, rand.Uint32())
-	buf[0] = 169
-	buf[1] = 254
-	ipv4 := net.IP(buf)
-	ipv4Net := net.IPNet{IP: ipv4, Mask: net.CIDRMask(16, 32)}
-
 	// Determine logical router port name.
 	lrpName := networkOVN.OVNRouterPort(tsName)
 
 	// Create the logical router port.
-	err = n.ovnnb.CreateLogicalRouterPort(ctx, n.getRouterName(), lrpName, routerMAC, uint32(bridgeMTU), []*net.IPNet{&ipv4Net, &ipv6Net}, cgName, false)
+	err = n.ovnnb.CreateLogicalRouterPort(ctx, n.getRouterName(), lrpName, routerMAC, uint32(bridgeMTU), []*net.IPNet{ipv4Net, ipv6Net}, cgName, false)
 	if err != nil {
 		return err
 	}
@@ -6166,6 +6154,18 @@ func (n *ovn) remotePeerDelete(peer *api.NetworkPeer) error {
 		if err != nil && err != networkOVN.ErrNotManaged {
 			return err
 		}
+	} else {
+		// Get the OVN AZ name.
+		azName, err := n.ovnnb.GetName(ctx)
+		if err != nil {
+			return err
+		}
+
+		// Release peering addresses.
+		err = icnb.DeleteTransitSwitchAllocation(ctx, string(tsName), azName)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil

internal/server/network/ovn/ovn_icnb_actions.go

@@ -2,6 +2,13 @@ package ovn
 
 import (
 	"context"
+	"encoding/binary"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/netip"
+	"slices"
+	"strings"
 
 	ovsdbClient "github.com/ovn-org/libovsdb/client"
 	"github.com/ovn-org/libovsdb/ovsdb"
@@ -30,8 +37,20 @@ func (o *ICNB) CreateTransitSwitch(ctx context.Context, name string, mayExist bo
 		return nil
 	}
 
+	// Generate a random IPv4 subnet (/28).
+	buf := make([]byte, 4)
+	binary.LittleEndian.PutUint32(buf, rand.Uint32())
+	buf[0] = 169
+	buf[1] = 254
+	ipv4 := net.IP(buf)
+	ipv4Net := net.IPNet{IP: ipv4.Mask(net.CIDRMask(28, 32)), Mask: net.CIDRMask(28, 32)}
+
 	// Mark new switches as managed by Incus.
-	transitSwitch.ExternalIDs = map[string]string{"incus-managed": "true"}
+	transitSwitch.ExternalIDs = map[string]string{
+		"incus-managed":     "true",
+		"incus-subnet-ipv4": ipv4Net.String(),
+		"incus-subnet-ipv6": fmt.Sprintf("fd42:%x:%x:%x::/64", rand.Intn(65535), rand.Intn(65535), rand.Intn(65535)),
+	}
 
 	// Create the switch.
 	ops, err := o.client.Create(&transitSwitch)
@@ -52,6 +71,140 @@ func (o *ICNB) CreateTransitSwitch(ctx context.Context, name string, mayExist bo
 	return nil
 }
 
+// CreateTransitSwitchAllocation creates a new allocation on the switch.
+func (o *ICNB) CreateTransitSwitchAllocation(ctx context.Context, switchName string, azName string) (*net.IPNet, *net.IPNet, error) {
+	// Get the switch.
+	transitSwitch := ovnICNB.TransitSwitch{
+		Name: switchName,
+	}
+
+	err := o.client.Get(ctx, &transitSwitch)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Check that it's managed by us.
+	if transitSwitch.ExternalIDs == nil || transitSwitch.ExternalIDs["incus-managed"] != "true" {
+		return nil, nil, fmt.Errorf("Transit switch isn't Incus managed")
+	}
+
+	// Check that prefixes are set.
+	if transitSwitch.ExternalIDs["incus-subnet-ipv4"] == "" || transitSwitch.ExternalIDs["incus-subnet-ipv6"] == "" {
+		return nil, nil, fmt.Errorf("No configured subnets on the transit switch")
+	}
+
+	// Get the allocated addresses.
+	v4Addresses := []string{}
+	v6Addresses := []string{}
+	for k, v := range transitSwitch.ExternalIDs {
+		if !strings.HasPrefix(k, "incus-allocation-") {
+			continue
+		}
+
+		fields := strings.Split(v, ",")
+		if len(fields) != 2 {
+			continue
+		}
+
+		v4Addresses = append(v4Addresses, fields[0])
+		v6Addresses = append(v6Addresses, fields[1])
+	}
+
+	// Get the prefixes.
+	v4Prefix, err := netip.ParsePrefix(transitSwitch.ExternalIDs["incus-subnet-ipv4"])
+	if err != nil {
+		return nil, nil, err
+	}
+
+	v6Prefix, err := netip.ParsePrefix(transitSwitch.ExternalIDs["incus-subnet-ipv6"])
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Allocate new IPs in the subnet.
+	v4Addr := v4Prefix.Addr()
+	for {
+		v4Addr = v4Addr.Next()
+		if !v4Prefix.Contains(v4Addr) {
+			return nil, nil, fmt.Errorf("Transit switch is out of IPv4 addresses")
+		}
+
+		if !slices.Contains(v4Addresses, v4Addr.String()) {
+			break
+		}
+	}
+
+	v6Addr := v6Prefix.Addr()
+	for {
+		v6Addr = v6Addr.Next()
+		if !v6Prefix.Contains(v6Addr) {
+			return nil, nil, fmt.Errorf("Transit switch is out of IPv6 addresses")
+		}
+
+		if !slices.Contains(v6Addresses, v6Addr.String()) {
+			break
+		}
+	}
+
+	// Update the record.
+	transitSwitch.ExternalIDs[fmt.Sprintf("incus-allocation-%s", azName)] = fmt.Sprintf("%s,%s", v4Addr.String(), v6Addr.String())
+
+	ops, err := o.client.Where(&transitSwitch).Update(&transitSwitch)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	resp, err := o.client.Transact(ctx, ops...)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	_, err = ovsdb.CheckOperationResults(resp, ops)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &net.IPNet{IP: net.IP(v4Addr.AsSlice()), Mask: net.CIDRMask(v4Prefix.Bits(), 32)}, &net.IPNet{IP: net.IP(v6Addr.AsSlice()), Mask: net.CIDRMask(v6Prefix.Bits(), 128)}, nil
+}
+
+// DeleteTransitSwitchAllocation removes a current allocation from the switch.
+func (o *ICNB) DeleteTransitSwitchAllocation(ctx context.Context, switchName string, azName string) error {
+	// Get the switch.
+	transitSwitch := ovnICNB.TransitSwitch{
+		Name: switchName,
+	}
+
+	err := o.client.Get(ctx, &transitSwitch)
+	if err != nil {
+		return err
+	}
+
+	// Check that it's managed by us.
+	if transitSwitch.ExternalIDs == nil || transitSwitch.ExternalIDs["incus-managed"] != "true" {
+		return fmt.Errorf("Transit switch isn't Incus managed")
+	}
+
+	// Update the record.
+	delete(transitSwitch.ExternalIDs, fmt.Sprintf("incus-allocation-%s", azName))
+
+	ops, err := o.client.Where(&transitSwitch).Update(&transitSwitch)
+	if err != nil {
+		return err
+	}
+
+	resp, err := o.client.Transact(ctx, ops...)
+	if err != nil {
+		return err
+	}
+
+	_, err = ovsdb.CheckOperationResults(resp, ops)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // DeleteTransitSwitch deletes an existing transit switch.
 // The force parameter is required to delete a transit switch which wasn't created by Incus.
 func (o *ICNB) DeleteTransitSwitch(ctx context.Context, name string, force bool) error {