[tunnel] Allocate v4 addrs via IPAM

Author: dilyevsky

api/core/v1alpha/tunnel_types.go

@@ -56,6 +56,10 @@ type AgentStatus struct {
 	// Overlay address of the agent. Currently we're using a /96 prefix which
 	// can be used for 4in6 tunneling.
 	AgentAddress string `json:"agentAddress,omitempty"`
+
+	// Extra addresses of the agent (for additional v4/v6 overlays, if configured).
+	// +optional
+	AgentAddresses []string `json:"agentAddresses,omitempty"`
 }
 
 type TunnelNodeCredentials struct {

api/core/v1alpha/zz_generated.deepcopy.go

@@ -33,7 +33,7 @@ type TunnelNodeReconciler struct {
 	jwtPrivateKeyPEM      []byte
 	jwtPublicKeyPEM       []byte
 	tokenRefreshThreshold time.Duration
-	agentIPAM             tunnet.IPAM
+	ipamv6, ipamv4        tunnet.IPAM
 
 	validator *token.InMemoryValidator
 	issuer    *token.Issuer
@@ -46,7 +46,7 @@ func NewTunnelNodeReconciler(
 	jwtPrivateKey []byte,
 	jwtPublicKey []byte,
 	tokenRefreshThreshold time.Duration,
-	agentIPAM tunnet.IPAM,
+	ipamv6, ipamv4 tunnet.IPAM,
 ) *TunnelNodeReconciler {
 	return &TunnelNodeReconciler{
 		Client:                c,
@@ -55,7 +55,8 @@ func NewTunnelNodeReconciler(
 		jwtPrivateKeyPEM:      jwtPrivateKey,
 		jwtPublicKeyPEM:       jwtPublicKey,
 		tokenRefreshThreshold: tokenRefreshThreshold,
-		agentIPAM:             agentIPAM,
+		ipamv6:                ipamv6,
+		ipamv4:                ipamv4,
 	}
 }
 
@@ -133,7 +134,7 @@ func (r *TunnelNodeReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 				log.Error(err, "Failed to parse IP address", "addr", agent.AgentAddress)
 				continue
 			}
-			if err := r.agentIPAM.Release(netip.PrefixFrom(addr, 96)); err != nil {
+			if err := r.ipamv6.Release(netip.PrefixFrom(addr, 96)); err != nil {
 				log.Error(err, "Failed to release IP address", "addr", addr)
 			}
 		}
@@ -192,18 +193,27 @@ func (r *TunnelNodeReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 			continue
 		}
 
-		addr, err := r.agentIPAM.Allocate()
+		addrv6, err := r.ipamv6.Allocate()
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to allocate agent address: %w", err)
 		}
 
-		log.Info("Allocated agent address", "agent", agent.Name, "addr", addr)
+		log.Info("Allocated agent v6 address", "agent", agent.Name, "addr", addrv6)
 
-		tn.Status.Agents[i].AgentAddress = addr.Addr().String()
+		tn.Status.Agents[i].AgentAddress = addrv6.Addr().String()
+
+		addrv4, err := r.ipamv4.Allocate()
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to allocate agent address: %w", err)
+		}
+
+		log.Info("Allocated agent v4 address", "agent", agent.Name, "addr", addrv4)
+
+		tn.Status.Agents[i].AgentAddresses = append(tn.Status.Agents[i].AgentAddresses, addrv4.Addr().String())
 
 		if err := r.Status().Update(ctx, tn); err != nil {
-			if err := r.agentIPAM.Release(addr); err != nil {
-				log.Error(err, "Failed to release agent address", "agent", agent.Name, "addr", addr)
+			if err := r.ipamv6.Release(addrv6); err != nil {
+				log.Error(err, "Failed to release agent address", "agent", agent.Name, "addr", addrv6)
 			}
 			return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err)
 		}

api/generated/zz_generated.openapi.go

@@ -44,7 +44,7 @@ func TestTunnelNodeReconciler(t *testing.T) {
 
 	systemULA := tunnet.NewULA(context.Background(), net.SystemNetworkID)
 	// Agent prefixes are /96 subnets that can embed IPv4 suffixes.
-	agentIPAM, err := systemULA.IPAM(context.Background(), 96)
+	ipamv6, err := systemULA.IPAM(context.Background(), 96)
 	require.NoError(t, err)
 
 	r := NewTunnelNodeReconciler(
@@ -54,15 +54,15 @@ func TestTunnelNodeReconciler(t *testing.T) {
 		privKey,
 		pubKey,
 		time.Minute,
-		agentIPAM,
+		ipamv6,
+		net.NewIPAMv4(context.Background()),
 	)
 
 	r.validator, err = token.NewInMemoryValidator(r.jwtPublicKeyPEM)
 	require.NoError(t, err)
 	r.issuer, err = token.NewIssuer(r.jwtPrivateKeyPEM)
 	require.NoError(t, err)
 
-	// Call the reconcile method.
 	_, err = r.Reconcile(context.TODO(), reconcile.Request{
 		NamespacedName: types.NamespacedName{
 			Name:      "test-tunnelnode",

pkg/apiserver/controllers/tunnelnode.go

@@ -47,7 +47,7 @@ import (
 	"github.com/apoxy-dev/apoxy/pkg/cryptoutils"
 	gw "github.com/apoxy-dev/apoxy/pkg/gateway"
 	"github.com/apoxy-dev/apoxy/pkg/log"
-	"github.com/apoxy-dev/apoxy/pkg/tunnel/net"
+	apoxynet "github.com/apoxy-dev/apoxy/pkg/tunnel/net"
 	tunnet "github.com/apoxy-dev/apoxy/pkg/tunnel/net"
 
 	ctrlv1alpha1 "github.com/apoxy-dev/apoxy/api/controllers/v1alpha1"
@@ -352,15 +352,15 @@ func encodeSQLiteConnArgs(args map[string]string) string {
 
 // defaultOptions returns default options.
 func defaultOptions(ctx context.Context) (*options, error) {
-	systemULA := tunnet.NewULA(ctx, net.SystemNetworkID)
+	systemULA := tunnet.NewULA(ctx, apoxynet.SystemNetworkID)
 	// Agent prefixes are /96 subnets that can embed IPv4 suffixes.
 	agentIPAM, err := systemULA.IPAM(ctx, 96)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create agent IPAM: %w", err)
 	}
 
 	// Proxy prefixes are /128 leafs under reserved proxy endpoint.
-	epULA, err := systemULA.WithEndpoint(ctx, net.ProxyEndpointID)
+	epULA, err := systemULA.WithEndpoint(ctx, apoxynet.ProxyEndpointID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create proxy endpoint: %w", err)
 	}
@@ -486,6 +486,7 @@ func (m *Manager) Start(
 		dOpts.jwtPublicKey,
 		dOpts.jwtRefreshThreshold,
 		dOpts.agentIPAM,
+		apoxynet.NewIPAMv4(context.Background()),
 	)
 	if err := tunnelNodeReconciler.SetupWithManager(ctx, m.manager); err != nil {
 		return fmt.Errorf("failed to set up TunnelNode controller: %v", err)

pkg/apiserver/controllers/tunnelnode_test.go

@@ -586,7 +586,7 @@ func (t *TunnelServer) reconcile(ctx context.Context, request reconcile.Request)
 		}
 
 		// Check if we already allocated and assigned addresses.
-		if conn.addrv6.IsValid() /*&& conn.addrv4.IsValid()*/ {
+		if conn.addrv6.IsValid() && conn.addrv4.IsValid() {
 			log.V(1).Info("Agent address is already assigned")
 			continue
 		}
@@ -606,18 +606,18 @@ func (t *TunnelServer) reconcile(ctx context.Context, request reconcile.Request)
 		}
 		t.conns.Set(agent.Name, conn)
 
-		// TODO(dilyevsky): v4 prefix should also be delivered via agent status
-		// struct (needs api change to support multiple addresses).
-		/*
-			if !conn.addrv4.IsValid() {
-				var err error
-				conn.addrv4, err = t.options.ipamv4.Allocate()
-				if err != nil {
-					log.Error(err, "Failed to allocate IPv4 address")
-					continue
+		if !conn.addrv4.IsValid() {
+			for _, agentAddr := range agent.AgentAddresses {
+				if addr, err := netip.ParseAddr(agentAddr); err == nil && addr.Is4() {
+					conn.addrv4 = netip.PrefixFrom(addr, 32)
+					break
 				}
 			}
-		*/
+			if !conn.addrv4.IsValid() {
+				log.Info("No IPv4 address allocated")
+				continue
+			}
+		}
 		t.conns.Set(agent.Name, conn)
 
 		log.Info("Assigned addresses to connection",
@@ -642,18 +642,16 @@ func (t *TunnelServer) reconcile(ctx context.Context, request reconcile.Request)
 			metrics.TunnelConnectionFailures.WithLabelValues("route_addition_failed").Inc()
 			return reconcile.Result{}, nil
 		}
-		/*
-			if err := t.router.AddAddr(conn.addrv4, conn); err != nil {
-				log.Error(err, "Failed to add TUN peer")
-				metrics.TunnelConnectionFailures.WithLabelValues("tun_peer_add_failed").Inc()
-				return reconcile.Result{}, nil
-			}
-			if err := t.router.AddRoute(conn.addrv4); err != nil {
-				log.Error(err, "Failed to add route")
-				metrics.TunnelConnectionFailures.WithLabelValues("route_addition_failed").Inc()
-				return reconcile.Result{}, nil
-			}
-		*/
+		if err := t.router.AddAddr(conn.addrv4, conn); err != nil {
+			log.Error(err, "Failed to add TUN peer")
+			metrics.TunnelConnectionFailures.WithLabelValues("tun_peer_add_failed").Inc()
+			return reconcile.Result{}, nil
+		}
+		if err := t.router.AddRoute(conn.addrv4); err != nil {
+			log.Error(err, "Failed to add route")
+			metrics.TunnelConnectionFailures.WithLabelValues("route_addition_failed").Inc()
+			return reconcile.Result{}, nil
+		}
 
 		metrics.TunnelConnectionsActive.Inc()
 		defer metrics.TunnelConnectionsActive.Dec()