net: publish node assignments and replace stale agents

api/net/v1alpha1/types.go

@@ -19,6 +19,18 @@ const (
 	// and read intentionally until then.
 	SiteLabelKey = "net.unbounded-cloud.io/site"
 
+	// GatewayPoolLabelKey identifies the single GatewayPool assignment adopted
+	// by the node agent. An absent label means the node is a worker.
+	GatewayPoolLabelKey = "net.unbounded-cloud.io/gateway-pool"
+
+	// EffectiveSiteAnnotation records the Site adopted by a node-agent Pod.
+	EffectiveSiteAnnotation = "net.unbounded-cloud.io/effective-site"
+
+	// EffectiveGatewayPoolAnnotation records the GatewayPool adopted by a
+	// node-agent Pod. A present empty value means the worker role; an absent
+	// key means acknowledgment is pending.
+	EffectiveGatewayPoolAnnotation = "net.unbounded-cloud.io/effective-gateway-pool"
+
 	// NodeDiscoveredPublicIPAnnotation contains the translated source IP
 	// discovered by the node agent. It does not prove that the WireGuard port is
 	// reachable through symmetric NAT or CGNAT.

cmd/unbounded-net-controller/main.go

@@ -727,6 +727,21 @@ func run(cfg *config.Config, forceNotLeader bool) error {
 			}()
 		}
 
+		nodeAssignmentCtrl, err := controller.NewNodeAssignmentController(
+			clientset,
+			informerFactory.Core().V1().Nodes(),
+			nodeAuthCaches.pods,
+		)
+		if err != nil {
+			klog.Errorf("Failed to create Node assignment controller: %v", err)
+		} else {
+			go func() {
+				if err := nodeAssignmentCtrl.Run(ctx, 2); err != nil {
+					klog.Errorf("Node assignment controller error: %v", err)
+				}
+			}()
+		}
+
 		// Create and start peering aggregation controller
 		peeringAggCtrl, err := controller.NewPeeringAggregationController(dynamicClient, dynamicInformerFactory)
 		if err != nil {

cmd/unbounded-net-controller/node_auth_informers.go

@@ -5,14 +5,17 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
 
 	"github.com/Azure/unbounded/internal/net/authn"
+	netcontroller "github.com/Azure/unbounded/internal/net/controller"
 )
 
 // These watches run on every serving replica, independently of leadership.
@@ -43,6 +46,12 @@ func newNodeAuthInformers(ctx context.Context, client kubernetes.Interface, name
 	result.pods.Informer()
 	result.serviceAccounts.Informer()
 
+	if err := result.pods.Informer().AddIndexers(cache.Indexers{
+		netcontroller.NodeAgentPodNodeNameIndex: netcontroller.NodeAgentPodNodeNameIndexFunc,
+	}); err != nil {
+		panic(fmt.Sprintf("add node-agent Pod Node index: %v", err))
+	}
+
 	return result
 }
 

cmd/unbounded-net-controller/node_auth_informers_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 
 	"github.com/Azure/unbounded/internal/net/authn"
+	netcontroller "github.com/Azure/unbounded/internal/net/controller"
 )
 
 func nodeAuthObjects() (*corev1.Pod, *corev1.ServiceAccount, *authn.KubernetesServiceAccountIdentity) {
@@ -182,6 +183,11 @@ func TestNodeAuthInformersScopeAndLifecycle(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	indexed, err := caches.pods.Informer().GetIndexer().ByIndex(netcontroller.NodeAgentPodNodeNameIndex, pod.Spec.NodeName)
+	if err != nil || len(indexed) != 1 || indexed[0].(*corev1.Pod).UID != pod.UID {
+		t.Fatalf("node-agent Pods indexed by Node = %#v, err=%v", indexed, err)
+	}
+
 	if _, err := caches.serviceAccounts.Lister().ServiceAccounts(sa.Namespace).Get(sa.Name); err != nil {
 		t.Fatalf("unlabeled service account must be watched: %v", err)
 	}

deploy/net/controller/02-rbac.yaml.tmpl

@@ -13,8 +13,9 @@ rules:
   # Nodes: read, patch, and update for CIDR assignment and site labeling.
   # Cannot be further scoped -- the controller must operate on any node in the cluster.
   # Fields restricted by ValidatingAdmissionPolicy (09-vap.yaml.tmpl):
-  #   labels/annotations: only net.unbounded-cloud.io/* prefix
-  #   spec: only podCIDR and podCIDRs (Kubernetes enforces immutability after initial set)
+  #   labels: only net.unbounded-cloud.io/* and the canonical Site label
+  #   annotations: only net.unbounded-cloud.io/*
+  #   spec: only pod CIDRs, plus the node agent's exact gateway taint
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "watch", "patch", "update"]
@@ -186,10 +187,12 @@ rules:
     resources: ["services"]
     resourceNames: ["unbounded-net-controller"]
     verbs: ["get"]
-  # Pods: dashboard status and local OIDC bound-object validation.
+  # Pods: dashboard status, local OIDC bound-object validation, and targeted
+  # replacement of node-agent Pods after acknowledged assignment changes.
+  # ValidatingAdmissionPolicy restricts delete to DaemonSet-owned node Pods.
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch", "delete"]
   # ServiceAccounts: local OIDC bound-object validation (controller namespace only).
   - apiGroups: [""]
     resources: ["serviceaccounts"]

deploy/net/controller/09-vap.yaml.tmpl

@@ -63,11 +63,10 @@ spec:
 
 ---
 # Restrict what fields unbounded-net service accounts may modify on Node objects.
-# The controller and node agent may ONLY modify:
-#   - metadata.labels with the net.unbounded-cloud.io/ prefix
-#   - metadata.annotations with the net.unbounded-cloud.io/ prefix
-#   - spec.podCIDR and spec.podCIDRs (Kubernetes enforces immutability after initial set)
-# All other node fields (spec.taints, spec.unschedulable, status, etc.) must remain unchanged.
+# The controller owns networking labels and pod CIDRs.
+# The node agent may only modify its public key, MTU, discovery annotations,
+# and exact gateway taint on its token-bound Node.
+# All other node fields (spec.unschedulable, status, etc.) must remain unchanged.
 # The CEL expressions iterate over map keys dynamically rather than enumerating fields,
 # so new fields added to the Node spec in future Kubernetes versions are automatically protected.
 apiVersion: admissionregistration.k8s.io/v1
@@ -91,14 +90,41 @@ spec:
         request.userInfo.username == 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-controller' ||
         request.userInfo.username == 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node'
   validations:
+    - expression: >-
+        request.userInfo.username != 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' ||
+        object.metadata.?labels.orValue({}) == oldObject.metadata.?labels.orValue({})
+      message: "unbounded-net-node may not modify Node labels"
+    - expression: >-
+        request.userInfo.username != 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' ||
+        (object.spec.?podCIDR.orValue('') == oldObject.spec.?podCIDR.orValue('') &&
+          object.spec.?podCIDRs.orValue([]) == oldObject.spec.?podCIDRs.orValue([]))
+      message: "unbounded-net-node may not modify controller-owned pod CIDRs"
     # Status must be unchanged.
     - expression: "object.status == oldObject.status"
       message: "unbounded-net may not modify node status"
-    # Spec: only podCIDR and podCIDRs may change. All other spec fields are protected.
+    # Spec: only podCIDR and podCIDRs may change generally. The node agent may
+    # additionally add or remove its exact gateway taint.
+    - expression: >-
+        object.spec.all(k,
+          k in ['podCIDR', 'podCIDRs'] ||
+          (request.userInfo.username == 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' && k == 'taints') ||
+          (k in oldObject.spec && object.spec[k] == oldObject.spec[k])) &&
+        oldObject.spec.all(k,
+          k in ['podCIDR', 'podCIDRs'] ||
+          (request.userInfo.username == 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' && k == 'taints') ||
+          (k in object.spec && oldObject.spec[k] == object.spec[k]))
+      message: "unbounded-net may only modify pod CIDRs and the node agent's gateway taint"
     - expression: >-
-        object.spec.all(k, k in ['podCIDR', 'podCIDRs'] || (k in oldObject.spec && object.spec[k] == oldObject.spec[k])) &&
-        oldObject.spec.all(k, k in ['podCIDR', 'podCIDRs'] || (k in object.spec && oldObject.spec[k] == object.spec[k]))
-      message: "unbounded-net may only modify spec.podCIDR and spec.podCIDRs on nodes"
+        request.userInfo.username != 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' ||
+        (request.userInfo.extra[?'authentication.kubernetes.io/node-name'][0].orValue('') == object.metadata.name &&
+          request.userInfo.extra[?'authentication.kubernetes.io/node-uid'][0].orValue('') == object.metadata.uid &&
+          object.spec.?taints.orValue([]).filter(t, !(t.key == 'net.unbounded-cloud.io/gateway-node' &&
+            t.?value.orValue('') == 'true' && t.effect == 'NoSchedule')) ==
+            oldObject.spec.?taints.orValue([]).filter(t, !(t.key == 'net.unbounded-cloud.io/gateway-node' &&
+              t.?value.orValue('') == 'true' && t.effect == 'NoSchedule')) &&
+          object.spec.?taints.orValue([]).filter(t, t.key == 'net.unbounded-cloud.io/gateway-node' &&
+            t.?value.orValue('') == 'true' && t.effect == 'NoSchedule').size() <= 1)
+      message: "unbounded-net-node may only add or remove its exact gateway taint on its own Node"
     - expression: >-
         request.userInfo.username != 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' ||
         (((!has(object.metadata.annotations) || !('net.unbounded-cloud.io/wireguard-port' in object.metadata.annotations)) &&
@@ -133,6 +159,19 @@ spec:
           oldObject.metadata.annotations.all(k, k.startsWith('net.unbounded-cloud.io/') ||
             (k in object.metadata.annotations && oldObject.metadata.annotations[k] == object.metadata.annotations[k])))
       message: "unbounded-net may only modify annotations with net.unbounded-cloud.io/ prefix"
+    - expression: >-
+        request.userInfo.username != 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node' ||
+        (object.metadata.?annotations.orValue({}).all(k,
+          k in ['net.unbounded-cloud.io/wg-pubkey', 'net.unbounded-cloud.io/tunnel-mtu',
+            'net.unbounded-cloud.io/discovered-public-ip', 'net.unbounded-cloud.io/discovered-public-ip-expires-at'] ||
+          (k in oldObject.metadata.?annotations.orValue({}) &&
+            object.metadata.annotations[k] == oldObject.metadata.annotations[k])) &&
+        oldObject.metadata.?annotations.orValue({}).all(k,
+          k in ['net.unbounded-cloud.io/wg-pubkey', 'net.unbounded-cloud.io/tunnel-mtu',
+            'net.unbounded-cloud.io/discovered-public-ip', 'net.unbounded-cloud.io/discovered-public-ip-expires-at'] ||
+          (k in object.metadata.?annotations.orValue({}) &&
+            oldObject.metadata.annotations[k] == object.metadata.annotations[k])))
+      message: "unbounded-net-node may only modify agent-owned Node annotations"
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingAdmissionPolicyBinding
@@ -145,6 +184,40 @@ spec:
   policyName: unbounded-net-node-field-restriction
   validationActions: ["Deny"]
 
+---
+# GatewayPool names are written verbatim into Node label values. Names are
+# immutable, so only CREATE is checked; existing longer pools can still be
+# updated, drained, and finalized.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: unbounded-net-gateway-pool-name
+  labels:
+    app.kubernetes.io/name: unbounded-net-controller
+    app.kubernetes.io/component: controller
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+      - apiGroups: ["net.unbounded-cloud.io"]
+        apiVersions: ["v1alpha1"]
+        operations: ["CREATE"]
+        resources: ["gatewaypools"]
+  validations:
+    - expression: "object.metadata.name.size() <= 63"
+      message: "GatewayPool names must fit the 63-character Kubernetes label value limit"
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: unbounded-net-gateway-pool-name
+  labels:
+    app.kubernetes.io/name: unbounded-net-controller
+    app.kubernetes.io/component: controller
+spec:
+  policyName: unbounded-net-gateway-pool-name
+  validationActions: ["Deny"]
+
 ---
 # Protect the ownership of public IP annotations on every Node. The discovered
 # address and its expiry belong to the agent running on that Node. The declared
@@ -196,3 +269,110 @@ metadata:
 spec:
   policyName: unbounded-net-public-ip-annotation-ownership
   validationActions: ["Deny"]
+
+---
+# Restrict node-agent Pod updates to effective assignment acknowledgment by
+# the service account token bound to that exact Pod.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: unbounded-net-effective-assignment-ownership
+  labels:
+    app.kubernetes.io/name: unbounded-net-node
+    app.kubernetes.io/component: node
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["UPDATE"]
+        resources: ["pods"]
+  matchConditions:
+    - name: "is-unbounded-net-node-sa"
+      expression: "request.userInfo.username == 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-node'"
+  validations:
+    - expression: >-
+        request.userInfo.extra[?'authentication.kubernetes.io/pod-name'][0].orValue('') == object.metadata.name &&
+        request.userInfo.extra[?'authentication.kubernetes.io/pod-uid'][0].orValue('') == object.metadata.uid
+      message: "unbounded-net-node may only acknowledge assignment on its own bound Pod"
+    - expression: >-
+        object.spec == oldObject.spec &&
+        has(object.status) == has(oldObject.status) &&
+        (!has(object.status) || object.status == oldObject.status) &&
+        has(object.metadata.labels) == has(oldObject.metadata.labels) &&
+        (!has(object.metadata.labels) || object.metadata.labels == oldObject.metadata.labels) &&
+        has(object.metadata.ownerReferences) == has(oldObject.metadata.ownerReferences) &&
+        (!has(object.metadata.ownerReferences) || object.metadata.ownerReferences == oldObject.metadata.ownerReferences) &&
+        has(object.metadata.finalizers) == has(oldObject.metadata.finalizers) &&
+        (!has(object.metadata.finalizers) || object.metadata.finalizers == oldObject.metadata.finalizers)
+      message: "unbounded-net-node may only modify effective assignment annotations on its own Pod"
+    - expression: >-
+        (!has(object.metadata.annotations) && !has(oldObject.metadata.annotations)) ||
+        (!has(oldObject.metadata.annotations) && object.metadata.annotations.all(k,
+          k in ['net.unbounded-cloud.io/effective-site', 'net.unbounded-cloud.io/effective-gateway-pool'])) ||
+        (!has(object.metadata.annotations) && oldObject.metadata.annotations.all(k,
+          k in ['net.unbounded-cloud.io/effective-site', 'net.unbounded-cloud.io/effective-gateway-pool'])) ||
+        (has(object.metadata.annotations) && has(oldObject.metadata.annotations) &&
+          object.metadata.annotations.all(k,
+            k in ['net.unbounded-cloud.io/effective-site', 'net.unbounded-cloud.io/effective-gateway-pool'] ||
+            (k in oldObject.metadata.annotations && object.metadata.annotations[k] == oldObject.metadata.annotations[k])) &&
+          oldObject.metadata.annotations.all(k,
+            k in ['net.unbounded-cloud.io/effective-site', 'net.unbounded-cloud.io/effective-gateway-pool'] ||
+            (k in object.metadata.annotations && oldObject.metadata.annotations[k] == object.metadata.annotations[k])))
+      message: "unbounded-net-node may only modify effective assignment annotations"
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: unbounded-net-effective-assignment-ownership
+  labels:
+    app.kubernetes.io/name: unbounded-net-node
+    app.kubernetes.io/component: node
+spec:
+  policyName: unbounded-net-effective-assignment-ownership
+  validationActions: ["Deny"]
+
+---
+# Restrict controller Pod deletion to the node-agent Pods owned by the
+# unbounded-net-node DaemonSet.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: unbounded-net-node-pod-delete-restriction
+  labels:
+    app.kubernetes.io/name: unbounded-net-controller
+    app.kubernetes.io/component: controller
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["DELETE"]
+        resources: ["pods"]
+  matchConditions:
+    - name: "is-unbounded-net-controller-sa"
+      expression: "request.userInfo.username == 'system:serviceaccount:{{ default "unbounded-system" .Namespace }}:unbounded-net-controller'"
+  validations:
+    - expression: >-
+        oldObject.metadata.namespace == '{{ default "unbounded-system" .Namespace }}' &&
+        oldObject.metadata.labels['app.kubernetes.io/name'] == 'unbounded-net-node' &&
+        oldObject.metadata.labels['app.kubernetes.io/component'] == 'node' &&
+        oldObject.metadata.ownerReferences.exists(owner,
+          owner.controller == true &&
+          owner.apiVersion == 'apps/v1' &&
+          owner.kind == 'DaemonSet' &&
+          owner.name == 'unbounded-net-node')
+      message: "unbounded-net controller may only delete DaemonSet-owned unbounded-net-node Pods"
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: unbounded-net-node-pod-delete-restriction
+  labels:
+    app.kubernetes.io/name: unbounded-net-controller
+    app.kubernetes.io/component: controller
+spec:
+  policyName: unbounded-net-node-pod-delete-restriction
+  validationActions: ["Deny"]

deploy/net/controller/rbac_test.go

@@ -62,6 +62,10 @@ func TestBoundObjectCacheRBAC(t *testing.T) {
 							t.Fatalf("missing Pod %s permission in controller namespace", verb)
 						}
 					}
+
+					if !grantsPodRead(role.Rules, "delete") {
+						t.Fatal("missing targeted Pod delete permission in controller namespace")
+					}
 				}
 			}