Make idle reconciles issue no Kubernetes API requests

Cargo.toml

@@ -65,6 +65,7 @@ opentelemetry_sdk = "0.32.1"
 opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["tokio", "grpc-tonic", "trace"], optional = true }
 prometheus = "0.14.0"
 prost = "0.14.1"
+rand = "0.9"
 regex = "1.12.2"
 reqwest = { version = "0.12.28", default-features = false, features = ["json", "rustls-tls", ] }
 schemars = { version = "1.2.0", features = ["chrono04"] }

README.md

@@ -851,6 +851,18 @@ docker tag alpine:3.21 my-private-registry.example.com/alpine:3.21
 docker push my-private-registry.example.com/alpine:3.21
 ```
 
+### Resync Interval
+
+Every object the operator writes is watched, so out-of-band changes are undone as soon as the
+watch reports them, and reconciles that find nothing to change make no Kubernetes API requests.
+On top of that, each resource that reconciled cleanly is reconciled again every 30 minutes
+(±10% jitter) as a backstop for state the watches can't see, such as Restate's own view of a
+deployment. Change it with the `resyncIntervalSeconds` Helm value:
+
+```yaml
+resyncIntervalSeconds: 600
+```
+
 ### EKS Security Groups for Pods
 
 [EKS Security Groups for Pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) allows

charts/restate-operator-helm/templates/deployment.yaml

@@ -86,6 +86,10 @@ spec:
             - name: CLUSTER_DNS
               value: {{ .Values.clusterDns }}
           {{- end }}
+          {{- if .Values.resyncIntervalSeconds }}
+            - name: RESYNC_INTERVAL_SECONDS
+              value: {{ .Values.resyncIntervalSeconds | quote }}
+          {{- end }}
           {{- with .Values.env }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

charts/restate-operator-helm/values.yaml

@@ -20,6 +20,9 @@ awsPodIdentityAssociationCluster: null
 gcpWorkloadIdentity: null
 clusterDns: null # defaults to "cluster.local" in the operator binary
 canaryImage: null # defaults to "alpine:3.21"; image must provide cat, grep, wget and a CA bundle at /etc/ssl/certs/ca-certificates.crt
+# How often a resource that reconciled cleanly is reconciled again with no change to prompt it, in seconds.
+# Defaults to 1800 in the operator binary. Each resync is jittered by +/-10%.
+resyncIntervalSeconds: null
 
 podSecurityContext:
   fsGroup: 2000

release-notes/unreleased/fewer-idle-api-requests.md

@@ -0,0 +1,33 @@
+# Release Notes: Far fewer Kubernetes API requests when nothing has changed
+
+## Behavioral Change
+
+### What Changed
+- Idle reconciles no longer write to the Kubernetes API. The operator compares what it would
+  apply with its watch caches and skips server-side applies, deletes of objects that are already
+  gone, and status patches that would change nothing. RestateDeployment no longer creates the
+  current ReplicaSet only to get a `409 AlreadyExists` back.
+- Each RestateCluster and RestateCloudEnvironment is now reconciled once per event or resync.
+  Previously every namespaced child queued a second reconcile of its cluster-scoped owner, so
+  each RestateCluster was reconciled twice.
+- The steady-state resync is configurable and defaults to **30 minutes** (was 5): CLI
+  `--resync-interval-seconds`, env `RESYNC_INTERVAL_SECONDS`, Helm `resyncIntervalSeconds`.
+  Each resync is jittered by ±10%. The shorter requeues used while a resource is not ready, while
+  an old version drains, or after an error are unchanged.
+- Out-of-band edits to the per-version Services of a RestateDeployment, and to the tunnel
+  Deployment and Service of a RestateCloudEnvironment, are now reverted as soon as the watch
+  reports them rather than at the next resync.
+
+### Why This Matters
+On a local kind cluster with audit logging, an idle RestateCluster with three replicas cost 21
+API requests per reconcile, twice every 5 minutes: about 12,000 requests (about 18 MB of
+audit log) per cluster per day. On EKS each one is an audit record billed by CloudWatch. With
+this change an idle reconcile makes no requests at all.
+
+### Impact on Users
+- Drift is still repaired: every object the operator writes is watched, and a write to it by
+  anyone else is undone when the watch delivers it, as before.
+- After an operator restart, the first reconcile of each resource applies everything once.
+- The resync is now a backstop for state the watches cannot see (for example a deployment
+  removed from Restate by hand), so such changes take up to 30 minutes to be noticed. Lower
+  `resyncIntervalSeconds` if that is too long.

src/controllers/mod.rs

@@ -8,8 +8,10 @@ use kube::Resource;
 use kube::client::Client;
 use kube::runtime::WatchStreamExt;
 use kube::runtime::events::{Event, EventType, Recorder};
+use kube::runtime::reflector::ObjectRef;
 use kube::runtime::{reflector, watcher};
 use serde::Serialize;
+use serde::de::DeserializeOwned;
 use tokio::sync::RwLock;
 use tokio::time::sleep;
 use tracing::{debug, info, warn};
@@ -18,6 +20,7 @@ use url::Url;
 use crate::Metrics;
 use crate::resources::ReconciliationState;
 
+pub mod owned_objects;
 pub mod restatecloudenvironment;
 pub mod restatecluster;
 pub mod restatedeployment;
@@ -251,6 +254,9 @@ pub struct State {
     /// the operator itself rather than about a resource it manages. `None` when the pod name
     /// was not supplied, eg when running outside the cluster.
     operator_object_ref: Option<ObjectReference>,
+
+    /// When a resource that reconciled cleanly is next reconciled unprompted
+    pub resync: ResyncInterval,
 }
 
 /// State wrapper around the controller outputs for the web server
@@ -267,6 +273,7 @@ impl State {
         canary_image: String,
         operator_pod_name: Option<String>,
         operator_pod_uid: Option<String>,
+        resync: ResyncInterval,
     ) -> Self {
         let operator_object_ref = operator_pod_name.map(|name| ObjectReference {
             api_version: Some("v1".into()),
@@ -290,6 +297,7 @@ impl State {
             cluster_dns,
             canary_image,
             operator_object_ref,
+            resync,
         }
     }
 
@@ -304,6 +312,101 @@ impl State {
     }
 }
 
+/// How long a resource that reconciled cleanly waits before it is reconciled again with no
+/// event to prompt it.
+///
+/// The controllers watch every object they write, so drift is repaired as soon as the watch
+/// reports it. This is the backstop for what the watches cannot see, such as a deployment
+/// removed from Restate behind the operator's back, or a missed watch event.
+#[derive(Clone, Copy, Debug)]
+pub struct ResyncInterval(Duration);
+
+impl ResyncInterval {
+    pub fn new(interval: Duration) -> Self {
+        Self(interval)
+    }
+
+    /// The delay until the next resync: the interval, give or take 10%. Without the jitter,
+    /// resources reconciled together (all of them, after an operator restart) would resync
+    /// in lockstep forever.
+    pub fn next(&self) -> Duration {
+        self.0.mul_f64(rand::random_range(0.9..1.1))
+    }
+}
+
+/// A watch of `api` that keeps the returned store current, as a stream of the objects it saw
+/// change, for triggering reconciles. Deletions bump the generation, so a predicate on it
+/// still lets them through.
+pub fn reflected<K>(
+    api: kube::Api<K>,
+    cfg: watcher::Config,
+) -> (
+    reflector::Store<K>,
+    impl futures::Stream<Item = Result<K, watcher::Error>> + Send + 'static,
+)
+where
+    K: Resource<DynamicType = ()>
+        + Clone
+        + std::fmt::Debug
+        + DeserializeOwned
+        + Send
+        + Sync
+        + 'static,
+{
+    let (store, writer) = reflector::store();
+    let stream = reflector(writer, watcher(api, cfg))
+        .map(ensure_deletion_change)
+        .touched_objects()
+        .default_backoff();
+    (store, stream)
+}
+
+// deletion apparently doesn't lead to any change in metadata otherwise, which means the changed_predicate
+// would drop them.
+fn ensure_deletion_change<K: Resource, E>(
+    mut event: Result<kube::runtime::watcher::Event<K>, E>,
+) -> Result<kube::runtime::watcher::Event<K>, E> {
+    if let Ok(kube::runtime::watcher::Event::Delete(ref mut object)) = event {
+        let meta = object.meta_mut();
+        meta.generation = match meta.generation {
+            Some(val) => Some(val + 1),
+            None => Some(0),
+        }
+    }
+    event
+}
+
+/// The owners of `object` of the cluster-scoped kind `Owner`, for `watches_stream`.
+///
+/// Use this instead of `owns_stream` for namespaced children of a cluster-scoped owner:
+/// `owns_stream` keys the owner by the child's namespace, so the same owner would be queued
+/// under a second, namespaced key that reconciles (and requeues) alongside the real one.
+pub fn cluster_scoped_owners<Owner, K>(object: K) -> impl Iterator<Item = ObjectRef<Owner>>
+where
+    Owner: Resource<DynamicType = ()>,
+    K: Resource,
+{
+    object
+        .meta()
+        .owner_references
+        .clone()
+        .into_iter()
+        .flatten()
+        .filter(|owner| {
+            owner.api_version == Owner::api_version(&()) && owner.kind == Owner::kind(&())
+        })
+        .map(|owner| ObjectRef::new(&owner.name))
+}
+
+/// Whether applying `desired` as the status would leave `current` as it is, so the write can
+/// be skipped.
+pub fn status_unchanged<S: Serialize>(current: Option<&S>, desired: &S) -> bool {
+    current.is_some_and(|current| {
+        serde_json::to_value(current).expect("status to serialize")
+            == serde_json::to_value(desired).expect("status to serialize")
+    })
+}
+
 pub fn service_url(
     service_name: &str,
     service_namespace: &str,