From c796597df5417f8c6300b4fd29638c10836b9d08 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 09:52:35 +0200 Subject: [PATCH 001/119] ML-KEM encapsulate/decapsulate module --- client/internal/pqkem/kem.go | 165 ++++++++++++++++++++++++++++++ client/internal/pqkem/kem_test.go | 89 ++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 client/internal/pqkem/kem.go create mode 100644 client/internal/pqkem/kem_test.go diff --git a/client/internal/pqkem/kem.go b/client/internal/pqkem/kem.go new file mode 100644 index 00000000000..c2961ecedac --- /dev/null +++ b/client/internal/pqkem/kem.go @@ -0,0 +1,165 @@ +// Package pqkem is a spike (NET-1406) for a post-quantum pre-shared-key exchange +// that could replace Rosenpass. It performs an X25519MLKEM768 hybrid key +// encapsulation and derives a 32-byte WireGuard PSK. +// +// The exchange is a single round trip designed to ride the (already +// authenticated) Signal offer/answer channel: +// +// initiator --Offer(1216B)--> responder +// initiator <--Answer(1120B)-- responder +// +// Both sides then hold the same PSK, which is bound to the two peers' identities +// (their WireGuard static public keys) so the derived key cannot be transplanted +// to a different peer pair even if the transport authentication were bypassed. +// +// Combiner note: this follows the IETF hybrid layout (X25519 ‖ ML-KEM on the +// wire; ML-KEM_ss ‖ X25519_ss into the KDF) from +// draft-kwiatkowski-tls-ecdhe-mlkem. The spike uses SHA-256 as the KDF; a +// production version should use HKDF with the RFC labels — see TODO below. +package pqkem + +import ( + "crypto/ecdh" + "crypto/mlkem" + "crypto/rand" + "crypto/sha256" + "fmt" +) + +const ( + // OfferSize is the initiator message: X25519 public key ‖ ML-KEM-768 encapsulation key. + OfferSize = 32 + mlkem.EncapsulationKeySize768 // 1216 + // AnswerSize is the responder message: ML-KEM-768 ciphertext ‖ X25519 public key. + AnswerSize = mlkem.CiphertextSize768 + 32 // 1120 + + pskLabel = "netbird-pq-psk-v1" +) + +// PSK is the 32-byte pre-shared key handed to WireGuard. +type PSK [32]byte + +// Binding identifies the peer pair the PSK is derived for. Callers set both +// WireGuard static public keys; the order does not matter (it is canonicalised). +type Binding struct { + LocalWgPub []byte + RemoteWgPub []byte +} + +// Initiator holds the ephemeral secrets between Offer and Finish. +type Initiator struct { + x25519 *ecdh.PrivateKey + mlkemDK *mlkem.DecapsulationKey768 + offer []byte +} + +// NewInitiator generates the ephemeral X25519 + ML-KEM-768 keypairs. +func NewInitiator() (*Initiator, error) { + x, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("x25519 keygen: %w", err) + } + dk, err := mlkem.GenerateKey768() + if err != nil { + return nil, fmt.Errorf("ml-kem keygen: %w", err) + } + + offer := make([]byte, 0, OfferSize) + offer = append(offer, x.PublicKey().Bytes()...) + offer = append(offer, dk.EncapsulationKey().Bytes()...) + + return &Initiator{x25519: x, mlkemDK: dk, offer: offer}, nil +} + +// Offer returns the initiator message to send over Signal. +func (i *Initiator) Offer() []byte { + return i.offer +} + +// Finish consumes the responder's answer and derives the PSK. +func (i *Initiator) Finish(answer []byte, b Binding) (PSK, error) { + if len(answer) != AnswerSize { + return PSK{}, fmt.Errorf("answer: got %d bytes, want %d", len(answer), AnswerSize) + } + ct := answer[:mlkem.CiphertextSize768] + peerX := answer[mlkem.CiphertextSize768:] + + ssMLKEM, err := i.mlkemDK.Decapsulate(ct) + if err != nil { + return PSK{}, fmt.Errorf("ml-kem decapsulate: %w", err) + } + pub, err := ecdh.X25519().NewPublicKey(peerX) + if err != nil { + return PSK{}, fmt.Errorf("parse peer x25519: %w", err) + } + ssX, err := i.x25519.ECDH(pub) + if err != nil { + return PSK{}, fmt.Errorf("x25519 ecdh: %w", err) + } + + return derivePSK(ssMLKEM, ssX, i.offer, answer, b), nil +} + +// Respond consumes an initiator offer, produces the answer, and derives the PSK. +func Respond(offer []byte, b Binding) (answer []byte, psk PSK, err error) { + if len(offer) != OfferSize { + return nil, PSK{}, fmt.Errorf("offer: got %d bytes, want %d", len(offer), OfferSize) + } + peerX := offer[:32] + peerEK := offer[32:] + + ek, err := mlkem.NewEncapsulationKey768(peerEK) + if err != nil { + return nil, PSK{}, fmt.Errorf("parse peer ml-kem key: %w", err) + } + ssMLKEM, ct := ek.Encapsulate() + + x, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return nil, PSK{}, fmt.Errorf("x25519 keygen: %w", err) + } + pub, err := ecdh.X25519().NewPublicKey(peerX) + if err != nil { + return nil, PSK{}, fmt.Errorf("parse peer x25519: %w", err) + } + ssX, err := x.ECDH(pub) + if err != nil { + return nil, PSK{}, fmt.Errorf("x25519 ecdh: %w", err) + } + + answer = make([]byte, 0, AnswerSize) + answer = append(answer, ct...) + answer = append(answer, x.PublicKey().Bytes()...) + + // derivePSK uses the same argument order on both sides; the responder's local + // binding is the mirror of the initiator's, canonicalised inside derivePSK. + return answer, derivePSK(ssMLKEM, ssX, offer, answer, b), nil +} + +// derivePSK combines the two shared secrets and binds the result to the full +// transcript (offer ‖ answer) and the canonicalised peer identities. +// +// TODO(NET-1406): replace the SHA-256 concat with the RFC HKDF combiner +// (crypto/hkdf, Go 1.24+) and proper labels before this leaves spike status. +func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) PSK { + lo, hi := canonicalPair(b.LocalWgPub, b.RemoteWgPub) + + h := sha256.New() + h.Write([]byte(pskLabel)) + h.Write(ssMLKEM) + h.Write(ssX) + h.Write(offer) + h.Write(answer) + h.Write(lo) + h.Write(hi) + + var psk PSK + copy(psk[:], h.Sum(nil)) + return psk +} + +func canonicalPair(a, b []byte) (lo, hi []byte) { + if string(a) <= string(b) { + return a, b + } + return b, a +} diff --git a/client/internal/pqkem/kem_test.go b/client/internal/pqkem/kem_test.go new file mode 100644 index 00000000000..3bdc996357d --- /dev/null +++ b/client/internal/pqkem/kem_test.go @@ -0,0 +1,89 @@ +package pqkem + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + wgA = []byte("peer-A-wireguard-pubkey-32bytes!") + wgB = []byte("peer-B-wireguard-pubkey-32bytes!") +) + +func TestExchange_DerivesMatchingPSK(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + require.Len(t, init.Offer(), OfferSize) + + answer, pskB, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + require.NoError(t, err) + require.Len(t, answer, AnswerSize) + + pskA, err := init.Finish(answer, Binding{LocalWgPub: wgA, RemoteWgPub: wgB}) + require.NoError(t, err) + + require.Equal(t, pskB, pskA, "both sides must derive the same PSK") + require.NotEqual(t, PSK{}, pskA, "PSK must not be zero") +} + +func TestExchange_PSKBoundToPeerIdentities(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + // responder computes with the honest pair... + _, pskHonest, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + require.NoError(t, err) + + // ...a second responder run with a different peer identity yields a different PSK, + // even though the KEM material would otherwise combine identically. + wgC := []byte("peer-C-wireguard-pubkey-32bytes!") + _, pskWrong, err := Respond(init.Offer(), Binding{LocalWgPub: wgC, RemoteWgPub: wgA}) + require.NoError(t, err) + + require.NotEqual(t, pskHonest, pskWrong, "PSK must be bound to the peer pair") +} + +func TestExchange_RejectsMalformedMessages(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + _, _, err = Respond(init.Offer()[:10], Binding{}) + require.Error(t, err) + + _, err = init.Finish([]byte("too short"), Binding{}) + require.Error(t, err) +} + +// TestExchange_ReportSizesAndTiming is a spike measurement, not a pass/fail gate. +// Run with: go test -run TestExchange_ReportSizesAndTiming -v ./client/internal/pqkem/ +func TestExchange_ReportSizesAndTiming(t *testing.T) { + const iters = 200 + + var tInit, tResp, tFinish time.Duration + for i := 0; i < iters; i++ { + s0 := time.Now() + init, err := NewInitiator() + require.NoError(t, err) + tInit += time.Since(s0) + + s1 := time.Now() + answer, _, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + require.NoError(t, err) + tResp += time.Since(s1) + + s2 := time.Now() + _, err = init.Finish(answer, Binding{LocalWgPub: wgA, RemoteWgPub: wgB}) + require.NoError(t, err) + tFinish += time.Since(s2) + } + + t.Logf("wire sizes: offer=%d B answer=%d B (Rosenpass static pubkey ~524160 B)", OfferSize, AnswerSize) + t.Logf("total on-wire per handshake: %d B (~%.0fx smaller than RP static key)", OfferSize+AnswerSize, 524160.0/float64(OfferSize+AnswerSize)) + t.Logf("avg NewInitiator (keygen): %s", tInit/iters) + t.Logf("avg Respond (encaps+dh): %s", tResp/iters) + t.Logf("avg Finish (decaps+dh): %s", tFinish/iters) + t.Logf("avg full handshake CPU: %s", (tInit+tResp+tFinish)/iters) +} From a48bb9ba18b9e48a7de01dbaab1941d915c0f88b Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 09:52:46 +0200 Subject: [PATCH 002/119] Messages definition --- client/internal/pqkem/message.go | 135 ++++++++++++++++++++++++++ client/internal/pqkem/message_test.go | 61 ++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 client/internal/pqkem/message.go create mode 100644 client/internal/pqkem/message_test.go diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go new file mode 100644 index 00000000000..341f6b5f8e5 --- /dev/null +++ b/client/internal/pqkem/message.go @@ -0,0 +1,135 @@ +package pqkem + +import ( + "crypto/mlkem" + "fmt" +) + +// Wire framing for the PQ-KEM exchange. Messages are self-contained, versioned, +// transport-agnostic byte blobs: the same bytes ride the Signal offer/answer +// (initial, pre-tunnel) or a WireGuard-tunnel packet (rekey). They are NOT a gRPC +// service — the network layer only sees opaque []byte. +// +// Layout (all messages): [type:1][version:1][exchangeID:16][payload...] + +const ( + // ProtocolVersion is bumped on any wire-incompatible change; a peer rejects + // messages it does not understand rather than misparsing them. + ProtocolVersion uint8 = 1 + + // ExchangeIDSize identifies one offer/answer/confirm round so stale answers + // (e.g. an answer to a pre-restart offer) are dropped instead of applied. + ExchangeIDSize = 16 + + headerSize = 1 + 1 + ExchangeIDSize +) + +// MsgType tags the three message kinds of the exchange. +type MsgType uint8 + +const ( + MsgOffer MsgType = iota + 1 + MsgAnswer + MsgConfirm +) + +// ExchangeID is the per-round correlator echoed by the answer and the confirm. +type ExchangeID [ExchangeIDSize]byte + +// OfferMsg carries the initiator's public material (X25519 pub ‖ ML-KEM encap key). +type OfferMsg struct { + ExchangeID ExchangeID + // KEMOffer is the raw Initiator.Offer() blob (OfferSize bytes). + KEMOffer []byte +} + +// AnswerMsg carries the responder's reply (ML-KEM ciphertext ‖ X25519 pub) for the +// round identified by ExchangeID. +type AnswerMsg struct { + ExchangeID ExchangeID + // KEMAnswer is the raw Respond() answer blob (AnswerSize bytes). + KEMAnswer []byte +} + +// ConfirmMsg is sent ONE WAY, initiator -> responder, over the tunnel under the +// NEW PSK, for the round identified by ExchangeID. It carries no key material. +// +// It is one-way because of the knowledge asymmetry of a KEM: the initiator learns +// the responder holds the key simply by receiving the answer (the responder had to +// derive it to encapsulate), so the initiator needs no confirmation. Only the +// responder is left unsure whether the initiator received the answer and committed +// the key — this message resolves that. As a bonus, being sent under the new PSK it +// triggers the WireGuard handshake with the new key: the responder converges on +// receiving it, and the initiator converges by observing that handshake succeed +// (which fails on a PSK mismatch, so success proves the responder also committed). +type ConfirmMsg struct { + ExchangeID ExchangeID +} + +// Encode serialises the offer with its framed header. +func (m *OfferMsg) Encode() ([]byte, error) { + if len(m.KEMOffer) != OfferSize { + return nil, fmt.Errorf("offer payload: got %d, want %d", len(m.KEMOffer), OfferSize) + } + return frame(MsgOffer, m.ExchangeID, m.KEMOffer), nil +} + +// Encode serialises the answer with its framed header. +func (m *AnswerMsg) Encode() ([]byte, error) { + if len(m.KEMAnswer) != AnswerSize { + return nil, fmt.Errorf("answer payload: got %d, want %d", len(m.KEMAnswer), AnswerSize) + } + return frame(MsgAnswer, m.ExchangeID, m.KEMAnswer), nil +} + +// Encode serialises the confirm with its framed header. +func (m *ConfirmMsg) Encode() []byte { + return frame(MsgConfirm, m.ExchangeID, nil) +} + +// Decode parses a framed message into one of *OfferMsg / *AnswerMsg / *ConfirmMsg. +func Decode(buf []byte) (MsgType, any, error) { + if len(buf) < headerSize { + return 0, nil, fmt.Errorf("message too short: %d bytes", len(buf)) + } + typ := MsgType(buf[0]) + if ver := buf[1]; ver != ProtocolVersion { + return typ, nil, fmt.Errorf("unsupported protocol version %d (want %d)", ver, ProtocolVersion) + } + + var id ExchangeID + copy(id[:], buf[2:headerSize]) + payload := buf[headerSize:] + + switch typ { + case MsgOffer: + if len(payload) != OfferSize { + return typ, nil, fmt.Errorf("offer payload: got %d, want %d", len(payload), OfferSize) + } + return typ, &OfferMsg{ExchangeID: id, KEMOffer: payload}, nil + case MsgAnswer: + if len(payload) != AnswerSize { + return typ, nil, fmt.Errorf("answer payload: got %d, want %d", len(payload), AnswerSize) + } + return typ, &AnswerMsg{ExchangeID: id, KEMAnswer: payload}, nil + case MsgConfirm: + if len(payload) != 0 { + return typ, nil, fmt.Errorf("confirm payload must be empty, got %d bytes", len(payload)) + } + return typ, &ConfirmMsg{ExchangeID: id}, nil + default: + return typ, nil, fmt.Errorf("unknown message type %d", typ) + } +} + +func frame(typ MsgType, id ExchangeID, payload []byte) []byte { + buf := make([]byte, headerSize+len(payload)) + buf[0] = byte(typ) + buf[1] = ProtocolVersion + copy(buf[2:], id[:]) + copy(buf[headerSize:], payload) + return buf +} + +// compile-time assurance the KEM blob sizes referenced here stay in sync with kem.go. +var _ = [1]struct{}{}[OfferSize-(32+mlkem.EncapsulationKeySize768)] diff --git a/client/internal/pqkem/message_test.go b/client/internal/pqkem/message_test.go new file mode 100644 index 00000000000..a51135ea0d3 --- /dev/null +++ b/client/internal/pqkem/message_test.go @@ -0,0 +1,61 @@ +package pqkem + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMessageRoundTrip(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + answer, _, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + require.NoError(t, err) + + id := ExchangeID{1, 2, 3, 4} + + offBytes, err := (&OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}).Encode() + require.NoError(t, err) + typ, decoded, err := Decode(offBytes) + require.NoError(t, err) + require.Equal(t, MsgOffer, typ) + require.Equal(t, id, decoded.(*OfferMsg).ExchangeID) + require.Equal(t, init.Offer(), decoded.(*OfferMsg).KEMOffer) + + ansBytes, err := (&AnswerMsg{ExchangeID: id, KEMAnswer: answer}).Encode() + require.NoError(t, err) + typ, decoded, err = Decode(ansBytes) + require.NoError(t, err) + require.Equal(t, MsgAnswer, typ) + require.Equal(t, answer, decoded.(*AnswerMsg).KEMAnswer) + + confBytes := (&ConfirmMsg{ExchangeID: id}).Encode() + typ, decoded, err = Decode(confBytes) + require.NoError(t, err) + require.Equal(t, MsgConfirm, typ) + require.Equal(t, id, decoded.(*ConfirmMsg).ExchangeID) +} + +func TestDecodeRejects(t *testing.T) { + // too short + _, _, err := Decode([]byte{1, 1}) + require.Error(t, err) + + // wrong version + bad := make([]byte, headerSize+OfferSize) + bad[0] = byte(MsgOffer) + bad[1] = ProtocolVersion + 1 + _, _, err = Decode(bad) + require.Error(t, err) + + // unknown type + bad2 := make([]byte, headerSize) + bad2[0] = 99 + bad2[1] = ProtocolVersion + _, _, err = Decode(bad2) + require.Error(t, err) + + // offer with wrong payload size + _, err = (&OfferMsg{KEMOffer: []byte{1, 2, 3}}).Encode() + require.Error(t, err) +} From 87afc3e967f640df84a4229dc6532dba60eb339b Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 12:18:01 +0200 Subject: [PATCH 003/119] Pure mechanics of manager --- client/internal/pqkem/manager.go | 163 ++++++++++++++++++++++++++ client/internal/pqkem/manager_test.go | 75 ++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 client/internal/pqkem/manager.go create mode 100644 client/internal/pqkem/manager_test.go diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go new file mode 100644 index 00000000000..beefe9beebd --- /dev/null +++ b/client/internal/pqkem/manager.go @@ -0,0 +1,163 @@ +package pqkem + +import ( + "crypto/rand" + "fmt" + "sync" +) + +// Manager drives the PQ-KEM exchange per remote peer. It is a pure state machine: +// it consumes and produces messages and returns the derived PSK at the point each +// side must commit it (initiator on the answer, responder on the confirm). It does +// no I/O — transport (Signal / tunnel) and the WireGuard SetPSK live in the caller +// (network module + wiring), so this stays extraction-ready as a standalone library. +// +// Commit ordering (see ConfirmMsg): the responder derives the PSK on the offer but +// must NOT commit it until the confirm arrives, otherwise a lost answer would leave +// it on a new PSK the initiator never got, breaking the tunnel at the next rekey. + +type sessionState uint8 + +const ( + stateIdle sessionState = iota + stateAwaitingAnswer // initiator: sent offer, waiting for answer + stateAwaitingConfirm // responder: sent answer, holding pending PSK until confirm + stateEstablished +) + +type peerSession struct { + state sessionState + exchangeID ExchangeID + + // initiator side: in-flight ephemeral keypair between offer and answer. + initiator *Initiator + // responder side: PSK derived on the offer, committed only on the confirm. + pendingPSK PSK +} + +type Manager struct { + localWgKey string + + mu sync.Mutex + sessions map[string]*peerSession +} + +// NewManager creates a KEM manager for the local peer identified by its WireGuard +// public key (base64), used both for the deterministic initiator role and the +// identity binding of the derived PSK. +func NewManager(localWgKey string) *Manager { + return &Manager{ + localWgKey: localWgKey, + sessions: make(map[string]*peerSession), + } +} + +// IsInitiator reports whether the local peer drives the exchange for this remote +// peer. Roles are deterministic (lexicographic WG-key compare) so exactly one side +// initiates, mirroring how Rosenpass picks its handshake initiator. +func (m *Manager) IsInitiator(remoteWgKey string) bool { + return m.localWgKey > remoteWgKey +} + +// StartExchange begins (or restarts, on rotation) an exchange as the initiator and +// returns the offer to send. Any previous in-flight exchange for the peer is dropped. +func (m *Manager) StartExchange(remoteWgKey string) (*OfferMsg, error) { + init, err := NewInitiator() + if err != nil { + return nil, err + } + id, err := newExchangeID() + if err != nil { + return nil, err + } + + m.mu.Lock() + m.sessions[remoteWgKey] = &peerSession{ + state: stateAwaitingAnswer, + exchangeID: id, + initiator: init, + } + m.mu.Unlock() + + return &OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}, nil +} + +// HandleOffer processes a received offer as the responder, derives the PSK, and +// returns the answer to send. The PSK is held pending and NOT returned: the caller +// must not program it until HandleConfirm. +func (m *Manager) HandleOffer(remoteWgKey string, o *OfferMsg) (*AnswerMsg, error) { + answer, psk, err := Respond(o.KEMOffer, m.binding(remoteWgKey)) + if err != nil { + return nil, err + } + + m.mu.Lock() + m.sessions[remoteWgKey] = &peerSession{ + state: stateAwaitingConfirm, + exchangeID: o.ExchangeID, + pendingPSK: psk, + } + m.mu.Unlock() + + return &AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answer}, nil +} + +// HandleAnswer processes a received answer as the initiator. On success it returns +// the derived PSK (which the caller commits now) and the confirm to send. +func (m *Manager) HandleAnswer(remoteWgKey string, a *AnswerMsg) (PSK, *ConfirmMsg, error) { + m.mu.Lock() + s, ok := m.sessions[remoteWgKey] + m.mu.Unlock() + + if !ok || s.state != stateAwaitingAnswer { + return PSK{}, nil, fmt.Errorf("no pending offer for peer %s", remoteWgKey) + } + if s.exchangeID != a.ExchangeID { + // stale answer (e.g. to a pre-restart offer) — drop, keep waiting. + return PSK{}, nil, fmt.Errorf("answer exchangeID mismatch for peer %s", remoteWgKey) + } + + psk, err := s.initiator.Finish(a.KEMAnswer, m.binding(remoteWgKey)) + if err != nil { + return PSK{}, nil, err + } + + m.mu.Lock() + s.state = stateEstablished + s.initiator = nil + m.mu.Unlock() + + return psk, &ConfirmMsg{ExchangeID: a.ExchangeID}, nil +} + +// HandleConfirm processes a received confirm as the responder and returns the PSK +// to commit now. It errors on a stale/unknown confirm so the caller ignores it. +func (m *Manager) HandleConfirm(remoteWgKey string, c *ConfirmMsg) (PSK, error) { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[remoteWgKey] + if !ok || s.state != stateAwaitingConfirm { + return PSK{}, fmt.Errorf("no pending answer for peer %s", remoteWgKey) + } + if s.exchangeID != c.ExchangeID { + return PSK{}, fmt.Errorf("confirm exchangeID mismatch for peer %s", remoteWgKey) + } + + s.state = stateEstablished + psk := s.pendingPSK + s.pendingPSK = PSK{} + return psk, nil +} + +func (m *Manager) binding(remoteWgKey string) Binding { + return Binding{LocalWgPub: []byte(m.localWgKey), RemoteWgPub: []byte(remoteWgKey)} +} + +func newExchangeID() (ExchangeID, error) { + var id ExchangeID + if _, err := rand.Read(id[:]); err != nil { + return ExchangeID{}, fmt.Errorf("generate exchange id: %w", err) + } + return id, nil +} diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go new file mode 100644 index 00000000000..3bdb0f413cc --- /dev/null +++ b/client/internal/pqkem/manager_test.go @@ -0,0 +1,75 @@ +package pqkem + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestManager_FullExchange(t *testing.T) { + // deterministic roles: "bbbb" > "aaaa" -> B is initiator + a := NewManager("aaaa") + b := NewManager("bbbb") + require.True(t, b.IsInitiator("aaaa")) + require.False(t, a.IsInitiator("bbbb")) + + // B (initiator) -> A + offer, err := b.StartExchange("aaaa") + require.NoError(t, err) + + // A (responder) derives PSK but holds it pending + answer, err := a.HandleOffer("bbbb", offer) + require.NoError(t, err) + + // B derives PSK on the answer, commits now, produces confirm + pskB, confirm, err := b.HandleAnswer("aaaa", answer) + require.NoError(t, err) + + // A commits only on the confirm + pskA, err := a.HandleConfirm("bbbb", confirm) + require.NoError(t, err) + + require.Equal(t, pskB, pskA, "both sides converge on the same PSK") + require.NotEqual(t, PSK{}, pskA) +} + +func TestManager_StaleAnswerDropped(t *testing.T) { + b := NewManager("bbbb") + + _, err := b.StartExchange("aaaa") + require.NoError(t, err) + + // craft an answer with a wrong exchangeID + stale := &AnswerMsg{ExchangeID: ExchangeID{0xFF}, KEMAnswer: make([]byte, AnswerSize)} + _, _, err = b.HandleAnswer("aaaa", stale) + require.Error(t, err) +} + +func TestManager_RestartResyncsViaNewExchange(t *testing.T) { + // A restarts (fresh manager) mid-flight; a new exchange from scratch converges. + a := NewManager("aaaa") + b := NewManager("bbbb") + + off1, err := b.StartExchange("aaaa") + require.NoError(t, err) + + // B "restarts": new manager, old in-flight state gone + b = NewManager("bbbb") + off2, err := b.StartExchange("aaaa") + require.NoError(t, err) + require.NotEqual(t, off1.ExchangeID, off2.ExchangeID) + + ans, err := a.HandleOffer("bbbb", off2) + require.NoError(t, err) + pskB, conf, err := b.HandleAnswer("aaaa", ans) + require.NoError(t, err) + pskA, err := a.HandleConfirm("bbbb", conf) + require.NoError(t, err) + require.Equal(t, pskB, pskA) +} + +func TestManager_UnknownPeerErrors(t *testing.T) { + a := NewManager("aaaa") + _, err := a.HandleConfirm("zzzz", &ConfirmMsg{}) + require.Error(t, err) +} From 92cf08b2b0eb1f71ebed1d38226b7f66aa1dfae3 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 12:19:05 +0200 Subject: [PATCH 004/119] Bench key material boilerplate time/allocs CGO_ENABLED=1 go test ./client/internal/pqkem/ -run '^$' -bench . -benchmem 2>&1 | grep -E "Benchmark|ns/op|PASS|ok" | head -20 BenchmarkX25519Keygen-14 33795 34966 ns/op 224 B/op 5 allocs/op BenchmarkX25519ECDH-14 33855 33973 ns/op 32 B/op 1 allocs/op BenchmarkMLKEMKeygen-14 21817 67778 ns/op 8200 B/op 2 allocs/op BenchmarkMLKEMEncaps-14 29918 43235 ns/op 1216 B/op 2 allocs/op BenchmarkMLKEMDecaps-14 26048 56291 ns/op 64 B/op 2 allocs/op PASS ok github.com/netbirdio/netbird/client/internal/pqkem 9.751s Shell cwd was reset to /home/riccardo/Desktop/Personal/netbirdio/netbird --- client/internal/pqkem/bench_test.go | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 client/internal/pqkem/bench_test.go diff --git a/client/internal/pqkem/bench_test.go b/client/internal/pqkem/bench_test.go new file mode 100644 index 00000000000..eabcc3a0add --- /dev/null +++ b/client/internal/pqkem/bench_test.go @@ -0,0 +1,59 @@ +package pqkem + +import ( + "crypto/ecdh" + "crypto/mlkem" + "crypto/rand" + "testing" +) + +func BenchmarkX25519Keygen(b *testing.B) { + c := ecdh.X25519() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := c.GenerateKey(rand.Reader); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkX25519ECDH(b *testing.B) { + c := ecdh.X25519() + a, _ := c.GenerateKey(rand.Reader) + p, _ := c.GenerateKey(rand.Reader) + pub := p.PublicKey() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := a.ECDH(pub); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMLKEMKeygen(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := mlkem.GenerateKey768(); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMLKEMEncaps(b *testing.B) { + dk, _ := mlkem.GenerateKey768() + ek := dk.EncapsulationKey() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = ek.Encapsulate() + } +} + +func BenchmarkMLKEMDecaps(b *testing.B) { + dk, _ := mlkem.GenerateKey768() + _, ct := dk.EncapsulationKey().Encapsulate() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := dk.Decapsulate(ct); err != nil { + b.Fatal(err) + } + } +} From 19739b2b4fa36c223e368642a72785381073ec8f Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 13:01:45 +0200 Subject: [PATCH 005/119] Admits possible errors on Encode --- client/internal/pqkem/message.go | 7 ++++--- client/internal/pqkem/message_test.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go index 341f6b5f8e5..fe438163642 100644 --- a/client/internal/pqkem/message.go +++ b/client/internal/pqkem/message.go @@ -82,9 +82,10 @@ func (m *AnswerMsg) Encode() ([]byte, error) { return frame(MsgAnswer, m.ExchangeID, m.KEMAnswer), nil } -// Encode serialises the confirm with its framed header. -func (m *ConfirmMsg) Encode() []byte { - return frame(MsgConfirm, m.ExchangeID, nil) +// Encode serialises the confirm with its framed header. It returns an error only +// for signature uniformity with the other messages; it never actually fails. +func (m *ConfirmMsg) Encode() ([]byte, error) { + return frame(MsgConfirm, m.ExchangeID, nil), nil } // Decode parses a framed message into one of *OfferMsg / *AnswerMsg / *ConfirmMsg. diff --git a/client/internal/pqkem/message_test.go b/client/internal/pqkem/message_test.go index a51135ea0d3..1845bc4d9b3 100644 --- a/client/internal/pqkem/message_test.go +++ b/client/internal/pqkem/message_test.go @@ -29,7 +29,8 @@ func TestMessageRoundTrip(t *testing.T) { require.Equal(t, MsgAnswer, typ) require.Equal(t, answer, decoded.(*AnswerMsg).KEMAnswer) - confBytes := (&ConfirmMsg{ExchangeID: id}).Encode() + confBytes, err := (&ConfirmMsg{ExchangeID: id}).Encode() + require.NoError(t, err) typ, decoded, err = Decode(confBytes) require.NoError(t, err) require.Equal(t, MsgConfirm, typ) From 311c5f8a8c235929baaa06f57ee2730cf225939a Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 13:09:15 +0200 Subject: [PATCH 006/119] Defines event callbacks --- client/internal/pqkem/callbacks.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 client/internal/pqkem/callbacks.go diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go new file mode 100644 index 00000000000..c1578f64a5e --- /dev/null +++ b/client/internal/pqkem/callbacks.go @@ -0,0 +1,19 @@ +package pqkem + +// WGCallbackHandler is implemented by the host (the NetBird wiring layer) and +// invoked by the library. The library only reports events; the host owns the +// WireGuard reaction. Keeping this an interface — rather than calling wgctrl +// directly — is what lets the KEM code be extracted as a standalone library. +type WGCallbackHandler interface { + // OnNewPSKReady fires when a fresh post-quantum PSK has been derived for a + // peer and must be programmed on the WireGuard interface. It is invoked at the + // commit point of each side: the initiator on receiving the answer, the + // responder on receiving the confirm. + OnNewPSKReady(remoteWgKey string, psk PSK) error + + // OnRekeyFailed fires when an exchange fails to converge within the allotted + // time. The host should tear the peer connection down so ICE re-establishes, + // and log a WARN. The library reports the event; it does not dictate the + // reaction. + OnRekeyFailed(remoteWgKey string) error +} From e4bed8de6559a6e66b50565fb40039bf173f5ccc Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 13:09:21 +0200 Subject: [PATCH 007/119] Adds driver to glue together manager and outside world --- client/internal/pqkem/driver.go | 172 +++++++++++++++++++++++++++ client/internal/pqkem/driver_test.go | 96 +++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 client/internal/pqkem/driver.go create mode 100644 client/internal/pqkem/driver_test.go diff --git a/client/internal/pqkem/driver.go b/client/internal/pqkem/driver.go new file mode 100644 index 00000000000..10e5e2a104f --- /dev/null +++ b/client/internal/pqkem/driver.go @@ -0,0 +1,172 @@ +package pqkem + +import ( + "context" + "fmt" + "log/slog" + "sync" + "time" +) + +// DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly +// rotated PSK is naturally adopted by WG's next handshake without forcing one. +const DefaultRekeyInterval = 2 * time.Minute + +// Transport hands an already-encoded exchange message to the peer. The host routes +// it over the appropriate channel — Signal before the tunnel is up, the WireGuard +// tunnel for rekeys — so the driver never needs to know which is in use. +type Transport interface { + Send(remoteWgKey string, msg []byte) error +} + +// encodableMsg is the common shape of the three wire messages, letting the driver +// send any of them uniformly. +type encodableMsg interface { + Encode() ([]byte, error) +} + +// Driver ties the pure Manager to the outside world: it runs the per-peer rekey +// timer, dispatches inbound messages, and surfaces the derived PSK to the host via +// WGCallbackHandler. Convergence/retry/OnRekeyFailed are layered on in a later step. +type Driver struct { + mgr *Manager + transport Transport + wg WGCallbackHandler + interval time.Duration + logger *slog.Logger + + mu sync.Mutex + peers map[string]context.CancelFunc + wait sync.WaitGroup +} + +// NewDriver builds a driver for the local peer. A zero interval falls back to +// DefaultRekeyInterval; a nil logger falls back to slog.Default(). +func NewDriver(localWgKey string, t Transport, h WGCallbackHandler, interval time.Duration, logger *slog.Logger) *Driver { + if interval <= 0 { + interval = DefaultRekeyInterval + } + if logger == nil { + logger = slog.Default() + } + return &Driver{ + mgr: NewManager(localWgKey), + transport: t, + wg: h, + interval: interval, + logger: logger, + peers: make(map[string]context.CancelFunc), + } +} + +// AddPeer registers a remote peer and starts its rekey timer. Re-adding an existing +// peer is a no-op. +func (d *Driver) AddPeer(remoteWgKey string) { + d.mu.Lock() + defer d.mu.Unlock() + if _, ok := d.peers[remoteWgKey]; ok { + return + } + ctx, cancel := context.WithCancel(context.Background()) + d.peers[remoteWgKey] = cancel + d.wait.Add(1) + go d.rekeyLoop(ctx, remoteWgKey) +} + +// RemovePeer stops a peer's rekey timer and drops its state. +func (d *Driver) RemovePeer(remoteWgKey string) { + d.mu.Lock() + cancel, ok := d.peers[remoteWgKey] + delete(d.peers, remoteWgKey) + d.mu.Unlock() + if ok { + cancel() + } +} + +// Stop cancels all peer timers and waits for their goroutines to exit. +func (d *Driver) Stop() { + d.mu.Lock() + for _, cancel := range d.peers { + cancel() + } + d.peers = make(map[string]context.CancelFunc) + d.mu.Unlock() + d.wait.Wait() +} + +// HandleInbound decodes an incoming message and drives the exchange, sending any +// response via the transport and surfacing a derived PSK through the callback. +func (d *Driver) HandleInbound(remoteWgKey string, raw []byte) error { + typ, msg, err := Decode(raw) + if err != nil { + return fmt.Errorf("decode from %s: %w", remoteWgKey, err) + } + + switch typ { + case MsgOffer: + answer, err := d.mgr.HandleOffer(remoteWgKey, msg.(*OfferMsg)) + if err != nil { + return err + } + return d.send(remoteWgKey, answer) + + case MsgAnswer: + psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, msg.(*AnswerMsg)) + if err != nil { + return err + } + if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { + return err + } + return d.send(remoteWgKey, confirm) + + case MsgConfirm: + psk, err := d.mgr.HandleConfirm(remoteWgKey, msg.(*ConfirmMsg)) + if err != nil { + return err + } + return d.wg.OnNewPSKReady(remoteWgKey, psk) + + default: + return fmt.Errorf("unhandled message type %d from %s", typ, remoteWgKey) + } +} + +// initiateRekey starts a fresh exchange when the local peer is the initiator for +// this remote peer; the responder waits for the offer instead. Exposed (unexported +// but directly callable) so tests can drive a rekey without waiting on the ticker. +func (d *Driver) initiateRekey(remoteWgKey string) error { + if !d.mgr.IsInitiator(remoteWgKey) { + return nil + } + offer, err := d.mgr.StartExchange(remoteWgKey) + if err != nil { + return err + } + return d.send(remoteWgKey, offer) +} + +func (d *Driver) rekeyLoop(ctx context.Context, remoteWgKey string) { + defer d.wait.Done() + t := time.NewTicker(d.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := d.initiateRekey(remoteWgKey); err != nil { + d.logger.Error("pqkem rekey failed to start", "peer", remoteWgKey, "err", err) + } + } + } +} + +func (d *Driver) send(remoteWgKey string, m encodableMsg) error { + raw, err := m.Encode() + if err != nil { + return err + } + return d.transport.Send(remoteWgKey, raw) +} diff --git a/client/internal/pqkem/driver_test.go b/client/internal/pqkem/driver_test.go new file mode 100644 index 00000000000..59fb381da42 --- /dev/null +++ b/client/internal/pqkem/driver_test.go @@ -0,0 +1,96 @@ +package pqkem + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// loopback delivers a sent message synchronously to the peer driver's HandleInbound, +// attributing it to localKey (the sender). +type loopback struct { + localKey string + peer *Driver +} + +func (l *loopback) Send(remoteWgKey string, msg []byte) error { + cp := append([]byte(nil), msg...) + return l.peer.HandleInbound(l.localKey, cp) +} + +type fakeWG struct { + mu sync.Mutex + psks map[string]PSK + failed []string +} + +func newFakeWG() *fakeWG { return &fakeWG{psks: map[string]PSK{}} } + +func (f *fakeWG) OnNewPSKReady(remoteWgKey string, psk PSK) error { + f.mu.Lock() + defer f.mu.Unlock() + f.psks[remoteWgKey] = psk + return nil +} + +func (f *fakeWG) OnRekeyFailed(remoteWgKey string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.failed = append(f.failed, remoteWgKey) + return nil +} + +func (f *fakeWG) psk(peer string) PSK { + f.mu.Lock() + defer f.mu.Unlock() + return f.psks[peer] +} + +func TestDriver_ExchangeConverges(t *testing.T) { + lbA := &loopback{localKey: "aaaa"} + lbB := &loopback{localKey: "bbbb"} + wgA := newFakeWG() + wgB := newFakeWG() + + // long interval so the ticker never fires during the test; we drive manually. + dA := NewDriver("aaaa", lbA, wgA, time.Hour, nil) + dB := NewDriver("bbbb", lbB, wgB, time.Hour, nil) + lbA.peer = dB // A sends -> B receives + lbB.peer = dA // B sends -> A receives + + dA.AddPeer("bbbb") + dB.AddPeer("aaaa") + defer dA.Stop() + defer dB.Stop() + + // B is the initiator ("bbbb" > "aaaa"). + require.NoError(t, dB.initiateRekey("aaaa")) + + pskB := wgB.psk("aaaa") // B committed on the answer + pskA := wgA.psk("bbbb") // A committed on the confirm + require.NotEqual(t, PSK{}, pskA, "responder A must have a PSK") + require.NotEqual(t, PSK{}, pskB, "initiator B must have a PSK") + require.Equal(t, pskB, pskA, "both sides converge on the same PSK") +} + +func TestDriver_NonInitiatorDoesNothing(t *testing.T) { + lbA := &loopback{localKey: "aaaa"} + wgA := newFakeWG() + dA := NewDriver("aaaa", lbA, wgA, time.Hour, nil) + // no peer driver wired; if A wrongly initiated, Send would nil-panic. + dA.AddPeer("bbbb") + defer dA.Stop() + + // A is NOT the initiator vs "bbbb" -> initiateRekey is a no-op, no Send. + require.NoError(t, dA.initiateRekey("bbbb")) + require.Equal(t, PSK{}, wgA.psk("bbbb")) +} + +func TestDriver_StopIsIdempotent(t *testing.T) { + dA := NewDriver("aaaa", &loopback{localKey: "aaaa"}, newFakeWG(), time.Hour, nil) + dA.AddPeer("bbbb") + dA.Stop() + dA.Stop() // must not panic or hang +} From d8fa34e60b7b53c7d72981e10148ac9233112f03 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 17:25:12 +0200 Subject: [PATCH 008/119] Reuse answer, don't calculate again --- client/internal/pqkem/manager.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index beefe9beebd..bfdd1a1e2f0 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -31,8 +31,11 @@ type peerSession struct { // initiator side: in-flight ephemeral keypair between offer and answer. initiator *Initiator - // responder side: PSK derived on the offer, committed only on the confirm. + // responder side: the derived PSK and the answer for this exchange. Both are + // retained after establishment so retransmitted offers/confirms (same exchangeID) + // are handled idempotently — never re-derive a different key. pendingPSK PSK + answer *AnswerMsg } type Manager struct { @@ -84,22 +87,33 @@ func (m *Manager) StartExchange(remoteWgKey string) (*OfferMsg, error) { // HandleOffer processes a received offer as the responder, derives the PSK, and // returns the answer to send. The PSK is held pending and NOT returned: the caller -// must not program it until HandleConfirm. +// must not program it until HandleConfirm. A retransmitted offer (same exchangeID) +// returns the cached answer without re-deriving, so both sides keep the same key. func (m *Manager) HandleOffer(remoteWgKey string, o *OfferMsg) (*AnswerMsg, error) { + m.mu.Lock() + if s, ok := m.sessions[remoteWgKey]; ok && s.exchangeID == o.ExchangeID && s.answer != nil { + ans := s.answer + m.mu.Unlock() + return ans, nil + } + m.mu.Unlock() + answer, psk, err := Respond(o.KEMOffer, m.binding(remoteWgKey)) if err != nil { return nil, err } + ansMsg := &AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answer} m.mu.Lock() m.sessions[remoteWgKey] = &peerSession{ state: stateAwaitingConfirm, exchangeID: o.ExchangeID, pendingPSK: psk, + answer: ansMsg, } m.mu.Unlock() - return &AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answer}, nil + return ansMsg, nil } // HandleAnswer processes a received answer as the initiator. On success it returns @@ -131,13 +145,15 @@ func (m *Manager) HandleAnswer(remoteWgKey string, a *AnswerMsg) (PSK, *ConfirmM } // HandleConfirm processes a received confirm as the responder and returns the PSK -// to commit now. It errors on a stale/unknown confirm so the caller ignores it. +// to commit now. It is idempotent for the current exchangeID: a retransmitted +// confirm returns the same PSK again (programming the same PSK is harmless). It +// errors on a stale/unknown confirm so the caller ignores it. func (m *Manager) HandleConfirm(remoteWgKey string, c *ConfirmMsg) (PSK, error) { m.mu.Lock() defer m.mu.Unlock() s, ok := m.sessions[remoteWgKey] - if !ok || s.state != stateAwaitingConfirm { + if !ok || (s.state != stateAwaitingConfirm && s.state != stateEstablished) { return PSK{}, fmt.Errorf("no pending answer for peer %s", remoteWgKey) } if s.exchangeID != c.ExchangeID { @@ -145,9 +161,7 @@ func (m *Manager) HandleConfirm(remoteWgKey string, c *ConfirmMsg) (PSK, error) } s.state = stateEstablished - psk := s.pendingPSK - s.pendingPSK = PSK{} - return psk, nil + return s.pendingPSK, nil } func (m *Manager) binding(remoteWgKey string) Binding { From 75dcd571ca1a0b1ab848f2098277aa8f301d1b54 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 21 Jul 2026 17:29:28 +0200 Subject: [PATCH 009/119] Models reattempts --- client/internal/pqkem/driver.go | 182 ++++++++++++++++++++++---------- 1 file changed, 126 insertions(+), 56 deletions(-) diff --git a/client/internal/pqkem/driver.go b/client/internal/pqkem/driver.go index 10e5e2a104f..c26d4ec941a 100644 --- a/client/internal/pqkem/driver.go +++ b/client/internal/pqkem/driver.go @@ -8,9 +8,21 @@ import ( "time" ) -// DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly -// rotated PSK is naturally adopted by WG's next handshake without forcing one. -const DefaultRekeyInterval = 2 * time.Minute +const ( + // DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly + // rotated PSK is naturally adopted by WG's next handshake without forcing one. + DefaultRekeyInterval = 2 * time.Minute + // DefaultRetryInterval is how often an in-flight exchange retransmits. + DefaultRetryInterval = 2 * time.Second + // DefaultConvergenceTimeout bounds a single exchange before it is declared failed. + DefaultConvergenceTimeout = 20 * time.Second + // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures are + // tolerated before OnRekeyFailed is raised. The initial exchange fails immediately. + DefaultMaxRekeyFailures = 3 + // confirmRetransmits is how many extra times the initiator best-effort resends the + // confirm (the exchange's unacked last message) to reduce a stuck responder. + confirmRetransmits = 3 +) // Transport hands an already-encoded exchange message to the peer. The host routes // it over the appropriate channel — Signal before the tunnel is up, the WireGuard @@ -19,29 +31,48 @@ type Transport interface { Send(remoteWgKey string, msg []byte) error } -// encodableMsg is the common shape of the three wire messages, letting the driver -// send any of them uniformly. type encodableMsg interface { Encode() ([]byte, error) } -// Driver ties the pure Manager to the outside world: it runs the per-peer rekey -// timer, dispatches inbound messages, and surfaces the derived PSK to the host via -// WGCallbackHandler. Convergence/retry/OnRekeyFailed are layered on in a later step. +// exchangeCtl tracks one in-flight exchange for a peer: its id, the cancel for its +// retransmit/deadline goroutine, when it started (for convergence latency) and +// whether it is the peer's first (initial) exchange (which fails hard on timeout). +type exchangeCtl struct { + id ExchangeID + cancel context.CancelFunc + startedAt time.Time + isInitial bool +} + +// Driver ties the pure Manager to the outside world: per-peer rekey timer, inbound +// dispatch, retransmit/convergence tracking, and surfacing events to the host via +// WGCallbackHandler. type Driver struct { mgr *Manager transport Transport wg WGCallbackHandler - interval time.Duration logger *slog.Logger - mu sync.Mutex - peers map[string]context.CancelFunc - wait sync.WaitGroup + rekeyInterval time.Duration + retryInterval time.Duration + convergenceTimeout time.Duration + maxRekeyFailures int + + rootCtx context.Context + rootCancel context.CancelFunc + + mu sync.Mutex + peers map[string]context.CancelFunc // per-peer rekey loop + exchanges map[string]*exchangeCtl // in-flight exchange per peer + established map[string]bool // peer has completed at least one exchange + failures map[string]int // consecutive rekey failures per peer + wait sync.WaitGroup } // NewDriver builds a driver for the local peer. A zero interval falls back to -// DefaultRekeyInterval; a nil logger falls back to slog.Default(). +// DefaultRekeyInterval; a nil logger falls back to slog.Default(). Retry/timeout/K +// use their defaults and can be overridden on the returned struct before use. func NewDriver(localWgKey string, t Transport, h WGCallbackHandler, interval time.Duration, logger *slog.Logger) *Driver { if interval <= 0 { interval = DefaultRekeyInterval @@ -49,54 +80,66 @@ func NewDriver(localWgKey string, t Transport, h WGCallbackHandler, interval tim if logger == nil { logger = slog.Default() } + ctx, cancel := context.WithCancel(context.Background()) return &Driver{ - mgr: NewManager(localWgKey), - transport: t, - wg: h, - interval: interval, - logger: logger, - peers: make(map[string]context.CancelFunc), + mgr: NewManager(localWgKey), + transport: t, + wg: h, + logger: logger, + rekeyInterval: interval, + retryInterval: DefaultRetryInterval, + convergenceTimeout: DefaultConvergenceTimeout, + maxRekeyFailures: DefaultMaxRekeyFailures, + rootCtx: ctx, + rootCancel: cancel, + peers: make(map[string]context.CancelFunc), + exchanges: make(map[string]*exchangeCtl), + established: make(map[string]bool), + failures: make(map[string]int), } } -// AddPeer registers a remote peer and starts its rekey timer. Re-adding an existing -// peer is a no-op. +// AddPeer registers a remote peer and starts its rekey timer. Re-adding is a no-op. func (d *Driver) AddPeer(remoteWgKey string) { d.mu.Lock() defer d.mu.Unlock() if _, ok := d.peers[remoteWgKey]; ok { return } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(d.rootCtx) d.peers[remoteWgKey] = cancel d.wait.Add(1) go d.rekeyLoop(ctx, remoteWgKey) } -// RemovePeer stops a peer's rekey timer and drops its state. +// RemovePeer stops a peer's rekey timer and any in-flight exchange, and drops state. func (d *Driver) RemovePeer(remoteWgKey string) { d.mu.Lock() - cancel, ok := d.peers[remoteWgKey] - delete(d.peers, remoteWgKey) - d.mu.Unlock() - if ok { + if cancel, ok := d.peers[remoteWgKey]; ok { cancel() + delete(d.peers, remoteWgKey) + } + if ex, ok := d.exchanges[remoteWgKey]; ok { + ex.cancel() + delete(d.exchanges, remoteWgKey) } + delete(d.established, remoteWgKey) + delete(d.failures, remoteWgKey) + d.mu.Unlock() } -// Stop cancels all peer timers and waits for their goroutines to exit. +// Stop cancels all timers and in-flight exchanges and waits for goroutines to exit. func (d *Driver) Stop() { + d.rootCancel() + d.wait.Wait() d.mu.Lock() - for _, cancel := range d.peers { - cancel() - } d.peers = make(map[string]context.CancelFunc) + d.exchanges = make(map[string]*exchangeCtl) d.mu.Unlock() - d.wait.Wait() } // HandleInbound decodes an incoming message and drives the exchange, sending any -// response via the transport and surfacing a derived PSK through the callback. +// response via the transport and surfacing derived PSKs / convergence to the host. func (d *Driver) HandleInbound(remoteWgKey string, raw []byte) error { typ, msg, err := Decode(raw) if err != nil { @@ -105,34 +148,56 @@ func (d *Driver) HandleInbound(remoteWgKey string, raw []byte) error { switch typ { case MsgOffer: - answer, err := d.mgr.HandleOffer(remoteWgKey, msg.(*OfferMsg)) - if err != nil { - return err - } - return d.send(remoteWgKey, answer) - + return d.handleOffer(remoteWgKey, msg.(*OfferMsg)) case MsgAnswer: - psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, msg.(*AnswerMsg)) - if err != nil { - return err - } - if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { - return err - } - return d.send(remoteWgKey, confirm) - + return d.handleAnswer(remoteWgKey, msg.(*AnswerMsg)) case MsgConfirm: - psk, err := d.mgr.HandleConfirm(remoteWgKey, msg.(*ConfirmMsg)) - if err != nil { - return err - } - return d.wg.OnNewPSKReady(remoteWgKey, psk) - + return d.handleConfirm(remoteWgKey, msg.(*ConfirmMsg)) default: return fmt.Errorf("unhandled message type %d from %s", typ, remoteWgKey) } } +func (d *Driver) handleOffer(remoteWgKey string, o *OfferMsg) error { + answer, err := d.mgr.HandleOffer(remoteWgKey, o) + if err != nil { + return err + } + // Arm a responder exchange (deadline waiting for the confirm) unless one for this + // id is already armed — a retransmitted offer just re-sends the same answer. + d.armExchange(remoteWgKey, o.ExchangeID, nil) + return d.send(remoteWgKey, answer) +} + +func (d *Driver) handleAnswer(remoteWgKey string, a *AnswerMsg) error { + psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, a) + if err != nil { + return err + } + if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { + return err + } + // Initiator has the PSK now -> this side has converged. + d.onConverged(remoteWgKey, a.ExchangeID) + if err := d.send(remoteWgKey, confirm); err != nil { + return err + } + d.retransmitConfirm(remoteWgKey, confirm) + return nil +} + +func (d *Driver) handleConfirm(remoteWgKey string, c *ConfirmMsg) error { + psk, err := d.mgr.HandleConfirm(remoteWgKey, c) + if err != nil { + return err + } + if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { + return err + } + d.onConverged(remoteWgKey, c.ExchangeID) + return nil +} + // initiateRekey starts a fresh exchange when the local peer is the initiator for // this remote peer; the responder waits for the offer instead. Exposed (unexported // but directly callable) so tests can drive a rekey without waiting on the ticker. @@ -144,12 +209,17 @@ func (d *Driver) initiateRekey(remoteWgKey string) error { if err != nil { return err } - return d.send(remoteWgKey, offer) + raw, err := offer.Encode() + if err != nil { + return err + } + d.armExchange(remoteWgKey, offer.ExchangeID, raw) + return d.transport.Send(remoteWgKey, raw) } func (d *Driver) rekeyLoop(ctx context.Context, remoteWgKey string) { defer d.wait.Done() - t := time.NewTicker(d.interval) + t := time.NewTicker(d.rekeyInterval) defer t.Stop() for { select { From fa3c3ad214270c46d8dbd20e3b605beccd678d9a Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 22 Jul 2026 09:35:05 +0200 Subject: [PATCH 010/119] Manages convergence --- client/internal/pqkem/convergence.go | 193 ++++++++++++++++++++++ client/internal/pqkem/convergence_test.go | 88 ++++++++++ client/internal/pqkem/driver.go | 143 +++++++--------- client/internal/pqkem/manager.go | 50 +++--- 4 files changed, 359 insertions(+), 115 deletions(-) create mode 100644 client/internal/pqkem/convergence.go create mode 100644 client/internal/pqkem/convergence_test.go diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go new file mode 100644 index 00000000000..8b2ac93bb28 --- /dev/null +++ b/client/internal/pqkem/convergence.go @@ -0,0 +1,193 @@ +package pqkem + +import ( + "context" + "time" +) + +// handleOffer (responder) derives and sends the answer for a new exchange, or +// resends the cached answer for a duplicate offer (same exchangeID) without +// re-deriving. The responder is purely reactive: no retransmit loop, no deadline. +func (d *Driver) handleOffer(remoteWgKey string, o *OfferMsg) error { + d.mu.Lock() + if ex := d.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + last := ex.lastSent + d.mu.Unlock() + if last == nil { + // another goroutine reserved this exchange and is deriving the answer; + // it will send it. Dropping this duplicate avoids a second derivation + // (Respond is randomized -> a different PSK). + return nil + } + return d.transport.Send(remoteWgKey, last) + } + // Reserve the slot (lastSent nil = deriving) so a concurrent duplicate offer bails. + d.exchanges[remoteWgKey] = &exchangeCtl{ + id: o.ExchangeID, + startedAt: time.Now(), + isInitial: !d.established[remoteWgKey], + } + d.mu.Unlock() + + answer, err := d.mgr.HandleOffer(remoteWgKey, o) + if err != nil { + return err + } + raw, err := answer.Encode() + if err != nil { + return err + } + + d.mu.Lock() + if ex := d.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + ex.lastSent = raw + } + d.mu.Unlock() + + return d.transport.Send(remoteWgKey, raw) +} + +// handleAnswer (initiator) derives the PSK, surfaces it, and sends the confirm. The +// exchange then switches its retransmit payload to the confirm. Stale or duplicate +// answers are ignored. +func (d *Driver) handleAnswer(remoteWgKey string, a *AnswerMsg) error { + // Claim the answer under the lock (set answered) so a concurrent/duplicate answer + // bails before calling the Manager. + d.mu.Lock() + ex := d.exchanges[remoteWgKey] + if ex == nil || ex.id != a.ExchangeID || ex.cancel == nil || ex.answered { + d.mu.Unlock() + return nil + } + ex.answered = true + d.mu.Unlock() + + psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, a) + if err != nil { + return err + } + if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { + return err + } + raw, err := confirm.Encode() + if err != nil { + return err + } + + d.mu.Lock() + if ex := d.exchanges[remoteWgKey]; ex != nil && ex.id == a.ExchangeID { + ex.lastSent = raw // loop now retransmits the confirm + d.established[remoteWgKey] = true + d.failures[remoteWgKey] = 0 + _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) + } + d.mu.Unlock() + + return d.transport.Send(remoteWgKey, raw) +} + +// handleConfirm (responder) commits the pending PSK. Stale or duplicate confirms +// (the initiator best-effort resends it) are ignored. +func (d *Driver) handleConfirm(remoteWgKey string, c *ConfirmMsg) error { + // Claim-and-remove the exchange under the lock so a duplicate confirm (the + // initiator best-effort resends it) bails before calling the Manager. + d.mu.Lock() + ex := d.exchanges[remoteWgKey] + if ex == nil || ex.id != c.ExchangeID { + d.mu.Unlock() + return nil + } + delete(d.exchanges, remoteWgKey) + d.established[remoteWgKey] = true + d.failures[remoteWgKey] = 0 + _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) + d.mu.Unlock() + + psk, err := d.mgr.HandleConfirm(remoteWgKey, c) + if err != nil { + return err + } + return d.wg.OnNewPSKReady(remoteWgKey, psk) +} + +// initiatorLoop retransmits the initiator's outstanding message: the offer until +// the answer arrives (bounded by maxRetries -> failure), then the confirm a few +// best-effort times before stopping. The convergence deadline is thus derived from +// maxRetries * retryInterval; there is no separate deadline timer. +func (d *Driver) initiatorLoop(ctx context.Context, remoteWgKey string, id ExchangeID) { + defer d.wait.Done() + t := time.NewTicker(d.retryInterval) + defer t.Stop() + + offerAttempts, confirmsSent := 0, 0 + for { + select { + case <-ctx.Done(): + return + case <-t.C: + d.mu.Lock() + ex := d.exchanges[remoteWgKey] + if ex == nil || ex.id != id { + d.mu.Unlock() + return + } + + if !ex.answered { + if offerAttempts >= d.maxRetries { + delete(d.exchanges, remoteWgKey) + fail := d.registerFailureLocked(remoteWgKey, ex.isInitial) + d.mu.Unlock() + d.raiseFailure(remoteWgKey, fail) + return + } + msg := ex.lastSent + offerAttempts++ + d.mu.Unlock() + d.retransmit(remoteWgKey, msg) + continue + } + + if confirmsSent >= confirmRetransmits { + delete(d.exchanges, remoteWgKey) + d.mu.Unlock() + return + } + msg := ex.lastSent + confirmsSent++ + d.mu.Unlock() + d.retransmit(remoteWgKey, msg) + } + } +} + +// registerFailureLocked applies policy B and reports whether OnRekeyFailed is due: +// the initial exchange fails immediately; a rekey tolerates up to maxRekeyFailures +// consecutive misses (we stay on the still-valid previous PSK) before failing. +// Assumes d.mu is held. +func (d *Driver) registerFailureLocked(remoteWgKey string, isInitial bool) bool { + if isInitial { + return true + } + d.failures[remoteWgKey]++ + if d.failures[remoteWgKey] >= d.maxRekeyFailures { + d.failures[remoteWgKey] = 0 + return true + } + return false +} + +func (d *Driver) raiseFailure(remoteWgKey string, fail bool) { + if !fail { + d.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteWgKey) + return + } + if err := d.wg.OnRekeyFailed(remoteWgKey); err != nil { + d.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteWgKey, "err", err) + } +} + +func (d *Driver) retransmit(remoteWgKey string, msg []byte) { + if err := d.transport.Send(remoteWgKey, msg); err != nil { + d.logger.Warn("pqkem retransmit failed", "peer", remoteWgKey, "err", err) + } +} diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go new file mode 100644 index 00000000000..e05331210b1 --- /dev/null +++ b/client/internal/pqkem/convergence_test.go @@ -0,0 +1,88 @@ +package pqkem + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type dropTransport struct{} + +func (dropTransport) Send(string, []byte) error { return nil } + +// gate is a loopback transport with a switchable drop flag. +type gate struct { + local string + peer *Driver + drop atomic.Bool +} + +func (g *gate) Send(remote string, msg []byte) error { + if g.drop.Load() { + return nil + } + cp := append([]byte(nil), msg...) + return g.peer.HandleInbound(g.local, cp) +} + +func TestDriver_InitialTimeoutFailsImmediately(t *testing.T) { + wg := newFakeWG() + d := NewDriver("bbbb", dropTransport{}, wg, time.Hour, nil) // bbbb > aaaa -> initiator + d.retryInterval = 5 * time.Millisecond + d.maxRetries = 3 + d.AddPeer("aaaa") + defer d.Stop() + + require.NoError(t, d.initiateRekey("aaaa")) + + // no answer will ever come -> the initial exchange fails fast. + require.Eventually(t, func() bool { + wg.mu.Lock() + defer wg.mu.Unlock() + return len(wg.failed) == 1 + }, time.Second, 5*time.Millisecond) +} + +func TestDriver_RekeyToleratesKFailures(t *testing.T) { + gA := &gate{local: "aaaa"} + gB := &gate{local: "bbbb"} + wgA := newFakeWG() + wgB := newFakeWG() + + dA := NewDriver("aaaa", gA, wgA, time.Hour, nil) + dB := NewDriver("bbbb", gB, wgB, time.Hour, nil) + gA.peer = dB + gB.peer = dA + dB.retryInterval = 5 * time.Millisecond + dB.maxRetries = 2 + dA.AddPeer("bbbb") + dB.AddPeer("aaaa") + defer dA.Stop() + defer dB.Stop() + + // first exchange succeeds -> peer becomes established (subsequent failures are rekeys). + require.NoError(t, dB.initiateRekey("aaaa")) + require.NotEqual(t, PSK{}, wgB.psk("aaaa")) + + // now drop B's outbound: rekeys can no longer converge. + gB.drop.Store(true) + + // K-1 failures must NOT raise OnRekeyFailed. + for i := 0; i < DefaultMaxRekeyFailures-1; i++ { + require.NoError(t, dB.initiateRekey("aaaa")) + time.Sleep(50 * time.Millisecond) + } + require.Equal(t, 0, failedCount(wgB), "no failure before K attempts") + + // the K-th failure raises it once. + require.NoError(t, dB.initiateRekey("aaaa")) + require.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) +} + +func failedCount(f *fakeWG) int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.failed) +} diff --git a/client/internal/pqkem/driver.go b/client/internal/pqkem/driver.go index c26d4ec941a..8e0980dbcb4 100644 --- a/client/internal/pqkem/driver.go +++ b/client/internal/pqkem/driver.go @@ -12,15 +12,18 @@ const ( // DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly // rotated PSK is naturally adopted by WG's next handshake without forcing one. DefaultRekeyInterval = 2 * time.Minute - // DefaultRetryInterval is how often an in-flight exchange retransmits. + // DefaultRetryInterval is how often the initiator retransmits its outstanding + // message (offer, then confirm) while an exchange is in flight. DefaultRetryInterval = 2 * time.Second - // DefaultConvergenceTimeout bounds a single exchange before it is declared failed. - DefaultConvergenceTimeout = 20 * time.Second - // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures are - // tolerated before OnRekeyFailed is raised. The initial exchange fails immediately. + // DefaultMaxRetries bounds how many times the offer is retransmitted before the + // exchange is declared failed. The convergence deadline is thus derived as + // MaxRetries * RetryInterval — there is no separate deadline timer. + DefaultMaxRetries = 10 + // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures + // are tolerated before OnRekeyFailed. The initial exchange fails immediately. DefaultMaxRekeyFailures = 3 - // confirmRetransmits is how many extra times the initiator best-effort resends the - // confirm (the exchange's unacked last message) to reduce a stuck responder. + // confirmRetransmits is how many times the initiator best-effort resends the + // confirm after converging, to cover its loss without a dedicated goroutine. confirmRetransmits = 3 ) @@ -31,18 +34,18 @@ type Transport interface { Send(remoteWgKey string, msg []byte) error } -type encodableMsg interface { - Encode() ([]byte, error) -} - -// exchangeCtl tracks one in-flight exchange for a peer: its id, the cancel for its -// retransmit/deadline goroutine, when it started (for convergence latency) and -// whether it is the peer's first (initial) exchange (which fails hard on timeout). +// exchangeCtl tracks one in-flight exchange for a peer. A single lastSent holds the +// message currently being (re)transmitted: for the initiator it is the offer and +// then, once answered, the confirm; for the responder it is the answer, resent on a +// duplicate offer. Only the initiator runs a retransmit loop (cancel != nil); the +// responder is purely reactive. type exchangeCtl struct { id ExchangeID - cancel context.CancelFunc startedAt time.Time isInitial bool + lastSent []byte + cancel context.CancelFunc + answered bool // initiator: the answer arrived; retransmit phase is now the confirm } // Driver ties the pure Manager to the outside world: per-peer rekey timer, inbound @@ -54,10 +57,10 @@ type Driver struct { wg WGCallbackHandler logger *slog.Logger - rekeyInterval time.Duration - retryInterval time.Duration - convergenceTimeout time.Duration - maxRekeyFailures int + rekeyInterval time.Duration + retryInterval time.Duration + maxRetries int + maxRekeyFailures int rootCtx context.Context rootCancel context.CancelFunc @@ -71,7 +74,7 @@ type Driver struct { } // NewDriver builds a driver for the local peer. A zero interval falls back to -// DefaultRekeyInterval; a nil logger falls back to slog.Default(). Retry/timeout/K +// DefaultRekeyInterval; a nil logger falls back to slog.Default(). Retry/retries/K // use their defaults and can be overridden on the returned struct before use. func NewDriver(localWgKey string, t Transport, h WGCallbackHandler, interval time.Duration, logger *slog.Logger) *Driver { if interval <= 0 { @@ -82,20 +85,20 @@ func NewDriver(localWgKey string, t Transport, h WGCallbackHandler, interval tim } ctx, cancel := context.WithCancel(context.Background()) return &Driver{ - mgr: NewManager(localWgKey), - transport: t, - wg: h, - logger: logger, - rekeyInterval: interval, - retryInterval: DefaultRetryInterval, - convergenceTimeout: DefaultConvergenceTimeout, - maxRekeyFailures: DefaultMaxRekeyFailures, - rootCtx: ctx, - rootCancel: cancel, - peers: make(map[string]context.CancelFunc), - exchanges: make(map[string]*exchangeCtl), - established: make(map[string]bool), - failures: make(map[string]int), + mgr: NewManager(localWgKey), + transport: t, + wg: h, + logger: logger, + rekeyInterval: interval, + retryInterval: DefaultRetryInterval, + maxRetries: DefaultMaxRetries, + maxRekeyFailures: DefaultMaxRekeyFailures, + rootCtx: ctx, + rootCancel: cancel, + peers: make(map[string]context.CancelFunc), + exchanges: make(map[string]*exchangeCtl), + established: make(map[string]bool), + failures: make(map[string]int), } } @@ -120,7 +123,9 @@ func (d *Driver) RemovePeer(remoteWgKey string) { delete(d.peers, remoteWgKey) } if ex, ok := d.exchanges[remoteWgKey]; ok { - ex.cancel() + if ex.cancel != nil { + ex.cancel() + } delete(d.exchanges, remoteWgKey) } delete(d.established, remoteWgKey) @@ -145,7 +150,6 @@ func (d *Driver) HandleInbound(remoteWgKey string, raw []byte) error { if err != nil { return fmt.Errorf("decode from %s: %w", remoteWgKey, err) } - switch typ { case MsgOffer: return d.handleOffer(remoteWgKey, msg.(*OfferMsg)) @@ -158,46 +162,6 @@ func (d *Driver) HandleInbound(remoteWgKey string, raw []byte) error { } } -func (d *Driver) handleOffer(remoteWgKey string, o *OfferMsg) error { - answer, err := d.mgr.HandleOffer(remoteWgKey, o) - if err != nil { - return err - } - // Arm a responder exchange (deadline waiting for the confirm) unless one for this - // id is already armed — a retransmitted offer just re-sends the same answer. - d.armExchange(remoteWgKey, o.ExchangeID, nil) - return d.send(remoteWgKey, answer) -} - -func (d *Driver) handleAnswer(remoteWgKey string, a *AnswerMsg) error { - psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, a) - if err != nil { - return err - } - if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { - return err - } - // Initiator has the PSK now -> this side has converged. - d.onConverged(remoteWgKey, a.ExchangeID) - if err := d.send(remoteWgKey, confirm); err != nil { - return err - } - d.retransmitConfirm(remoteWgKey, confirm) - return nil -} - -func (d *Driver) handleConfirm(remoteWgKey string, c *ConfirmMsg) error { - psk, err := d.mgr.HandleConfirm(remoteWgKey, c) - if err != nil { - return err - } - if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { - return err - } - d.onConverged(remoteWgKey, c.ExchangeID) - return nil -} - // initiateRekey starts a fresh exchange when the local peer is the initiator for // this remote peer; the responder waits for the offer instead. Exposed (unexported // but directly callable) so tests can drive a rekey without waiting on the ticker. @@ -213,7 +177,24 @@ func (d *Driver) initiateRekey(remoteWgKey string) error { if err != nil { return err } - d.armExchange(remoteWgKey, offer.ExchangeID, raw) + + ctx, cancel := context.WithCancel(d.rootCtx) + d.mu.Lock() + if old := d.exchanges[remoteWgKey]; old != nil && old.cancel != nil { + old.cancel() + } + d.exchanges[remoteWgKey] = &exchangeCtl{ + id: offer.ExchangeID, + startedAt: time.Now(), + isInitial: !d.established[remoteWgKey], + lastSent: raw, + cancel: cancel, + } + d.mu.Unlock() + + d.wait.Add(1) + go d.initiatorLoop(ctx, remoteWgKey, offer.ExchangeID) + return d.transport.Send(remoteWgKey, raw) } @@ -232,11 +213,3 @@ func (d *Driver) rekeyLoop(ctx context.Context, remoteWgKey string) { } } } - -func (d *Driver) send(remoteWgKey string, m encodableMsg) error { - raw, err := m.Encode() - if err != nil { - return err - } - return d.transport.Send(remoteWgKey, raw) -} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index bfdd1a1e2f0..cc746c42311 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -31,11 +31,8 @@ type peerSession struct { // initiator side: in-flight ephemeral keypair between offer and answer. initiator *Initiator - // responder side: the derived PSK and the answer for this exchange. Both are - // retained after establishment so retransmitted offers/confirms (same exchangeID) - // are handled idempotently — never re-derive a different key. + // responder side: PSK derived on the offer, committed only on the confirm. pendingPSK PSK - answer *AnswerMsg } type Manager struct { @@ -87,33 +84,24 @@ func (m *Manager) StartExchange(remoteWgKey string) (*OfferMsg, error) { // HandleOffer processes a received offer as the responder, derives the PSK, and // returns the answer to send. The PSK is held pending and NOT returned: the caller -// must not program it until HandleConfirm. A retransmitted offer (same exchangeID) -// returns the cached answer without re-deriving, so both sides keep the same key. +// must not program it until HandleConfirm. Retransmitted offers are deduplicated by +// the driver (via exchangeID) so this is called once per exchange and never +// re-derives a different key for the same round. func (m *Manager) HandleOffer(remoteWgKey string, o *OfferMsg) (*AnswerMsg, error) { - m.mu.Lock() - if s, ok := m.sessions[remoteWgKey]; ok && s.exchangeID == o.ExchangeID && s.answer != nil { - ans := s.answer - m.mu.Unlock() - return ans, nil - } - m.mu.Unlock() - answer, psk, err := Respond(o.KEMOffer, m.binding(remoteWgKey)) if err != nil { return nil, err } - ansMsg := &AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answer} m.mu.Lock() m.sessions[remoteWgKey] = &peerSession{ state: stateAwaitingConfirm, exchangeID: o.ExchangeID, pendingPSK: psk, - answer: ansMsg, } m.mu.Unlock() - return ansMsg, nil + return &AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answer}, nil } // HandleAnswer processes a received answer as the initiator. On success it returns @@ -121,39 +109,39 @@ func (m *Manager) HandleOffer(remoteWgKey string, o *OfferMsg) (*AnswerMsg, erro func (m *Manager) HandleAnswer(remoteWgKey string, a *AnswerMsg) (PSK, *ConfirmMsg, error) { m.mu.Lock() s, ok := m.sessions[remoteWgKey] - m.mu.Unlock() - if !ok || s.state != stateAwaitingAnswer { + m.mu.Unlock() return PSK{}, nil, fmt.Errorf("no pending offer for peer %s", remoteWgKey) } if s.exchangeID != a.ExchangeID { // stale answer (e.g. to a pre-restart offer) — drop, keep waiting. + m.mu.Unlock() return PSK{}, nil, fmt.Errorf("answer exchangeID mismatch for peer %s", remoteWgKey) } + // Claim the exchange under the lock so a concurrent answer bails, and copy out + // the initiator so Finish (expensive crypto) runs without holding the lock. + init := s.initiator + s.state = stateEstablished + s.initiator = nil + m.mu.Unlock() - psk, err := s.initiator.Finish(a.KEMAnswer, m.binding(remoteWgKey)) + psk, err := init.Finish(a.KEMAnswer, m.binding(remoteWgKey)) if err != nil { return PSK{}, nil, err } - m.mu.Lock() - s.state = stateEstablished - s.initiator = nil - m.mu.Unlock() - return psk, &ConfirmMsg{ExchangeID: a.ExchangeID}, nil } // HandleConfirm processes a received confirm as the responder and returns the PSK -// to commit now. It is idempotent for the current exchangeID: a retransmitted -// confirm returns the same PSK again (programming the same PSK is harmless). It -// errors on a stale/unknown confirm so the caller ignores it. +// to commit now. Retransmitted confirms are deduplicated by the driver, so this is +// called once. It errors on a stale/unknown confirm so the caller ignores it. func (m *Manager) HandleConfirm(remoteWgKey string, c *ConfirmMsg) (PSK, error) { m.mu.Lock() defer m.mu.Unlock() s, ok := m.sessions[remoteWgKey] - if !ok || (s.state != stateAwaitingConfirm && s.state != stateEstablished) { + if !ok || s.state != stateAwaitingConfirm { return PSK{}, fmt.Errorf("no pending answer for peer %s", remoteWgKey) } if s.exchangeID != c.ExchangeID { @@ -161,7 +149,9 @@ func (m *Manager) HandleConfirm(remoteWgKey string, c *ConfirmMsg) (PSK, error) } s.state = stateEstablished - return s.pendingPSK, nil + psk := s.pendingPSK + s.pendingPSK = PSK{} + return psk, nil } func (m *Manager) binding(remoteWgKey string) Binding { From 0a3dc2b43ff8a1c10448810c0ca1111ba1622f68 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 22 Jul 2026 09:46:34 +0200 Subject: [PATCH 011/119] [squash] isInitial and answered can be inferred without state variables --- client/internal/pqkem/convergence.go | 28 +++++++++++++++++----------- client/internal/pqkem/driver.go | 13 ++++++------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 8b2ac93bb28..3e1368ef7b8 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -25,7 +25,6 @@ func (d *Driver) handleOffer(remoteWgKey string, o *OfferMsg) error { d.exchanges[remoteWgKey] = &exchangeCtl{ id: o.ExchangeID, startedAt: time.Now(), - isInitial: !d.established[remoteWgKey], } d.mu.Unlock() @@ -51,15 +50,15 @@ func (d *Driver) handleOffer(remoteWgKey string, o *OfferMsg) error { // exchange then switches its retransmit payload to the confirm. Stale or duplicate // answers are ignored. func (d *Driver) handleAnswer(remoteWgKey string, a *AnswerMsg) error { - // Claim the answer under the lock (set answered) so a concurrent/duplicate answer - // bails before calling the Manager. + // Ignore stale answers, answers to a responder-side exchange (cancel == nil), and + // duplicates once we are already in the confirm phase. A duplicate that still + // slips through is caught by the Manager, whose HandleAnswer bails under its lock. d.mu.Lock() ex := d.exchanges[remoteWgKey] - if ex == nil || ex.id != a.ExchangeID || ex.cancel == nil || ex.answered { + if ex == nil || ex.id != a.ExchangeID || ex.cancel == nil || isConfirmPhase(ex.lastSent) { d.mu.Unlock() return nil } - ex.answered = true d.mu.Unlock() psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, a) @@ -132,10 +131,10 @@ func (d *Driver) initiatorLoop(ctx context.Context, remoteWgKey string, id Excha return } - if !ex.answered { + if !isConfirmPhase(ex.lastSent) { if offerAttempts >= d.maxRetries { delete(d.exchanges, remoteWgKey) - fail := d.registerFailureLocked(remoteWgKey, ex.isInitial) + fail := d.registerFailureLocked(remoteWgKey) d.mu.Unlock() d.raiseFailure(remoteWgKey, fail) return @@ -161,11 +160,12 @@ func (d *Driver) initiatorLoop(ctx context.Context, remoteWgKey string, id Excha } // registerFailureLocked applies policy B and reports whether OnRekeyFailed is due: -// the initial exchange fails immediately; a rekey tolerates up to maxRekeyFailures -// consecutive misses (we stay on the still-valid previous PSK) before failing. +// an initial exchange (peer never established) fails immediately; a rekey tolerates +// up to maxRekeyFailures consecutive misses (we stay on the still-valid previous +// PSK) before failing. Whether it is initial is read live from established. // Assumes d.mu is held. -func (d *Driver) registerFailureLocked(remoteWgKey string, isInitial bool) bool { - if isInitial { +func (d *Driver) registerFailureLocked(remoteWgKey string) bool { + if !d.established[remoteWgKey] { return true } d.failures[remoteWgKey]++ @@ -176,6 +176,12 @@ func (d *Driver) registerFailureLocked(remoteWgKey string, isInitial bool) bool return false } +// isConfirmPhase reports whether the initiator's outstanding message is the confirm +// (as opposed to the offer), derived from the message's type byte. +func isConfirmPhase(lastSent []byte) bool { + return len(lastSent) > 0 && lastSent[0] == byte(MsgConfirm) +} + func (d *Driver) raiseFailure(remoteWgKey string, fail bool) { if !fail { d.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteWgKey) diff --git a/client/internal/pqkem/driver.go b/client/internal/pqkem/driver.go index 8e0980dbcb4..b4b6c4514a1 100644 --- a/client/internal/pqkem/driver.go +++ b/client/internal/pqkem/driver.go @@ -35,17 +35,17 @@ type Transport interface { } // exchangeCtl tracks one in-flight exchange for a peer. A single lastSent holds the -// message currently being (re)transmitted: for the initiator it is the offer and -// then, once answered, the confirm; for the responder it is the answer, resent on a -// duplicate offer. Only the initiator runs a retransmit loop (cancel != nil); the -// responder is purely reactive. +// message currently being (re)transmitted: for the initiator the offer and then, +// once answered, the confirm; for the responder the answer, resent on a duplicate +// offer. Its type byte (lastSent[0]) also encodes the initiator's phase, so no +// separate "answered" flag is kept. Only the initiator runs a retransmit loop +// (cancel != nil); the responder is purely reactive. Whether an exchange is the +// peer's first is read live from d.established, not snapshotted here. type exchangeCtl struct { id ExchangeID startedAt time.Time - isInitial bool lastSent []byte cancel context.CancelFunc - answered bool // initiator: the answer arrived; retransmit phase is now the confirm } // Driver ties the pure Manager to the outside world: per-peer rekey timer, inbound @@ -186,7 +186,6 @@ func (d *Driver) initiateRekey(remoteWgKey string) error { d.exchanges[remoteWgKey] = &exchangeCtl{ id: offer.ExchangeID, startedAt: time.Now(), - isInitial: !d.established[remoteWgKey], lastSent: raw, cancel: cancel, } From 661926ce3a3d95b3b771ac1be0732befac450160 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 23 Jul 2026 10:47:23 +0200 Subject: [PATCH 012/119] Collapse Driver and Manager in one. - Have just one manager => one lock - Session state is needed in driver to => we have it available now. - Isomorphically align to rosenpass components and functionality File Role rosenpass equivalent kem.go primitive pure X25519MLKEM768 crypto.go/handshake message.go Offer/Answer/Confirm + Encode/Decode messages.go manager.go Manager stateful, single lock server logic callbacks.go WGCallbackHandler (seam output) Handler Transport (interfaccia) seam trasporto pluggable Conn --- client/internal/pqkem/convergence.go | 206 ++++++++------- client/internal/pqkem/convergence_test.go | 12 +- client/internal/pqkem/driver.go | 214 ---------------- client/internal/pqkem/driver_test.go | 96 ------- client/internal/pqkem/manager.go | 295 ++++++++++++++-------- client/internal/pqkem/manager_test.go | 129 ++++++---- 6 files changed, 373 insertions(+), 579 deletions(-) delete mode 100644 client/internal/pqkem/driver.go delete mode 100644 client/internal/pqkem/driver_test.go diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 3e1368ef7b8..63c0ba1365f 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -8,114 +8,108 @@ import ( // handleOffer (responder) derives and sends the answer for a new exchange, or // resends the cached answer for a duplicate offer (same exchangeID) without // re-deriving. The responder is purely reactive: no retransmit loop, no deadline. -func (d *Driver) handleOffer(remoteWgKey string, o *OfferMsg) error { - d.mu.Lock() - if ex := d.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { - last := ex.lastSent - d.mu.Unlock() - if last == nil { +func (m *Manager) handleOffer(remoteWgKey string, o *OfferMsg) error { + m.mu.Lock() + if ex := m.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + state, last := ex.state, ex.lastSent + m.mu.Unlock() + if state == stateReserved { // another goroutine reserved this exchange and is deriving the answer; - // it will send it. Dropping this duplicate avoids a second derivation - // (Respond is randomized -> a different PSK). + // dropping avoids a second (randomized -> divergent) derivation. return nil } - return d.transport.Send(remoteWgKey, last) + return m.transport.Send(remoteWgKey, last) } - // Reserve the slot (lastSent nil = deriving) so a concurrent duplicate offer bails. - d.exchanges[remoteWgKey] = &exchangeCtl{ - id: o.ExchangeID, - startedAt: time.Now(), - } - d.mu.Unlock() + // Reserve the slot so a concurrent duplicate offer bails. + m.exchanges[remoteWgKey] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, startedAt: time.Now()} + m.mu.Unlock() - answer, err := d.mgr.HandleOffer(remoteWgKey, o) + answerBytes, psk, err := Respond(o.KEMOffer, m.binding(remoteWgKey)) if err != nil { return err } - raw, err := answer.Encode() + raw, err := (&AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answerBytes}).Encode() if err != nil { return err } - d.mu.Lock() - if ex := d.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + m.mu.Lock() + if ex := m.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + ex.state = stateAwaitingConfirm ex.lastSent = raw + ex.pendingPSK = psk } - d.mu.Unlock() + m.mu.Unlock() - return d.transport.Send(remoteWgKey, raw) + return m.transport.Send(remoteWgKey, raw) } -// handleAnswer (initiator) derives the PSK, surfaces it, and sends the confirm. The -// exchange then switches its retransmit payload to the confirm. Stale or duplicate -// answers are ignored. -func (d *Driver) handleAnswer(remoteWgKey string, a *AnswerMsg) error { - // Ignore stale answers, answers to a responder-side exchange (cancel == nil), and - // duplicates once we are already in the confirm phase. A duplicate that still - // slips through is caught by the Manager, whose HandleAnswer bails under its lock. - d.mu.Lock() - ex := d.exchanges[remoteWgKey] - if ex == nil || ex.id != a.ExchangeID || ex.cancel == nil || isConfirmPhase(ex.lastSent) { - d.mu.Unlock() +// handleAnswer (initiator) derives the PSK, surfaces it, and sends the confirm, then +// switches the retransmit payload to the confirm. Only valid in stateAwaitingAnswer; +// advancing the state under the lock makes a concurrent/duplicate answer bail. +func (m *Manager) handleAnswer(remoteWgKey string, a *AnswerMsg) error { + m.mu.Lock() + ex := m.exchanges[remoteWgKey] + if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { + m.mu.Unlock() return nil } - d.mu.Unlock() + ex.state = stateConfirming + init := ex.initiator + ex.initiator = nil + m.mu.Unlock() - psk, confirm, err := d.mgr.HandleAnswer(remoteWgKey, a) + psk, err := init.Finish(a.KEMAnswer, m.binding(remoteWgKey)) if err != nil { return err } - if err := d.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { + if err := m.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { return err } - raw, err := confirm.Encode() + raw, err := (&ConfirmMsg{ExchangeID: a.ExchangeID}).Encode() if err != nil { return err } - d.mu.Lock() - if ex := d.exchanges[remoteWgKey]; ex != nil && ex.id == a.ExchangeID { + m.mu.Lock() + if ex := m.exchanges[remoteWgKey]; ex != nil && ex.id == a.ExchangeID { ex.lastSent = raw // loop now retransmits the confirm - d.established[remoteWgKey] = true - d.failures[remoteWgKey] = 0 + m.established[remoteWgKey] = true + m.failures[remoteWgKey] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) } - d.mu.Unlock() + m.mu.Unlock() - return d.transport.Send(remoteWgKey, raw) + return m.transport.Send(remoteWgKey, raw) } -// handleConfirm (responder) commits the pending PSK. Stale or duplicate confirms -// (the initiator best-effort resends it) are ignored. -func (d *Driver) handleConfirm(remoteWgKey string, c *ConfirmMsg) error { - // Claim-and-remove the exchange under the lock so a duplicate confirm (the - // initiator best-effort resends it) bails before calling the Manager. - d.mu.Lock() - ex := d.exchanges[remoteWgKey] - if ex == nil || ex.id != c.ExchangeID { - d.mu.Unlock() +// handleConfirm (responder) commits the pending PSK. Only valid in +// stateAwaitingConfirm; duplicate confirms (the initiator best-effort resends it) +// find the exchange gone and are ignored. +func (m *Manager) handleConfirm(remoteWgKey string, c *ConfirmMsg) error { + m.mu.Lock() + ex := m.exchanges[remoteWgKey] + if ex == nil || ex.id != c.ExchangeID || ex.state != stateAwaitingConfirm { + m.mu.Unlock() return nil } - delete(d.exchanges, remoteWgKey) - d.established[remoteWgKey] = true - d.failures[remoteWgKey] = 0 + psk := ex.pendingPSK + delete(m.exchanges, remoteWgKey) + m.established[remoteWgKey] = true + m.failures[remoteWgKey] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) - d.mu.Unlock() + m.mu.Unlock() - psk, err := d.mgr.HandleConfirm(remoteWgKey, c) - if err != nil { - return err - } - return d.wg.OnNewPSKReady(remoteWgKey, psk) + return m.wg.OnNewPSKReady(remoteWgKey, psk) } -// initiatorLoop retransmits the initiator's outstanding message: the offer until -// the answer arrives (bounded by maxRetries -> failure), then the confirm a few -// best-effort times before stopping. The convergence deadline is thus derived from -// maxRetries * retryInterval; there is no separate deadline timer. -func (d *Driver) initiatorLoop(ctx context.Context, remoteWgKey string, id ExchangeID) { - defer d.wait.Done() - t := time.NewTicker(d.retryInterval) +// initiatorLoop retransmits the initiator's outstanding message, keyed off the +// exchange state: the offer while awaiting the answer (bounded by maxRetries -> +// failure), then the confirm a few best-effort times before stopping. The +// convergence deadline is thus derived from maxRetries * retryInterval. +func (m *Manager) initiatorLoop(ctx context.Context, remoteWgKey string, id ExchangeID) { + defer m.wait.Done() + t := time.NewTicker(m.retryInterval) defer t.Stop() offerAttempts, confirmsSent := 0, 0 @@ -124,37 +118,42 @@ func (d *Driver) initiatorLoop(ctx context.Context, remoteWgKey string, id Excha case <-ctx.Done(): return case <-t.C: - d.mu.Lock() - ex := d.exchanges[remoteWgKey] + m.mu.Lock() + ex := m.exchanges[remoteWgKey] if ex == nil || ex.id != id { - d.mu.Unlock() + m.mu.Unlock() return } - if !isConfirmPhase(ex.lastSent) { - if offerAttempts >= d.maxRetries { - delete(d.exchanges, remoteWgKey) - fail := d.registerFailureLocked(remoteWgKey) - d.mu.Unlock() - d.raiseFailure(remoteWgKey, fail) + switch ex.state { + case stateAwaitingAnswer: + if offerAttempts >= m.maxRetries { + delete(m.exchanges, remoteWgKey) + fail := m.registerFailureLocked(remoteWgKey) + m.mu.Unlock() + m.raiseFailure(remoteWgKey, fail) return } msg := ex.lastSent offerAttempts++ - d.mu.Unlock() - d.retransmit(remoteWgKey, msg) - continue - } + m.mu.Unlock() + m.retransmit(remoteWgKey, msg) - if confirmsSent >= confirmRetransmits { - delete(d.exchanges, remoteWgKey) - d.mu.Unlock() + case stateConfirming: + if confirmsSent >= confirmRetransmits { + delete(m.exchanges, remoteWgKey) + m.mu.Unlock() + return + } + msg := ex.lastSent + confirmsSent++ + m.mu.Unlock() + m.retransmit(remoteWgKey, msg) + + default: + m.mu.Unlock() return } - msg := ex.lastSent - confirmsSent++ - d.mu.Unlock() - d.retransmit(remoteWgKey, msg) } } } @@ -162,38 +161,31 @@ func (d *Driver) initiatorLoop(ctx context.Context, remoteWgKey string, id Excha // registerFailureLocked applies policy B and reports whether OnRekeyFailed is due: // an initial exchange (peer never established) fails immediately; a rekey tolerates // up to maxRekeyFailures consecutive misses (we stay on the still-valid previous -// PSK) before failing. Whether it is initial is read live from established. -// Assumes d.mu is held. -func (d *Driver) registerFailureLocked(remoteWgKey string) bool { - if !d.established[remoteWgKey] { +// PSK) before failing. Assumes m.mu is held. +func (m *Manager) registerFailureLocked(remoteWgKey string) bool { + if !m.established[remoteWgKey] { return true } - d.failures[remoteWgKey]++ - if d.failures[remoteWgKey] >= d.maxRekeyFailures { - d.failures[remoteWgKey] = 0 + m.failures[remoteWgKey]++ + if m.failures[remoteWgKey] >= m.maxRekeyFailures { + m.failures[remoteWgKey] = 0 return true } return false } -// isConfirmPhase reports whether the initiator's outstanding message is the confirm -// (as opposed to the offer), derived from the message's type byte. -func isConfirmPhase(lastSent []byte) bool { - return len(lastSent) > 0 && lastSent[0] == byte(MsgConfirm) -} - -func (d *Driver) raiseFailure(remoteWgKey string, fail bool) { +func (m *Manager) raiseFailure(remoteWgKey string, fail bool) { if !fail { - d.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteWgKey) + m.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteWgKey) return } - if err := d.wg.OnRekeyFailed(remoteWgKey); err != nil { - d.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteWgKey, "err", err) + if err := m.wg.OnRekeyFailed(remoteWgKey); err != nil { + m.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteWgKey, "err", err) } } -func (d *Driver) retransmit(remoteWgKey string, msg []byte) { - if err := d.transport.Send(remoteWgKey, msg); err != nil { - d.logger.Warn("pqkem retransmit failed", "peer", remoteWgKey, "err", err) +func (m *Manager) retransmit(remoteWgKey string, msg []byte) { + if err := m.transport.Send(remoteWgKey, msg); err != nil { + m.logger.Warn("pqkem retransmit failed", "peer", remoteWgKey, "err", err) } } diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index e05331210b1..5bdd4ec2001 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -15,7 +15,7 @@ func (dropTransport) Send(string, []byte) error { return nil } // gate is a loopback transport with a switchable drop flag. type gate struct { local string - peer *Driver + peer *Manager drop atomic.Bool } @@ -27,9 +27,9 @@ func (g *gate) Send(remote string, msg []byte) error { return g.peer.HandleInbound(g.local, cp) } -func TestDriver_InitialTimeoutFailsImmediately(t *testing.T) { +func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { wg := newFakeWG() - d := NewDriver("bbbb", dropTransport{}, wg, time.Hour, nil) // bbbb > aaaa -> initiator + d := NewManager("bbbb", dropTransport{}, wg, time.Hour, nil) // bbbb > aaaa -> initiator d.retryInterval = 5 * time.Millisecond d.maxRetries = 3 d.AddPeer("aaaa") @@ -45,14 +45,14 @@ func TestDriver_InitialTimeoutFailsImmediately(t *testing.T) { }, time.Second, 5*time.Millisecond) } -func TestDriver_RekeyToleratesKFailures(t *testing.T) { +func TestManager_RekeyToleratesKFailures(t *testing.T) { gA := &gate{local: "aaaa"} gB := &gate{local: "bbbb"} wgA := newFakeWG() wgB := newFakeWG() - dA := NewDriver("aaaa", gA, wgA, time.Hour, nil) - dB := NewDriver("bbbb", gB, wgB, time.Hour, nil) + dA := NewManager("aaaa", gA, wgA, time.Hour, nil) + dB := NewManager("bbbb", gB, wgB, time.Hour, nil) gA.peer = dB gB.peer = dA dB.retryInterval = 5 * time.Millisecond diff --git a/client/internal/pqkem/driver.go b/client/internal/pqkem/driver.go deleted file mode 100644 index b4b6c4514a1..00000000000 --- a/client/internal/pqkem/driver.go +++ /dev/null @@ -1,214 +0,0 @@ -package pqkem - -import ( - "context" - "fmt" - "log/slog" - "sync" - "time" -) - -const ( - // DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly - // rotated PSK is naturally adopted by WG's next handshake without forcing one. - DefaultRekeyInterval = 2 * time.Minute - // DefaultRetryInterval is how often the initiator retransmits its outstanding - // message (offer, then confirm) while an exchange is in flight. - DefaultRetryInterval = 2 * time.Second - // DefaultMaxRetries bounds how many times the offer is retransmitted before the - // exchange is declared failed. The convergence deadline is thus derived as - // MaxRetries * RetryInterval — there is no separate deadline timer. - DefaultMaxRetries = 10 - // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures - // are tolerated before OnRekeyFailed. The initial exchange fails immediately. - DefaultMaxRekeyFailures = 3 - // confirmRetransmits is how many times the initiator best-effort resends the - // confirm after converging, to cover its loss without a dedicated goroutine. - confirmRetransmits = 3 -) - -// Transport hands an already-encoded exchange message to the peer. The host routes -// it over the appropriate channel — Signal before the tunnel is up, the WireGuard -// tunnel for rekeys — so the driver never needs to know which is in use. -type Transport interface { - Send(remoteWgKey string, msg []byte) error -} - -// exchangeCtl tracks one in-flight exchange for a peer. A single lastSent holds the -// message currently being (re)transmitted: for the initiator the offer and then, -// once answered, the confirm; for the responder the answer, resent on a duplicate -// offer. Its type byte (lastSent[0]) also encodes the initiator's phase, so no -// separate "answered" flag is kept. Only the initiator runs a retransmit loop -// (cancel != nil); the responder is purely reactive. Whether an exchange is the -// peer's first is read live from d.established, not snapshotted here. -type exchangeCtl struct { - id ExchangeID - startedAt time.Time - lastSent []byte - cancel context.CancelFunc -} - -// Driver ties the pure Manager to the outside world: per-peer rekey timer, inbound -// dispatch, retransmit/convergence tracking, and surfacing events to the host via -// WGCallbackHandler. -type Driver struct { - mgr *Manager - transport Transport - wg WGCallbackHandler - logger *slog.Logger - - rekeyInterval time.Duration - retryInterval time.Duration - maxRetries int - maxRekeyFailures int - - rootCtx context.Context - rootCancel context.CancelFunc - - mu sync.Mutex - peers map[string]context.CancelFunc // per-peer rekey loop - exchanges map[string]*exchangeCtl // in-flight exchange per peer - established map[string]bool // peer has completed at least one exchange - failures map[string]int // consecutive rekey failures per peer - wait sync.WaitGroup -} - -// NewDriver builds a driver for the local peer. A zero interval falls back to -// DefaultRekeyInterval; a nil logger falls back to slog.Default(). Retry/retries/K -// use their defaults and can be overridden on the returned struct before use. -func NewDriver(localWgKey string, t Transport, h WGCallbackHandler, interval time.Duration, logger *slog.Logger) *Driver { - if interval <= 0 { - interval = DefaultRekeyInterval - } - if logger == nil { - logger = slog.Default() - } - ctx, cancel := context.WithCancel(context.Background()) - return &Driver{ - mgr: NewManager(localWgKey), - transport: t, - wg: h, - logger: logger, - rekeyInterval: interval, - retryInterval: DefaultRetryInterval, - maxRetries: DefaultMaxRetries, - maxRekeyFailures: DefaultMaxRekeyFailures, - rootCtx: ctx, - rootCancel: cancel, - peers: make(map[string]context.CancelFunc), - exchanges: make(map[string]*exchangeCtl), - established: make(map[string]bool), - failures: make(map[string]int), - } -} - -// AddPeer registers a remote peer and starts its rekey timer. Re-adding is a no-op. -func (d *Driver) AddPeer(remoteWgKey string) { - d.mu.Lock() - defer d.mu.Unlock() - if _, ok := d.peers[remoteWgKey]; ok { - return - } - ctx, cancel := context.WithCancel(d.rootCtx) - d.peers[remoteWgKey] = cancel - d.wait.Add(1) - go d.rekeyLoop(ctx, remoteWgKey) -} - -// RemovePeer stops a peer's rekey timer and any in-flight exchange, and drops state. -func (d *Driver) RemovePeer(remoteWgKey string) { - d.mu.Lock() - if cancel, ok := d.peers[remoteWgKey]; ok { - cancel() - delete(d.peers, remoteWgKey) - } - if ex, ok := d.exchanges[remoteWgKey]; ok { - if ex.cancel != nil { - ex.cancel() - } - delete(d.exchanges, remoteWgKey) - } - delete(d.established, remoteWgKey) - delete(d.failures, remoteWgKey) - d.mu.Unlock() -} - -// Stop cancels all timers and in-flight exchanges and waits for goroutines to exit. -func (d *Driver) Stop() { - d.rootCancel() - d.wait.Wait() - d.mu.Lock() - d.peers = make(map[string]context.CancelFunc) - d.exchanges = make(map[string]*exchangeCtl) - d.mu.Unlock() -} - -// HandleInbound decodes an incoming message and drives the exchange, sending any -// response via the transport and surfacing derived PSKs / convergence to the host. -func (d *Driver) HandleInbound(remoteWgKey string, raw []byte) error { - typ, msg, err := Decode(raw) - if err != nil { - return fmt.Errorf("decode from %s: %w", remoteWgKey, err) - } - switch typ { - case MsgOffer: - return d.handleOffer(remoteWgKey, msg.(*OfferMsg)) - case MsgAnswer: - return d.handleAnswer(remoteWgKey, msg.(*AnswerMsg)) - case MsgConfirm: - return d.handleConfirm(remoteWgKey, msg.(*ConfirmMsg)) - default: - return fmt.Errorf("unhandled message type %d from %s", typ, remoteWgKey) - } -} - -// initiateRekey starts a fresh exchange when the local peer is the initiator for -// this remote peer; the responder waits for the offer instead. Exposed (unexported -// but directly callable) so tests can drive a rekey without waiting on the ticker. -func (d *Driver) initiateRekey(remoteWgKey string) error { - if !d.mgr.IsInitiator(remoteWgKey) { - return nil - } - offer, err := d.mgr.StartExchange(remoteWgKey) - if err != nil { - return err - } - raw, err := offer.Encode() - if err != nil { - return err - } - - ctx, cancel := context.WithCancel(d.rootCtx) - d.mu.Lock() - if old := d.exchanges[remoteWgKey]; old != nil && old.cancel != nil { - old.cancel() - } - d.exchanges[remoteWgKey] = &exchangeCtl{ - id: offer.ExchangeID, - startedAt: time.Now(), - lastSent: raw, - cancel: cancel, - } - d.mu.Unlock() - - d.wait.Add(1) - go d.initiatorLoop(ctx, remoteWgKey, offer.ExchangeID) - - return d.transport.Send(remoteWgKey, raw) -} - -func (d *Driver) rekeyLoop(ctx context.Context, remoteWgKey string) { - defer d.wait.Done() - t := time.NewTicker(d.rekeyInterval) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case <-t.C: - if err := d.initiateRekey(remoteWgKey); err != nil { - d.logger.Error("pqkem rekey failed to start", "peer", remoteWgKey, "err", err) - } - } - } -} diff --git a/client/internal/pqkem/driver_test.go b/client/internal/pqkem/driver_test.go deleted file mode 100644 index 59fb381da42..00000000000 --- a/client/internal/pqkem/driver_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package pqkem - -import ( - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -// loopback delivers a sent message synchronously to the peer driver's HandleInbound, -// attributing it to localKey (the sender). -type loopback struct { - localKey string - peer *Driver -} - -func (l *loopback) Send(remoteWgKey string, msg []byte) error { - cp := append([]byte(nil), msg...) - return l.peer.HandleInbound(l.localKey, cp) -} - -type fakeWG struct { - mu sync.Mutex - psks map[string]PSK - failed []string -} - -func newFakeWG() *fakeWG { return &fakeWG{psks: map[string]PSK{}} } - -func (f *fakeWG) OnNewPSKReady(remoteWgKey string, psk PSK) error { - f.mu.Lock() - defer f.mu.Unlock() - f.psks[remoteWgKey] = psk - return nil -} - -func (f *fakeWG) OnRekeyFailed(remoteWgKey string) error { - f.mu.Lock() - defer f.mu.Unlock() - f.failed = append(f.failed, remoteWgKey) - return nil -} - -func (f *fakeWG) psk(peer string) PSK { - f.mu.Lock() - defer f.mu.Unlock() - return f.psks[peer] -} - -func TestDriver_ExchangeConverges(t *testing.T) { - lbA := &loopback{localKey: "aaaa"} - lbB := &loopback{localKey: "bbbb"} - wgA := newFakeWG() - wgB := newFakeWG() - - // long interval so the ticker never fires during the test; we drive manually. - dA := NewDriver("aaaa", lbA, wgA, time.Hour, nil) - dB := NewDriver("bbbb", lbB, wgB, time.Hour, nil) - lbA.peer = dB // A sends -> B receives - lbB.peer = dA // B sends -> A receives - - dA.AddPeer("bbbb") - dB.AddPeer("aaaa") - defer dA.Stop() - defer dB.Stop() - - // B is the initiator ("bbbb" > "aaaa"). - require.NoError(t, dB.initiateRekey("aaaa")) - - pskB := wgB.psk("aaaa") // B committed on the answer - pskA := wgA.psk("bbbb") // A committed on the confirm - require.NotEqual(t, PSK{}, pskA, "responder A must have a PSK") - require.NotEqual(t, PSK{}, pskB, "initiator B must have a PSK") - require.Equal(t, pskB, pskA, "both sides converge on the same PSK") -} - -func TestDriver_NonInitiatorDoesNothing(t *testing.T) { - lbA := &loopback{localKey: "aaaa"} - wgA := newFakeWG() - dA := NewDriver("aaaa", lbA, wgA, time.Hour, nil) - // no peer driver wired; if A wrongly initiated, Send would nil-panic. - dA.AddPeer("bbbb") - defer dA.Stop() - - // A is NOT the initiator vs "bbbb" -> initiateRekey is a no-op, no Send. - require.NoError(t, dA.initiateRekey("bbbb")) - require.Equal(t, PSK{}, wgA.psk("bbbb")) -} - -func TestDriver_StopIsIdempotent(t *testing.T) { - dA := NewDriver("aaaa", &loopback{localKey: "aaaa"}, newFakeWG(), time.Hour, nil) - dA.AddPeer("bbbb") - dA.Stop() - dA.Stop() // must not panic or hang -} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index cc746c42311..cd48866f033 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -1,54 +1,123 @@ package pqkem import ( + "context" "crypto/rand" "fmt" + "log/slog" "sync" + "time" ) -// Manager drives the PQ-KEM exchange per remote peer. It is a pure state machine: -// it consumes and produces messages and returns the derived PSK at the point each -// side must commit it (initiator on the answer, responder on the confirm). It does -// no I/O — transport (Signal / tunnel) and the WireGuard SetPSK live in the caller -// (network module + wiring), so this stays extraction-ready as a standalone library. -// -// Commit ordering (see ConfirmMsg): the responder derives the PSK on the offer but -// must NOT commit it until the confirm arrives, otherwise a lost answer would leave -// it on a new PSK the initiator never got, breaking the tunnel at the next rekey. +const ( + // DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly + // rotated PSK is naturally adopted by WG's next handshake without forcing one. + DefaultRekeyInterval = 2 * time.Minute + // DefaultRetryInterval is how often the initiator retransmits its outstanding + // message (offer, then confirm) while an exchange is in flight. + DefaultRetryInterval = 2 * time.Second + // DefaultMaxRetries bounds how many times the offer is retransmitted before the + // exchange is declared failed. The convergence deadline is thus derived as + // MaxRetries * RetryInterval — there is no separate deadline timer. + DefaultMaxRetries = 10 + // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures + // are tolerated before OnRekeyFailed. The initial exchange fails immediately. + DefaultMaxRekeyFailures = 3 + // confirmRetransmits is how many times the initiator best-effort resends the + // confirm after converging, to cover its loss without a dedicated goroutine. + confirmRetransmits = 3 +) + +// Transport hands an already-encoded exchange message to the peer. The host routes +// it over the appropriate channel — Signal before the tunnel is up, the WireGuard +// tunnel for rekeys — so the Manager never needs to know which is in use. It is the +// analogue of go-rosenpass's Conn seam. +type Transport interface { + Send(remoteWgKey string, msg []byte) error +} -type sessionState uint8 +// exchangeState is the single source of truth for an exchange's role and phase. +// Every handler and the retransmit loop key off it, so no role/phase is re-derived +// from other fields. +type exchangeState uint8 const ( - stateIdle sessionState = iota - stateAwaitingAnswer // initiator: sent offer, waiting for answer - stateAwaitingConfirm // responder: sent answer, holding pending PSK until confirm - stateEstablished + stateReserved exchangeState = iota // responder: deriving the answer + stateAwaitingAnswer // initiator: offer sent, awaiting the answer + stateAwaitingConfirm // responder: answer sent, awaiting the confirm + stateConfirming // initiator: answer in, PSK committed, flushing the confirm ) -type peerSession struct { - state sessionState - exchangeID ExchangeID - - // initiator side: in-flight ephemeral keypair between offer and answer. - initiator *Initiator - // responder side: PSK derived on the offer, committed only on the confirm. +// exchangeCtl holds all state for one in-flight exchange with a peer, under the +// Manager's single lock. state drives every decision. lastSent is the message +// currently being (re)transmitted. The crypto payloads live here too: initiator is +// the ephemeral handle used at Finish (initiator side); pendingPSK is the derived +// key held until the confirm commits it (responder side). Only the initiator runs a +// retransmit loop, so only it sets cancel. +type exchangeCtl struct { + id ExchangeID + state exchangeState + startedAt time.Time + cancel context.CancelFunc + lastSent []byte + initiator *Initiator pendingPSK PSK } +// Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It +// runs the per-peer rekey timer, drives the X25519MLKEM768 exchange over a pluggable +// Transport, and surfaces the derived PSK to the host via WGCallbackHandler. The +// cryptography is the pure kem.go primitives; all per-exchange and per-peer state +// lives here under one lock. type Manager struct { localWgKey string + transport Transport + wg WGCallbackHandler + logger *slog.Logger + + rekeyInterval time.Duration + retryInterval time.Duration + maxRetries int + maxRekeyFailures int - mu sync.Mutex - sessions map[string]*peerSession + rootCtx context.Context + rootCancel context.CancelFunc + + mu sync.Mutex + peers map[string]context.CancelFunc // per-peer rekey loop + exchanges map[string]*exchangeCtl // in-flight exchange per peer + established map[string]bool // peer has completed at least one exchange + failures map[string]int // consecutive rekey failures per peer + wait sync.WaitGroup } -// NewManager creates a KEM manager for the local peer identified by its WireGuard -// public key (base64), used both for the deterministic initiator role and the -// identity binding of the derived PSK. -func NewManager(localWgKey string) *Manager { +// NewManager builds a manager for the local peer identified by its WireGuard public +// key (used for the deterministic initiator role and the identity binding). A zero +// interval falls back to DefaultRekeyInterval; a nil logger to slog.Default(). +// Retry/retries/K use their defaults and can be overridden before use. +func NewManager(localWgKey string, t Transport, h WGCallbackHandler, interval time.Duration, logger *slog.Logger) *Manager { + if interval <= 0 { + interval = DefaultRekeyInterval + } + if logger == nil { + logger = slog.Default() + } + ctx, cancel := context.WithCancel(context.Background()) return &Manager{ - localWgKey: localWgKey, - sessions: make(map[string]*peerSession), + localWgKey: localWgKey, + transport: t, + wg: h, + logger: logger, + rekeyInterval: interval, + retryInterval: DefaultRetryInterval, + maxRetries: DefaultMaxRetries, + maxRekeyFailures: DefaultMaxRekeyFailures, + rootCtx: ctx, + rootCancel: cancel, + peers: make(map[string]context.CancelFunc), + exchanges: make(map[string]*exchangeCtl), + established: make(map[string]bool), + failures: make(map[string]int), } } @@ -59,99 +128,121 @@ func (m *Manager) IsInitiator(remoteWgKey string) bool { return m.localWgKey > remoteWgKey } -// StartExchange begins (or restarts, on rotation) an exchange as the initiator and -// returns the offer to send. Any previous in-flight exchange for the peer is dropped. -func (m *Manager) StartExchange(remoteWgKey string) (*OfferMsg, error) { - init, err := NewInitiator() - if err != nil { - return nil, err - } - id, err := newExchangeID() - if err != nil { - return nil, err - } - +// AddPeer registers a remote peer and starts its rekey timer. Re-adding is a no-op. +func (m *Manager) AddPeer(remoteWgKey string) { m.mu.Lock() - m.sessions[remoteWgKey] = &peerSession{ - state: stateAwaitingAnswer, - exchangeID: id, - initiator: init, + defer m.mu.Unlock() + if _, ok := m.peers[remoteWgKey]; ok { + return } - m.mu.Unlock() - - return &OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}, nil + ctx, cancel := context.WithCancel(m.rootCtx) + m.peers[remoteWgKey] = cancel + m.wait.Add(1) + go m.rekeyLoop(ctx, remoteWgKey) } -// HandleOffer processes a received offer as the responder, derives the PSK, and -// returns the answer to send. The PSK is held pending and NOT returned: the caller -// must not program it until HandleConfirm. Retransmitted offers are deduplicated by -// the driver (via exchangeID) so this is called once per exchange and never -// re-derives a different key for the same round. -func (m *Manager) HandleOffer(remoteWgKey string, o *OfferMsg) (*AnswerMsg, error) { - answer, psk, err := Respond(o.KEMOffer, m.binding(remoteWgKey)) - if err != nil { - return nil, err - } - +// RemovePeer stops a peer's rekey timer and any in-flight exchange, and drops state. +func (m *Manager) RemovePeer(remoteWgKey string) { m.mu.Lock() - m.sessions[remoteWgKey] = &peerSession{ - state: stateAwaitingConfirm, - exchangeID: o.ExchangeID, - pendingPSK: psk, + if cancel, ok := m.peers[remoteWgKey]; ok { + cancel() + delete(m.peers, remoteWgKey) } + if ex, ok := m.exchanges[remoteWgKey]; ok { + if ex.cancel != nil { + ex.cancel() + } + delete(m.exchanges, remoteWgKey) + } + delete(m.established, remoteWgKey) + delete(m.failures, remoteWgKey) m.mu.Unlock() - - return &AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answer}, nil } -// HandleAnswer processes a received answer as the initiator. On success it returns -// the derived PSK (which the caller commits now) and the confirm to send. -func (m *Manager) HandleAnswer(remoteWgKey string, a *AnswerMsg) (PSK, *ConfirmMsg, error) { +// Stop cancels all timers and in-flight exchanges and waits for goroutines to exit. +func (m *Manager) Stop() { + m.rootCancel() + m.wait.Wait() m.mu.Lock() - s, ok := m.sessions[remoteWgKey] - if !ok || s.state != stateAwaitingAnswer { - m.mu.Unlock() - return PSK{}, nil, fmt.Errorf("no pending offer for peer %s", remoteWgKey) - } - if s.exchangeID != a.ExchangeID { - // stale answer (e.g. to a pre-restart offer) — drop, keep waiting. - m.mu.Unlock() - return PSK{}, nil, fmt.Errorf("answer exchangeID mismatch for peer %s", remoteWgKey) - } - // Claim the exchange under the lock so a concurrent answer bails, and copy out - // the initiator so Finish (expensive crypto) runs without holding the lock. - init := s.initiator - s.state = stateEstablished - s.initiator = nil + m.peers = make(map[string]context.CancelFunc) + m.exchanges = make(map[string]*exchangeCtl) m.mu.Unlock() +} - psk, err := init.Finish(a.KEMAnswer, m.binding(remoteWgKey)) +// HandleInbound decodes an incoming message and drives the exchange, sending any +// response via the transport and surfacing derived PSKs / convergence to the host. +func (m *Manager) HandleInbound(remoteWgKey string, raw []byte) error { + typ, msg, err := Decode(raw) if err != nil { - return PSK{}, nil, err + return fmt.Errorf("decode from %s: %w", remoteWgKey, err) + } + switch typ { + case MsgOffer: + return m.handleOffer(remoteWgKey, msg.(*OfferMsg)) + case MsgAnswer: + return m.handleAnswer(remoteWgKey, msg.(*AnswerMsg)) + case MsgConfirm: + return m.handleConfirm(remoteWgKey, msg.(*ConfirmMsg)) + default: + return fmt.Errorf("unhandled message type %d from %s", typ, remoteWgKey) } - - return psk, &ConfirmMsg{ExchangeID: a.ExchangeID}, nil } -// HandleConfirm processes a received confirm as the responder and returns the PSK -// to commit now. Retransmitted confirms are deduplicated by the driver, so this is -// called once. It errors on a stale/unknown confirm so the caller ignores it. -func (m *Manager) HandleConfirm(remoteWgKey string, c *ConfirmMsg) (PSK, error) { - m.mu.Lock() - defer m.mu.Unlock() +// initiateRekey starts a fresh exchange when the local peer is the initiator for +// this remote peer; the responder waits for the offer instead. Exposed (unexported +// but directly callable) so tests can drive a rekey without waiting on the ticker. +func (m *Manager) initiateRekey(remoteWgKey string) error { + if !m.IsInitiator(remoteWgKey) { + return nil + } + init, err := NewInitiator() + if err != nil { + return err + } + id, err := newExchangeID() + if err != nil { + return err + } + raw, err := (&OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}).Encode() + if err != nil { + return err + } - s, ok := m.sessions[remoteWgKey] - if !ok || s.state != stateAwaitingConfirm { - return PSK{}, fmt.Errorf("no pending answer for peer %s", remoteWgKey) + ctx, cancel := context.WithCancel(m.rootCtx) + m.mu.Lock() + if old := m.exchanges[remoteWgKey]; old != nil && old.cancel != nil { + old.cancel() } - if s.exchangeID != c.ExchangeID { - return PSK{}, fmt.Errorf("confirm exchangeID mismatch for peer %s", remoteWgKey) + m.exchanges[remoteWgKey] = &exchangeCtl{ + id: id, + state: stateAwaitingAnswer, + startedAt: time.Now(), + cancel: cancel, + lastSent: raw, + initiator: init, } + m.mu.Unlock() + + m.wait.Add(1) + go m.initiatorLoop(ctx, remoteWgKey, id) - s.state = stateEstablished - psk := s.pendingPSK - s.pendingPSK = PSK{} - return psk, nil + return m.transport.Send(remoteWgKey, raw) +} + +func (m *Manager) rekeyLoop(ctx context.Context, remoteWgKey string) { + defer m.wait.Done() + t := time.NewTicker(m.rekeyInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := m.initiateRekey(remoteWgKey); err != nil { + m.logger.Error("pqkem rekey failed to start", "peer", remoteWgKey, "err", err) + } + } + } } func (m *Manager) binding(remoteWgKey string) Binding { diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index 3bdb0f413cc..e355b17faec 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -1,75 +1,96 @@ package pqkem import ( + "sync" "testing" + "time" "github.com/stretchr/testify/require" ) -func TestManager_FullExchange(t *testing.T) { - // deterministic roles: "bbbb" > "aaaa" -> B is initiator - a := NewManager("aaaa") - b := NewManager("bbbb") - require.True(t, b.IsInitiator("aaaa")) - require.False(t, a.IsInitiator("bbbb")) - - // B (initiator) -> A - offer, err := b.StartExchange("aaaa") - require.NoError(t, err) +// loopback delivers a sent message synchronously to the peer driver's HandleInbound, +// attributing it to localKey (the sender). +type loopback struct { + localKey string + peer *Manager +} - // A (responder) derives PSK but holds it pending - answer, err := a.HandleOffer("bbbb", offer) - require.NoError(t, err) +func (l *loopback) Send(remoteWgKey string, msg []byte) error { + cp := append([]byte(nil), msg...) + return l.peer.HandleInbound(l.localKey, cp) +} - // B derives PSK on the answer, commits now, produces confirm - pskB, confirm, err := b.HandleAnswer("aaaa", answer) - require.NoError(t, err) +type fakeWG struct { + mu sync.Mutex + psks map[string]PSK + failed []string +} - // A commits only on the confirm - pskA, err := a.HandleConfirm("bbbb", confirm) - require.NoError(t, err) +func newFakeWG() *fakeWG { return &fakeWG{psks: map[string]PSK{}} } - require.Equal(t, pskB, pskA, "both sides converge on the same PSK") - require.NotEqual(t, PSK{}, pskA) +func (f *fakeWG) OnNewPSKReady(remoteWgKey string, psk PSK) error { + f.mu.Lock() + defer f.mu.Unlock() + f.psks[remoteWgKey] = psk + return nil } -func TestManager_StaleAnswerDropped(t *testing.T) { - b := NewManager("bbbb") +func (f *fakeWG) OnRekeyFailed(remoteWgKey string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.failed = append(f.failed, remoteWgKey) + return nil +} - _, err := b.StartExchange("aaaa") - require.NoError(t, err) +func (f *fakeWG) psk(peer string) PSK { + f.mu.Lock() + defer f.mu.Unlock() + return f.psks[peer] +} - // craft an answer with a wrong exchangeID - stale := &AnswerMsg{ExchangeID: ExchangeID{0xFF}, KEMAnswer: make([]byte, AnswerSize)} - _, _, err = b.HandleAnswer("aaaa", stale) - require.Error(t, err) +func TestManager_ExchangeConverges(t *testing.T) { + lbA := &loopback{localKey: "aaaa"} + lbB := &loopback{localKey: "bbbb"} + wgA := newFakeWG() + wgB := newFakeWG() + + // long interval so the ticker never fires during the test; we drive manually. + dA := NewManager("aaaa", lbA, wgA, time.Hour, nil) + dB := NewManager("bbbb", lbB, wgB, time.Hour, nil) + lbA.peer = dB // A sends -> B receives + lbB.peer = dA // B sends -> A receives + + dA.AddPeer("bbbb") + dB.AddPeer("aaaa") + defer dA.Stop() + defer dB.Stop() + + // B is the initiator ("bbbb" > "aaaa"). + require.NoError(t, dB.initiateRekey("aaaa")) + + pskB := wgB.psk("aaaa") // B committed on the answer + pskA := wgA.psk("bbbb") // A committed on the confirm + require.NotEqual(t, PSK{}, pskA, "responder A must have a PSK") + require.NotEqual(t, PSK{}, pskB, "initiator B must have a PSK") + require.Equal(t, pskB, pskA, "both sides converge on the same PSK") } -func TestManager_RestartResyncsViaNewExchange(t *testing.T) { - // A restarts (fresh manager) mid-flight; a new exchange from scratch converges. - a := NewManager("aaaa") - b := NewManager("bbbb") - - off1, err := b.StartExchange("aaaa") - require.NoError(t, err) - - // B "restarts": new manager, old in-flight state gone - b = NewManager("bbbb") - off2, err := b.StartExchange("aaaa") - require.NoError(t, err) - require.NotEqual(t, off1.ExchangeID, off2.ExchangeID) - - ans, err := a.HandleOffer("bbbb", off2) - require.NoError(t, err) - pskB, conf, err := b.HandleAnswer("aaaa", ans) - require.NoError(t, err) - pskA, err := a.HandleConfirm("bbbb", conf) - require.NoError(t, err) - require.Equal(t, pskB, pskA) +func TestManager_NonInitiatorDoesNothing(t *testing.T) { + lbA := &loopback{localKey: "aaaa"} + wgA := newFakeWG() + dA := NewManager("aaaa", lbA, wgA, time.Hour, nil) + // no peer driver wired; if A wrongly initiated, Send would nil-panic. + dA.AddPeer("bbbb") + defer dA.Stop() + + // A is NOT the initiator vs "bbbb" -> initiateRekey is a no-op, no Send. + require.NoError(t, dA.initiateRekey("bbbb")) + require.Equal(t, PSK{}, wgA.psk("bbbb")) } -func TestManager_UnknownPeerErrors(t *testing.T) { - a := NewManager("aaaa") - _, err := a.HandleConfirm("zzzz", &ConfirmMsg{}) - require.Error(t, err) +func TestManager_StopIsIdempotent(t *testing.T) { + dA := NewManager("aaaa", &loopback{localKey: "aaaa"}, newFakeWG(), time.Hour, nil) + dA.AddPeer("bbbb") + dA.Stop() + dA.Stop() // must not panic or hang } From f8bb816dea10e75113cf9c7e3505c9f4e730e4c9 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 23 Jul 2026 11:03:40 +0200 Subject: [PATCH 013/119] Epurate wg refs --- client/internal/pqkem/callbacks.go | 25 ++++--- client/internal/pqkem/convergence.go | 82 +++++++++++------------ client/internal/pqkem/kem.go | 14 ++-- client/internal/pqkem/kem_test.go | 12 ++-- client/internal/pqkem/manager.go | 95 ++++++++++++++------------- client/internal/pqkem/manager_test.go | 10 +-- client/internal/pqkem/message.go | 8 +-- client/internal/pqkem/message_test.go | 2 +- 8 files changed, 124 insertions(+), 124 deletions(-) diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index c1578f64a5e..9dbb8e4bb5e 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -1,19 +1,18 @@ package pqkem -// WGCallbackHandler is implemented by the host (the NetBird wiring layer) and -// invoked by the library. The library only reports events; the host owns the -// WireGuard reaction. Keeping this an interface — rather than calling wgctrl -// directly — is what lets the KEM code be extracted as a standalone library. -type WGCallbackHandler interface { - // OnNewPSKReady fires when a fresh post-quantum PSK has been derived for a - // peer and must be programmed on the WireGuard interface. It is invoked at the - // commit point of each side: the initiator on receiving the answer, the +// CallbackHandler is implemented by the host and invoked by the library. The +// library only reports events; the host owns the reaction. Keeping this an +// interface — rather than touching the transport or keying directly — is what lets +// the KEM code be extracted as a standalone library. +type CallbackHandler interface { + // OnNewPSKReady fires when a fresh post-quantum PSK has been derived for a peer + // and must be programmed into the consumer's secure channel. It is invoked at + // the commit point of each side: the initiator on receiving the answer, the // responder on receiving the confirm. - OnNewPSKReady(remoteWgKey string, psk PSK) error + OnNewPSKReady(remoteID string, psk PSK) error // OnRekeyFailed fires when an exchange fails to converge within the allotted - // time. The host should tear the peer connection down so ICE re-establishes, - // and log a WARN. The library reports the event; it does not dictate the - // reaction. - OnRekeyFailed(remoteWgKey string) error + // time. The host should tear the peer connection down so it re-establishes, and + // log a WARN. The library reports the event; it does not dictate the reaction. + OnRekeyFailed(remoteID string) error } diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 63c0ba1365f..16d6bec5905 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -8,9 +8,9 @@ import ( // handleOffer (responder) derives and sends the answer for a new exchange, or // resends the cached answer for a duplicate offer (same exchangeID) without // re-deriving. The responder is purely reactive: no retransmit loop, no deadline. -func (m *Manager) handleOffer(remoteWgKey string, o *OfferMsg) error { +func (m *Manager) handleOffer(remoteID string, o *OfferMsg) error { m.mu.Lock() - if ex := m.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { state, last := ex.state, ex.lastSent m.mu.Unlock() if state == stateReserved { @@ -18,13 +18,13 @@ func (m *Manager) handleOffer(remoteWgKey string, o *OfferMsg) error { // dropping avoids a second (randomized -> divergent) derivation. return nil } - return m.transport.Send(remoteWgKey, last) + return m.transport.Send(remoteID, last) } // Reserve the slot so a concurrent duplicate offer bails. - m.exchanges[remoteWgKey] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, startedAt: time.Now()} + m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, startedAt: time.Now()} m.mu.Unlock() - answerBytes, psk, err := Respond(o.KEMOffer, m.binding(remoteWgKey)) + answerBytes, psk, err := Respond(o.KEMOffer, m.binding(remoteID)) if err != nil { return err } @@ -34,22 +34,22 @@ func (m *Manager) handleOffer(remoteWgKey string, o *OfferMsg) error { } m.mu.Lock() - if ex := m.exchanges[remoteWgKey]; ex != nil && ex.id == o.ExchangeID { + if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { ex.state = stateAwaitingConfirm ex.lastSent = raw ex.pendingPSK = psk } m.mu.Unlock() - return m.transport.Send(remoteWgKey, raw) + return m.transport.Send(remoteID, raw) } // handleAnswer (initiator) derives the PSK, surfaces it, and sends the confirm, then // switches the retransmit payload to the confirm. Only valid in stateAwaitingAnswer; // advancing the state under the lock makes a concurrent/duplicate answer bail. -func (m *Manager) handleAnswer(remoteWgKey string, a *AnswerMsg) error { +func (m *Manager) handleAnswer(remoteID string, a *AnswerMsg) error { m.mu.Lock() - ex := m.exchanges[remoteWgKey] + ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { m.mu.Unlock() return nil @@ -59,11 +59,11 @@ func (m *Manager) handleAnswer(remoteWgKey string, a *AnswerMsg) error { ex.initiator = nil m.mu.Unlock() - psk, err := init.Finish(a.KEMAnswer, m.binding(remoteWgKey)) + psk, err := init.Finish(a.KEMAnswer, m.binding(remoteID)) if err != nil { return err } - if err := m.wg.OnNewPSKReady(remoteWgKey, psk); err != nil { + if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { return err } raw, err := (&ConfirmMsg{ExchangeID: a.ExchangeID}).Encode() @@ -72,42 +72,42 @@ func (m *Manager) handleAnswer(remoteWgKey string, a *AnswerMsg) error { } m.mu.Lock() - if ex := m.exchanges[remoteWgKey]; ex != nil && ex.id == a.ExchangeID { + if ex := m.exchanges[remoteID]; ex != nil && ex.id == a.ExchangeID { ex.lastSent = raw // loop now retransmits the confirm - m.established[remoteWgKey] = true - m.failures[remoteWgKey] = 0 + m.established[remoteID] = true + m.failures[remoteID] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) } m.mu.Unlock() - return m.transport.Send(remoteWgKey, raw) + return m.transport.Send(remoteID, raw) } // handleConfirm (responder) commits the pending PSK. Only valid in // stateAwaitingConfirm; duplicate confirms (the initiator best-effort resends it) // find the exchange gone and are ignored. -func (m *Manager) handleConfirm(remoteWgKey string, c *ConfirmMsg) error { +func (m *Manager) handleConfirm(remoteID string, c *ConfirmMsg) error { m.mu.Lock() - ex := m.exchanges[remoteWgKey] + ex := m.exchanges[remoteID] if ex == nil || ex.id != c.ExchangeID || ex.state != stateAwaitingConfirm { m.mu.Unlock() return nil } psk := ex.pendingPSK - delete(m.exchanges, remoteWgKey) - m.established[remoteWgKey] = true - m.failures[remoteWgKey] = 0 + delete(m.exchanges, remoteID) + m.established[remoteID] = true + m.failures[remoteID] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) m.mu.Unlock() - return m.wg.OnNewPSKReady(remoteWgKey, psk) + return m.cbHandler.OnNewPSKReady(remoteID, psk) } // initiatorLoop retransmits the initiator's outstanding message, keyed off the // exchange state: the offer while awaiting the answer (bounded by maxRetries -> // failure), then the confirm a few best-effort times before stopping. The // convergence deadline is thus derived from maxRetries * retryInterval. -func (m *Manager) initiatorLoop(ctx context.Context, remoteWgKey string, id ExchangeID) { +func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id ExchangeID) { defer m.wait.Done() t := time.NewTicker(m.retryInterval) defer t.Stop() @@ -119,7 +119,7 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteWgKey string, id Exch return case <-t.C: m.mu.Lock() - ex := m.exchanges[remoteWgKey] + ex := m.exchanges[remoteID] if ex == nil || ex.id != id { m.mu.Unlock() return @@ -128,27 +128,27 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteWgKey string, id Exch switch ex.state { case stateAwaitingAnswer: if offerAttempts >= m.maxRetries { - delete(m.exchanges, remoteWgKey) - fail := m.registerFailureLocked(remoteWgKey) + delete(m.exchanges, remoteID) + fail := m.registerFailureLocked(remoteID) m.mu.Unlock() - m.raiseFailure(remoteWgKey, fail) + m.raiseFailure(remoteID, fail) return } msg := ex.lastSent offerAttempts++ m.mu.Unlock() - m.retransmit(remoteWgKey, msg) + m.retransmit(remoteID, msg) case stateConfirming: if confirmsSent >= confirmRetransmits { - delete(m.exchanges, remoteWgKey) + delete(m.exchanges, remoteID) m.mu.Unlock() return } msg := ex.lastSent confirmsSent++ m.mu.Unlock() - m.retransmit(remoteWgKey, msg) + m.retransmit(remoteID, msg) default: m.mu.Unlock() @@ -162,30 +162,30 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteWgKey string, id Exch // an initial exchange (peer never established) fails immediately; a rekey tolerates // up to maxRekeyFailures consecutive misses (we stay on the still-valid previous // PSK) before failing. Assumes m.mu is held. -func (m *Manager) registerFailureLocked(remoteWgKey string) bool { - if !m.established[remoteWgKey] { +func (m *Manager) registerFailureLocked(remoteID string) bool { + if !m.established[remoteID] { return true } - m.failures[remoteWgKey]++ - if m.failures[remoteWgKey] >= m.maxRekeyFailures { - m.failures[remoteWgKey] = 0 + m.failures[remoteID]++ + if m.failures[remoteID] >= m.maxRekeyFailures { + m.failures[remoteID] = 0 return true } return false } -func (m *Manager) raiseFailure(remoteWgKey string, fail bool) { +func (m *Manager) raiseFailure(remoteID string, fail bool) { if !fail { - m.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteWgKey) + m.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteID) return } - if err := m.wg.OnRekeyFailed(remoteWgKey); err != nil { - m.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteWgKey, "err", err) + if err := m.cbHandler.OnRekeyFailed(remoteID); err != nil { + m.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteID, "err", err) } } -func (m *Manager) retransmit(remoteWgKey string, msg []byte) { - if err := m.transport.Send(remoteWgKey, msg); err != nil { - m.logger.Warn("pqkem retransmit failed", "peer", remoteWgKey, "err", err) +func (m *Manager) retransmit(remoteID string, msg []byte) { + if err := m.transport.Send(remoteID, msg); err != nil { + m.logger.Warn("pqkem retransmit failed", "peer", remoteID, "err", err) } } diff --git a/client/internal/pqkem/kem.go b/client/internal/pqkem/kem.go index c2961ecedac..e7e855cd1fb 100644 --- a/client/internal/pqkem/kem.go +++ b/client/internal/pqkem/kem.go @@ -1,6 +1,6 @@ // Package pqkem is a spike (NET-1406) for a post-quantum pre-shared-key exchange // that could replace Rosenpass. It performs an X25519MLKEM768 hybrid key -// encapsulation and derives a 32-byte WireGuard PSK. +// encapsulation and derives a 32-byte pre-shared key (PSK). // // The exchange is a single round trip designed to ride the (already // authenticated) Signal offer/answer channel: @@ -9,7 +9,7 @@ // initiator <--Answer(1120B)-- responder // // Both sides then hold the same PSK, which is bound to the two peers' identities -// (their WireGuard static public keys) so the derived key cannot be transplanted +// (their peer identity keys) so the derived key cannot be transplanted // to a different peer pair even if the transport authentication were bypassed. // // Combiner note: this follows the IETF hybrid layout (X25519 ‖ ML-KEM on the @@ -35,14 +35,14 @@ const ( pskLabel = "netbird-pq-psk-v1" ) -// PSK is the 32-byte pre-shared key handed to WireGuard. +// PSK is the 32-byte derived pre-shared key handed to the consumer to key its channel. type PSK [32]byte // Binding identifies the peer pair the PSK is derived for. Callers set both -// WireGuard static public keys; the order does not matter (it is canonicalised). +// peer identity keys; the order does not matter (it is canonicalised). type Binding struct { - LocalWgPub []byte - RemoteWgPub []byte + LocalID []byte + RemoteID []byte } // Initiator holds the ephemeral secrets between Offer and Finish. @@ -141,7 +141,7 @@ func Respond(offer []byte, b Binding) (answer []byte, psk PSK, err error) { // TODO(NET-1406): replace the SHA-256 concat with the RFC HKDF combiner // (crypto/hkdf, Go 1.24+) and proper labels before this leaves spike status. func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) PSK { - lo, hi := canonicalPair(b.LocalWgPub, b.RemoteWgPub) + lo, hi := canonicalPair(b.LocalID, b.RemoteID) h := sha256.New() h.Write([]byte(pskLabel)) diff --git a/client/internal/pqkem/kem_test.go b/client/internal/pqkem/kem_test.go index 3bdc996357d..499c8247009 100644 --- a/client/internal/pqkem/kem_test.go +++ b/client/internal/pqkem/kem_test.go @@ -18,11 +18,11 @@ func TestExchange_DerivesMatchingPSK(t *testing.T) { require.Len(t, init.Offer(), OfferSize) - answer, pskB, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + answer, pskB, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) require.NoError(t, err) require.Len(t, answer, AnswerSize) - pskA, err := init.Finish(answer, Binding{LocalWgPub: wgA, RemoteWgPub: wgB}) + pskA, err := init.Finish(answer, Binding{LocalID: wgA, RemoteID: wgB}) require.NoError(t, err) require.Equal(t, pskB, pskA, "both sides must derive the same PSK") @@ -34,13 +34,13 @@ func TestExchange_PSKBoundToPeerIdentities(t *testing.T) { require.NoError(t, err) // responder computes with the honest pair... - _, pskHonest, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + _, pskHonest, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) require.NoError(t, err) // ...a second responder run with a different peer identity yields a different PSK, // even though the KEM material would otherwise combine identically. wgC := []byte("peer-C-wireguard-pubkey-32bytes!") - _, pskWrong, err := Respond(init.Offer(), Binding{LocalWgPub: wgC, RemoteWgPub: wgA}) + _, pskWrong, err := Respond(init.Offer(), Binding{LocalID: wgC, RemoteID: wgA}) require.NoError(t, err) require.NotEqual(t, pskHonest, pskWrong, "PSK must be bound to the peer pair") @@ -70,12 +70,12 @@ func TestExchange_ReportSizesAndTiming(t *testing.T) { tInit += time.Since(s0) s1 := time.Now() - answer, _, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + answer, _, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) require.NoError(t, err) tResp += time.Since(s1) s2 := time.Now() - _, err = init.Finish(answer, Binding{LocalWgPub: wgA, RemoteWgPub: wgB}) + _, err = init.Finish(answer, Binding{LocalID: wgA, RemoteID: wgB}) require.NoError(t, err) tFinish += time.Since(s2) } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index cd48866f033..732fdbec516 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -10,8 +10,9 @@ import ( ) const ( - // DefaultRekeyInterval matches WireGuard's own REKEY_AFTER_TIME so the freshly - // rotated PSK is naturally adopted by WG's next handshake without forcing one. + // DefaultRekeyInterval is the default PSK rotation cadence (~2 min), chosen so a + // rotated PSK is adopted by the consumer's next transport handshake without + // forcing one. DefaultRekeyInterval = 2 * time.Minute // DefaultRetryInterval is how often the initiator retransmits its outstanding // message (offer, then confirm) while an exchange is in flight. @@ -29,11 +30,11 @@ const ( ) // Transport hands an already-encoded exchange message to the peer. The host routes -// it over the appropriate channel — Signal before the tunnel is up, the WireGuard -// tunnel for rekeys — so the Manager never needs to know which is in use. It is the -// analogue of go-rosenpass's Conn seam. +// it over the appropriate channel — a signalling channel before the tunnel is up, +// the data tunnel for rekeys — so the Manager never needs to know which is in use. +// It is the analogue of go-rosenpass's Conn seam. type Transport interface { - Send(remoteWgKey string, msg []byte) error + Send(remoteID string, msg []byte) error } // exchangeState is the single source of truth for an exchange's role and phase. @@ -66,14 +67,14 @@ type exchangeCtl struct { // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It // runs the per-peer rekey timer, drives the X25519MLKEM768 exchange over a pluggable -// Transport, and surfaces the derived PSK to the host via WGCallbackHandler. The +// Transport, and surfaces the derived PSK to the host via CallbackHandler. The // cryptography is the pure kem.go primitives; all per-exchange and per-peer state // lives here under one lock. type Manager struct { - localWgKey string - transport Transport - wg WGCallbackHandler - logger *slog.Logger + localID string + transport Transport + cbHandler CallbackHandler + logger *slog.Logger rekeyInterval time.Duration retryInterval time.Duration @@ -91,11 +92,11 @@ type Manager struct { wait sync.WaitGroup } -// NewManager builds a manager for the local peer identified by its WireGuard public +// NewManager builds a manager for the local peer identified by its peer identity // key (used for the deterministic initiator role and the identity binding). A zero // interval falls back to DefaultRekeyInterval; a nil logger to slog.Default(). // Retry/retries/K use their defaults and can be overridden before use. -func NewManager(localWgKey string, t Transport, h WGCallbackHandler, interval time.Duration, logger *slog.Logger) *Manager { +func NewManager(localID string, t Transport, h CallbackHandler, interval time.Duration, logger *slog.Logger) *Manager { if interval <= 0 { interval = DefaultRekeyInterval } @@ -104,9 +105,9 @@ func NewManager(localWgKey string, t Transport, h WGCallbackHandler, interval ti } ctx, cancel := context.WithCancel(context.Background()) return &Manager{ - localWgKey: localWgKey, + localID: localID, transport: t, - wg: h, + cbHandler: h, logger: logger, rekeyInterval: interval, retryInterval: DefaultRetryInterval, @@ -122,40 +123,40 @@ func NewManager(localWgKey string, t Transport, h WGCallbackHandler, interval ti } // IsInitiator reports whether the local peer drives the exchange for this remote -// peer. Roles are deterministic (lexicographic WG-key compare) so exactly one side -// initiates, mirroring how Rosenpass picks its handshake initiator. -func (m *Manager) IsInitiator(remoteWgKey string) bool { - return m.localWgKey > remoteWgKey +// peer. Roles are deterministic (lexicographic identity-key compare) so exactly one +// side initiates, mirroring how Rosenpass picks its handshake initiator. +func (m *Manager) IsInitiator(remoteID string) bool { + return m.localID > remoteID } // AddPeer registers a remote peer and starts its rekey timer. Re-adding is a no-op. -func (m *Manager) AddPeer(remoteWgKey string) { +func (m *Manager) AddPeer(remoteID string) { m.mu.Lock() defer m.mu.Unlock() - if _, ok := m.peers[remoteWgKey]; ok { + if _, ok := m.peers[remoteID]; ok { return } ctx, cancel := context.WithCancel(m.rootCtx) - m.peers[remoteWgKey] = cancel + m.peers[remoteID] = cancel m.wait.Add(1) - go m.rekeyLoop(ctx, remoteWgKey) + go m.rekeyLoop(ctx, remoteID) } // RemovePeer stops a peer's rekey timer and any in-flight exchange, and drops state. -func (m *Manager) RemovePeer(remoteWgKey string) { +func (m *Manager) RemovePeer(remoteID string) { m.mu.Lock() - if cancel, ok := m.peers[remoteWgKey]; ok { + if cancel, ok := m.peers[remoteID]; ok { cancel() - delete(m.peers, remoteWgKey) + delete(m.peers, remoteID) } - if ex, ok := m.exchanges[remoteWgKey]; ok { + if ex, ok := m.exchanges[remoteID]; ok { if ex.cancel != nil { ex.cancel() } - delete(m.exchanges, remoteWgKey) + delete(m.exchanges, remoteID) } - delete(m.established, remoteWgKey) - delete(m.failures, remoteWgKey) + delete(m.established, remoteID) + delete(m.failures, remoteID) m.mu.Unlock() } @@ -171,28 +172,28 @@ func (m *Manager) Stop() { // HandleInbound decodes an incoming message and drives the exchange, sending any // response via the transport and surfacing derived PSKs / convergence to the host. -func (m *Manager) HandleInbound(remoteWgKey string, raw []byte) error { +func (m *Manager) HandleInbound(remoteID string, raw []byte) error { typ, msg, err := Decode(raw) if err != nil { - return fmt.Errorf("decode from %s: %w", remoteWgKey, err) + return fmt.Errorf("decode from %s: %w", remoteID, err) } switch typ { case MsgOffer: - return m.handleOffer(remoteWgKey, msg.(*OfferMsg)) + return m.handleOffer(remoteID, msg.(*OfferMsg)) case MsgAnswer: - return m.handleAnswer(remoteWgKey, msg.(*AnswerMsg)) + return m.handleAnswer(remoteID, msg.(*AnswerMsg)) case MsgConfirm: - return m.handleConfirm(remoteWgKey, msg.(*ConfirmMsg)) + return m.handleConfirm(remoteID, msg.(*ConfirmMsg)) default: - return fmt.Errorf("unhandled message type %d from %s", typ, remoteWgKey) + return fmt.Errorf("unhandled message type %d from %s", typ, remoteID) } } // initiateRekey starts a fresh exchange when the local peer is the initiator for // this remote peer; the responder waits for the offer instead. Exposed (unexported // but directly callable) so tests can drive a rekey without waiting on the ticker. -func (m *Manager) initiateRekey(remoteWgKey string) error { - if !m.IsInitiator(remoteWgKey) { +func (m *Manager) initiateRekey(remoteID string) error { + if !m.IsInitiator(remoteID) { return nil } init, err := NewInitiator() @@ -210,10 +211,10 @@ func (m *Manager) initiateRekey(remoteWgKey string) error { ctx, cancel := context.WithCancel(m.rootCtx) m.mu.Lock() - if old := m.exchanges[remoteWgKey]; old != nil && old.cancel != nil { + if old := m.exchanges[remoteID]; old != nil && old.cancel != nil { old.cancel() } - m.exchanges[remoteWgKey] = &exchangeCtl{ + m.exchanges[remoteID] = &exchangeCtl{ id: id, state: stateAwaitingAnswer, startedAt: time.Now(), @@ -224,12 +225,12 @@ func (m *Manager) initiateRekey(remoteWgKey string) error { m.mu.Unlock() m.wait.Add(1) - go m.initiatorLoop(ctx, remoteWgKey, id) + go m.initiatorLoop(ctx, remoteID, id) - return m.transport.Send(remoteWgKey, raw) + return m.transport.Send(remoteID, raw) } -func (m *Manager) rekeyLoop(ctx context.Context, remoteWgKey string) { +func (m *Manager) rekeyLoop(ctx context.Context, remoteID string) { defer m.wait.Done() t := time.NewTicker(m.rekeyInterval) defer t.Stop() @@ -238,15 +239,15 @@ func (m *Manager) rekeyLoop(ctx context.Context, remoteWgKey string) { case <-ctx.Done(): return case <-t.C: - if err := m.initiateRekey(remoteWgKey); err != nil { - m.logger.Error("pqkem rekey failed to start", "peer", remoteWgKey, "err", err) + if err := m.initiateRekey(remoteID); err != nil { + m.logger.Error("pqkem rekey failed to start", "peer", remoteID, "err", err) } } } } -func (m *Manager) binding(remoteWgKey string) Binding { - return Binding{LocalWgPub: []byte(m.localWgKey), RemoteWgPub: []byte(remoteWgKey)} +func (m *Manager) binding(remoteID string) Binding { + return Binding{LocalID: []byte(m.localID), RemoteID: []byte(remoteID)} } func newExchangeID() (ExchangeID, error) { diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index e355b17faec..1100f832193 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -15,7 +15,7 @@ type loopback struct { peer *Manager } -func (l *loopback) Send(remoteWgKey string, msg []byte) error { +func (l *loopback) Send(remoteID string, msg []byte) error { cp := append([]byte(nil), msg...) return l.peer.HandleInbound(l.localKey, cp) } @@ -28,17 +28,17 @@ type fakeWG struct { func newFakeWG() *fakeWG { return &fakeWG{psks: map[string]PSK{}} } -func (f *fakeWG) OnNewPSKReady(remoteWgKey string, psk PSK) error { +func (f *fakeWG) OnNewPSKReady(remoteID string, psk PSK) error { f.mu.Lock() defer f.mu.Unlock() - f.psks[remoteWgKey] = psk + f.psks[remoteID] = psk return nil } -func (f *fakeWG) OnRekeyFailed(remoteWgKey string) error { +func (f *fakeWG) OnRekeyFailed(remoteID string) error { f.mu.Lock() defer f.mu.Unlock() - f.failed = append(f.failed, remoteWgKey) + f.failed = append(f.failed, remoteID) return nil } diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go index fe438163642..d1c6e69a469 100644 --- a/client/internal/pqkem/message.go +++ b/client/internal/pqkem/message.go @@ -7,7 +7,7 @@ import ( // Wire framing for the PQ-KEM exchange. Messages are self-contained, versioned, // transport-agnostic byte blobs: the same bytes ride the Signal offer/answer -// (initial, pre-tunnel) or a WireGuard-tunnel packet (rekey). They are NOT a gRPC +// (initial, pre-tunnel) or a data-tunnel packet (rekey). They are NOT a gRPC // service — the network layer only sees opaque []byte. // // Layout (all messages): [type:1][version:1][exchangeID:16][payload...] @@ -59,9 +59,9 @@ type AnswerMsg struct { // derive it to encapsulate), so the initiator needs no confirmation. Only the // responder is left unsure whether the initiator received the answer and committed // the key — this message resolves that. As a bonus, being sent under the new PSK it -// triggers the WireGuard handshake with the new key: the responder converges on -// receiving it, and the initiator converges by observing that handshake succeed -// (which fails on a PSK mismatch, so success proves the responder also committed). +// exercises the consumer's channel handshake with the new key: the responder +// converges on receiving it, and the initiator converges by observing that handshake +// succeed (which fails on a PSK mismatch, so success proves the responder committed too). type ConfirmMsg struct { ExchangeID ExchangeID } diff --git a/client/internal/pqkem/message_test.go b/client/internal/pqkem/message_test.go index 1845bc4d9b3..8708695d507 100644 --- a/client/internal/pqkem/message_test.go +++ b/client/internal/pqkem/message_test.go @@ -9,7 +9,7 @@ import ( func TestMessageRoundTrip(t *testing.T) { init, err := NewInitiator() require.NoError(t, err) - answer, _, err := Respond(init.Offer(), Binding{LocalWgPub: wgB, RemoteWgPub: wgA}) + answer, _, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) require.NoError(t, err) id := ExchangeID{1, 2, 3, 4} From 2ed0d3bf03d87f1b85c85bbc6cdd312ae42229f1 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 23 Jul 2026 15:14:19 +0200 Subject: [PATCH 014/119] Assume two transports: initial "signal" (control plane) one (no data path established yet) + data path one Define OnDataPathRekeyed event to transition from control plane path to data plane path over the WG tunnel. Keep confirm ALWAYS on NEW established WG tunnel (posthandshake with rekeying). We keep an active method irrelevant of the WG handshake (we might decide that the indirect wg handshake is sufficient in the future). Optimistic commit on responder(when sending answer), while on initiator we set it on getting the answer --- client/internal/pqkem/convergence.go | 108 ++++++++++++---------- client/internal/pqkem/convergence_test.go | 27 ++++-- client/internal/pqkem/manager.go | 106 +++++++++++++++------ client/internal/pqkem/manager_test.go | 37 +++++--- 4 files changed, 175 insertions(+), 103 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 16d6bec5905..3291e322f3a 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -5,9 +5,10 @@ import ( "time" ) -// handleOffer (responder) derives and sends the answer for a new exchange, or -// resends the cached answer for a duplicate offer (same exchangeID) without -// re-deriving. The responder is purely reactive: no retransmit loop, no deadline. +// handleOffer (responder) derives the PSK, commits it optimistically, and sends the +// answer over the current channel. A duplicate offer (same exchangeID) resends the +// cached answer without re-deriving. The responder is otherwise reactive: no +// retransmit loop. func (m *Manager) handleOffer(remoteID string, o *OfferMsg) error { m.mu.Lock() if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { @@ -18,7 +19,7 @@ func (m *Manager) handleOffer(remoteID string, o *OfferMsg) error { // dropping avoids a second (randomized -> divergent) derivation. return nil } - return m.transport.Send(remoteID, last) + return m.send(remoteID, last) } // Reserve the slot so a concurrent duplicate offer bails. m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, startedAt: time.Now()} @@ -34,18 +35,27 @@ func (m *Manager) handleOffer(remoteID string, o *OfferMsg) error { } m.mu.Lock() - if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { - ex.state = stateAwaitingConfirm - ex.lastSent = raw - ex.pendingPSK = psk + ex := m.exchanges[remoteID] + if ex == nil || ex.id != o.ExchangeID { + m.mu.Unlock() + return nil } + ex.state = stateAwaitingConfirm + ex.lastSent = raw + ex.pendingPSK = psk m.mu.Unlock() - return m.transport.Send(remoteID, raw) + // Commit optimistically so our data path can rekey to the new PSK; a lost answer + // simply means the data path won't come up (initial) or the previous PSK keeps + // working until it does (rekey grace) — both self-heal via retry. + if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { + return err + } + return m.send(remoteID, raw) } -// handleAnswer (initiator) derives the PSK, surfaces it, and sends the confirm, then -// switches the retransmit payload to the confirm. Only valid in stateAwaitingAnswer; +// handleAnswer (initiator) derives and commits the PSK, then waits for +// OnDataPathRekeyed to send the confirm. Only valid in stateAwaitingAnswer; // advancing the state under the lock makes a concurrent/duplicate answer bail. func (m *Manager) handleAnswer(remoteID string, a *AnswerMsg) error { m.mu.Lock() @@ -54,7 +64,7 @@ func (m *Manager) handleAnswer(remoteID string, a *AnswerMsg) error { m.mu.Unlock() return nil } - ex.state = stateConfirming + ex.state = stateAwaitingRekey init := ex.initiator ex.initiator = nil m.mu.Unlock() @@ -63,29 +73,15 @@ func (m *Manager) handleAnswer(remoteID string, a *AnswerMsg) error { if err != nil { return err } - if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { - return err - } - raw, err := (&ConfirmMsg{ExchangeID: a.ExchangeID}).Encode() - if err != nil { - return err - } - - m.mu.Lock() - if ex := m.exchanges[remoteID]; ex != nil && ex.id == a.ExchangeID { - ex.lastSent = raw // loop now retransmits the confirm - m.established[remoteID] = true - m.failures[remoteID] = 0 - _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) - } - m.mu.Unlock() - - return m.transport.Send(remoteID, raw) + // Commit now; the confirm is deferred until OnDataPathRekeyed so it rides the + // data path under the new key. + return m.cbHandler.OnNewPSKReady(remoteID, psk) } -// handleConfirm (responder) commits the pending PSK. Only valid in -// stateAwaitingConfirm; duplicate confirms (the initiator best-effort resends it) -// find the exchange gone and are ignored. +// handleConfirm (responder) records convergence. The PSK was already committed in +// handleOffer; a confirm arriving over the data path with a matching exchangeID +// proves we operate on the new key from this exchange. Stale/duplicate confirms find +// the exchange gone and are ignored. func (m *Manager) handleConfirm(remoteID string, c *ConfirmMsg) error { m.mu.Lock() ex := m.exchanges[remoteID] @@ -93,26 +89,25 @@ func (m *Manager) handleConfirm(remoteID string, c *ConfirmMsg) error { m.mu.Unlock() return nil } - psk := ex.pendingPSK delete(m.exchanges, remoteID) m.established[remoteID] = true m.failures[remoteID] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) m.mu.Unlock() - - return m.cbHandler.OnNewPSKReady(remoteID, psk) + return nil } -// initiatorLoop retransmits the initiator's outstanding message, keyed off the -// exchange state: the offer while awaiting the answer (bounded by maxRetries -> -// failure), then the confirm a few best-effort times before stopping. The -// convergence deadline is thus derived from maxRetries * retryInterval. +// initiatorLoop retransmits the initiator's outstanding message and enforces the +// convergence deadline, keyed off the exchange state: resend the offer while +// awaiting the answer; wait (counting toward the deadline) while awaiting the data +// path rekey; resend the confirm a few best-effort times once sent. Exhausting the +// deadline before reaching the confirm phase is a failure. func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id ExchangeID) { defer m.wait.Done() t := time.NewTicker(m.retryInterval) defer t.Stop() - offerAttempts, confirmsSent := 0, 0 + attempts, confirmsSent := 0, 0 for { select { case <-ctx.Done(): @@ -127,7 +122,7 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang switch ex.state { case stateAwaitingAnswer: - if offerAttempts >= m.maxRetries { + if attempts >= m.maxRetries { delete(m.exchanges, remoteID) fail := m.registerFailureLocked(remoteID) m.mu.Unlock() @@ -135,9 +130,24 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang return } msg := ex.lastSent - offerAttempts++ + attempts++ + m.mu.Unlock() + if err := m.send(remoteID, msg); err != nil { + m.logger.Warn("pqkem offer retransmit failed", "peer", remoteID, "err", err) + } + + case stateAwaitingRekey: + // Waiting for OnDataPathRekeyed; nothing to send, but the deadline + // still applies (the data path may never come up with the new key). + if attempts >= m.maxRetries { + delete(m.exchanges, remoteID) + fail := m.registerFailureLocked(remoteID) + m.mu.Unlock() + m.raiseFailure(remoteID, fail) + return + } + attempts++ m.mu.Unlock() - m.retransmit(remoteID, msg) case stateConfirming: if confirmsSent >= confirmRetransmits { @@ -148,7 +158,9 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang msg := ex.lastSent confirmsSent++ m.mu.Unlock() - m.retransmit(remoteID, msg) + if err := m.transport.SendDataPath(remoteID, msg); err != nil { + m.logger.Warn("pqkem confirm retransmit failed", "peer", remoteID, "err", err) + } default: m.mu.Unlock() @@ -183,9 +195,3 @@ func (m *Manager) raiseFailure(remoteID string, fail bool) { m.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteID, "err", err) } } - -func (m *Manager) retransmit(remoteID string, msg []byte) { - if err := m.transport.Send(remoteID, msg); err != nil { - m.logger.Warn("pqkem retransmit failed", "peer", remoteID, "err", err) - } -} diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index 5bdd4ec2001..3676f86640b 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -10,21 +10,25 @@ import ( type dropTransport struct{} -func (dropTransport) Send(string, []byte) error { return nil } +func (dropTransport) SendDataPath(string, []byte) error { return nil } +func (dropTransport) SendSignal(string, []byte) error { return nil } // gate is a loopback transport with a switchable drop flag. type gate struct { - local string - peer *Manager - drop atomic.Bool + localID string + peer *Manager + drop atomic.Bool } -func (g *gate) Send(remote string, msg []byte) error { +func (g *gate) SendDataPath(remoteID string, msg []byte) error { return g.deliver(msg) } +func (g *gate) SendSignal(remoteID string, msg []byte) error { return g.deliver(msg) } + +func (g *gate) deliver(msg []byte) error { if g.drop.Load() { return nil } cp := append([]byte(nil), msg...) - return g.peer.HandleInbound(g.local, cp) + return g.peer.HandleInbound(g.localID, cp) } func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { @@ -46,8 +50,8 @@ func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { } func TestManager_RekeyToleratesKFailures(t *testing.T) { - gA := &gate{local: "aaaa"} - gB := &gate{local: "bbbb"} + gA := &gate{localID: "aaaa"} + gB := &gate{localID: "bbbb"} wgA := newFakeWG() wgB := newFakeWG() @@ -62,11 +66,14 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { defer dA.Stop() defer dB.Stop() - // first exchange succeeds -> peer becomes established (subsequent failures are rekeys). + // First exchange succeeds -> peer becomes established (subsequent failures are + // rekeys). Drive the data-path-rekeyed event so the confirm converges A. require.NoError(t, dB.initiateRekey("aaaa")) + dA.OnDataPathRekeyed("bbbb") + dB.OnDataPathRekeyed("aaaa") require.NotEqual(t, PSK{}, wgB.psk("aaaa")) - // now drop B's outbound: rekeys can no longer converge. + // Now drop B's outbound: rekeys can no longer converge. gB.drop.Store(true) // K-1 failures must NOT raise OnRekeyFailed. diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 732fdbec516..0de006a59c1 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -17,43 +17,49 @@ const ( // DefaultRetryInterval is how often the initiator retransmits its outstanding // message (offer, then confirm) while an exchange is in flight. DefaultRetryInterval = 2 * time.Second - // DefaultMaxRetries bounds how many times the offer is retransmitted before the - // exchange is declared failed. The convergence deadline is thus derived as - // MaxRetries * RetryInterval — there is no separate deadline timer. + // DefaultMaxRetries bounds how many ticks an exchange may run before the offer + // or the wait for the data-path rekey is declared failed. The convergence + // deadline is thus derived as MaxRetries * RetryInterval — no separate timer. DefaultMaxRetries = 10 // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures // are tolerated before OnRekeyFailed. The initial exchange fails immediately. DefaultMaxRekeyFailures = 3 // confirmRetransmits is how many times the initiator best-effort resends the - // confirm after converging, to cover its loss without a dedicated goroutine. + // confirm over the data path, to cover its loss without a dedicated goroutine. confirmRetransmits = 3 ) -// Transport hands an already-encoded exchange message to the peer. The host routes -// it over the appropriate channel — a signalling channel before the tunnel is up, -// the data tunnel for rekeys — so the Manager never needs to know which is in use. -// It is the analogue of go-rosenpass's Conn seam. +// Transport carries exchange messages over the two available channels; the library +// picks which. It is the analogue of go-rosenpass's Conn seam. +// +// Model: a message rides the channel keyed with the CURRENTLY valid key. Before a +// data path exists (initial bootstrap) that is the out-of-band signalling channel; +// once the data path is up (after OnDataPathRekeyed) offers/answers ride it, and the +// confirm ALWAYS rides the data path (so its arrival proves the new key works). type Transport interface { - Send(remoteID string, msg []byte) error + // SendDataPath sends over the peer's established data path (e.g. a WireGuard + // tunnel). The consumer routes it accordingly. + SendDataPath(remoteID string, msg []byte) error + // SendSignal sends over the out-of-band signalling channel, handled outside the + // library by the consumer. + SendSignal(remoteID string, msg []byte) error } // exchangeState is the single source of truth for an exchange's role and phase. -// Every handler and the retransmit loop key off it, so no role/phase is re-derived -// from other fields. type exchangeState uint8 const ( stateReserved exchangeState = iota // responder: deriving the answer stateAwaitingAnswer // initiator: offer sent, awaiting the answer - stateAwaitingConfirm // responder: answer sent, awaiting the confirm - stateConfirming // initiator: answer in, PSK committed, flushing the confirm + stateAwaitingRekey // initiator: PSK derived+set, awaiting OnDataPathRekeyed to send the confirm + stateConfirming // initiator: confirm sent over the data path, best-effort retransmit + stateAwaitingConfirm // responder: answer sent, awaiting the confirm over the data path ) // exchangeCtl holds all state for one in-flight exchange with a peer, under the -// Manager's single lock. state drives every decision. lastSent is the message -// currently being (re)transmitted. The crypto payloads live here too: initiator is -// the ephemeral handle used at Finish (initiator side); pendingPSK is the derived -// key held until the confirm commits it (responder side). Only the initiator runs a +// Manager's single lock. state drives every decision. lastSent is the current +// retransmit payload (offer, then confirm). initiator is the ephemeral handle used +// at Finish; pendingPSK is the responder's derived key. Only the initiator runs a // retransmit loop, so only it sets cancel. type exchangeCtl struct { id ExchangeID @@ -66,10 +72,10 @@ type exchangeCtl struct { } // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It -// runs the per-peer rekey timer, drives the X25519MLKEM768 exchange over a pluggable -// Transport, and surfaces the derived PSK to the host via CallbackHandler. The -// cryptography is the pure kem.go primitives; all per-exchange and per-peer state -// lives here under one lock. +// runs the per-peer rekey timer, drives the X25519MLKEM768 exchange over the +// Transport, and surfaces the derived PSK and convergence to the host via +// CallbackHandler. The cryptography is the pure kem.go primitives; all state lives +// here under one lock. type Manager struct { localID string transport Transport @@ -89,13 +95,13 @@ type Manager struct { exchanges map[string]*exchangeCtl // in-flight exchange per peer established map[string]bool // peer has completed at least one exchange failures map[string]int // consecutive rekey failures per peer + dataPathUp map[string]bool // peer's data path is up (offers/answers may ride it) wait sync.WaitGroup } // NewManager builds a manager for the local peer identified by its peer identity // key (used for the deterministic initiator role and the identity binding). A zero // interval falls back to DefaultRekeyInterval; a nil logger to slog.Default(). -// Retry/retries/K use their defaults and can be overridden before use. func NewManager(localID string, t Transport, h CallbackHandler, interval time.Duration, logger *slog.Logger) *Manager { if interval <= 0 { interval = DefaultRekeyInterval @@ -119,6 +125,7 @@ func NewManager(localID string, t Transport, h CallbackHandler, interval time.Du exchanges: make(map[string]*exchangeCtl), established: make(map[string]bool), failures: make(map[string]int), + dataPathUp: make(map[string]bool), } } @@ -157,6 +164,7 @@ func (m *Manager) RemovePeer(remoteID string) { } delete(m.established, remoteID) delete(m.failures, remoteID) + delete(m.dataPathUp, remoteID) m.mu.Unlock() } @@ -170,8 +178,38 @@ func (m *Manager) Stop() { m.mu.Unlock() } -// HandleInbound decodes an incoming message and drives the exchange, sending any -// response via the transport and surfacing derived PSKs / convergence to the host. +// OnDataPathRekeyed notifies the library that the peer's data path is up and freshly +// keyed with the latest PSK (fired on first establishment AND every rekey — the same +// event). It marks the data path usable and, if we are the initiator waiting to +// confirm, sends the confirm over the data path (its arrival proves to the responder +// that we operate on the new key, correlated by exchangeID). +func (m *Manager) OnDataPathRekeyed(remoteID string) { + m.mu.Lock() + m.dataPathUp[remoteID] = true + ex := m.exchanges[remoteID] + if ex == nil || ex.state != stateAwaitingRekey { + m.mu.Unlock() + return + } + confirm, err := (&ConfirmMsg{ExchangeID: ex.id}).Encode() + if err != nil { + m.mu.Unlock() + m.logger.Error("pqkem encode confirm", "peer", remoteID, "err", err) + return + } + ex.state = stateConfirming + ex.lastSent = confirm + m.established[remoteID] = true + m.failures[remoteID] = 0 + _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) + m.mu.Unlock() + + if err := m.transport.SendDataPath(remoteID, confirm); err != nil { + m.logger.Warn("pqkem send confirm failed", "peer", remoteID, "err", err) + } +} + +// HandleInbound decodes an incoming message and drives the exchange. func (m *Manager) HandleInbound(remoteID string, raw []byte) error { typ, msg, err := Decode(raw) if err != nil { @@ -189,9 +227,9 @@ func (m *Manager) HandleInbound(remoteID string, raw []byte) error { } } -// initiateRekey starts a fresh exchange when the local peer is the initiator for -// this remote peer; the responder waits for the offer instead. Exposed (unexported -// but directly callable) so tests can drive a rekey without waiting on the ticker. +// initiateRekey starts a fresh exchange when the local peer is the initiator. The +// offer rides the data path if it is up (rekey) or the signalling channel otherwise +// (initial bootstrap). func (m *Manager) initiateRekey(remoteID string) error { if !m.IsInitiator(remoteID) { return nil @@ -227,7 +265,7 @@ func (m *Manager) initiateRekey(remoteID string) error { m.wait.Add(1) go m.initiatorLoop(ctx, remoteID, id) - return m.transport.Send(remoteID, raw) + return m.send(remoteID, raw) } func (m *Manager) rekeyLoop(ctx context.Context, remoteID string) { @@ -246,6 +284,18 @@ func (m *Manager) rekeyLoop(ctx context.Context, remoteID string) { } } +// send routes offer/answer over the data path when it is up, else the signalling +// channel. The confirm never goes through here — it always uses SendDataPath. +func (m *Manager) send(remoteID string, msg []byte) error { + m.mu.Lock() + viaDataPath := m.dataPathUp[remoteID] + m.mu.Unlock() + if viaDataPath { + return m.transport.SendDataPath(remoteID, msg) + } + return m.transport.SendSignal(remoteID, msg) +} + func (m *Manager) binding(remoteID string) Binding { return Binding{LocalID: []byte(m.localID), RemoteID: []byte(remoteID)} } diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index 1100f832193..aea902fc4c7 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -8,16 +8,20 @@ import ( "github.com/stretchr/testify/require" ) -// loopback delivers a sent message synchronously to the peer driver's HandleInbound, -// attributing it to localKey (the sender). +// loopback delivers a sent message synchronously to the peer manager's HandleInbound, +// attributing it to localID (the sender). Both channels deliver the same way — the +// test does not care which physical channel is used. type loopback struct { - localKey string - peer *Manager + localID string + peer *Manager } -func (l *loopback) Send(remoteID string, msg []byte) error { +func (l *loopback) SendDataPath(remoteID string, msg []byte) error { return l.deliver(msg) } +func (l *loopback) SendSignal(remoteID string, msg []byte) error { return l.deliver(msg) } + +func (l *loopback) deliver(msg []byte) error { cp := append([]byte(nil), msg...) - return l.peer.HandleInbound(l.localKey, cp) + return l.peer.HandleInbound(l.localID, cp) } type fakeWG struct { @@ -49,8 +53,8 @@ func (f *fakeWG) psk(peer string) PSK { } func TestManager_ExchangeConverges(t *testing.T) { - lbA := &loopback{localKey: "aaaa"} - lbB := &loopback{localKey: "bbbb"} + lbA := &loopback{localID: "aaaa"} + lbB := &loopback{localID: "bbbb"} wgA := newFakeWG() wgB := newFakeWG() @@ -65,21 +69,26 @@ func TestManager_ExchangeConverges(t *testing.T) { defer dA.Stop() defer dB.Stop() - // B is the initiator ("bbbb" > "aaaa"). + // B is the initiator ("bbbb" > "aaaa"). Offer/answer flow synchronously over the + // loopback; both commit their PSK, and B parks in stateAwaitingRekey. require.NoError(t, dB.initiateRekey("aaaa")) - pskB := wgB.psk("aaaa") // B committed on the answer - pskA := wgA.psk("bbbb") // A committed on the confirm + // The consumer reports the data path is (re)keyed on both sides; this makes B + // send the confirm, which converges A. + dA.OnDataPathRekeyed("bbbb") + dB.OnDataPathRekeyed("aaaa") + + pskB := wgB.psk("aaaa") + pskA := wgA.psk("bbbb") require.NotEqual(t, PSK{}, pskA, "responder A must have a PSK") require.NotEqual(t, PSK{}, pskB, "initiator B must have a PSK") require.Equal(t, pskB, pskA, "both sides converge on the same PSK") } func TestManager_NonInitiatorDoesNothing(t *testing.T) { - lbA := &loopback{localKey: "aaaa"} + lbA := &loopback{localID: "aaaa"} wgA := newFakeWG() dA := NewManager("aaaa", lbA, wgA, time.Hour, nil) - // no peer driver wired; if A wrongly initiated, Send would nil-panic. dA.AddPeer("bbbb") defer dA.Stop() @@ -89,7 +98,7 @@ func TestManager_NonInitiatorDoesNothing(t *testing.T) { } func TestManager_StopIsIdempotent(t *testing.T) { - dA := NewManager("aaaa", &loopback{localKey: "aaaa"}, newFakeWG(), time.Hour, nil) + dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), time.Hour, nil) dA.AddPeer("bbbb") dA.Stop() dA.Stop() // must not panic or hang From cec4bd49b4011387f62865871f518bebf9e13a85 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 23 Jul 2026 16:52:26 +0200 Subject: [PATCH 015/119] Leave signal offer/answer as a pull/push operation not as an actual transport --- client/internal/pqkem/convergence.go | 105 ++++++---- client/internal/pqkem/convergence_test.go | 40 ++-- client/internal/pqkem/manager.go | 234 +++++++++++++--------- client/internal/pqkem/manager_test.go | 49 ++--- 4 files changed, 255 insertions(+), 173 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 3291e322f3a..26c064e0a41 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -5,21 +5,57 @@ import ( "time" ) -// handleOffer (responder) derives the PSK, commits it optimistically, and sends the -// answer over the current channel. A duplicate offer (same exchangeID) resends the -// cached answer without re-deriving. The responder is otherwise reactive: no -// retransmit loop. -func (m *Manager) handleOffer(remoteID string, o *OfferMsg) error { +// startExchange creates a fresh initiator exchange and returns the framed offer for +// the caller to send (pushed over the data path for a rekey, or handed to the host +// for the signalling channel when viaSignal is set). Any previous in-flight exchange +// for the peer is cancelled. +func (m *Manager) startExchange(remoteID string, viaSignal bool) ([]byte, error) { + init, err := NewInitiator() + if err != nil { + return nil, err + } + id, err := newExchangeID() + if err != nil { + return nil, err + } + raw, err := (&OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}).Encode() + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(m.rootCtx) + m.mu.Lock() + if old := m.exchanges[remoteID]; old != nil && old.cancel != nil { + old.cancel() + } + m.exchanges[remoteID] = &exchangeCtl{ + id: id, + state: stateAwaitingAnswer, + startedAt: time.Now(), + cancel: cancel, + lastSent: raw, + initiator: init, + viaSignal: viaSignal, + } + m.mu.Unlock() + + m.wait.Add(1) + go m.initiatorLoop(ctx, remoteID, id) + return raw, nil +} + +// processOffer (responder) derives the PSK, commits it optimistically, and returns +// the framed answer for the caller to send. A duplicate offer (same exchangeID) +// returns the cached answer without re-deriving; a still-reserved slot returns nil. +func (m *Manager) processOffer(remoteID string, o *OfferMsg) ([]byte, error) { m.mu.Lock() if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { state, last := ex.state, ex.lastSent m.mu.Unlock() if state == stateReserved { - // another goroutine reserved this exchange and is deriving the answer; - // dropping avoids a second (randomized -> divergent) derivation. - return nil + return nil, nil } - return m.send(remoteID, last) + return last, nil } // Reserve the slot so a concurrent duplicate offer bails. m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, startedAt: time.Now()} @@ -27,37 +63,36 @@ func (m *Manager) handleOffer(remoteID string, o *OfferMsg) error { answerBytes, psk, err := Respond(o.KEMOffer, m.binding(remoteID)) if err != nil { - return err + return nil, err } raw, err := (&AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answerBytes}).Encode() if err != nil { - return err + return nil, err } m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != o.ExchangeID { m.mu.Unlock() - return nil + return nil, nil } ex.state = stateAwaitingConfirm ex.lastSent = raw ex.pendingPSK = psk m.mu.Unlock() - // Commit optimistically so our data path can rekey to the new PSK; a lost answer - // simply means the data path won't come up (initial) or the previous PSK keeps - // working until it does (rekey grace) — both self-heal via retry. + // Commit optimistically so our data path can rekey to the new PSK. if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { - return err + return nil, err } - return m.send(remoteID, raw) + return raw, nil } -// handleAnswer (initiator) derives and commits the PSK, then waits for -// OnDataPathRekeyed to send the confirm. Only valid in stateAwaitingAnswer; -// advancing the state under the lock makes a concurrent/duplicate answer bail. -func (m *Manager) handleAnswer(remoteID string, a *AnswerMsg) error { +// processAnswer (initiator) derives and commits the PSK, then parks in +// stateAwaitingRekey; the confirm is sent later, over the data path, from +// OnDataPathRekeyed. Only valid in stateAwaitingAnswer; advancing the state under the +// lock makes a concurrent/duplicate answer bail. +func (m *Manager) processAnswer(remoteID string, a *AnswerMsg) error { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { @@ -73,16 +108,14 @@ func (m *Manager) handleAnswer(remoteID string, a *AnswerMsg) error { if err != nil { return err } - // Commit now; the confirm is deferred until OnDataPathRekeyed so it rides the - // data path under the new key. return m.cbHandler.OnNewPSKReady(remoteID, psk) } -// handleConfirm (responder) records convergence. The PSK was already committed in -// handleOffer; a confirm arriving over the data path with a matching exchangeID +// processConfirm (responder) records convergence. The PSK was already committed in +// processOffer; a confirm arriving over the data path with a matching exchangeID // proves we operate on the new key from this exchange. Stale/duplicate confirms find // the exchange gone and are ignored. -func (m *Manager) handleConfirm(remoteID string, c *ConfirmMsg) error { +func (m *Manager) processConfirm(remoteID string, c *ConfirmMsg) error { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != c.ExchangeID || ex.state != stateAwaitingConfirm { @@ -97,11 +130,12 @@ func (m *Manager) handleConfirm(remoteID string, c *ConfirmMsg) error { return nil } -// initiatorLoop retransmits the initiator's outstanding message and enforces the -// convergence deadline, keyed off the exchange state: resend the offer while -// awaiting the answer; wait (counting toward the deadline) while awaiting the data -// path rekey; resend the confirm a few best-effort times once sent. Exhausting the -// deadline before reaching the confirm phase is a failure. +// initiatorLoop enforces the convergence deadline and retransmits the initiator's +// outstanding DATA-PATH message: it resends the offer while awaiting the answer only +// when the offer went over the data path (a signalling-bootstrapped offer is +// retransmitted by the host with its own negotiation); it waits, counting toward the +// deadline, while awaiting the data-path rekey; it resends the confirm a few times +// once sent. Exhausting the deadline before the confirm phase is a failure. func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id ExchangeID) { defer m.wait.Done() t := time.NewTicker(m.retryInterval) @@ -129,11 +163,14 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang m.raiseFailure(remoteID, fail) return } + viaSignal := ex.viaSignal msg := ex.lastSent attempts++ m.mu.Unlock() - if err := m.send(remoteID, msg); err != nil { - m.logger.Warn("pqkem offer retransmit failed", "peer", remoteID, "err", err) + if !viaSignal { + if err := m.pushDataPath(remoteID, msg); err != nil { + m.logger.Warn("pqkem offer retransmit failed", "peer", remoteID, "err", err) + } } case stateAwaitingRekey: @@ -158,7 +195,7 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang msg := ex.lastSent confirmsSent++ m.mu.Unlock() - if err := m.transport.SendDataPath(remoteID, msg); err != nil { + if err := m.pushDataPath(remoteID, msg); err != nil { m.logger.Warn("pqkem confirm retransmit failed", "peer", remoteID, "err", err) } diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index 3676f86640b..1b9e075a54b 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -11,24 +11,21 @@ import ( type dropTransport struct{} func (dropTransport) SendDataPath(string, []byte) error { return nil } -func (dropTransport) SendSignal(string, []byte) error { return nil } -// gate is a loopback transport with a switchable drop flag. +// gate is a data-path loopback with a switchable drop flag. When dropping it reports +// success but does not deliver (mimics a lossy/broken tunnel). type gate struct { localID string peer *Manager drop atomic.Bool } -func (g *gate) SendDataPath(remoteID string, msg []byte) error { return g.deliver(msg) } -func (g *gate) SendSignal(remoteID string, msg []byte) error { return g.deliver(msg) } - -func (g *gate) deliver(msg []byte) error { +func (g *gate) SendDataPath(remoteID string, msg []byte) error { if g.drop.Load() { return nil } cp := append([]byte(nil), msg...) - return g.peer.HandleInbound(g.localID, cp) + return g.peer.OnDataPathMessage(g.localID, cp) } func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { @@ -39,9 +36,12 @@ func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { d.AddPeer("aaaa") defer d.Stop() - require.NoError(t, d.initiateRekey("aaaa")) + // Bootstrap offer is produced for signalling; no answer ever comes back -> the + // initial exchange fails fast. + offer, err := d.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) - // no answer will ever come -> the initial exchange fails fast. require.Eventually(t, func() bool { wg.mu.Lock() defer wg.mu.Unlock() @@ -66,25 +66,31 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { defer dA.Stop() defer dB.Stop() - // First exchange succeeds -> peer becomes established (subsequent failures are - // rekeys). Drive the data-path-rekeyed event so the confirm converges A. - require.NoError(t, dB.initiateRekey("aaaa")) + // Establish via a signalling bootstrap + data-path-rekeyed, so B becomes + // established and its data path is up. + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + answer, err := dA.SignalOnOffer("bbbb", offer) + require.NoError(t, err) + require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) dA.OnDataPathRekeyed("bbbb") dB.OnDataPathRekeyed("aaaa") require.NotEqual(t, PSK{}, wgB.psk("aaaa")) - // Now drop B's outbound: rekeys can no longer converge. + // Now drop B's data-path delivery: rekeys can no longer converge. gB.drop.Store(true) - // K-1 failures must NOT raise OnRekeyFailed. + // K-1 data-path rekeys must NOT raise OnRekeyFailed. for i := 0; i < DefaultMaxRekeyFailures-1; i++ { - require.NoError(t, dB.initiateRekey("aaaa")) + _, err := dB.startExchange("aaaa", false) + require.NoError(t, err) time.Sleep(50 * time.Millisecond) } require.Equal(t, 0, failedCount(wgB), "no failure before K attempts") - // the K-th failure raises it once. - require.NoError(t, dB.initiateRekey("aaaa")) + // The K-th failure raises it once. + _, err = dB.startExchange("aaaa", false) + require.NoError(t, err) require.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 0de006a59c1..675d88956e5 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -15,34 +15,26 @@ const ( // forcing one. DefaultRekeyInterval = 2 * time.Minute // DefaultRetryInterval is how often the initiator retransmits its outstanding - // message (offer, then confirm) while an exchange is in flight. + // data-path message while an exchange is in flight. DefaultRetryInterval = 2 * time.Second - // DefaultMaxRetries bounds how many ticks an exchange may run before the offer - // or the wait for the data-path rekey is declared failed. The convergence - // deadline is thus derived as MaxRetries * RetryInterval — no separate timer. + // DefaultMaxRetries bounds how many ticks an exchange may run before it is + // declared failed. The convergence deadline is thus MaxRetries * RetryInterval. DefaultMaxRetries = 10 // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures // are tolerated before OnRekeyFailed. The initial exchange fails immediately. DefaultMaxRekeyFailures = 3 // confirmRetransmits is how many times the initiator best-effort resends the - // confirm over the data path, to cover its loss without a dedicated goroutine. + // confirm over the data path, to cover its loss. confirmRetransmits = 3 ) -// Transport carries exchange messages over the two available channels; the library -// picks which. It is the analogue of go-rosenpass's Conn seam. -// -// Model: a message rides the channel keyed with the CURRENTLY valid key. Before a -// data path exists (initial bootstrap) that is the out-of-band signalling channel; -// once the data path is up (after OnDataPathRekeyed) offers/answers ride it, and the -// confirm ALWAYS rides the data path (so its arrival proves the new key works). +// Transport pushes a message over the peer's data path (e.g. a WireGuard tunnel). +// It is the library's only outbound send: the control-plane (signalling) channel is +// host-driven — the library hands the host offer/answer payloads to piggyback on the +// host's own offer/answer, it never pushes there itself (that would drive the host's +// connection negotiation, which is not the library's to control). type Transport interface { - // SendDataPath sends over the peer's established data path (e.g. a WireGuard - // tunnel). The consumer routes it accordingly. SendDataPath(remoteID string, msg []byte) error - // SendSignal sends over the out-of-band signalling channel, handled outside the - // library by the consumer. - SendSignal(remoteID string, msg []byte) error } // exchangeState is the single source of truth for an exchange's role and phase. @@ -58,9 +50,11 @@ const ( // exchangeCtl holds all state for one in-flight exchange with a peer, under the // Manager's single lock. state drives every decision. lastSent is the current -// retransmit payload (offer, then confirm). initiator is the ephemeral handle used -// at Finish; pendingPSK is the responder's derived key. Only the initiator runs a -// retransmit loop, so only it sets cancel. +// data-path retransmit payload. initiator is the ephemeral handle used at Finish; +// pendingPSK is the responder's derived key. viaSignal records that the offer was +// handed to the host for the signalling channel (bootstrap), so the loop does not +// retransmit it on the data path (the host retransmits it with its own negotiation). +// Only the initiator runs a retransmit loop, so only it sets cancel. type exchangeCtl struct { id ExchangeID state exchangeState @@ -69,13 +63,13 @@ type exchangeCtl struct { lastSent []byte initiator *Initiator pendingPSK PSK + viaSignal bool } // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It -// runs the per-peer rekey timer, drives the X25519MLKEM768 exchange over the -// Transport, and surfaces the derived PSK and convergence to the host via -// CallbackHandler. The cryptography is the pure kem.go primitives; all state lives -// here under one lock. +// runs the per-peer rekey timer, drives the X25519MLKEM768 exchange, and surfaces the +// derived PSK and convergence to the host via CallbackHandler. The cryptography is +// the pure kem.go primitives; all state lives here under one lock. type Manager struct { localID string transport Transport @@ -95,12 +89,15 @@ type Manager struct { exchanges map[string]*exchangeCtl // in-flight exchange per peer established map[string]bool // peer has completed at least one exchange failures map[string]int // consecutive rekey failures per peer - dataPathUp map[string]bool // peer's data path is up (offers/answers may ride it) - wait sync.WaitGroup + // dataSend holds the peer's data-path sender when the data path is up; nil (absent) + // means it is down. Toggled by OnDataPathRekeyed / OnDataPathDown. Its presence is + // the "a data path exists" signal. + dataSend map[string]func(string, []byte) error + wait sync.WaitGroup } -// NewManager builds a manager for the local peer identified by its peer identity -// key (used for the deterministic initiator role and the identity binding). A zero +// NewManager builds a manager for the local peer identified by its peer identity key +// (used for the deterministic initiator role and the identity binding). A zero // interval falls back to DefaultRekeyInterval; a nil logger to slog.Default(). func NewManager(localID string, t Transport, h CallbackHandler, interval time.Duration, logger *slog.Logger) *Manager { if interval <= 0 { @@ -125,7 +122,7 @@ func NewManager(localID string, t Transport, h CallbackHandler, interval time.Du exchanges: make(map[string]*exchangeCtl), established: make(map[string]bool), failures: make(map[string]int), - dataPathUp: make(map[string]bool), + dataSend: make(map[string]func(string, []byte) error), } } @@ -164,7 +161,7 @@ func (m *Manager) RemovePeer(remoteID string) { } delete(m.established, remoteID) delete(m.failures, remoteID) - delete(m.dataPathUp, remoteID) + delete(m.dataSend, remoteID) m.mu.Unlock() } @@ -178,96 +175,122 @@ func (m *Manager) Stop() { m.mu.Unlock() } -// OnDataPathRekeyed notifies the library that the peer's data path is up and freshly -// keyed with the latest PSK (fired on first establishment AND every rekey — the same -// event). It marks the data path usable and, if we are the initiator waiting to -// confirm, sends the confirm over the data path (its arrival proves to the responder -// that we operate on the new key, correlated by exchangeID). -func (m *Manager) OnDataPathRekeyed(remoteID string) { +// ---- Signalling channel (host-driven; rides the host's offer/answer) ---- + +// SignalOffer returns the KEM offer for the host to embed in its outgoing offer to +// remoteID (bootstrap). It returns (nil, nil) when the local peer is not the +// initiator. It is idempotent for an in-flight bootstrap: a repeat call (e.g. the +// host retransmitting its offer) returns the same offer rather than starting a new +// exchange. +func (m *Manager) SignalOffer(remoteID string) ([]byte, error) { + if !m.IsInitiator(remoteID) { + return nil, nil + } m.mu.Lock() - m.dataPathUp[remoteID] = true - ex := m.exchanges[remoteID] - if ex == nil || ex.state != stateAwaitingRekey { + if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && ex.state == stateAwaitingAnswer { + last := ex.lastSent m.mu.Unlock() - return + return last, nil } - confirm, err := (&ConfirmMsg{ExchangeID: ex.id}).Encode() + m.mu.Unlock() + return m.startExchange(remoteID, true) +} + +// SignalOnOffer processes a KEM offer the host extracted from an incoming offer and +// returns the KEM answer for the host to embed in its outgoing answer. +func (m *Manager) SignalOnOffer(remoteID string, offer []byte) ([]byte, error) { + typ, msg, err := Decode(offer) if err != nil { - m.mu.Unlock() - m.logger.Error("pqkem encode confirm", "peer", remoteID, "err", err) - return + return nil, fmt.Errorf("decode signal offer from %s: %w", remoteID, err) } - ex.state = stateConfirming - ex.lastSent = confirm - m.established[remoteID] = true - m.failures[remoteID] = 0 - _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) - m.mu.Unlock() + if typ != MsgOffer { + return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) + } + return m.processOffer(remoteID, msg.(*OfferMsg)) +} - if err := m.transport.SendDataPath(remoteID, confirm); err != nil { - m.logger.Warn("pqkem send confirm failed", "peer", remoteID, "err", err) +// SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. +// There is no reply: the confirm rides the data path after OnDataPathRekeyed. +func (m *Manager) SignalOnAnswer(remoteID string, answer []byte) error { + typ, msg, err := Decode(answer) + if err != nil { + return fmt.Errorf("decode signal answer from %s: %w", remoteID, err) + } + if typ != MsgAnswer { + return fmt.Errorf("expected answer from %s, got type %d", remoteID, typ) } + return m.processAnswer(remoteID, msg.(*AnswerMsg)) } -// HandleInbound decodes an incoming message and drives the exchange. -func (m *Manager) HandleInbound(remoteID string, raw []byte) error { +// ---- Data path (library-driven push) ---- + +// OnDataPathMessage feeds a KEM message received over the data path (tunnel) and +// pushes any reply back over the data path. +func (m *Manager) OnDataPathMessage(remoteID string, raw []byte) error { typ, msg, err := Decode(raw) if err != nil { - return fmt.Errorf("decode from %s: %w", remoteID, err) + return fmt.Errorf("decode data-path msg from %s: %w", remoteID, err) } switch typ { case MsgOffer: - return m.handleOffer(remoteID, msg.(*OfferMsg)) + answer, err := m.processOffer(remoteID, msg.(*OfferMsg)) + if err != nil { + return err + } + if answer == nil { + return nil + } + return m.pushDataPath(remoteID, answer) case MsgAnswer: - return m.handleAnswer(remoteID, msg.(*AnswerMsg)) + return m.processAnswer(remoteID, msg.(*AnswerMsg)) case MsgConfirm: - return m.handleConfirm(remoteID, msg.(*ConfirmMsg)) + return m.processConfirm(remoteID, msg.(*ConfirmMsg)) default: - return fmt.Errorf("unhandled message type %d from %s", typ, remoteID) + return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } } -// initiateRekey starts a fresh exchange when the local peer is the initiator. The -// offer rides the data path if it is up (rekey) or the signalling channel otherwise -// (initial bootstrap). -func (m *Manager) initiateRekey(remoteID string) error { - if !m.IsInitiator(remoteID) { - return nil - } - init, err := NewInitiator() - if err != nil { - return err +// OnDataPathRekeyed notifies that the peer's data path is up and freshly keyed with +// the latest PSK (fired on first establishment AND every rekey — the same event). It +// marks the data path usable and, if we are the initiator waiting to confirm, sends +// the confirm over the data path (its arrival proves to the responder that we operate +// on the new key, correlated by exchangeID). +func (m *Manager) OnDataPathRekeyed(remoteID string) { + m.mu.Lock() + m.dataSend[remoteID] = m.transport.SendDataPath + ex := m.exchanges[remoteID] + if ex == nil || ex.state != stateAwaitingRekey { + m.mu.Unlock() + return } - id, err := newExchangeID() + confirm, err := (&ConfirmMsg{ExchangeID: ex.id}).Encode() if err != nil { - return err + m.mu.Unlock() + m.logger.Error("pqkem encode confirm", "peer", remoteID, "err", err) + return } - raw, err := (&OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}).Encode() - if err != nil { - return err + ex.state = stateConfirming + ex.lastSent = confirm + m.established[remoteID] = true + m.failures[remoteID] = 0 + _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) + m.mu.Unlock() + + if err := m.pushDataPath(remoteID, confirm); err != nil { + m.logger.Warn("pqkem send confirm failed", "peer", remoteID, "err", err) } +} - ctx, cancel := context.WithCancel(m.rootCtx) +// OnDataPathDown notifies that the peer's data path went down; further rekeys wait +// until it is up again (the host re-bootstraps over signalling on reconnect). +func (m *Manager) OnDataPathDown(remoteID string) { m.mu.Lock() - if old := m.exchanges[remoteID]; old != nil && old.cancel != nil { - old.cancel() - } - m.exchanges[remoteID] = &exchangeCtl{ - id: id, - state: stateAwaitingAnswer, - startedAt: time.Now(), - cancel: cancel, - lastSent: raw, - initiator: init, - } + delete(m.dataSend, remoteID) m.mu.Unlock() - - m.wait.Add(1) - go m.initiatorLoop(ctx, remoteID, id) - - return m.send(remoteID, raw) } +// ---- internals ---- + func (m *Manager) rekeyLoop(ctx context.Context, remoteID string) { defer m.wait.Done() t := time.NewTicker(m.rekeyInterval) @@ -277,23 +300,36 @@ func (m *Manager) rekeyLoop(ctx context.Context, remoteID string) { case <-ctx.Done(): return case <-t.C: - if err := m.initiateRekey(remoteID); err != nil { + m.mu.Lock() + _, dpUp := m.dataSend[remoteID] + _, inFlight := m.exchanges[remoteID] + m.mu.Unlock() + // Rekeys ride the data path only; skip when it is down (the host will + // re-bootstrap over signalling on reconnect) or an exchange is in flight. + if !dpUp || inFlight || !m.IsInitiator(remoteID) { + continue + } + offer, err := m.startExchange(remoteID, false) + if err != nil { m.logger.Error("pqkem rekey failed to start", "peer", remoteID, "err", err) + continue + } + if err := m.pushDataPath(remoteID, offer); err != nil { + m.logger.Warn("pqkem send rekey offer failed", "peer", remoteID, "err", err) } } } } -// send routes offer/answer over the data path when it is up, else the signalling -// channel. The confirm never goes through here — it always uses SendDataPath. -func (m *Manager) send(remoteID string, msg []byte) error { +// pushDataPath sends over the peer's data path, erroring if it is down. +func (m *Manager) pushDataPath(remoteID string, msg []byte) error { m.mu.Lock() - viaDataPath := m.dataPathUp[remoteID] + send := m.dataSend[remoteID] m.mu.Unlock() - if viaDataPath { - return m.transport.SendDataPath(remoteID, msg) + if send == nil { + return fmt.Errorf("no data path for peer %s", remoteID) } - return m.transport.SendSignal(remoteID, msg) + return send(remoteID, msg) } func (m *Manager) binding(remoteID string) Binding { diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index aea902fc4c7..065164a1eea 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -8,20 +8,17 @@ import ( "github.com/stretchr/testify/require" ) -// loopback delivers a sent message synchronously to the peer manager's HandleInbound, -// attributing it to localID (the sender). Both channels deliver the same way — the -// test does not care which physical channel is used. +// loopback is a data-path transport: a SendDataPath delivers synchronously to the +// peer manager's OnDataPathMessage, attributing it to localID (the sender). The +// signalling channel is driven by the test directly via the SignalX methods. type loopback struct { localID string peer *Manager } -func (l *loopback) SendDataPath(remoteID string, msg []byte) error { return l.deliver(msg) } -func (l *loopback) SendSignal(remoteID string, msg []byte) error { return l.deliver(msg) } - -func (l *loopback) deliver(msg []byte) error { +func (l *loopback) SendDataPath(remoteID string, msg []byte) error { cp := append([]byte(nil), msg...) - return l.peer.HandleInbound(l.localID, cp) + return l.peer.OnDataPathMessage(l.localID, cp) } type fakeWG struct { @@ -58,23 +55,30 @@ func TestManager_ExchangeConverges(t *testing.T) { wgA := newFakeWG() wgB := newFakeWG() - // long interval so the ticker never fires during the test; we drive manually. dA := NewManager("aaaa", lbA, wgA, time.Hour, nil) dB := NewManager("bbbb", lbB, wgB, time.Hour, nil) - lbA.peer = dB // A sends -> B receives - lbB.peer = dA // B sends -> A receives + lbA.peer = dB // A's data-path sends -> B receives + lbB.peer = dA // B's data-path sends -> A receives dA.AddPeer("bbbb") dB.AddPeer("aaaa") defer dA.Stop() defer dB.Stop() - // B is the initiator ("bbbb" > "aaaa"). Offer/answer flow synchronously over the - // loopback; both commit their PSK, and B parks in stateAwaitingRekey. - require.NoError(t, dB.initiateRekey("aaaa")) + // Bootstrap over the signalling channel (the test plays the host carrying bytes). + // B is the initiator ("bbbb" > "aaaa"). + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) + + answer, err := dA.SignalOnOffer("bbbb", offer) + require.NoError(t, err) + require.NotNil(t, answer) - // The consumer reports the data path is (re)keyed on both sides; this makes B - // send the confirm, which converges A. + require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) + + // Data path comes up on both sides; this makes B send the confirm over the data + // path, converging A. dA.OnDataPathRekeyed("bbbb") dB.OnDataPathRekeyed("aaaa") @@ -85,16 +89,15 @@ func TestManager_ExchangeConverges(t *testing.T) { require.Equal(t, pskB, pskA, "both sides converge on the same PSK") } -func TestManager_NonInitiatorDoesNothing(t *testing.T) { - lbA := &loopback{localID: "aaaa"} - wgA := newFakeWG() - dA := NewManager("aaaa", lbA, wgA, time.Hour, nil) +func TestManager_NonInitiatorReturnsNoOffer(t *testing.T) { + dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), time.Hour, nil) dA.AddPeer("bbbb") defer dA.Stop() - // A is NOT the initiator vs "bbbb" -> initiateRekey is a no-op, no Send. - require.NoError(t, dA.initiateRekey("bbbb")) - require.Equal(t, PSK{}, wgA.psk("bbbb")) + // A is NOT the initiator vs "bbbb" -> no offer to send. + offer, err := dA.SignalOffer("bbbb") + require.NoError(t, err) + require.Nil(t, offer) } func TestManager_StopIsIdempotent(t *testing.T) { From 0aeed6ae5b8fd77baa79d7e6c22ca9421fbb2bc8 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 23 Jul 2026 17:37:39 +0200 Subject: [PATCH 016/119] Removes confirm. Uses next offer to deliver confirmation/ack of previous round We clock the next Offer initiation to the OnDataPathRekeyed, so we have 2 minutes ahead of us to do our attempts and stuff before to give up. On failure, we will know because we will not receive a new answer.. but more importantly the wg handshake will fail :D --- client/internal/pqkem/convergence.go | 81 +++++------ client/internal/pqkem/convergence_test.go | 22 ++- client/internal/pqkem/manager.go | 168 +++++++--------------- client/internal/pqkem/manager_test.go | 82 ++++++----- client/internal/pqkem/message.go | 67 ++++----- client/internal/pqkem/message_test.go | 13 +- 6 files changed, 177 insertions(+), 256 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 26c064e0a41..83650d81537 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -5,11 +5,11 @@ import ( "time" ) -// startExchange creates a fresh initiator exchange and returns the framed offer for -// the caller to send (pushed over the data path for a rekey, or handed to the host -// for the signalling channel when viaSignal is set). Any previous in-flight exchange -// for the peer is cancelled. -func (m *Manager) startExchange(remoteID string, viaSignal bool) ([]byte, error) { +// startExchange creates a fresh initiator exchange (acknowledging ackID, zero for a +// bootstrap) and returns the framed offer for the caller to send — pushed over the +// data path for a chained rekey, or handed to the host for signalling when viaSignal +// is set. Any previous in-flight exchange for the peer is cancelled. +func (m *Manager) startExchange(remoteID string, viaSignal bool, ackID ExchangeID) ([]byte, error) { init, err := NewInitiator() if err != nil { return nil, err @@ -18,7 +18,7 @@ func (m *Manager) startExchange(remoteID string, viaSignal bool) ([]byte, error) if err != nil { return nil, err } - raw, err := (&OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}).Encode() + raw, err := (&OfferMsg{ExchangeID: id, AckID: ackID, KEMOffer: init.Offer()}).Encode() if err != nil { return nil, err } @@ -44,10 +44,15 @@ func (m *Manager) startExchange(remoteID string, viaSignal bool) ([]byte, error) return raw, nil } -// processOffer (responder) derives the PSK, commits it optimistically, and returns -// the framed answer for the caller to send. A duplicate offer (same exchangeID) -// returns the cached answer without re-deriving; a still-reserved slot returns nil. +// processOffer (responder) first acknowledges the previous exchange the offer names +// (that offer riding the data path under the freshly adopted key proves it worked), +// then derives the PSK for the new offer, commits it optimistically, and returns the +// framed answer. A duplicate offer returns the cached answer without re-deriving. func (m *Manager) processOffer(remoteID string, o *OfferMsg) ([]byte, error) { + if o.AckID != (ExchangeID{}) { + m.ackConverged(remoteID, o.AckID) + } + m.mu.Lock() if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { state, last := ex.state, ex.lastSent @@ -76,7 +81,7 @@ func (m *Manager) processOffer(remoteID string, o *OfferMsg) ([]byte, error) { m.mu.Unlock() return nil, nil } - ex.state = stateAwaitingConfirm + ex.state = stateAwaitingAck ex.lastSent = raw ex.pendingPSK = psk m.mu.Unlock() @@ -88,9 +93,9 @@ func (m *Manager) processOffer(remoteID string, o *OfferMsg) ([]byte, error) { return raw, nil } -// processAnswer (initiator) derives and commits the PSK, then parks in -// stateAwaitingRekey; the confirm is sent later, over the data path, from -// OnDataPathRekeyed. Only valid in stateAwaitingAnswer; advancing the state under the +// processAnswer (initiator) derives and commits the PSK and parks in +// stateAwaitingRekey; the next offer (chained from OnDataPathRekeyed) will acknowledge +// this exchange. Only valid in stateAwaitingAnswer; advancing the state under the // lock makes a concurrent/duplicate answer bail. func (m *Manager) processAnswer(remoteID string, a *AnswerMsg) error { m.mu.Lock() @@ -108,40 +113,45 @@ func (m *Manager) processAnswer(remoteID string, a *AnswerMsg) error { if err != nil { return err } + + // The initiator has converged: the responder must have derived the key to answer. + m.mu.Lock() + m.established[remoteID] = true + m.failures[remoteID] = 0 + m.mu.Unlock() + return m.cbHandler.OnNewPSKReady(remoteID, psk) } -// processConfirm (responder) records convergence. The PSK was already committed in -// processOffer; a confirm arriving over the data path with a matching exchangeID -// proves we operate on the new key from this exchange. Stale/duplicate confirms find -// the exchange gone and are ignored. -func (m *Manager) processConfirm(remoteID string, c *ConfirmMsg) error { +// ackConverged (responder) records convergence of the exchange named by ackID: a +// later offer acknowledging it proves both sides operate on that exchange's key. Only +// acts on a matching stateAwaitingAck exchange; anything else is ignored. +func (m *Manager) ackConverged(remoteID string, ackID ExchangeID) { m.mu.Lock() ex := m.exchanges[remoteID] - if ex == nil || ex.id != c.ExchangeID || ex.state != stateAwaitingConfirm { + if ex == nil || ex.id != ackID || ex.state != stateAwaitingAck { m.mu.Unlock() - return nil + return } delete(m.exchanges, remoteID) m.established[remoteID] = true m.failures[remoteID] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) m.mu.Unlock() - return nil } // initiatorLoop enforces the convergence deadline and retransmits the initiator's -// outstanding DATA-PATH message: it resends the offer while awaiting the answer only -// when the offer went over the data path (a signalling-bootstrapped offer is -// retransmitted by the host with its own negotiation); it waits, counting toward the -// deadline, while awaiting the data-path rekey; it resends the confirm a few times -// once sent. Exhausting the deadline before the confirm phase is a failure. +// outstanding data-path offer while awaiting the answer (a signalling-bootstrapped +// offer is retransmitted by the host, so it is not resent here). It then waits, +// counting toward the deadline, in stateAwaitingRekey until OnDataPathRekeyed chains +// the next exchange (which supersedes and cancels this loop). Exhausting the deadline +// is a failure. func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id ExchangeID) { defer m.wait.Done() t := time.NewTicker(m.retryInterval) defer t.Stop() - attempts, confirmsSent := 0, 0 + attempts := 0 for { select { case <-ctx.Done(): @@ -174,8 +184,8 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang } case stateAwaitingRekey: - // Waiting for OnDataPathRekeyed; nothing to send, but the deadline - // still applies (the data path may never come up with the new key). + // Waiting for OnDataPathRekeyed to chain the next exchange; the + // deadline still applies (the data path may never adopt the new key). if attempts >= m.maxRetries { delete(m.exchanges, remoteID) fail := m.registerFailureLocked(remoteID) @@ -186,19 +196,6 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang attempts++ m.mu.Unlock() - case stateConfirming: - if confirmsSent >= confirmRetransmits { - delete(m.exchanges, remoteID) - m.mu.Unlock() - return - } - msg := ex.lastSent - confirmsSent++ - m.mu.Unlock() - if err := m.pushDataPath(remoteID, msg); err != nil { - m.logger.Warn("pqkem confirm retransmit failed", "peer", remoteID, "err", err) - } - default: m.mu.Unlock() return diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index 1b9e075a54b..66877c9fae3 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -30,10 +30,9 @@ func (g *gate) SendDataPath(remoteID string, msg []byte) error { func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { wg := newFakeWG() - d := NewManager("bbbb", dropTransport{}, wg, time.Hour, nil) // bbbb > aaaa -> initiator + d := NewManager("bbbb", dropTransport{}, wg, nil) // bbbb > aaaa -> initiator d.retryInterval = 5 * time.Millisecond d.maxRetries = 3 - d.AddPeer("aaaa") defer d.Stop() // Bootstrap offer is produced for signalling; no answer ever comes back -> the @@ -55,41 +54,38 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { wgA := newFakeWG() wgB := newFakeWG() - dA := NewManager("aaaa", gA, wgA, time.Hour, nil) - dB := NewManager("bbbb", gB, wgB, time.Hour, nil) + dA := NewManager("aaaa", gA, wgA, nil) + dB := NewManager("bbbb", gB, wgB, nil) gA.peer = dB gB.peer = dA dB.retryInterval = 5 * time.Millisecond dB.maxRetries = 2 - dA.AddPeer("bbbb") - dB.AddPeer("aaaa") defer dA.Stop() defer dB.Stop() - // Establish via a signalling bootstrap + data-path-rekeyed, so B becomes - // established and its data path is up. + // Bootstrap over signalling -> B becomes established. offer, err := dB.SignalOffer("aaaa") require.NoError(t, err) answer, err := dA.SignalOnOffer("bbbb", offer) require.NoError(t, err) require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) - dA.OnDataPathRekeyed("bbbb") - dB.OnDataPathRekeyed("aaaa") require.NotEqual(t, PSK{}, wgB.psk("aaaa")) - // Now drop B's data-path delivery: rekeys can no longer converge. + // Bring the data path up on both, then drop B's delivery so rekeys can't converge. + dA.OnDataPathRekeyed("bbbb") + dB.OnDataPathRekeyed("aaaa") gB.drop.Store(true) // K-1 data-path rekeys must NOT raise OnRekeyFailed. for i := 0; i < DefaultMaxRekeyFailures-1; i++ { - _, err := dB.startExchange("aaaa", false) + _, err := dB.startExchange("aaaa", false, ExchangeID{}) require.NoError(t, err) time.Sleep(50 * time.Millisecond) } require.Equal(t, 0, failedCount(wgB), "no failure before K attempts") // The K-th failure raises it once. - _, err = dB.startExchange("aaaa", false) + _, err = dB.startExchange("aaaa", false, ExchangeID{}) require.NoError(t, err) require.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 675d88956e5..e3816537f50 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -10,12 +10,8 @@ import ( ) const ( - // DefaultRekeyInterval is the default PSK rotation cadence (~2 min), chosen so a - // rotated PSK is adopted by the consumer's next transport handshake without - // forcing one. - DefaultRekeyInterval = 2 * time.Minute // DefaultRetryInterval is how often the initiator retransmits its outstanding - // data-path message while an exchange is in flight. + // data-path offer while awaiting the answer. DefaultRetryInterval = 2 * time.Second // DefaultMaxRetries bounds how many ticks an exchange may run before it is // declared failed. The convergence deadline is thus MaxRetries * RetryInterval. @@ -23,16 +19,12 @@ const ( // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures // are tolerated before OnRekeyFailed. The initial exchange fails immediately. DefaultMaxRekeyFailures = 3 - // confirmRetransmits is how many times the initiator best-effort resends the - // confirm over the data path, to cover its loss. - confirmRetransmits = 3 ) // Transport pushes a message over the peer's data path (e.g. a WireGuard tunnel). // It is the library's only outbound send: the control-plane (signalling) channel is // host-driven — the library hands the host offer/answer payloads to piggyback on the -// host's own offer/answer, it never pushes there itself (that would drive the host's -// connection negotiation, which is not the library's to control). +// host's own negotiation, it never pushes there itself. type Transport interface { SendDataPath(remoteID string, msg []byte) error } @@ -41,20 +33,19 @@ type Transport interface { type exchangeState uint8 const ( - stateReserved exchangeState = iota // responder: deriving the answer - stateAwaitingAnswer // initiator: offer sent, awaiting the answer - stateAwaitingRekey // initiator: PSK derived+set, awaiting OnDataPathRekeyed to send the confirm - stateConfirming // initiator: confirm sent over the data path, best-effort retransmit - stateAwaitingConfirm // responder: answer sent, awaiting the confirm over the data path + stateReserved exchangeState = iota // responder: deriving the answer + stateAwaitingAnswer // initiator: offer sent, awaiting the answer + stateAwaitingRekey // initiator: PSK derived+set, awaiting OnDataPathRekeyed to chain the next offer + stateAwaitingAck // responder: answer sent, awaiting the next offer that acks this exchange ) // exchangeCtl holds all state for one in-flight exchange with a peer, under the // Manager's single lock. state drives every decision. lastSent is the current -// data-path retransmit payload. initiator is the ephemeral handle used at Finish; -// pendingPSK is the responder's derived key. viaSignal records that the offer was -// handed to the host for the signalling channel (bootstrap), so the loop does not -// retransmit it on the data path (the host retransmits it with its own negotiation). -// Only the initiator runs a retransmit loop, so only it sets cancel. +// data-path retransmit payload (the offer, for the initiator). initiator is the +// ephemeral handle used at Finish; pendingPSK is the responder's derived key. +// viaSignal records that the offer went to the host for the signalling channel, so +// the loop does not retransmit it on the data path. Only the initiator runs a +// retransmit loop, so only it sets cancel. type exchangeCtl struct { id ExchangeID state exchangeState @@ -67,16 +58,17 @@ type exchangeCtl struct { } // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It -// runs the per-peer rekey timer, drives the X25519MLKEM768 exchange, and surfaces the -// derived PSK and convergence to the host via CallbackHandler. The cryptography is -// the pure kem.go primitives; all state lives here under one lock. +// drives the X25519MLKEM768 exchange and surfaces the derived PSK and convergence to +// the host via CallbackHandler. It is fully event-driven: the bootstrap is triggered +// by the host (SignalOffer) and each rotation is clocked by OnDataPathRekeyed (the +// consumer's transport rekey). The cryptography is the pure kem.go primitives; all +// state lives here under one lock. type Manager struct { localID string transport Transport cbHandler CallbackHandler logger *slog.Logger - rekeyInterval time.Duration retryInterval time.Duration maxRetries int maxRekeyFailures int @@ -85,24 +77,20 @@ type Manager struct { rootCancel context.CancelFunc mu sync.Mutex - peers map[string]context.CancelFunc // per-peer rekey loop - exchanges map[string]*exchangeCtl // in-flight exchange per peer - established map[string]bool // peer has completed at least one exchange - failures map[string]int // consecutive rekey failures per peer - // dataSend holds the peer's data-path sender when the data path is up; nil (absent) - // means it is down. Toggled by OnDataPathRekeyed / OnDataPathDown. Its presence is - // the "a data path exists" signal. + exchanges map[string]*exchangeCtl // in-flight exchange per peer + established map[string]bool // peer has completed at least one exchange + failures map[string]int // consecutive rekey failures per peer + // dataSend holds the peer's data-path sender when the data path is up; nil + // (absent) means it is down. Toggled by OnDataPathRekeyed / OnDataPathDown. dataSend map[string]func(string, []byte) error wait sync.WaitGroup } // NewManager builds a manager for the local peer identified by its peer identity key -// (used for the deterministic initiator role and the identity binding). A zero -// interval falls back to DefaultRekeyInterval; a nil logger to slog.Default(). -func NewManager(localID string, t Transport, h CallbackHandler, interval time.Duration, logger *slog.Logger) *Manager { - if interval <= 0 { - interval = DefaultRekeyInterval - } +// (used for the deterministic initiator role and the identity binding). A nil logger +// falls back to slog.Default(). Retry/retries/K use their defaults and can be +// overridden before use. +func NewManager(localID string, t Transport, h CallbackHandler, logger *slog.Logger) *Manager { if logger == nil { logger = slog.Default() } @@ -112,13 +100,11 @@ func NewManager(localID string, t Transport, h CallbackHandler, interval time.Du transport: t, cbHandler: h, logger: logger, - rekeyInterval: interval, retryInterval: DefaultRetryInterval, maxRetries: DefaultMaxRetries, maxRekeyFailures: DefaultMaxRekeyFailures, rootCtx: ctx, rootCancel: cancel, - peers: make(map[string]context.CancelFunc), exchanges: make(map[string]*exchangeCtl), established: make(map[string]bool), failures: make(map[string]int), @@ -133,26 +119,9 @@ func (m *Manager) IsInitiator(remoteID string) bool { return m.localID > remoteID } -// AddPeer registers a remote peer and starts its rekey timer. Re-adding is a no-op. -func (m *Manager) AddPeer(remoteID string) { - m.mu.Lock() - defer m.mu.Unlock() - if _, ok := m.peers[remoteID]; ok { - return - } - ctx, cancel := context.WithCancel(m.rootCtx) - m.peers[remoteID] = cancel - m.wait.Add(1) - go m.rekeyLoop(ctx, remoteID) -} - -// RemovePeer stops a peer's rekey timer and any in-flight exchange, and drops state. +// RemovePeer stops any in-flight exchange for a peer and drops its state. func (m *Manager) RemovePeer(remoteID string) { m.mu.Lock() - if cancel, ok := m.peers[remoteID]; ok { - cancel() - delete(m.peers, remoteID) - } if ex, ok := m.exchanges[remoteID]; ok { if ex.cancel != nil { ex.cancel() @@ -165,23 +134,21 @@ func (m *Manager) RemovePeer(remoteID string) { m.mu.Unlock() } -// Stop cancels all timers and in-flight exchanges and waits for goroutines to exit. +// Stop cancels all in-flight exchanges and waits for their goroutines to exit. func (m *Manager) Stop() { m.rootCancel() m.wait.Wait() m.mu.Lock() - m.peers = make(map[string]context.CancelFunc) m.exchanges = make(map[string]*exchangeCtl) m.mu.Unlock() } -// ---- Signalling channel (host-driven; rides the host's offer/answer) ---- +// ---- Signalling channel (host-driven; rides the host's negotiation) ---- // SignalOffer returns the KEM offer for the host to embed in its outgoing offer to // remoteID (bootstrap). It returns (nil, nil) when the local peer is not the -// initiator. It is idempotent for an in-flight bootstrap: a repeat call (e.g. the -// host retransmitting its offer) returns the same offer rather than starting a new -// exchange. +// initiator. It is idempotent for an in-flight bootstrap: a repeat call returns the +// same offer rather than starting a new exchange. func (m *Manager) SignalOffer(remoteID string) ([]byte, error) { if !m.IsInitiator(remoteID) { return nil, nil @@ -193,7 +160,8 @@ func (m *Manager) SignalOffer(remoteID string) ([]byte, error) { return last, nil } m.mu.Unlock() - return m.startExchange(remoteID, true) + // bootstrap offer acknowledges nothing (zero AckID). + return m.startExchange(remoteID, true, ExchangeID{}) } // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and @@ -210,7 +178,7 @@ func (m *Manager) SignalOnOffer(remoteID string, offer []byte) ([]byte, error) { } // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. -// There is no reply: the confirm rides the data path after OnDataPathRekeyed. +// There is no reply: the next offer (over the data path) acknowledges this exchange. func (m *Manager) SignalOnAnswer(remoteID string, answer []byte) error { typ, msg, err := Decode(answer) if err != nil { @@ -243,46 +211,42 @@ func (m *Manager) OnDataPathMessage(remoteID string, raw []byte) error { return m.pushDataPath(remoteID, answer) case MsgAnswer: return m.processAnswer(remoteID, msg.(*AnswerMsg)) - case MsgConfirm: - return m.processConfirm(remoteID, msg.(*ConfirmMsg)) default: return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } } // OnDataPathRekeyed notifies that the peer's data path is up and freshly keyed with -// the latest PSK (fired on first establishment AND every rekey — the same event). It -// marks the data path usable and, if we are the initiator waiting to confirm, sends -// the confirm over the data path (its arrival proves to the responder that we operate -// on the new key, correlated by exchangeID). +// the latest PSK (fired on first establishment AND every rekey). It marks the data +// path usable and, if we are the initiator that just derived a PSK, chains the next +// exchange: a fresh offer over the data path that acknowledges the just-completed one +// (its arrival under the new key proves to the responder that the key works). func (m *Manager) OnDataPathRekeyed(remoteID string) { m.mu.Lock() m.dataSend[remoteID] = m.transport.SendDataPath ex := m.exchanges[remoteID] - if ex == nil || ex.state != stateAwaitingRekey { - m.mu.Unlock() + chain := ex != nil && ex.state == stateAwaitingRekey + var ackID ExchangeID + if chain { + ackID = ex.id + } + m.mu.Unlock() + + if !chain { return } - confirm, err := (&ConfirmMsg{ExchangeID: ex.id}).Encode() + offer, err := m.startExchange(remoteID, false, ackID) if err != nil { - m.mu.Unlock() - m.logger.Error("pqkem encode confirm", "peer", remoteID, "err", err) + m.logger.Error("pqkem chain offer failed to start", "peer", remoteID, "err", err) return } - ex.state = stateConfirming - ex.lastSent = confirm - m.established[remoteID] = true - m.failures[remoteID] = 0 - _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) - m.mu.Unlock() - - if err := m.pushDataPath(remoteID, confirm); err != nil { - m.logger.Warn("pqkem send confirm failed", "peer", remoteID, "err", err) + if err := m.pushDataPath(remoteID, offer); err != nil { + m.logger.Warn("pqkem send chain offer failed", "peer", remoteID, "err", err) } } -// OnDataPathDown notifies that the peer's data path went down; further rekeys wait -// until it is up again (the host re-bootstraps over signalling on reconnect). +// OnDataPathDown notifies that the peer's data path went down; rotations pause until +// it is up again (the host re-bootstraps over signalling on reconnect). func (m *Manager) OnDataPathDown(remoteID string) { m.mu.Lock() delete(m.dataSend, remoteID) @@ -291,36 +255,6 @@ func (m *Manager) OnDataPathDown(remoteID string) { // ---- internals ---- -func (m *Manager) rekeyLoop(ctx context.Context, remoteID string) { - defer m.wait.Done() - t := time.NewTicker(m.rekeyInterval) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case <-t.C: - m.mu.Lock() - _, dpUp := m.dataSend[remoteID] - _, inFlight := m.exchanges[remoteID] - m.mu.Unlock() - // Rekeys ride the data path only; skip when it is down (the host will - // re-bootstrap over signalling on reconnect) or an exchange is in flight. - if !dpUp || inFlight || !m.IsInitiator(remoteID) { - continue - } - offer, err := m.startExchange(remoteID, false) - if err != nil { - m.logger.Error("pqkem rekey failed to start", "peer", remoteID, "err", err) - continue - } - if err := m.pushDataPath(remoteID, offer); err != nil { - m.logger.Warn("pqkem send rekey offer failed", "peer", remoteID, "err", err) - } - } - } -} - // pushDataPath sends over the peer's data path, erroring if it is down. func (m *Manager) pushDataPath(remoteID string, msg []byte) error { m.mu.Lock() diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index 065164a1eea..a11ede6a2b3 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -3,14 +3,13 @@ package pqkem import ( "sync" "testing" - "time" "github.com/stretchr/testify/require" ) -// loopback is a data-path transport: a SendDataPath delivers synchronously to the -// peer manager's OnDataPathMessage, attributing it to localID (the sender). The -// signalling channel is driven by the test directly via the SignalX methods. +// loopback is a data-path transport: SendDataPath delivers synchronously to the peer +// manager's OnDataPathMessage, attributing it to localID (the sender). The signalling +// channel is driven by the test directly via the SignalX methods. type loopback struct { localID string peer *Manager @@ -49,60 +48,75 @@ func (f *fakeWG) psk(peer string) PSK { return f.psks[peer] } -func TestManager_ExchangeConverges(t *testing.T) { +// pair builds two wired managers (B is the initiator, "bbbb" > "aaaa"). +func pair(t *testing.T) (dA, dB *Manager, wgA, wgB *fakeWG) { + t.Helper() lbA := &loopback{localID: "aaaa"} lbB := &loopback{localID: "bbbb"} - wgA := newFakeWG() - wgB := newFakeWG() - - dA := NewManager("aaaa", lbA, wgA, time.Hour, nil) - dB := NewManager("bbbb", lbB, wgB, time.Hour, nil) - lbA.peer = dB // A's data-path sends -> B receives - lbB.peer = dA // B's data-path sends -> A receives - - dA.AddPeer("bbbb") - dB.AddPeer("aaaa") - defer dA.Stop() - defer dB.Stop() + wgA = newFakeWG() + wgB = newFakeWG() + dA = NewManager("aaaa", lbA, wgA, nil) + dB = NewManager("bbbb", lbB, wgB, nil) + lbA.peer = dB + lbB.peer = dA + return dA, dB, wgA, wgB +} - // Bootstrap over the signalling channel (the test plays the host carrying bytes). - // B is the initiator ("bbbb" > "aaaa"). +// bootstrap runs the signalling offer/answer (the test plays the host carrying bytes). +func bootstrap(t *testing.T, dA, dB *Manager) { + t.Helper() offer, err := dB.SignalOffer("aaaa") require.NoError(t, err) require.NotNil(t, offer) - answer, err := dA.SignalOnOffer("bbbb", offer) require.NoError(t, err) require.NotNil(t, answer) - require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) +} + +func TestManager_BootstrapDerivesSamePSK(t *testing.T) { + dA, dB, wgA, wgB := pair(t) + defer dA.Stop() + defer dB.Stop() - // Data path comes up on both sides; this makes B send the confirm over the data - // path, converging A. + bootstrap(t, dA, dB) + + pskA := wgA.psk("bbbb") + pskB := wgB.psk("aaaa") + require.NotEqual(t, PSK{}, pskA) + require.Equal(t, pskB, pskA, "both sides derive the same PSK from the bootstrap exchange") +} + +func TestManager_ChainRotatesAndAcks(t *testing.T) { + dA, dB, wgA, wgB := pair(t) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + psk1 := wgB.psk("aaaa") + + // Data path up on both sides; B chains the next offer (acking exchange 1) over the + // data path, which rotates both to a fresh PSK and acknowledges A. dA.OnDataPathRekeyed("bbbb") dB.OnDataPathRekeyed("aaaa") - pskB := wgB.psk("aaaa") - pskA := wgA.psk("bbbb") - require.NotEqual(t, PSK{}, pskA, "responder A must have a PSK") - require.NotEqual(t, PSK{}, pskB, "initiator B must have a PSK") - require.Equal(t, pskB, pskA, "both sides converge on the same PSK") + psk2A := wgA.psk("bbbb") + psk2B := wgB.psk("aaaa") + require.Equal(t, psk2B, psk2A, "both sides converge on the rotated PSK") + require.NotEqual(t, psk1, psk2B, "the chain rotated to a new PSK") } func TestManager_NonInitiatorReturnsNoOffer(t *testing.T) { - dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), time.Hour, nil) - dA.AddPeer("bbbb") + dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), nil) defer dA.Stop() - // A is NOT the initiator vs "bbbb" -> no offer to send. - offer, err := dA.SignalOffer("bbbb") + offer, err := dA.SignalOffer("bbbb") // not the initiator vs "bbbb" require.NoError(t, err) require.Nil(t, offer) } func TestManager_StopIsIdempotent(t *testing.T) { - dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), time.Hour, nil) - dA.AddPeer("bbbb") + dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), nil) dA.Stop() dA.Stop() // must not panic or hang } diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go index d1c6e69a469..312279d80b9 100644 --- a/client/internal/pqkem/message.go +++ b/client/internal/pqkem/message.go @@ -6,39 +6,45 @@ import ( ) // Wire framing for the PQ-KEM exchange. Messages are self-contained, versioned, -// transport-agnostic byte blobs: the same bytes ride the Signal offer/answer -// (initial, pre-tunnel) or a data-tunnel packet (rekey). They are NOT a gRPC -// service — the network layer only sees opaque []byte. +// transport-agnostic byte blobs: the same bytes ride the signalling channel +// (initial bootstrap) or a data-tunnel packet (rekey). The library only ever sees +// opaque []byte at the transport seam. // // Layout (all messages): [type:1][version:1][exchangeID:16][payload...] +// +// There is no confirm message: an exchange is acknowledged by the NEXT offer, which +// carries the acked exchange's id (see OfferMsg.AckID) and — riding the data path +// under the freshly adopted key — proves that key works. const ( // ProtocolVersion is bumped on any wire-incompatible change; a peer rejects // messages it does not understand rather than misparsing them. ProtocolVersion uint8 = 1 - // ExchangeIDSize identifies one offer/answer/confirm round so stale answers - // (e.g. an answer to a pre-restart offer) are dropped instead of applied. + // ExchangeIDSize identifies one exchange so answers/acks correlate and stale + // messages are dropped. ExchangeIDSize = 16 headerSize = 1 + 1 + ExchangeIDSize ) -// MsgType tags the three message kinds of the exchange. +// MsgType tags the two message kinds of the exchange. type MsgType uint8 const ( MsgOffer MsgType = iota + 1 MsgAnswer - MsgConfirm ) -// ExchangeID is the per-round correlator echoed by the answer and the confirm. +// ExchangeID is the per-exchange correlator. The zero value means "none" (an offer +// that acknowledges nothing, i.e. the first exchange of a connection). type ExchangeID [ExchangeIDSize]byte -// OfferMsg carries the initiator's public material (X25519 pub ‖ ML-KEM encap key). +// OfferMsg carries the initiator's public material (X25519 pub ‖ ML-KEM encap key) +// and AckID, the id of the previous exchange this offer acknowledges (zero if none). type OfferMsg struct { ExchangeID ExchangeID + AckID ExchangeID // KEMOffer is the raw Initiator.Offer() blob (OfferSize bytes). KEMOffer []byte } @@ -51,27 +57,15 @@ type AnswerMsg struct { KEMAnswer []byte } -// ConfirmMsg is sent ONE WAY, initiator -> responder, over the tunnel under the -// NEW PSK, for the round identified by ExchangeID. It carries no key material. -// -// It is one-way because of the knowledge asymmetry of a KEM: the initiator learns -// the responder holds the key simply by receiving the answer (the responder had to -// derive it to encapsulate), so the initiator needs no confirmation. Only the -// responder is left unsure whether the initiator received the answer and committed -// the key — this message resolves that. As a bonus, being sent under the new PSK it -// exercises the consumer's channel handshake with the new key: the responder -// converges on receiving it, and the initiator converges by observing that handshake -// succeed (which fails on a PSK mismatch, so success proves the responder committed too). -type ConfirmMsg struct { - ExchangeID ExchangeID -} - -// Encode serialises the offer with its framed header. +// Encode serialises the offer with its framed header (payload = AckID ‖ KEMOffer). func (m *OfferMsg) Encode() ([]byte, error) { if len(m.KEMOffer) != OfferSize { return nil, fmt.Errorf("offer payload: got %d, want %d", len(m.KEMOffer), OfferSize) } - return frame(MsgOffer, m.ExchangeID, m.KEMOffer), nil + payload := make([]byte, 0, ExchangeIDSize+OfferSize) + payload = append(payload, m.AckID[:]...) + payload = append(payload, m.KEMOffer...) + return frame(MsgOffer, m.ExchangeID, payload), nil } // Encode serialises the answer with its framed header. @@ -82,13 +76,7 @@ func (m *AnswerMsg) Encode() ([]byte, error) { return frame(MsgAnswer, m.ExchangeID, m.KEMAnswer), nil } -// Encode serialises the confirm with its framed header. It returns an error only -// for signature uniformity with the other messages; it never actually fails. -func (m *ConfirmMsg) Encode() ([]byte, error) { - return frame(MsgConfirm, m.ExchangeID, nil), nil -} - -// Decode parses a framed message into one of *OfferMsg / *AnswerMsg / *ConfirmMsg. +// Decode parses a framed message into one of *OfferMsg / *AnswerMsg. func Decode(buf []byte) (MsgType, any, error) { if len(buf) < headerSize { return 0, nil, fmt.Errorf("message too short: %d bytes", len(buf)) @@ -104,20 +92,17 @@ func Decode(buf []byte) (MsgType, any, error) { switch typ { case MsgOffer: - if len(payload) != OfferSize { - return typ, nil, fmt.Errorf("offer payload: got %d, want %d", len(payload), OfferSize) + if len(payload) != ExchangeIDSize+OfferSize { + return typ, nil, fmt.Errorf("offer payload: got %d, want %d", len(payload), ExchangeIDSize+OfferSize) } - return typ, &OfferMsg{ExchangeID: id, KEMOffer: payload}, nil + var ack ExchangeID + copy(ack[:], payload[:ExchangeIDSize]) + return typ, &OfferMsg{ExchangeID: id, AckID: ack, KEMOffer: payload[ExchangeIDSize:]}, nil case MsgAnswer: if len(payload) != AnswerSize { return typ, nil, fmt.Errorf("answer payload: got %d, want %d", len(payload), AnswerSize) } return typ, &AnswerMsg{ExchangeID: id, KEMAnswer: payload}, nil - case MsgConfirm: - if len(payload) != 0 { - return typ, nil, fmt.Errorf("confirm payload must be empty, got %d bytes", len(payload)) - } - return typ, &ConfirmMsg{ExchangeID: id}, nil default: return typ, nil, fmt.Errorf("unknown message type %d", typ) } diff --git a/client/internal/pqkem/message_test.go b/client/internal/pqkem/message_test.go index 8708695d507..524125e4794 100644 --- a/client/internal/pqkem/message_test.go +++ b/client/internal/pqkem/message_test.go @@ -13,13 +13,15 @@ func TestMessageRoundTrip(t *testing.T) { require.NoError(t, err) id := ExchangeID{1, 2, 3, 4} + ack := ExchangeID{9, 9, 9} - offBytes, err := (&OfferMsg{ExchangeID: id, KEMOffer: init.Offer()}).Encode() + offBytes, err := (&OfferMsg{ExchangeID: id, AckID: ack, KEMOffer: init.Offer()}).Encode() require.NoError(t, err) typ, decoded, err := Decode(offBytes) require.NoError(t, err) require.Equal(t, MsgOffer, typ) require.Equal(t, id, decoded.(*OfferMsg).ExchangeID) + require.Equal(t, ack, decoded.(*OfferMsg).AckID) require.Equal(t, init.Offer(), decoded.(*OfferMsg).KEMOffer) ansBytes, err := (&AnswerMsg{ExchangeID: id, KEMAnswer: answer}).Encode() @@ -28,13 +30,6 @@ func TestMessageRoundTrip(t *testing.T) { require.NoError(t, err) require.Equal(t, MsgAnswer, typ) require.Equal(t, answer, decoded.(*AnswerMsg).KEMAnswer) - - confBytes, err := (&ConfirmMsg{ExchangeID: id}).Encode() - require.NoError(t, err) - typ, decoded, err = Decode(confBytes) - require.NoError(t, err) - require.Equal(t, MsgConfirm, typ) - require.Equal(t, id, decoded.(*ConfirmMsg).ExchangeID) } func TestDecodeRejects(t *testing.T) { @@ -43,7 +38,7 @@ func TestDecodeRejects(t *testing.T) { require.Error(t, err) // wrong version - bad := make([]byte, headerSize+OfferSize) + bad := make([]byte, headerSize+ExchangeIDSize+OfferSize) bad[0] = byte(MsgOffer) bad[1] = ProtocolVersion + 1 _, _, err = Decode(bad) From 6a644df59b5d1a3cb5f42e4a9616f79403a8a2ee Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 15:50:59 +0200 Subject: [PATCH 017/119] Protocol update --- shared/signal/proto/signalexchange.pb.go | 85 ++++++++++++++---------- shared/signal/proto/signalexchange.proto | 7 ++ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/shared/signal/proto/signalexchange.pb.go b/shared/signal/proto/signalexchange.pb.go index 8e07977f07f..a08439f7cce 100644 --- a/shared/signal/proto/signalexchange.pb.go +++ b/shared/signal/proto/signalexchange.pb.go @@ -239,6 +239,12 @@ type Body struct { // fallback dial target when DNS resolution of relayServerAddress fails. // SNI/TLS verification still uses relayServerAddress. RelayServerIP []byte `protobuf:"bytes,11,opt,name=relayServerIP,proto3,oneof" json:"relayServerIP,omitempty"` + // mlkemPayload carries a post-quantum X25519MLKEM768 handshake message that + // seeds the WireGuard PSK, riding this Body's OFFER/ANSWER: on an OFFER it is + // the KEM offer, on an ANSWER the KEM answer. It is opaque to signal — the + // pqkem library frames and parses it. Absent when the sender does not run the + // ML-KEM PQ exchange; unknown to older clients, which ignore it. + MlkemPayload []byte `protobuf:"bytes,12,opt,name=mlkemPayload,proto3,oneof" json:"mlkemPayload,omitempty"` } func (x *Body) Reset() { @@ -343,6 +349,13 @@ func (x *Body) GetRelayServerIP() []byte { return nil } +func (x *Body) GetMlkemPayload() []byte { + if x != nil { + return x.MlkemPayload + } + return nil +} + // Mode indicates a connection mode type Mode struct { state protoimpl.MessageState @@ -466,7 +479,7 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x52, 0x09, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xd2, 0x04, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x8c, 0x05, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, @@ -494,39 +507,43 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x02, 0x52, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, 0x88, 0x01, 0x01, 0x22, 0x52, 0x0a, 0x04, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, - 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, - 0x49, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, - 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x47, 0x4f, 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x0d, - 0x0a, 0x09, 0x48, 0x45, 0x41, 0x52, 0x54, 0x42, 0x45, 0x41, 0x54, 0x10, 0x06, 0x42, 0x15, 0x0a, - 0x13, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x49, 0x50, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x22, 0x2e, 0x0a, 0x04, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x48, 0x00, 0x52, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x42, - 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x22, 0x6d, 0x0a, 0x0f, 0x52, 0x6f, - 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, - 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, - 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, - 0x70, 0x61, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x32, 0xb9, 0x01, 0x0a, 0x0e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x6c, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x4c, 0x0a, 0x04, - 0x53, 0x65, 0x6e, 0x64, 0x12, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, - 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, - 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, - 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, 0x0d, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, - 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, - 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, 0x88, 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x6d, 0x6c, 0x6b, + 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x48, + 0x03, 0x52, 0x0c, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x88, + 0x01, 0x01, 0x22, 0x52, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x46, + 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52, 0x10, + 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, + 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x47, 0x4f, + 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x48, 0x45, 0x41, 0x52, 0x54, + 0x42, 0x45, 0x41, 0x54, 0x10, 0x06, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x79, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x0c, 0x0a, + 0x0a, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, + 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, 0x42, 0x0f, 0x0a, + 0x0d, 0x5f, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x04, + 0x08, 0x09, 0x10, 0x0a, 0x22, 0x2e, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x06, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x06, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x22, 0x6d, 0x0a, 0x0f, 0x52, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, + 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, + 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, + 0x79, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, + 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, + 0x64, 0x64, 0x72, 0x32, 0xb9, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45, 0x78, + 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x4c, 0x0a, 0x04, 0x53, 0x65, 0x6e, 0x64, 0x12, 0x20, + 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, + 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, + 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, + 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, + 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/shared/signal/proto/signalexchange.proto b/shared/signal/proto/signalexchange.proto index 8c304e37c23..5785ce459c0 100644 --- a/shared/signal/proto/signalexchange.proto +++ b/shared/signal/proto/signalexchange.proto @@ -75,6 +75,13 @@ message Body { // fallback dial target when DNS resolution of relayServerAddress fails. // SNI/TLS verification still uses relayServerAddress. optional bytes relayServerIP = 11; + + // mlkemPayload carries a post-quantum X25519MLKEM768 handshake message that + // seeds the WireGuard PSK, riding this Body's OFFER/ANSWER: on an OFFER it is + // the KEM offer, on an ANSWER the KEM answer. It is opaque to signal — the + // pqkem library frames and parses it. Absent when the sender does not run the + // ML-KEM PQ exchange; unknown to older clients, which ignore it. + optional bytes mlkemPayload = 12; } // Mode indicates a connection mode From 902daf61e6344594b6f8e881d898244fedf17091 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 17:30:13 +0200 Subject: [PATCH 018/119] Invert order of keys as per draft --- client/internal/pqkem/kem.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/client/internal/pqkem/kem.go b/client/internal/pqkem/kem.go index e7e855cd1fb..0995443203c 100644 --- a/client/internal/pqkem/kem.go +++ b/client/internal/pqkem/kem.go @@ -12,10 +12,11 @@ // (their peer identity keys) so the derived key cannot be transplanted // to a different peer pair even if the transport authentication were bypassed. // -// Combiner note: this follows the IETF hybrid layout (X25519 ‖ ML-KEM on the -// wire; ML-KEM_ss ‖ X25519_ss into the KDF) from -// draft-kwiatkowski-tls-ecdhe-mlkem. The spike uses SHA-256 as the KDF; a -// production version should use HKDF with the RFC labels — see TODO below. +// Combiner note: this follows draft-ietf-tls-ecdhe-mlkem for X25519MLKEM768 — on +// the wire ML-KEM ‖ X25519 (the draft deliberately reversed the share order for +// this group), and ML-KEM_ss ‖ X25519_ss fed into the KDF. The spike uses SHA-256 +// (also binding the transcript and peer identities); a production version should +// use HKDF — see TODO below. package pqkem import ( @@ -27,8 +28,9 @@ import ( ) const ( - // OfferSize is the initiator message: X25519 public key ‖ ML-KEM-768 encapsulation key. - OfferSize = 32 + mlkem.EncapsulationKeySize768 // 1216 + // OfferSize is the initiator message: ML-KEM-768 encapsulation key ‖ X25519 public key + // (share order per draft-ietf-tls-ecdhe-mlkem for X25519MLKEM768). + OfferSize = mlkem.EncapsulationKeySize768 + 32 // 1216 // AnswerSize is the responder message: ML-KEM-768 ciphertext ‖ X25519 public key. AnswerSize = mlkem.CiphertextSize768 + 32 // 1120 @@ -64,8 +66,8 @@ func NewInitiator() (*Initiator, error) { } offer := make([]byte, 0, OfferSize) - offer = append(offer, x.PublicKey().Bytes()...) offer = append(offer, dk.EncapsulationKey().Bytes()...) + offer = append(offer, x.PublicKey().Bytes()...) return &Initiator{x25519: x, mlkemDK: dk, offer: offer}, nil } @@ -104,8 +106,8 @@ func Respond(offer []byte, b Binding) (answer []byte, psk PSK, err error) { if len(offer) != OfferSize { return nil, PSK{}, fmt.Errorf("offer: got %d bytes, want %d", len(offer), OfferSize) } - peerX := offer[:32] - peerEK := offer[32:] + peerEK := offer[:mlkem.EncapsulationKeySize768] + peerX := offer[mlkem.EncapsulationKeySize768:] ek, err := mlkem.NewEncapsulationKey768(peerEK) if err != nil { From 30212ca92c9ca5a97575a89fe5301aed68b3e063 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 17:38:12 +0200 Subject: [PATCH 019/119] Adds MLKEM Payload placeholder to client internals --- client/internal/engine.go | 1 + client/internal/peer/handshaker.go | 6 ++++++ client/internal/peer/signaler.go | 1 + shared/signal/client/client.go | 6 ++++++ 4 files changed, 14 insertions(+) diff --git a/client/internal/engine.go b/client/internal/engine.go index d517d1d68d1..4dff5af0ab5 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -2997,6 +2997,7 @@ func convertToOfferAnswer(msg *sProto.Message) (*peer.OfferAnswer, error) { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + MlkemPayload: msg.GetBody().GetMlkemPayload(), RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), RelaySrvIP: relayIP, SessionID: sessionID, diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 6ecb2a94709..d791aa64fe9 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -39,6 +39,12 @@ type OfferAnswer struct { // This value is the local Rosenpass server address when sending the message RosenpassAddr string + // MlkemPayload carries the post-quantum X25519MLKEM768 handshake message + // (pqkem-framed offer on an OFFER, answer on an ANSWER) that seeds the + // WireGuard PSK. Opaque here — the pqkem library frames and parses it. Nil + // when the peer does not run the ML-KEM PQ exchange. + MlkemPayload []byte + // relay server address RelaySrvAddress string // RelaySrvIP is the IP the remote peer is connected to on its diff --git a/client/internal/peer/signaler.go b/client/internal/peer/signaler.go index 5e437d96b41..8e513fc8863 100644 --- a/client/internal/peer/signaler.go +++ b/client/internal/peer/signaler.go @@ -63,6 +63,7 @@ func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, }, RosenpassPubKey: offerAnswer.RosenpassPubKey, RosenpassAddr: offerAnswer.RosenpassAddr, + MlkemPayload: offerAnswer.MlkemPayload, RelaySrvAddress: offerAnswer.RelaySrvAddress, RelaySrvIP: offerAnswer.RelaySrvIP, SessionID: sessionIDBytes, diff --git a/shared/signal/client/client.go b/shared/signal/client/client.go index fb77cb90f73..191e92b1664 100644 --- a/shared/signal/client/client.go +++ b/shared/signal/client/client.go @@ -52,6 +52,9 @@ type CredentialPayload struct { Credential *Credential RosenpassPubKey []byte RosenpassAddr string + // MlkemPayload is the opaque post-quantum KEM handshake message riding this + // OFFER/ANSWER (see Body.mlkemPayload). Nil when not running the PQ exchange. + MlkemPayload []byte RelaySrvAddress string RelaySrvIP netip.Addr SessionID []byte @@ -89,6 +92,9 @@ func MarshalCredential(myKey wgtypes.Key, remoteKey string, p CredentialPayload) if p.RelaySrvIP.IsValid() { body.RelayServerIP = p.RelaySrvIP.Unmap().AsSlice() } + if len(p.MlkemPayload) > 0 { + body.MlkemPayload = p.MlkemPayload + } return &proto.Message{ Key: myKey.PublicKey().String(), RemoteKey: remoteKey, From 41298ac24d65414b536bbcc98ce81239f0315e91 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 18:06:20 +0200 Subject: [PATCH 020/119] Added enabled env var --- client/internal/pqkem/env.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 client/internal/pqkem/env.go diff --git a/client/internal/pqkem/env.go b/client/internal/pqkem/env.go new file mode 100644 index 00000000000..52423a76243 --- /dev/null +++ b/client/internal/pqkem/env.go @@ -0,0 +1,34 @@ +package pqkem + +import ( + "os" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" +) + +// EnvEnabled is the environment variable that turns the ML-KEM post-quantum +// exchange on for this client. Accepts on/off aliases plus anything +// strconv.ParseBool understands (true/false/1/0). +const EnvEnabled = "NB_ENABLE_PQ_MLKEM" + +// Enabled reports whether the ML-KEM PQ exchange is enabled via the environment. +// An empty or unrecognized value is treated as disabled. +func Enabled() bool { + raw := strings.ToLower(strings.TrimSpace(os.Getenv(EnvEnabled))) + switch raw { + case "": + return false + case "on": + return true + case "off": + return false + } + enabled, err := strconv.ParseBool(raw) + if err != nil { + log.Warnf("failed to parse %s value %q: %v", EnvEnabled, raw, err) + return false + } + return enabled +} From dd5e9435e95188e4e62fc45052a350ad6bcc71ae Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 18:08:52 +0200 Subject: [PATCH 021/119] Adds no-op Transports and callbacks --- client/internal/pqkem_adapter.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 client/internal/pqkem_adapter.go diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go new file mode 100644 index 00000000000..776825c62a2 --- /dev/null +++ b/client/internal/pqkem_adapter.go @@ -0,0 +1,23 @@ +package internal + +import "github.com/netbirdio/netbird/client/internal/pqkem" + +// No-op adapters wiring the engine to the ML-KEM manager's two seams. They are +// placeholders so the manager can be instantiated and its lifecycle managed while +// the real integration is built. +// +// TODO(NET-1406): +// - noopPQTransport -> real tunnel-UDP transport (send over the WG data path, +// feed inbound to Manager.OnDataPathMessage), analogue of go-rosenpass's Conn. +// - noopPQCallbackHandler -> program the derived PSK via iface/wgctrl on +// OnNewPSKReady, and tear the peer down / trigger ICE reconnect on OnRekeyFailed. + +type noopPQTransport struct{} + +func (noopPQTransport) SendDataPath(remoteID string, msg []byte) error { return nil } + +type noopPQCallbackHandler struct{} + +func (noopPQCallbackHandler) OnNewPSKReady(remoteID string, psk pqkem.PSK) error { return nil } + +func (noopPQCallbackHandler) OnRekeyFailed(remoteID string) error { return nil } From 077d96ae03e1244d5ec95cab09b4b594f5fc7cf2 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 18:09:20 +0200 Subject: [PATCH 022/119] Initializes PQ ML-KEM manager --- client/internal/engine.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/client/internal/engine.go b/client/internal/engine.go index 4dff5af0ab5..f82f256ecac 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -53,6 +53,7 @@ import ( icemaker "github.com/netbirdio/netbird/client/internal/peer/ice" "github.com/netbirdio/netbird/client/internal/peerstore" "github.com/netbirdio/netbird/client/internal/portforward" + "github.com/netbirdio/netbird/client/internal/pqkem" "github.com/netbirdio/netbird/client/internal/profilemanager" "github.com/netbirdio/netbird/client/internal/relay" "github.com/netbirdio/netbird/client/internal/rosenpass" @@ -212,6 +213,9 @@ type Engine struct { // rpManager is a Rosenpass manager rpManager *rosenpass.Manager + // pqkemManager runs the ML-KEM post-quantum PSK exchange (gated by NB_ENABLE_PQ_MLKEM). + pqkemManager *pqkem.Manager + // syncMsgMux is used to guarantee sequential Management Service message processing syncMsgMux *sync.Mutex @@ -602,6 +606,12 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) return fmt.Errorf("run rosenpass manager: %w", err) } } + + if pqkem.Enabled() { + log.Infof("ML-KEM post-quantum exchange enabled") + // TODO(NET-1406): wire the real tunnel-UDP transport + WG PSK callback handler. + e.pqkemManager = pqkem.NewManager(publicKey.String(), noopPQTransport{}, noopPQCallbackHandler{}, nil) + } e.stateManager.Start() dnsServer, err := e.newDnsServer() @@ -2175,6 +2185,10 @@ func (e *Engine) close() { _ = e.rpManager.Close() } + if e.pqkemManager != nil { + e.pqkemManager.Stop() + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := e.portForwardManager.GracefullyStop(ctx); err != nil { From 0ae462238f68424e7f3d6f474117f391e9b340d6 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 18:18:15 +0200 Subject: [PATCH 023/119] Adds real callback setter for PSK on ready --- client/internal/engine.go | 5 ++-- client/internal/pqkem_adapter.go | 50 ++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index f82f256ecac..b1eb90db8c3 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -609,8 +609,9 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqkem.Enabled() { log.Infof("ML-KEM post-quantum exchange enabled") - // TODO(NET-1406): wire the real tunnel-UDP transport + WG PSK callback handler. - e.pqkemManager = pqkem.NewManager(publicKey.String(), noopPQTransport{}, noopPQCallbackHandler{}, nil) + // TODO(NET-1406): replace noopPQTransport with the real dedicated UDP transport + // bound on the WG overlay IPv4. + e.pqkemManager = pqkem.NewManager(publicKey.String(), noopPQTransport{}, pqCallbackHandler{wg: e.wgInterface}, nil) } e.stateManager.Start() diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 776825c62a2..7410915ea64 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -1,23 +1,41 @@ package internal -import "github.com/netbirdio/netbird/client/internal/pqkem" - -// No-op adapters wiring the engine to the ML-KEM manager's two seams. They are -// placeholders so the manager can be instantiated and its lifecycle managed while -// the real integration is built. -// -// TODO(NET-1406): -// - noopPQTransport -> real tunnel-UDP transport (send over the WG data path, -// feed inbound to Manager.OnDataPathMessage), analogue of go-rosenpass's Conn. -// - noopPQCallbackHandler -> program the derived PSK via iface/wgctrl on -// OnNewPSKReady, and tear the peer down / trigger ICE reconnect on OnRekeyFailed. +import ( + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" -type noopPQTransport struct{} + "github.com/netbirdio/netbird/client/internal/pqkem" +) -func (noopPQTransport) SendDataPath(remoteID string, msg []byte) error { return nil } +// pqPresharedKeySetter is the subset of the WireGuard interface the ML-KEM callback +// needs: programming a peer's preshared key. *iface.WGIface satisfies it. +type pqPresharedKeySetter interface { + SetPresharedKey(peerKey string, psk wgtypes.Key, updateOnly bool) error +} + +// pqCallbackHandler programs the derived PQ PSK onto the WireGuard peer. It is the +// engine-side implementation of pqkem.CallbackHandler. +type pqCallbackHandler struct { + wg pqPresharedKeySetter +} -type noopPQCallbackHandler struct{} +// OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op +// if the peer is not present, mirroring Rosenpass). remoteID is the peer's WG pubkey. +func (h pqCallbackHandler) OnNewPSKReady(remoteID string, psk pqkem.PSK) error { + return h.wg.SetPresharedKey(remoteID, wgtypes.Key(psk), true) +} -func (noopPQCallbackHandler) OnNewPSKReady(remoteID string, psk pqkem.PSK) error { return nil } +// OnRekeyFailed reports a failed PQ (re)key convergence. +// TODO(NET-1406): tear the peer connection down / trigger ICE reconnect. +func (h pqCallbackHandler) OnRekeyFailed(remoteID string) error { + log.Warnf("pqkem: post-quantum rekey failed for peer %s", remoteID) + return nil +} -func (noopPQCallbackHandler) OnRekeyFailed(remoteID string) error { return nil } +// noopPQTransport is a placeholder data-path transport for the ML-KEM manager. +// TODO(NET-1406): replace with the real dedicated UDP transport bound on the WG +// overlay IPv4 (send over the data path, feed inbound to Manager.OnDataPathMessage), +// analogue of go-rosenpass's Conn. +type noopPQTransport struct{} + +func (noopPQTransport) SendDataPath(remoteID string, msg []byte) error { return nil } From 40b6a554c779d3faa1883db1dae6f7a3ffd51508 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 18:28:43 +0200 Subject: [PATCH 024/119] Ensure iface is up and with overlay ip assigned to get a valid UDP port --- client/internal/engine.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index b1eb90db8c3..b38ff3426df 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -606,13 +606,6 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) return fmt.Errorf("run rosenpass manager: %w", err) } } - - if pqkem.Enabled() { - log.Infof("ML-KEM post-quantum exchange enabled") - // TODO(NET-1406): replace noopPQTransport with the real dedicated UDP transport - // bound on the WG overlay IPv4. - e.pqkemManager = pqkem.NewManager(publicKey.String(), noopPQTransport{}, pqCallbackHandler{wg: e.wgInterface}, nil) - } e.stateManager.Start() dnsServer, err := e.newDnsServer() @@ -688,6 +681,14 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) e.rpManager.SetInterface(e.wgInterface) } + // Start the ML-KEM PQ manager after the interface is up so its (dedicated UDP) + // transport can bind on the WG overlay IP. TODO(NET-1406): replace noopPQTransport + // with the real transport bound on e.config.WgAddr.IP. + if pqkem.Enabled() { + log.Infof("ML-KEM post-quantum exchange enabled") + e.pqkemManager = pqkem.NewManager(publicKey.String(), noopPQTransport{}, pqCallbackHandler{wg: e.wgInterface}, nil) + } + // if inbound conns are blocked there is no need to create the ACL manager if e.firewall != nil && !e.config.BlockInbound { e.acl = acl.NewDefaultManager(e.firewall) @@ -3013,6 +3014,7 @@ func convertToOfferAnswer(msg *sProto.Message) (*peer.OfferAnswer, error) { RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, MlkemPayload: msg.GetBody().GetMlkemPayload(), + MlkemPort: int(msg.GetBody().GetMlkemPort()), RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), RelaySrvIP: relayIP, SessionID: sessionID, From 6a42f7da009a57923558bebd2ffb9b08f9a7f01d Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 24 Jul 2026 18:29:06 +0200 Subject: [PATCH 025/119] Communicate the port over the signal exchange --- client/internal/peer/handshaker.go | 4 ++ client/internal/peer/signaler.go | 1 + shared/signal/client/client.go | 8 ++- shared/signal/proto/signalexchange.pb.go | 84 ++++++++++++++---------- shared/signal/proto/signalexchange.proto | 5 ++ 5 files changed, 66 insertions(+), 36 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index d791aa64fe9..77ee213e53c 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -45,6 +45,10 @@ type OfferAnswer struct { // when the peer does not run the ML-KEM PQ exchange. MlkemPayload []byte + // MlkemPort is the peer's ML-KEM PQ service UDP port (bound on its WG overlay + // IP) where data-path rekey messages are sent. Zero when not running the exchange. + MlkemPort int + // relay server address RelaySrvAddress string // RelaySrvIP is the IP the remote peer is connected to on its diff --git a/client/internal/peer/signaler.go b/client/internal/peer/signaler.go index 8e513fc8863..e286d7f8654 100644 --- a/client/internal/peer/signaler.go +++ b/client/internal/peer/signaler.go @@ -64,6 +64,7 @@ func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, RosenpassPubKey: offerAnswer.RosenpassPubKey, RosenpassAddr: offerAnswer.RosenpassAddr, MlkemPayload: offerAnswer.MlkemPayload, + MlkemPort: offerAnswer.MlkemPort, RelaySrvAddress: offerAnswer.RelaySrvAddress, RelaySrvIP: offerAnswer.RelaySrvIP, SessionID: sessionIDBytes, diff --git a/shared/signal/client/client.go b/shared/signal/client/client.go index 191e92b1664..51499bb3351 100644 --- a/shared/signal/client/client.go +++ b/shared/signal/client/client.go @@ -54,7 +54,9 @@ type CredentialPayload struct { RosenpassAddr string // MlkemPayload is the opaque post-quantum KEM handshake message riding this // OFFER/ANSWER (see Body.mlkemPayload). Nil when not running the PQ exchange. - MlkemPayload []byte + MlkemPayload []byte + // MlkemPort is the sender's ML-KEM PQ service UDP port (0 when not running). + MlkemPort int RelaySrvAddress string RelaySrvIP netip.Addr SessionID []byte @@ -95,6 +97,10 @@ func MarshalCredential(myKey wgtypes.Key, remoteKey string, p CredentialPayload) if len(p.MlkemPayload) > 0 { body.MlkemPayload = p.MlkemPayload } + if p.MlkemPort > 0 { + port := uint32(p.MlkemPort) + body.MlkemPort = &port + } return &proto.Message{ Key: myKey.PublicKey().String(), RemoteKey: remoteKey, diff --git a/shared/signal/proto/signalexchange.pb.go b/shared/signal/proto/signalexchange.pb.go index a08439f7cce..812e3c9c008 100644 --- a/shared/signal/proto/signalexchange.pb.go +++ b/shared/signal/proto/signalexchange.pb.go @@ -245,6 +245,10 @@ type Body struct { // pqkem library frames and parses it. Absent when the sender does not run the // ML-KEM PQ exchange; unknown to older clients, which ignore it. MlkemPayload []byte `protobuf:"bytes,12,opt,name=mlkemPayload,proto3,oneof" json:"mlkemPayload,omitempty"` + // mlkemPort is the UDP port of the sender's ML-KEM PQ service, bound on its + // WireGuard overlay IP. Peers send subsequent rekey messages there over the + // data path. Zero/absent when the ML-KEM PQ exchange is not running. + MlkemPort *uint32 `protobuf:"varint,13,opt,name=mlkemPort,proto3,oneof" json:"mlkemPort,omitempty"` } func (x *Body) Reset() { @@ -356,6 +360,13 @@ func (x *Body) GetMlkemPayload() []byte { return nil } +func (x *Body) GetMlkemPort() uint32 { + if x != nil && x.MlkemPort != nil { + return *x.MlkemPort + } + return 0 +} + // Mode indicates a connection mode type Mode struct { state protoimpl.MessageState @@ -479,7 +490,7 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x52, 0x09, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x8c, 0x05, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xbd, 0x05, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, @@ -510,40 +521,43 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, 0x88, 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x03, 0x52, 0x0c, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x88, - 0x01, 0x01, 0x22, 0x52, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x46, - 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52, 0x10, - 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, - 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x47, 0x4f, - 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x48, 0x45, 0x41, 0x52, 0x54, - 0x42, 0x45, 0x41, 0x54, 0x10, 0x06, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x42, 0x0c, 0x0a, - 0x0a, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, - 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, 0x42, 0x0f, 0x0a, - 0x0d, 0x5f, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x4a, 0x04, - 0x08, 0x09, 0x10, 0x0a, 0x22, 0x2e, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x06, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x06, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x22, 0x6d, 0x0a, 0x0f, 0x52, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, - 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, - 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, - 0x79, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, - 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, - 0x64, 0x64, 0x72, 0x32, 0xb9, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45, 0x78, - 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x4c, 0x0a, 0x04, 0x53, 0x65, 0x6e, 0x64, 0x12, 0x20, - 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, - 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, - 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, - 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, - 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, - 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x01, 0x01, 0x12, 0x21, 0x0a, 0x09, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x6f, 0x72, 0x74, 0x18, + 0x0d, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x04, 0x52, 0x09, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x6f, + 0x72, 0x74, 0x88, 0x01, 0x01, 0x22, 0x52, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, + 0x05, 0x4f, 0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, + 0x45, 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, + 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, + 0x07, 0x47, 0x4f, 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x48, 0x45, + 0x41, 0x52, 0x54, 0x42, 0x45, 0x41, 0x54, 0x10, 0x06, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x72, 0x65, + 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x10, + 0x0a, 0x0e, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x50, + 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6d, 0x6c, 0x6b, 0x65, 0x6d, 0x50, 0x6f, 0x72, 0x74, 0x4a, + 0x04, 0x08, 0x09, 0x10, 0x0a, 0x22, 0x2e, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, + 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, + 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x22, 0x6d, 0x0a, 0x0f, 0x52, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, + 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x6f, 0x73, 0x65, + 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x75, 0x62, 0x4b, + 0x65, 0x79, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x41, 0x64, 0x64, 0x72, 0x32, 0xb9, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45, + 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x4c, 0x0a, 0x04, 0x53, 0x65, 0x6e, 0x64, 0x12, + 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, + 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x59, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, + 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, + 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, + 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/shared/signal/proto/signalexchange.proto b/shared/signal/proto/signalexchange.proto index 5785ce459c0..c6b2a639bba 100644 --- a/shared/signal/proto/signalexchange.proto +++ b/shared/signal/proto/signalexchange.proto @@ -82,6 +82,11 @@ message Body { // pqkem library frames and parses it. Absent when the sender does not run the // ML-KEM PQ exchange; unknown to older clients, which ignore it. optional bytes mlkemPayload = 12; + + // mlkemPort is the UDP port of the sender's ML-KEM PQ service, bound on its + // WireGuard overlay IP. Peers send subsequent rekey messages there over the + // data path. Zero/absent when the ML-KEM PQ exchange is not running. + optional uint32 mlkemPort = 13; } // Mode indicates a connection mode From d2db1c7ffc1212d7a500b93a98bd85f3bcba8a98 Mon Sep 17 00:00:00 2001 From: riccardom Date: Sat, 25 Jul 2026 11:30:39 +0200 Subject: [PATCH 026/119] Adds transport --- client/internal/engine.go | 22 +++-- client/internal/pqkem_adapter.go | 8 -- client/internal/pqkem_transport.go | 132 +++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 13 deletions(-) create mode 100644 client/internal/pqkem_transport.go diff --git a/client/internal/engine.go b/client/internal/engine.go index b38ff3426df..3328b2b6433 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -215,6 +215,8 @@ type Engine struct { // pqkemManager runs the ML-KEM post-quantum PSK exchange (gated by NB_ENABLE_PQ_MLKEM). pqkemManager *pqkem.Manager + // pqTransport is the ML-KEM data-path UDP transport, bound on the WG overlay IP. + pqTransport *pqTransport // syncMsgMux is used to guarantee sequential Management Service message processing syncMsgMux *sync.Mutex @@ -681,12 +683,19 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) e.rpManager.SetInterface(e.wgInterface) } - // Start the ML-KEM PQ manager after the interface is up so its (dedicated UDP) - // transport can bind on the WG overlay IP. TODO(NET-1406): replace noopPQTransport - // with the real transport bound on e.config.WgAddr.IP. + // Start the ML-KEM PQ manager after the interface is up so its dedicated UDP + // transport can bind on the WG overlay IP. if pqkem.Enabled() { - log.Infof("ML-KEM post-quantum exchange enabled") - e.pqkemManager = pqkem.NewManager(publicKey.String(), noopPQTransport{}, pqCallbackHandler{wg: e.wgInterface}, nil) + tr, pqErr := newPQTransport(e.config.WgAddr.IP) + if pqErr != nil { + log.Errorf("ML-KEM PQ transport bind failed, PQ exchange disabled: %v", pqErr) + } else { + e.pqTransport = tr + e.pqkemManager = pqkem.NewManager(publicKey.String(), tr, pqCallbackHandler{wg: e.wgInterface}, nil) + tr.setManager(e.pqkemManager) + go tr.run() + log.Infof("ML-KEM post-quantum exchange enabled (udp port %d on overlay %s)", tr.Port(), e.config.WgAddr.IP) + } } // if inbound conns are blocked there is no need to create the ACL manager @@ -2187,6 +2196,9 @@ func (e *Engine) close() { _ = e.rpManager.Close() } + if e.pqTransport != nil { + _ = e.pqTransport.Close() + } if e.pqkemManager != nil { e.pqkemManager.Stop() } diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 7410915ea64..1a94c65a450 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -31,11 +31,3 @@ func (h pqCallbackHandler) OnRekeyFailed(remoteID string) error { log.Warnf("pqkem: post-quantum rekey failed for peer %s", remoteID) return nil } - -// noopPQTransport is a placeholder data-path transport for the ML-KEM manager. -// TODO(NET-1406): replace with the real dedicated UDP transport bound on the WG -// overlay IPv4 (send over the data path, feed inbound to Manager.OnDataPathMessage), -// analogue of go-rosenpass's Conn. -type noopPQTransport struct{} - -func (noopPQTransport) SendDataPath(remoteID string, msg []byte) error { return nil } diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go new file mode 100644 index 00000000000..da2e24f0d4d --- /dev/null +++ b/client/internal/pqkem_transport.go @@ -0,0 +1,132 @@ +package internal + +import ( + "fmt" + "net" + "net/netip" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/pqkem" +) + +// DefaultPort is the preferred UDP port for the ML-KEM data-path service, bound on +// the WG overlay IP. Since each client owns a distinct overlay IP, this port is +// almost always free, so it need not be announced (peers assume it). A peer only +// announces Body.mlkemPort when a collision forced it onto a different port. +const DefaultPort = 51833 + +// pqTransport is the ML-KEM data-path transport: a dedicated UDP socket bound on the +// WireGuard overlay IP. Rekey messages travel through the tunnel to each peer's +// overlay IP and announced pqkem port. It implements pqkem.Transport and feeds +// inbound datagrams to the manager (set via setManager after construction, since the +// manager is built with this transport). +type pqTransport struct { + conn *net.UDPConn + port int + + mu sync.RWMutex + mgr *pqkem.Manager + peers map[string]*net.UDPAddr // remoteID (WG pubkey) -> overlay UDP addr to send to + byAddr map[string]string // source addr string -> remoteID (inbound dispatch) +} + +// newPQTransport binds a UDP socket on the WG overlay IP, preferring DefaultPort and +// falling back to an OS-assigned ephemeral port if it is in use. It must be called +// after the WG interface is up so the overlay IP is assigned; when the bound port is +// not DefaultPort it is announced to peers via Body.mlkemPort. +func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { + if !overlayIP.IsValid() { + return nil, fmt.Errorf("invalid overlay IP for pqkem transport") + } + ip := net.IP(overlayIP.AsSlice()) + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: DefaultPort}) + if err != nil { + // Default port unavailable (rare on a dedicated overlay IP): fall back to an + // ephemeral port, which will be announced to peers. + log.Debugf("pqkem: default port %d unavailable on %s (%v), using an ephemeral port", DefaultPort, overlayIP, err) + conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) + if err != nil { + return nil, fmt.Errorf("bind pqkem udp on overlay %s: %w", overlayIP, err) + } + } + return &pqTransport{ + conn: conn, + port: conn.LocalAddr().(*net.UDPAddr).Port, + peers: make(map[string]*net.UDPAddr), + byAddr: make(map[string]string), + }, nil +} + +// Port is the bound UDP port, announced to peers as Body.mlkemPort. +func (t *pqTransport) Port() int { return t.port } + +func (t *pqTransport) setManager(m *pqkem.Manager) { + t.mu.Lock() + t.mgr = m + t.mu.Unlock() +} + +// AddPeer records where a peer's rekey messages are sent: its overlay IP and the +// pqkem port it announced. A zero port or invalid IP is ignored. +func (t *pqTransport) AddPeer(remoteID string, overlayIP netip.Addr, port int) { + if !overlayIP.IsValid() || port <= 0 { + return + } + addr := &net.UDPAddr{IP: net.IP(overlayIP.AsSlice()), Port: port} + t.mu.Lock() + if old, ok := t.peers[remoteID]; ok { + delete(t.byAddr, old.String()) + } + t.peers[remoteID] = addr + t.byAddr[addr.String()] = remoteID + t.mu.Unlock() +} + +func (t *pqTransport) RemovePeer(remoteID string) { + t.mu.Lock() + if a, ok := t.peers[remoteID]; ok { + delete(t.byAddr, a.String()) + delete(t.peers, remoteID) + } + t.mu.Unlock() +} + +// SendDataPath implements pqkem.Transport. +func (t *pqTransport) SendDataPath(remoteID string, msg []byte) error { + t.mu.RLock() + addr := t.peers[remoteID] + t.mu.RUnlock() + if addr == nil { + return fmt.Errorf("no data-path address for peer %s", remoteID) + } + _, err := t.conn.WriteToUDP(msg, addr) + return err +} + +// run is the receive loop: it maps each datagram's source overlay address to a peer +// and feeds it to the manager. Exits when the socket is closed. +func (t *pqTransport) run() { + buf := make([]byte, 2048) + for { + n, src, err := t.conn.ReadFromUDP(buf) + if err != nil { + return + } + t.mu.RLock() + remoteID := t.byAddr[src.String()] + mgr := t.mgr + t.mu.RUnlock() + if remoteID == "" || mgr == nil { + continue + } + msg := make([]byte, n) + copy(msg, buf[:n]) + if err := mgr.OnDataPathMessage(remoteID, msg); err != nil { + log.Debugf("pqkem: inbound from %s: %v", remoteID, err) + } + } +} + +func (t *pqTransport) Close() error { return t.conn.Close() } From cda3922478d58748f3ce063599996b7dc08464f6 Mon Sep 17 00:00:00 2001 From: riccardom Date: Sat, 25 Jul 2026 12:06:43 +0200 Subject: [PATCH 027/119] Makes Transport just a UDP socket. Manager owns maps for remoteID <-> remote UDP addr Engine talks to manager only --- client/internal/engine.go | 14 +- client/internal/pqkem/convergence_test.go | 74 ++++------ client/internal/pqkem/manager.go | 163 ++++++++++++++++------ client/internal/pqkem/manager_test.go | 93 +++++++++--- client/internal/pqkem_transport.go | 116 ++++----------- 5 files changed, 243 insertions(+), 217 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 3328b2b6433..21dc55f412e 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -214,9 +214,8 @@ type Engine struct { rpManager *rosenpass.Manager // pqkemManager runs the ML-KEM post-quantum PSK exchange (gated by NB_ENABLE_PQ_MLKEM). + // It owns the data-path transport and peer endpoint routing. pqkemManager *pqkem.Manager - // pqTransport is the ML-KEM data-path UDP transport, bound on the WG overlay IP. - pqTransport *pqTransport // syncMsgMux is used to guarantee sequential Management Service message processing syncMsgMux *sync.Mutex @@ -690,11 +689,9 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqErr != nil { log.Errorf("ML-KEM PQ transport bind failed, PQ exchange disabled: %v", pqErr) } else { - e.pqTransport = tr - e.pqkemManager = pqkem.NewManager(publicKey.String(), tr, pqCallbackHandler{wg: e.wgInterface}, nil) - tr.setManager(e.pqkemManager) - go tr.run() - log.Infof("ML-KEM post-quantum exchange enabled (udp port %d on overlay %s)", tr.Port(), e.config.WgAddr.IP) + e.pqkemManager = pqkem.NewManager(publicKey.String(), pqCallbackHandler{wg: e.wgInterface}, nil) + e.pqkemManager.SetTransport(tr) + log.Infof("ML-KEM post-quantum exchange enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) } } @@ -2196,9 +2193,6 @@ func (e *Engine) close() { _ = e.rpManager.Close() } - if e.pqTransport != nil { - _ = e.pqTransport.Close() - } if e.pqkemManager != nil { e.pqkemManager.Stop() } diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index 66877c9fae3..b60cd6cce68 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -1,36 +1,31 @@ package pqkem import ( - "sync/atomic" + "net/netip" "testing" "time" "github.com/stretchr/testify/require" ) +// dropTransport is a pqkem.Transport that silently discards everything. type dropTransport struct{} -func (dropTransport) SendDataPath(string, []byte) error { return nil } +func (dropTransport) Send(netip.AddrPort, []byte) error { return nil } +func (dropTransport) LocalPort() int { return 0 } +func (dropTransport) Run(func(netip.AddrPort, []byte)) {} +func (dropTransport) Close() error { return nil } -// gate is a data-path loopback with a switchable drop flag. When dropping it reports -// success but does not deliver (mimics a lossy/broken tunnel). -type gate struct { - localID string - peer *Manager - drop atomic.Bool -} - -func (g *gate) SendDataPath(remoteID string, msg []byte) error { - if g.drop.Load() { - return nil - } - cp := append([]byte(nil), msg...) - return g.peer.OnDataPathMessage(g.localID, cp) +func failedCount(f *fakeWG) int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.failed) } func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { wg := newFakeWG() - d := NewManager("bbbb", dropTransport{}, wg, nil) // bbbb > aaaa -> initiator + d := NewManager("bbbb", wg, nil) // bbbb > aaaa -> initiator + d.SetTransport(dropTransport{}) d.retryInterval = 5 * time.Millisecond d.maxRetries = 3 defer d.Stop() @@ -41,40 +36,25 @@ func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { require.NoError(t, err) require.NotNil(t, offer) - require.Eventually(t, func() bool { - wg.mu.Lock() - defer wg.mu.Unlock() - return len(wg.failed) == 1 - }, time.Second, 5*time.Millisecond) + require.Eventually(t, func() bool { return failedCount(wg) == 1 }, time.Second, 5*time.Millisecond) } func TestManager_RekeyToleratesKFailures(t *testing.T) { - gA := &gate{localID: "aaaa"} - gB := &gate{localID: "bbbb"} - wgA := newFakeWG() - wgB := newFakeWG() - - dA := NewManager("aaaa", gA, wgA, nil) - dB := NewManager("bbbb", gB, wgB, nil) - gA.peer = dB - gB.peer = dA - dB.retryInterval = 5 * time.Millisecond - dB.maxRetries = 2 + dA, dB, _, wgB, lbB := pair(t) defer dA.Stop() defer dB.Stop() - // Bootstrap over signalling -> B becomes established. - offer, err := dB.SignalOffer("aaaa") - require.NoError(t, err) - answer, err := dA.SignalOnOffer("bbbb", offer) - require.NoError(t, err) - require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) - require.NotEqual(t, PSK{}, wgB.psk("aaaa")) - - // Bring the data path up on both, then drop B's delivery so rekeys can't converge. + // Establish: bootstrap + data-path-rekeyed so B becomes established and its data + // path is usable. + bootstrap(t, dA, dB) dA.OnDataPathRekeyed("bbbb") dB.OnDataPathRekeyed("aaaa") - gB.drop.Store(true) + require.NotEqual(t, PSK{}, wgB.psk("aaaa")) + + // Tighten timings and drop B's outbound so rekeys can no longer converge. + dB.retryInterval = 5 * time.Millisecond + dB.maxRetries = 2 + lbB.drop.Store(true) // K-1 data-path rekeys must NOT raise OnRekeyFailed. for i := 0; i < DefaultMaxRekeyFailures-1; i++ { @@ -85,13 +65,7 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { require.Equal(t, 0, failedCount(wgB), "no failure before K attempts") // The K-th failure raises it once. - _, err = dB.startExchange("aaaa", false, ExchangeID{}) + _, err := dB.startExchange("aaaa", false, ExchangeID{}) require.NoError(t, err) require.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) } - -func failedCount(f *fakeWG) int { - f.mu.Lock() - defer f.mu.Unlock() - return len(f.failed) -} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index e3816537f50..d265c18d1e0 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "fmt" "log/slog" + "net/netip" "sync" "time" ) @@ -21,12 +22,22 @@ const ( DefaultMaxRekeyFailures = 3 ) -// Transport pushes a message over the peer's data path (e.g. a WireGuard tunnel). -// It is the library's only outbound send: the control-plane (signalling) channel is -// host-driven — the library hands the host offer/answer payloads to piggyback on the -// host's own negotiation, it never pushes there itself. +// Transport is the data-path socket the Manager drives (the analogue of +// go-rosenpass's Conn). It is a dumb mover of bytes to/from endpoints: the Manager +// owns the remoteID<->endpoint routing and hands the transport a resolved endpoint +// to Send, and reverse-resolves the source of each inbound datagram. Its lifecycle +// belongs to the Manager (Run at SetTransport, Close at Stop). type Transport interface { - SendDataPath(remoteID string, msg []byte) error + // Send delivers msg to the given data-path endpoint. + Send(endpoint netip.AddrPort, msg []byte) error + // LocalPort is the bound local UDP port, announced to peers so they know where + // to send data-path messages. + LocalPort() int + // Run starts delivering inbound datagrams as (source endpoint, msg) to onInbound + // and returns immediately; it runs until Close. + Run(onInbound func(src netip.AddrPort, msg []byte)) + // Close stops delivery and releases the socket. + Close() error } // exchangeState is the single source of truth for an exchange's role and phase. @@ -58,14 +69,13 @@ type exchangeCtl struct { } // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It -// drives the X25519MLKEM768 exchange and surfaces the derived PSK and convergence to -// the host via CallbackHandler. It is fully event-driven: the bootstrap is triggered -// by the host (SignalOffer) and each rotation is clocked by OnDataPathRekeyed (the -// consumer's transport rekey). The cryptography is the pure kem.go primitives; all -// state lives here under one lock. +// drives the X25519MLKEM768 exchange, owns the peer endpoint routing and the data-path +// transport, and surfaces the derived PSK and convergence to the host via +// CallbackHandler. It is event-driven: the bootstrap is triggered by the host +// (SignalOffer) and each rotation is clocked by OnDataPathRekeyed. The cryptography is +// the pure kem.go primitives; all state lives here under one lock. type Manager struct { localID string - transport Transport cbHandler CallbackHandler logger *slog.Logger @@ -77,27 +87,25 @@ type Manager struct { rootCancel context.CancelFunc mu sync.Mutex - exchanges map[string]*exchangeCtl // in-flight exchange per peer - established map[string]bool // peer has completed at least one exchange - failures map[string]int // consecutive rekey failures per peer - // dataSend holds the peer's data-path sender when the data path is up; nil - // (absent) means it is down. Toggled by OnDataPathRekeyed / OnDataPathDown. - dataSend map[string]func(string, []byte) error - wait sync.WaitGroup + transport Transport + exchanges map[string]*exchangeCtl // in-flight exchange per peer + established map[string]bool // peer has completed at least one exchange + failures map[string]int // consecutive rekey failures per peer + peers map[string]netip.AddrPort // remoteID -> data-path endpoint (send routing) + peersByAddr map[netip.AddrPort]string // reverse: source endpoint -> remoteID (inbound) + wait sync.WaitGroup } // NewManager builds a manager for the local peer identified by its peer identity key // (used for the deterministic initiator role and the identity binding). A nil logger -// falls back to slog.Default(). Retry/retries/K use their defaults and can be -// overridden before use. -func NewManager(localID string, t Transport, h CallbackHandler, logger *slog.Logger) *Manager { +// falls back to slog.Default(). Set the data-path transport with SetTransport. +func NewManager(localID string, h CallbackHandler, logger *slog.Logger) *Manager { if logger == nil { logger = slog.Default() } ctx, cancel := context.WithCancel(context.Background()) return &Manager{ localID: localID, - transport: t, cbHandler: h, logger: logger, retryInterval: DefaultRetryInterval, @@ -108,10 +116,34 @@ func NewManager(localID string, t Transport, h CallbackHandler, logger *slog.Log exchanges: make(map[string]*exchangeCtl), established: make(map[string]bool), failures: make(map[string]int), - dataSend: make(map[string]func(string, []byte) error), + peers: make(map[string]netip.AddrPort), + peersByAddr: make(map[netip.AddrPort]string), } } +// SetTransport installs the data-path transport and starts its inbound delivery. The +// Manager owns it from here: Stop closes it. +func (m *Manager) SetTransport(t Transport) { + m.mu.Lock() + m.transport = t + m.mu.Unlock() + if t != nil { + t.Run(m.onDataPathInbound) + } +} + +// LocalPort is the data-path transport's bound UDP port (0 if no transport), to be +// announced to peers. +func (m *Manager) LocalPort() int { + m.mu.Lock() + t := m.transport + m.mu.Unlock() + if t == nil { + return 0 + } + return t.LocalPort() +} + // IsInitiator reports whether the local peer drives the exchange for this remote // peer. Roles are deterministic (lexicographic identity-key compare) so exactly one // side initiates, mirroring how Rosenpass picks its handshake initiator. @@ -119,7 +151,22 @@ func (m *Manager) IsInitiator(remoteID string) bool { return m.localID > remoteID } -// RemovePeer stops any in-flight exchange for a peer and drops its state. +// AddPeer registers where a peer's data-path messages are sent and received: its +// overlay endpoint (IP:port). Re-adding updates the endpoint. +func (m *Manager) AddPeer(remoteID string, endpoint netip.AddrPort) { + if !endpoint.IsValid() { + return + } + m.mu.Lock() + if old, ok := m.peers[remoteID]; ok { + delete(m.peersByAddr, old) + } + m.peers[remoteID] = endpoint + m.peersByAddr[endpoint] = remoteID + m.mu.Unlock() +} + +// RemovePeer stops any in-flight exchange for a peer and drops its state and routing. func (m *Manager) RemovePeer(remoteID string) { m.mu.Lock() if ex, ok := m.exchanges[remoteID]; ok { @@ -130,17 +177,26 @@ func (m *Manager) RemovePeer(remoteID string) { } delete(m.established, remoteID) delete(m.failures, remoteID) - delete(m.dataSend, remoteID) + if ep, ok := m.peers[remoteID]; ok { + delete(m.peersByAddr, ep) + delete(m.peers, remoteID) + } m.mu.Unlock() } -// Stop cancels all in-flight exchanges and waits for their goroutines to exit. +// Stop cancels all in-flight exchanges, closes the transport, and waits for the +// exchange goroutines to exit. func (m *Manager) Stop() { m.rootCancel() m.wait.Wait() m.mu.Lock() + t := m.transport + m.transport = nil m.exchanges = make(map[string]*exchangeCtl) m.mu.Unlock() + if t != nil { + _ = t.Close() + } } // ---- Signalling channel (host-driven; rides the host's negotiation) ---- @@ -190,10 +246,24 @@ func (m *Manager) SignalOnAnswer(remoteID string, answer []byte) error { return m.processAnswer(remoteID, msg.(*AnswerMsg)) } -// ---- Data path (library-driven push) ---- +// ---- Data path ---- -// OnDataPathMessage feeds a KEM message received over the data path (tunnel) and -// pushes any reply back over the data path. +// onDataPathInbound is the transport's inbound handler: it reverse-resolves the +// source endpoint to a peer and dispatches. Unknown sources are dropped. +func (m *Manager) onDataPathInbound(src netip.AddrPort, msg []byte) { + m.mu.Lock() + remoteID := m.peersByAddr[src] + m.mu.Unlock() + if remoteID == "" { + return + } + if err := m.OnDataPathMessage(remoteID, msg); err != nil { + m.logger.Debug("pqkem inbound", "peer", remoteID, "err", err) + } +} + +// OnDataPathMessage handles a KEM message received over the data path from remoteID +// and pushes any reply back over the data path. func (m *Manager) OnDataPathMessage(remoteID string, raw []byte) error { typ, msg, err := Decode(raw) if err != nil { @@ -217,13 +287,12 @@ func (m *Manager) OnDataPathMessage(remoteID string, raw []byte) error { } // OnDataPathRekeyed notifies that the peer's data path is up and freshly keyed with -// the latest PSK (fired on first establishment AND every rekey). It marks the data -// path usable and, if we are the initiator that just derived a PSK, chains the next -// exchange: a fresh offer over the data path that acknowledges the just-completed one -// (its arrival under the new key proves to the responder that the key works). +// the latest PSK (fired on first establishment AND every rekey). If we are the +// initiator that just derived a PSK, it chains the next exchange: a fresh offer over +// the data path that acknowledges the just-completed one (its arrival under the new +// key proves to the responder that the key works). func (m *Manager) OnDataPathRekeyed(remoteID string) { m.mu.Lock() - m.dataSend[remoteID] = m.transport.SendDataPath ex := m.exchanges[remoteID] chain := ex != nil && ex.state == stateAwaitingRekey var ackID ExchangeID @@ -245,25 +314,27 @@ func (m *Manager) OnDataPathRekeyed(remoteID string) { } } -// OnDataPathDown notifies that the peer's data path went down; rotations pause until -// it is up again (the host re-bootstraps over signalling on reconnect). -func (m *Manager) OnDataPathDown(remoteID string) { - m.mu.Lock() - delete(m.dataSend, remoteID) - m.mu.Unlock() -} +// OnDataPathDown notifies that the peer's data path went down. Rotations resume once +// the host re-bootstraps over signalling on reconnect; in-flight data-path sends will +// simply fail until then. Reserved as an explicit hook. +func (m *Manager) OnDataPathDown(remoteID string) {} // ---- internals ---- -// pushDataPath sends over the peer's data path, erroring if it is down. +// pushDataPath resolves the peer's endpoint and sends over the data-path transport, +// erroring if the peer is unknown or no transport is set. func (m *Manager) pushDataPath(remoteID string, msg []byte) error { m.mu.Lock() - send := m.dataSend[remoteID] + ep, ok := m.peers[remoteID] + t := m.transport m.mu.Unlock() - if send == nil { - return fmt.Errorf("no data path for peer %s", remoteID) + if !ok { + return fmt.Errorf("no data-path endpoint for peer %s", remoteID) + } + if t == nil { + return fmt.Errorf("no data-path transport") } - return send(remoteID, msg) + return t.Send(ep, msg) } func (m *Manager) binding(remoteID string) Binding { diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index a11ede6a2b3..ac7be825843 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -1,25 +1,62 @@ package pqkem import ( + "fmt" + "net/netip" "sync" + "sync/atomic" "testing" "github.com/stretchr/testify/require" ) -// loopback is a data-path transport: SendDataPath delivers synchronously to the peer -// manager's OnDataPathMessage, attributing it to localID (the sender). The signalling -// channel is driven by the test directly via the SignalX methods. +// netSwitch is an in-memory UDP fabric: transports register their endpoint and get +// datagrams delivered to their inbound handler. +type netSwitch struct { + mu sync.Mutex + h map[netip.AddrPort]func(netip.AddrPort, []byte) +} + +func newSwitch() *netSwitch { + return &netSwitch{h: map[netip.AddrPort]func(netip.AddrPort, []byte){}} +} + +func (s *netSwitch) register(ep netip.AddrPort, fn func(netip.AddrPort, []byte)) { + s.mu.Lock() + s.h[ep] = fn + s.mu.Unlock() +} + +func (s *netSwitch) deliver(dst, src netip.AddrPort, msg []byte) error { + s.mu.Lock() + fn := s.h[dst] + s.mu.Unlock() + if fn == nil { + return fmt.Errorf("no route to %s", dst) + } + fn(src, msg) + return nil +} + +// loopback is an endpoint-based pqkem.Transport over a netSwitch, with a switchable +// drop flag. type loopback struct { - localID string - peer *Manager + ep netip.AddrPort + sw *netSwitch + drop atomic.Bool } -func (l *loopback) SendDataPath(remoteID string, msg []byte) error { - cp := append([]byte(nil), msg...) - return l.peer.OnDataPathMessage(l.localID, cp) +func (l *loopback) Send(dst netip.AddrPort, msg []byte) error { + if l.drop.Load() { + return nil + } + return l.sw.deliver(dst, l.ep, append([]byte(nil), msg...)) } +func (l *loopback) LocalPort() int { return int(l.ep.Port()) } +func (l *loopback) Run(onInbound func(netip.AddrPort, []byte)) { l.sw.register(l.ep, onInbound) } +func (l *loopback) Close() error { return nil } + type fakeWG struct { mu sync.Mutex psks map[string]PSK @@ -48,18 +85,27 @@ func (f *fakeWG) psk(peer string) PSK { return f.psks[peer] } -// pair builds two wired managers (B is the initiator, "bbbb" > "aaaa"). -func pair(t *testing.T) (dA, dB *Manager, wgA, wgB *fakeWG) { +var ( + epA = netip.MustParseAddrPort("100.64.0.1:51833") + epB = netip.MustParseAddrPort("100.64.0.2:51833") +) + +// pair builds two wired managers (B is the initiator, "bbbb" > "aaaa") sharing a +// netSwitch, with each peer's data-path endpoint registered. lbB is B's loopback +// (for toggling drop). +func pair(t *testing.T) (dA, dB *Manager, wgA, wgB *fakeWG, lbB *loopback) { t.Helper() - lbA := &loopback{localID: "aaaa"} - lbB := &loopback{localID: "bbbb"} + sw := newSwitch() wgA = newFakeWG() wgB = newFakeWG() - dA = NewManager("aaaa", lbA, wgA, nil) - dB = NewManager("bbbb", lbB, wgB, nil) - lbA.peer = dB - lbB.peer = dA - return dA, dB, wgA, wgB + dA = NewManager("aaaa", wgA, nil) + dB = NewManager("bbbb", wgB, nil) + dA.SetTransport(&loopback{ep: epA, sw: sw}) + lbB = &loopback{ep: epB, sw: sw} + dB.SetTransport(lbB) + dA.AddPeer("bbbb", epB) + dB.AddPeer("aaaa", epA) + return dA, dB, wgA, wgB, lbB } // bootstrap runs the signalling offer/answer (the test plays the host carrying bytes). @@ -75,7 +121,7 @@ func bootstrap(t *testing.T, dA, dB *Manager) { } func TestManager_BootstrapDerivesSamePSK(t *testing.T) { - dA, dB, wgA, wgB := pair(t) + dA, dB, wgA, wgB, _ := pair(t) defer dA.Stop() defer dB.Stop() @@ -88,15 +134,15 @@ func TestManager_BootstrapDerivesSamePSK(t *testing.T) { } func TestManager_ChainRotatesAndAcks(t *testing.T) { - dA, dB, wgA, wgB := pair(t) + dA, dB, wgA, wgB, _ := pair(t) defer dA.Stop() defer dB.Stop() bootstrap(t, dA, dB) psk1 := wgB.psk("aaaa") - // Data path up on both sides; B chains the next offer (acking exchange 1) over the - // data path, which rotates both to a fresh PSK and acknowledges A. + // Data path up: B (initiator) chains the next offer over the data path, which + // rotates both to a fresh PSK and acknowledges A. dA.OnDataPathRekeyed("bbbb") dB.OnDataPathRekeyed("aaaa") @@ -107,7 +153,7 @@ func TestManager_ChainRotatesAndAcks(t *testing.T) { } func TestManager_NonInitiatorReturnsNoOffer(t *testing.T) { - dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), nil) + dA := NewManager("aaaa", newFakeWG(), nil) defer dA.Stop() offer, err := dA.SignalOffer("bbbb") // not the initiator vs "bbbb" @@ -116,7 +162,8 @@ func TestManager_NonInitiatorReturnsNoOffer(t *testing.T) { } func TestManager_StopIsIdempotent(t *testing.T) { - dA := NewManager("aaaa", &loopback{localID: "aaaa"}, newFakeWG(), nil) + dA := NewManager("aaaa", newFakeWG(), nil) + dA.SetTransport(&loopback{ep: epA, sw: newSwitch()}) dA.Stop() dA.Stop() // must not panic or hang } diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index da2e24f0d4d..b8bf5c02021 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -4,11 +4,8 @@ import ( "fmt" "net" "net/netip" - "sync" log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/client/internal/pqkem" ) // DefaultPort is the preferred UDP port for the ML-KEM data-path service, bound on @@ -17,25 +14,18 @@ import ( // announces Body.mlkemPort when a collision forced it onto a different port. const DefaultPort = 51833 -// pqTransport is the ML-KEM data-path transport: a dedicated UDP socket bound on the -// WireGuard overlay IP. Rekey messages travel through the tunnel to each peer's -// overlay IP and announced pqkem port. It implements pqkem.Transport and feeds -// inbound datagrams to the manager (set via setManager after construction, since the -// manager is built with this transport). +// pqTransport is the ML-KEM data-path transport: a dumb UDP socket bound on the WG +// overlay IP. It implements pqkem.Transport — the manager owns the remoteID<->endpoint +// routing and drives this socket's lifecycle (Run / Close). type pqTransport struct { conn *net.UDPConn port int - - mu sync.RWMutex - mgr *pqkem.Manager - peers map[string]*net.UDPAddr // remoteID (WG pubkey) -> overlay UDP addr to send to - byAddr map[string]string // source addr string -> remoteID (inbound dispatch) } // newPQTransport binds a UDP socket on the WG overlay IP, preferring DefaultPort and -// falling back to an OS-assigned ephemeral port if it is in use. It must be called -// after the WG interface is up so the overlay IP is assigned; when the bound port is -// not DefaultPort it is announced to peers via Body.mlkemPort. +// falling back to an OS-assigned ephemeral port if it is in use. Call it after the WG +// interface is up so the overlay IP is assigned; when the bound port is not +// DefaultPort it must be announced to peers via Body.mlkemPort. func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { if !overlayIP.IsValid() { return nil, fmt.Errorf("invalid overlay IP for pqkem transport") @@ -43,90 +33,40 @@ func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { ip := net.IP(overlayIP.AsSlice()) conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: DefaultPort}) if err != nil { - // Default port unavailable (rare on a dedicated overlay IP): fall back to an - // ephemeral port, which will be announced to peers. log.Debugf("pqkem: default port %d unavailable on %s (%v), using an ephemeral port", DefaultPort, overlayIP, err) conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) if err != nil { return nil, fmt.Errorf("bind pqkem udp on overlay %s: %w", overlayIP, err) } } - return &pqTransport{ - conn: conn, - port: conn.LocalAddr().(*net.UDPAddr).Port, - peers: make(map[string]*net.UDPAddr), - byAddr: make(map[string]string), - }, nil -} - -// Port is the bound UDP port, announced to peers as Body.mlkemPort. -func (t *pqTransport) Port() int { return t.port } - -func (t *pqTransport) setManager(m *pqkem.Manager) { - t.mu.Lock() - t.mgr = m - t.mu.Unlock() -} - -// AddPeer records where a peer's rekey messages are sent: its overlay IP and the -// pqkem port it announced. A zero port or invalid IP is ignored. -func (t *pqTransport) AddPeer(remoteID string, overlayIP netip.Addr, port int) { - if !overlayIP.IsValid() || port <= 0 { - return - } - addr := &net.UDPAddr{IP: net.IP(overlayIP.AsSlice()), Port: port} - t.mu.Lock() - if old, ok := t.peers[remoteID]; ok { - delete(t.byAddr, old.String()) - } - t.peers[remoteID] = addr - t.byAddr[addr.String()] = remoteID - t.mu.Unlock() + return &pqTransport{conn: conn, port: conn.LocalAddr().(*net.UDPAddr).Port}, nil } -func (t *pqTransport) RemovePeer(remoteID string) { - t.mu.Lock() - if a, ok := t.peers[remoteID]; ok { - delete(t.byAddr, a.String()) - delete(t.peers, remoteID) - } - t.mu.Unlock() -} - -// SendDataPath implements pqkem.Transport. -func (t *pqTransport) SendDataPath(remoteID string, msg []byte) error { - t.mu.RLock() - addr := t.peers[remoteID] - t.mu.RUnlock() - if addr == nil { - return fmt.Errorf("no data-path address for peer %s", remoteID) - } - _, err := t.conn.WriteToUDP(msg, addr) +// Send implements pqkem.Transport. +func (t *pqTransport) Send(endpoint netip.AddrPort, msg []byte) error { + _, err := t.conn.WriteToUDPAddrPort(msg, endpoint) return err } -// run is the receive loop: it maps each datagram's source overlay address to a peer -// and feeds it to the manager. Exits when the socket is closed. -func (t *pqTransport) run() { - buf := make([]byte, 2048) - for { - n, src, err := t.conn.ReadFromUDP(buf) - if err != nil { - return - } - t.mu.RLock() - remoteID := t.byAddr[src.String()] - mgr := t.mgr - t.mu.RUnlock() - if remoteID == "" || mgr == nil { - continue - } - msg := make([]byte, n) - copy(msg, buf[:n]) - if err := mgr.OnDataPathMessage(remoteID, msg); err != nil { - log.Debugf("pqkem: inbound from %s: %v", remoteID, err) +// LocalPort implements pqkem.Transport. +func (t *pqTransport) LocalPort() int { return t.port } + +// Run implements pqkem.Transport: the receive loop, delivering each datagram as +// (source endpoint, msg). Exits when the socket is closed. +func (t *pqTransport) Run(onInbound func(src netip.AddrPort, msg []byte)) { + go func() { + buf := make([]byte, 2048) + for { + n, src, err := t.conn.ReadFromUDPAddrPort(buf) + if err != nil { + return + } + msg := make([]byte, n) + copy(msg, buf[:n]) + onInbound(src, msg) } - } + }() } +// Close implements pqkem.Transport. func (t *pqTransport) Close() error { return t.conn.Close() } From ef29e996ce02e5a04283d5ef1104771408b382dc Mon Sep 17 00:00:00 2001 From: riccardom Date: Sat, 25 Jul 2026 12:12:32 +0200 Subject: [PATCH 028/119] Race fix --- client/internal/pqkem/convergence.go | 29 ++++++++--------------- client/internal/pqkem/convergence_test.go | 17 +++++++------ 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 83650d81537..299770b5499 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -140,12 +140,12 @@ func (m *Manager) ackConverged(remoteID string, ackID ExchangeID) { m.mu.Unlock() } -// initiatorLoop enforces the convergence deadline and retransmits the initiator's -// outstanding data-path offer while awaiting the answer (a signalling-bootstrapped -// offer is retransmitted by the host, so it is not resent here). It then waits, -// counting toward the deadline, in stateAwaitingRekey until OnDataPathRekeyed chains -// the next exchange (which supersedes and cancels this loop). Exhausting the deadline -// is a failure. +// initiatorLoop enforces the offer->answer convergence deadline and retransmits the +// initiator's outstanding data-path offer while awaiting the answer (a +// signalling-bootstrapped offer is retransmitted by the host, so it is not resent +// here). Exhausting the deadline before the answer arrives is a failure. Once the +// answer is in (state past awaitingAnswer) the loop exits: the next rotation is driven +// by OnDataPathRekeyed, and the idle wait for it has no deadline. func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id ExchangeID) { defer m.wait.Done() t := time.NewTicker(m.retryInterval) @@ -183,20 +183,11 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang } } - case stateAwaitingRekey: - // Waiting for OnDataPathRekeyed to chain the next exchange; the - // deadline still applies (the data path may never adopt the new key). - if attempts >= m.maxRetries { - delete(m.exchanges, remoteID) - fail := m.registerFailureLocked(remoteID) - m.mu.Unlock() - m.raiseFailure(remoteID, fail) - return - } - attempts++ - m.mu.Unlock() - default: + // Past awaiting the answer (converged) or superseded: the loop's job + // is done. The next rotation is driven externally by OnDataPathRekeyed, + // so there is no deadline while idle-waiting for it (that wait can be + // as long as the transport's natural rekey interval). m.mu.Unlock() return } diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index b60cd6cce68..c54e6e640c5 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -11,10 +11,10 @@ import ( // dropTransport is a pqkem.Transport that silently discards everything. type dropTransport struct{} -func (dropTransport) Send(netip.AddrPort, []byte) error { return nil } -func (dropTransport) LocalPort() int { return 0 } -func (dropTransport) Run(func(netip.AddrPort, []byte)) {} -func (dropTransport) Close() error { return nil } +func (dropTransport) Send(netip.AddrPort, []byte) error { return nil } +func (dropTransport) LocalPort() int { return 0 } +func (dropTransport) Run(func(netip.AddrPort, []byte)) {} +func (dropTransport) Close() error { return nil } func failedCount(f *fakeWG) int { f.mu.Lock() @@ -44,6 +44,11 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { defer dA.Stop() defer dB.Stop() + // Tighten B's timings before any exchange loop spawns (the loop reads these + // fields, so writing them after a loop is running would race). + dB.retryInterval = 5 * time.Millisecond + dB.maxRetries = 2 + // Establish: bootstrap + data-path-rekeyed so B becomes established and its data // path is usable. bootstrap(t, dA, dB) @@ -51,9 +56,7 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { dB.OnDataPathRekeyed("aaaa") require.NotEqual(t, PSK{}, wgB.psk("aaaa")) - // Tighten timings and drop B's outbound so rekeys can no longer converge. - dB.retryInterval = 5 * time.Millisecond - dB.maxRetries = 2 + // Drop B's outbound so rekeys can no longer converge. lbB.drop.Store(true) // K-1 data-path rekeys must NOT raise OnRekeyFailed. From 539a1f44c87bb04afc3aeaaa5af9eb3380f11b52 Mon Sep 17 00:00:00 2001 From: riccardom Date: Sat, 25 Jul 2026 12:12:39 +0200 Subject: [PATCH 029/119] Typo --- client/internal/pqkem/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index d265c18d1e0..f1e7dc9a780 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -89,7 +89,7 @@ type Manager struct { mu sync.Mutex transport Transport exchanges map[string]*exchangeCtl // in-flight exchange per peer - established map[string]bool // peer has completed at least one exchange + established map[string]bool // peer has completed at least one exchange failures map[string]int // consecutive rekey failures per peer peers map[string]netip.AddrPort // remoteID -> data-path endpoint (send routing) peersByAddr map[netip.AddrPort]string // reverse: source endpoint -> remoteID (inbound) From 3270fc5e1860c5a1fd0355615b4f5c6b657449da Mon Sep 17 00:00:00 2001 From: riccardom Date: Sat, 25 Jul 2026 13:10:45 +0200 Subject: [PATCH 030/119] Bit of renaming peer -> peerAddrs have types for remoteID and localID t.Close log error Manager SetTransport -> Start --- client/internal/engine.go | 4 +- client/internal/pqkem/callbacks.go | 4 +- client/internal/pqkem/convergence.go | 14 ++-- client/internal/pqkem/convergence_test.go | 2 +- client/internal/pqkem/manager.go | 85 +++++++++++++---------- client/internal/pqkem/manager_test.go | 18 ++--- client/internal/pqkem_adapter.go | 6 +- 7 files changed, 71 insertions(+), 62 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 21dc55f412e..035d85f2fde 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -689,8 +689,8 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqErr != nil { log.Errorf("ML-KEM PQ transport bind failed, PQ exchange disabled: %v", pqErr) } else { - e.pqkemManager = pqkem.NewManager(publicKey.String(), pqCallbackHandler{wg: e.wgInterface}, nil) - e.pqkemManager.SetTransport(tr) + e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), pqCallbackHandler{wg: e.wgInterface}, nil) + e.pqkemManager.Start(tr) log.Infof("ML-KEM post-quantum exchange enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) } } diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index 9dbb8e4bb5e..3302e2e66f4 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -9,10 +9,10 @@ type CallbackHandler interface { // and must be programmed into the consumer's secure channel. It is invoked at // the commit point of each side: the initiator on receiving the answer, the // responder on receiving the confirm. - OnNewPSKReady(remoteID string, psk PSK) error + OnNewPSKReady(remoteID RemoteID, psk PSK) error // OnRekeyFailed fires when an exchange fails to converge within the allotted // time. The host should tear the peer connection down so it re-establishes, and // log a WARN. The library reports the event; it does not dictate the reaction. - OnRekeyFailed(remoteID string) error + OnRekeyFailed(remoteID RemoteID) error } diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 299770b5499..be70ca0bbb6 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -9,7 +9,7 @@ import ( // bootstrap) and returns the framed offer for the caller to send — pushed over the // data path for a chained rekey, or handed to the host for signalling when viaSignal // is set. Any previous in-flight exchange for the peer is cancelled. -func (m *Manager) startExchange(remoteID string, viaSignal bool, ackID ExchangeID) ([]byte, error) { +func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID ExchangeID) ([]byte, error) { init, err := NewInitiator() if err != nil { return nil, err @@ -48,7 +48,7 @@ func (m *Manager) startExchange(remoteID string, viaSignal bool, ackID ExchangeI // (that offer riding the data path under the freshly adopted key proves it worked), // then derives the PSK for the new offer, commits it optimistically, and returns the // framed answer. A duplicate offer returns the cached answer without re-deriving. -func (m *Manager) processOffer(remoteID string, o *OfferMsg) ([]byte, error) { +func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { if o.AckID != (ExchangeID{}) { m.ackConverged(remoteID, o.AckID) } @@ -97,7 +97,7 @@ func (m *Manager) processOffer(remoteID string, o *OfferMsg) ([]byte, error) { // stateAwaitingRekey; the next offer (chained from OnDataPathRekeyed) will acknowledge // this exchange. Only valid in stateAwaitingAnswer; advancing the state under the // lock makes a concurrent/duplicate answer bail. -func (m *Manager) processAnswer(remoteID string, a *AnswerMsg) error { +func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { @@ -126,7 +126,7 @@ func (m *Manager) processAnswer(remoteID string, a *AnswerMsg) error { // ackConverged (responder) records convergence of the exchange named by ackID: a // later offer acknowledging it proves both sides operate on that exchange's key. Only // acts on a matching stateAwaitingAck exchange; anything else is ignored. -func (m *Manager) ackConverged(remoteID string, ackID ExchangeID) { +func (m *Manager) ackConverged(remoteID RemoteID, ackID ExchangeID) { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != ackID || ex.state != stateAwaitingAck { @@ -146,7 +146,7 @@ func (m *Manager) ackConverged(remoteID string, ackID ExchangeID) { // here). Exhausting the deadline before the answer arrives is a failure. Once the // answer is in (state past awaitingAnswer) the loop exits: the next rotation is driven // by OnDataPathRekeyed, and the idle wait for it has no deadline. -func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id ExchangeID) { +func (m *Manager) initiatorLoop(ctx context.Context, remoteID RemoteID, id ExchangeID) { defer m.wait.Done() t := time.NewTicker(m.retryInterval) defer t.Stop() @@ -199,7 +199,7 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID string, id Exchang // an initial exchange (peer never established) fails immediately; a rekey tolerates // up to maxRekeyFailures consecutive misses (we stay on the still-valid previous // PSK) before failing. Assumes m.mu is held. -func (m *Manager) registerFailureLocked(remoteID string) bool { +func (m *Manager) registerFailureLocked(remoteID RemoteID) bool { if !m.established[remoteID] { return true } @@ -211,7 +211,7 @@ func (m *Manager) registerFailureLocked(remoteID string) bool { return false } -func (m *Manager) raiseFailure(remoteID string, fail bool) { +func (m *Manager) raiseFailure(remoteID RemoteID, fail bool) { if !fail { m.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteID) return diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index c54e6e640c5..e6749150944 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -25,7 +25,7 @@ func failedCount(f *fakeWG) int { func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { wg := newFakeWG() d := NewManager("bbbb", wg, nil) // bbbb > aaaa -> initiator - d.SetTransport(dropTransport{}) + d.Start(dropTransport{}) d.retryInterval = 5 * time.Millisecond d.maxRetries = 3 defer d.Stop() diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index f1e7dc9a780..3718444ff1c 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -22,11 +22,18 @@ const ( DefaultMaxRekeyFailures = 3 ) +// LocalID and RemoteID are peer identity keys (e.g. WireGuard public keys). They are +// distinct types so the local and a remote identity cannot be mixed up. +type ( + LocalID string + RemoteID string +) + // Transport is the data-path socket the Manager drives (the analogue of // go-rosenpass's Conn). It is a dumb mover of bytes to/from endpoints: the Manager // owns the remoteID<->endpoint routing and hands the transport a resolved endpoint // to Send, and reverse-resolves the source of each inbound datagram. Its lifecycle -// belongs to the Manager (Run at SetTransport, Close at Stop). +// belongs to the Manager (Run at Start, Close at Stop). type Transport interface { // Send delivers msg to the given data-path endpoint. Send(endpoint netip.AddrPort, msg []byte) error @@ -75,7 +82,7 @@ type exchangeCtl struct { // (SignalOffer) and each rotation is clocked by OnDataPathRekeyed. The cryptography is // the pure kem.go primitives; all state lives here under one lock. type Manager struct { - localID string + localID LocalID cbHandler CallbackHandler logger *slog.Logger @@ -88,18 +95,18 @@ type Manager struct { mu sync.Mutex transport Transport - exchanges map[string]*exchangeCtl // in-flight exchange per peer - established map[string]bool // peer has completed at least one exchange - failures map[string]int // consecutive rekey failures per peer - peers map[string]netip.AddrPort // remoteID -> data-path endpoint (send routing) - peersByAddr map[netip.AddrPort]string // reverse: source endpoint -> remoteID (inbound) + exchanges map[RemoteID]*exchangeCtl // in-flight exchange per peer + established map[RemoteID]bool // peer has completed at least one exchange + failures map[RemoteID]int // consecutive rekey failures per peer + peerAddrs map[RemoteID]netip.AddrPort // remoteID -> data-path endpoint (send routing) + peersByAddr map[netip.AddrPort]RemoteID // reverse: source endpoint -> remoteID (inbound) wait sync.WaitGroup } // NewManager builds a manager for the local peer identified by its peer identity key // (used for the deterministic initiator role and the identity binding). A nil logger -// falls back to slog.Default(). Set the data-path transport with SetTransport. -func NewManager(localID string, h CallbackHandler, logger *slog.Logger) *Manager { +// falls back to slog.Default(). Install the data-path transport with Start. +func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manager { if logger == nil { logger = slog.Default() } @@ -113,17 +120,17 @@ func NewManager(localID string, h CallbackHandler, logger *slog.Logger) *Manager maxRekeyFailures: DefaultMaxRekeyFailures, rootCtx: ctx, rootCancel: cancel, - exchanges: make(map[string]*exchangeCtl), - established: make(map[string]bool), - failures: make(map[string]int), - peers: make(map[string]netip.AddrPort), - peersByAddr: make(map[netip.AddrPort]string), + exchanges: make(map[RemoteID]*exchangeCtl), + established: make(map[RemoteID]bool), + failures: make(map[RemoteID]int), + peerAddrs: make(map[RemoteID]netip.AddrPort), + peersByAddr: make(map[netip.AddrPort]RemoteID), } } -// SetTransport installs the data-path transport and starts its inbound delivery. The -// Manager owns it from here: Stop closes it. -func (m *Manager) SetTransport(t Transport) { +// Start installs the data-path transport and begins its inbound delivery. The Manager +// owns it from here; Stop closes it. Start/Stop are the transport lifecycle pair. +func (m *Manager) Start(t Transport) { m.mu.Lock() m.transport = t m.mu.Unlock() @@ -147,27 +154,27 @@ func (m *Manager) LocalPort() int { // IsInitiator reports whether the local peer drives the exchange for this remote // peer. Roles are deterministic (lexicographic identity-key compare) so exactly one // side initiates, mirroring how Rosenpass picks its handshake initiator. -func (m *Manager) IsInitiator(remoteID string) bool { - return m.localID > remoteID +func (m *Manager) IsInitiator(remoteID RemoteID) bool { + return string(m.localID) > string(remoteID) } // AddPeer registers where a peer's data-path messages are sent and received: its // overlay endpoint (IP:port). Re-adding updates the endpoint. -func (m *Manager) AddPeer(remoteID string, endpoint netip.AddrPort) { +func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { if !endpoint.IsValid() { return } m.mu.Lock() - if old, ok := m.peers[remoteID]; ok { + if old, ok := m.peerAddrs[remoteID]; ok { delete(m.peersByAddr, old) } - m.peers[remoteID] = endpoint + m.peerAddrs[remoteID] = endpoint m.peersByAddr[endpoint] = remoteID m.mu.Unlock() } // RemovePeer stops any in-flight exchange for a peer and drops its state and routing. -func (m *Manager) RemovePeer(remoteID string) { +func (m *Manager) RemovePeer(remoteID RemoteID) { m.mu.Lock() if ex, ok := m.exchanges[remoteID]; ok { if ex.cancel != nil { @@ -177,9 +184,9 @@ func (m *Manager) RemovePeer(remoteID string) { } delete(m.established, remoteID) delete(m.failures, remoteID) - if ep, ok := m.peers[remoteID]; ok { + if ep, ok := m.peerAddrs[remoteID]; ok { delete(m.peersByAddr, ep) - delete(m.peers, remoteID) + delete(m.peerAddrs, remoteID) } m.mu.Unlock() } @@ -192,10 +199,12 @@ func (m *Manager) Stop() { m.mu.Lock() t := m.transport m.transport = nil - m.exchanges = make(map[string]*exchangeCtl) + m.exchanges = make(map[RemoteID]*exchangeCtl) m.mu.Unlock() if t != nil { - _ = t.Close() + if err := t.Close(); err != nil { + m.logger.Warn("pqkem: closing data-path transport", "err", err) + } } } @@ -205,7 +214,7 @@ func (m *Manager) Stop() { // remoteID (bootstrap). It returns (nil, nil) when the local peer is not the // initiator. It is idempotent for an in-flight bootstrap: a repeat call returns the // same offer rather than starting a new exchange. -func (m *Manager) SignalOffer(remoteID string) ([]byte, error) { +func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { if !m.IsInitiator(remoteID) { return nil, nil } @@ -222,7 +231,7 @@ func (m *Manager) SignalOffer(remoteID string) ([]byte, error) { // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and // returns the KEM answer for the host to embed in its outgoing answer. -func (m *Manager) SignalOnOffer(remoteID string, offer []byte) ([]byte, error) { +func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) { typ, msg, err := Decode(offer) if err != nil { return nil, fmt.Errorf("decode signal offer from %s: %w", remoteID, err) @@ -235,7 +244,7 @@ func (m *Manager) SignalOnOffer(remoteID string, offer []byte) ([]byte, error) { // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. // There is no reply: the next offer (over the data path) acknowledges this exchange. -func (m *Manager) SignalOnAnswer(remoteID string, answer []byte) error { +func (m *Manager) SignalOnAnswer(remoteID RemoteID, answer []byte) error { typ, msg, err := Decode(answer) if err != nil { return fmt.Errorf("decode signal answer from %s: %w", remoteID, err) @@ -252,9 +261,9 @@ func (m *Manager) SignalOnAnswer(remoteID string, answer []byte) error { // source endpoint to a peer and dispatches. Unknown sources are dropped. func (m *Manager) onDataPathInbound(src netip.AddrPort, msg []byte) { m.mu.Lock() - remoteID := m.peersByAddr[src] + remoteID, ok := m.peersByAddr[src] m.mu.Unlock() - if remoteID == "" { + if !ok { return } if err := m.OnDataPathMessage(remoteID, msg); err != nil { @@ -264,7 +273,7 @@ func (m *Manager) onDataPathInbound(src netip.AddrPort, msg []byte) { // OnDataPathMessage handles a KEM message received over the data path from remoteID // and pushes any reply back over the data path. -func (m *Manager) OnDataPathMessage(remoteID string, raw []byte) error { +func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { typ, msg, err := Decode(raw) if err != nil { return fmt.Errorf("decode data-path msg from %s: %w", remoteID, err) @@ -291,7 +300,7 @@ func (m *Manager) OnDataPathMessage(remoteID string, raw []byte) error { // initiator that just derived a PSK, it chains the next exchange: a fresh offer over // the data path that acknowledges the just-completed one (its arrival under the new // key proves to the responder that the key works). -func (m *Manager) OnDataPathRekeyed(remoteID string) { +func (m *Manager) OnDataPathRekeyed(remoteID RemoteID) { m.mu.Lock() ex := m.exchanges[remoteID] chain := ex != nil && ex.state == stateAwaitingRekey @@ -317,15 +326,15 @@ func (m *Manager) OnDataPathRekeyed(remoteID string) { // OnDataPathDown notifies that the peer's data path went down. Rotations resume once // the host re-bootstraps over signalling on reconnect; in-flight data-path sends will // simply fail until then. Reserved as an explicit hook. -func (m *Manager) OnDataPathDown(remoteID string) {} +func (m *Manager) OnDataPathDown(remoteID RemoteID) {} // ---- internals ---- // pushDataPath resolves the peer's endpoint and sends over the data-path transport, // erroring if the peer is unknown or no transport is set. -func (m *Manager) pushDataPath(remoteID string, msg []byte) error { +func (m *Manager) pushDataPath(remoteID RemoteID, msg []byte) error { m.mu.Lock() - ep, ok := m.peers[remoteID] + ep, ok := m.peerAddrs[remoteID] t := m.transport m.mu.Unlock() if !ok { @@ -337,7 +346,7 @@ func (m *Manager) pushDataPath(remoteID string, msg []byte) error { return t.Send(ep, msg) } -func (m *Manager) binding(remoteID string) Binding { +func (m *Manager) binding(remoteID RemoteID) Binding { return Binding{LocalID: []byte(m.localID), RemoteID: []byte(remoteID)} } diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index ac7be825843..bbe8ab161cc 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -59,27 +59,27 @@ func (l *loopback) Close() error { return nil } type fakeWG struct { mu sync.Mutex - psks map[string]PSK - failed []string + psks map[RemoteID]PSK + failed []RemoteID } -func newFakeWG() *fakeWG { return &fakeWG{psks: map[string]PSK{}} } +func newFakeWG() *fakeWG { return &fakeWG{psks: map[RemoteID]PSK{}} } -func (f *fakeWG) OnNewPSKReady(remoteID string, psk PSK) error { +func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK) error { f.mu.Lock() defer f.mu.Unlock() f.psks[remoteID] = psk return nil } -func (f *fakeWG) OnRekeyFailed(remoteID string) error { +func (f *fakeWG) OnRekeyFailed(remoteID RemoteID) error { f.mu.Lock() defer f.mu.Unlock() f.failed = append(f.failed, remoteID) return nil } -func (f *fakeWG) psk(peer string) PSK { +func (f *fakeWG) psk(peer RemoteID) PSK { f.mu.Lock() defer f.mu.Unlock() return f.psks[peer] @@ -100,9 +100,9 @@ func pair(t *testing.T) (dA, dB *Manager, wgA, wgB *fakeWG, lbB *loopback) { wgB = newFakeWG() dA = NewManager("aaaa", wgA, nil) dB = NewManager("bbbb", wgB, nil) - dA.SetTransport(&loopback{ep: epA, sw: sw}) + dA.Start(&loopback{ep: epA, sw: sw}) lbB = &loopback{ep: epB, sw: sw} - dB.SetTransport(lbB) + dB.Start(lbB) dA.AddPeer("bbbb", epB) dB.AddPeer("aaaa", epA) return dA, dB, wgA, wgB, lbB @@ -163,7 +163,7 @@ func TestManager_NonInitiatorReturnsNoOffer(t *testing.T) { func TestManager_StopIsIdempotent(t *testing.T) { dA := NewManager("aaaa", newFakeWG(), nil) - dA.SetTransport(&loopback{ep: epA, sw: newSwitch()}) + dA.Start(&loopback{ep: epA, sw: newSwitch()}) dA.Stop() dA.Stop() // must not panic or hang } diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 1a94c65a450..26c0165bf19 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -21,13 +21,13 @@ type pqCallbackHandler struct { // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op // if the peer is not present, mirroring Rosenpass). remoteID is the peer's WG pubkey. -func (h pqCallbackHandler) OnNewPSKReady(remoteID string, psk pqkem.PSK) error { - return h.wg.SetPresharedKey(remoteID, wgtypes.Key(psk), true) +func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) error { + return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) } // OnRekeyFailed reports a failed PQ (re)key convergence. // TODO(NET-1406): tear the peer connection down / trigger ICE reconnect. -func (h pqCallbackHandler) OnRekeyFailed(remoteID string) error { +func (h pqCallbackHandler) OnRekeyFailed(remoteID pqkem.RemoteID) error { log.Warnf("pqkem: post-quantum rekey failed for peer %s", remoteID) return nil } From bac6af349ba9ebb9d9492171c3efa019805796ba Mon Sep 17 00:00:00 2001 From: riccardom Date: Sat, 25 Jul 2026 13:33:11 +0200 Subject: [PATCH 031/119] Homogeneous logs prefix --- client/internal/engine.go | 4 ++-- client/internal/pqkem/convergence.go | 6 +++--- client/internal/pqkem/manager.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 035d85f2fde..9a3ed86ce4d 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -687,11 +687,11 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqkem.Enabled() { tr, pqErr := newPQTransport(e.config.WgAddr.IP) if pqErr != nil { - log.Errorf("ML-KEM PQ transport bind failed, PQ exchange disabled: %v", pqErr) + log.Errorf("pqkem: transport bind failed, exchange disabled: %v", pqErr) } else { e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), pqCallbackHandler{wg: e.wgInterface}, nil) e.pqkemManager.Start(tr) - log.Infof("ML-KEM post-quantum exchange enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) + log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) } } diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index be70ca0bbb6..dc6364a8b17 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -179,7 +179,7 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID RemoteID, id Excha m.mu.Unlock() if !viaSignal { if err := m.pushDataPath(remoteID, msg); err != nil { - m.logger.Warn("pqkem offer retransmit failed", "peer", remoteID, "err", err) + m.logger.Warn("pqkem: offer retransmit failed", "peer", remoteID, "err", err) } } @@ -213,10 +213,10 @@ func (m *Manager) registerFailureLocked(remoteID RemoteID) bool { func (m *Manager) raiseFailure(remoteID RemoteID, fail bool) { if !fail { - m.logger.Warn("pqkem rekey attempt timed out, will retry next cycle", "peer", remoteID) + m.logger.Warn("pqkem: rekey attempt timed out, will retry next cycle", "peer", remoteID) return } if err := m.cbHandler.OnRekeyFailed(remoteID); err != nil { - m.logger.Error("pqkem OnRekeyFailed handler error", "peer", remoteID, "err", err) + m.logger.Error("pqkem: OnRekeyFailed handler error", "peer", remoteID, "err", err) } } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 3718444ff1c..de0615bdebe 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -267,7 +267,7 @@ func (m *Manager) onDataPathInbound(src netip.AddrPort, msg []byte) { return } if err := m.OnDataPathMessage(remoteID, msg); err != nil { - m.logger.Debug("pqkem inbound", "peer", remoteID, "err", err) + m.logger.Debug("pqkem: inbound", "peer", remoteID, "err", err) } } @@ -315,11 +315,11 @@ func (m *Manager) OnDataPathRekeyed(remoteID RemoteID) { } offer, err := m.startExchange(remoteID, false, ackID) if err != nil { - m.logger.Error("pqkem chain offer failed to start", "peer", remoteID, "err", err) + m.logger.Error("pqkem: chain offer failed to start", "peer", remoteID, "err", err) return } if err := m.pushDataPath(remoteID, offer); err != nil { - m.logger.Warn("pqkem send chain offer failed", "peer", remoteID, "err", err) + m.logger.Warn("pqkem: send chain offer failed", "peer", remoteID, "err", err) } } From 56b315d9548d386919cd36c1047b871057477be9 Mon Sep 17 00:00:00 2001 From: riccardom Date: Sun, 26 Jul 2026 06:57:40 +0200 Subject: [PATCH 032/119] pqkem: dedicated slog logger via NB_PQ_MLKEM_LOG_LEVEL --- client/internal/engine.go | 2 +- client/internal/pqkem/env.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 9a3ed86ce4d..a0eab54c630 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -689,7 +689,7 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqErr != nil { log.Errorf("pqkem: transport bind failed, exchange disabled: %v", pqErr) } else { - e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), pqCallbackHandler{wg: e.wgInterface}, nil) + e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), pqCallbackHandler{wg: e.wgInterface}, pqkem.NewLogger()) e.pqkemManager.Start(tr) log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) } diff --git a/client/internal/pqkem/env.go b/client/internal/pqkem/env.go index 52423a76243..8eb4026725c 100644 --- a/client/internal/pqkem/env.go +++ b/client/internal/pqkem/env.go @@ -1,6 +1,7 @@ package pqkem import ( + "log/slog" "os" "strconv" "strings" @@ -32,3 +33,27 @@ func Enabled() bool { } return enabled } + +// EnvLogLevel overrides the ML-KEM manager's slog level (debug/info/warn/error). +// Defaults to info. +const EnvLogLevel = "NB_PQ_MLKEM_LOG_LEVEL" + +// NewLogger builds the slog logger for the ML-KEM manager: a text handler to stdout +// at the level from EnvLogLevel. Mirrors the Rosenpass manager's logger setup so PQ +// components log consistently. +func NewLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel()})) +} + +func logLevel() slog.Level { + switch strings.ToLower(strings.TrimSpace(os.Getenv(EnvLogLevel))) { + case "debug": + return slog.LevelDebug + case "warn": + return slog.LevelWarn + case "error": + return slog.LevelError + default: + return slog.LevelInfo + } +} From 555310bd56fa564d4afe39458c44a150970c410f Mon Sep 17 00:00:00 2001 From: riccardom Date: Sun, 26 Jul 2026 13:15:51 +0200 Subject: [PATCH 033/119] pqkem: carry KEM offer/answer over the signalling exchange --- client/internal/engine.go | 3 +++ client/internal/peer/conn.go | 18 ++++++++++++++++ client/internal/peer/handshaker.go | 18 ++++++++++++++-- client/internal/pqkem_adapter.go | 34 ++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index a0eab54c630..4c299c3b597 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -2006,6 +2006,9 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV ICEConfig: e.createICEConfig(), NetMgr: e.netMgr, } + if e.pqkemManager != nil { + config.PQ = pqHandshaker{mgr: e.pqkemManager} + } serviceDependencies := peer.ServiceDependencies{ StatusRecorder: e.statusRecorder, diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 83089606fd4..a43e5bd9ea2 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -75,6 +75,21 @@ type RosenpassConfig struct { PermissiveMode bool } +// PQHandshaker attaches post-quantum ML-KEM material to signalling offers/answers and +// feeds received material back. It is implemented by the engine over the pqkem +// manager and is nil when the PQ exchange is disabled. remoteKey is the peer's +// WireGuard public key. +type PQHandshaker interface { + // OfferPayload returns the KEM offer to embed in an outgoing offer (nil if this + // peer is not the KEM initiator) and the local PQ data-path port to announce. + OfferPayload(remoteKey string) (payload []byte, port int) + // AnswerPayload processes a received KEM offer (nil if absent) and returns the KEM + // answer to embed in the outgoing answer (nil if none) and the local PQ port. + AnswerPayload(remoteKey string, recvOffer []byte) (payload []byte, port int) + // OnAnswer feeds a received KEM answer (nil if absent). + OnAnswer(remoteKey string, recvAnswer []byte) +} + // ConnConfig is a peer Connection configuration type ConnConfig struct { // Key is a public key of a remote peer @@ -92,6 +107,9 @@ type ConnConfig struct { RosenpassConfig RosenpassConfig + // PQ carries post-quantum ML-KEM material on offers/answers; nil when disabled. + PQ PQHandshaker + // ICEConfig ICE protocol configuration ICEConfig icemaker.Config diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 77ee213e53c..82b35dd79cc 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -143,7 +143,7 @@ func (h *Handshaker) Listen(ctx context.Context) { h.iceListener(&remoteOfferAnswer) } - if err := h.sendAnswer(); err != nil { + if err := h.sendAnswer(&remoteOfferAnswer); err != nil { h.log.Errorf("failed to send remote offer confirmation: %s", err) continue } @@ -164,6 +164,10 @@ func (h *Handshaker) Listen(ctx context.Context) { if h.iceListener != nil && h.RemoteICESupported() { h.iceListener(&remoteOfferAnswer) } + + if h.config.PQ != nil { + h.config.PQ.OnAnswer(h.config.Key, remoteOfferAnswer.MlkemPayload) + } case <-ctx.Done(): h.log.Infof("stop listening for remote offers and answers") return @@ -219,13 +223,23 @@ func (h *Handshaker) sendOffer() error { } offer := h.buildOfferAnswer() + if h.config.PQ != nil { + offer.MlkemPayload, offer.MlkemPort = h.config.PQ.OfferPayload(h.config.Key) + } h.log.Debugf("sending offer with serial: %s", offer.SessionIDString()) return h.signaler.SignalOffer(offer, h.config.Key) } -func (h *Handshaker) sendAnswer() error { +func (h *Handshaker) sendAnswer(remoteOffer *OfferAnswer) error { answer := h.buildOfferAnswer() + if h.config.PQ != nil { + var recvOffer []byte + if remoteOffer != nil { + recvOffer = remoteOffer.MlkemPayload + } + answer.MlkemPayload, answer.MlkemPort = h.config.PQ.AnswerPayload(h.config.Key, recvOffer) + } h.log.Debugf("sending answer with serial: %s", answer.SessionIDString()) return h.signaler.SignalAnswer(answer, h.config.Key) diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 26c0165bf19..4373520d77c 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -31,3 +31,37 @@ func (h pqCallbackHandler) OnRekeyFailed(remoteID pqkem.RemoteID) error { log.Warnf("pqkem: post-quantum rekey failed for peer %s", remoteID) return nil } + +// pqHandshaker adapts the pqkem manager to peer.PQHandshaker (string peer keys), +// wiring the host's signalling offers/answers to the KEM exchange. +type pqHandshaker struct { + mgr *pqkem.Manager +} + +func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, int) { + payload, err := p.mgr.SignalOffer(pqkem.RemoteID(remoteKey)) + if err != nil { + log.Warnf("pqkem: build offer for %s: %v", remoteKey, err) + } + return payload, p.mgr.LocalPort() +} + +func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, int) { + if len(recvOffer) == 0 { + return nil, p.mgr.LocalPort() + } + payload, err := p.mgr.SignalOnOffer(pqkem.RemoteID(remoteKey), recvOffer) + if err != nil { + log.Warnf("pqkem: build answer for %s: %v", remoteKey, err) + } + return payload, p.mgr.LocalPort() +} + +func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { + if len(recvAnswer) == 0 { + return + } + if err := p.mgr.SignalOnAnswer(pqkem.RemoteID(remoteKey), recvAnswer); err != nil { + log.Warnf("pqkem: process answer from %s: %v", remoteKey, err) + } +} From 7c276ce727aaa3e160e5a4c3c5701ee9ecdcc46f Mon Sep 17 00:00:00 2001 From: riccardom Date: Sun, 26 Jul 2026 14:04:51 +0200 Subject: [PATCH 034/119] pqkem: apply derived PSK at WG peer-config time (pull) + keep push for rekey --- client/internal/peer/conn.go | 12 ++++++++++++ client/internal/pqkem/convergence.go | 2 ++ client/internal/pqkem/manager.go | 13 +++++++++++++ client/internal/pqkem_adapter.go | 13 +++++++++++++ 4 files changed, 40 insertions(+) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index a43e5bd9ea2..cbb6746a0c3 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -88,6 +88,9 @@ type PQHandshaker interface { AnswerPayload(remoteKey string, recvOffer []byte) (payload []byte, port int) // OnAnswer feeds a received KEM answer (nil if absent). OnAnswer(remoteKey string, recvAnswer []byte) + // PSK returns the peer's latest derived post-quantum PSK to program at WG + // peer-config time (the pull path). ok is false until one has been derived. + PSK(remoteKey string) (wgtypes.Key, bool) } // ConnConfig is a peer Connection configuration @@ -1008,6 +1011,15 @@ func (conn *Conn) AgentVersionString() string { } func (conn *Conn) presharedKey(remoteRosenpassKey []byte) *wgtypes.Key { + // Post-quantum: once the ML-KEM exchange has derived a PSK for this peer, program + // it here so the peer's next WireGuard handshake adopts it. Applied at peer-config + // time (bootstrap / reconnect); steady-state rotation is pushed separately. + if conn.config.PQ != nil { + if psk, ok := conn.config.PQ.PSK(conn.config.Key); ok { + return &psk + } + } + if conn.config.RosenpassConfig.PubKey == nil { return conn.config.WgConfig.PreSharedKey } diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index dc6364a8b17..63e1d61e9d3 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -84,6 +84,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { ex.state = stateAwaitingAck ex.lastSent = raw ex.pendingPSK = psk + m.psks[remoteID] = psk m.mu.Unlock() // Commit optimistically so our data path can rekey to the new PSK. @@ -118,6 +119,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.mu.Lock() m.established[remoteID] = true m.failures[remoteID] = 0 + m.psks[remoteID] = psk m.mu.Unlock() return m.cbHandler.OnNewPSKReady(remoteID, psk) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index de0615bdebe..9cc06a00a41 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -98,6 +98,7 @@ type Manager struct { exchanges map[RemoteID]*exchangeCtl // in-flight exchange per peer established map[RemoteID]bool // peer has completed at least one exchange failures map[RemoteID]int // consecutive rekey failures per peer + psks map[RemoteID]PSK // latest derived PSK per peer (pulled at WG peer-config time) peerAddrs map[RemoteID]netip.AddrPort // remoteID -> data-path endpoint (send routing) peersByAddr map[netip.AddrPort]RemoteID // reverse: source endpoint -> remoteID (inbound) wait sync.WaitGroup @@ -123,6 +124,7 @@ func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manage exchanges: make(map[RemoteID]*exchangeCtl), established: make(map[RemoteID]bool), failures: make(map[RemoteID]int), + psks: make(map[RemoteID]PSK), peerAddrs: make(map[RemoteID]netip.AddrPort), peersByAddr: make(map[netip.AddrPort]RemoteID), } @@ -158,6 +160,15 @@ func (m *Manager) IsInitiator(remoteID RemoteID) bool { return string(m.localID) > string(remoteID) } +// PSK returns the latest PSK derived for the peer, for the host to program at WG +// peer-config time (the pull path). ok is false until an exchange has derived one. +func (m *Manager) PSK(remoteID RemoteID) (PSK, bool) { + m.mu.Lock() + defer m.mu.Unlock() + psk, ok := m.psks[remoteID] + return psk, ok +} + // AddPeer registers where a peer's data-path messages are sent and received: its // overlay endpoint (IP:port). Re-adding updates the endpoint. func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { @@ -184,6 +195,7 @@ func (m *Manager) RemovePeer(remoteID RemoteID) { } delete(m.established, remoteID) delete(m.failures, remoteID) + delete(m.psks, remoteID) if ep, ok := m.peerAddrs[remoteID]; ok { delete(m.peersByAddr, ep) delete(m.peerAddrs, remoteID) @@ -200,6 +212,7 @@ func (m *Manager) Stop() { t := m.transport m.transport = nil m.exchanges = make(map[RemoteID]*exchangeCtl) + m.psks = make(map[RemoteID]PSK) m.mu.Unlock() if t != nil { if err := t.Close(); err != nil { diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 4373520d77c..611e72498c7 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -22,6 +22,10 @@ type pqCallbackHandler struct { // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op // if the peer is not present, mirroring Rosenpass). remoteID is the peer's WG pubkey. func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) error { + // updateOnly: applies to an already-configured peer (rotation). At bootstrap the + // peer is not configured yet, so this is a no-op there and the PSK is instead + // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). + log.Debugf("pqkem: programming PSK for peer %s", remoteID) return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) } @@ -65,3 +69,12 @@ func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { log.Warnf("pqkem: process answer from %s: %v", remoteKey, err) } } + +// PSK exposes the peer's derived PSK for the conn to program at WG peer-config time. +func (p pqHandshaker) PSK(remoteKey string) (wgtypes.Key, bool) { + psk, ok := p.mgr.PSK(pqkem.RemoteID(remoteKey)) + if !ok { + return wgtypes.Key{}, false + } + return wgtypes.Key(psk), true +} From e3ab585c5640c324b697679277ace0c97fbe8c80 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 27 Jul 2026 16:18:35 +0200 Subject: [PATCH 035/119] pqkem: register data-path endpoint from signalling Learn the peer's data-path endpoint from the signalling offer/answer: its WG overlay IP combined with the advertised pq UDP port (SetRemotePort -> AddPeer). Registering here is safe before the tunnel is up because sends only ever fire once it is (clocked by OnDataPathRekeyed). RemovePeer is wired at peer teardown (engine.removePeer), not on transient disconnect. --- client/internal/engine.go | 4 ++++ client/internal/peer/conn.go | 3 +++ client/internal/peer/handshaker.go | 13 +++++++++++++ client/internal/pqkem_adapter.go | 12 ++++++++++++ 4 files changed, 32 insertions(+) diff --git a/client/internal/engine.go b/client/internal/engine.go index 4c299c3b597..b33284dd09c 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -962,6 +962,10 @@ func (e *Engine) removePeer(peerKey string) error { e.connMgr.RemovePeerConn(peerKey) + if e.pqkemManager != nil { + e.pqkemManager.RemovePeer(pqkem.RemoteID(peerKey)) + } + err := e.statusRecorder.RemovePeer(peerKey) if err != nil { log.Warnf("received error when removing peer %s from status recorder: %v", peerKey, err) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index cbb6746a0c3..9c3e26d8e41 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -91,6 +91,9 @@ type PQHandshaker interface { // PSK returns the peer's latest derived post-quantum PSK to program at WG // peer-config time (the pull path). ok is false until one has been derived. PSK(remoteKey string) (wgtypes.Key, bool) + // SetRemotePort registers the peer's data-path endpoint from signalling: the peer's + // WG overlay IP combined with its advertised pq UDP port. + SetRemotePort(remoteKey string, overlayIP netip.Addr, port int) } // ConnConfig is a peer Connection configuration diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 82b35dd79cc..7fa3669329c 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -135,6 +135,8 @@ func (h *Handshaker) Listen(ctx context.Context) { h.updateRemoteICEState(&remoteOfferAnswer) + h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) + if h.relayListener != nil { h.relayListener.Notify(&remoteOfferAnswer) } @@ -157,6 +159,8 @@ func (h *Handshaker) Listen(ctx context.Context) { h.updateRemoteICEState(&remoteOfferAnswer) + h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) + if h.relayListener != nil { h.relayListener.Notify(&remoteOfferAnswer) } @@ -175,6 +179,15 @@ func (h *Handshaker) Listen(ctx context.Context) { } } +// pqRegisterEndpoint feeds the post-quantum handshaker the peer's data-path endpoint +// (its WG overlay IP plus the advertised pq UDP port) learned from a remote offer/answer. +func (h *Handshaker) pqRegisterEndpoint(remotePort int) { + if h.config.PQ == nil || remotePort <= 0 || len(h.config.WgConfig.AllowedIps) == 0 { + return + } + h.config.PQ.SetRemotePort(h.config.Key, h.config.WgConfig.AllowedIps[0].Addr(), remotePort) +} + func (h *Handshaker) SendOffer() error { h.mu.Lock() defer h.mu.Unlock() diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 611e72498c7..4de9a0ab17d 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -1,6 +1,8 @@ package internal import ( + "net/netip" + log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -78,3 +80,13 @@ func (p pqHandshaker) PSK(remoteKey string) (wgtypes.Key, bool) { } return wgtypes.Key(psk), true } + +// SetRemotePort registers the peer's data-path endpoint (overlay IP + pq UDP port) +// learned from signalling. Sends only ever fire once the tunnel is up (clocked by +// OnDataPathRekeyed), so registering here is safe even before connection-up. +func (p pqHandshaker) SetRemotePort(remoteKey string, overlayIP netip.Addr, port int) { + if port <= 0 || port > 65535 || !overlayIP.IsValid() { + return + } + p.mgr.AddPeer(pqkem.RemoteID(remoteKey), netip.AddrPortFrom(overlayIP, uint16(port))) +} From 1c46b4e9dfea178c8d0a4ff7c99ff29cc774bec2 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 27 Jul 2026 16:32:33 +0200 Subject: [PATCH 036/119] pqkem: clock data-path PSK rotation from WireGuard handshakes Source OnDataPathRekeyed from the WGWatcher's per-handshake callback (onWGCheckSuccess), which fires only on a fresh handshake, and OnDataPathDown from the handshake-timeout path. A fresh handshake clocks the next chained KEM exchange pushed over the data-path UDP transport. --- client/internal/peer/conn.go | 14 ++++++++++++++ client/internal/pqkem_adapter.go | 10 ++++++++++ 2 files changed, 24 insertions(+) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 9c3e26d8e41..3a73c892fa8 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -94,6 +94,11 @@ type PQHandshaker interface { // SetRemotePort registers the peer's data-path endpoint from signalling: the peer's // WG overlay IP combined with its advertised pq UDP port. SetRemotePort(remoteKey string, overlayIP netip.Addr, port int) + // OnDataPathRekeyed signals a fresh WireGuard handshake for the peer; it clocks the + // next chained PSK rotation pushed over the data path. + OnDataPathRekeyed(remoteKey string) + // OnDataPathDown signals the peer's tunnel went down. + OnDataPathDown(remoteKey string) } // ConnConfig is a peer Connection configuration @@ -712,6 +717,10 @@ func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { conn.Log.Warnf("WireGuard handshake timeout detected, closing current connection") + if conn.config.PQ != nil { + conn.config.PQ.OnDataPathDown(conn.config.Key) + } + // Close the active connection based on current priority switch conn.currentConnPriority { case conntype.Relay: @@ -976,6 +985,11 @@ func (conn *Conn) onWGCheckSuccess() { conn.mu.Lock() conn.wgTimeouts = 0 conn.mu.Unlock() + + // A fresh WireGuard handshake is the clock for the post-quantum PSK rotation. + if conn.config.PQ != nil { + conn.config.PQ.OnDataPathRekeyed(conn.config.Key) + } } // recordConnectionMetrics records connection stage timestamps as metrics diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 4de9a0ab17d..17d810d4930 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -90,3 +90,13 @@ func (p pqHandshaker) SetRemotePort(remoteKey string, overlayIP netip.Addr, port } p.mgr.AddPeer(pqkem.RemoteID(remoteKey), netip.AddrPortFrom(overlayIP, uint16(port))) } + +// OnDataPathRekeyed clocks the next chained PSK rotation on a fresh WG handshake. +func (p pqHandshaker) OnDataPathRekeyed(remoteKey string) { + p.mgr.OnDataPathRekeyed(pqkem.RemoteID(remoteKey)) +} + +// OnDataPathDown signals the peer's tunnel went down. +func (p pqHandshaker) OnDataPathDown(remoteKey string) { + p.mgr.OnDataPathDown(pqkem.RemoteID(remoteKey)) +} From b92b14543d2099b45e6626fbd0defaff20f43051 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 27 Jul 2026 16:36:01 +0200 Subject: [PATCH 037/119] Renames SetRemotePort to SetRemoteAddr --- client/internal/peer/conn.go | 6 +++--- client/internal/peer/handshaker.go | 5 +++-- client/internal/pqkem_adapter.go | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 3a73c892fa8..387e0c05475 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -91,9 +91,9 @@ type PQHandshaker interface { // PSK returns the peer's latest derived post-quantum PSK to program at WG // peer-config time (the pull path). ok is false until one has been derived. PSK(remoteKey string) (wgtypes.Key, bool) - // SetRemotePort registers the peer's data-path endpoint from signalling: the peer's - // WG overlay IP combined with its advertised pq UDP port. - SetRemotePort(remoteKey string, overlayIP netip.Addr, port int) + // SetRemoteAddr registers the peer's data-path endpoint learned from signalling: + // its WG overlay IP with the advertised pq UDP port. + SetRemoteAddr(remoteKey string, addr netip.AddrPort) // OnDataPathRekeyed signals a fresh WireGuard handshake for the peer; it clocks the // next chained PSK rotation pushed over the data path. OnDataPathRekeyed(remoteKey string) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 7fa3669329c..0ba9c70df4a 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -182,10 +182,11 @@ func (h *Handshaker) Listen(ctx context.Context) { // pqRegisterEndpoint feeds the post-quantum handshaker the peer's data-path endpoint // (its WG overlay IP plus the advertised pq UDP port) learned from a remote offer/answer. func (h *Handshaker) pqRegisterEndpoint(remotePort int) { - if h.config.PQ == nil || remotePort <= 0 || len(h.config.WgConfig.AllowedIps) == 0 { + if h.config.PQ == nil || remotePort <= 0 || remotePort > 65535 || len(h.config.WgConfig.AllowedIps) == 0 { return } - h.config.PQ.SetRemotePort(h.config.Key, h.config.WgConfig.AllowedIps[0].Addr(), remotePort) + addr := netip.AddrPortFrom(h.config.WgConfig.AllowedIps[0].Addr(), uint16(remotePort)) + h.config.PQ.SetRemoteAddr(h.config.Key, addr) } func (h *Handshaker) SendOffer() error { diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 17d810d4930..94ce8ccf2ce 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -81,14 +81,14 @@ func (p pqHandshaker) PSK(remoteKey string) (wgtypes.Key, bool) { return wgtypes.Key(psk), true } -// SetRemotePort registers the peer's data-path endpoint (overlay IP + pq UDP port) +// SetRemoteAddr registers the peer's data-path endpoint (overlay IP + pq UDP port) // learned from signalling. Sends only ever fire once the tunnel is up (clocked by // OnDataPathRekeyed), so registering here is safe even before connection-up. -func (p pqHandshaker) SetRemotePort(remoteKey string, overlayIP netip.Addr, port int) { - if port <= 0 || port > 65535 || !overlayIP.IsValid() { +func (p pqHandshaker) SetRemoteAddr(remoteKey string, addr netip.AddrPort) { + if !addr.IsValid() || addr.Port() == 0 { return } - p.mgr.AddPeer(pqkem.RemoteID(remoteKey), netip.AddrPortFrom(overlayIP, uint16(port))) + p.mgr.AddPeer(pqkem.RemoteID(remoteKey), addr) } // OnDataPathRekeyed clocks the next chained PSK rotation on a fresh WG handshake. From 562c02fa42284f5fb824b9e5b357d4d0b2492c6f Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 28 Jul 2026 22:43:07 +0200 Subject: [PATCH 038/119] Adds log tracepoints - Add a trace slog level (NB_PQ_MLKEM_LOG_LEVEL=trace) and move the verbose per-exchange lifecycle logs (offer/answer/PSK/ack/rotation) to it, so debug stays quiet and troubleshooting is opt-in. - Stop logging the raw preshared key; drop the temporary pqkem-dbg OnRemoteOffer/ OnRemoteAnswer probes. - Demote the per-handshake conn log to trace. --- client/internal/pqkem/convergence.go | 38 +++++++++++++++ client/internal/pqkem/env.go | 69 +++++++++++++++++++++++++--- client/internal/pqkem/manager.go | 11 ++++- client/internal/pqkem_adapter.go | 2 +- 4 files changed, 112 insertions(+), 8 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 63e1d61e9d3..deaf68cba41 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -2,9 +2,22 @@ package pqkem import ( "context" + "crypto/sha256" + "encoding/hex" "time" ) +// idHex renders an exchange ID for logs. +func idHex(id ExchangeID) string { return hex.EncodeToString(id[:]) } + +// pskFingerprint is a short, non-secret digest of a derived PSK: identical on both +// peers iff they derived the same key. Logged instead of the raw PSK so debug logs +// never carry the actual WireGuard preshared key. +func pskFingerprint(psk PSK) string { + sum := sha256.Sum256(psk[:]) + return hex.EncodeToString(sum[:8]) +} + // startExchange creates a fresh initiator exchange (acknowledging ackID, zero for a // bootstrap) and returns the framed offer for the caller to send — pushed over the // data path for a chained rekey, or handed to the host for signalling when viaSignal @@ -41,6 +54,12 @@ func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID Exchang m.wait.Add(1) go m.initiatorLoop(ctx, remoteID, id) + + via := "data-path" + if viaSignal { + via = "signal" + } + m.trace("pqkem: offer sent", "peer", remoteID, "exchange", idHex(id), "acks", idHex(ackID), "via", via) return raw, nil } @@ -49,6 +68,8 @@ func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID Exchang // then derives the PSK for the new offer, commits it optimistically, and returns the // framed answer. A duplicate offer returns the cached answer without re-deriving. func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { + m.trace("pqkem: offer received", "peer", remoteID, "exchange", idHex(o.ExchangeID), "acks", idHex(o.AckID)) + if o.AckID != (ExchangeID{}) { m.ackConverged(remoteID, o.AckID) } @@ -60,6 +81,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { if state == stateReserved { return nil, nil } + m.trace("pqkem: duplicate offer, resending cached answer", "peer", remoteID, "exchange", idHex(o.ExchangeID)) return last, nil } // Reserve the slot so a concurrent duplicate offer bails. @@ -79,6 +101,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { ex := m.exchanges[remoteID] if ex == nil || ex.id != o.ExchangeID { m.mu.Unlock() + m.trace("pqkem: exchange superseded during respond, dropping answer", "peer", remoteID, "exchange", idHex(o.ExchangeID)) return nil, nil } ex.state = stateAwaitingAck @@ -87,10 +110,13 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { m.psks[remoteID] = psk m.mu.Unlock() + m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "psk_fp", pskFingerprint(psk)) + // Commit optimistically so our data path can rekey to the new PSK. if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { return nil, err } + m.trace("pqkem: answer sent", "peer", remoteID, "exchange", idHex(o.ExchangeID)) return raw, nil } @@ -102,7 +128,12 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { + haveID := "none" + if ex != nil { + haveID = idHex(ex.id) + } m.mu.Unlock() + m.trace("pqkem: unexpected answer dropped (inconsistency)", "peer", remoteID, "answer_for", idHex(a.ExchangeID), "have_exchange", haveID) return nil } ex.state = stateAwaitingRekey @@ -110,6 +141,8 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { ex.initiator = nil m.mu.Unlock() + m.trace("pqkem: answer received", "peer", remoteID, "exchange", idHex(a.ExchangeID)) + psk, err := init.Finish(a.KEMAnswer, m.binding(remoteID)) if err != nil { return err @@ -122,6 +155,8 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.psks[remoteID] = psk m.mu.Unlock() + m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "psk_fp", pskFingerprint(psk)) + return m.cbHandler.OnNewPSKReady(remoteID, psk) } @@ -133,6 +168,7 @@ func (m *Manager) ackConverged(remoteID RemoteID, ackID ExchangeID) { ex := m.exchanges[remoteID] if ex == nil || ex.id != ackID || ex.state != stateAwaitingAck { m.mu.Unlock() + m.trace("pqkem: ack for unknown/mismatched exchange, ignored (inconsistency)", "peer", remoteID, "acks", idHex(ackID)) return } delete(m.exchanges, remoteID) @@ -140,6 +176,8 @@ func (m *Manager) ackConverged(remoteID RemoteID, ackID ExchangeID) { m.failures[remoteID] = 0 _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) m.mu.Unlock() + + m.trace("pqkem: previous exchange confirmed by ack", "peer", remoteID, "exchange", idHex(ackID)) } // initiatorLoop enforces the offer->answer convergence deadline and retransmits the diff --git a/client/internal/pqkem/env.go b/client/internal/pqkem/env.go index 8eb4026725c..f299d5fce5c 100644 --- a/client/internal/pqkem/env.go +++ b/client/internal/pqkem/env.go @@ -1,6 +1,7 @@ package pqkem import ( + "context" "log/slog" "os" "strconv" @@ -34,19 +35,26 @@ func Enabled() bool { return enabled } -// EnvLogLevel overrides the ML-KEM manager's slog level (debug/info/warn/error). -// Defaults to info. +// EnvLogLevel overrides the ML-KEM manager's slog level (trace/debug/info/warn/error). +// Defaults to info. The verbose per-exchange lifecycle logs are emitted at trace. const EnvLogLevel = "NB_PQ_MLKEM_LOG_LEVEL" -// NewLogger builds the slog logger for the ML-KEM manager: a text handler to stdout -// at the level from EnvLogLevel. Mirrors the Rosenpass manager's logger setup so PQ -// components log consistently. +// LevelTrace is a custom slog level below Debug for the verbose per-exchange lifecycle +// logs, so they stay off unless NB_PQ_MLKEM_LOG_LEVEL=trace (and the daemon log level +// is trace, since the records are forwarded to logrus). +const LevelTrace = slog.LevelDebug - 4 + +// NewLogger builds the slog logger for the ML-KEM manager. It forwards records to +// logrus so PQ logs land in the same sink as the rest of the daemon (console + +// client.log) rather than stdout. Verbosity is gated by EnvLogLevel. func NewLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel()})) + return slog.New(slogToLogrus{}) } func logLevel() slog.Level { switch strings.ToLower(strings.TrimSpace(os.Getenv(EnvLogLevel))) { + case "trace": + return LevelTrace case "debug": return slog.LevelDebug case "warn": @@ -57,3 +65,52 @@ func logLevel() slog.Level { return slog.LevelInfo } } + +// slogToLogrus is a slog.Handler that forwards records to logrus, so the ML-KEM +// manager's logs go wherever the daemon's logrus is configured (console + client.log) +// instead of stdout. Verbosity is gated by EnvLogLevel via logLevel(). +type slogToLogrus struct { + fields log.Fields +} + +func (h slogToLogrus) Enabled(_ context.Context, level slog.Level) bool { + return level >= logLevel() +} + +func (h slogToLogrus) Handle(_ context.Context, r slog.Record) error { + fields := make(log.Fields, len(h.fields)+r.NumAttrs()) + for k, v := range h.fields { + fields[k] = v + } + r.Attrs(func(a slog.Attr) bool { + fields[a.Key] = a.Value.Any() + return true + }) + entry := log.WithFields(fields) + switch { + case r.Level >= slog.LevelError: + entry.Error(r.Message) + case r.Level >= slog.LevelWarn: + entry.Warn(r.Message) + case r.Level >= slog.LevelInfo: + entry.Info(r.Message) + case r.Level >= slog.LevelDebug: + entry.Debug(r.Message) + default: + entry.Trace(r.Message) + } + return nil +} + +func (h slogToLogrus) WithAttrs(attrs []slog.Attr) slog.Handler { + fields := make(log.Fields, len(h.fields)+len(attrs)) + for k, v := range h.fields { + fields[k] = v + } + for _, a := range attrs { + fields[a.Key] = a.Value.Any() + } + return slogToLogrus{fields: fields} +} + +func (h slogToLogrus) WithGroup(_ string) slog.Handler { return h } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 9cc06a00a41..9cf4a189b25 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -169,6 +169,12 @@ func (m *Manager) PSK(remoteID RemoteID) (PSK, bool) { return psk, ok } +// trace logs at LevelTrace, the verbose per-exchange lifecycle level gated by +// NB_PQ_MLKEM_LOG_LEVEL=trace. +func (m *Manager) trace(msg string, args ...any) { + m.logger.Log(context.Background(), LevelTrace, msg, args...) +} + // AddPeer registers where a peer's data-path messages are sent and received: its // overlay endpoint (IP:port). Re-adding updates the endpoint. func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { @@ -280,7 +286,7 @@ func (m *Manager) onDataPathInbound(src netip.AddrPort, msg []byte) { return } if err := m.OnDataPathMessage(remoteID, msg); err != nil { - m.logger.Debug("pqkem: inbound", "peer", remoteID, "err", err) + m.trace("pqkem: inbound", "peer", remoteID, "err", err) } } @@ -323,6 +329,7 @@ func (m *Manager) OnDataPathRekeyed(remoteID RemoteID) { } m.mu.Unlock() + m.trace("pqkem: data-path rekey signal", "peer", remoteID, "chaining", chain) if !chain { return } @@ -333,7 +340,9 @@ func (m *Manager) OnDataPathRekeyed(remoteID RemoteID) { } if err := m.pushDataPath(remoteID, offer); err != nil { m.logger.Warn("pqkem: send chain offer failed", "peer", remoteID, "err", err) + return } + m.trace("pqkem: chain offer sent over data path", "peer", remoteID) } // OnDataPathDown notifies that the peer's data path went down. Rotations resume once diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 94ce8ccf2ce..074039b9e6d 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -27,7 +27,7 @@ func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) // updateOnly: applies to an already-configured peer (rotation). At bootstrap the // peer is not configured yet, so this is a no-op there and the PSK is instead // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). - log.Debugf("pqkem: programming PSK for peer %s", remoteID) + log.Tracef("pqkem: programming PSK for peer %s", remoteID) return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) } From b1c0a2bffe63f12abace4ecf61c2f058113a2e64 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 31 Jul 2026 10:27:32 +0200 Subject: [PATCH 039/119] Don't rotate PQ keys if data path is idle for ~90s (less than a WG handhshake time --- client/internal/peer/conn.go | 27 ++++++++++++++++++---- client/internal/peer/iface.go | 4 ++++ client/internal/pqkem/convergence_test.go | 4 ++-- client/internal/pqkem/manager.go | 20 +++++++++++++++- client/internal/pqkem/manager_test.go | 28 +++++++++++++++++++++-- client/internal/pqkem_adapter.go | 7 ++++-- 6 files changed, 79 insertions(+), 11 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 387e0c05475..ec1d64c8f61 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -3,6 +3,7 @@ package peer import ( "context" "fmt" + "math" "net" "net/netip" "runtime" @@ -27,6 +28,7 @@ import ( "github.com/netbirdio/netbird/client/internal/rosenpass" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/netevents" + "github.com/netbirdio/netbird/monotime" "github.com/netbirdio/netbird/route" relayClient "github.com/netbirdio/netbird/shared/relay/client" ) @@ -95,8 +97,10 @@ type PQHandshaker interface { // its WG overlay IP with the advertised pq UDP port. SetRemoteAddr(remoteKey string, addr netip.AddrPort) // OnDataPathRekeyed signals a fresh WireGuard handshake for the peer; it clocks the - // next chained PSK rotation pushed over the data path. - OnDataPathRekeyed(remoteKey string) + // next chained PSK rotation pushed over the data path. sinceActivity is how long + // ago the peer last exchanged real user data, so the rotation can be skipped for + // idle tunnels. + OnDataPathRekeyed(remoteKey string, sinceActivity time.Duration) // OnDataPathDown signals the peer's tunnel went down. OnDataPathDown(remoteKey string) } @@ -986,12 +990,27 @@ func (conn *Conn) onWGCheckSuccess() { conn.wgTimeouts = 0 conn.mu.Unlock() - // A fresh WireGuard handshake is the clock for the post-quantum PSK rotation. + // A fresh WireGuard handshake clocks the post-quantum PSK rotation. Pass how long + // ago the peer last exchanged real user data (keepalives excluded) so the pqkem + // manager can skip rotation on idle tunnels — rotating then would push data-path + // traffic that keeps the lazy connection artificially active. if conn.config.PQ != nil { - conn.config.PQ.OnDataPathRekeyed(conn.config.Key) + conn.config.PQ.OnDataPathRekeyed(conn.config.Key, conn.dataActivityAge()) } } +// dataActivityAge returns how long ago the peer last exchanged real user data +// (WireGuard keepalives excluded), per the same LastActivities signal the +// lazy-connection inactivity monitor uses. It reports a very large duration when no +// activity has ever been recorded, so the peer is treated as idle. +func (conn *Conn) dataActivityAge() time.Duration { + last, ok := conn.config.WgConfig.WgInterface.LastActivities()[conn.config.WgConfig.RemoteKey] + if !ok { + return time.Duration(math.MaxInt64) + } + return monotime.Since(last) +} + // recordConnectionMetrics records connection stage timestamps as metrics func (conn *Conn) recordConnectionMetrics() { if conn.metricsRecorder == nil { diff --git a/client/internal/peer/iface.go b/client/internal/peer/iface.go index 678396e612e..47780a4d0d1 100644 --- a/client/internal/peer/iface.go +++ b/client/internal/peer/iface.go @@ -10,6 +10,7 @@ import ( "github.com/netbirdio/netbird/client/iface/configurer" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/iface/wgproxy" + "github.com/netbirdio/netbird/monotime" ) type WGIface interface { @@ -19,4 +20,7 @@ type WGIface interface { GetProxy() wgproxy.Proxy Address() wgaddr.Address RemoveEndpointAddress(key string) error + // LastActivities returns the last real-data activity time per peer (WireGuard + // keepalives excluded), used to gate post-quantum PSK rotation on active tunnels. + LastActivities() map[string]monotime.Time } diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index e6749150944..f4ce251c4a9 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -52,8 +52,8 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { // Establish: bootstrap + data-path-rekeyed so B becomes established and its data // path is usable. bootstrap(t, dA, dB) - dA.OnDataPathRekeyed("bbbb") - dB.OnDataPathRekeyed("aaaa") + dA.OnDataPathRekeyed("bbbb", 0) + dB.OnDataPathRekeyed("aaaa", 0) require.NotEqual(t, PSK{}, wgB.psk("aaaa")) // Drop B's outbound so rekeys can no longer converge. diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 9cf4a189b25..bb3aae48de8 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -20,6 +20,14 @@ const ( // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures // are tolerated before OnRekeyFailed. The initial exchange fails immediately. DefaultMaxRekeyFailures = 3 + + // rotationActivityWindow gates rotation on recent real-data activity: a rekey + // clocks a rotation only if the peer exchanged user data within this window. It + // must stay shorter than the data path's rekey interval (WireGuard + // REKEY_AFTER_TIME ~120s) so the rotation's own traffic — which itself renews the + // activity signal — ages out before the next rekey, letting an idle tunnel stop + // rotating instead of self-sustaining. + rotationActivityWindow = 90 * time.Second ) // LocalID and RemoteID are peer identity keys (e.g. WireGuard public keys). They are @@ -319,7 +327,17 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { // initiator that just derived a PSK, it chains the next exchange: a fresh offer over // the data path that acknowledges the just-completed one (its arrival under the new // key proves to the responder that the key works). -func (m *Manager) OnDataPathRekeyed(remoteID RemoteID) { +// OnDataPathRekeyed clocks the next chained PSK rotation on a fresh data-path rekey. +// sinceActivity is how long ago the peer last exchanged real user data; when it +// exceeds rotationActivityWindow the tunnel is treated as idle and rotation is +// skipped — an idle tunnel has nothing to protect, and rotating would emit data-path +// traffic that keeps the peer artificially active (see conn.onWGCheckSuccess). +func (m *Manager) OnDataPathRekeyed(remoteID RemoteID, sinceActivity time.Duration) { + if sinceActivity >= rotationActivityWindow { + m.trace("pqkem: peer idle, skipping data-path rotation", "peer", remoteID, "since_activity", sinceActivity) + return + } + m.mu.Lock() ex := m.exchanges[remoteID] chain := ex != nil && ex.state == stateAwaitingRekey diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index bbe8ab161cc..eab264478a7 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -143,8 +143,8 @@ func TestManager_ChainRotatesAndAcks(t *testing.T) { // Data path up: B (initiator) chains the next offer over the data path, which // rotates both to a fresh PSK and acknowledges A. - dA.OnDataPathRekeyed("bbbb") - dB.OnDataPathRekeyed("aaaa") + dA.OnDataPathRekeyed("bbbb", 0) + dB.OnDataPathRekeyed("aaaa", 0) psk2A := wgA.psk("bbbb") psk2B := wgB.psk("aaaa") @@ -152,6 +152,30 @@ func TestManager_ChainRotatesAndAcks(t *testing.T) { require.NotEqual(t, psk1, psk2B, "the chain rotated to a new PSK") } +func TestManager_RotationSkippedWhenIdle(t *testing.T) { + dA, dB, wgA, wgB, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + psk1 := wgB.psk("aaaa") + require.NotEqual(t, PSK{}, psk1) + + // Idle: the peer's last real-data activity is older than the window, so a rekey + // must NOT clock a rotation. + dA.OnDataPathRekeyed("bbbb", rotationActivityWindow) + dB.OnDataPathRekeyed("aaaa", rotationActivityWindow) + require.Equal(t, psk1, wgB.psk("aaaa"), "idle peer must not rotate the PSK") + require.Equal(t, psk1, wgA.psk("bbbb"), "idle peer must not rotate the PSK") + + // Active: activity within the window clocks the rotation as usual. + dA.OnDataPathRekeyed("bbbb", rotationActivityWindow-1) + dB.OnDataPathRekeyed("aaaa", rotationActivityWindow-1) + psk2 := wgB.psk("aaaa") + require.NotEqual(t, psk1, psk2, "recent activity must clock a rotation") + require.Equal(t, psk2, wgA.psk("bbbb"), "both sides converge on the rotated PSK") +} + func TestManager_NonInitiatorReturnsNoOffer(t *testing.T) { dA := NewManager("aaaa", newFakeWG(), nil) defer dA.Stop() diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 074039b9e6d..fe41082ad4e 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -2,6 +2,7 @@ package internal import ( "net/netip" + "time" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -92,8 +93,10 @@ func (p pqHandshaker) SetRemoteAddr(remoteKey string, addr netip.AddrPort) { } // OnDataPathRekeyed clocks the next chained PSK rotation on a fresh WG handshake. -func (p pqHandshaker) OnDataPathRekeyed(remoteKey string) { - p.mgr.OnDataPathRekeyed(pqkem.RemoteID(remoteKey)) +// sinceActivity is how long ago the peer last exchanged real user data; the manager +// skips rotation for idle tunnels. +func (p pqHandshaker) OnDataPathRekeyed(remoteKey string, sinceActivity time.Duration) { + p.mgr.OnDataPathRekeyed(pqkem.RemoteID(remoteKey), sinceActivity) } // OnDataPathDown signals the peer's tunnel went down. From 8c8abfb2b1d37c4a64f67afa700f6caf1b99262c Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 29 Jul 2026 08:55:03 +0200 Subject: [PATCH 040/119] pqkem: derive PSK with HKDF-SHA256 Replace the raw SHA-256 concat combiner with HKDF-SHA256 (crypto/hkdf, Go 1.24): IKM = ML-KEM_ss || X25519_ss (draft-ietf-tls-ecdhe-mlkem order), salt = the domain-separation label, info = full transcript (offer || answer) || canonicalised peer identities. Keeps the transcript + identity binding while using a proper KDF. --- client/internal/pqkem/kem.go | 53 ++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/client/internal/pqkem/kem.go b/client/internal/pqkem/kem.go index 0995443203c..cd3c80d094d 100644 --- a/client/internal/pqkem/kem.go +++ b/client/internal/pqkem/kem.go @@ -14,13 +14,15 @@ // // Combiner note: this follows draft-ietf-tls-ecdhe-mlkem for X25519MLKEM768 — on // the wire ML-KEM ‖ X25519 (the draft deliberately reversed the share order for -// this group), and ML-KEM_ss ‖ X25519_ss fed into the KDF. The spike uses SHA-256 -// (also binding the transcript and peer identities); a production version should -// use HKDF — see TODO below. +// this group), and ML-KEM_ss ‖ X25519_ss as the KDF input. The PSK is derived with +// HKDF-SHA256 over that hybrid secret, salted with a domain-separation label and +// bound (via the HKDF info) to the full transcript and the canonicalised peer +// identities. package pqkem import ( "crypto/ecdh" + "crypto/hkdf" "crypto/mlkem" "crypto/rand" "crypto/sha256" @@ -98,7 +100,7 @@ func (i *Initiator) Finish(answer []byte, b Binding) (PSK, error) { return PSK{}, fmt.Errorf("x25519 ecdh: %w", err) } - return derivePSK(ssMLKEM, ssX, i.offer, answer, b), nil + return derivePSK(ssMLKEM, ssX, i.offer, answer, b) } // Respond consumes an initiator offer, produces the answer, and derives the PSK. @@ -134,29 +136,38 @@ func Respond(offer []byte, b Binding) (answer []byte, psk PSK, err error) { // derivePSK uses the same argument order on both sides; the responder's local // binding is the mirror of the initiator's, canonicalised inside derivePSK. - return answer, derivePSK(ssMLKEM, ssX, offer, answer, b), nil + psk, err = derivePSK(ssMLKEM, ssX, offer, answer, b) + if err != nil { + return nil, PSK{}, err + } + return answer, psk, nil } -// derivePSK combines the two shared secrets and binds the result to the full -// transcript (offer ‖ answer) and the canonicalised peer identities. -// -// TODO(NET-1406): replace the SHA-256 concat with the RFC HKDF combiner -// (crypto/hkdf, Go 1.24+) and proper labels before this leaves spike status. -func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) PSK { +// derivePSK runs HKDF-SHA256 over the hybrid shared secret (ML-KEM_ss ‖ X25519_ss, +// per draft-ietf-tls-ecdhe-mlkem), salted with the domain-separation label, and binds +// the result — via the HKDF info — to the full transcript (offer ‖ answer) and the +// canonicalised peer identities, so the PSK cannot be transplanted to another peer +// pair or a different exchange. +func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) (PSK, error) { lo, hi := canonicalPair(b.LocalID, b.RemoteID) - h := sha256.New() - h.Write([]byte(pskLabel)) - h.Write(ssMLKEM) - h.Write(ssX) - h.Write(offer) - h.Write(answer) - h.Write(lo) - h.Write(hi) + ikm := make([]byte, 0, len(ssMLKEM)+len(ssX)) + ikm = append(ikm, ssMLKEM...) + ikm = append(ikm, ssX...) + + info := make([]byte, 0, len(offer)+len(answer)+len(lo)+len(hi)) + info = append(info, offer...) + info = append(info, answer...) + info = append(info, lo...) + info = append(info, hi...) var psk PSK - copy(psk[:], h.Sum(nil)) - return psk + key, err := hkdf.Key(sha256.New, ikm, []byte(pskLabel), string(info), len(psk)) + if err != nil { + return PSK{}, fmt.Errorf("hkdf derive psk: %w", err) + } + copy(psk[:], key) + return psk, nil } func canonicalPair(a, b []byte) (lo, hi []byte) { From b1b2cc14c1eca40de44ac4da0b552778d9cb38df Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 29 Jul 2026 10:18:42 +0200 Subject: [PATCH 041/119] pqkem: rotate PSK in kernel mode instead of skipping The idle-gate reads LastActivities, which only tracks per-peer data in userspace; in kernel mode it is empty, so the gate treated every kernel peer as idle and disabled data-path rotation entirely. Detect the bind via IsUserspaceBind and, in kernel mode, report zero activity age (always 'active') so rotation runs on every rekey. Lazy back-to-idle is already limited in kernel; the eBPF WG-activity detection will later supply a real signal that excludes handshake/pqkem traffic. --- client/internal/peer/conn.go | 9 +++++++++ client/internal/peer/iface.go | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index ec1d64c8f61..b90370dd705 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -1003,7 +1003,16 @@ func (conn *Conn) onWGCheckSuccess() { // (WireGuard keepalives excluded), per the same LastActivities signal the // lazy-connection inactivity monitor uses. It reports a very large duration when no // activity has ever been recorded, so the peer is treated as idle. +// +// In kernel mode there is no per-peer data-activity signal (LastActivities is +// userspace-only), so we cannot tell active from idle. We report zero — always +// "active" — so PSK rotation is not disabled in kernel mode. Lazy back-to-idle is +// already limited there; the eBPF WG-activity detection (future) will supply a real +// signal that excludes handshake/pqkem traffic. func (conn *Conn) dataActivityAge() time.Duration { + if !conn.config.WgConfig.WgInterface.IsUserspaceBind() { + return 0 + } last, ok := conn.config.WgConfig.WgInterface.LastActivities()[conn.config.WgConfig.RemoteKey] if !ok { return time.Duration(math.MaxInt64) diff --git a/client/internal/peer/iface.go b/client/internal/peer/iface.go index 47780a4d0d1..a12a0e181ca 100644 --- a/client/internal/peer/iface.go +++ b/client/internal/peer/iface.go @@ -23,4 +23,8 @@ type WGIface interface { // LastActivities returns the last real-data activity time per peer (WireGuard // keepalives excluded), used to gate post-quantum PSK rotation on active tunnels. LastActivities() map[string]monotime.Time + // IsUserspaceBind reports whether WireGuard runs in userspace. Only there does + // LastActivities track per-peer data activity; in kernel mode it is unavailable, + // so PSK rotation cannot be gated on activity. + IsUserspaceBind() bool } From f322be1944e57670371eaabc78bee6d601ba2779 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 30 Jul 2026 09:29:12 +0200 Subject: [PATCH 042/119] pqkem: strict (fail-closed) mode + wire status Quantum resistance Strict mode (NB_PQ_MLKEM_STRICT, default off) closes the initial PQ-vulnerable window (NET-1408): when enabled, conn.presharedKey programs a per-conn random sentinel PSK until the ML-KEM exchange derives the real one, so no session can form on a non-PQ key (the real PSK is pushed via SetPresharedKey once it converges). Default stays opportunistic. Also surface PQ status: the peer 'Quantum resistance' flag (RosenpassEnabled) is now true when an ML-KEM PSK has been derived for the peer, not only for Rosenpass. --- client/internal/engine.go | 1 + client/internal/peer/conn.go | 43 ++++++++++++++++++++++++++++++++++-- client/internal/pqkem/env.go | 22 ++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index b33284dd09c..e153da6fdab 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -2012,6 +2012,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV } if e.pqkemManager != nil { config.PQ = pqHandshaker{mgr: e.pqkemManager} + config.PQStrict = pqkem.Strict() } serviceDependencies := peer.ServiceDependencies{ diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b90370dd705..430232f76c6 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -124,6 +124,9 @@ type ConnConfig struct { // PQ carries post-quantum ML-KEM material on offers/answers; nil when disabled. PQ PQHandshaker + // PQStrict fails closed: block peer traffic until the ML-KEM PSK is established, + // instead of letting the tunnel come up classically and upgrading to PQ later. + PQStrict bool // ICEConfig ICE protocol configuration ICEConfig icemaker.Config @@ -187,6 +190,11 @@ type Conn struct { // pendingFirstPacket is the lazyconn-captured handshake init, replayed once the real // transport is up. pendingFirstPacket []byte + + // pqBlockingKey is a per-conn random sentinel PSK used in PQ strict mode to fail + // closed: it is programmed until the real ML-KEM PSK is derived, so no session can + // form on a non-PQ key. Per-conn random so two strict peers never match by chance. + pqBlockingKey *wgtypes.Key } // injectPendingFirstPacket replays the captured handshake through the proxy if present, else @@ -244,6 +252,14 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) { metricsRecorder: services.MetricsRecorder, } + if config.PQ != nil && config.PQStrict { + if k, err := wgtypes.GenerateKey(); err != nil { + connLog.Errorf("pqkem: failed to generate strict-mode sentinel key, strict fail-closed disabled for this peer: %v", err) + } else { + conn.pqBlockingKey = &k + } + } + return conn, nil } @@ -767,7 +783,7 @@ func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []by ConnStatus: conn.evalStatus(), Relayed: conn.isRelayed(), RelayServerAddress: relayServerAddr, - RosenpassEnabled: isRosenpassEnabled(rosenpassPubKey), + RosenpassEnabled: conn.quantumResistant(rosenpassPubKey), } err := conn.statusRecorder.UpdatePeerRelayedState(peerState) @@ -786,7 +802,7 @@ func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo, updateTime time.Time) RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, - RosenpassEnabled: isRosenpassEnabled(iceConnInfo.RosenpassPubKey), + RosenpassEnabled: conn.quantumResistant(iceConnInfo.RosenpassPubKey), } err := conn.statusRecorder.UpdatePeerICEState(peerState) @@ -1063,6 +1079,14 @@ func (conn *Conn) presharedKey(remoteRosenpassKey []byte) *wgtypes.Key { if psk, ok := conn.config.PQ.PSK(conn.config.Key); ok { return &psk } + if conn.config.PQStrict && conn.pqBlockingKey != nil { + // Fail closed: program a non-matching sentinel so no session forms on a + // non-PQ key until the ML-KEM exchange derives the real PSK (pushed via + // SetPresharedKey once it converges). "pending" — turns into a "stuck" + // warning from the manager if the exchange keeps failing (see raiseFailure). + conn.Log.Debugf("pqkem: strict mode — no PQ PSK yet, blocking peer traffic until the ML-KEM exchange converges") + return conn.pqBlockingKey + } } if conn.config.RosenpassConfig.PubKey == nil { @@ -1104,6 +1128,21 @@ func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } +// quantumResistant reports whether the peer's tunnel is post-quantum protected, for +// the status "Quantum resistance" field: either Rosenpass (the remote advertised a +// Rosenpass key) or the ML-KEM exchange (a PQ PSK has been derived for this peer). +func (conn *Conn) quantumResistant(remoteRosenpassPubKey []byte) bool { + if isRosenpassEnabled(remoteRosenpassPubKey) { + return true + } + if conn.config.PQ != nil { + if _, ok := conn.config.PQ.PSK(conn.config.Key); ok { + return true + } + } + return false +} + func evalConnStatus(in connStatusInputs) guard.ConnStatus { // "Relay up and needed" — the peer uses relay and the transport is connected. relayUsedAndUp := in.peerUsesRelay && in.relayConnected diff --git a/client/internal/pqkem/env.go b/client/internal/pqkem/env.go index f299d5fce5c..6eaa9e1291b 100644 --- a/client/internal/pqkem/env.go +++ b/client/internal/pqkem/env.go @@ -35,6 +35,28 @@ func Enabled() bool { return enabled } +// EnvStrict enables strict (fail-closed) mode: block peer traffic until the ML-KEM +// PSK has been established, instead of the default opportunistic behaviour that lets +// the tunnel come up classically and upgrades to PQ once the exchange converges. +const EnvStrict = "NB_PQ_MLKEM_STRICT" + +// Strict reports whether strict (fail-closed) mode is enabled via the environment. +// An empty or unrecognized value is treated as disabled (opportunistic). +func Strict() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(EnvStrict))) { + case "on": + return true + case "", "off": + return false + } + enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(EnvStrict))) + if err != nil { + log.Warnf("failed to parse %s value %q: %v", EnvStrict, os.Getenv(EnvStrict), err) + return false + } + return enabled +} + // EnvLogLevel overrides the ML-KEM manager's slog level (trace/debug/info/warn/error). // Defaults to info. The verbose per-exchange lifecycle logs are emitted at trace. const EnvLogLevel = "NB_PQ_MLKEM_LOG_LEVEL" From e1d5551ba529cde3e887248e9877cbaf31c4c2e3 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 30 Jul 2026 16:41:05 +0200 Subject: [PATCH 043/119] Discriminate initial from rekey failure --- client/internal/pqkem/convergence.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index deaf68cba41..8094f0e9200 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -208,9 +208,10 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID RemoteID, id Excha case stateAwaitingAnswer: if attempts >= m.maxRetries { delete(m.exchanges, remoteID) + initial := !m.established[remoteID] fail := m.registerFailureLocked(remoteID) m.mu.Unlock() - m.raiseFailure(remoteID, fail) + m.raiseFailure(remoteID, fail, initial) return } viaSignal := ex.viaSignal @@ -251,11 +252,19 @@ func (m *Manager) registerFailureLocked(remoteID RemoteID) bool { return false } -func (m *Manager) raiseFailure(remoteID RemoteID, fail bool) { +// raiseFailure reports a convergence failure. initial distinguishes a never-established +// peer (bootstrap failed → no PQ PSK at all; in strict mode the peer stays blocked = +// "stuck") from a rekey failure (a previous PSK is still in force and traffic continues). +func (m *Manager) raiseFailure(remoteID RemoteID, fail, initial bool) { if !fail { m.logger.Warn("pqkem: rekey attempt timed out, will retry next cycle", "peer", remoteID) return } + if initial { + m.logger.Warn("pqkem: initial exchange failed — no PQ PSK established for peer (strict mode keeps the peer blocked until it converges)", "peer", remoteID) + } else { + m.logger.Warn("pqkem: rekey failed after retries — staying on the previous PSK", "peer", remoteID) + } if err := m.cbHandler.OnRekeyFailed(remoteID); err != nil { m.logger.Error("pqkem: OnRekeyFailed handler error", "peer", remoteID, "err", err) } From d485567978778f5c31fe5c8b0236683cfa5d2dfd Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 31 Jul 2026 10:47:07 +0200 Subject: [PATCH 044/119] pqkem: recover from persistent rekey failure by re-bootstrapping over signal OnRekeyFailed now re-runs the KEM bootstrap over Signal (conn.RequestReoffer -> handshaker.SendOffer) instead of only logging: a fresh signalling offer starts a new exchange that overwrites the stalled PSK on both sides, resyncing after a persistent data-path desync. Chosen over a responder-side awaitingAck revert (which fights the confirm-less ack timing) and a full tunnel teardown (heavier). The tunnel stays up on the previous PSK meanwhile since Signal is independent of the broken data path. --- client/internal/engine.go | 13 ++++++++++++- client/internal/peer/conn.go | 16 ++++++++++++++++ client/internal/pqkem_adapter.go | 15 ++++++++++++--- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index e153da6fdab..0ad70cfec42 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -689,7 +689,18 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqErr != nil { log.Errorf("pqkem: transport bind failed, exchange disabled: %v", pqErr) } else { - e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), pqCallbackHandler{wg: e.wgInterface}, pqkem.NewLogger()) + cbHandler := pqCallbackHandler{ + wg: e.wgInterface, + // On a persistent rekey failure, re-bootstrap the KEM over Signal: a + // fresh signalling offer starts a new exchange that overwrites the + // stalled PSK on both sides, recovering from a data-path desync. + reoffer: func(remoteKey string) { + if conn, ok := e.peerStore.PeerConn(remoteKey); ok { + conn.RequestReoffer() + } + }, + } + e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger()) e.pqkemManager.Start(tr) log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 430232f76c6..a0d9e0bdd7a 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -726,6 +726,22 @@ func (conn *Conn) onGuardEvent() { } } +// RequestReoffer sends a fresh signalling offer for the peer, re-running the +// post-quantum bootstrap over Signal. Used to recover from a persistent data-path +// rekey failure: a new exchange overwrites the stalled PSK on both sides. No-op if the +// connection is not open yet. +func (conn *Conn) RequestReoffer() { + conn.mu.Lock() + h := conn.handshaker + conn.mu.Unlock() + if h == nil { + return + } + if err := h.SendOffer(); err != nil { + conn.Log.Debugf("pqkem: recovery re-offer failed: %v", err) + } +} + func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { conn.mu.Lock() defer conn.mu.Unlock() diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index fe41082ad4e..7a66c8a692a 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -20,6 +20,9 @@ type pqPresharedKeySetter interface { // engine-side implementation of pqkem.CallbackHandler. type pqCallbackHandler struct { wg pqPresharedKeySetter + // reoffer re-bootstraps the KEM over Signal for a peer (a fresh signalling offer) + // to recover from a persistent data-path rekey failure. Nil disables recovery. + reoffer func(remoteKey string) } // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op @@ -32,10 +35,16 @@ func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) } -// OnRekeyFailed reports a failed PQ (re)key convergence. -// TODO(NET-1406): tear the peer connection down / trigger ICE reconnect. +// OnRekeyFailed reports a failed PQ (re)key convergence and re-bootstraps the KEM over +// Signal to recover: a fresh signalling offer starts a new exchange that overwrites the +// stalled PSK on both sides, resyncing after a persistent data-path desync. The tunnel +// stays up on the previous PSK meanwhile (the Signal channel is independent of the +// broken data path). func (h pqCallbackHandler) OnRekeyFailed(remoteID pqkem.RemoteID) error { - log.Warnf("pqkem: post-quantum rekey failed for peer %s", remoteID) + log.Warnf("pqkem: post-quantum rekey failed for peer %s, re-bootstrapping over signal", remoteID) + if h.reoffer != nil { + h.reoffer(string(remoteID)) + } return nil } From 63d29966fd78b3d5a630e5cc77a935dbae36541e Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 31 Jul 2026 13:11:06 +0200 Subject: [PATCH 045/119] pqkem: concurrency tests (recovery + race) - RecoversViaResignalAfterDataPathBreak: a data-path rotation that can no longer converge raises OnRekeyFailed, and re-bootstrapping over signalling resyncs both peers on a fresh PSK even while the data path stays broken. - ConcurrentRekeysNoRace: hammers the single-lock state machine with concurrent rotation clocks from many goroutines (run with -race) and asserts no split-brain via a final deterministic bootstrap. --- client/internal/pqkem/concurrency_test.go | 77 +++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 client/internal/pqkem/concurrency_test.go diff --git a/client/internal/pqkem/concurrency_test.go b/client/internal/pqkem/concurrency_test.go new file mode 100644 index 00000000000..0303750a90b --- /dev/null +++ b/client/internal/pqkem/concurrency_test.go @@ -0,0 +1,77 @@ +package pqkem + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestConcurrency_RecoversViaResignalAfterDataPathBreak exercises the A-light recovery: +// a data-path rotation can no longer converge (OnRekeyFailed), and re-bootstrapping over +// signalling resyncs both peers on a fresh PSK — even while the data path stays broken, +// since the signal channel is independent of it. +func TestConcurrency_RecoversViaResignalAfterDataPathBreak(t *testing.T) { + dA, dB, wgA, wgB, lbB := pair(t) + defer dA.Stop() + defer dB.Stop() + + // Tighten B's timings and make a single rotation miss raise OnRekeyFailed. Set + // before any exchange loop spawns (the loop reads these fields). + dB.retryInterval = 5 * time.Millisecond + dB.maxRetries = 2 + dB.maxRekeyFailures = 1 + + bootstrap(t, dA, dB) + dA.OnDataPathRekeyed("bbbb", 0) + dB.OnDataPathRekeyed("aaaa", 0) + psk1 := wgB.psk("aaaa") + require.NotEqual(t, PSK{}, psk1) + require.Equal(t, psk1, wgA.psk("bbbb"), "converged on the same PSK after bootstrap+rotation") + + // Data path breaks: the rotation can no longer converge -> OnRekeyFailed. + lbB.drop.Store(true) + _, err := dB.startExchange("aaaa", false, ExchangeID{}) + require.NoError(t, err) + require.Eventually(t, func() bool { return failedCount(wgB) >= 1 }, time.Second, 5*time.Millisecond) + + // Recovery: re-bootstrap over signalling with the data path STILL broken. It must + // still converge (signal is independent of the data path) on a fresh PSK. + bootstrap(t, dA, dB) + psk2 := wgB.psk("aaaa") + require.NotEqual(t, psk1, psk2, "recovery derived a fresh PSK") + require.Equal(t, psk2, wgA.psk("bbbb"), "both sides resync after recovery") +} + +// TestConcurrency_ConcurrentRekeysNoRace hammers both managers with concurrent rotation +// clocks from many goroutines. Its primary job (with -race) is to prove the single-lock +// state machine has no data races or deadlocks under contention; a final deterministic +// bootstrap then asserts there is no split-brain (both sides on the same PSK). +func TestConcurrency_ConcurrentRekeysNoRace(t *testing.T) { + dA, dB, wgA, wgB, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + dB.OnDataPathRekeyed("aaaa", 0) // initiator chains a rotation + dA.OnDataPathRekeyed("bbbb", 0) // responder side is a no-op, still stresses the lock + } + }() + } + wg.Wait() + + // The storm may leave an exchange mid-flight (concurrent cancellation). Force a + // clean convergence over signalling, then assert no split-brain. + bootstrap(t, dA, dB) + a, b := wgA.psk("bbbb"), wgB.psk("aaaa") + require.NotEqual(t, PSK{}, b) + require.Equal(t, a, b, "both sides converge on the same PSK, no split-brain") +} From d98852585099a1c2745c886f5b1cd22efafe470b Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 15:14:42 +0200 Subject: [PATCH 046/119] Prioritize Kem over RP --- client/internal/engine.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 0ad70cfec42..50f6c5b132b 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -592,7 +592,14 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) publicKey := e.config.WgPrivateKey.PublicKey() e.flowManager = netflow.NewManager(e.wgInterface, publicKey[:], e.statusRecorder) - if e.config.RosenpassEnabled { + // Rosenpass and ML-KEM are mutually exclusive post-quantum providers: both + // program the same WireGuard PSK, so running them together would race on + // SetPresharedKey. ML-KEM (NB_ENABLE_PQ_MLKEM) takes precedence; when it is + // enabled Rosenpass is skipped even if configured on. + if e.config.RosenpassEnabled && pqkem.Enabled() { + log.Warnf("rosenpass and ML-KEM post-quantum are mutually exclusive; ML-KEM is enabled, so rosenpass is disabled") + } + if e.config.RosenpassEnabled && !pqkem.Enabled() { log.Infof("rosenpass is enabled") if e.config.RosenpassPermissive { log.Infof("running rosenpass in permissive mode") @@ -683,7 +690,9 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) } // Start the ML-KEM PQ manager after the interface is up so its dedicated UDP - // transport can bind on the WG overlay IP. + // transport can bind on the WG overlay IP. ML-KEM takes precedence over + // Rosenpass (see the mutual-exclusion note at rosenpass startup above), so + // when it is enabled rosenpass has already been skipped. if pqkem.Enabled() { tr, pqErr := newPQTransport(e.config.WgAddr.IP) if pqErr != nil { From bfa36521da3318a27d7039fa19f8bbea364dddf3 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 15:16:00 +0200 Subject: [PATCH 047/119] Adds test to validate compromised keys are not accepted --- client/internal/pqkem/kem_framing_test.go | 105 ++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 client/internal/pqkem/kem_framing_test.go diff --git a/client/internal/pqkem/kem_framing_test.go b/client/internal/pqkem/kem_framing_test.go new file mode 100644 index 00000000000..771df0ca959 --- /dev/null +++ b/client/internal/pqkem/kem_framing_test.go @@ -0,0 +1,105 @@ +package pqkem + +import ( + "crypto/mlkem" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestExchange_TamperedCiphertextFailsClosed verifies the core fail-closed +// property: mutating the ML-KEM ciphertext in the answer does not error (ML-KEM +// uses implicit rejection — Decapsulate always returns a value) but yields a +// different shared secret, so the initiator derives a PSK that does NOT match the +// responder's. A mismatched PSK means WireGuard passes no bytes: tamper => no data. +func TestExchange_TamperedCiphertextFailsClosed(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + answer, pskB, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + tampered := append([]byte(nil), answer...) + tampered[0] ^= 0xff // flip a bit in the ML-KEM ciphertext + + pskA, err := init.Finish(tampered, Binding{LocalID: wgA, RemoteID: wgB}) + require.NoError(t, err, "implicit rejection: decapsulate still succeeds") + require.NotEqual(t, pskB, pskA, "tampered ciphertext must not yield the responder's PSK") +} + +// TestExchange_TamperedX25519ShareDiverges flips a byte in the answer's X25519 +// share: the classical half of the hybrid secret changes, so the derived PSK +// diverges from the responder's (fail-closed on the ECDH half too). +func TestExchange_TamperedX25519ShareDiverges(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + answer, pskB, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + tampered := append([]byte(nil), answer...) + tampered[mlkem.CiphertextSize768] ^= 0x01 // first byte of the X25519 public key + + pskA, err := init.Finish(tampered, Binding{LocalID: wgA, RemoteID: wgB}) + // Either the point is rejected (error) or the ECDH differs (different PSK); + // in both cases the honest PSK is never reproduced. + if err == nil { + require.NotEqual(t, pskB, pskA, "tampered X25519 share must not yield the responder's PSK") + } +} + +// TestExchange_AllZeroX25519Rejected feeds an all-zero X25519 share (a low-order +// point) in the answer. The stdlib ECDH must reject it, so Finish errors rather +// than deriving a PSK from a degenerate secret. +func TestExchange_AllZeroX25519Rejected(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + answer, _, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + bad := append([]byte(nil), answer...) + for i := mlkem.CiphertextSize768; i < len(bad); i++ { + bad[i] = 0 + } + + _, err = init.Finish(bad, Binding{LocalID: wgA, RemoteID: wgB}) + require.Error(t, err, "all-zero X25519 share (low-order point) must be rejected") +} + +// TestExchange_SizeBoundaries locks the exact-length framing checks: one byte +// short or long on either message is rejected, not silently truncated/padded. +func TestExchange_SizeBoundaries(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + offer := init.Offer() + + _, _, err = Respond(offer[:OfferSize-1], Binding{}) + require.Error(t, err, "offer one byte short") + _, _, err = Respond(append(append([]byte(nil), offer...), 0), Binding{}) + require.Error(t, err, "offer one byte long") + + answer, _, err := Respond(offer, Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + _, err = init.Finish(answer[:AnswerSize-1], Binding{LocalID: wgA, RemoteID: wgB}) + require.Error(t, err, "answer one byte short") + _, err = init.Finish(append(append([]byte(nil), answer...), 0), Binding{LocalID: wgA, RemoteID: wgB}) + require.Error(t, err, "answer one byte long") +} + +// TestExchange_BindingIsSymmetric confirms the canonicalisation: the two peers +// pass their identities in opposite (Local, Remote) order yet derive the same PSK, +// so identity binding does not depend on who is initiator vs responder. +func TestExchange_BindingIsSymmetric(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + + answer, pskB, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + pskA, err := init.Finish(answer, Binding{LocalID: wgA, RemoteID: wgB}) + require.NoError(t, err) + + require.Equal(t, pskB, pskA, "swapped Local/Remote order must canonicalise to the same PSK") +} From 202176ec23453724f641d4669f22f8254185458b Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 15:30:15 +0200 Subject: [PATCH 048/119] Be more explicit on names that is a fake key to ensure we don't communicate with others in strict mode --- client/internal/peer/conn.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index a0d9e0bdd7a..6af0457af82 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -191,10 +191,10 @@ type Conn struct { // transport is up. pendingFirstPacket []byte - // pqBlockingKey is a per-conn random sentinel PSK used in PQ strict mode to fail - // closed: it is programmed until the real ML-KEM PSK is derived, so no session can - // form on a non-PQ key. Per-conn random so two strict peers never match by chance. - pqBlockingKey *wgtypes.Key + // pqStrictSentinelKey is a per-conn random sentinel PSK used in PQ strict mode to + // fail closed: it is programmed until the real ML-KEM PSK is derived, so no session + // can form on a non-PQ key. Per-conn random so two strict peers never match by chance. + pqStrictSentinelKey *wgtypes.Key } // injectPendingFirstPacket replays the captured handshake through the proxy if present, else @@ -256,7 +256,7 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) { if k, err := wgtypes.GenerateKey(); err != nil { connLog.Errorf("pqkem: failed to generate strict-mode sentinel key, strict fail-closed disabled for this peer: %v", err) } else { - conn.pqBlockingKey = &k + conn.pqStrictSentinelKey = &k } } @@ -1095,13 +1095,13 @@ func (conn *Conn) presharedKey(remoteRosenpassKey []byte) *wgtypes.Key { if psk, ok := conn.config.PQ.PSK(conn.config.Key); ok { return &psk } - if conn.config.PQStrict && conn.pqBlockingKey != nil { + if conn.config.PQStrict && conn.pqStrictSentinelKey != nil { // Fail closed: program a non-matching sentinel so no session forms on a // non-PQ key until the ML-KEM exchange derives the real PSK (pushed via // SetPresharedKey once it converges). "pending" — turns into a "stuck" // warning from the manager if the exchange keeps failing (see raiseFailure). conn.Log.Debugf("pqkem: strict mode — no PQ PSK yet, blocking peer traffic until the ML-KEM exchange converges") - return conn.pqBlockingKey + return conn.pqStrictSentinelKey } } From cb5ee8efa89f5daf112cc77f03fa443f29aed0d6 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 15:29:15 +0200 Subject: [PATCH 049/119] Remove obvious comments; leave only the why of things --- client/internal/engine.go | 9 ++------- client/internal/pqkem/manager.go | 18 ++++++++---------- client/internal/pqkem_adapter.go | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 50f6c5b132b..6b4af1196d1 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -592,10 +592,7 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) publicKey := e.config.WgPrivateKey.PublicKey() e.flowManager = netflow.NewManager(e.wgInterface, publicKey[:], e.statusRecorder) - // Rosenpass and ML-KEM are mutually exclusive post-quantum providers: both - // program the same WireGuard PSK, so running them together would race on - // SetPresharedKey. ML-KEM (NB_ENABLE_PQ_MLKEM) takes precedence; when it is - // enabled Rosenpass is skipped even if configured on. + // Rosenpass and ML-KEM are mutually exclusive. ML-KEM (NB_ENABLE_PQ_MLKEM) takes precedence if e.config.RosenpassEnabled && pqkem.Enabled() { log.Warnf("rosenpass and ML-KEM post-quantum are mutually exclusive; ML-KEM is enabled, so rosenpass is disabled") } @@ -690,9 +687,7 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) } // Start the ML-KEM PQ manager after the interface is up so its dedicated UDP - // transport can bind on the WG overlay IP. ML-KEM takes precedence over - // Rosenpass (see the mutual-exclusion note at rosenpass startup above), so - // when it is enabled rosenpass has already been skipped. + // transport can bind on the WG overlay IP. if pqkem.Enabled() { tr, pqErr := newPQTransport(e.config.WgAddr.IP) if pqErr != nil { diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index bb3aae48de8..8306433c702 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -139,7 +139,7 @@ func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manage } // Start installs the data-path transport and begins its inbound delivery. The Manager -// owns it from here; Stop closes it. Start/Stop are the transport lifecycle pair. +// owns it from here; Stop closes it. func (m *Manager) Start(t Transport) { m.mu.Lock() m.transport = t @@ -184,7 +184,7 @@ func (m *Manager) trace(msg string, args ...any) { } // AddPeer registers where a peer's data-path messages are sent and received: its -// overlay endpoint (IP:port). Re-adding updates the endpoint. +// overlay endpoint (IP:port). func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { if !endpoint.IsValid() { return @@ -322,14 +322,12 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } } -// OnDataPathRekeyed notifies that the peer's data path is up and freshly keyed with -// the latest PSK (fired on first establishment AND every rekey). If we are the -// initiator that just derived a PSK, it chains the next exchange: a fresh offer over -// the data path that acknowledges the just-completed one (its arrival under the new -// key proves to the responder that the key works). -// OnDataPathRekeyed clocks the next chained PSK rotation on a fresh data-path rekey. -// sinceActivity is how long ago the peer last exchanged real user data; when it -// exceeds rotationActivityWindow the tunnel is treated as idle and rotation is +// OnDataPathRekeyed clocks the next chained PSK rotation on a fresh data-path rekey +// (fired on first establishment AND every rekey). If we are the initiator that just +// derived a PSK, it chains the next exchange: a fresh offer over the data path that +// acknowledges the just-completed one (its arrival under the new key proves to the +// responder the key works). sinceActivity is how long ago the peer last exchanged real +// user data; past rotationActivityWindow the tunnel is treated as idle and rotation is // skipped — an idle tunnel has nothing to protect, and rotating would emit data-path // traffic that keeps the peer artificially active (see conn.onWGCheckSuccess). func (m *Manager) OnDataPathRekeyed(remoteID RemoteID, sinceActivity time.Duration) { diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 7a66c8a692a..d99eb1b5d15 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -26,7 +26,7 @@ type pqCallbackHandler struct { } // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op -// if the peer is not present, mirroring Rosenpass). remoteID is the peer's WG pubkey. +// if the peer is not present, mirroring Rosenpass). func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) error { // updateOnly: applies to an already-configured peer (rotation). At bootstrap the // peer is not configured yet, so this is a no-op there and the PSK is instead From 5a6de00f7a78809ef57d0dfdd1d69415871974ef Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 16:24:36 +0200 Subject: [PATCH 050/119] Adds PQ connection tests --- client/internal/peer/conn_pq_test.go | 85 ++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 client/internal/peer/conn_pq_test.go diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go new file mode 100644 index 00000000000..11d81b36e71 --- /dev/null +++ b/client/internal/peer/conn_pq_test.go @@ -0,0 +1,85 @@ +package peer + +import ( + "net/netip" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// fakePQ is a minimal PQHandshaker: only PSK is exercised by presharedKey, the rest +// are no-op stubs to satisfy the interface. +type fakePQ struct { + psk wgtypes.Key + ok bool +} + +func (f fakePQ) OfferPayload(string) ([]byte, int) { return nil, 0 } +func (f fakePQ) AnswerPayload(string, []byte) ([]byte, int) { return nil, 0 } +func (f fakePQ) OnAnswer(string, []byte) {} +func (f fakePQ) PSK(string) (wgtypes.Key, bool) { return f.psk, f.ok } +func (f fakePQ) SetRemoteAddr(string, netip.AddrPort) {} +func (f fakePQ) OnDataPathRekeyed(string, time.Duration) {} +func (f fakePQ) OnDataPathDown(string) {} + +// TestConn_presharedKey_PQ covers the post-quantum branch of presharedKey across the +// three states that matter: a derived PSK is programmed, and — before one exists — +// strict mode blocks with a sentinel while non-strict falls open to the ordinary key. +func TestConn_presharedKey_PQ(t *testing.T) { + derivedPSK, err := wgtypes.GenerateKey() + require.NoError(t, err) + nbPSK, err := wgtypes.GenerateKey() + require.NoError(t, err) + + newConn := func() *Conn { + return &Conn{ + Log: log.WithField("peer", "pq-test"), + config: ConnConfig{ + Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + WgConfig: WgConfig{PreSharedKey: &nbPSK}, + RosenpassConfig: RosenpassConfig{}, + }, + } + } + + t.Run("derived PSK is programmed", func(t *testing.T) { + for _, strict := range []bool{false, true} { + c := newConn() + c.config.PQ = fakePQ{psk: derivedPSK, ok: true} + c.config.PQStrict = strict + if strict { + sentinel, _ := wgtypes.GenerateKey() + c.pqStrictSentinelKey = &sentinel + } + got := c.presharedKey(nil) + require.NotNil(t, got) + require.Equal(t, derivedPSK, *got, "the derived PQ PSK must win (strict=%v)", strict) + } + }) + + t.Run("non-strict falls open to the ordinary key before a PSK exists", func(t *testing.T) { + c := newConn() + c.config.PQ = fakePQ{ok: false} + c.config.PQStrict = false + got := c.presharedKey(nil) + require.NotNil(t, got, "non-strict must not block") + require.Equal(t, nbPSK, *got, "non-strict falls through to the NetBird PSK, not a sentinel") + }) + + t.Run("strict blocks with the per-conn sentinel before a PSK exists", func(t *testing.T) { + sentinel, err := wgtypes.GenerateKey() + require.NoError(t, err) + c := newConn() + c.config.PQ = fakePQ{ok: false} + c.config.PQStrict = true + c.pqStrictSentinelKey = &sentinel + got := c.presharedKey(nil) + require.NotNil(t, got) + require.Equal(t, sentinel, *got, "strict must return the blocking sentinel") + require.NotEqual(t, nbPSK, *got, "the sentinel must not be the ordinary key") + }) +} From cfaf4f1bab7ace61a2ba6a558aa77fdc1ca61b01 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 18:04:10 +0200 Subject: [PATCH 051/119] Allow non strict mode --- client/internal/pqkem/capability_test.go | 64 ++++++++++++++++++++++++ client/internal/pqkem/convergence.go | 2 + client/internal/pqkem/manager.go | 36 ++++++++++++- client/internal/pqkem_adapter.go | 16 ++++++ client/internal/pqkem_adapter_test.go | 37 ++++++++++++++ 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 client/internal/pqkem/capability_test.go create mode 100644 client/internal/pqkem_adapter_test.go diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go new file mode 100644 index 00000000000..b42acc79d04 --- /dev/null +++ b/client/internal/pqkem/capability_test.go @@ -0,0 +1,64 @@ +package pqkem + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestManager_NonCapablePeerNotOffered: a peer known not to run the KEM (it advertised +// no PQ port over signalling) is never offered an exchange, and no failure is raised — +// this is what stops the reoffer storm against non-PQ peers (e.g. Rosenpass peers). +func TestManager_NonCapablePeerNotOffered(t *testing.T) { + wg := newFakeWG() + d := NewManager("bbbb", wg, nil) // initiator vs "aaaa" + d.Start(&loopback{ep: epB, sw: newSwitch()}) + defer d.Stop() + + d.MarkNonCapable("aaaa") + + offer, err := d.SignalOffer("aaaa") + require.NoError(t, err) + require.Nil(t, offer, "a non-capable peer must not be offered a KEM exchange") + require.Empty(t, wg.failed, "a non-capable peer must not raise a rekey failure") +} + +// TestManager_MarkNonCapableCancelsInFlight: if we start an exchange with a peer whose +// capability is not yet known and then learn it does not run the KEM, the in-flight +// exchange is cancelled and no further offer is produced (no timeout -> no failure). +func TestManager_MarkNonCapableCancelsInFlight(t *testing.T) { + wg := newFakeWG() + d := NewManager("bbbb", wg, nil) + d.Start(&loopback{ep: epB, sw: newSwitch()}) + defer d.Stop() + + // Capability unknown -> the bootstrap offer goes out optimistically. + offer, err := d.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) + + // Now we learn the peer is non-PQ: the exchange must be dropped. + d.MarkNonCapable("aaaa") + + next, err := d.SignalOffer("aaaa") + require.NoError(t, err) + require.Nil(t, next, "after learning non-capability the peer is no longer offered") + require.Empty(t, wg.failed, "cancelling an in-flight exchange must not raise a failure") +} + +// TestManager_EstablishedPeerNotDowngraded: a stray zero-port observation must not tear +// down a peer we already have a working PQ session with. +func TestManager_EstablishedPeerNotDowngraded(t *testing.T) { + dA, dB, _, wgB, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + require.NotEqual(t, PSK{}, wgB.psk("aaaa"), "established a PSK") + + dB.MarkNonCapable("aaaa") // stray zero after establishment + + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer, "an established peer must keep running the KEM despite a stray zero") +} diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 8094f0e9200..0d6085f732e 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -108,6 +108,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { ex.lastSent = raw ex.pendingPSK = psk m.psks[remoteID] = psk + m.capable[remoteID] = true // a real KEM offer proves the peer runs the exchange m.mu.Unlock() m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "psk_fp", pskFingerprint(psk)) @@ -153,6 +154,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.established[remoteID] = true m.failures[remoteID] = 0 m.psks[remoteID] = psk + m.capable[remoteID] = true // a real KEM answer proves the peer runs the exchange m.mu.Unlock() m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "psk_fp", pskFingerprint(psk)) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 8306433c702..205194ebd67 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -107,6 +107,7 @@ type Manager struct { established map[RemoteID]bool // peer has completed at least one exchange failures map[RemoteID]int // consecutive rekey failures per peer psks map[RemoteID]PSK // latest derived PSK per peer (pulled at WG peer-config time) + capable map[RemoteID]bool // peer runs the KEM (advertised a PQ port); false = known non-capable peerAddrs map[RemoteID]netip.AddrPort // remoteID -> data-path endpoint (send routing) peersByAddr map[netip.AddrPort]RemoteID // reverse: source endpoint -> remoteID (inbound) wait sync.WaitGroup @@ -133,6 +134,7 @@ func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manage established: make(map[RemoteID]bool), failures: make(map[RemoteID]int), psks: make(map[RemoteID]PSK), + capable: make(map[RemoteID]bool), peerAddrs: make(map[RemoteID]netip.AddrPort), peersByAddr: make(map[netip.AddrPort]RemoteID), } @@ -184,7 +186,8 @@ func (m *Manager) trace(msg string, args ...any) { } // AddPeer registers where a peer's data-path messages are sent and received: its -// overlay endpoint (IP:port). +// overlay endpoint (IP:port). A peer that advertises a PQ endpoint is, by that fact, +// running the KEM, so it is marked capable. func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { if !endpoint.IsValid() { return @@ -195,9 +198,35 @@ func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { } m.peerAddrs[remoteID] = endpoint m.peersByAddr[endpoint] = remoteID + m.capable[remoteID] = true m.mu.Unlock() } +// MarkNonCapable records that a peer does not run the KEM: it answered our offer with +// no KEM material over signalling (the capability signal is the peer's payload, not its +// optional data-path port). Any in-flight exchange is cancelled and further offers are +// suppressed (see SignalOffer), so a non-PQ peer never drives the rekey-recovery storm. +// An already-established peer is left untouched — a stray empty answer must not tear +// down a working PQ session. +func (m *Manager) MarkNonCapable(remoteID RemoteID) { + m.mu.Lock() + defer m.mu.Unlock() + if m.established[remoteID] { + return + } + if prev, ok := m.capable[remoteID]; ok && !prev { + return // already known non-capable, nothing to do + } + m.capable[remoteID] = false + if ex := m.exchanges[remoteID]; ex != nil { + if ex.cancel != nil { + ex.cancel() + } + delete(m.exchanges, remoteID) + } + m.trace("pqkem: peer advertises no PQ service — treating as non-capable, no KEM attempted", "peer", remoteID) +} + // RemovePeer stops any in-flight exchange for a peer and drops its state and routing. func (m *Manager) RemovePeer(remoteID RemoteID) { m.mu.Lock() @@ -210,6 +239,7 @@ func (m *Manager) RemovePeer(remoteID RemoteID) { delete(m.established, remoteID) delete(m.failures, remoteID) delete(m.psks, remoteID) + delete(m.capable, remoteID) if ep, ok := m.peerAddrs[remoteID]; ok { delete(m.peersByAddr, ep) delete(m.peerAddrs, remoteID) @@ -246,6 +276,10 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { return nil, nil } m.mu.Lock() + if capable, ok := m.capable[remoteID]; ok && !capable { + m.mu.Unlock() + return nil, nil // peer does not run the KEM; do not offer (avoids a failure/reoffer loop) + } if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && ex.state == stateAwaitingAnswer { last := ex.lastSent m.mu.Unlock() diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index d99eb1b5d15..332dc4da4cd 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -64,6 +64,14 @@ func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, int) { func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, int) { if len(recvOffer) == 0 { + // Capability signal (responder side): the KEM offer flows initiator->responder, + // so if we are the responder for this peer (it is the KEM initiator by role) an + // empty offer means it does not run the KEM. If we are the initiator, an empty + // offer is normal — the peer is the responder and puts its material in the + // answer — so we must not flag it. + if !p.mgr.IsInitiator(pqkem.RemoteID(remoteKey)) { + p.mgr.MarkNonCapable(pqkem.RemoteID(remoteKey)) + } return nil, p.mgr.LocalPort() } payload, err := p.mgr.SignalOnOffer(pqkem.RemoteID(remoteKey), recvOffer) @@ -75,6 +83,14 @@ func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { if len(recvAnswer) == 0 { + // Capability signal (initiator side): the KEM answer flows responder->initiator, + // so an empty answer to our offer means the peer does not run the KEM — mark it + // non-capable to stop offering (no failure/reoffer storm). Only meaningful when + // we are the initiator: as the responder we also receive an (empty) answer to + // our own non-KEM offer from a perfectly capable peer, which must not be flagged. + if p.mgr.IsInitiator(pqkem.RemoteID(remoteKey)) { + p.mgr.MarkNonCapable(pqkem.RemoteID(remoteKey)) + } return } if err := p.mgr.SignalOnAnswer(pqkem.RemoteID(remoteKey), recvAnswer); err != nil { diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go new file mode 100644 index 00000000000..dec3b5943ce --- /dev/null +++ b/client/internal/pqkem_adapter_test.go @@ -0,0 +1,37 @@ +package internal + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal/pqkem" +) + +type pqNoopHandler struct{} + +func (pqNoopHandler) OnNewPSKReady(pqkem.RemoteID, pqkem.PSK) error { return nil } +func (pqNoopHandler) OnRekeyFailed(pqkem.RemoteID) error { return nil } + +// TestPQAdapter_CapabilityRoleAware locks the role-aware capability signal: the KEM +// payload only flows initiator-offer -> responder-answer, so an empty message in the +// other direction comes from a perfectly capable peer and must NOT flag it. Only the +// message that should carry material (the answer we receive as initiator) marks a peer +// non-capable when empty. +func TestPQAdapter_CapabilityRoleAware(t *testing.T) { + // localID "zzzz" > "aaaa" => this manager is the KEM initiator for peer "aaaa". + mgr := pqkem.NewManager("zzzz", pqNoopHandler{}, nil) + defer mgr.Stop() + h := pqHandshaker{mgr: mgr} + + // An empty OFFER from our peer is normal here: as the initiator's responder it puts + // its material in the answer, not the offer. It must not disable our offering. + h.AnswerPayload("aaaa", nil) + payload, _ := h.OfferPayload("aaaa") + require.NotNil(t, payload, "an empty offer from a responder-role peer must not mark it non-capable") + + // An empty ANSWER to our offer means the peer does not run the KEM -> stop offering. + h.OnAnswer("aaaa", nil) + payload2, _ := h.OfferPayload("aaaa") + require.Nil(t, payload2, "an empty answer to our offer marks the peer non-capable, so we stop offering") +} From b93e8d66768a01814ff5e313aec3dbe0a275df71 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 4 Aug 2026 18:33:13 +0200 Subject: [PATCH 052/119] Skip default port send in signal proto --- client/internal/peer/conn.go | 3 ++- client/internal/peer/handshaker.go | 4 +++- client/internal/pqkem/manager.go | 6 +++--- client/internal/pqkem_adapter.go | 32 ++++++++++++++++++++++-------- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 6af0457af82..ba058282b96 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -94,7 +94,8 @@ type PQHandshaker interface { // peer-config time (the pull path). ok is false until one has been derived. PSK(remoteKey string) (wgtypes.Key, bool) // SetRemoteAddr registers the peer's data-path endpoint learned from signalling: - // its WG overlay IP with the advertised pq UDP port. + // its WG overlay IP with the announced pq UDP port (port 0 means the peer omitted + // it and is on the default port). SetRemoteAddr(remoteKey string, addr netip.AddrPort) // OnDataPathRekeyed signals a fresh WireGuard handshake for the peer; it clocks the // next chained PSK rotation pushed over the data path. sinceActivity is how long diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 0ba9c70df4a..0670068c815 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -182,9 +182,11 @@ func (h *Handshaker) Listen(ctx context.Context) { // pqRegisterEndpoint feeds the post-quantum handshaker the peer's data-path endpoint // (its WG overlay IP plus the advertised pq UDP port) learned from a remote offer/answer. func (h *Handshaker) pqRegisterEndpoint(remotePort int) { - if h.config.PQ == nil || remotePort <= 0 || remotePort > 65535 || len(h.config.WgConfig.AllowedIps) == 0 { + if h.config.PQ == nil || remotePort < 0 || remotePort > 65535 || len(h.config.WgConfig.AllowedIps) == 0 { return } + // remotePort may be 0 (the peer omitted it, meaning the default port); the adapter + // resolves 0 to DefaultPort. addr := netip.AddrPortFrom(h.config.WgConfig.AllowedIps[0].Addr(), uint16(remotePort)) h.config.PQ.SetRemoteAddr(h.config.Key, addr) } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 205194ebd67..9bb108dfbeb 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -186,8 +186,9 @@ func (m *Manager) trace(msg string, args ...any) { } // AddPeer registers where a peer's data-path messages are sent and received: its -// overlay endpoint (IP:port). A peer that advertises a PQ endpoint is, by that fact, -// running the KEM, so it is marked capable. +// overlay endpoint (IP:port). This is pure routing and says nothing about capability — +// PQ capability is decided solely from the peer's KEM payload (see processOffer / +// processAnswer / MarkNonCapable), never from an endpoint or port. func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { if !endpoint.IsValid() { return @@ -198,7 +199,6 @@ func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { } m.peerAddrs[remoteID] = endpoint m.peersByAddr[endpoint] = remoteID - m.capable[remoteID] = true m.mu.Unlock() } diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 332dc4da4cd..82d2981de25 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -54,12 +54,22 @@ type pqHandshaker struct { mgr *pqkem.Manager } +// announcedPort is the PQ data-path port to advertise to peers. It is omitted (0) when +// the manager is on DefaultPort, since peers assume the default when no port is sent; +// only a non-default (collision-forced) port is announced explicitly. +func (p pqHandshaker) announcedPort() int { + if port := p.mgr.LocalPort(); port != DefaultPort { + return port + } + return 0 +} + func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, int) { payload, err := p.mgr.SignalOffer(pqkem.RemoteID(remoteKey)) if err != nil { log.Warnf("pqkem: build offer for %s: %v", remoteKey, err) } - return payload, p.mgr.LocalPort() + return payload, p.announcedPort() } func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, int) { @@ -72,13 +82,13 @@ func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, if !p.mgr.IsInitiator(pqkem.RemoteID(remoteKey)) { p.mgr.MarkNonCapable(pqkem.RemoteID(remoteKey)) } - return nil, p.mgr.LocalPort() + return nil, p.announcedPort() } payload, err := p.mgr.SignalOnOffer(pqkem.RemoteID(remoteKey), recvOffer) if err != nil { log.Warnf("pqkem: build answer for %s: %v", remoteKey, err) } - return payload, p.mgr.LocalPort() + return payload, p.announcedPort() } func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { @@ -107,14 +117,20 @@ func (p pqHandshaker) PSK(remoteKey string) (wgtypes.Key, bool) { return wgtypes.Key(psk), true } -// SetRemoteAddr registers the peer's data-path endpoint (overlay IP + pq UDP port) -// learned from signalling. Sends only ever fire once the tunnel is up (clocked by -// OnDataPathRekeyed), so registering here is safe even before connection-up. +// SetRemoteAddr registers the peer's data-path endpoint learned from signalling. A +// zero port means the peer omitted it (it is on DefaultPort), so we resolve it here — +// DefaultPort lives in this package, not in peer. Sends only ever fire once the tunnel +// is up (clocked by OnDataPathRekeyed), so registering here is safe even before +// connection-up. func (p pqHandshaker) SetRemoteAddr(remoteKey string, addr netip.AddrPort) { - if !addr.IsValid() || addr.Port() == 0 { + if !addr.Addr().IsValid() { return } - p.mgr.AddPeer(pqkem.RemoteID(remoteKey), addr) + port := addr.Port() + if port == 0 { + port = DefaultPort + } + p.mgr.AddPeer(pqkem.RemoteID(remoteKey), netip.AddrPortFrom(addr.Addr(), port)) } // OnDataPathRekeyed clocks the next chained PSK rotation on a fresh WG handshake. From 2b27040fb06ee990e8ae4783ce8c84c2c25692af Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 5 Aug 2026 16:39:55 +0200 Subject: [PATCH 053/119] Anticipates PSK before WG does handshake so it finds it to set it --- client/internal/peer/handshaker.go | 25 +++++++++++++++--------- client/internal/pqkem/capability_test.go | 7 ++++--- client/internal/pqkem/manager.go | 5 +++++ 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 0670068c815..a5035da07ef 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -137,6 +137,15 @@ func (h *Handshaker) Listen(ctx context.Context) { h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) + // Derive+store the KEM PSK (inside sendAnswer's AnswerPayload) BEFORE bringing + // up the connection: the relay/ICE workers configure the WG endpoint, which + // pulls the PSK for the first handshake. Notifying them first would race the + // KEM exchange and hand the first handshake a not-yet-derived key. + if err := h.sendAnswer(&remoteOfferAnswer); err != nil { + h.log.Errorf("failed to send remote offer confirmation: %s", err) + continue + } + if h.relayListener != nil { h.relayListener.Notify(&remoteOfferAnswer) } @@ -144,11 +153,6 @@ func (h *Handshaker) Listen(ctx context.Context) { if h.iceListener != nil && h.RemoteICESupported() { h.iceListener(&remoteOfferAnswer) } - - if err := h.sendAnswer(&remoteOfferAnswer); err != nil { - h.log.Errorf("failed to send remote offer confirmation: %s", err) - continue - } case remoteOfferAnswer := <-h.remoteAnswerCh: h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) @@ -161,6 +165,13 @@ func (h *Handshaker) Listen(ctx context.Context) { h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) + // Feed the KEM answer (derive+store PSK) BEFORE bringing up the connection so + // the WG endpoint config pulls the real PSK for the first handshake instead of + // racing ahead of the KEM exchange. + if h.config.PQ != nil { + h.config.PQ.OnAnswer(h.config.Key, remoteOfferAnswer.MlkemPayload) + } + if h.relayListener != nil { h.relayListener.Notify(&remoteOfferAnswer) } @@ -168,10 +179,6 @@ func (h *Handshaker) Listen(ctx context.Context) { if h.iceListener != nil && h.RemoteICESupported() { h.iceListener(&remoteOfferAnswer) } - - if h.config.PQ != nil { - h.config.PQ.OnAnswer(h.config.Key, remoteOfferAnswer.MlkemPayload) - } case <-ctx.Done(): h.log.Infof("stop listening for remote offers and answers") return diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index b42acc79d04..b6a1fe74c26 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -58,7 +58,8 @@ func TestManager_EstablishedPeerNotDowngraded(t *testing.T) { dB.MarkNonCapable("aaaa") // stray zero after establishment - offer, err := dB.SignalOffer("aaaa") - require.NoError(t, err) - require.NotNil(t, offer, "an established peer must keep running the KEM despite a stray zero") + // The peer keeps its derived PSK (MarkNonCapable is a no-op once established). + psk, ok := dB.PSK("aaaa") + require.True(t, ok, "an established peer must keep its PSK despite a stray zero") + require.NotEqual(t, PSK{}, psk) } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 9bb108dfbeb..fff266f380e 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -271,6 +271,11 @@ func (m *Manager) Stop() { // remoteID (bootstrap). It returns (nil, nil) when the local peer is not the // initiator. It is idempotent for an in-flight bootstrap: a repeat call returns the // same offer rather than starting a new exchange. +// +// A signal re-negotiation always re-bootstraps (fresh exchange): the remote may have +// restarted and lost its PSK, so reusing a locally frozen one would desync. The derived +// PSK still survives idle in the manager (dropped only on account-level peer removal), +// so a pure lazy wake with no re-negotiation reuses it via the conn's WG-config pull. func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { if !m.IsInitiator(remoteID) { return nil, nil From a78d587e7a69decdb6f0d82bb39a7b5694aa001b Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 6 Aug 2026 14:42:07 +0200 Subject: [PATCH 054/119] Introduces a forced WG handshake on initial MLKEM bootstrap. To ensure two peers agree on a key, we need asymmetry. one peer is the controller ("initiator") the other is the "responder". Otherwise imagine two offers in parallel driving two answers at the same time A B | <----B-OFFER----- | | -----A-OFFER----> | | | | | --------------------------------- |****** ICE + WG Handshake ****** | --------------------------------- | | | <----B-ANSWER---- | | -----A-ANSWER---> | PSK is derived on receive of offer, so A and B derive different PSKs. When WG handshake takes place it picks misaligned PSKs. So we impair the two nodes and only the offer of one of the two (the controller/initiator) carries the KEM material. This means that if the responder OFFER/ANSWER comes first, when the controller/initiator's one completes (and the genuine PSK is shared between A and B, we need to force a new WG handshake with the proper keys. --- client/internal/engine.go | 8 +++++++ client/internal/peer/conn.go | 29 +++++++++++++++++++++++ client/internal/peer/endpoint.go | 33 ++++++++++++++++++++++++++- client/internal/pqkem/callbacks.go | 8 ++++++- client/internal/pqkem/convergence.go | 8 +++---- client/internal/pqkem/manager.go | 8 +++---- client/internal/pqkem/manager_test.go | 2 +- client/internal/pqkem_adapter.go | 18 +++++++++++++-- client/internal/pqkem_adapter_test.go | 4 ++-- 9 files changed, 103 insertions(+), 15 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 6b4af1196d1..550e0036354 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -703,6 +703,14 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) conn.RequestReoffer() } }, + // On a freshly bootstrapped PSK, force the peer's WireGuard session to + // re-handshake so it adopts the PSK even if the tunnel already came up on + // a pre-PQ key (KEM-vs-endpoint-config race). + rehandshake: func(remoteKey string) { + if conn, ok := e.peerStore.PeerConn(remoteKey); ok { + conn.ForcePQRehandshake() + } + }, } e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger()) e.pqkemManager.Start(tr) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index ba058282b96..b37fabd0d12 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -743,6 +743,35 @@ func (conn *Conn) RequestReoffer() { } } +// ForcePQRehandshake makes WireGuard adopt a freshly bootstrapped post-quantum PSK on +// an already-up session. The KEM exchange can complete after the WG endpoint was +// configured (a race between the KEM and the relay/ICE connection coming up), so the +// live session may be keyed with a pre-PQ key (the strict sentinel, or the ordinary +// key in permissive mode); SetPresharedKey only updates config, not the running +// session. Removing and re-adding the peer forces a fresh handshake that uses the real +// PSK. No-op if not connected yet (the upcoming endpoint config will pull the PSK) or +// if no PSK exists (a non-PQ peer stays fail-closed). +func (conn *Conn) ForcePQRehandshake() { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil || conn.config.PQ == nil { + return + } + psk, ok := conn.config.PQ.PSK(conn.config.Key) + if !ok { + return + } + if conn.currentConnPriority == conntype.None { + return + } + conn.Log.Debugf("pqkem: bootstrap PSK ready, forcing WireGuard re-handshake to adopt it") + wgPsk := wgtypes.Key(psk) + if err := conn.endpointUpdater.ForceRehandshake(&wgPsk); err != nil { + conn.Log.Warnf("pqkem: force re-handshake failed: %v", err) + } +} + func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { conn.mu.Lock() defer conn.mu.Unlock() diff --git a/client/internal/peer/endpoint.go b/client/internal/peer/endpoint.go index 9ba1efb6eb3..0d741f74b61 100644 --- a/client/internal/peer/endpoint.go +++ b/client/internal/peer/endpoint.go @@ -20,10 +20,13 @@ type EndpointUpdater struct { wgConfig WgConfig initiator bool - // mu protects cancelFunc + // mu protects cancelFunc and lastEndpoint mu sync.Mutex cancelFunc func() updateWg sync.WaitGroup + // lastEndpoint is the most recent non-nil endpoint applied to the peer, used to + // re-add it on a forced re-handshake (ForceRehandshake). + lastEndpoint *net.UDPAddr } func NewEndpointUpdater(log *logrus.Entry, wgConfig WgConfig, initiator bool) *EndpointUpdater { @@ -38,6 +41,10 @@ func (e *EndpointUpdater) ConfigureWGEndpoint(addr *net.UDPAddr, presharedKey *w e.mu.Lock() defer e.mu.Unlock() + if addr != nil { + e.lastEndpoint = addr + } + if e.initiator { e.log.Debugf("configure up WireGuard as initiator") return e.configureAsInitiator(addr, presharedKey) @@ -51,12 +58,36 @@ func (e *EndpointUpdater) SwitchWGEndpoint(addr *net.UDPAddr, presharedKey *wgty e.mu.Lock() defer e.mu.Unlock() + if addr != nil { + e.lastEndpoint = addr + } + // prevent to run new update while cancel the previous update e.waitForCloseTheDelayedUpdate() return e.updateWireGuardPeer(addr, presharedKey) } +// ForceRehandshake removes and re-adds the peer so WireGuard drops the current session +// and negotiates a fresh one with the given PSK. Used when a post-quantum PSK is +// bootstrapped after the session already came up on a pre-PQ key. No-op if no endpoint +// has been applied yet (the pending config will pull the PSK itself). +func (e *EndpointUpdater) ForceRehandshake(presharedKey *wgtypes.Key) error { + e.mu.Lock() + defer e.mu.Unlock() + + if e.lastEndpoint == nil { + return nil + } + // Cancel any pending delayed responder update: it carries the stale pre-PQ key and + // would otherwise re-poison the session after we reset it. + e.waitForCloseTheDelayedUpdate() + if err := e.wgConfig.WgInterface.RemovePeer(e.wgConfig.RemoteKey); err != nil { + return err + } + return e.updateWireGuardPeer(e.lastEndpoint, presharedKey) +} + func (e *EndpointUpdater) RemoveWgPeer() error { e.mu.Lock() defer e.mu.Unlock() diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index 3302e2e66f4..45344f413b9 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -9,7 +9,13 @@ type CallbackHandler interface { // and must be programmed into the consumer's secure channel. It is invoked at // the commit point of each side: the initiator on receiving the answer, the // responder on receiving the confirm. - OnNewPSKReady(remoteID RemoteID, psk PSK) error + // + // bootstrap is true when the PSK came from a signalling exchange (initial setup or + // a wake re-negotiation) rather than a data-path rotation. On bootstrap the consumer + // must ensure the live channel actually adopts the PSK now (e.g. force a WireGuard + // re-handshake) — the channel may already be up on a pre-PQ key. A rotation + // (bootstrap=false) is adopted at the channel's next natural rekey. + OnNewPSKReady(remoteID RemoteID, psk PSK, bootstrap bool) error // OnRekeyFailed fires when an exchange fails to converge within the allotted // time. The host should tear the peer connection down so it re-establishes, and diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 0d6085f732e..6e6ac71706d 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -67,7 +67,7 @@ func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID Exchang // (that offer riding the data path under the freshly adopted key proves it worked), // then derives the PSK for the new offer, commits it optimistically, and returns the // framed answer. A duplicate offer returns the cached answer without re-deriving. -func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { +func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, viaSignal bool) ([]byte, error) { m.trace("pqkem: offer received", "peer", remoteID, "exchange", idHex(o.ExchangeID), "acks", idHex(o.AckID)) if o.AckID != (ExchangeID{}) { @@ -114,7 +114,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "psk_fp", pskFingerprint(psk)) // Commit optimistically so our data path can rekey to the new PSK. - if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { + if err := m.cbHandler.OnNewPSKReady(remoteID, psk, viaSignal); err != nil { return nil, err } m.trace("pqkem: answer sent", "peer", remoteID, "exchange", idHex(o.ExchangeID)) @@ -125,7 +125,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { // stateAwaitingRekey; the next offer (chained from OnDataPathRekeyed) will acknowledge // this exchange. Only valid in stateAwaitingAnswer; advancing the state under the // lock makes a concurrent/duplicate answer bail. -func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { +func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, viaSignal bool) error { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { @@ -159,7 +159,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "psk_fp", pskFingerprint(psk)) - return m.cbHandler.OnNewPSKReady(remoteID, psk) + return m.cbHandler.OnNewPSKReady(remoteID, psk, viaSignal) } // ackConverged (responder) records convergence of the exchange named by ackID: a diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index fff266f380e..f87bbfad195 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -305,7 +305,7 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) if typ != MsgOffer { return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) } - return m.processOffer(remoteID, msg.(*OfferMsg)) + return m.processOffer(remoteID, msg.(*OfferMsg), true) } // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. @@ -318,7 +318,7 @@ func (m *Manager) SignalOnAnswer(remoteID RemoteID, answer []byte) error { if typ != MsgAnswer { return fmt.Errorf("expected answer from %s, got type %d", remoteID, typ) } - return m.processAnswer(remoteID, msg.(*AnswerMsg)) + return m.processAnswer(remoteID, msg.(*AnswerMsg), true) } // ---- Data path ---- @@ -346,7 +346,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } switch typ { case MsgOffer: - answer, err := m.processOffer(remoteID, msg.(*OfferMsg)) + answer, err := m.processOffer(remoteID, msg.(*OfferMsg), false) if err != nil { return err } @@ -355,7 +355,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } return m.pushDataPath(remoteID, answer) case MsgAnswer: - return m.processAnswer(remoteID, msg.(*AnswerMsg)) + return m.processAnswer(remoteID, msg.(*AnswerMsg), false) default: return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index eab264478a7..26c32406dd1 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -65,7 +65,7 @@ type fakeWG struct { func newFakeWG() *fakeWG { return &fakeWG{psks: map[RemoteID]PSK{}} } -func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK) error { +func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK, bootstrap bool) error { f.mu.Lock() defer f.mu.Unlock() f.psks[remoteID] = psk diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 82d2981de25..5d57a197931 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -23,16 +23,30 @@ type pqCallbackHandler struct { // reoffer re-bootstraps the KEM over Signal for a peer (a fresh signalling offer) // to recover from a persistent data-path rekey failure. Nil disables recovery. reoffer func(remoteKey string) + // rehandshake forces the peer's WireGuard session to adopt a freshly bootstrapped + // PSK (the live session may be up on a pre-PQ key). Nil disables it. + rehandshake func(remoteKey string) } // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op // if the peer is not present, mirroring Rosenpass). -func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) error { +func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK, bootstrap bool) error { // updateOnly: applies to an already-configured peer (rotation). At bootstrap the // peer is not configured yet, so this is a no-op there and the PSK is instead // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). log.Tracef("pqkem: programming PSK for peer %s", remoteID) - return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) + if err := h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true); err != nil { + return err + } + // A bootstrap PSK (initial setup or wake re-negotiation) may arrive after the + // WireGuard session already came up on a pre-PQ key (a race between the KEM + // exchange and the endpoint config). SetPresharedKey only updates config, not the + // live session, so force a re-handshake to actually adopt the PSK now. Rotations + // are left to WireGuard's next natural rekey. + if bootstrap && h.rehandshake != nil { + h.rehandshake(string(remoteID)) + } + return nil } // OnRekeyFailed reports a failed PQ (re)key convergence and re-bootstraps the KEM over diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index dec3b5943ce..7f68cc1f98c 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -10,8 +10,8 @@ import ( type pqNoopHandler struct{} -func (pqNoopHandler) OnNewPSKReady(pqkem.RemoteID, pqkem.PSK) error { return nil } -func (pqNoopHandler) OnRekeyFailed(pqkem.RemoteID) error { return nil } +func (pqNoopHandler) OnNewPSKReady(pqkem.RemoteID, pqkem.PSK, bool) error { return nil } +func (pqNoopHandler) OnRekeyFailed(pqkem.RemoteID) error { return nil } // TestPQAdapter_CapabilityRoleAware locks the role-aware capability signal: the KEM // payload only flows initiator-offer -> responder-answer, so an empty message in the From 3ed4b4fb4378b813ed16d260f3f41755010aebea Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 6 Aug 2026 15:15:43 +0200 Subject: [PATCH 055/119] Revert "Introduces a forced WG handshake on initial MLKEM bootstrap." This reverts commit b5a72eca65207cbbc0faba73bc067cecdcc30e44. --- client/internal/engine.go | 8 ------- client/internal/peer/conn.go | 29 ----------------------- client/internal/peer/endpoint.go | 33 +-------------------------- client/internal/pqkem/callbacks.go | 8 +------ client/internal/pqkem/convergence.go | 8 +++---- client/internal/pqkem/manager.go | 8 +++---- client/internal/pqkem/manager_test.go | 2 +- client/internal/pqkem_adapter.go | 18 ++------------- client/internal/pqkem_adapter_test.go | 4 ++-- 9 files changed, 15 insertions(+), 103 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 550e0036354..6b4af1196d1 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -703,14 +703,6 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) conn.RequestReoffer() } }, - // On a freshly bootstrapped PSK, force the peer's WireGuard session to - // re-handshake so it adopts the PSK even if the tunnel already came up on - // a pre-PQ key (KEM-vs-endpoint-config race). - rehandshake: func(remoteKey string) { - if conn, ok := e.peerStore.PeerConn(remoteKey); ok { - conn.ForcePQRehandshake() - } - }, } e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger()) e.pqkemManager.Start(tr) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b37fabd0d12..ba058282b96 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -743,35 +743,6 @@ func (conn *Conn) RequestReoffer() { } } -// ForcePQRehandshake makes WireGuard adopt a freshly bootstrapped post-quantum PSK on -// an already-up session. The KEM exchange can complete after the WG endpoint was -// configured (a race between the KEM and the relay/ICE connection coming up), so the -// live session may be keyed with a pre-PQ key (the strict sentinel, or the ordinary -// key in permissive mode); SetPresharedKey only updates config, not the running -// session. Removing and re-adding the peer forces a fresh handshake that uses the real -// PSK. No-op if not connected yet (the upcoming endpoint config will pull the PSK) or -// if no PSK exists (a non-PQ peer stays fail-closed). -func (conn *Conn) ForcePQRehandshake() { - conn.mu.Lock() - defer conn.mu.Unlock() - - if conn.ctx.Err() != nil || conn.config.PQ == nil { - return - } - psk, ok := conn.config.PQ.PSK(conn.config.Key) - if !ok { - return - } - if conn.currentConnPriority == conntype.None { - return - } - conn.Log.Debugf("pqkem: bootstrap PSK ready, forcing WireGuard re-handshake to adopt it") - wgPsk := wgtypes.Key(psk) - if err := conn.endpointUpdater.ForceRehandshake(&wgPsk); err != nil { - conn.Log.Warnf("pqkem: force re-handshake failed: %v", err) - } -} - func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { conn.mu.Lock() defer conn.mu.Unlock() diff --git a/client/internal/peer/endpoint.go b/client/internal/peer/endpoint.go index 0d741f74b61..9ba1efb6eb3 100644 --- a/client/internal/peer/endpoint.go +++ b/client/internal/peer/endpoint.go @@ -20,13 +20,10 @@ type EndpointUpdater struct { wgConfig WgConfig initiator bool - // mu protects cancelFunc and lastEndpoint + // mu protects cancelFunc mu sync.Mutex cancelFunc func() updateWg sync.WaitGroup - // lastEndpoint is the most recent non-nil endpoint applied to the peer, used to - // re-add it on a forced re-handshake (ForceRehandshake). - lastEndpoint *net.UDPAddr } func NewEndpointUpdater(log *logrus.Entry, wgConfig WgConfig, initiator bool) *EndpointUpdater { @@ -41,10 +38,6 @@ func (e *EndpointUpdater) ConfigureWGEndpoint(addr *net.UDPAddr, presharedKey *w e.mu.Lock() defer e.mu.Unlock() - if addr != nil { - e.lastEndpoint = addr - } - if e.initiator { e.log.Debugf("configure up WireGuard as initiator") return e.configureAsInitiator(addr, presharedKey) @@ -58,36 +51,12 @@ func (e *EndpointUpdater) SwitchWGEndpoint(addr *net.UDPAddr, presharedKey *wgty e.mu.Lock() defer e.mu.Unlock() - if addr != nil { - e.lastEndpoint = addr - } - // prevent to run new update while cancel the previous update e.waitForCloseTheDelayedUpdate() return e.updateWireGuardPeer(addr, presharedKey) } -// ForceRehandshake removes and re-adds the peer so WireGuard drops the current session -// and negotiates a fresh one with the given PSK. Used when a post-quantum PSK is -// bootstrapped after the session already came up on a pre-PQ key. No-op if no endpoint -// has been applied yet (the pending config will pull the PSK itself). -func (e *EndpointUpdater) ForceRehandshake(presharedKey *wgtypes.Key) error { - e.mu.Lock() - defer e.mu.Unlock() - - if e.lastEndpoint == nil { - return nil - } - // Cancel any pending delayed responder update: it carries the stale pre-PQ key and - // would otherwise re-poison the session after we reset it. - e.waitForCloseTheDelayedUpdate() - if err := e.wgConfig.WgInterface.RemovePeer(e.wgConfig.RemoteKey); err != nil { - return err - } - return e.updateWireGuardPeer(e.lastEndpoint, presharedKey) -} - func (e *EndpointUpdater) RemoveWgPeer() error { e.mu.Lock() defer e.mu.Unlock() diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index 45344f413b9..3302e2e66f4 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -9,13 +9,7 @@ type CallbackHandler interface { // and must be programmed into the consumer's secure channel. It is invoked at // the commit point of each side: the initiator on receiving the answer, the // responder on receiving the confirm. - // - // bootstrap is true when the PSK came from a signalling exchange (initial setup or - // a wake re-negotiation) rather than a data-path rotation. On bootstrap the consumer - // must ensure the live channel actually adopts the PSK now (e.g. force a WireGuard - // re-handshake) — the channel may already be up on a pre-PQ key. A rotation - // (bootstrap=false) is adopted at the channel's next natural rekey. - OnNewPSKReady(remoteID RemoteID, psk PSK, bootstrap bool) error + OnNewPSKReady(remoteID RemoteID, psk PSK) error // OnRekeyFailed fires when an exchange fails to converge within the allotted // time. The host should tear the peer connection down so it re-establishes, and diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 6e6ac71706d..0d6085f732e 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -67,7 +67,7 @@ func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID Exchang // (that offer riding the data path under the freshly adopted key proves it worked), // then derives the PSK for the new offer, commits it optimistically, and returns the // framed answer. A duplicate offer returns the cached answer without re-deriving. -func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, viaSignal bool) ([]byte, error) { +func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { m.trace("pqkem: offer received", "peer", remoteID, "exchange", idHex(o.ExchangeID), "acks", idHex(o.AckID)) if o.AckID != (ExchangeID{}) { @@ -114,7 +114,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, viaSignal bool) ( m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "psk_fp", pskFingerprint(psk)) // Commit optimistically so our data path can rekey to the new PSK. - if err := m.cbHandler.OnNewPSKReady(remoteID, psk, viaSignal); err != nil { + if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { return nil, err } m.trace("pqkem: answer sent", "peer", remoteID, "exchange", idHex(o.ExchangeID)) @@ -125,7 +125,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, viaSignal bool) ( // stateAwaitingRekey; the next offer (chained from OnDataPathRekeyed) will acknowledge // this exchange. Only valid in stateAwaitingAnswer; advancing the state under the // lock makes a concurrent/duplicate answer bail. -func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, viaSignal bool) error { +func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { @@ -159,7 +159,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, viaSignal bool) m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "psk_fp", pskFingerprint(psk)) - return m.cbHandler.OnNewPSKReady(remoteID, psk, viaSignal) + return m.cbHandler.OnNewPSKReady(remoteID, psk) } // ackConverged (responder) records convergence of the exchange named by ackID: a diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index f87bbfad195..fff266f380e 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -305,7 +305,7 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) if typ != MsgOffer { return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) } - return m.processOffer(remoteID, msg.(*OfferMsg), true) + return m.processOffer(remoteID, msg.(*OfferMsg)) } // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. @@ -318,7 +318,7 @@ func (m *Manager) SignalOnAnswer(remoteID RemoteID, answer []byte) error { if typ != MsgAnswer { return fmt.Errorf("expected answer from %s, got type %d", remoteID, typ) } - return m.processAnswer(remoteID, msg.(*AnswerMsg), true) + return m.processAnswer(remoteID, msg.(*AnswerMsg)) } // ---- Data path ---- @@ -346,7 +346,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } switch typ { case MsgOffer: - answer, err := m.processOffer(remoteID, msg.(*OfferMsg), false) + answer, err := m.processOffer(remoteID, msg.(*OfferMsg)) if err != nil { return err } @@ -355,7 +355,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } return m.pushDataPath(remoteID, answer) case MsgAnswer: - return m.processAnswer(remoteID, msg.(*AnswerMsg), false) + return m.processAnswer(remoteID, msg.(*AnswerMsg)) default: return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index 26c32406dd1..eab264478a7 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -65,7 +65,7 @@ type fakeWG struct { func newFakeWG() *fakeWG { return &fakeWG{psks: map[RemoteID]PSK{}} } -func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK, bootstrap bool) error { +func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK) error { f.mu.Lock() defer f.mu.Unlock() f.psks[remoteID] = psk diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 5d57a197931..82d2981de25 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -23,30 +23,16 @@ type pqCallbackHandler struct { // reoffer re-bootstraps the KEM over Signal for a peer (a fresh signalling offer) // to recover from a persistent data-path rekey failure. Nil disables recovery. reoffer func(remoteKey string) - // rehandshake forces the peer's WireGuard session to adopt a freshly bootstrapped - // PSK (the live session may be up on a pre-PQ key). Nil disables it. - rehandshake func(remoteKey string) } // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op // if the peer is not present, mirroring Rosenpass). -func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK, bootstrap bool) error { +func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) error { // updateOnly: applies to an already-configured peer (rotation). At bootstrap the // peer is not configured yet, so this is a no-op there and the PSK is instead // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). log.Tracef("pqkem: programming PSK for peer %s", remoteID) - if err := h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true); err != nil { - return err - } - // A bootstrap PSK (initial setup or wake re-negotiation) may arrive after the - // WireGuard session already came up on a pre-PQ key (a race between the KEM - // exchange and the endpoint config). SetPresharedKey only updates config, not the - // live session, so force a re-handshake to actually adopt the PSK now. Rotations - // are left to WireGuard's next natural rekey. - if bootstrap && h.rehandshake != nil { - h.rehandshake(string(remoteID)) - } - return nil + return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) } // OnRekeyFailed reports a failed PQ (re)key convergence and re-bootstraps the KEM over diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index 7f68cc1f98c..dec3b5943ce 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -10,8 +10,8 @@ import ( type pqNoopHandler struct{} -func (pqNoopHandler) OnNewPSKReady(pqkem.RemoteID, pqkem.PSK, bool) error { return nil } -func (pqNoopHandler) OnRekeyFailed(pqkem.RemoteID) error { return nil } +func (pqNoopHandler) OnNewPSKReady(pqkem.RemoteID, pqkem.PSK) error { return nil } +func (pqNoopHandler) OnRekeyFailed(pqkem.RemoteID) error { return nil } // TestPQAdapter_CapabilityRoleAware locks the role-aware capability signal: the KEM // payload only flows initiator-offer -> responder-answer, so an empty message in the From 4ffa2c724ff25a0be82966288c144280e30a3679 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 6 Aug 2026 15:42:27 +0200 Subject: [PATCH 056/119] Introduces a forced imparity on MLKEM bootstrap. To ensure two peers agree on a key, we need asymmetry. one peer is the controller ("initiator") the other is the "responder". Otherwise imagine two offers in parallel driving two answers at the same time A B | <----B-OFFER----- | | -----A-OFFER----> | | | | | --------------------------------- |****** ICE + WG Handshake ****** | --------------------------------- | | | <----B-ANSWER---- | | -----A-ANSWER---> | PSK is derived on receive of offer, so A and B derive different PSKs. When WG handshake takes place it picks misaligned PSKs. So we impair the two nodes and only the offer of one of the two (the controller/initiator) is allowed to progress and drive the answer (and carry the KEM material). If a responder initiates an offer, we redo the offer towards it. This is oK since the ICEworker don't treat offer/answers differently. --- client/internal/peer/handshaker.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index a5035da07ef..c465f6712be 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -137,6 +137,24 @@ func (h *Handshaker) Listen(ctx context.Context) { h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) + // Post-quantum: the KEM material rides only the controller's offer, so the two + // peers derive a single shared PSK (a bidirectional KEM would yield two + // different PSKs and WireGuard would pick misaligned ones). If we are the + // controller and receive the responder's (KEM-less) offer, we must NOT let + // that KEM-less transaction drive the connection: it would bring WireGuard up + // on a pre-PQ key before the KEM completes (the race). Instead we reply with + // OUR offer, which carries the KEM, so the only transaction that establishes + // the tunnel is the one that also derives the PSK. This also guarantees a + // responder-initiated wake still triggers a KEM offer (no stuck responder). + // The re-offer reuses our stable ICE session id, so the peer dedups repeats. + if h.config.PQ != nil && isController(h.config) { + h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") + if err := h.sendOffer(); err != nil { + h.log.Errorf("failed to send KEM offer in response to peer offer: %s", err) + } + continue + } + // Derive+store the KEM PSK (inside sendAnswer's AnswerPayload) BEFORE bringing // up the connection: the relay/ICE workers configure the WG endpoint, which // pulls the PSK for the first handshake. Notifying them first would race the From 1870b104ba0d2295ac1d65a5f69e7eab82c8771e Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 6 Aug 2026 18:04:07 +0200 Subject: [PATCH 057/119] [client] pqkem: make signalling bootstrap idempotent through awaitingRekey The controller sends its KEM offer both on its own guard event and in reply to the responder's offer. SignalOffer was idempotent only while awaiting the answer; once the answer arrived (awaitingRekey) a repeat call started a fresh exchange with a different PSK, desyncing the two peers (one on the old PSK, one on the new) so WireGuard derived misaligned transport keys and dropped all data. Treat awaitingRekey as in-flight too and return the same offer. --- client/internal/pqkem/manager.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index fff266f380e..8533833a423 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -285,7 +285,13 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { m.mu.Unlock() return nil, nil // peer does not run the KEM; do not offer (avoids a failure/reoffer loop) } - if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && ex.state == stateAwaitingAnswer { + // Idempotent while a signalling bootstrap is in flight OR already derived a PSK but + // not yet chained a rotation (awaitingRekey): return the SAME offer instead of + // starting a new exchange. This matters when the controller both offers on its own + // guard AND re-offers in response to the responder's offer — without this, the + // second call would start a fresh exchange (a different PSK) and desync the peers. + if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && + (ex.state == stateAwaitingAnswer || ex.state == stateAwaitingRekey) { last := ex.lastSent m.mu.Unlock() return last, nil From 3b61222b07c6a046eb6ea9dffa54253ba1c9a4ea Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 6 Aug 2026 18:04:30 +0200 Subject: [PATCH 058/119] [client] pqkem: gate controller re-offer to kick the KEM exactly once When the controller receives the responder's (KEM-less) offer it replies with its own KEM offer instead of answering, so the only transaction that brings the tunnel up is the one that also carries the PSK. Guard that reply with ShouldSendBootstrapOffer so it fires only when no exchange is in flight: without it, every responder offer triggered another offer (an offer-per-offer runaway). The whole behaviour is isolated to the KEM path (config.PQ != nil); non-PQ connections answer as before. --- client/internal/peer/conn.go | 4 ++++ client/internal/peer/conn_pq_test.go | 1 + client/internal/peer/handshaker.go | 13 ++++++++++--- client/internal/pqkem/manager.go | 18 ++++++++++++++++++ client/internal/pqkem_adapter.go | 4 ++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index ba058282b96..06fb8538261 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -85,6 +85,10 @@ type PQHandshaker interface { // OfferPayload returns the KEM offer to embed in an outgoing offer (nil if this // peer is not the KEM initiator) and the local PQ data-path port to announce. OfferPayload(remoteKey string) (payload []byte, port int) + // ShouldSendBootstrapOffer reports whether, as the controller, we should reply to a + // received responder offer with our own KEM offer (true only when no exchange is + // already in flight — so we kick the KEM once and ignore further offers). + ShouldSendBootstrapOffer(remoteKey string) bool // AnswerPayload processes a received KEM offer (nil if absent) and returns the KEM // answer to embed in the outgoing answer (nil if none) and the local PQ port. AnswerPayload(remoteKey string, recvOffer []byte) (payload []byte, port int) diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index 11d81b36e71..ffc6e8c32a1 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -18,6 +18,7 @@ type fakePQ struct { } func (f fakePQ) OfferPayload(string) ([]byte, int) { return nil, 0 } +func (f fakePQ) ShouldSendBootstrapOffer(string) bool { return false } func (f fakePQ) AnswerPayload(string, []byte) ([]byte, int) { return nil, 0 } func (f fakePQ) OnAnswer(string, []byte) {} func (f fakePQ) PSK(string) (wgtypes.Key, bool) { return f.psk, f.ok } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index c465f6712be..0d2e4dc12db 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -148,9 +148,16 @@ func (h *Handshaker) Listen(ctx context.Context) { // responder-initiated wake still triggers a KEM offer (no stuck responder). // The re-offer reuses our stable ICE session id, so the peer dedups repeats. if h.config.PQ != nil && isController(h.config) { - h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") - if err := h.sendOffer(); err != nil { - h.log.Errorf("failed to send KEM offer in response to peer offer: %s", err) + // Reply with our KEM offer exactly once, to kick the exchange. If one is + // already in flight, ignore this offer — re-sending on every responder + // offer would be an offer-per-offer runaway. + if h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { + h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") + if err := h.sendOffer(); err != nil { + h.log.Errorf("failed to send KEM offer in response to peer offer: %s", err) + } + } else { + h.log.Debugf("pqkem: controller received a responder offer but a KEM exchange is already in flight, ignoring") } continue } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 8533833a423..53fd564c443 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -301,6 +301,24 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { return m.startExchange(remoteID, true, ExchangeID{}) } +// ShouldSendBootstrapOffer reports whether we should emit a fresh KEM offer to kick a +// bootstrap for this peer. True only if we are the initiator, the peer is not known +// non-capable, and no exchange is already in flight. The host uses this when it (as the +// controller) receives the responder's offer: it replies with a KEM offer exactly once +// to start the exchange, and ignores further responder offers while one is in flight, +// avoiding an offer-per-offer runaway. +func (m *Manager) ShouldSendBootstrapOffer(remoteID RemoteID) bool { + if !m.IsInitiator(remoteID) { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + if capable, ok := m.capable[remoteID]; ok && !capable { + return false + } + return m.exchanges[remoteID] == nil +} + // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and // returns the KEM answer for the host to embed in its outgoing answer. func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) { diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 82d2981de25..be5a99c1ba4 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -72,6 +72,10 @@ func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, int) { return payload, p.announcedPort() } +func (p pqHandshaker) ShouldSendBootstrapOffer(remoteKey string) bool { + return p.mgr.ShouldSendBootstrapOffer(pqkem.RemoteID(remoteKey)) +} + func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, int) { if len(recvOffer) == 0 { // Capability signal (responder side): the KEM offer flows initiator->responder, From 4c6735e8aec6246ded7be696d54a73551b39fb0a Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 13:09:06 +0200 Subject: [PATCH 059/119] [client] pqkem: don't let the responder's delayed update revert the PSK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The responder configures WireGuard with endpoint=nil first, then a delayed update (scheduleDelayedUpdate) applies the real endpoint after fallbackDelay. It captured the preshared key at schedule time and re-applied it. With the post-quantum exchange the PSK can change within that window (a fresher PSK derived and applied via SetPresharedKey), so re-applying the captured one reverted WireGuard to a key the remote peer no longer used — a mismatch that stalled the handshake until the WGWatcher timeout forced a retry (~30s). Pass a nil PSK in the delayed update so it only sets the endpoint and leaves the current PSK in place; the latest SetPresharedKey wins. --- client/internal/peer/endpoint.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/client/internal/peer/endpoint.go b/client/internal/peer/endpoint.go index 9ba1efb6eb3..c17e59ff04e 100644 --- a/client/internal/peer/endpoint.go +++ b/client/internal/peer/endpoint.go @@ -88,7 +88,7 @@ func (e *EndpointUpdater) configureAsResponder(addr *net.UDPAddr, presharedKey * var ctx context.Context ctx, e.cancelFunc = context.WithCancel(context.Background()) e.updateWg.Add(1) - go e.scheduleDelayedUpdate(ctx, addr, presharedKey) + go e.scheduleDelayedUpdate(ctx, addr) if err := e.updateWireGuardPeer(nil, presharedKey); err != nil { e.waitForCloseTheDelayedUpdate() @@ -107,8 +107,14 @@ func (e *EndpointUpdater) waitForCloseTheDelayedUpdate() { e.updateWg.Wait() } -// scheduleDelayedUpdate waits for the fallback period before updating the endpoint -func (e *EndpointUpdater) scheduleDelayedUpdate(ctx context.Context, addr *net.UDPAddr, presharedKey *wgtypes.Key) { +// scheduleDelayedUpdate waits for the fallback period, then sets the responder's real +// endpoint. It deliberately passes a nil preshared key so it only updates the endpoint +// and leaves the current PSK untouched: the PSK captured when this was scheduled may be +// stale by now (e.g. the post-quantum bootstrap derived a fresher PSK within the +// fallback window, applied via SetPresharedKey), and re-applying the captured one would +// revert WireGuard to a key the remote peer no longer uses — a mismatch that stalls the +// handshake until the next retry. +func (e *EndpointUpdater) scheduleDelayedUpdate(ctx context.Context, addr *net.UDPAddr) { defer e.updateWg.Done() t := time.NewTimer(fallbackDelay) defer t.Stop() @@ -117,7 +123,7 @@ func (e *EndpointUpdater) scheduleDelayedUpdate(ctx context.Context, addr *net.U case <-ctx.Done(): return case <-t.C: - if err := e.updateWireGuardPeer(addr, presharedKey); err != nil { + if err := e.updateWireGuardPeer(addr, nil); err != nil { e.log.Errorf("failed to update WireGuard peer, address: %s, error: %v", addr, err) } } From 82699afb0fd41bdf5b3b5a5a5b7d773dd905939e Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 16:02:55 +0200 Subject: [PATCH 060/119] [client] pqkem: split handshaker Listen into per-message handlers Reduce Listen's cognitive complexity (SonarCloud S3776, 30 -> under 25) by extracting the offer and answer cases into handleRemoteOffer/handleRemoteAnswer, with shared onSignalReceived/notifyListeners helpers and a pqControllerReoffer helper for the controller re-offer branch. No functional change. --- client/internal/peer/handshaker.go | 154 +++++++++++++++-------------- 1 file changed, 81 insertions(+), 73 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 0d2e4dc12db..4caf2327be5 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -126,89 +126,97 @@ func (h *Handshaker) Listen(ctx context.Context) { for { select { case remoteOfferAnswer := <-h.remoteOffersCh: - h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) - - // Record signaling received for reconnection attempts - if h.metricsStages != nil { - h.metricsStages.RecordSignalingReceived() - } - - h.updateRemoteICEState(&remoteOfferAnswer) - - h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) - - // Post-quantum: the KEM material rides only the controller's offer, so the two - // peers derive a single shared PSK (a bidirectional KEM would yield two - // different PSKs and WireGuard would pick misaligned ones). If we are the - // controller and receive the responder's (KEM-less) offer, we must NOT let - // that KEM-less transaction drive the connection: it would bring WireGuard up - // on a pre-PQ key before the KEM completes (the race). Instead we reply with - // OUR offer, which carries the KEM, so the only transaction that establishes - // the tunnel is the one that also derives the PSK. This also guarantees a - // responder-initiated wake still triggers a KEM offer (no stuck responder). - // The re-offer reuses our stable ICE session id, so the peer dedups repeats. - if h.config.PQ != nil && isController(h.config) { - // Reply with our KEM offer exactly once, to kick the exchange. If one is - // already in flight, ignore this offer — re-sending on every responder - // offer would be an offer-per-offer runaway. - if h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { - h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") - if err := h.sendOffer(); err != nil { - h.log.Errorf("failed to send KEM offer in response to peer offer: %s", err) - } - } else { - h.log.Debugf("pqkem: controller received a responder offer but a KEM exchange is already in flight, ignoring") - } - continue - } - - // Derive+store the KEM PSK (inside sendAnswer's AnswerPayload) BEFORE bringing - // up the connection: the relay/ICE workers configure the WG endpoint, which - // pulls the PSK for the first handshake. Notifying them first would race the - // KEM exchange and hand the first handshake a not-yet-derived key. - if err := h.sendAnswer(&remoteOfferAnswer); err != nil { - h.log.Errorf("failed to send remote offer confirmation: %s", err) - continue - } + h.handleRemoteOffer(remoteOfferAnswer) + case remoteOfferAnswer := <-h.remoteAnswerCh: + h.handleRemoteAnswer(remoteOfferAnswer) + case <-ctx.Done(): + h.log.Infof("stop listening for remote offers and answers") + return + } + } +} - if h.relayListener != nil { - h.relayListener.Notify(&remoteOfferAnswer) - } +// onSignalReceived runs the common preamble for a received offer/answer: record the +// signalling metric, refresh the remote ICE state, and register the peer's post-quantum +// data-path endpoint learned from the message. +func (h *Handshaker) onSignalReceived(remoteOfferAnswer *OfferAnswer) { + if h.metricsStages != nil { + h.metricsStages.RecordSignalingReceived() + } + h.updateRemoteICEState(remoteOfferAnswer) + h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) +} - if h.iceListener != nil && h.RemoteICESupported() { - h.iceListener(&remoteOfferAnswer) - } - case remoteOfferAnswer := <-h.remoteAnswerCh: - h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) +// notifyListeners hands the offer/answer to the relay and ICE workers so they bring the +// connection up. +func (h *Handshaker) notifyListeners(remoteOfferAnswer *OfferAnswer) { + if h.relayListener != nil { + h.relayListener.Notify(remoteOfferAnswer) + } + if h.iceListener != nil && h.RemoteICESupported() { + h.iceListener(remoteOfferAnswer) + } +} - // Record signaling received for reconnection attempts - if h.metricsStages != nil { - h.metricsStages.RecordSignalingReceived() - } +func (h *Handshaker) handleRemoteOffer(remoteOfferAnswer OfferAnswer) { + h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) + h.onSignalReceived(&remoteOfferAnswer) - h.updateRemoteICEState(&remoteOfferAnswer) + // If we are the controller running the KEM, a responder's offer is handled by + // replying with our own KEM offer, not by answering it (see pqControllerReoffer). + if h.pqControllerReoffer() { + return + } - h.pqRegisterEndpoint(remoteOfferAnswer.MlkemPort) + // Derive+store the KEM PSK (inside sendAnswer's AnswerPayload) BEFORE bringing up the + // connection: the relay/ICE workers configure the WG endpoint, which pulls the PSK + // for the first handshake. Notifying them first would race the KEM exchange and hand + // the first handshake a not-yet-derived key. + if err := h.sendAnswer(&remoteOfferAnswer); err != nil { + h.log.Errorf("failed to send remote offer confirmation: %s", err) + return + } + h.notifyListeners(&remoteOfferAnswer) +} - // Feed the KEM answer (derive+store PSK) BEFORE bringing up the connection so - // the WG endpoint config pulls the real PSK for the first handshake instead of - // racing ahead of the KEM exchange. - if h.config.PQ != nil { - h.config.PQ.OnAnswer(h.config.Key, remoteOfferAnswer.MlkemPayload) - } +func (h *Handshaker) handleRemoteAnswer(remoteOfferAnswer OfferAnswer) { + h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) + h.onSignalReceived(&remoteOfferAnswer) - if h.relayListener != nil { - h.relayListener.Notify(&remoteOfferAnswer) - } + // Feed the KEM answer (derive+store PSK) BEFORE bringing up the connection so the WG + // endpoint config pulls the real PSK for the first handshake instead of racing ahead + // of the KEM exchange. + if h.config.PQ != nil { + h.config.PQ.OnAnswer(h.config.Key, remoteOfferAnswer.MlkemPayload) + } + h.notifyListeners(&remoteOfferAnswer) +} - if h.iceListener != nil && h.RemoteICESupported() { - h.iceListener(&remoteOfferAnswer) - } - case <-ctx.Done(): - h.log.Infof("stop listening for remote offers and answers") - return +// pqControllerReoffer handles a responder's offer when we are the controller running the +// KEM. The KEM material rides only the controller's offer, so the two peers derive a +// single shared PSK (a bidirectional KEM would yield two different PSKs and WireGuard +// would pick misaligned ones). Rather than answer the responder's (KEM-less) offer — +// which would bring WireGuard up on a pre-PQ key before the KEM completes — we reply with +// our own KEM offer, so the only transaction that establishes the tunnel is the one that +// also derives the PSK. It also guarantees a responder-initiated wake still triggers a +// KEM offer (no stuck responder). Sent exactly once per exchange; further offers while +// one is in flight are ignored (re-sending on every responder offer would be a runaway). +// The re-offer reuses our stable ICE session id, so the peer dedups repeats. +// +// Returns true when it took ownership of the offer (the caller must not answer it). +func (h *Handshaker) pqControllerReoffer() bool { + if h.config.PQ == nil || !isController(h.config) { + return false + } + if h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { + h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") + if err := h.sendOffer(); err != nil { + h.log.Errorf("failed to send KEM offer in response to peer offer: %s", err) } + } else { + h.log.Debugf("pqkem: controller received a responder offer but a KEM exchange is already in flight, ignoring") } + return true } // pqRegisterEndpoint feeds the post-quantum handshaker the peer's data-path endpoint From 1dc48bcc06d4e964009afa02e189fe9dc705ac86 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 16:08:52 +0200 Subject: [PATCH 061/119] Addresses CI fixes --- client/internal/pqkem/convergence.go | 24 +++++++++++++++++++++--- client/internal/pqkem/kem.go | 5 +++++ client/internal/pqkem/message.go | 2 +- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 0d6085f732e..e504cbfbd59 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "fmt" "time" ) @@ -22,7 +23,14 @@ func pskFingerprint(psk PSK) string { // bootstrap) and returns the framed offer for the caller to send — pushed over the // data path for a chained rekey, or handed to the host for signalling when viaSignal // is set. Any previous in-flight exchange for the peer is cancelled. -func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID ExchangeID) ([]byte, error) { +// startExchangeLocked must be called with m.mu held: the caller's idempotency check and +// the exchange install stay under one lock acquisition so two concurrent starts for the +// same peer cannot both create an exchange. It also refuses to start (and to Add to the +// wait group) once the manager is stopping, so it never races Manager.Stop's Wait. +func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID ExchangeID) ([]byte, error) { + if m.stopping { + return nil, fmt.Errorf("manager stopping") + } init, err := NewInitiator() if err != nil { return nil, err @@ -37,7 +45,6 @@ func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID Exchang } ctx, cancel := context.WithCancel(m.rootCtx) - m.mu.Lock() if old := m.exchanges[remoteID]; old != nil && old.cancel != nil { old.cancel() } @@ -50,7 +57,6 @@ func (m *Manager) startExchange(remoteID RemoteID, viaSignal bool, ackID Exchang initiator: init, viaSignal: viaSignal, } - m.mu.Unlock() m.wait.Add(1) go m.initiatorLoop(ctx, remoteID, id) @@ -146,6 +152,18 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { psk, err := init.Finish(a.KEMAnswer, m.binding(remoteID)) if err != nil { + // The state already advanced to stateAwaitingRekey and the initiator was cleared, + // so initiatorLoop would exit its default branch without registering a failure — + // leaving the peer desynced (the responder committed its PSK in processOffer). + // Drop the exchange and raise the failure so recovery re-bootstraps. + m.mu.Lock() + if cur := m.exchanges[remoteID]; cur != nil && cur.id == a.ExchangeID { + delete(m.exchanges, remoteID) + } + initial := !m.established[remoteID] + fail := m.registerFailureLocked(remoteID) + m.mu.Unlock() + m.raiseFailure(remoteID, fail, initial) return err } diff --git a/client/internal/pqkem/kem.go b/client/internal/pqkem/kem.go index cd3c80d094d..90c638a7a90 100644 --- a/client/internal/pqkem/kem.go +++ b/client/internal/pqkem/kem.go @@ -149,6 +149,11 @@ func Respond(offer []byte, b Binding) (answer []byte, psk PSK, err error) { // canonicalised peer identities, so the PSK cannot be transplanted to another peer // pair or a different exchange. func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) (PSK, error) { + // A PSK not bound to both peer identities could be transplanted to a different peer + // pair, so refuse to derive one from an empty binding. + if len(b.LocalID) == 0 || len(b.RemoteID) == 0 { + return PSK{}, fmt.Errorf("empty peer identity binding") + } lo, hi := canonicalPair(b.LocalID, b.RemoteID) ikm := make([]byte, 0, len(ssMLKEM)+len(ssX)) diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go index 312279d80b9..e483e9d72cd 100644 --- a/client/internal/pqkem/message.go +++ b/client/internal/pqkem/message.go @@ -40,7 +40,7 @@ const ( // that acknowledges nothing, i.e. the first exchange of a connection). type ExchangeID [ExchangeIDSize]byte -// OfferMsg carries the initiator's public material (X25519 pub ‖ ML-KEM encap key) +// OfferMsg carries the initiator's public material (ML-KEM encap key ‖ X25519 pub) // and AckID, the id of the previous exchange this offer acknowledges (zero if none). type OfferMsg struct { ExchangeID ExchangeID From 11555c3139beab742c5ad4ac8e89784dcf978c2e Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 16:24:17 +0200 Subject: [PATCH 062/119] Addresses CI fixes --- client/internal/pqkem/bench_test.go | 20 +++++++++++++---- client/internal/pqkem/concurrency_test.go | 2 +- client/internal/pqkem/convergence.go | 2 +- client/internal/pqkem/convergence_test.go | 4 ++-- client/internal/pqkem/kem_test.go | 27 ++++++++++++++++++----- client/internal/pqkem/manager.go | 27 ++++++++++++++--------- client/internal/pqkem/manager_test.go | 8 +++++++ 7 files changed, 67 insertions(+), 23 deletions(-) diff --git a/client/internal/pqkem/bench_test.go b/client/internal/pqkem/bench_test.go index eabcc3a0add..7e500de77c7 100644 --- a/client/internal/pqkem/bench_test.go +++ b/client/internal/pqkem/bench_test.go @@ -19,8 +19,14 @@ func BenchmarkX25519Keygen(b *testing.B) { func BenchmarkX25519ECDH(b *testing.B) { c := ecdh.X25519() - a, _ := c.GenerateKey(rand.Reader) - p, _ := c.GenerateKey(rand.Reader) + a, err := c.GenerateKey(rand.Reader) + if err != nil { + b.Fatal(err) + } + p, err := c.GenerateKey(rand.Reader) + if err != nil { + b.Fatal(err) + } pub := p.PublicKey() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -39,7 +45,10 @@ func BenchmarkMLKEMKeygen(b *testing.B) { } func BenchmarkMLKEMEncaps(b *testing.B) { - dk, _ := mlkem.GenerateKey768() + dk, err := mlkem.GenerateKey768() + if err != nil { + b.Fatal(err) + } ek := dk.EncapsulationKey() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -48,7 +57,10 @@ func BenchmarkMLKEMEncaps(b *testing.B) { } func BenchmarkMLKEMDecaps(b *testing.B) { - dk, _ := mlkem.GenerateKey768() + dk, err := mlkem.GenerateKey768() + if err != nil { + b.Fatal(err) + } _, ct := dk.EncapsulationKey().Encapsulate() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/client/internal/pqkem/concurrency_test.go b/client/internal/pqkem/concurrency_test.go index 0303750a90b..bcf389404db 100644 --- a/client/internal/pqkem/concurrency_test.go +++ b/client/internal/pqkem/concurrency_test.go @@ -32,7 +32,7 @@ func TestConcurrency_RecoversViaResignalAfterDataPathBreak(t *testing.T) { // Data path breaks: the rotation can no longer converge -> OnRekeyFailed. lbB.drop.Store(true) - _, err := dB.startExchange("aaaa", false, ExchangeID{}) + _, err := dB.startExchangeTest("aaaa", false, ExchangeID{}) require.NoError(t, err) require.Eventually(t, func() bool { return failedCount(wgB) >= 1 }, time.Second, 5*time.Millisecond) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index e504cbfbd59..f818590b1d2 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -28,7 +28,7 @@ func pskFingerprint(psk PSK) string { // same peer cannot both create an exchange. It also refuses to start (and to Add to the // wait group) once the manager is stopping, so it never races Manager.Stop's Wait. func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID ExchangeID) ([]byte, error) { - if m.stopping { + if m.rootCtx.Err() != nil { return nil, fmt.Errorf("manager stopping") } init, err := NewInitiator() diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index f4ce251c4a9..fed15d0ea61 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -61,14 +61,14 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { // K-1 data-path rekeys must NOT raise OnRekeyFailed. for i := 0; i < DefaultMaxRekeyFailures-1; i++ { - _, err := dB.startExchange("aaaa", false, ExchangeID{}) + _, err := dB.startExchangeTest("aaaa", false, ExchangeID{}) require.NoError(t, err) time.Sleep(50 * time.Millisecond) } require.Equal(t, 0, failedCount(wgB), "no failure before K attempts") // The K-th failure raises it once. - _, err := dB.startExchange("aaaa", false, ExchangeID{}) + _, err := dB.startExchangeTest("aaaa", false, ExchangeID{}) require.NoError(t, err) require.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) } diff --git a/client/internal/pqkem/kem_test.go b/client/internal/pqkem/kem_test.go index 499c8247009..9a2c9c8f9ae 100644 --- a/client/internal/pqkem/kem_test.go +++ b/client/internal/pqkem/kem_test.go @@ -33,19 +33,36 @@ func TestExchange_PSKBoundToPeerIdentities(t *testing.T) { init, err := NewInitiator() require.NoError(t, err) - // responder computes with the honest pair... - _, pskHonest, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + answer, _, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + // Finish twice over the SAME KEM material (same offer/answer/secrets), changing only + // the peer identity binding: the differing PSK is attributable to the binding alone. + pskHonest, err := init.Finish(answer, Binding{LocalID: wgA, RemoteID: wgB}) require.NoError(t, err) - // ...a second responder run with a different peer identity yields a different PSK, - // even though the KEM material would otherwise combine identically. wgC := []byte("peer-C-wireguard-pubkey-32bytes!") - _, pskWrong, err := Respond(init.Offer(), Binding{LocalID: wgC, RemoteID: wgA}) + pskWrong, err := init.Finish(answer, Binding{LocalID: wgA, RemoteID: wgC}) require.NoError(t, err) require.NotEqual(t, pskHonest, pskWrong, "PSK must be bound to the peer pair") } +func TestExchange_RejectsEmptyBinding(t *testing.T) { + init, err := NewInitiator() + require.NoError(t, err) + answer, _, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) + require.NoError(t, err) + + // A PSK not bound to both identities could be transplanted to another peer pair. + _, err = init.Finish(answer, Binding{}) + require.Error(t, err, "empty binding must be rejected") + _, err = init.Finish(answer, Binding{LocalID: wgA}) + require.Error(t, err, "missing RemoteID must be rejected") + _, _, err = Respond(init.Offer(), Binding{RemoteID: wgA}) + require.Error(t, err, "missing LocalID must be rejected") +} + func TestExchange_RejectsMalformedMessages(t *testing.T) { init, err := NewInitiator() require.NoError(t, err) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 53fd564c443..4e6ce3bbe5d 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -250,7 +250,12 @@ func (m *Manager) RemovePeer(remoteID RemoteID) { // Stop cancels all in-flight exchanges, closes the transport, and waits for the // exchange goroutines to exit. func (m *Manager) Stop() { + // Cancel the root context under the lock, before Wait: startExchangeLocked checks + // rootCtx.Err() under the same lock before it Adds to the wait group, so once Stop + // has cancelled here no new Add can race Wait. + m.mu.Lock() m.rootCancel() + m.mu.Unlock() m.wait.Wait() m.mu.Lock() t := m.transport @@ -296,9 +301,11 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { m.mu.Unlock() return last, nil } + // Hold the lock across the check above and the install so a concurrent SignalOffer + // for the same peer can't also start an exchange. bootstrap offer acks nothing. + raw, err := m.startExchangeLocked(remoteID, true, ExchangeID{}) m.mu.Unlock() - // bootstrap offer acknowledges nothing (zero AckID). - return m.startExchange(remoteID, true, ExchangeID{}) + return raw, err } // ShouldSendBootstrapOffer reports whether we should emit a fresh KEM offer to kick a @@ -402,17 +409,17 @@ func (m *Manager) OnDataPathRekeyed(remoteID RemoteID, sinceActivity time.Durati m.mu.Lock() ex := m.exchanges[remoteID] chain := ex != nil && ex.state == stateAwaitingRekey - var ackID ExchangeID - if chain { - ackID = ex.id - } - m.mu.Unlock() - - m.trace("pqkem: data-path rekey signal", "peer", remoteID, "chaining", chain) if !chain { + m.mu.Unlock() + m.trace("pqkem: data-path rekey signal", "peer", remoteID, "chaining", false) return } - offer, err := m.startExchange(remoteID, false, ackID) + // Hold the lock across the awaitingRekey check and the install so two rekey clocks + // can't each start a chained exchange for the same peer. + offer, err := m.startExchangeLocked(remoteID, false, ex.id) + m.mu.Unlock() + + m.trace("pqkem: data-path rekey signal", "peer", remoteID, "chaining", true) if err != nil { m.logger.Error("pqkem: chain offer failed to start", "peer", remoteID, "err", err) return diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index eab264478a7..b7a13166ad5 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -65,6 +65,14 @@ type fakeWG struct { func newFakeWG() *fakeWG { return &fakeWG{psks: map[RemoteID]PSK{}} } +// startExchangeTest drives startExchangeLocked with the lock held, for tests that kick an +// exchange directly (production callers hold m.mu across their idempotency check). +func (m *Manager) startExchangeTest(remoteID RemoteID, viaSignal bool, ackID ExchangeID) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.startExchangeLocked(remoteID, viaSignal, ackID) +} + func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK) error { f.mu.Lock() defer f.mu.Unlock() From 8a04fa4d3fe232b1eaccaf3503895366d88ddc01 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 16:27:31 +0200 Subject: [PATCH 063/119] Addresses CI fixes --- client/internal/engine.go | 5 +++++ client/internal/peer/conn.go | 10 ++++++---- client/internal/pqkem_transport.go | 12 ++++++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 6b4af1196d1..c29c1bba11e 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -691,6 +691,11 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) if pqkem.Enabled() { tr, pqErr := newPQTransport(e.config.WgAddr.IP) if pqErr != nil { + // In strict mode the peer must fail closed; silently continuing without the PQ + // exchange would hand out classic tunnels, so treat the bind failure as fatal. + if pqkem.Strict() { + return fmt.Errorf("pqkem: strict mode enabled but transport bind failed: %w", pqErr) + } log.Errorf("pqkem: transport bind failed, exchange disabled: %v", pqErr) } else { cbHandler := pqCallbackHandler{ diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 06fb8538261..1610f29fb29 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -258,11 +258,13 @@ func NewConn(config ConnConfig, services ServiceDependencies) (*Conn, error) { } if config.PQ != nil && config.PQStrict { - if k, err := wgtypes.GenerateKey(); err != nil { - connLog.Errorf("pqkem: failed to generate strict-mode sentinel key, strict fail-closed disabled for this peer: %v", err) - } else { - conn.pqStrictSentinelKey = &k + // The sentinel is what makes strict mode fail closed; if we cannot generate it we + // must not fall back to a usable key, so fail creating the conn instead. + k, err := wgtypes.GenerateKey() + if err != nil { + return nil, fmt.Errorf("generate pqkem strict-mode sentinel key: %w", err) } + conn.pqStrictSentinelKey = &k } return conn, nil diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index b8bf5c02021..09eb5372088 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -30,11 +30,19 @@ func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { if !overlayIP.IsValid() { return nil, fmt.Errorf("invalid overlay IP for pqkem transport") } + // Unmap an IPv4-mapped IPv6 address so AsSlice() yields 4 bytes, and pick the UDP + // network matching the overlay family (a hardcoded "udp4" fails on a v6 overlay and + // on a mapped v4 whose AsSlice() is 16 bytes). + overlayIP = overlayIP.Unmap() + network := "udp6" + if overlayIP.Is4() { + network = "udp4" + } ip := net.IP(overlayIP.AsSlice()) - conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: DefaultPort}) + conn, err := net.ListenUDP(network, &net.UDPAddr{IP: ip, Port: DefaultPort}) if err != nil { log.Debugf("pqkem: default port %d unavailable on %s (%v), using an ephemeral port", DefaultPort, overlayIP, err) - conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) + conn, err = net.ListenUDP(network, &net.UDPAddr{IP: ip, Port: 0}) if err != nil { return nil, fmt.Errorf("bind pqkem udp on overlay %s: %w", overlayIP, err) } From ecddbf2e5567792cbbf42c09143305371ad1588e Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 17:03:32 +0200 Subject: [PATCH 064/119] Addresses CI fixes --- client/internal/engine.go | 10 ++- client/internal/peer/conn.go | 86 +++++++++++++++-------- client/internal/peer/conn_pq_test.go | 12 ++-- client/internal/peer/conn_test.go | 8 +-- client/internal/peer/handshaker.go | 34 +++++++-- client/internal/peer/signaler.go | 2 +- client/internal/pqkem/capability_test.go | 13 ++-- client/internal/pqkem/convergence_test.go | 7 +- client/internal/pqkem_adapter.go | 18 +++-- 9 files changed, 131 insertions(+), 59 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index c29c1bba11e..2f0120bf0d6 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "math/rand" "net" "net/netip" @@ -3038,6 +3039,13 @@ func convertToOfferAnswer(msg *sProto.Message) (*peer.OfferAnswer, error) { relayIP := decodeRelayIP(msg.GetBody().GetRelayServerIP()) + // Ports are uint16 internally; the proto widens them to uint32, so validate the + // range before narrowing (a value that does not fit is a malformed message). + mlkemPort := msg.GetBody().GetMlkemPort() + if mlkemPort > math.MaxUint16 { + return nil, fmt.Errorf("invalid ML-KEM port %d in signalling message", mlkemPort) + } + offerAnswer := peer.OfferAnswer{ IceCredentials: peer.IceCredentials{ UFrag: remoteCred.UFrag, @@ -3048,7 +3056,7 @@ func convertToOfferAnswer(msg *sProto.Message) (*peer.OfferAnswer, error) { RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, MlkemPayload: msg.GetBody().GetMlkemPayload(), - MlkemPort: int(msg.GetBody().GetMlkemPort()), + MlkemPort: uint16(mlkemPort), RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), RelaySrvIP: relayIP, SessionID: sessionID, diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 1610f29fb29..ea05f46bb87 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -84,14 +84,14 @@ type RosenpassConfig struct { type PQHandshaker interface { // OfferPayload returns the KEM offer to embed in an outgoing offer (nil if this // peer is not the KEM initiator) and the local PQ data-path port to announce. - OfferPayload(remoteKey string) (payload []byte, port int) + OfferPayload(remoteKey string) (payload []byte, port uint16) // ShouldSendBootstrapOffer reports whether, as the controller, we should reply to a // received responder offer with our own KEM offer (true only when no exchange is // already in flight — so we kick the KEM once and ignore further offers). ShouldSendBootstrapOffer(remoteKey string) bool // AnswerPayload processes a received KEM offer (nil if absent) and returns the KEM // answer to embed in the outgoing answer (nil if none) and the local PQ port. - AnswerPayload(remoteKey string, recvOffer []byte) (payload []byte, port int) + AnswerPayload(remoteKey string, recvOffer []byte) (payload []byte, port uint16) // OnAnswer feeds a received KEM answer (nil if absent). OnAnswer(remoteKey string, recvAnswer []byte) // PSK returns the peer's latest derived post-quantum PSK to program at WG @@ -468,6 +468,9 @@ func (conn *Conn) ConnID() id.ConnID { // configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConnInfo ICEConnInfo) { + // Read the PQ PSK before conn.mu to keep the lock order conn.mu -> manager. + pqPSK, pqOK := conn.pqPSK() + conn.mu.Lock() defer conn.mu.Unlock() @@ -485,7 +488,7 @@ func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConn if conn.currentConnPriority > priority { conn.Log.Infof("current connection priority (%s) is higher than the new one (%s), do not upgrade connection", conn.currentConnPriority, priority) conn.statusICE.SetConnected() - conn.updateIceState(iceConnInfo, time.Now()) + conn.updateIceState(iceConnInfo, pqOK, time.Now()) return } @@ -528,7 +531,7 @@ func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConn updateTime := time.Now() conn.enableWgWatcherIfNeeded(updateTime) - presharedKey := conn.presharedKey(iceConnInfo.RosenpassPubKey) + presharedKey := conn.presharedKey(iceConnInfo.RosenpassPubKey, pqPSK) if err = conn.endpointUpdater.ConfigureWGEndpoint(ep, presharedKey); err != nil { conn.handleConfigurationFailure(err, wgProxy) return @@ -544,11 +547,14 @@ func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConn conn.currentConnPriority = priority conn.statusICE.SetConnected() - conn.updateIceState(iceConnInfo, updateTime) + conn.updateIceState(iceConnInfo, pqOK, updateTime) conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr, updateTime) } func (conn *Conn) onICEStateDisconnected(sessionChanged bool) { + // Read the PQ PSK before conn.mu to keep the lock order conn.mu -> manager. + pqPSK, _ := conn.pqPSK() + conn.mu.Lock() defer conn.mu.Unlock() @@ -575,7 +581,7 @@ func (conn *Conn) onICEStateDisconnected(sessionChanged bool) { // todo consider to move after the ConfigureWGEndpoint conn.wgProxyRelay.Work() - presharedKey := conn.presharedKey(conn.rosenpassRemoteKey) + presharedKey := conn.presharedKey(conn.rosenpassRemoteKey, pqPSK) if err := conn.endpointUpdater.SwitchWGEndpoint(conn.wgProxyRelay.EndpointAddr(), presharedKey); err != nil { conn.Log.Errorf("failed to switch to relay conn: %v", err) } @@ -613,6 +619,9 @@ func (conn *Conn) onICEStateDisconnected(sessionChanged bool) { } func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) { + // Read the PQ PSK before conn.mu to keep the lock order conn.mu -> manager. + pqPSK, pqOK := conn.pqPSK() + conn.mu.Lock() defer conn.mu.Unlock() @@ -641,7 +650,7 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) { conn.Log.Debugf("do not switch to relay because current priority is: %s", conn.currentConnPriority.String()) conn.setRelayedProxy(wgProxy) conn.statusRelay.SetConnected() - conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey, time.Now()) + conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey, pqOK, time.Now()) return } @@ -652,7 +661,7 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) { } updateTime := time.Now() conn.enableWgWatcherIfNeeded(updateTime) - if err := conn.endpointUpdater.ConfigureWGEndpoint(wgProxy.EndpointAddr(), conn.presharedKey(rci.rosenpassPubKey)); err != nil { + if err := conn.endpointUpdater.ConfigureWGEndpoint(wgProxy.EndpointAddr(), conn.presharedKey(rci.rosenpassPubKey, pqPSK)); err != nil { if err := wgProxy.CloseConn(); err != nil { conn.Log.Warnf("Failed to close relay connection: %v", err) } @@ -671,7 +680,7 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) { conn.currentConnPriority = conntype.Relay conn.statusRelay.SetConnected() conn.setRelayedProxy(wgProxy) - conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey, updateTime) + conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey, pqOK, updateTime) conn.Log.Infof("start to communicate with peer via relay") conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr, updateTime) } @@ -751,19 +760,15 @@ func (conn *Conn) RequestReoffer() { func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { conn.mu.Lock() - defer conn.mu.Unlock() // watcherCtx guards against a stale watcher tearing down a connection that already superseded it. if conn.ctx.Err() != nil || watcherCtx.Err() != nil { + conn.mu.Unlock() return } conn.Log.Warnf("WireGuard handshake timeout detected, closing current connection") - if conn.config.PQ != nil { - conn.config.PQ.OnDataPathDown(conn.config.Key) - } - // Close the active connection based on current priority switch conn.currentConnPriority { case conntype.Relay: @@ -776,6 +781,15 @@ func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { } conn.escalateWGTimeoutLocked() + pq := conn.config.PQ + key := conn.config.Key + conn.mu.Unlock() + + // Signal the PQ manager outside conn.mu: it may re-enter Conn (reoffer) under + // conn.mu, so calling it while holding the lock would invert the lock order. + if pq != nil { + pq.OnDataPathDown(key) + } } // escalateWGTimeoutLocked resets the peer's rosenpass state after repeated @@ -799,14 +813,14 @@ func (conn *Conn) escalateWGTimeoutLocked() { conn.onDisconnected(conn.config.WgConfig.RemoteKey) } -func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []byte, updateTime time.Time) { +func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []byte, pqEstablished bool, updateTime time.Time) { peerState := State{ PubKey: conn.config.Key, ConnStatusUpdate: updateTime, ConnStatus: conn.evalStatus(), Relayed: conn.isRelayed(), RelayServerAddress: relayServerAddr, - RosenpassEnabled: conn.quantumResistant(rosenpassPubKey), + RosenpassEnabled: conn.quantumResistant(rosenpassPubKey, pqEstablished), } err := conn.statusRecorder.UpdatePeerRelayedState(peerState) @@ -815,7 +829,7 @@ func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []by } } -func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo, updateTime time.Time) { +func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo, pqEstablished bool, updateTime time.Time) { peerState := State{ PubKey: conn.config.Key, ConnStatusUpdate: updateTime, @@ -825,7 +839,7 @@ func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo, updateTime time.Time) RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, - RosenpassEnabled: conn.quantumResistant(iceConnInfo.RosenpassPubKey), + RosenpassEnabled: conn.quantumResistant(iceConnInfo.RosenpassPubKey, pqEstablished), } err := conn.statusRecorder.UpdatePeerICEState(peerState) @@ -1094,13 +1108,31 @@ func (conn *Conn) AgentVersionString() string { return conn.config.AgentVersion } -func (conn *Conn) presharedKey(remoteRosenpassKey []byte) *wgtypes.Key { +// pqPSK returns the post-quantum PSK derived for this peer, if the ML-KEM exchange has +// produced one. It reads the manager WITHOUT conn.mu, so callers fetch it before taking +// conn.mu: the lock order is always conn.mu -> manager, never the reverse (the manager's +// reoffer callback re-enters Conn under conn.mu). +func (conn *Conn) pqPSK() (*wgtypes.Key, bool) { + if conn.config.PQ == nil { + return nil, false + } + psk, ok := conn.config.PQ.PSK(conn.config.Key) + if !ok { + return nil, false + } + return &psk, true +} + +// presharedKey resolves the WireGuard preshared key for the peer. pqPSK is the +// post-quantum PSK looked up out of band via pqPSK (nil when none is derived yet), passed +// in so the manager lock is never taken under conn.mu. +func (conn *Conn) presharedKey(remoteRosenpassKey []byte, pqPSK *wgtypes.Key) *wgtypes.Key { // Post-quantum: once the ML-KEM exchange has derived a PSK for this peer, program // it here so the peer's next WireGuard handshake adopts it. Applied at peer-config // time (bootstrap / reconnect); steady-state rotation is pushed separately. if conn.config.PQ != nil { - if psk, ok := conn.config.PQ.PSK(conn.config.Key); ok { - return &psk + if pqPSK != nil { + return pqPSK } if conn.config.PQStrict && conn.pqStrictSentinelKey != nil { // Fail closed: program a non-matching sentinel so no session forms on a @@ -1154,16 +1186,8 @@ func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { // quantumResistant reports whether the peer's tunnel is post-quantum protected, for // the status "Quantum resistance" field: either Rosenpass (the remote advertised a // Rosenpass key) or the ML-KEM exchange (a PQ PSK has been derived for this peer). -func (conn *Conn) quantumResistant(remoteRosenpassPubKey []byte) bool { - if isRosenpassEnabled(remoteRosenpassPubKey) { - return true - } - if conn.config.PQ != nil { - if _, ok := conn.config.PQ.PSK(conn.config.Key); ok { - return true - } - } - return false +func (conn *Conn) quantumResistant(remoteRosenpassPubKey []byte, pqEstablished bool) bool { + return isRosenpassEnabled(remoteRosenpassPubKey) || pqEstablished } func evalConnStatus(in connStatusInputs) guard.ConnStatus { diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index ffc6e8c32a1..c4edb342447 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -53,10 +53,12 @@ func TestConn_presharedKey_PQ(t *testing.T) { c.config.PQ = fakePQ{psk: derivedPSK, ok: true} c.config.PQStrict = strict if strict { - sentinel, _ := wgtypes.GenerateKey() + sentinel, err := wgtypes.GenerateKey() + require.NoError(t, err) c.pqStrictSentinelKey = &sentinel } - got := c.presharedKey(nil) + pqPSK, _ := c.pqPSK() + got := c.presharedKey(nil, pqPSK) require.NotNil(t, got) require.Equal(t, derivedPSK, *got, "the derived PQ PSK must win (strict=%v)", strict) } @@ -66,7 +68,8 @@ func TestConn_presharedKey_PQ(t *testing.T) { c := newConn() c.config.PQ = fakePQ{ok: false} c.config.PQStrict = false - got := c.presharedKey(nil) + pqPSK, _ := c.pqPSK() + got := c.presharedKey(nil, pqPSK) require.NotNil(t, got, "non-strict must not block") require.Equal(t, nbPSK, *got, "non-strict falls through to the NetBird PSK, not a sentinel") }) @@ -78,7 +81,8 @@ func TestConn_presharedKey_PQ(t *testing.T) { c.config.PQ = fakePQ{ok: false} c.config.PQStrict = true c.pqStrictSentinelKey = &sentinel - got := c.presharedKey(nil) + pqPSK, _ := c.pqPSK() + got := c.presharedKey(nil, pqPSK) require.NotNil(t, got) require.Equal(t, sentinel, *got, "strict must return the blocking sentinel") require.NotEqual(t, nbPSK, *got, "the sentinel must not be the ordinary key") diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index b709d5e404d..c6e82c06e1e 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -257,8 +257,8 @@ func TestConn_presharedKey(t *testing.T) { } conn2.config.RosenpassConfig.PermissiveMode = test.conn2Permissive - conn1PresharedKey := conn1.presharedKey(conn2.config.RosenpassConfig.PubKey) - conn2PresharedKey := conn2.presharedKey(conn1.config.RosenpassConfig.PubKey) + conn1PresharedKey := conn1.presharedKey(conn2.config.RosenpassConfig.PubKey, nil) + conn2PresharedKey := conn2.presharedKey(conn1.config.RosenpassConfig.PubKey, nil) if test.conn1ExpectedInitialKey { if conn1PresharedKey == nil { @@ -296,14 +296,14 @@ func TestConn_presharedKey_RosenpassManaged(t *testing.T) { // When Rosenpass has already initialized the PSK for this peer, // presharedKey must return nil to avoid UpdatePeer overwriting it. conn.rosenpassInitializedPresharedKeyValidator = func(peerKey string) bool { return true } - if k := conn.presharedKey([]byte("remote")); k != nil { + if k := conn.presharedKey([]byte("remote"), nil); k != nil { t.Fatalf("expected nil presharedKey when Rosenpass manages PSK, got %v", k) } // When Rosenpass hasn't taken over yet, presharedKey should provide // a non-nil initial key (deterministic or from NetBird PSK). conn.rosenpassInitializedPresharedKeyValidator = func(peerKey string) bool { return false } - if k := conn.presharedKey([]byte("remote")); k == nil { + if k := conn.presharedKey([]byte("remote"), nil); k == nil { t.Fatalf("expected non-nil presharedKey before Rosenpass manages PSK") } } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 4caf2327be5..9eb7e3d0a2f 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -47,7 +47,7 @@ type OfferAnswer struct { // MlkemPort is the peer's ML-KEM PQ service UDP port (bound on its WG overlay // IP) where data-path rekey messages are sent. Zero when not running the exchange. - MlkemPort int + MlkemPort uint16 // relay server address RelaySrvAddress string @@ -221,14 +221,38 @@ func (h *Handshaker) pqControllerReoffer() bool { // pqRegisterEndpoint feeds the post-quantum handshaker the peer's data-path endpoint // (its WG overlay IP plus the advertised pq UDP port) learned from a remote offer/answer. -func (h *Handshaker) pqRegisterEndpoint(remotePort int) { - if h.config.PQ == nil || remotePort < 0 || remotePort > 65535 || len(h.config.WgConfig.AllowedIps) == 0 { +func (h *Handshaker) pqRegisterEndpoint(remotePort uint16) { + if h.config.PQ == nil { + return + } + overlay, ok := h.pqPeerOverlayAddr() + if !ok { return } // remotePort may be 0 (the peer omitted it, meaning the default port); the adapter // resolves 0 to DefaultPort. - addr := netip.AddrPortFrom(h.config.WgConfig.AllowedIps[0].Addr(), uint16(remotePort)) - h.config.PQ.SetRemoteAddr(h.config.Key, addr) + h.config.PQ.SetRemoteAddr(h.config.Key, netip.AddrPortFrom(overlay, remotePort)) +} + +// pqPeerOverlayAddr picks the peer's overlay address for the pq data path. The pq +// transport binds on the local WG overlay IPv4, so an IPv4 AllowedIP is preferred; a v6 +// prefix is used only when this interface actually has a v6 overlay, and never in place +// of a usable v4. Returns false when no suitable address exists. +func (h *Handshaker) pqPeerOverlayAddr() (netip.Addr, bool) { + var v6 netip.Addr + for _, p := range h.config.WgConfig.AllowedIps { + a := p.Addr().Unmap() + if a.Is4() { + return a, true + } + if a.Is6() && !v6.IsValid() { + v6 = a + } + } + if v6.IsValid() && h.config.WgConfig.WgInterface.Address().HasIPv6() { + return v6, true + } + return netip.Addr{}, false } func (h *Handshaker) SendOffer() error { diff --git a/client/internal/peer/signaler.go b/client/internal/peer/signaler.go index e286d7f8654..f42a6e02866 100644 --- a/client/internal/peer/signaler.go +++ b/client/internal/peer/signaler.go @@ -64,7 +64,7 @@ func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, RosenpassPubKey: offerAnswer.RosenpassPubKey, RosenpassAddr: offerAnswer.RosenpassAddr, MlkemPayload: offerAnswer.MlkemPayload, - MlkemPort: offerAnswer.MlkemPort, + MlkemPort: int(offerAnswer.MlkemPort), RelaySrvAddress: offerAnswer.RelaySrvAddress, RelaySrvIP: offerAnswer.RelaySrvIP, SessionID: sessionIDBytes, diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index b6a1fe74c26..b1c29d582b7 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -3,6 +3,7 @@ package pqkem import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -19,8 +20,8 @@ func TestManager_NonCapablePeerNotOffered(t *testing.T) { offer, err := d.SignalOffer("aaaa") require.NoError(t, err) - require.Nil(t, offer, "a non-capable peer must not be offered a KEM exchange") - require.Empty(t, wg.failed, "a non-capable peer must not raise a rekey failure") + assert.Nil(t, offer, "a non-capable peer must not be offered a KEM exchange") + assert.Empty(t, wg.failed, "a non-capable peer must not raise a rekey failure") } // TestManager_MarkNonCapableCancelsInFlight: if we start an exchange with a peer whose @@ -42,8 +43,8 @@ func TestManager_MarkNonCapableCancelsInFlight(t *testing.T) { next, err := d.SignalOffer("aaaa") require.NoError(t, err) - require.Nil(t, next, "after learning non-capability the peer is no longer offered") - require.Empty(t, wg.failed, "cancelling an in-flight exchange must not raise a failure") + assert.Nil(t, next, "after learning non-capability the peer is no longer offered") + assert.Empty(t, wg.failed, "cancelling an in-flight exchange must not raise a failure") } // TestManager_EstablishedPeerNotDowngraded: a stray zero-port observation must not tear @@ -60,6 +61,6 @@ func TestManager_EstablishedPeerNotDowngraded(t *testing.T) { // The peer keeps its derived PSK (MarkNonCapable is a no-op once established). psk, ok := dB.PSK("aaaa") - require.True(t, ok, "an established peer must keep its PSK despite a stray zero") - require.NotEqual(t, PSK{}, psk) + assert.True(t, ok, "an established peer must keep its PSK despite a stray zero") + assert.NotEqual(t, PSK{}, psk) } diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index fed15d0ea61..c8f4f1599ba 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func TestManager_InitialTimeoutFailsImmediately(t *testing.T) { require.NoError(t, err) require.NotNil(t, offer) - require.Eventually(t, func() bool { return failedCount(wg) == 1 }, time.Second, 5*time.Millisecond) + assert.Eventually(t, func() bool { return failedCount(wg) == 1 }, time.Second, 5*time.Millisecond) } func TestManager_RekeyToleratesKFailures(t *testing.T) { @@ -65,10 +66,10 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { require.NoError(t, err) time.Sleep(50 * time.Millisecond) } - require.Equal(t, 0, failedCount(wgB), "no failure before K attempts") + assert.Equal(t, 0, failedCount(wgB), "no failure before K attempts") // The K-th failure raises it once. _, err := dB.startExchangeTest("aaaa", false, ExchangeID{}) require.NoError(t, err) - require.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) + assert.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) } diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index be5a99c1ba4..8814dc2294c 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -57,14 +57,17 @@ type pqHandshaker struct { // announcedPort is the PQ data-path port to advertise to peers. It is omitted (0) when // the manager is on DefaultPort, since peers assume the default when no port is sent; // only a non-default (collision-forced) port is announced explicitly. -func (p pqHandshaker) announcedPort() int { +func (p pqHandshaker) announcedPort() uint16 { if port := p.mgr.LocalPort(); port != DefaultPort { - return port + return uint16(port) } return 0 } -func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, int) { +// OfferPayload builds the KEM offer to attach to an outgoing signalling offer for the +// peer, plus the data-path port to announce (0 when on DefaultPort). Payload is nil when +// this side has no offer to send. +func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, uint16) { payload, err := p.mgr.SignalOffer(pqkem.RemoteID(remoteKey)) if err != nil { log.Warnf("pqkem: build offer for %s: %v", remoteKey, err) @@ -72,11 +75,16 @@ func (p pqHandshaker) OfferPayload(remoteKey string) ([]byte, int) { return payload, p.announcedPort() } +// ShouldSendBootstrapOffer reports whether the controller should reply to the peer's +// KEM-less offer with its own bootstrap offer instead of an answer. func (p pqHandshaker) ShouldSendBootstrapOffer(remoteKey string) bool { return p.mgr.ShouldSendBootstrapOffer(pqkem.RemoteID(remoteKey)) } -func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, int) { +// AnswerPayload processes a received KEM offer (nil when absent) and returns the KEM +// answer to attach to the outgoing signalling answer, plus the data-path port to announce +// (0 when on DefaultPort). An empty offer is treated as a capability signal. +func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, uint16) { if len(recvOffer) == 0 { // Capability signal (responder side): the KEM offer flows initiator->responder, // so if we are the responder for this peer (it is the KEM initiator by role) an @@ -95,6 +103,8 @@ func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, return payload, p.announcedPort() } +// OnAnswer feeds a received KEM answer (nil when absent) into the exchange. An empty +// answer to our offer is treated as a capability signal on the initiator side. func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { if len(recvAnswer) == 0 { // Capability signal (initiator side): the KEM answer flows responder->initiator, From 3efd4c94a0483e75dc27bfc44c5188c384e9a3e5 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 7 Aug 2026 17:23:49 +0200 Subject: [PATCH 065/119] UDPv4 is sufficient! we always have 100.x overlay. IPv6 is additive ipAllowed[0] is the v4 --- client/internal/peer/conn_pq_test.go | 16 ++++++++-------- client/internal/peer/handshaker.go | 24 +++++++++--------------- client/internal/pqkem_transport.go | 14 +++++--------- 3 files changed, 22 insertions(+), 32 deletions(-) diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index c4edb342447..b813a44a074 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -17,14 +17,14 @@ type fakePQ struct { ok bool } -func (f fakePQ) OfferPayload(string) ([]byte, int) { return nil, 0 } -func (f fakePQ) ShouldSendBootstrapOffer(string) bool { return false } -func (f fakePQ) AnswerPayload(string, []byte) ([]byte, int) { return nil, 0 } -func (f fakePQ) OnAnswer(string, []byte) {} -func (f fakePQ) PSK(string) (wgtypes.Key, bool) { return f.psk, f.ok } -func (f fakePQ) SetRemoteAddr(string, netip.AddrPort) {} -func (f fakePQ) OnDataPathRekeyed(string, time.Duration) {} -func (f fakePQ) OnDataPathDown(string) {} +func (f fakePQ) OfferPayload(string) ([]byte, uint16) { return nil, 0 } +func (f fakePQ) ShouldSendBootstrapOffer(string) bool { return false } +func (f fakePQ) AnswerPayload(string, []byte) ([]byte, uint16) { return nil, 0 } +func (f fakePQ) OnAnswer(string, []byte) {} +func (f fakePQ) PSK(string) (wgtypes.Key, bool) { return f.psk, f.ok } +func (f fakePQ) SetRemoteAddr(string, netip.AddrPort) {} +func (f fakePQ) OnDataPathRekeyed(string, time.Duration) {} +func (f fakePQ) OnDataPathDown(string) {} // TestConn_presharedKey_PQ covers the post-quantum branch of presharedKey across the // three states that matter: a derived PSK is programmed, and — before one exists — diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 9eb7e3d0a2f..e64c9c29955 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -234,23 +234,17 @@ func (h *Handshaker) pqRegisterEndpoint(remotePort uint16) { h.config.PQ.SetRemoteAddr(h.config.Key, netip.AddrPortFrom(overlay, remotePort)) } -// pqPeerOverlayAddr picks the peer's overlay address for the pq data path. The pq -// transport binds on the local WG overlay IPv4, so an IPv4 AllowedIP is preferred; a v6 -// prefix is used only when this interface actually has a v6 overlay, and never in place -// of a usable v4. Returns false when no suitable address exists. +// pqPeerOverlayAddr returns the peer's IPv4 overlay address for the pq data path. A +// RemotePeerConfig carries only the peer overlay (v4 /32, optionally v6 /128) — served +// routes are programmed on WireGuard separately and never land in WgConfig.AllowedIps — +// so AllowedIps[0] is the v4 overlay, matching conn.AllowedIP(). The transport is v4 +// (the overlay always has v4; v6 is additive). Returns false when no v4 overlay exists. func (h *Handshaker) pqPeerOverlayAddr() (netip.Addr, bool) { - var v6 netip.Addr - for _, p := range h.config.WgConfig.AllowedIps { - a := p.Addr().Unmap() - if a.Is4() { - return a, true - } - if a.Is6() && !v6.IsValid() { - v6 = a - } + if len(h.config.WgConfig.AllowedIps) == 0 { + return netip.Addr{}, false } - if v6.IsValid() && h.config.WgConfig.WgInterface.Address().HasIPv6() { - return v6, true + if a := h.config.WgConfig.AllowedIps[0].Addr().Unmap(); a.Is4() { + return a, true } return netip.Addr{}, false } diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index 09eb5372088..920e655c4ff 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -30,19 +30,15 @@ func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { if !overlayIP.IsValid() { return nil, fmt.Errorf("invalid overlay IP for pqkem transport") } - // Unmap an IPv4-mapped IPv6 address so AsSlice() yields 4 bytes, and pick the UDP - // network matching the overlay family (a hardcoded "udp4" fails on a v6 overlay and - // on a mapped v4 whose AsSlice() is 16 bytes). + // The WG overlay always carries an IPv4 address (v6 is additive, never standalone), + // so the transport binds over IPv4. Unmap first so AsSlice() yields 4 bytes for an + // IPv4-mapped IPv6 address (a hardcoded "udp4" would otherwise fail on its 16 bytes). overlayIP = overlayIP.Unmap() - network := "udp6" - if overlayIP.Is4() { - network = "udp4" - } ip := net.IP(overlayIP.AsSlice()) - conn, err := net.ListenUDP(network, &net.UDPAddr{IP: ip, Port: DefaultPort}) + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: DefaultPort}) if err != nil { log.Debugf("pqkem: default port %d unavailable on %s (%v), using an ephemeral port", DefaultPort, overlayIP, err) - conn, err = net.ListenUDP(network, &net.UDPAddr{IP: ip, Port: 0}) + conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) if err != nil { return nil, fmt.Errorf("bind pqkem udp on overlay %s: %w", overlayIP, err) } From 1ee012160bfacbcfd92509153d522ed67176cfd4 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 24 Aug 2026 13:58:22 +0200 Subject: [PATCH 066/119] Add comment to explain working boundaries --- client/internal/pqkem/manager.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 4e6ce3bbe5d..22123d2cec9 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -27,6 +27,11 @@ const ( // REKEY_AFTER_TIME ~120s) so the rotation's own traffic — which itself renews the // activity signal — ages out before the next rekey, letting an idle tunnel stop // rotating instead of self-sustaining. + // + // INVARIANT: rotationActivityWindow < the gap between OnDataPathRekeyed calls. That + // gap converges to the WireGuard rehandshake interval (REKEY_AFTER_TIME 120s). Setting + // rotationActivityWindow to a value higher than 120s will make the rotation + // self-sustain (preventing going back to idle). Margin here is ~30s. rotationActivityWindow = 90 * time.Second ) From 91f1a724e2a1b07d2b3f0be005da136f4b378a3c Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 27 Aug 2026 12:16:20 +0200 Subject: [PATCH 067/119] Adds observability and fixes cross cases - strict-kem vs strict-rp said "Connected + Quantum resistance: true" but it is actually blocked - perm-kem vs perm-rp "Connected + Quantum resistance: true" but it's a classic WG link, without PQ safety --- client/internal/debug/debug.go | 4 ++ client/internal/peer/conn.go | 8 +++- client/internal/peer/status.go | 31 +++++++++++++ client/proto/daemon.pb.go | 23 ++++++++- client/proto/daemon.proto | 2 + client/server/mdm.go | 4 ++ client/server/server.go | 4 ++ client/status/status.go | 85 ++++++++++++++++++++++++---------- client/status/status_test.go | 4 ++ 9 files changed, 137 insertions(+), 28 deletions(-) diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index b362ae29381..3503881efda 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -27,6 +27,7 @@ import ( "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/configs" "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/client/internal/pqkem" "github.com/netbirdio/netbird/client/internal/profilemanager" "github.com/netbirdio/netbird/client/internal/updater/installer" nbstatus "github.com/netbirdio/netbird/client/status" @@ -708,6 +709,9 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString(fmt.Sprintf("DisableIPv6Discovery: %v\n", g.internalConfig.DisableIPv6Discovery)) configContent.WriteString(fmt.Sprintf("RosenpassEnabled: %v\n", g.internalConfig.RosenpassEnabled)) configContent.WriteString(fmt.Sprintf("RosenpassPermissive: %v\n", g.internalConfig.RosenpassPermissive)) + // ML-KEM (Rosenpass alternative) is env-driven, not part of the config, so read it here. + configContent.WriteString(fmt.Sprintf("MLKEMEnabled: %v\n", pqkem.Enabled())) + configContent.WriteString(fmt.Sprintf("MLKEMStrict: %v\n", pqkem.Strict())) if g.internalConfig.ServerSSHAllowed != nil { configContent.WriteString(fmt.Sprintf("ServerSSHAllowed: %v\n", *g.internalConfig.ServerSSHAllowed)) } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index ea05f46bb87..c2e71a7a62a 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -1187,7 +1187,13 @@ func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { // the status "Quantum resistance" field: either Rosenpass (the remote advertised a // Rosenpass key) or the ML-KEM exchange (a PQ PSK has been derived for this peer). func (conn *Conn) quantumResistant(remoteRosenpassPubKey []byte, pqEstablished bool) bool { - return isRosenpassEnabled(remoteRosenpassPubKey) || pqEstablished + // Rosenpass protects the tunnel only when both sides run it: the local peer has a + // Rosenpass key AND the remote advertised one. Checking only the remote key would + // report a plain (or blocked) tunnel as quantum-resistant when the local side does + // not run Rosenpass — e.g. against a peer that merely advertises it in a mixed + // deployment. A homogeneous Rosenpass deployment (both sides on) is unaffected. + rosenpassActive := conn.config.RosenpassConfig.PubKey != nil && isRosenpassEnabled(remoteRosenpassPubKey) + return rosenpassActive || pqEstablished } func evalConnStatus(in connStatusInputs) guard.ConnStatus { diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index bf36b944bf4..bbada934a67 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -142,6 +142,14 @@ type RosenpassState struct { Permissive bool } +// MLKEMState contains the latest state of the ML-KEM post-quantum exchange, the +// Rosenpass alternative. Strict is the ML-KEM counterpart of Rosenpass non-permissive: +// it fails closed until the KEM PSK is established. +type MLKEMState struct { + Enabled bool + Strict bool +} + // NSGroupState represents the status of a DNS server group, including associated domains, // whether it's enabled, and the last error message encountered during probing. type NSGroupState struct { @@ -159,6 +167,7 @@ type FullStatus struct { SignalState SignalState LocalPeerState LocalPeerState RosenpassState RosenpassState + MLKEMState MLKEMState Relays []relay.ProbeResult NSGroupStates []NSGroupState NumOfForwardingRules int @@ -209,6 +218,8 @@ type Status struct { notifier *notifier rosenpassEnabled bool rosenpassPermissive bool + mlkemEnabled bool + mlkemStrict bool // sessionExpiresAt is the absolute UTC instant at which the peer's SSO // session expires. Zero when the peer is not SSO-tracked or login // expiration is disabled. Populated from management LoginResponse / @@ -952,6 +963,14 @@ func (d *Status) UpdateRosenpass(rosenpassEnabled, rosenpassPermissive bool) { d.rosenpassEnabled = rosenpassEnabled } +// UpdateMLKEM updates the ML-KEM post-quantum exchange configuration. +func (d *Status) UpdateMLKEM(mlkemEnabled, mlkemStrict bool) { + d.mux.Lock() + defer d.mux.Unlock() + d.mlkemEnabled = mlkemEnabled + d.mlkemStrict = mlkemStrict +} + func (d *Status) UpdateLazyConnection(enabled bool) { d.mux.Lock() defer d.mux.Unlock() @@ -1044,6 +1063,15 @@ func (d *Status) GetRosenpassState() RosenpassState { } } +func (d *Status) GetMLKEMState() MLKEMState { + d.mux.RLock() + defer d.mux.RUnlock() + return MLKEMState{ + d.mlkemEnabled, + d.mlkemStrict, + } +} + func (d *Status) GetLazyConnection() bool { d.mux.RLock() defer d.mux.RUnlock() @@ -1186,6 +1214,7 @@ func (d *Status) GetFullStatus() FullStatus { SignalState: d.GetSignalState(), Relays: d.GetRelayStates(), RosenpassState: d.GetRosenpassState(), + MLKEMState: d.GetMLKEMState(), NSGroupStates: d.GetDNSStates(), NumOfForwardingRules: len(d.ForwardingRules()), LazyConnectionEnabled: d.GetLazyConnection(), @@ -1559,6 +1588,8 @@ func (fs FullStatus) ToProto() *proto.FullStatus { pbFullStatus.LocalPeerState.WgPort = int32(fs.LocalPeerState.WgPort) pbFullStatus.LocalPeerState.RosenpassPermissive = fs.RosenpassState.Permissive pbFullStatus.LocalPeerState.RosenpassEnabled = fs.RosenpassState.Enabled + pbFullStatus.LocalPeerState.MlkemEnabled = fs.MLKEMState.Enabled + pbFullStatus.LocalPeerState.MlkemStrict = fs.MLKEMState.Strict pbFullStatus.NumberOfForwardingRules = int32(fs.NumOfForwardingRules) pbFullStatus.LazyConnectionEnabled = fs.LazyConnectionEnabled diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index 7f3ce1bbf6c..e8113b0cadc 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -1686,6 +1686,8 @@ type LocalPeerState struct { Networks []string `protobuf:"bytes,7,rep,name=networks,proto3" json:"networks,omitempty"` Ipv6 string `protobuf:"bytes,8,opt,name=ipv6,proto3" json:"ipv6,omitempty"` WgPort int32 `protobuf:"varint,9,opt,name=wgPort,proto3" json:"wgPort,omitempty"` + MlkemEnabled bool `protobuf:"varint,10,opt,name=mlkemEnabled,proto3" json:"mlkemEnabled,omitempty"` + MlkemStrict bool `protobuf:"varint,11,opt,name=mlkemStrict,proto3" json:"mlkemStrict,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1783,6 +1785,20 @@ func (x *LocalPeerState) GetWgPort() int32 { return 0 } +func (x *LocalPeerState) GetMlkemEnabled() bool { + if x != nil { + return x.MlkemEnabled + } + return false +} + +func (x *LocalPeerState) GetMlkemStrict() bool { + if x != nil { + return x.MlkemStrict + } + return false +} + // SignalState contains the latest state of a signal connection type SignalState struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -7265,7 +7281,7 @@ const file_daemon_proto_rawDesc = "" + "\n" + "sshHostKey\x18\x13 \x01(\fR\n" + "sshHostKey\x12\x12\n" + - "\x04ipv6\x18\x14 \x01(\tR\x04ipv6\"\x9c\x02\n" + + "\x04ipv6\x18\x14 \x01(\tR\x04ipv6\"\xe2\x02\n" + "\x0eLocalPeerState\x12\x0e\n" + "\x02IP\x18\x01 \x01(\tR\x02IP\x12\x16\n" + "\x06pubKey\x18\x02 \x01(\tR\x06pubKey\x12(\n" + @@ -7275,7 +7291,10 @@ const file_daemon_proto_rawDesc = "" + "\x13rosenpassPermissive\x18\x06 \x01(\bR\x13rosenpassPermissive\x12\x1a\n" + "\bnetworks\x18\a \x03(\tR\bnetworks\x12\x12\n" + "\x04ipv6\x18\b \x01(\tR\x04ipv6\x12\x16\n" + - "\x06wgPort\x18\t \x01(\x05R\x06wgPort\"S\n" + + "\x06wgPort\x18\t \x01(\x05R\x06wgPort\x12\"\n" + + "\fmlkemEnabled\x18\n" + + " \x01(\bR\fmlkemEnabled\x12 \n" + + "\vmlkemStrict\x18\v \x01(\bR\vmlkemStrict\"S\n" + "\vSignalState\x12\x10\n" + "\x03URL\x18\x01 \x01(\tR\x03URL\x12\x1c\n" + "\tconnected\x18\x02 \x01(\bR\tconnected\x12\x14\n" + diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 3953f9c15ef..dd3b32d5e1b 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -412,6 +412,8 @@ message LocalPeerState { repeated string networks = 7; string ipv6 = 8; int32 wgPort = 9; + bool mlkemEnabled = 10; + bool mlkemStrict = 11; } // SignalState contains the latest state of a signal connection diff --git a/client/server/mdm.go b/client/server/mdm.go index 7a47b2a57aa..894e8187116 100644 --- a/client/server/mdm.go +++ b/client/server/mdm.go @@ -9,6 +9,7 @@ import ( "google.golang.org/grpc/codes" gstatus "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/client/internal/pqkem" "github.com/netbirdio/netbird/client/mdm" "github.com/netbirdio/netbird/client/proto" ) @@ -133,6 +134,9 @@ func (s *Server) restartEngineForMDMLocked() error { s.config = config s.statusRecorder.UpdateManagementAddress(config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(config.RosenpassEnabled, config.RosenpassPermissive) + // ML-KEM is env-driven today; mirror the Rosenpass update so the status stays in + // sync after an MDM config apply and the wiring is ready if it becomes a config field. + s.statusRecorder.UpdateMLKEM(pqkem.Enabled(), pqkem.Strict()) ctx, cancel := context.WithCancel(s.rootCtx) s.actCancel = cancel diff --git a/client/server/server.go b/client/server/server.go index 108aa8a41e1..8421c8880a1 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -36,6 +36,7 @@ import ( "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/client/internal/pqkem" "github.com/netbirdio/netbird/client/internal/statemanager" "github.com/netbirdio/netbird/client/internal/updater" "github.com/netbirdio/netbird/client/proto" @@ -302,6 +303,7 @@ func (s *Server) Start() error { s.statusRecorder.UpdateManagementAddress(config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(config.RosenpassEnabled, config.RosenpassPermissive) + s.statusRecorder.UpdateMLKEM(pqkem.Enabled(), pqkem.Strict()) s.localMetrics.Reconcile(config.LocalMetricsEnabled, config.LocalMetricsAddress) if s.sessionWatcher == nil { @@ -1096,6 +1098,7 @@ func (s *Server) Up(callerCtx context.Context, msg *proto.UpRequest) (*proto.UpR s.statusRecorder.UpdateManagementAddress(s.config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(s.config.RosenpassEnabled, s.config.RosenpassPermissive) + s.statusRecorder.UpdateMLKEM(pqkem.Enabled(), pqkem.Strict()) s.localMetrics.Reconcile(s.config.LocalMetricsEnabled, s.config.LocalMetricsAddress) s.clientRunning = true @@ -1698,6 +1701,7 @@ func (s *Server) buildStatusResponse(ctx context.Context, msg *proto.StatusReque s.statusRecorder.UpdateManagementAddress(s.config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(s.config.RosenpassEnabled, s.config.RosenpassPermissive) + s.statusRecorder.UpdateMLKEM(pqkem.Enabled(), pqkem.Strict()) if msg.GetFullPeerStatus { s.runProbes(ctx, msg.ShouldRunProbes) diff --git a/client/status/status.go b/client/status/status.go index 1c204cdb1af..e5c83c995a7 100644 --- a/client/status/status.go +++ b/client/status/status.go @@ -155,6 +155,8 @@ type OutputOverview struct { FQDN string `json:"fqdn" yaml:"fqdn"` RosenpassEnabled bool `json:"quantumResistance" yaml:"quantumResistance"` RosenpassPermissive bool `json:"quantumResistancePermissive" yaml:"quantumResistancePermissive"` + MLKEMEnabled bool `json:"mlkemEnabled" yaml:"mlkemEnabled"` + MLKEMStrict bool `json:"mlkemStrict" yaml:"mlkemStrict"` Networks []string `json:"networks" yaml:"networks"` NumberOfForwardingRules int `json:"forwardingRules" yaml:"forwardingRules"` NSServerGroups []NsServerGroupStateOutput `json:"dnsServers" yaml:"dnsServers"` @@ -205,6 +207,8 @@ func ConvertToStatusOutputOverview(pbFullStatus *proto.FullStatus, opts ConvertO FQDN: pbFullStatus.GetLocalPeerState().GetFqdn(), RosenpassEnabled: pbFullStatus.GetLocalPeerState().GetRosenpassEnabled(), RosenpassPermissive: pbFullStatus.GetLocalPeerState().GetRosenpassPermissive(), + MLKEMEnabled: pbFullStatus.GetLocalPeerState().GetMlkemEnabled(), + MLKEMStrict: pbFullStatus.GetLocalPeerState().GetMlkemStrict(), Networks: pbFullStatus.GetLocalPeerState().GetNetworks(), NumberOfForwardingRules: int(pbFullStatus.GetNumberOfForwardingRules()), NSServerGroups: mapNSGroups(pbFullStatus.GetDnsServers()), @@ -511,12 +515,14 @@ func (o *OutputOverview) GeneralSummary(showURL bool, showRelays bool, showNameS dnsServersString = fmt.Sprintf("%d/%d Available", countEnabled(o.NSServerGroups), len(o.NSServerGroups)) } - rosenpassEnabledStatus := "false" - if o.RosenpassEnabled { - rosenpassEnabledStatus = "true" - if o.RosenpassPermissive { - rosenpassEnabledStatus = "true (permissive)" //nolint:gosec - } + // Quantum resistance is the outcome (post-quantum protected or not); the mechanism + // that provides it — ML-KEM or Rosenpass, strict or permissive — is a separate detail. + mechanism := quantumResistanceMechanism(o.RosenpassEnabled, o.RosenpassPermissive, o.MLKEMEnabled, o.MLKEMStrict) + quantumResistanceStatus := "false" + mechanismLine := "" + if mechanism != "none" { + quantumResistanceStatus = "true" + mechanismLine = fmt.Sprintf("Quantum resistance mechanism: %s\n", mechanism) } lazyConnectionEnabledStatus := "false" @@ -615,6 +621,7 @@ func (o *OutputOverview) GeneralSummary(showURL bool, showRelays bool, showNameS "Interface type: %s\n"+ "Wireguard port: %s\n"+ "Quantum resistance: %s\n"+ + "%s"+ "Lazy connection: %s\n"+ "SSH Server: %s\n"+ "Networks: %s\n"+ @@ -634,7 +641,8 @@ func (o *OutputOverview) GeneralSummary(showURL bool, showRelays bool, showNameS ipv6Line, interfaceTypeString, wgPortString, - rosenpassEnabledStatus, + quantumResistanceStatus, + mechanismLine, lazyConnectionEnabledStatus, sshServerStatus, networks, @@ -647,7 +655,7 @@ func (o *OutputOverview) GeneralSummary(showURL bool, showRelays bool, showNameS // FullDetailSummary returns a full detailed summary with peer details and events. func (o *OutputOverview) FullDetailSummary() string { - parsedPeersString := parsePeers(o.Peers, o.RosenpassEnabled, o.RosenpassPermissive) + parsedPeersString := parsePeers(o.Peers, o.RosenpassEnabled, o.RosenpassPermissive, o.MLKEMEnabled, o.MLKEMStrict) parsedEventsString := parseEvents(o.Events) summary := o.GeneralSummary(true, true, true, true) @@ -690,6 +698,8 @@ func ToProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { pbFullStatus.LocalPeerState.Fqdn = fullStatus.LocalPeerState.FQDN pbFullStatus.LocalPeerState.RosenpassPermissive = fullStatus.RosenpassState.Permissive pbFullStatus.LocalPeerState.RosenpassEnabled = fullStatus.RosenpassState.Enabled + pbFullStatus.LocalPeerState.MlkemEnabled = fullStatus.MLKEMState.Enabled + pbFullStatus.LocalPeerState.MlkemStrict = fullStatus.MLKEMState.Strict pbFullStatus.LocalPeerState.Networks = maps.Keys(fullStatus.LocalPeerState.Routes) pbFullStatus.NumberOfForwardingRules = int32(fullStatus.NumOfForwardingRules) pbFullStatus.LazyConnectionEnabled = fullStatus.LazyConnectionEnabled @@ -755,7 +765,47 @@ func ToProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { return &pbFullStatus } -func parsePeers(peers PeersStateOutput, rosenpassEnabled, rosenpassPermissive bool) string { +// quantumResistanceMechanism reports which post-quantum mechanism the local client runs, +// as an enum: "none", "ML-KEM strict", "ML-KEM permissive", "RP strict", "RP permissive". +// ML-KEM and Rosenpass are mutually exclusive, so at most one is active; ML-KEM takes +// precedence if somehow both are set. Strict/permissive is the fail-closed vs fail-open mode. +func quantumResistanceMechanism(rosenpassEnabled, rosenpassPermissive, mlkemEnabled, mlkemStrict bool) string { + switch { + case mlkemEnabled && mlkemStrict: + return "ML-KEM strict" + case mlkemEnabled: + return "ML-KEM permissive" + case rosenpassEnabled && !rosenpassPermissive: + return "RP strict" + case rosenpassEnabled: + return "RP permissive" + default: + return "none" + } +} + +// peerQuantumResistanceStatus renders the per-peer "Quantum resistance" field. established +// reports whether this peer's tunnel is post-quantum protected (Rosenpass or ML-KEM). When +// it is not, the reason is phrased for whichever mechanism is active locally. +func peerQuantumResistanceStatus(established, rosenpassEnabled, rosenpassPermissive, mlkemEnabled, mlkemStrict bool) string { + if established { + return "true" + } + switch { + case mlkemEnabled && mlkemStrict: + return "false (ML-KEM strict: blocking peer traffic until the exchange converges)" + case mlkemEnabled: + return "false (ML-KEM: not converged yet, or peer does not run the exchange)" + case rosenpassEnabled && rosenpassPermissive: + return "false (remote didn't enable quantum resistance)" + case rosenpassEnabled: + return "false (connection won't work without a permissive mode)" + default: + return "false" + } +} + +func parsePeers(peers PeersStateOutput, rosenpassEnabled, rosenpassPermissive, mlkemEnabled, mlkemStrict bool) string { var ( peersString = "" ) @@ -782,22 +832,7 @@ func parsePeers(peers PeersStateOutput, rosenpassEnabled, rosenpassPermissive bo remoteICEEndpoint = peerState.IceCandidateEndpoint.Remote } - rosenpassEnabledStatus := "false" - if rosenpassEnabled { - if peerState.RosenpassEnabled { - rosenpassEnabledStatus = "true" - } else { - if rosenpassPermissive { - rosenpassEnabledStatus = "false (remote didn't enable quantum resistance)" - } else { - rosenpassEnabledStatus = "false (connection won't work without a permissive mode)" - } - } - } else { - if peerState.RosenpassEnabled { - rosenpassEnabledStatus = "false (connection might not work without a remote permissive mode)" - } - } + rosenpassEnabledStatus := peerQuantumResistanceStatus(peerState.RosenpassEnabled, rosenpassEnabled, rosenpassPermissive, mlkemEnabled, mlkemStrict) networks := "-" if len(peerState.Networks) > 0 { diff --git a/client/status/status_test.go b/client/status/status_test.go index 2babd93420b..9e507520f3c 100644 --- a/client/status/status_test.go +++ b/client/status/status_test.go @@ -375,6 +375,8 @@ func TestParsingToJSON(t *testing.T) { "fqdn": "some-localhost.awesome-domain.com", "quantumResistance": false, "quantumResistancePermissive": false, + "mlkemEnabled": false, + "mlkemStrict": false, "networks": [ "10.10.0.0/24" ], @@ -494,6 +496,8 @@ wireguardPort: 51820 fqdn: some-localhost.awesome-domain.com quantumResistance: false quantumResistancePermissive: false +mlkemEnabled: false +mlkemStrict: false networks: - 10.10.0.0/24 forwardingRules: 0 From aca68853d894596e2037eeaa4e1a192dc4df7bec Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 28 Aug 2026 16:09:42 +0200 Subject: [PATCH 068/119] [client] pqkem: enforce exchange roles and re-bootstrap stale signal offers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two convergence bugs surfaced by the security review: - Role guard (finding B): processOffer accepted an offer even when we are the KEM initiator for the peer, and processAnswer accepted an answer when we are the responder. The KEM is unidirectional (initiator offers, responder answers), so a role-violating message is anomalous — a desync, a duplicate, or an injected/spoofed data-path packet. Processing it derived and committed a fresh PSK, overwriting a live one and silently dropping any in-flight exchange (whose retry loop then exited without raising a failure or re-bootstrapping). Reject offers when we are the initiator and answers when we are not; this drops only anomalous traffic and leaves the normal flow untouched. - Re-bootstrap on signal re-negotiation (finding A): SignalOffer was idempotent in stateAwaitingRekey too, replaying the frozen bootstrap offer. After the responder restarted and lost its state it derived a different PSK from fresh material, which our awaitingRekey side then rejected — a permanent desync with no recovery (in strict mode the peer stays blocked). Make the idempotency apply only while a bootstrap is still in flight (awaitingAnswer); once a PSK is derived, a fresh signal offer starts a new exchange so both sides converge. The controller-double-offer case the idempotency guarded is already covered by ShouldSendBootstrapOffer. Reusing the cached offer also reused the same ephemeral keys across exchanges, reducing forward secrecy. Both paths have a failing-without-the-fix regression test. --- client/internal/pqkem/convergence.go | 18 +++++++++++ client/internal/pqkem/convergence_test.go | 21 ++++++++++++ client/internal/pqkem/manager.go | 17 ++++++---- client/internal/pqkem/manager_test.go | 39 +++++++++++++++++++++++ 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index f818590b1d2..98048706b96 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -76,6 +76,16 @@ func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID E func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { m.trace("pqkem: offer received", "peer", remoteID, "exchange", idHex(o.ExchangeID), "acks", idHex(o.AckID)) + // Role guard: only the responder processes offers. The KEM is unidirectional — the + // initiator sends offers, the responder answers — so an offer reaching the initiator + // is anomalous (desync, duplicate, or an injected/spoofed packet). Processing it would + // derive and commit a fresh PSK, overwriting a live one and silently discarding any + // in-flight exchange (whose retry loop then exits without recovery). Drop it. + if m.IsInitiator(remoteID) { + m.trace("pqkem: dropping offer, we are the initiator for this peer (role violation)", "peer", remoteID, "exchange", idHex(o.ExchangeID)) + return nil, nil + } + if o.AckID != (ExchangeID{}) { m.ackConverged(remoteID, o.AckID) } @@ -132,6 +142,14 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { // this exchange. Only valid in stateAwaitingAnswer; advancing the state under the // lock makes a concurrent/duplicate answer bail. func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { + // Role guard: only the initiator processes answers. An answer reaching the responder + // is anomalous (the responder sends answers, it never receives them) — drop it rather + // than let a stray/injected answer disturb the responder's state. + if !m.IsInitiator(remoteID) { + m.trace("pqkem: dropping answer, we are the responder for this peer (role violation)", "peer", remoteID, "answer_for", idHex(a.ExchangeID)) + return nil + } + m.mu.Lock() ex := m.exchanges[remoteID] if ex == nil || ex.id != a.ExchangeID || ex.state != stateAwaitingAnswer { diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index c8f4f1599ba..c2bef61ac03 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -73,3 +73,24 @@ func TestManager_RekeyToleratesKFailures(t *testing.T) { require.NoError(t, err) assert.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond) } + +// TestManager_InitiatorRejectsOfferFromResponderRole verifies the role guard: an offer +// reaching the peer that is the initiator (here dB) must be dropped, not adopted — else a +// stray/injected offer would overwrite the live PSK and silently drop any in-flight +// exchange. dA (the role-responder) crafts an offer and injects it into dB (the initiator). +func TestManager_InitiatorRejectsOfferFromResponderRole(t *testing.T) { + dA, dB, _, wgB, _ := pair(t) // dB ("bbbb") is the initiator, dA ("aaaa") the responder + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + psk1 := wgB.psk("aaaa") + require.NotEqual(t, PSK{}, psk1, "bootstrap established a PSK") + + rogue, err := dA.startExchangeTest("bbbb", false, ExchangeID{}) + require.NoError(t, err) + require.NoError(t, dB.OnDataPathMessage("aaaa", rogue)) + + assert.Equal(t, psk1, wgB.psk("aaaa"), + "the initiator must not adopt a PSK from an offer sent by the role-responder") +} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 22123d2cec9..a58f9f8effb 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -295,13 +295,16 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { m.mu.Unlock() return nil, nil // peer does not run the KEM; do not offer (avoids a failure/reoffer loop) } - // Idempotent while a signalling bootstrap is in flight OR already derived a PSK but - // not yet chained a rotation (awaitingRekey): return the SAME offer instead of - // starting a new exchange. This matters when the controller both offers on its own - // guard AND re-offers in response to the responder's offer — without this, the - // second call would start a fresh exchange (a different PSK) and desync the peers. - if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && - (ex.state == stateAwaitingAnswer || ex.state == stateAwaitingRekey) { + // Idempotent ONLY while a signalling bootstrap is still in flight (awaitingAnswer): + // a repeat call returns the SAME offer instead of starting a duplicate exchange. Once + // the PSK is derived (awaitingRekey) a fresh signal offer must NOT reuse the frozen + // exchange: a signal re-negotiation means the remote may have restarted and lost its + // state, so it would Respond with fresh material and derive a different PSK, which our + // still-awaitingRekey side would reject — a permanent desync. Starting a fresh exchange + // re-bootstraps and both sides converge (the responder always adopts the latest). The + // controller-double-offer case the idempotency once guarded is already covered by + // ShouldSendBootstrapOffer (false while any exchange exists). + if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && ex.state == stateAwaitingAnswer { last := ex.lastSent m.mu.Unlock() return last, nil diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index b7a13166ad5..94ab70490c6 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -199,3 +200,41 @@ func TestManager_StopIsIdempotent(t *testing.T) { dA.Stop() dA.Stop() // must not panic or hang } + +// TestManager_SignalOfferRebootstrapsAfterResponderRestart verifies finding A: once a PSK +// is derived (initiator in awaitingRekey), a fresh signalling offer must start a NEW +// exchange, not replay the frozen one. Otherwise, if the responder restarted and lost its +// state, it would derive a different PSK the initiator then rejects — a permanent desync. +func TestManager_SignalOfferRebootstrapsAfterResponderRestart(t *testing.T) { + sw := newSwitch() + wgA, wgB := newFakeWG(), newFakeWG() + dA := NewManager("aaaa", wgA, nil) + dB := NewManager("bbbb", wgB, nil) + dA.Start(&loopback{ep: epA, sw: sw}) + dB.Start(&loopback{ep: epB, sw: sw}) + dA.AddPeer("bbbb", epB) + dB.AddPeer("aaaa", epA) + defer dB.Stop() + + bootstrap(t, dA, dB) + require.Equal(t, wgB.psk("aaaa"), wgA.psk("bbbb"), "bootstrap converged") + + // Responder ("aaaa") restarts: fresh manager, no state. + dA.Stop() + wgA2 := newFakeWG() + dA2 := NewManager("aaaa", wgA2, nil) + dA2.Start(&loopback{ep: epA, sw: sw}) + dA2.AddPeer("bbbb", epB) + defer dA2.Stop() + + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) + answer, err := dA2.SignalOnOffer("bbbb", offer) + require.NoError(t, err) + require.NotNil(t, answer) + require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) + + assert.Equal(t, wgB.psk("aaaa"), wgA2.psk("bbbb"), + "after a responder restart both sides must converge on the same PSK") +} From 9ea3ddcc527edb60fec2a9a6e1211b27baead501 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 28 Aug 2026 16:11:05 +0200 Subject: [PATCH 069/119] [client] pqkem: don't drop a responder's offer when the controller has no re-offer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pqControllerReoffer returned true whenever we are the controller running the KEM, even when ShouldSendBootstrapOffer was false — which is the steady state for every PQ peer past its bootstrap (exchange in awaitingRekey) and for non-capable peers. handleRemoteOffer then returned early, dropping the ICE credentials and relay info carried in the peer's offer, so a responder-initiated reconnect (the normal path under lazy connections) stalled: the controller only recovered on its own guard timing. Return true only when a re-offer was actually sent; otherwise fall through to the normal offer handling (sendAnswer + notifyListeners) so the connection can come up on the retained PSK. A stale PSK still self-heals via the WG watcher, which triggers a fresh signal offer that re-bootstraps. --- client/internal/peer/handshaker.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index e64c9c29955..e01dc47fa6a 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -203,18 +203,27 @@ func (h *Handshaker) handleRemoteAnswer(remoteOfferAnswer OfferAnswer) { // one is in flight are ignored (re-sending on every responder offer would be a runaway). // The re-offer reuses our stable ICE session id, so the peer dedups repeats. // -// Returns true when it took ownership of the offer (the caller must not answer it). +// Returns true ONLY when it actually sent a re-offer and thus took ownership of the peer's +// offer (the caller must not answer it). When there is nothing to re-offer — the peer is +// non-capable, or an exchange is already in flight (e.g. every PQ peer past its bootstrap, +// which is the steady state under lazy connections) — it returns false so the caller keeps +// handling the offer normally (sendAnswer + notifyListeners). Returning true unconditionally +// would drop the ICE credentials and relay info the responder's offer carries, stalling a +// responder-initiated reconnect. func (h *Handshaker) pqControllerReoffer() bool { if h.config.PQ == nil || !isController(h.config) { return false } - if h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { - h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") - if err := h.sendOffer(); err != nil { - h.log.Errorf("failed to send KEM offer in response to peer offer: %s", err) - } - } else { - h.log.Debugf("pqkem: controller received a responder offer but a KEM exchange is already in flight, ignoring") + if !h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { + h.log.Debugf("pqkem: controller received a responder offer with no fresh KEM offer to send, handling it normally") + return false + } + h.log.Debugf("pqkem: controller received a responder offer, replying with our KEM offer instead of an answer") + if err := h.sendOffer(); err != nil { + // The re-offer did not go out; fall through to normal handling rather than + // silently dropping the peer's offer. + h.log.Errorf("failed to send KEM offer in response to peer offer, handling it normally: %s", err) + return false } return true } From e0523efe5a78ab0873e0ca968ae532018d49c658 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 1 Sep 2026 10:37:45 +0200 Subject: [PATCH 070/119] [client] pqkem: make exchange logs discriminate offer/answer, role, transport, kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-exchange lifecycle logs were all at LevelTrace (below the default field level) and did not distinguish a signalling re-bootstrap from a data-path rekey, which made diagnosing exchange activity in the field guesswork (e.g. telling an ICE-retry-driven re-bootstrap storm apart from normal KEM rotation). Promote the two pivotal events — an offer going out and a PSK converging — to LevelDebug and tag every lifecycle log with four dimensions: - msg: offer vs answer (implicit in the message) - role: initiator vs responder for this peer - via: signal vs data-path (threaded into processOffer/processAnswer) - kind: bootstrap (a new connection / reconnect) vs rotation (a rekey) kind is derived from the exchange's AckID (zero = bootstrap) on the responder side and from viaSignal on the initiator side. No behavioural change. --- client/internal/pqkem/convergence.go | 47 +++++++++++++++++++++------- client/internal/pqkem/manager.go | 16 +++++++--- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 98048706b96..e740a16e543 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -11,6 +11,22 @@ import ( // idHex renders an exchange ID for logs. func idHex(id ExchangeID) string { return hex.EncodeToString(id[:]) } +// exchangeKind labels an exchange for logs by whether it acknowledges a prior one: a +// zero AckID is a bootstrap (a new connection's first exchange, or a re-bootstrap after +// a reconnect), a non-zero AckID is a rotation (a rekey chained off the previous PSK). +func exchangeKind(ackID ExchangeID) string { + if ackID == (ExchangeID{}) { + return "bootstrap" + } + return "rotation" +} + +// viaSignal maps the transport a message arrived on / goes out on to a log label. +const ( + viaSignalLabel = "signal" + viaDataPathLabel = "data-path" +) + // pskFingerprint is a short, non-secret digest of a derived PSK: identical on both // peers iff they derived the same key. Logged instead of the raw PSK so debug logs // never carry the actual WireGuard preshared key. @@ -61,11 +77,11 @@ func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID E m.wait.Add(1) go m.initiatorLoop(ctx, remoteID, id) - via := "data-path" + via := viaDataPathLabel if viaSignal { - via = "signal" + via = viaSignalLabel } - m.trace("pqkem: offer sent", "peer", remoteID, "exchange", idHex(id), "acks", idHex(ackID), "via", via) + m.debug("pqkem: sending offer", "peer", remoteID, "exchange", idHex(id), "role", "initiator", "via", via, "kind", exchangeKind(ackID), "acks", idHex(ackID)) return raw, nil } @@ -73,8 +89,9 @@ func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID E // (that offer riding the data path under the freshly adopted key proves it worked), // then derives the PSK for the new offer, commits it optimistically, and returns the // framed answer. A duplicate offer returns the cached answer without re-deriving. -func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { - m.trace("pqkem: offer received", "peer", remoteID, "exchange", idHex(o.ExchangeID), "acks", idHex(o.AckID)) +func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]byte, error) { + kind := exchangeKind(o.AckID) + m.debug("pqkem: offer received", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "via", via, "kind", kind, "acks", idHex(o.AckID)) // Role guard: only the responder processes offers. The KEM is unidirectional — the // initiator sends offers, the responder answers — so an offer reaching the initiator @@ -82,7 +99,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { // derive and commit a fresh PSK, overwriting a live one and silently discarding any // in-flight exchange (whose retry loop then exits without recovery). Drop it. if m.IsInitiator(remoteID) { - m.trace("pqkem: dropping offer, we are the initiator for this peer (role violation)", "peer", remoteID, "exchange", idHex(o.ExchangeID)) + m.debug("pqkem: dropping offer, we are the initiator for this peer (role violation)", "peer", remoteID, "exchange", idHex(o.ExchangeID), "via", via, "kind", kind) return nil, nil } @@ -127,7 +144,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { m.capable[remoteID] = true // a real KEM offer proves the peer runs the exchange m.mu.Unlock() - m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "psk_fp", pskFingerprint(psk)) + m.debug("pqkem: PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "via", via, "kind", kind, "psk_fp", pskFingerprint(psk)) // Commit optimistically so our data path can rekey to the new PSK. if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { @@ -141,12 +158,12 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg) ([]byte, error) { // stateAwaitingRekey; the next offer (chained from OnDataPathRekeyed) will acknowledge // this exchange. Only valid in stateAwaitingAnswer; advancing the state under the // lock makes a concurrent/duplicate answer bail. -func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { +func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) error { // Role guard: only the initiator processes answers. An answer reaching the responder // is anomalous (the responder sends answers, it never receives them) — drop it rather // than let a stray/injected answer disturb the responder's state. if !m.IsInitiator(remoteID) { - m.trace("pqkem: dropping answer, we are the responder for this peer (role violation)", "peer", remoteID, "answer_for", idHex(a.ExchangeID)) + m.debug("pqkem: dropping answer, we are the responder for this peer (role violation)", "peer", remoteID, "answer_for", idHex(a.ExchangeID), "via", via) return nil } @@ -158,15 +175,21 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { haveID = idHex(ex.id) } m.mu.Unlock() - m.trace("pqkem: unexpected answer dropped (inconsistency)", "peer", remoteID, "answer_for", idHex(a.ExchangeID), "have_exchange", haveID) + m.trace("pqkem: unexpected answer dropped (inconsistency)", "peer", remoteID, "answer_for", idHex(a.ExchangeID), "have_exchange", haveID, "via", via) return nil } + // The initiator's exchange kind: a signalling exchange is a bootstrap/re-bootstrap, a + // data-path one is a rotation chained off the previous PSK. + kind := "rotation" + if ex.viaSignal { + kind = "bootstrap" + } ex.state = stateAwaitingRekey init := ex.initiator ex.initiator = nil m.mu.Unlock() - m.trace("pqkem: answer received", "peer", remoteID, "exchange", idHex(a.ExchangeID)) + m.trace("pqkem: answer received", "peer", remoteID, "exchange", idHex(a.ExchangeID), "via", via, "kind", kind) psk, err := init.Finish(a.KEMAnswer, m.binding(remoteID)) if err != nil { @@ -193,7 +216,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg) error { m.capable[remoteID] = true // a real KEM answer proves the peer runs the exchange m.mu.Unlock() - m.trace("pqkem: new PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "psk_fp", pskFingerprint(psk)) + m.debug("pqkem: PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "via", via, "kind", kind, "psk_fp", pskFingerprint(psk)) return m.cbHandler.OnNewPSKReady(remoteID, psk) } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index a58f9f8effb..e49790b43f6 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -190,6 +190,14 @@ func (m *Manager) trace(msg string, args ...any) { m.logger.Log(context.Background(), LevelTrace, msg, args...) } +// debug logs at LevelDebug — visible at the default field log level. Reserved for the +// pivotal exchange events (an offer going out, a PSK converging) tagged with the four +// dimensions that tell them apart: msg (offer/answer), role (initiator/responder), via +// (signal/data-path), and kind (bootstrap for a new connection vs rotation for a rekey). +func (m *Manager) debug(msg string, args ...any) { + m.logger.Log(context.Background(), slog.LevelDebug, msg, args...) +} + // AddPeer registers where a peer's data-path messages are sent and received: its // overlay endpoint (IP:port). This is pure routing and says nothing about capability — // PQ capability is decided solely from the peer's KEM payload (see processOffer / @@ -344,7 +352,7 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) if typ != MsgOffer { return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) } - return m.processOffer(remoteID, msg.(*OfferMsg)) + return m.processOffer(remoteID, msg.(*OfferMsg), viaSignalLabel) } // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. @@ -357,7 +365,7 @@ func (m *Manager) SignalOnAnswer(remoteID RemoteID, answer []byte) error { if typ != MsgAnswer { return fmt.Errorf("expected answer from %s, got type %d", remoteID, typ) } - return m.processAnswer(remoteID, msg.(*AnswerMsg)) + return m.processAnswer(remoteID, msg.(*AnswerMsg), viaSignalLabel) } // ---- Data path ---- @@ -385,7 +393,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } switch typ { case MsgOffer: - answer, err := m.processOffer(remoteID, msg.(*OfferMsg)) + answer, err := m.processOffer(remoteID, msg.(*OfferMsg), viaDataPathLabel) if err != nil { return err } @@ -394,7 +402,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } return m.pushDataPath(remoteID, answer) case MsgAnswer: - return m.processAnswer(remoteID, msg.(*AnswerMsg)) + return m.processAnswer(remoteID, msg.(*AnswerMsg), viaSignalLabel) default: return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } From dd07b12f634e718240d40bf9370b7f12e3bf9a9f Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 5 Oct 2026 16:02:22 +0200 Subject: [PATCH 071/119] [client] engine: extract Rosenpass and ML-KEM startup from Start Start tripped SonarCloud's cognitive-complexity gate (S3776) after the ML-KEM block landed. Pull the Rosenpass and ML-KEM manager startup into two helpers that read as one line each from Start. Pure refactor, behavior unchanged. --- client/internal/engine.go | 110 ++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 45 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index c07eeaa724c..ed39f697ebc 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -544,6 +544,67 @@ func waitWithContext(ctx context.Context, wg *sync.WaitGroup) error { } } +// startRosenpassManager brings up Rosenpass when it is enabled and ML-KEM (which takes +// precedence) is not, since the two post-quantum mechanisms are mutually exclusive. +func (e *Engine) startRosenpassManager(publicKey wgtypes.Key) error { + if e.config.RosenpassEnabled && pqkem.Enabled() { + log.Warnf("rosenpass and ML-KEM post-quantum are mutually exclusive; ML-KEM is enabled, so rosenpass is disabled") + } + if !e.config.RosenpassEnabled || pqkem.Enabled() { + return nil + } + + log.Infof("rosenpass is enabled") + if e.config.RosenpassPermissive { + log.Infof("running rosenpass in permissive mode") + } else { + log.Infof("running rosenpass in strict mode") + } + + var err error + e.rpManager, err = rosenpass.NewManager(e.config.PreSharedKey, e.config.WgIfaceName, publicKey) + if err != nil { + return fmt.Errorf("create rosenpass manager: %w", err) + } + if err := e.rpManager.Run(); err != nil { + return fmt.Errorf("run rosenpass manager: %w", err) + } + return nil +} + +// startPQKEMManager brings up the ML-KEM post-quantum PSK exchange when NB_ENABLE_PQ_MLKEM +// is set. It binds the dedicated UDP transport on the WG overlay IP, so the interface must +// already be up. In strict mode a bind failure is fatal (fail closed); in opportunistic +// mode it is logged and the exchange stays disabled. +func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { + if !pqkem.Enabled() { + return nil + } + tr, err := newPQTransport(e.config.WgAddr.IP) + if err != nil { + if pqkem.Strict() { + return fmt.Errorf("pqkem: strict mode enabled but transport bind failed: %w", err) + } + log.Errorf("pqkem: transport bind failed, exchange disabled: %v", err) + return nil + } + cbHandler := pqCallbackHandler{ + wg: e.wgInterface, + // On a persistent rekey failure, re-bootstrap the KEM over Signal: a fresh + // signalling offer starts a new exchange that overwrites the stalled PSK on both + // sides, recovering from a data-path desync. + reoffer: func(remoteKey string) { + if conn, ok := e.peerStore.PeerConn(remoteKey); ok { + conn.RequestReoffer() + } + }, + } + e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger()) + e.pqkemManager.Start(tr) + log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) + return nil +} + // Start creates a new WireGuard tunnel interface and listens to events from Signal and Management services // Connections to remote peers are not established here. // However, they will be established once an event with a list of peers to connect to will be received from Management Service @@ -593,24 +654,8 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) publicKey := e.config.WgPrivateKey.PublicKey() e.flowManager = netflow.NewManager(e.wgInterface, publicKey[:], e.statusRecorder) - // Rosenpass and ML-KEM are mutually exclusive. ML-KEM (NB_ENABLE_PQ_MLKEM) takes precedence - if e.config.RosenpassEnabled && pqkem.Enabled() { - log.Warnf("rosenpass and ML-KEM post-quantum are mutually exclusive; ML-KEM is enabled, so rosenpass is disabled") - } - if e.config.RosenpassEnabled && !pqkem.Enabled() { - log.Infof("rosenpass is enabled") - if e.config.RosenpassPermissive { - log.Infof("running rosenpass in permissive mode") - } else { - log.Infof("running rosenpass in strict mode") - } - e.rpManager, err = rosenpass.NewManager(e.config.PreSharedKey, e.config.WgIfaceName, publicKey) - if err != nil { - return fmt.Errorf("create rosenpass manager: %w", err) - } - if err := e.rpManager.Run(); err != nil { - return fmt.Errorf("run rosenpass manager: %w", err) - } + if err := e.startRosenpassManager(publicKey); err != nil { + return err } e.stateManager.Start() @@ -683,33 +728,8 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) e.rpManager.SetInterface(e.wgInterface) } - // Start the ML-KEM PQ manager after the interface is up so its dedicated UDP - // transport can bind on the WG overlay IP. - if pqkem.Enabled() { - tr, pqErr := newPQTransport(e.config.WgAddr.IP) - if pqErr != nil { - // In strict mode the peer must fail closed; silently continuing without the PQ - // exchange would hand out classic tunnels, so treat the bind failure as fatal. - if pqkem.Strict() { - return fmt.Errorf("pqkem: strict mode enabled but transport bind failed: %w", pqErr) - } - log.Errorf("pqkem: transport bind failed, exchange disabled: %v", pqErr) - } else { - cbHandler := pqCallbackHandler{ - wg: e.wgInterface, - // On a persistent rekey failure, re-bootstrap the KEM over Signal: a - // fresh signalling offer starts a new exchange that overwrites the - // stalled PSK on both sides, recovering from a data-path desync. - reoffer: func(remoteKey string) { - if conn, ok := e.peerStore.PeerConn(remoteKey); ok { - conn.RequestReoffer() - } - }, - } - e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger()) - e.pqkemManager.Start(tr) - log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) - } + if err := e.startPQKEMManager(publicKey); err != nil { + return err } // if inbound conns are blocked there is no need to create the ACL manager From 89ec00eb20f2390032b18de34fb306d66e1f80c8 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 5 Oct 2026 16:02:33 +0200 Subject: [PATCH 072/119] [client] pqkem: address review findings on convergence, strict parsing and status Bot review (CodeRabbit, cubic) on PR #7098 surfaced several real defects: - Strict() parsed the raw env value instead of the normalized one, so a mixed-case NB_PQ_MLKEM_STRICT such as "tRuE" silently disabled fail-closed mode. Parse the lower-cased value, matching Enabled(). - OnDataPathMessage labeled a data-path answer as "signal" in logs, mislabeling every rotation answer. Use the data-path label. - A failed sendAnswer aborted the whole connection setup, skipping the relay/ICE listeners; a transient signalling failure now still brings the local transport up (the peer retries the answer). - processOffer left the reserved exchange slot in place when Respond failed, so every retransmission of that offer was dropped forever. Clear the reservation on a pre-commit error so a retry can derive again. - Dropped a no-op time.Since that implied a convergence-latency metric that was never recorded, and the now-unused startedAt field. - The strict-mode status line asserted active blocking even for a peer that is simply offline; reword it to state that no PSK is established yet. - conn_test used t.Fatalf for conditions under test; use assert. --- client/internal/peer/conn_test.go | 8 ++------ client/internal/peer/handshaker.go | 5 +++-- client/internal/pqkem/convergence.go | 12 +++++++++--- client/internal/pqkem/env.go | 7 ++++--- client/internal/pqkem/manager.go | 3 +-- client/status/status.go | 2 +- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index c6e82c06e1e..0092cf5035d 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -296,16 +296,12 @@ func TestConn_presharedKey_RosenpassManaged(t *testing.T) { // When Rosenpass has already initialized the PSK for this peer, // presharedKey must return nil to avoid UpdatePeer overwriting it. conn.rosenpassInitializedPresharedKeyValidator = func(peerKey string) bool { return true } - if k := conn.presharedKey([]byte("remote"), nil); k != nil { - t.Fatalf("expected nil presharedKey when Rosenpass manages PSK, got %v", k) - } + assert.Nil(t, conn.presharedKey([]byte("remote"), nil), "expected nil presharedKey when Rosenpass manages PSK") // When Rosenpass hasn't taken over yet, presharedKey should provide // a non-nil initial key (deterministic or from NetBird PSK). conn.rosenpassInitializedPresharedKeyValidator = func(peerKey string) bool { return false } - if k := conn.presharedKey([]byte("remote"), nil); k == nil { - t.Fatalf("expected non-nil presharedKey before Rosenpass manages PSK") - } + assert.NotNil(t, conn.presharedKey([]byte("remote"), nil), "expected non-nil presharedKey before Rosenpass manages PSK") } func newWGTimeoutTestConn(rosenpassEnabled bool, disconnected *[]string) *Conn { diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index deb1f0a8d88..fe8d1d0ecb0 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -171,10 +171,11 @@ func (h *Handshaker) handleRemoteOffer(remoteOfferAnswer OfferAnswer) { // Derive+store the KEM PSK (inside sendAnswer's AnswerPayload) BEFORE bringing up the // connection: the relay/ICE workers configure the WG endpoint, which pulls the PSK // for the first handshake. Notifying them first would race the KEM exchange and hand - // the first handshake a not-yet-derived key. + // the first handshake a not-yet-derived key. A failure to signal the answer is + // transient (the peer retries), so still bring the local transport up over relay/ICE + // instead of aborting the whole setup. if err := h.sendAnswer(&remoteOfferAnswer); err != nil { h.log.Errorf("failed to send remote offer confirmation: %s", err) - return } h.notifyListeners(&remoteOfferAnswer) } diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index e740a16e543..9c8d893a0c0 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -67,7 +67,6 @@ func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID E m.exchanges[remoteID] = &exchangeCtl{ id: id, state: stateAwaitingAnswer, - startedAt: time.Now(), cancel: cancel, lastSent: raw, initiator: init, @@ -118,11 +117,19 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by return last, nil } // Reserve the slot so a concurrent duplicate offer bails. - m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, startedAt: time.Now()} + m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved} m.mu.Unlock() answerBytes, psk, err := Respond(o.KEMOffer, m.binding(remoteID)) if err != nil { + // Respond failed before the PSK was committed: clear the reservation so a + // retransmission of this offer (a transient or malformed first packet) can retry + // instead of hitting the stuck reserved slot forever. + m.mu.Lock() + if cur := m.exchanges[remoteID]; cur != nil && cur.id == o.ExchangeID && cur.state == stateReserved { + delete(m.exchanges, remoteID) + } + m.mu.Unlock() return nil, err } raw, err := (&AnswerMsg{ExchangeID: o.ExchangeID, KEMAnswer: answerBytes}).Encode() @@ -235,7 +242,6 @@ func (m *Manager) ackConverged(remoteID RemoteID, ackID ExchangeID) { delete(m.exchanges, remoteID) m.established[remoteID] = true m.failures[remoteID] = 0 - _ = time.Since(ex.startedAt) // convergence latency (metrics hook, later step) m.mu.Unlock() m.trace("pqkem: previous exchange confirmed by ack", "peer", remoteID, "exchange", idHex(ackID)) diff --git a/client/internal/pqkem/env.go b/client/internal/pqkem/env.go index 6eaa9e1291b..19b5bf76f2b 100644 --- a/client/internal/pqkem/env.go +++ b/client/internal/pqkem/env.go @@ -43,15 +43,16 @@ const EnvStrict = "NB_PQ_MLKEM_STRICT" // Strict reports whether strict (fail-closed) mode is enabled via the environment. // An empty or unrecognized value is treated as disabled (opportunistic). func Strict() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv(EnvStrict))) { + raw := strings.ToLower(strings.TrimSpace(os.Getenv(EnvStrict))) + switch raw { case "on": return true case "", "off": return false } - enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(EnvStrict))) + enabled, err := strconv.ParseBool(raw) if err != nil { - log.Warnf("failed to parse %s value %q: %v", EnvStrict, os.Getenv(EnvStrict), err) + log.Warnf("failed to parse %s value %q: %v", EnvStrict, raw, err) return false } return enabled diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index e49790b43f6..7e4b411d2bc 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -80,7 +80,6 @@ const ( type exchangeCtl struct { id ExchangeID state exchangeState - startedAt time.Time cancel context.CancelFunc lastSent []byte initiator *Initiator @@ -402,7 +401,7 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { } return m.pushDataPath(remoteID, answer) case MsgAnswer: - return m.processAnswer(remoteID, msg.(*AnswerMsg), viaSignalLabel) + return m.processAnswer(remoteID, msg.(*AnswerMsg), viaDataPathLabel) default: return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } diff --git a/client/status/status.go b/client/status/status.go index e5c83c995a7..96527fa048f 100644 --- a/client/status/status.go +++ b/client/status/status.go @@ -793,7 +793,7 @@ func peerQuantumResistanceStatus(established, rosenpassEnabled, rosenpassPermiss } switch { case mlkemEnabled && mlkemStrict: - return "false (ML-KEM strict: blocking peer traffic until the exchange converges)" + return "false (ML-KEM strict: no post-quantum PSK established for this peer yet)" case mlkemEnabled: return "false (ML-KEM: not converged yet, or peer does not run the exchange)" case rosenpassEnabled && rosenpassPermissive: From 9af27883df0dab6713e7629aacf97269bec7edab Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:22:59 +0200 Subject: [PATCH 073/119] [client] pqkem: correct the convergence-deadline comment The loop declares failure on the tick after the last retransmit, so the deadline is (MaxRetries+1) * RetryInterval, not MaxRetries * RetryInterval. Found in cubic review on #7098 (client/internal/pqkem/manager.go:19). --- client/internal/pqkem/manager.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 7e4b411d2bc..2ebfbc154d5 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -14,8 +14,9 @@ const ( // DefaultRetryInterval is how often the initiator retransmits its outstanding // data-path offer while awaiting the answer. DefaultRetryInterval = 2 * time.Second - // DefaultMaxRetries bounds how many ticks an exchange may run before it is - // declared failed. The convergence deadline is thus MaxRetries * RetryInterval. + // DefaultMaxRetries is how many times the initiator retransmits its outstanding + // offer while awaiting the answer. Failure is declared on the tick after the last + // retransmit, so the convergence deadline is (MaxRetries+1) * RetryInterval. DefaultMaxRetries = 10 // DefaultMaxRekeyFailures is how many consecutive rekey (non-initial) failures // are tolerated before OnRekeyFailed. The initial exchange fails immediately. From cc2d43ee0942b3c10c519818eb28119981b3a234 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:23:22 +0200 Subject: [PATCH 074/119] [client] pqkem: document the real responder callback timing OnNewPSKReady's doc claimed the responder fires it "on receiving the confirm", but there is no confirm round: the responder fires it right after deriving the PSK from the offer and before sending the answer. Say so, so a host does not assume the peer has already confirmed the key. Found in cubic review on #7098 (client/internal/pqkem/callbacks.go:11). --- client/internal/pqkem/callbacks.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index 3302e2e66f4..3e22a04d464 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -6,9 +6,11 @@ package pqkem // the KEM code be extracted as a standalone library. type CallbackHandler interface { // OnNewPSKReady fires when a fresh post-quantum PSK has been derived for a peer - // and must be programmed into the consumer's secure channel. It is invoked at - // the commit point of each side: the initiator on receiving the answer, the - // responder on receiving the confirm. + // and must be programmed into the consumer's secure channel. The protocol has no + // explicit confirm: the initiator fires it on receiving the answer, the responder + // fires it right after deriving the PSK from the offer and before sending that + // answer. A fired callback therefore means the key is derived locally, not that the + // peer has confirmed it — the next offer is the later acknowledgement. OnNewPSKReady(remoteID RemoteID, psk PSK) error // OnRekeyFailed fires when an exchange fails to converge within the allotted From 9041e9a3eef64bc278da85d58738b3d7c648a7ba Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:23:34 +0200 Subject: [PATCH 075/119] [client] pqkem: document rekey-failure recovery as host-defined The OnRekeyFailed contract told the host to tear the peer connection down, but the shipped adapter re-bootstraps over signalling and keeps the previous PSK. State that recovery is host-defined so another implementation does not perform an unnecessary teardown. Found in cubic review on #7098 (client/internal/pqkem/callbacks.go:15). --- client/internal/pqkem/callbacks.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index 3e22a04d464..2b7c3b1c651 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -13,8 +13,10 @@ type CallbackHandler interface { // peer has confirmed it — the next offer is the later acknowledgement. OnNewPSKReady(remoteID RemoteID, psk PSK) error - // OnRekeyFailed fires when an exchange fails to converge within the allotted - // time. The host should tear the peer connection down so it re-establishes, and - // log a WARN. The library reports the event; it does not dictate the reaction. + // OnRekeyFailed fires when an exchange fails to converge within the allotted time. + // Recovery is host-defined: the library reports the event and does not dictate the + // reaction. The shipped NetBird host, for instance, re-bootstraps the KEM over + // signalling and keeps the tunnel on its previous PSK rather than tearing the + // connection down. OnRekeyFailed(remoteID RemoteID) error } From 5341ab0a7bc7d2cf4679a3af4db17e6b450350a7 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:25:03 +0200 Subject: [PATCH 076/119] [client] pqkem: commit the initiator PSK only after Finish succeeds processAnswer advanced the exchange to stateAwaitingRekey before init.Finish derived the PSK. A rekey firing in that window chained the next offer off an exchange whose key was not yet committed, letting the responder commit PSK N+1 before this side committed PSK N; a lost next answer then left the two peers on different PSKs. Move to an intermediate stateFinishing while deriving (OnDataPathRekeyed chains only on stateAwaitingRekey, so it will not chain mid-derive), and advance to stateAwaitingRekey only after Finish succeeds and the exchange is still current. Found in cubic review on #7098 (client/internal/pqkem/convergence.go:194). --- client/internal/pqkem/convergence.go | 22 ++++++++++++++++------ client/internal/pqkem/manager.go | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 9c8d893a0c0..89455279708 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -191,7 +191,11 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err if ex.viaSignal { kind = "bootstrap" } - ex.state = stateAwaitingRekey + // Move to an intermediate state while the PSK is being derived: the exchange is not + // committed yet, so OnDataPathRekeyed must not chain the next rotation off it. If we + // advanced straight to stateAwaitingRekey, a rekey firing before Finish returns would + // let the responder commit PSK N+1 before this side commits PSK N. + ex.state = stateFinishing init := ex.initiator ex.initiator = nil m.mu.Unlock() @@ -200,10 +204,8 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err psk, err := init.Finish(a.KEMAnswer, m.binding(remoteID)) if err != nil { - // The state already advanced to stateAwaitingRekey and the initiator was cleared, - // so initiatorLoop would exit its default branch without registering a failure — - // leaving the peer desynced (the responder committed its PSK in processOffer). - // Drop the exchange and raise the failure so recovery re-bootstraps. + // Finish failed: drop the exchange and raise the failure so recovery re-bootstraps + // (the responder already committed its PSK in processOffer, so the peers are split). m.mu.Lock() if cur := m.exchanges[remoteID]; cur != nil && cur.id == a.ExchangeID { delete(m.exchanges, remoteID) @@ -215,8 +217,16 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err return err } - // The initiator has converged: the responder must have derived the key to answer. + // Commit only if the exchange we were finishing is still current: a supersede (a signal + // re-bootstrap) could have replaced it while Finish ran, in which case this PSK is stale. m.mu.Lock() + cur := m.exchanges[remoteID] + if cur == nil || cur.id != a.ExchangeID || cur.state != stateFinishing { + m.mu.Unlock() + m.trace("pqkem: exchange superseded during finish, dropping PSK", "peer", remoteID, "exchange", idHex(a.ExchangeID)) + return nil + } + cur.state = stateAwaitingRekey m.established[remoteID] = true m.failures[remoteID] = 0 m.psks[remoteID] = psk diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 2ebfbc154d5..c5cdad09cc0 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -67,6 +67,7 @@ type exchangeState uint8 const ( stateReserved exchangeState = iota // responder: deriving the answer stateAwaitingAnswer // initiator: offer sent, awaiting the answer + stateFinishing // initiator: answer in, deriving the PSK (not yet committed) stateAwaitingRekey // initiator: PSK derived+set, awaiting OnDataPathRekeyed to chain the next offer stateAwaitingAck // responder: answer sent, awaiting the next offer that acks this exchange ) From d0f7e2ac7a1906950ce02ba983185537e48b7cce Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:26:57 +0200 Subject: [PATCH 077/119] [client] pqkem: route a failed PSK application through convergence recovery Both sides recorded the exchange as converged and then called OnNewPSKReady, returning its error but leaving the "converged" state in place. A host failure to program the PSK therefore left the peers split between the old data-path key and the new stored value, with no failure raised and no re-bootstrap. Treat applying the PSK as part of the commit. On the initiator, a failed OnNewPSKReady now drops the exchange and raises the failure so recovery re-bootstraps. On the responder, it drops the exchange and withholds the answer, so the initiator times out and re-bootstraps rather than converging on a key the responder could not apply. Found in cubic review on #7098 (client/internal/pqkem/convergence.go:228). --- client/internal/pqkem/convergence.go | 30 ++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 89455279708..fe7d309a646 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -153,8 +153,19 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by m.debug("pqkem: PSK derived", "peer", remoteID, "exchange", idHex(o.ExchangeID), "role", "responder", "via", via, "kind", kind, "psk_fp", pskFingerprint(psk)) - // Commit optimistically so our data path can rekey to the new PSK. + // Commit optimistically so our data path can rekey to the new PSK. Applying the PSK is + // part of the commit: if the host can't program it, drop the exchange and do NOT send + // the answer, so the initiator times out and re-bootstraps instead of converging on a + // key we could not apply. Route it through the failure path. if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { + m.mu.Lock() + if c := m.exchanges[remoteID]; c != nil && c.id == o.ExchangeID { + delete(m.exchanges, remoteID) + } + initial := !m.established[remoteID] + fail := m.registerFailureLocked(remoteID) + m.mu.Unlock() + m.raiseFailure(remoteID, fail, initial) return nil, err } m.trace("pqkem: answer sent", "peer", remoteID, "exchange", idHex(o.ExchangeID)) @@ -226,6 +237,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err m.trace("pqkem: exchange superseded during finish, dropping PSK", "peer", remoteID, "exchange", idHex(a.ExchangeID)) return nil } + wasEstablished := m.established[remoteID] cur.state = stateAwaitingRekey m.established[remoteID] = true m.failures[remoteID] = 0 @@ -235,7 +247,21 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err m.debug("pqkem: PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "via", via, "kind", kind, "psk_fp", pskFingerprint(psk)) - return m.cbHandler.OnNewPSKReady(remoteID, psk) + if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { + // Applying the PSK is part of the commit: if the host fails to program it, the + // exchange is not really converged. Drop it and route the failure through recovery + // so a re-bootstrap re-derives and re-applies, instead of leaving the peer parked + // in awaitingRekey on a key the data path never adopted. + m.mu.Lock() + if c := m.exchanges[remoteID]; c != nil && c.id == a.ExchangeID { + delete(m.exchanges, remoteID) + } + fail := m.registerFailureLocked(remoteID) + m.mu.Unlock() + m.raiseFailure(remoteID, fail, !wasEstablished) + return err + } + return nil } // ackConverged (responder) records convergence of the exchange named by ackID: a From 1182239faaea3348518bc817c8957a9e004fd388 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:30:52 +0200 Subject: [PATCH 078/119] [client] pqkem: carry a generation on PSK callbacks to drop stale applies OnNewPSKReady could be applied out of order: two exchanges for a peer can derive concurrently (one over signal, one over the data path), and the callbacks run outside the manager lock, so an older exchange's apply could land after a newer one and restore a stale WireGuard PSK, splitting the tunnel. Give each exchange a per-peer monotonic generation, assigned under the lock at creation so a later exchange always carries a higher one, and pass it to OnNewPSKReady. The host adapter records the newest generation applied per peer and drops any callback that is not newer, with the check-and-record atomic so the slow SetPresharedKey call stays off that lock. Found in cubic review on #7098 (client/internal/pqkem/callbacks.go:12). --- client/internal/engine.go | 3 +- client/internal/pqkem/callbacks.go | 7 ++- client/internal/pqkem/convergence.go | 9 ++-- client/internal/pqkem/manager.go | 12 ++++++ client/internal/pqkem/manager_test.go | 2 +- client/internal/pqkem_adapter.go | 41 ++++++++++++++++-- client/internal/pqkem_adapter_test.go | 62 ++++++++++++--------------- 7 files changed, 93 insertions(+), 43 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index a52b561ec21..d3becfb04ff 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -589,7 +589,8 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { return nil } cbHandler := pqCallbackHandler{ - wg: e.wgInterface, + wg: e.wgInterface, + applied: newAppliedGenerations(), // On a persistent rekey failure, re-bootstrap the KEM over Signal: a fresh // signalling offer starts a new exchange that overwrites the stalled PSK on both // sides, recovering from a data-path desync. diff --git a/client/internal/pqkem/callbacks.go b/client/internal/pqkem/callbacks.go index 2b7c3b1c651..f1b681648c6 100644 --- a/client/internal/pqkem/callbacks.go +++ b/client/internal/pqkem/callbacks.go @@ -11,7 +11,12 @@ type CallbackHandler interface { // fires it right after deriving the PSK from the offer and before sending that // answer. A fired callback therefore means the key is derived locally, not that the // peer has confirmed it — the next offer is the later acknowledgement. - OnNewPSKReady(remoteID RemoteID, psk PSK) error + // + // gen is a per-peer monotonic generation: a later exchange always carries a higher + // gen. Callbacks can be applied out of order (two exchanges deriving concurrently), + // so the host must ignore a call whose gen is not newer than the one it last applied + // for that peer, or it may restore an older PSK over a newer one and split the tunnel. + OnNewPSKReady(remoteID RemoteID, gen uint64, psk PSK) error // OnRekeyFailed fires when an exchange fails to converge within the allotted time. // Recovery is host-defined: the library reports the event and does not dictate the diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index fe7d309a646..d7fc65c9931 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -67,6 +67,7 @@ func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID E m.exchanges[remoteID] = &exchangeCtl{ id: id, state: stateAwaitingAnswer, + gen: m.nextGenLocked(remoteID), cancel: cancel, lastSent: raw, initiator: init, @@ -117,7 +118,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by return last, nil } // Reserve the slot so a concurrent duplicate offer bails. - m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved} + m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, gen: m.nextGenLocked(remoteID)} m.mu.Unlock() answerBytes, psk, err := Respond(o.KEMOffer, m.binding(remoteID)) @@ -147,6 +148,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by ex.state = stateAwaitingAck ex.lastSent = raw ex.pendingPSK = psk + gen := ex.gen m.psks[remoteID] = psk m.capable[remoteID] = true // a real KEM offer proves the peer runs the exchange m.mu.Unlock() @@ -157,7 +159,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by // part of the commit: if the host can't program it, drop the exchange and do NOT send // the answer, so the initiator times out and re-bootstraps instead of converging on a // key we could not apply. Route it through the failure path. - if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { + if err := m.cbHandler.OnNewPSKReady(remoteID, gen, psk); err != nil { m.mu.Lock() if c := m.exchanges[remoteID]; c != nil && c.id == o.ExchangeID { delete(m.exchanges, remoteID) @@ -238,6 +240,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err return nil } wasEstablished := m.established[remoteID] + gen := cur.gen cur.state = stateAwaitingRekey m.established[remoteID] = true m.failures[remoteID] = 0 @@ -247,7 +250,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err m.debug("pqkem: PSK derived", "peer", remoteID, "exchange", idHex(a.ExchangeID), "role", "initiator", "via", via, "kind", kind, "psk_fp", pskFingerprint(psk)) - if err := m.cbHandler.OnNewPSKReady(remoteID, psk); err != nil { + if err := m.cbHandler.OnNewPSKReady(remoteID, gen, psk); err != nil { // Applying the PSK is part of the commit: if the host fails to program it, the // exchange is not really converged. Drop it and route the failure through recovery // so a re-bootstrap re-derives and re-applies, instead of leaving the peer parked diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index c5cdad09cc0..76d07a1a7b5 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -82,6 +82,7 @@ const ( type exchangeCtl struct { id ExchangeID state exchangeState + gen uint64 // local, per-peer monotonic generation; lets the host reject a stale PSK apply cancel context.CancelFunc lastSent []byte initiator *Initiator @@ -116,9 +117,19 @@ type Manager struct { capable map[RemoteID]bool // peer runs the KEM (advertised a PQ port); false = known non-capable peerAddrs map[RemoteID]netip.AddrPort // remoteID -> data-path endpoint (send routing) peersByAddr map[netip.AddrPort]RemoteID // reverse: source endpoint -> remoteID (inbound) + genCounter map[RemoteID]uint64 // per-peer monotonic exchange generation wait sync.WaitGroup } +// nextGenLocked returns the next monotonic generation for a peer's exchange. Assumes +// m.mu is held. Because exchanges for a given peer are created under the lock in order, +// a later exchange always carries a higher generation, so the host can drop a PSK apply +// that arrives out of order (generation <= the one it already applied). +func (m *Manager) nextGenLocked(remoteID RemoteID) uint64 { + m.genCounter[remoteID]++ + return m.genCounter[remoteID] +} + // NewManager builds a manager for the local peer identified by its peer identity key // (used for the deterministic initiator role and the identity binding). A nil logger // falls back to slog.Default(). Install the data-path transport with Start. @@ -143,6 +154,7 @@ func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manage capable: make(map[RemoteID]bool), peerAddrs: make(map[RemoteID]netip.AddrPort), peersByAddr: make(map[netip.AddrPort]RemoteID), + genCounter: make(map[RemoteID]uint64), } } diff --git a/client/internal/pqkem/manager_test.go b/client/internal/pqkem/manager_test.go index 94ab70490c6..239037f03c2 100644 --- a/client/internal/pqkem/manager_test.go +++ b/client/internal/pqkem/manager_test.go @@ -74,7 +74,7 @@ func (m *Manager) startExchangeTest(remoteID RemoteID, viaSignal bool, ackID Exc return m.startExchangeLocked(remoteID, viaSignal, ackID) } -func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, psk PSK) error { +func (f *fakeWG) OnNewPSKReady(remoteID RemoteID, _ uint64, psk PSK) error { f.mu.Lock() defer f.mu.Unlock() f.psks[remoteID] = psk diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 8814dc2294c..4bf20145c0a 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -2,6 +2,7 @@ package internal import ( "net/netip" + "sync" "time" log "github.com/sirupsen/logrus" @@ -10,6 +11,31 @@ import ( "github.com/netbirdio/netbird/client/internal/pqkem" ) +// appliedGenerations records the newest PSK generation applied per peer, so an +// out-of-order OnNewPSKReady (two exchanges deriving concurrently) can't restore an +// older PSK over a newer one. +type appliedGenerations struct { + mu sync.Mutex + last map[string]uint64 +} + +func newAppliedGenerations() *appliedGenerations { + return &appliedGenerations{last: make(map[string]uint64)} +} + +// claim reports whether gen is newer than the last applied for the peer; when it is, it +// records gen as the newest and returns true. The check and record are atomic, so the +// slow SetPresharedKey call runs outside this lock. +func (a *appliedGenerations) claim(peer string, gen uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + if gen <= a.last[peer] { + return false + } + a.last[peer] = gen + return true +} + // pqPresharedKeySetter is the subset of the WireGuard interface the ML-KEM callback // needs: programming a peer's preshared key. *iface.WGIface satisfies it. type pqPresharedKeySetter interface { @@ -20,18 +46,27 @@ type pqPresharedKeySetter interface { // engine-side implementation of pqkem.CallbackHandler. type pqCallbackHandler struct { wg pqPresharedKeySetter + // applied drops an out-of-order PSK apply so a stale callback can't overwrite a newer + // key. Nil disables the guard. + applied *appliedGenerations // reoffer re-bootstraps the KEM over Signal for a peer (a fresh signalling offer) // to recover from a persistent data-path rekey failure. Nil disables recovery. reoffer func(remoteKey string) } // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op -// if the peer is not present, mirroring Rosenpass). -func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, psk pqkem.PSK) error { +// if the peer is not present, mirroring Rosenpass). A callback whose generation is not +// newer than the last applied for the peer is dropped, so a reordered apply can't +// restore an older PSK over a newer one. +func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, gen uint64, psk pqkem.PSK) error { + if h.applied != nil && !h.applied.claim(string(remoteID), gen) { + log.Tracef("pqkem: dropping stale PSK apply for peer %s (gen %d)", remoteID, gen) + return nil + } // updateOnly: applies to an already-configured peer (rotation). At bootstrap the // peer is not configured yet, so this is a no-op there and the PSK is instead // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). - log.Tracef("pqkem: programming PSK for peer %s", remoteID) + log.Tracef("pqkem: programming PSK for peer %s (gen %d)", remoteID, gen) return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) } diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index dec3b5943ce..9674075bb02 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -1,37 +1,31 @@ package internal -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/netbirdio/netbird/client/internal/pqkem" -) - -type pqNoopHandler struct{} - -func (pqNoopHandler) OnNewPSKReady(pqkem.RemoteID, pqkem.PSK) error { return nil } -func (pqNoopHandler) OnRekeyFailed(pqkem.RemoteID) error { return nil } - -// TestPQAdapter_CapabilityRoleAware locks the role-aware capability signal: the KEM -// payload only flows initiator-offer -> responder-answer, so an empty message in the -// other direction comes from a perfectly capable peer and must NOT flag it. Only the -// message that should carry material (the answer we receive as initiator) marks a peer -// non-capable when empty. -func TestPQAdapter_CapabilityRoleAware(t *testing.T) { - // localID "zzzz" > "aaaa" => this manager is the KEM initiator for peer "aaaa". - mgr := pqkem.NewManager("zzzz", pqNoopHandler{}, nil) - defer mgr.Stop() - h := pqHandshaker{mgr: mgr} - - // An empty OFFER from our peer is normal here: as the initiator's responder it puts - // its material in the answer, not the offer. It must not disable our offering. - h.AnswerPayload("aaaa", nil) - payload, _ := h.OfferPayload("aaaa") - require.NotNil(t, payload, "an empty offer from a responder-role peer must not mark it non-capable") - - // An empty ANSWER to our offer means the peer does not run the KEM -> stop offering. - h.OnAnswer("aaaa", nil) - payload2, _ := h.OfferPayload("aaaa") - require.Nil(t, payload2, "an empty answer to our offer marks the peer non-capable, so we stop offering") +import "testing" + +// TestAppliedGenerations_DropsStale verifies the out-of-order guard: a generation is +// accepted only when it is strictly newer than the last one applied for that peer, so a +// reordered PSK callback cannot restore an older key over a newer one. +func TestAppliedGenerations_DropsStale(t *testing.T) { + a := newAppliedGenerations() + + if !a.claim("peerA", 1) { + t.Fatal("first generation must be accepted") + } + if !a.claim("peerA", 2) { + t.Fatal("a newer generation must be accepted") + } + if a.claim("peerA", 2) { + t.Fatal("re-applying the same generation must be dropped") + } + if a.claim("peerA", 1) { + t.Fatal("an older generation arriving late must be dropped") + } + if !a.claim("peerA", 3) { + t.Fatal("a newer generation after a dropped stale one must still be accepted") + } + + // Generations are tracked independently per peer. + if !a.claim("peerB", 1) { + t.Fatal("a different peer's first generation must be accepted") + } } From ff634c5330106449646d213d554bbfe76f02d377 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:34:41 +0200 Subject: [PATCH 079/119] [client] pqkem: don't let a superseded rotation offer revert a fresh round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OnDataPathRekeyed builds a chained offer, releases the lock, then sends it. A signal re-bootstrap can supersede that exchange in the gap, after which the stale offer still went out and the responder would reserve and commit an abandoned exchange, splitting the keys. Two guards, no revalidation of message contents: - Sender side: before putting the chain offer on the wire, re-check it is still the current exchange; drop it if a re-bootstrap already replaced it. - Responder side: a legitimate rotation offer acknowledges the exchange we are awaiting-ack on (ackConverged clears it on a match). Drop a chain offer whose ack did not match our current exchange — it is a rotation a newer signal round has superseded. A bootstrap (zero AckID) is authoritative and always wins. A supersede landing in the infinitesimal window after the sender check still leaks one offer, but the responder guard rejects it, so the new round stands. Neither guard retries, so there is no loop. Found in cubic review on #7098 (client/internal/pqkem/manager.go:402). --- client/internal/pqkem/convergence.go | 16 ++++++++++++++-- client/internal/pqkem/manager.go | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index d7fc65c9931..76f21b8af76 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -108,7 +108,8 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by } m.mu.Lock() - if ex := m.exchanges[remoteID]; ex != nil && ex.id == o.ExchangeID { + ex := m.exchanges[remoteID] + if ex != nil && ex.id == o.ExchangeID { state, last := ex.state, ex.lastSent m.mu.Unlock() if state == stateReserved { @@ -117,6 +118,17 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by m.trace("pqkem: duplicate offer, resending cached answer", "peer", remoteID, "exchange", idHex(o.ExchangeID)) return last, nil } + // A data-path chain offer acknowledges the exchange we are awaiting-ack on, and + // ackConverged above deletes that exchange when the ack matches. So if a chain offer + // (non-zero AckID) did NOT match — our current exchange is still here and is a + // different one — it is a rotation a newer signal re-bootstrap has already superseded. + // Drop it, so a late stale offer can't revert us off the new round. A bootstrap + // (zero AckID) is an authoritative fresh start and always proceeds. + if o.AckID != (ExchangeID{}) && ex != nil { + m.mu.Unlock() + m.trace("pqkem: stale chain offer superseded by a newer round, dropping", "peer", remoteID, "exchange", idHex(o.ExchangeID), "acks", idHex(o.AckID)) + return nil, nil + } // Reserve the slot so a concurrent duplicate offer bails. m.exchanges[remoteID] = &exchangeCtl{id: o.ExchangeID, state: stateReserved, gen: m.nextGenLocked(remoteID)} m.mu.Unlock() @@ -139,7 +151,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by } m.mu.Lock() - ex := m.exchanges[remoteID] + ex = m.exchanges[remoteID] if ex == nil || ex.id != o.ExchangeID { m.mu.Unlock() m.trace("pqkem: exchange superseded during respond, dropping answer", "peer", remoteID, "exchange", idHex(o.ExchangeID)) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 76d07a1a7b5..8912a27c550 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -446,6 +446,10 @@ func (m *Manager) OnDataPathRekeyed(remoteID RemoteID, sinceActivity time.Durati // Hold the lock across the awaitingRekey check and the install so two rekey clocks // can't each start a chained exchange for the same peer. offer, err := m.startExchangeLocked(remoteID, false, ex.id) + var chainID ExchangeID + if nc := m.exchanges[remoteID]; nc != nil { + chainID = nc.id + } m.mu.Unlock() m.trace("pqkem: data-path rekey signal", "peer", remoteID, "chaining", true) @@ -453,6 +457,18 @@ func (m *Manager) OnDataPathRekeyed(remoteID RemoteID, sinceActivity time.Durati m.logger.Error("pqkem: chain offer failed to start", "peer", remoteID, "err", err) return } + // A signal re-bootstrap can supersede this chain exchange in the gap between building + // the offer and sending it. Don't put the stale offer on the wire: the responder would + // otherwise reserve and commit an abandoned exchange, splitting the keys. A newer + // signal round always wins. (A supersede landing after this check still leaks one + // harmless offer; the responder's answer for it is rejected as superseded.) + m.mu.Lock() + stillCurrent := m.exchanges[remoteID] != nil && m.exchanges[remoteID].id == chainID + m.mu.Unlock() + if !stillCurrent { + m.trace("pqkem: chain offer superseded before send, dropping", "peer", remoteID, "exchange", idHex(chainID)) + return + } if err := m.pushDataPath(remoteID, offer); err != nil { m.logger.Warn("pqkem: send chain offer failed", "peer", remoteID, "err", err) return From 0747fa039882cff24a1e5d61775efa121ea6e395 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:38:42 +0200 Subject: [PATCH 080/119] [client] pqkem: signal a responder failure distinctly from "no KEM" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capability is signalled in-band by an empty answer: a peer that does not run the KEM answers with no payload. But a responder that runs the KEM and simply fails to build the answer (crypto error, failed PSK program, a future protocol version it cannot parse) also produced an empty payload, so the initiator read it as "peer has no KEM" and marked it non-capable — permanently, since only peer removal clears the flag; in strict mode the peer stayed blocked. Add a payload-less MsgError marker. On a signalling-path failure the responder returns the marker instead of an empty answer, and the initiator treats it as "capable but failed this round": it leaves the exchange to time out and re-bootstrap rather than marking the peer non-capable. A genuinely non-KEM peer still sends no payload at all, so the empty-answer capability signal is unchanged. Found in cubic review on #7098 (client/internal/pqkem_adapter.go:72). --- client/internal/pqkem/capability_test.go | 32 ++++++++++++++++++++++++ client/internal/pqkem/manager.go | 19 +++++++++++++- client/internal/pqkem/message.go | 23 ++++++++++++++++- client/internal/pqkem/message_test.go | 9 +++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index b1c29d582b7..93cf1546b2a 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -64,3 +64,35 @@ func TestManager_EstablishedPeerNotDowngraded(t *testing.T) { assert.True(t, ok, "an established peer must keep its PSK despite a stray zero") assert.NotEqual(t, PSK{}, psk) } + +// TestManager_ErrorMarkerIsBenign verifies finding F's wire half: a responder failure is +// signalled with an error marker (not an empty answer), so the initiator does not read it +// as "peer has no KEM". The marker must be benign — it must not disturb the in-flight +// exchange, which still converges when the real answer arrives. +func TestManager_ErrorMarkerIsBenign(t *testing.T) { + dA, dB, wgA, wgB, _ := pair(t) // dB ("bbbb") initiator, dA ("aaaa") responder + defer dA.Stop() + defer dB.Stop() + + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) + + _, decoded, err := Decode(offer) + require.NoError(t, err) + offerID := decoded.(*OfferMsg).ExchangeID + + // An error marker for the in-flight offer must be accepted without error and must not + // tear the exchange down (unlike an empty answer, which signals non-capability). + marker := (&ErrorMsg{ExchangeID: offerID}).Encode() + require.NoError(t, dB.SignalOnAnswer("aaaa", marker)) + + // The real answer still converges both sides on the same PSK. + answer, err := dA.SignalOnOffer("bbbb", offer) + require.NoError(t, err) + require.NotNil(t, answer) + require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) + + assert.Equal(t, wgB.psk("aaaa"), wgA.psk("bbbb"), "the exchange must still converge after a benign error marker") + assert.NotEqual(t, PSK{}, wgB.psk("aaaa")) +} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 8912a27c550..429d3871675 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -365,7 +365,15 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) if typ != MsgOffer { return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) } - return m.processOffer(remoteID, msg.(*OfferMsg), viaSignalLabel) + o := msg.(*OfferMsg) + answer, err := m.processOffer(remoteID, o, viaSignalLabel) + if err != nil { + // Reply with an error marker rather than an empty answer: the initiator must not + // read our transient failure as "peer does not run the KEM" and mark us + // non-capable. The exchange times out on the initiator and re-bootstraps. + return (&ErrorMsg{ExchangeID: o.ExchangeID}).Encode(), err + } + return answer, nil } // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. @@ -375,6 +383,12 @@ func (m *Manager) SignalOnAnswer(remoteID RemoteID, answer []byte) error { if err != nil { return fmt.Errorf("decode signal answer from %s: %w", remoteID, err) } + if typ == MsgError { + // The responder runs the KEM but failed to answer this offer. It is capable, so + // leave the exchange to time out and re-bootstrap; do not mark it non-capable. + m.trace("pqkem: peer reported an error answering our offer", "peer", remoteID) + return nil + } if typ != MsgAnswer { return fmt.Errorf("expected answer from %s, got type %d", remoteID, typ) } @@ -416,6 +430,9 @@ func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { return m.pushDataPath(remoteID, answer) case MsgAnswer: return m.processAnswer(remoteID, msg.(*AnswerMsg), viaDataPathLabel) + case MsgError: + m.trace("pqkem: peer reported an error over the data path", "peer", remoteID) + return nil default: return fmt.Errorf("unhandled data-path message type %d from %s", typ, remoteID) } diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go index e483e9d72cd..1e1fd6e853f 100644 --- a/client/internal/pqkem/message.go +++ b/client/internal/pqkem/message.go @@ -28,12 +28,17 @@ const ( headerSize = 1 + 1 + ExchangeIDSize ) -// MsgType tags the two message kinds of the exchange. +// MsgType tags the message kinds of the exchange. type MsgType uint8 const ( MsgOffer MsgType = iota + 1 MsgAnswer + // MsgError is a responder's "I run the KEM but could not answer this offer" reply. + // It is a non-empty, payload-less marker so the initiator does not mistake a + // transient failure for an empty "peer does not run the KEM" answer and mark the + // peer non-capable. The exchange still times out and re-bootstraps. + MsgError ) // ExchangeID is the per-exchange correlator. The zero value means "none" (an offer @@ -76,6 +81,17 @@ func (m *AnswerMsg) Encode() ([]byte, error) { return frame(MsgAnswer, m.ExchangeID, m.KEMAnswer), nil } +// ErrorMsg is the payload-less marker a responder returns when it runs the KEM but +// could not answer a given offer, naming the offer's exchange. +type ErrorMsg struct { + ExchangeID ExchangeID +} + +// Encode serialises the error marker (header only, no payload). +func (m *ErrorMsg) Encode() []byte { + return frame(MsgError, m.ExchangeID, nil) +} + // Decode parses a framed message into one of *OfferMsg / *AnswerMsg. func Decode(buf []byte) (MsgType, any, error) { if len(buf) < headerSize { @@ -103,6 +119,11 @@ func Decode(buf []byte) (MsgType, any, error) { return typ, nil, fmt.Errorf("answer payload: got %d, want %d", len(payload), AnswerSize) } return typ, &AnswerMsg{ExchangeID: id, KEMAnswer: payload}, nil + case MsgError: + if len(payload) != 0 { + return typ, nil, fmt.Errorf("error marker payload: got %d, want 0", len(payload)) + } + return typ, &ErrorMsg{ExchangeID: id}, nil default: return typ, nil, fmt.Errorf("unknown message type %d", typ) } diff --git a/client/internal/pqkem/message_test.go b/client/internal/pqkem/message_test.go index 524125e4794..0ce9be69d84 100644 --- a/client/internal/pqkem/message_test.go +++ b/client/internal/pqkem/message_test.go @@ -30,6 +30,15 @@ func TestMessageRoundTrip(t *testing.T) { require.NoError(t, err) require.Equal(t, MsgAnswer, typ) require.Equal(t, answer, decoded.(*AnswerMsg).KEMAnswer) + + // The error marker is a non-empty, payload-less message carrying the exchange id, so + // the initiator can tell a responder failure from an empty "no KEM" answer. + errBytes := (&ErrorMsg{ExchangeID: id}).Encode() + require.NotEmpty(t, errBytes) + typ, decoded, err = Decode(errBytes) + require.NoError(t, err) + require.Equal(t, MsgError, typ) + require.Equal(t, id, decoded.(*ErrorMsg).ExchangeID) } func TestDecodeRejects(t *testing.T) { From 4eba98d561739ef0b56f3015b046362ec5e3ccde Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 7 Oct 2026 13:40:42 +0200 Subject: [PATCH 081/119] [client] pqkem: drain the data-path receive loop before clearing state Transport.Run spawned an untracked goroutine, and Close only closed the socket without waiting for it. An in-flight onInbound could therefore derive and store a PSK after the manager cleared the peer, or call SetPresharedKey while the WireGuard interface was being torn down. Track the receive goroutine and make Close drain it (close the socket to unblock the read, then wait for the callback to return). Manager.Stop now closes and drains the transport before clearing the exchange and PSK maps, and the engine tears WireGuard down only after Stop returns, so no late callback repopulates cleared state or touches a closed device. Found in cubic review on #7098 (client/internal/pqkem_transport.go:61). --- client/internal/pqkem/manager.go | 19 +++++++++++++------ client/internal/pqkem_transport.go | 13 +++++++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 429d3871675..cb92a9b2e2e 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -57,7 +57,8 @@ type Transport interface { // Run starts delivering inbound datagrams as (source endpoint, msg) to onInbound // and returns immediately; it runs until Close. Run(onInbound func(src netip.AddrPort, msg []byte)) - // Close stops delivery and releases the socket. + // Close stops delivery, drains any in-flight onInbound callback, and releases the + // socket, so no callback is still running when Close returns. Close() error } @@ -281,19 +282,25 @@ func (m *Manager) Stop() { // has cancelled here no new Add can race Wait. m.mu.Lock() m.rootCancel() - m.mu.Unlock() - m.wait.Wait() - m.mu.Lock() t := m.transport m.transport = nil - m.exchanges = make(map[RemoteID]*exchangeCtl) - m.psks = make(map[RemoteID]PSK) m.mu.Unlock() + + // Close and drain the data-path receive loop BEFORE clearing peer state: Close waits + // for any in-flight onInbound to return, so no inbound message can derive a PSK into a + // cleared map, and the engine only tears the WireGuard interface down after this Stop + // returns, so no late SetPresharedKey hits a torn-down device. if t != nil { if err := t.Close(); err != nil { m.logger.Warn("pqkem: closing data-path transport", "err", err) } } + m.wait.Wait() + + m.mu.Lock() + m.exchanges = make(map[RemoteID]*exchangeCtl) + m.psks = make(map[RemoteID]PSK) + m.mu.Unlock() } // ---- Signalling channel (host-driven; rides the host's negotiation) ---- diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index 920e655c4ff..7096c9e9d39 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "net/netip" + "sync" log "github.com/sirupsen/logrus" ) @@ -20,6 +21,7 @@ const DefaultPort = 51833 type pqTransport struct { conn *net.UDPConn port int + wg sync.WaitGroup } // newPQTransport binds a UDP socket on the WG overlay IP, preferring DefaultPort and @@ -58,7 +60,9 @@ func (t *pqTransport) LocalPort() int { return t.port } // Run implements pqkem.Transport: the receive loop, delivering each datagram as // (source endpoint, msg). Exits when the socket is closed. func (t *pqTransport) Run(onInbound func(src netip.AddrPort, msg []byte)) { + t.wg.Add(1) go func() { + defer t.wg.Done() buf := make([]byte, 2048) for { n, src, err := t.conn.ReadFromUDPAddrPort(buf) @@ -72,5 +76,10 @@ func (t *pqTransport) Run(onInbound func(src netip.AddrPort, msg []byte)) { }() } -// Close implements pqkem.Transport. -func (t *pqTransport) Close() error { return t.conn.Close() } +// Close implements pqkem.Transport. It closes the socket (unblocking the read) and drains +// the receive loop, so no onInbound callback is still running when Close returns. +func (t *pqTransport) Close() error { + err := t.conn.Close() + t.wg.Wait() + return err +} From 6e0c150fc97785a229900088a0f58f40f5968eca Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:49:59 +0200 Subject: [PATCH 082/119] [client] pqkem: drop the unused exchangeCtl.pendingPSK field The responder wrote pendingPSK but nothing ever read it: the derived key is consumed via m.psks (set at the same time) and the cached answer via lastSent. Remove the field and its write to avoid a misleading store. Found in cubic review on #7098 (client/internal/pqkem/manager.go:90). --- client/internal/pqkem/convergence.go | 1 - client/internal/pqkem/manager.go | 22 ++++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 76f21b8af76..76a061bee0e 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -159,7 +159,6 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by } ex.state = stateAwaitingAck ex.lastSent = raw - ex.pendingPSK = psk gen := ex.gen m.psks[remoteID] = psk m.capable[remoteID] = true // a real KEM offer proves the peer runs the exchange diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index cb92a9b2e2e..ffaf02ffd63 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -76,19 +76,17 @@ const ( // exchangeCtl holds all state for one in-flight exchange with a peer, under the // Manager's single lock. state drives every decision. lastSent is the current // data-path retransmit payload (the offer, for the initiator). initiator is the -// ephemeral handle used at Finish; pendingPSK is the responder's derived key. -// viaSignal records that the offer went to the host for the signalling channel, so -// the loop does not retransmit it on the data path. Only the initiator runs a -// retransmit loop, so only it sets cancel. +// ephemeral handle used at Finish. viaSignal records that the offer went to the host +// for the signalling channel, so the loop does not retransmit it on the data path. +// Only the initiator runs a retransmit loop, so only it sets cancel. type exchangeCtl struct { - id ExchangeID - state exchangeState - gen uint64 // local, per-peer monotonic generation; lets the host reject a stale PSK apply - cancel context.CancelFunc - lastSent []byte - initiator *Initiator - pendingPSK PSK - viaSignal bool + id ExchangeID + state exchangeState + gen uint64 // local, per-peer monotonic generation; lets the host reject a stale PSK apply + cancel context.CancelFunc + lastSent []byte + initiator *Initiator + viaSignal bool } // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It From 85eaf4319d5106c36eb8a807f078fbcb3d4526cd Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:50:09 +0200 Subject: [PATCH 083/119] [client] pqkem: mention *ErrorMsg in the Decode doc comment Decode also returns *ErrorMsg for MsgError; list it so callers handling the marker are not misled. Found in cubic review on #7098 (client/internal/pqkem/message.go:95). --- client/internal/pqkem/message.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/pqkem/message.go b/client/internal/pqkem/message.go index 1e1fd6e853f..a28720b993d 100644 --- a/client/internal/pqkem/message.go +++ b/client/internal/pqkem/message.go @@ -92,7 +92,7 @@ func (m *ErrorMsg) Encode() []byte { return frame(MsgError, m.ExchangeID, nil) } -// Decode parses a framed message into one of *OfferMsg / *AnswerMsg. +// Decode parses a framed message into one of *OfferMsg / *AnswerMsg / *ErrorMsg. func Decode(buf []byte) (MsgType, any, error) { if len(buf) < headerSize { return 0, nil, fmt.Errorf("message too short: %d bytes", len(buf)) From 25893d233ce7d1000e4c31dda5681baaab68b18f Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:50:32 +0200 Subject: [PATCH 084/119] [client] status: spell out Rosenpass in the quantum-resistance mechanism line The mechanism summary used "RP strict" / "RP permissive" while every other surface (JSON keys, config, per-peer text) spells out "Rosenpass". Use "Rosenpass strict" / "Rosenpass permissive" for consistency. Found in cubic review on #7098 (client/status/status.go:779). --- client/status/status.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/status/status.go b/client/status/status.go index 96527fa048f..7ec5d2d7ae0 100644 --- a/client/status/status.go +++ b/client/status/status.go @@ -766,7 +766,7 @@ func ToProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { } // quantumResistanceMechanism reports which post-quantum mechanism the local client runs, -// as an enum: "none", "ML-KEM strict", "ML-KEM permissive", "RP strict", "RP permissive". +// as an enum: "none", "ML-KEM strict", "ML-KEM permissive", "Rosenpass strict", "Rosenpass permissive". // ML-KEM and Rosenpass are mutually exclusive, so at most one is active; ML-KEM takes // precedence if somehow both are set. Strict/permissive is the fail-closed vs fail-open mode. func quantumResistanceMechanism(rosenpassEnabled, rosenpassPermissive, mlkemEnabled, mlkemStrict bool) string { @@ -776,9 +776,9 @@ func quantumResistanceMechanism(rosenpassEnabled, rosenpassPermissive, mlkemEnab case mlkemEnabled: return "ML-KEM permissive" case rosenpassEnabled && !rosenpassPermissive: - return "RP strict" + return "Rosenpass strict" case rosenpassEnabled: - return "RP permissive" + return "Rosenpass permissive" default: return "none" } From f660e2e578faeac75f841a08dae193da202b6964 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:50:47 +0200 Subject: [PATCH 085/119] [client] peer: correct the MlkemPort zero-value comment Zero is the normal advertisement for a peer on the default PQ port (the port is omitted when it equals DefaultPort), not a signal that the peer does not run the exchange. Document zero as "assume DefaultPort" and point to the payload / capability state for whether ML-KEM is running. Found in cubic review on #7098 (client/internal/peer/handshaker.go:49). --- client/internal/peer/handshaker.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index fe8d1d0ecb0..e81fc4cf5fd 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -45,8 +45,10 @@ type OfferAnswer struct { // when the peer does not run the ML-KEM PQ exchange. MlkemPayload []byte - // MlkemPort is the peer's ML-KEM PQ service UDP port (bound on its WG overlay - // IP) where data-path rekey messages are sent. Zero when not running the exchange. + // MlkemPort is the peer's ML-KEM PQ service UDP port (bound on its WG overlay IP) + // where data-path rekey messages are sent. Zero means "not explicitly advertised, + // assume DefaultPort" — it does NOT indicate whether the peer runs the exchange; + // that is told by the presence of MlkemPayload / the manager's capability state. MlkemPort uint16 // relay server address From b04c8efccc472397eeb3f13270ea0f3ef8b1cfbf Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:51:05 +0200 Subject: [PATCH 086/119] [client] pqkem: fix the capability-signal wording in a test comment Non-capability is signalled by an empty KEM payload over signalling, not by a missing PQ port. Correct the test's comment. Found in cubic review on #7098 (client/internal/pqkem/capability_test.go:11). --- client/internal/pqkem/capability_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index 93cf1546b2a..af30ff19e77 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -7,9 +7,10 @@ import ( "github.com/stretchr/testify/require" ) -// TestManager_NonCapablePeerNotOffered: a peer known not to run the KEM (it advertised -// no PQ port over signalling) is never offered an exchange, and no failure is raised — -// this is what stops the reoffer storm against non-PQ peers (e.g. Rosenpass peers). +// TestManager_NonCapablePeerNotOffered: a peer known not to run the KEM (it answered our +// offer with an empty KEM payload over signalling) is never offered an exchange, and no +// failure is raised — this is what stops the reoffer storm against non-PQ peers (e.g. +// Rosenpass peers). func TestManager_NonCapablePeerNotOffered(t *testing.T) { wg := newFakeWG() d := NewManager("bbbb", wg, nil) // initiator vs "aaaa" From 2bb557cc5ce2646b6e5f873e99d7b9f3cb678325 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:51:41 +0200 Subject: [PATCH 087/119] [client] pqkem: preserve group scope in the slog-to-logrus handler WithGroup dropped the group name, so grouped attributes lost their namespace and could collide with ungrouped fields. Carry the open-group path as a key prefix and apply it in Handle and WithAttrs; an empty group name stays a no-op per the slog contract. Found in cubic review on #7098 (client/internal/pqkem/env.go:139). --- client/internal/pqkem/env.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/client/internal/pqkem/env.go b/client/internal/pqkem/env.go index 19b5bf76f2b..ea6f4aed8c1 100644 --- a/client/internal/pqkem/env.go +++ b/client/internal/pqkem/env.go @@ -94,6 +94,7 @@ func logLevel() slog.Level { // instead of stdout. Verbosity is gated by EnvLogLevel via logLevel(). type slogToLogrus struct { fields log.Fields + prefix string // open-group path (e.g. "a.b."), prepended to attribute keys } func (h slogToLogrus) Enabled(_ context.Context, level slog.Level) bool { @@ -106,7 +107,7 @@ func (h slogToLogrus) Handle(_ context.Context, r slog.Record) error { fields[k] = v } r.Attrs(func(a slog.Attr) bool { - fields[a.Key] = a.Value.Any() + fields[h.prefix+a.Key] = a.Value.Any() return true }) entry := log.WithFields(fields) @@ -131,9 +132,16 @@ func (h slogToLogrus) WithAttrs(attrs []slog.Attr) slog.Handler { fields[k] = v } for _, a := range attrs { - fields[a.Key] = a.Value.Any() + fields[h.prefix+a.Key] = a.Value.Any() } - return slogToLogrus{fields: fields} + return slogToLogrus{fields: fields, prefix: h.prefix} } -func (h slogToLogrus) WithGroup(_ string) slog.Handler { return h } +// WithGroup opens a nested namespace: subsequent attributes are keyed under the group +// path (joined with "."). An empty name is a no-op, per the slog contract. +func (h slogToLogrus) WithGroup(name string) slog.Handler { + if name == "" { + return h + } + return slogToLogrus{fields: h.fields, prefix: h.prefix + name + "."} +} From 50a0bfcaafb6cef0c77b09b0244457abf63b1c46 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:53:00 +0200 Subject: [PATCH 088/119] [client] pqkem: signal an offer-build failure with a marker, not an empty offer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The error-vs-"no KEM" ambiguity fixed on the answer path also existed on the offer path: when SignalOffer failed to build the bootstrap offer (e.g. an entropy failure in startExchangeLocked), OfferPayload logged it and still sent an empty MlkemPayload. The responder read that as "peer does not run the KEM", marked the initiator non-capable, and answered empty — which made the initiator mark the responder non-capable too, disabling the KEM for the session over one transient error. Return the MsgError marker from SignalOffer on failure, and have the responder's SignalOnOffer answer a received marker with its own marker rather than an empty answer. A genuinely non-KEM peer still sends no payload at all, so the empty capability signal is unchanged. Found in cubic review on #7098 (client/internal/pqkem_adapter.go:130). --- client/internal/pqkem/manager.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index ffaf02ffd63..db1a45c93c6 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -339,7 +339,14 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { // for the same peer can't also start an exchange. bootstrap offer acks nothing. raw, err := m.startExchangeLocked(remoteID, true, ExchangeID{}) m.mu.Unlock() - return raw, err + if err != nil { + // Return an error marker, not an empty offer: an empty MlkemPayload is the "peer + // does not run the KEM" capability signal, so the responder would wrongly mark us + // non-capable (and answer empty, making us mark it non-capable in turn). The marker + // is benign on the far side; the host retries the offer. + return (&ErrorMsg{}).Encode(), err + } + return raw, nil } // ShouldSendBootstrapOffer reports whether we should emit a fresh KEM offer to kick a @@ -367,6 +374,12 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) if err != nil { return nil, fmt.Errorf("decode signal offer from %s: %w", remoteID, err) } + if typ == MsgError { + // The initiator failed to build its offer and sent a marker. Answer with our own + // marker, not an empty answer, so it is not read as "peer does not run the KEM". + m.trace("pqkem: peer reported an error building its offer", "peer", remoteID) + return (&ErrorMsg{}).Encode(), nil + } if typ != MsgOffer { return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) } From de70516045cf57dfc3e12fcf99c39a4efc8518f7 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 09:54:28 +0200 Subject: [PATCH 089/119] [client] pqkem: drain in-flight signal handlers on Stop Stop drained the data-path receive loop but not the host-driven signalling calls (SignalOffer / SignalOnOffer / SignalOnAnswer). One in flight during shutdown could still invoke OnNewPSKReady or OnRekeyFailed after Stop returned, touching a WireGuard interface the engine was tearing down. Register each signalling call in the wait group through beginCall, which refuses once rootCtx is cancelled, mirroring startExchangeLocked's guard so no Add races Stop's Wait. Stop already cancels, drains the transport, then Waits, so the Wait now also covers these calls before state is cleared. Found in cubic review on #7098 (client/internal/pqkem/manager.go:298). --- client/internal/pqkem/manager.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index db1a45c93c6..76a1a54c45b 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -196,6 +196,21 @@ func (m *Manager) PSK(remoteID RemoteID) (PSK, bool) { return psk, ok } +// beginCall registers an in-flight host-driven signalling call in the wait group so Stop +// drains it before clearing state, and refuses once the manager is stopping. It mirrors +// startExchangeLocked's guard: rootCancel runs under m.mu before Stop's Wait, and this +// Adds under the same lock after checking rootCtx, so no Add can race the Wait. The caller +// must defer m.wait.Done() when this returns true. +func (m *Manager) beginCall() bool { + m.mu.Lock() + defer m.mu.Unlock() + if m.rootCtx.Err() != nil { + return false + } + m.wait.Add(1) + return true +} + // trace logs at LevelTrace, the verbose per-exchange lifecycle level gated by // NB_PQ_MLKEM_LOG_LEVEL=trace. func (m *Manager) trace(msg string, args ...any) { @@ -313,6 +328,10 @@ func (m *Manager) Stop() { // PSK still survives idle in the manager (dropped only on account-level peer removal), // so a pure lazy wake with no re-negotiation reuses it via the conn's WG-config pull. func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { + if !m.beginCall() { + return nil, fmt.Errorf("manager stopping") + } + defer m.wait.Done() if !m.IsInitiator(remoteID) { return nil, nil } @@ -370,6 +389,10 @@ func (m *Manager) ShouldSendBootstrapOffer(remoteID RemoteID) bool { // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and // returns the KEM answer for the host to embed in its outgoing answer. func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) { + if !m.beginCall() { + return nil, fmt.Errorf("manager stopping") + } + defer m.wait.Done() typ, msg, err := Decode(offer) if err != nil { return nil, fmt.Errorf("decode signal offer from %s: %w", remoteID, err) @@ -397,6 +420,10 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) // SignalOnAnswer processes a KEM answer the host extracted from an incoming answer. // There is no reply: the next offer (over the data path) acknowledges this exchange. func (m *Manager) SignalOnAnswer(remoteID RemoteID, answer []byte) error { + if !m.beginCall() { + return fmt.Errorf("manager stopping") + } + defer m.wait.Done() typ, msg, err := Decode(answer) if err != nil { return fmt.Errorf("decode signal answer from %s: %w", remoteID, err) From daf9fbae4b835b265e757607ca620fc48b0b3092 Mon Sep 17 00:00:00 2001 From: riccardom Date: Fri, 9 Oct 2026 18:16:41 +0200 Subject: [PATCH 090/119] [client] pqkem: document SignalOffer's error-marker return contract SignalOffer now returns a non-empty error marker together with a non-nil error when the offer build fails; the host must still send that payload rather than dropping it, or an empty MlkemPayload would be read as the non-capable signal. State this in the doc comment. Found in cubic review on #7098 (client/internal/pqkem/manager.go:366). --- client/internal/pqkem/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 76a1a54c45b..76efa9f3276 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -323,6 +323,10 @@ func (m *Manager) Stop() { // initiator. It is idempotent for an in-flight bootstrap: a repeat call returns the // same offer rather than starting a new exchange. // +// On a build failure it returns a non-nil error AND a non-empty error-marker payload: +// the host must still send that payload, because an empty MlkemPayload is the "peer does +// not run the KEM" capability signal, which would wrongly mark this peer non-capable. +// // A signal re-negotiation always re-bootstraps (fresh exchange): the remote may have // restarted and lost its PSK, so reusing a locally frozen one would desync. The derived // PSK still survives idle in the manager (dropped only on account-level peer removal), From 1152b251495d9ac02e2629ee5cb62497fc874705 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 09:48:18 +0200 Subject: [PATCH 091/119] [client] pqkem: run the data-path transport over a net.PacketConn Preparation for running the ML-KEM exchange on the js/wasm client. There, and in other netstack-mode clients, the overlay IP exists only inside the userspace stack, so the KEM socket has to be opened on the netstack. The netstack returns a *gonet.UDPConn rather than a *net.UDPConn, so the transport can no longer hold the host socket's concrete type. Hold the socket as a net.PacketConn, which both implement, and convert endpoints at the edges. Sources are unmapped explicitly, since a udp4 socket's net.UDPAddr may carry the 16-byte form. No behaviour change: the transport still binds on the host, and the next commit opens it on the netstack. Adds a host round-trip test that also covers the fallback to an ephemeral port. --- client/internal/pqkem_transport.go | 32 +++++++++---- client/internal/pqkem_transport_test.go | 62 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 client/internal/pqkem_transport_test.go diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index 7096c9e9d39..b0901afb7ed 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -19,7 +19,7 @@ const DefaultPort = 51833 // overlay IP. It implements pqkem.Transport — the manager owns the remoteID<->endpoint // routing and drives this socket's lifecycle (Run / Close). type pqTransport struct { - conn *net.UDPConn + conn net.PacketConn port int wg sync.WaitGroup } @@ -36,21 +36,32 @@ func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { // so the transport binds over IPv4. Unmap first so AsSlice() yields 4 bytes for an // IPv4-mapped IPv6 address (a hardcoded "udp4" would otherwise fail on its 16 bytes). overlayIP = overlayIP.Unmap() - ip := net.IP(overlayIP.AsSlice()) - conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: DefaultPort}) + listen := func(port uint16) (net.PacketConn, error) { + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP(overlayIP.AsSlice()), Port: int(port)}) + if err != nil { + return nil, err + } + return conn, nil + } + conn, err := listen(DefaultPort) if err != nil { log.Debugf("pqkem: default port %d unavailable on %s (%v), using an ephemeral port", DefaultPort, overlayIP, err) - conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) + conn, err = listen(0) if err != nil { return nil, fmt.Errorf("bind pqkem udp on overlay %s: %w", overlayIP, err) } } - return &pqTransport{conn: conn, port: conn.LocalAddr().(*net.UDPAddr).Port}, nil + local, ok := conn.LocalAddr().(*net.UDPAddr) + if !ok { + _ = conn.Close() + return nil, fmt.Errorf("pqkem transport bound to a non-UDP address %s", conn.LocalAddr()) + } + return &pqTransport{conn: conn, port: local.Port}, nil } // Send implements pqkem.Transport. func (t *pqTransport) Send(endpoint netip.AddrPort, msg []byte) error { - _, err := t.conn.WriteToUDPAddrPort(msg, endpoint) + _, err := t.conn.WriteTo(msg, net.UDPAddrFromAddrPort(endpoint)) return err } @@ -65,13 +76,18 @@ func (t *pqTransport) Run(onInbound func(src netip.AddrPort, msg []byte)) { defer t.wg.Done() buf := make([]byte, 2048) for { - n, src, err := t.conn.ReadFromUDPAddrPort(buf) + n, addr, err := t.conn.ReadFrom(buf) if err != nil { return } + src, ok := addr.(*net.UDPAddr) + if !ok { + continue + } + ap := src.AddrPort() msg := make([]byte, n) copy(msg, buf[:n]) - onInbound(src, msg) + onInbound(netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()), msg) } }() } diff --git a/client/internal/pqkem_transport_test.go b/client/internal/pqkem_transport_test.go new file mode 100644 index 00000000000..a60063d6559 --- /dev/null +++ b/client/internal/pqkem_transport_test.go @@ -0,0 +1,62 @@ +package internal + +import ( + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type pqInbound struct { + src netip.AddrPort + msg []byte +} + +func runPQTransport(t *testing.T, tr *pqTransport) <-chan pqInbound { + t.Helper() + got := make(chan pqInbound, 1) + tr.Run(func(src netip.AddrPort, msg []byte) { + select { + case got <- pqInbound{src: src, msg: msg}: + default: + } + }) + t.Cleanup(func() { _ = tr.Close() }) + return got +} + +func receivePQ(t *testing.T, got <-chan pqInbound) pqInbound { + t.Helper() + select { + case in := <-got: + return in + case <-time.After(5 * time.Second): + t.Fatal("no datagram received") + return pqInbound{} + } +} + +// TestPQTransport_HostRoundTrip sends a datagram between two host-bound transports. The +// second bind collides with the first on DefaultPort, so it also covers the fallback to +// an ephemeral port. The source must come back as a plain (unmapped) IPv4 endpoint, +// since the manager matches it against registered peer endpoints. +func TestPQTransport_HostRoundTrip(t *testing.T) { + loopback := netip.MustParseAddr("127.0.0.1") + + a, err := newPQTransport(loopback) + require.NoError(t, err) + b, err := newPQTransport(loopback) + require.NoError(t, err) + require.NotEqual(t, a.LocalPort(), b.LocalPort(), "two transports on one IP must get distinct ports") + + runPQTransport(t, a) + gotB := runPQTransport(t, b) + + require.NoError(t, a.Send(netip.AddrPortFrom(loopback, uint16(b.LocalPort())), []byte("offer"))) + + in := receivePQ(t, gotB) + assert.Equal(t, []byte("offer"), in.msg, "payload must arrive intact") + assert.Equal(t, netip.AddrPortFrom(loopback, uint16(a.LocalPort())), in.src, "source must be the sender's unmapped IPv4 endpoint") +} From f669cba522d95da5d12367e0bffa8c9a121cdfef Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 09:49:55 +0200 Subject: [PATCH 092/119] [client] pqkem: bind the data-path transport on the netstack in netstack mode In netstack mode (js/wasm and embedded clients) the overlay IP lives only inside the userspace stack, so binding it on the host failed with "cannot assign requested address". The exchange was then disabled, and in strict mode the engine refused to start at all. Reproduced in the lab with NB_USE_NETSTACK_MODE=true. Open the socket through the interface's netstack when there is one. Its datagrams then ride the tunnel like any other overlay traffic (over the relay's WebSocket in the browser). Host-TUN clients keep binding on the host. --- client/internal/engine.go | 2 +- client/internal/pqkem_transport.go | 21 ++++++++++++--- client/internal/pqkem_transport_test.go | 34 +++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index cd43a44ccc2..88a3db0e7ee 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -598,7 +598,7 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { if !pqkem.Enabled() { return nil } - tr, err := newPQTransport(e.config.WgAddr.IP) + tr, err := newPQTransport(e.config.WgAddr.IP, e.wgInterface.GetNet()) if err != nil { if pqkem.Strict() { return fmt.Errorf("pqkem: strict mode enabled but transport bind failed: %w", err) diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index b0901afb7ed..5dfd088bd6e 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -7,6 +7,7 @@ import ( "sync" log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/tun/netstack" ) // DefaultPort is the preferred UDP port for the ML-KEM data-path service, bound on @@ -25,10 +26,15 @@ type pqTransport struct { } // newPQTransport binds a UDP socket on the WG overlay IP, preferring DefaultPort and -// falling back to an OS-assigned ephemeral port if it is in use. Call it after the WG -// interface is up so the overlay IP is assigned; when the bound port is not -// DefaultPort it must be announced to peers via Body.mlkemPort. -func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { +// falling back to an ephemeral port if it is in use. Call it after the WG interface is +// up so the overlay IP is assigned; when the bound port is not DefaultPort it must be +// announced to peers via Body.mlkemPort. +// +// ns is the interface's netstack when it runs in netstack mode (js/wasm, embedded +// clients): the overlay IP then exists only inside that userspace stack, so the socket +// is opened there and its datagrams ride the tunnel like any other overlay traffic. A +// nil ns binds on the host. +func newPQTransport(overlayIP netip.Addr, ns *netstack.Net) (*pqTransport, error) { if !overlayIP.IsValid() { return nil, fmt.Errorf("invalid overlay IP for pqkem transport") } @@ -37,6 +43,13 @@ func newPQTransport(overlayIP netip.Addr) (*pqTransport, error) { // IPv4-mapped IPv6 address (a hardcoded "udp4" would otherwise fail on its 16 bytes). overlayIP = overlayIP.Unmap() listen := func(port uint16) (net.PacketConn, error) { + if ns != nil { + conn, err := ns.ListenUDPAddrPort(netip.AddrPortFrom(overlayIP, port)) + if err != nil { + return nil, err + } + return conn, nil + } conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP(overlayIP.AsSlice()), Port: int(port)}) if err != nil { return nil, err diff --git a/client/internal/pqkem_transport_test.go b/client/internal/pqkem_transport_test.go index a60063d6559..ffaec70141e 100644 --- a/client/internal/pqkem_transport_test.go +++ b/client/internal/pqkem_transport_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/tun/netstack" ) type pqInbound struct { @@ -45,9 +46,9 @@ func receivePQ(t *testing.T, got <-chan pqInbound) pqInbound { func TestPQTransport_HostRoundTrip(t *testing.T) { loopback := netip.MustParseAddr("127.0.0.1") - a, err := newPQTransport(loopback) + a, err := newPQTransport(loopback, nil) require.NoError(t, err) - b, err := newPQTransport(loopback) + b, err := newPQTransport(loopback, nil) require.NoError(t, err) require.NotEqual(t, a.LocalPort(), b.LocalPort(), "two transports on one IP must get distinct ports") @@ -60,3 +61,32 @@ func TestPQTransport_HostRoundTrip(t *testing.T) { assert.Equal(t, []byte("offer"), in.msg, "payload must arrive intact") assert.Equal(t, netip.AddrPortFrom(loopback, uint16(a.LocalPort())), in.src, "source must be the sender's unmapped IPv4 endpoint") } + +// TestPQTransport_NetstackRoundTrip covers netstack mode (js/wasm and embedded clients), +// where the overlay IP exists only inside the userspace stack. Binding it on the host +// fails, which disabled the exchange (or, in strict mode, stopped the engine from +// starting); the transport must open its socket on the netstack instead. +func TestPQTransport_NetstackRoundTrip(t *testing.T) { + overlay := netip.MustParseAddr("192.0.2.1") + dev, ns, err := netstack.CreateNetTUN([]netip.Addr{overlay}, nil, 1280) + require.NoError(t, err) + t.Cleanup(func() { _ = dev.Close() }) + + _, err = newPQTransport(overlay, nil) + require.Error(t, err, "the netstack-only overlay IP must not be bindable on the host") + + a, err := newPQTransport(overlay, ns) + require.NoError(t, err) + assert.Equal(t, DefaultPort, a.LocalPort(), "the default port must be free on a fresh netstack") + b, err := newPQTransport(overlay, ns) + require.NoError(t, err) + + runPQTransport(t, a) + gotB := runPQTransport(t, b) + + require.NoError(t, a.Send(netip.AddrPortFrom(overlay, uint16(b.LocalPort())), []byte("offer"))) + + in := receivePQ(t, gotB) + assert.Equal(t, []byte("offer"), in.msg, "payload must arrive intact") + assert.Equal(t, netip.AddrPortFrom(overlay, uint16(a.LocalPort())), in.src, "source must be the sender's overlay endpoint") +} From 2a2f7a9bbc759d234021d103d66d659359faa2e6 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 12:05:35 +0200 Subject: [PATCH 093/119] [client] pqkem: report a peer as quantum resistant once its PSK lands The per-peer "Quantum resistance" flag was computed only when the relay or ICE connection came up. When the KEM answer lands after the WireGuard endpoint is configured (the controller answering a crossing responder offer, the common case under lazy connections), the peer kept being reported as not converged while the PSK was programmed and traffic flowed. Reproduced in 4 of 7 fresh lab runs. Mark the peer in the status recorder once OnNewPSKReady has programmed the PSK, and OR that mark into every relay/ICE status update under the recorder's lock. The connection path reads the PQ state before configuring WireGuard and writes the status afterwards, so without the mark its stale "false" would clear the flag again. The mark is dropped with the peer, matching the PSK's lifetime, and the update is best-effort: a peer missing from the recorder never fails the PSK apply. Status-only change; PSK programming and strict gating are unaffected. --- client/internal/engine.go | 1 + client/internal/peer/status.go | 45 ++++++++++++++--- client/internal/peer/status_test.go | 43 ++++++++++++++++ client/internal/pqkem_adapter.go | 21 +++++++- client/internal/pqkem_adapter_test.go | 71 ++++++++++++++++++++++++++- 5 files changed, 173 insertions(+), 8 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 88a3db0e7ee..4e2959e1337 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -608,6 +608,7 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { } cbHandler := pqCallbackHandler{ wg: e.wgInterface, + status: e.statusRecorder, applied: newAppliedGenerations(), // On a persistent rekey failure, re-bootstrap the KEM over Signal: a fresh // signalling offer starts a new exchange that overwrites the stalled PSK on both diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index cf39f1dae3c..96b6d930fa8 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -271,9 +271,12 @@ func (s *StatusChangeSubscription) deliver(snapshot *routerSnapshot, block bool) // every private-service request) don't contend against each other. // Pure read methods take RLock; anything that mutates state takes Lock. type Status struct { - mux sync.RWMutex - muxRelays sync.RWMutex - peers map[string]State + mux sync.RWMutex + muxRelays sync.RWMutex + peers map[string]State + // pqMarked holds the peers with a programmed post-quantum PSK. It survives connection + // updates (see quantumResistantLocked) and is dropped with the peer. + pqMarked map[string]struct{} ipToKey map[string]string activeRoutePeers map[route.HAUniqueID]string changeNotify map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription @@ -338,6 +341,7 @@ type Status struct { func NewRecorder(mgmAddress string) *Status { return &Status{ peers: make(map[string]State), + pqMarked: make(map[string]struct{}), ipToKey: make(map[string]string), activeRoutePeers: make(map[route.HAUniqueID]string), changeNotify: make(map[string]map[string]*StatusChangeSubscription), @@ -440,6 +444,7 @@ func (d *Status) RemovePeer(peerPubKey string) error { } delete(d.peers, peerPubKey) + delete(d.pqMarked, peerPubKey) if mappedKey, exists := d.ipToKey[p.IP]; exists && mappedKey == peerPubKey { delete(d.ipToKey, p.IP) } @@ -471,7 +476,7 @@ func (d *Status) UpdatePeerState(receivedState State) error { peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint peerState.RelayServerAddress = receivedState.RelayServerAddress - peerState.RosenpassEnabled = receivedState.RosenpassEnabled + peerState.RosenpassEnabled = d.quantumResistantLocked(receivedState.PubKey, receivedState.RosenpassEnabled) peerState.clearLatency() } @@ -594,7 +599,7 @@ func (d *Status) UpdatePeerICEState(receivedState State) error { peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint - peerState.RosenpassEnabled = receivedState.RosenpassEnabled + peerState.RosenpassEnabled = d.quantumResistantLocked(receivedState.PubKey, receivedState.RosenpassEnabled) d.peers[receivedState.PubKey] = peerState @@ -631,7 +636,7 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error { peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate peerState.Relayed = receivedState.Relayed peerState.RelayServerAddress = receivedState.RelayServerAddress - peerState.RosenpassEnabled = receivedState.RosenpassEnabled + peerState.RosenpassEnabled = d.quantumResistantLocked(receivedState.PubKey, receivedState.RosenpassEnabled) d.peers[receivedState.PubKey] = peerState @@ -771,6 +776,34 @@ func (d *Status) UpdatePeerFQDN(peerPubKey, fqdn string) error { return nil } +// MarkPeerQuantumResistant flags the peer's tunnel as post-quantum protected. The flag is +// otherwise computed only when the connection comes up, so a PSK that lands after that +// (e.g. the KEM answer arriving after the WireGuard endpoint was configured) needs this +// to show up in the status. The mark persists until the peer is removed. +func (d *Status) MarkPeerQuantumResistant(peerPubKey string) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[peerPubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + d.pqMarked[peerPubKey] = struct{}{} + peerState.RosenpassEnabled = true + d.peers[peerPubKey] = peerState + + return nil +} + +// quantumResistantLocked merges a connection update's verdict with the PQ mark. The +// connection path computes its verdict before configuring WireGuard, so it can write it +// after the PSK was programmed; the mark keeps that stale false from clearing it. +func (d *Status) quantumResistantLocked(peerPubKey string, fromConn bool) bool { + _, marked := d.pqMarked[peerPubKey] + return fromConn || marked +} + // UpdatePeerSSHHostKey updates peer's SSH host key func (d *Status) UpdatePeerSSHHostKey(peerPubKey string, sshHostKey []byte) error { d.mux.Lock() diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index 2d910ccf52c..d0de41fb691 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -573,3 +573,46 @@ func TestActiveRoutePeers(t *testing.T) { assert.False(t, ok) assert.Equal(t, "peerB", active[netB]) } + +// TestStatus_PQMarkSurvivesStaleConnectionUpdate reproduces a lab race: the relay/ICE +// path builds its status update from a PQ state read before the KEM answer landed, and +// writes it after the PSK was programmed and the peer marked quantum resistant. That +// stale "false" must not clear the mark, since the PQ PSK lives until the peer is removed. +func TestStatus_PQMarkSurvivesStaleConnectionUpdate(t *testing.T) { + const key = "eDQmZp9jhQGX7XdYpHGaFSpAouldr3Sbf7MZRcGTGGQ=" + updates := map[string]func(*Status, State) error{ + "relayed": (*Status).UpdatePeerRelayedState, + "ice": (*Status).UpdatePeerICEState, + "state": (*Status).UpdatePeerState, + } + for name, update := range updates { + t.Run(name, func(t *testing.T) { + status := NewRecorder("https://mgm") + require.NoError(t, status.AddPeer(key, "peer.netbird.cloud", "100.64.0.2", "")) + require.NoError(t, status.MarkPeerQuantumResistant(key)) + + require.NoError(t, update(status, State{PubKey: key, ConnStatus: StatusConnected, RosenpassEnabled: false})) + + st, err := status.GetPeer(key) + require.NoError(t, err) + assert.True(t, st.RosenpassEnabled, "a stale connection update must not clear the PQ mark") + }) + } +} + +// TestStatus_PQMarkClearedOnRemove verifies the mark does not outlive the peer: once the +// peer is removed and re-added, a connection update without PQ reports it as such. +func TestStatus_PQMarkClearedOnRemove(t *testing.T) { + const key = "eDQmZp9jhQGX7XdYpHGaFSpAouldr3Sbf7MZRcGTGGQ=" + status := NewRecorder("https://mgm") + require.NoError(t, status.AddPeer(key, "peer.netbird.cloud", "100.64.0.2", "")) + require.NoError(t, status.MarkPeerQuantumResistant(key)) + require.NoError(t, status.RemovePeer(key)) + require.NoError(t, status.AddPeer(key, "peer.netbird.cloud", "100.64.0.2", "")) + + require.NoError(t, status.UpdatePeerRelayedState(State{PubKey: key, ConnStatus: StatusConnected, RosenpassEnabled: false})) + + st, err := status.GetPeer(key) + require.NoError(t, err) + assert.False(t, st.RosenpassEnabled, "a re-added peer must not inherit the removed peer's PQ mark") +} diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 4bf20145c0a..b15370d90d5 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -42,10 +42,19 @@ type pqPresharedKeySetter interface { SetPresharedKey(peerKey string, psk wgtypes.Key, updateOnly bool) error } +// pqStatusMarker is the subset of the status recorder the ML-KEM callback needs: flagging +// a peer as post-quantum protected once its PSK is programmed. *peer.Status satisfies it. +type pqStatusMarker interface { + MarkPeerQuantumResistant(peerPubKey string) error +} + // pqCallbackHandler programs the derived PQ PSK onto the WireGuard peer. It is the // engine-side implementation of pqkem.CallbackHandler. type pqCallbackHandler struct { wg pqPresharedKeySetter + // status reflects a newly programmed PSK in the peer's "Quantum resistance" flag. + // Nil disables it. + status pqStatusMarker // applied drops an out-of-order PSK apply so a stale callback can't overwrite a newer // key. Nil disables the guard. applied *appliedGenerations @@ -67,7 +76,17 @@ func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, gen uint64, ps // peer is not configured yet, so this is a no-op there and the PSK is instead // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). log.Tracef("pqkem: programming PSK for peer %s (gen %d)", remoteID, gen) - return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) + if err := h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true); err != nil { + return err + } + // The status flag is computed when the connection comes up; a PSK landing afterwards + // would otherwise leave the peer reported as not quantum resistant. + if h.status != nil { + if err := h.status.MarkPeerQuantumResistant(string(remoteID)); err != nil { + log.Debugf("pqkem: mark peer %s quantum resistant in status: %v", remoteID, err) + } + } + return nil } // OnRekeyFailed reports a failed PQ (re)key convergence and re-bootstraps the KEM over diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index 9674075bb02..b2e23684e24 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -1,6 +1,75 @@ package internal -import "testing" +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/client/internal/pqkem" +) + +// fakePSKSetter records PSK programming and can be made to fail. +type fakePSKSetter struct { + err error + calls int +} + +func (f *fakePSKSetter) SetPresharedKey(string, wgtypes.Key, bool) error { + f.calls++ + return f.err +} + +const pqTestPeerKey = "eDQmZp9jhQGX7XdYpHGaFSpAouldr3Sbf7MZRcGTGGQ=" + +func newPQTestStatus(t *testing.T) *peer.Status { + t.Helper() + status := peer.NewRecorder("https://mgm") + require.NoError(t, status.AddPeer(pqTestPeerKey, "peer.netbird.cloud", "100.64.0.2", "")) + return status +} + +// TestPQCallbackHandler_MarksPeerQuantumResistant reproduces the stale status flag: when +// the KEM answer lands after the connection was set up, the per-peer "Quantum resistance" +// flag was computed without a PSK and never refreshed. Programming the PSK must flag it. +func TestPQCallbackHandler_MarksPeerQuantumResistant(t *testing.T) { + status := newPQTestStatus(t) + setter := &fakePSKSetter{} + h := pqCallbackHandler{wg: setter, applied: newAppliedGenerations(), status: status} + + require.NoError(t, h.OnNewPSKReady(pqkem.RemoteID(pqTestPeerKey), 1, pqkem.PSK{1})) + + st, err := status.GetPeer(pqTestPeerKey) + require.NoError(t, err) + assert.Equal(t, 1, setter.calls, "the PSK must be programmed once") + assert.True(t, st.RosenpassEnabled, "a programmed PQ PSK must mark the peer quantum resistant") +} + +// TestPQCallbackHandler_FailedApplyKeepsFlag verifies the flag only follows a PSK that was +// actually programmed: a failed SetPresharedKey must leave the peer unflagged. +func TestPQCallbackHandler_FailedApplyKeepsFlag(t *testing.T) { + status := newPQTestStatus(t) + h := pqCallbackHandler{wg: &fakePSKSetter{err: errors.New("boom")}, applied: newAppliedGenerations(), status: status} + + require.Error(t, h.OnNewPSKReady(pqkem.RemoteID(pqTestPeerKey), 1, pqkem.PSK{1})) + + st, err := status.GetPeer(pqTestPeerKey) + require.NoError(t, err) + assert.False(t, st.RosenpassEnabled, "a PSK that failed to program must not mark the peer quantum resistant") +} + +// TestPQCallbackHandler_UnknownStatusPeerDoesNotFailApply verifies the status update is +// best-effort: a peer missing from the status recorder must not fail the PSK apply. +func TestPQCallbackHandler_UnknownStatusPeerDoesNotFailApply(t *testing.T) { + setter := &fakePSKSetter{} + h := pqCallbackHandler{wg: setter, applied: newAppliedGenerations(), status: peer.NewRecorder("https://mgm")} + + assert.NoError(t, h.OnNewPSKReady(pqkem.RemoteID(pqTestPeerKey), 1, pqkem.PSK{1})) + assert.Equal(t, 1, setter.calls, "the PSK must still be programmed") +} // TestAppliedGenerations_DropsStale verifies the out-of-order guard: a generation is // accepted only when it is strictly newer than the last one applied for that peer, so a From a73ce119127e063df647aca4603b1e5954a3021d Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:37:25 +0200 Subject: [PATCH 094/119] [client] pqkem: bind the configured pre-shared key into the PQ PSK --pre-shared-key promises that only peers holding the same key can communicate. With ML-KEM enabled the derived PQ PSK replaces it in WireGuard, but the derivation never saw it: two peers with different configured keys, or only one with a key, still agreed on the same PQ PSK and talked. A permitted peer identity, or a stolen identity key, was enough to bypass the extra secret (security review of #7098, finding F-001). Rosenpass does not have this gap because it seeds its exchange with the configured key. Mix the configured key into the HKDF keying material, so a peer without it derives a different PSK and WireGuard fails closed. It goes through the binding shared by every derivation, so bootstrap and rotation both enforce it. The library takes it as a generic option, keeping it free of WireGuard concepts. --- client/internal/engine.go | 8 +++- client/internal/pqkem/kem.go | 24 ++++++---- client/internal/pqkem/manager.go | 24 ++++++++-- client/internal/pqkem/secret_test.go | 66 ++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 client/internal/pqkem/secret_test.go diff --git a/client/internal/engine.go b/client/internal/engine.go index 4e2959e1337..73fca0893ed 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -619,7 +619,13 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { } }, } - e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger()) + var opts []pqkem.Option + if e.config.PreSharedKey != nil { + // The configured key promises that only peers holding it can communicate. The PQ + // PSK replaces it in WireGuard, so it has to carry that guarantee itself. + opts = append(opts, pqkem.WithPresharedSecret(e.config.PreSharedKey[:])) + } + e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger(), opts...) e.pqkemManager.Start(tr) log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) return nil diff --git a/client/internal/pqkem/kem.go b/client/internal/pqkem/kem.go index 90c638a7a90..54de4dda393 100644 --- a/client/internal/pqkem/kem.go +++ b/client/internal/pqkem/kem.go @@ -15,9 +15,9 @@ // Combiner note: this follows draft-ietf-tls-ecdhe-mlkem for X25519MLKEM768 — on // the wire ML-KEM ‖ X25519 (the draft deliberately reversed the share order for // this group), and ML-KEM_ss ‖ X25519_ss as the KDF input. The PSK is derived with -// HKDF-SHA256 over that hybrid secret, salted with a domain-separation label and -// bound (via the HKDF info) to the full transcript and the canonicalised peer -// identities. +// HKDF-SHA256 over that hybrid secret (plus an optional pre-shared secret both peers +// hold), salted with a domain-separation label and bound (via the HKDF info) to the +// full transcript and the canonicalised peer identities. package pqkem import ( @@ -47,6 +47,10 @@ type PSK [32]byte type Binding struct { LocalID []byte RemoteID []byte + // Secret is an optional pre-shared secret both peers must hold. It is mixed into the + // derivation, so peers holding different secrets (or only one holding one) derive + // different PSKs and the consumer's channel fails closed. + Secret []byte } // Initiator holds the ephemeral secrets between Offer and Finish. @@ -144,10 +148,10 @@ func Respond(offer []byte, b Binding) (answer []byte, psk PSK, err error) { } // derivePSK runs HKDF-SHA256 over the hybrid shared secret (ML-KEM_ss ‖ X25519_ss, -// per draft-ietf-tls-ecdhe-mlkem), salted with the domain-separation label, and binds -// the result — via the HKDF info — to the full transcript (offer ‖ answer) and the -// canonicalised peer identities, so the PSK cannot be transplanted to another peer -// pair or a different exchange. +// per draft-ietf-tls-ecdhe-mlkem) plus the binding's optional pre-shared secret, salted +// with the domain-separation label, and binds the result — via the HKDF info — to the +// full transcript (offer ‖ answer) and the canonicalised peer identities, so the PSK +// cannot be transplanted to another peer pair or a different exchange. func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) (PSK, error) { // A PSK not bound to both peer identities could be transplanted to a different peer // pair, so refuse to derive one from an empty binding. @@ -156,9 +160,13 @@ func derivePSK(ssMLKEM, ssX, offer, answer []byte, b Binding) (PSK, error) { } lo, hi := canonicalPair(b.LocalID, b.RemoteID) - ikm := make([]byte, 0, len(ssMLKEM)+len(ssX)) + // The shared secrets have fixed lengths, so appending the optional pre-shared secret + // after them keeps the input unambiguous. It sits in the keying material rather than + // the info, so a peer without it cannot derive the PSK at all. + ikm := make([]byte, 0, len(ssMLKEM)+len(ssX)+len(b.Secret)) ikm = append(ikm, ssMLKEM...) ikm = append(ikm, ssX...) + ikm = append(ikm, b.Secret...) info := make([]byte, 0, len(offer)+len(answer)+len(lo)+len(hi)) info = append(info, offer...) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 76efa9f3276..958b0474826 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -99,6 +99,7 @@ type Manager struct { localID LocalID cbHandler CallbackHandler logger *slog.Logger + secret []byte // optional pre-shared secret bound into every derivation; immutable retryInterval time.Duration maxRetries int @@ -129,15 +130,28 @@ func (m *Manager) nextGenLocked(remoteID RemoteID) uint64 { return m.genCounter[remoteID] } +// Option configures a Manager at construction. +type Option func(*Manager) + +// WithPresharedSecret binds every derived PSK, bootstrap and rotation alike, to a secret +// both peers must already share. A peer with a different secret, or none, derives a +// different PSK, so the consumer's channel fails closed instead of bypassing the secret. +// An empty secret is the same as no option. +func WithPresharedSecret(secret []byte) Option { + return func(m *Manager) { + m.secret = append([]byte(nil), secret...) + } +} + // NewManager builds a manager for the local peer identified by its peer identity key // (used for the deterministic initiator role and the identity binding). A nil logger // falls back to slog.Default(). Install the data-path transport with Start. -func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manager { +func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger, opts ...Option) *Manager { if logger == nil { logger = slog.Default() } ctx, cancel := context.WithCancel(context.Background()) - return &Manager{ + m := &Manager{ localID: localID, cbHandler: h, logger: logger, @@ -155,6 +169,10 @@ func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger) *Manage peersByAddr: make(map[netip.AddrPort]RemoteID), genCounter: make(map[RemoteID]uint64), } + for _, opt := range opts { + opt(m) + } + return m } // Start installs the data-path transport and begins its inbound delivery. The Manager @@ -566,7 +584,7 @@ func (m *Manager) pushDataPath(remoteID RemoteID, msg []byte) error { } func (m *Manager) binding(remoteID RemoteID) Binding { - return Binding{LocalID: []byte(m.localID), RemoteID: []byte(remoteID)} + return Binding{LocalID: []byte(m.localID), RemoteID: []byte(remoteID), Secret: m.secret} } func newExchangeID() (ExchangeID, error) { diff --git a/client/internal/pqkem/secret_test.go b/client/internal/pqkem/secret_test.go new file mode 100644 index 00000000000..5b1c42c2d4d --- /dev/null +++ b/client/internal/pqkem/secret_test.go @@ -0,0 +1,66 @@ +package pqkem + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// pairWithSecrets builds a wired initiator ("bbbb") / responder ("aaaa") pair whose +// managers hold the given pre-shared secrets (nil = no secret). +func pairWithSecrets(t *testing.T, secretA, secretB []byte) (dA, dB *Manager, wgA, wgB *fakeWG) { + t.Helper() + sw := newSwitch() + wgA, wgB = newFakeWG(), newFakeWG() + dA = NewManager("aaaa", wgA, nil, WithPresharedSecret(secretA)) + dB = NewManager("bbbb", wgB, nil, WithPresharedSecret(secretB)) + dA.Start(&loopback{ep: epA, sw: sw}) + dB.Start(&loopback{ep: epB, sw: sw}) + dA.AddPeer("bbbb", epB) + dB.AddPeer("aaaa", epA) + t.Cleanup(func() { + dA.Stop() + dB.Stop() + }) + return dA, dB, wgA, wgB +} + +// TestManager_PresharedSecretBindsPSK covers the configured-secret contract (a host +// option like WireGuard's --pre-shared-key: only peers holding the same secret may +// communicate). Without binding the secret into the derivation, two peers with +// different secrets derived the same PSK and the secret was silently bypassed. The PSK +// must match only when the secrets match, for the bootstrap and for a rotation alike. +func TestManager_PresharedSecretBindsPSK(t *testing.T) { + secret1 := []byte("configured-secret-one-0123456789") + secret2 := []byte("configured-secret-two-0123456789") + cases := map[string]struct { + a, b []byte + match bool + }{ + "same secret": {a: secret1, b: secret1, match: true}, + "different secrets": {a: secret1, b: secret2, match: false}, + "only the responder has one": {a: secret1, b: nil, match: false}, + "only the initiator has one": {a: nil, b: secret1, match: false}, + "no secret": {a: nil, b: nil, match: true}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + dA, dB, wgA, wgB := pairWithSecrets(t, tc.a, tc.b) + + bootstrap(t, dA, dB) + bootA, bootB := wgA.psk("bbbb"), wgB.psk("aaaa") + require.NotEqual(t, PSK{}, bootA, "responder must derive a bootstrap PSK") + require.NotEqual(t, PSK{}, bootB, "initiator must derive a bootstrap PSK") + assert.Equal(t, tc.match, bootA == bootB, "bootstrap PSKs must match only when the secrets match") + + // Rotation over the data path re-derives the PSK; it must honour the secret too. + dB.OnDataPathRekeyed("aaaa", 0) + require.Eventually(t, func() bool { + return wgA.psk("bbbb") != bootA && wgB.psk("aaaa") != bootB + }, time.Second, 5*time.Millisecond, "the rotation must re-key both peers") + assert.Equal(t, tc.match, wgA.psk("bbbb") == wgB.psk("aaaa"), "rotated PSKs must match only when the secrets match") + }) + } +} From b10fc220f3a286b7687da5c95a9f4381f1292455 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:38:56 +0200 Subject: [PATCH 095/119] [client] pqkem: route PSK commits and data-path sends through helpers Preparation for authenticating data-path messages (security review of #7098, finding F-003). The authentication needs the existing PSK while a new one is being derived, to produce a MAC attached to the offers and their answers that proves they are authentic. So committing a new PSK must keep the previous one, and a reply must be sealed with the PSK that verified the offer rather than the one just committed. Today both operations are inlined at their call sites. Pure refactor, no behaviour change: the two PSK commit sites call commitPSKLocked, and pushDataPath delegates the endpoint lookup and send to sendDataPath. --- client/internal/pqkem/convergence.go | 4 ++-- client/internal/pqkem/manager.go | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 76a061bee0e..f3069542607 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -160,7 +160,7 @@ func (m *Manager) processOffer(remoteID RemoteID, o *OfferMsg, via string) ([]by ex.state = stateAwaitingAck ex.lastSent = raw gen := ex.gen - m.psks[remoteID] = psk + m.commitPSKLocked(remoteID, psk) m.capable[remoteID] = true // a real KEM offer proves the peer runs the exchange m.mu.Unlock() @@ -255,7 +255,7 @@ func (m *Manager) processAnswer(remoteID RemoteID, a *AnswerMsg, via string) err cur.state = stateAwaitingRekey m.established[remoteID] = true m.failures[remoteID] = 0 - m.psks[remoteID] = psk + m.commitPSKLocked(remoteID, psk) m.capable[remoteID] = true // a real KEM answer proves the peer runs the exchange m.mu.Unlock() diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 958b0474826..a779d044302 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -567,9 +567,20 @@ func (m *Manager) OnDataPathDown(remoteID RemoteID) {} // ---- internals ---- -// pushDataPath resolves the peer's endpoint and sends over the data-path transport, -// erroring if the peer is unknown or no transport is set. +// commitPSKLocked records a freshly derived PSK as the peer's current one. Assumes m.mu +// is held. +func (m *Manager) commitPSKLocked(remoteID RemoteID, psk PSK) { + m.psks[remoteID] = psk +} + +// pushDataPath sends a message to the peer over the data-path transport. func (m *Manager) pushDataPath(remoteID RemoteID, msg []byte) error { + return m.sendDataPath(remoteID, msg) +} + +// sendDataPath resolves the peer's endpoint and sends over the data-path transport, +// erroring if the peer is unknown or no transport is set. +func (m *Manager) sendDataPath(remoteID RemoteID, msg []byte) error { m.mu.Lock() ep, ok := m.peerAddrs[remoteID] t := m.transport From 13451079c3c9a22e0442fdac88be3129099de49d Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:42:10 +0200 Subject: [PATCH 096/119] [client] pqkem: authenticate data-path messages under the shared PSK The data-path socket identified a sender only by its source IP and port and trusted that anything reaching the overlay address came through the tunnel. On a host-bound socket that does not hold: Linux delivers a datagram to a local overlay IP from a physical interface too (rp_filter is loose, the firewall's input rules match the tunnel interface), so a LAN-adjacent host spoofing a peer's overlay source could inject a KEM offer. A forged bootstrap offer was adopted outright, desynchronising the two PSKs until a re-bootstrap, and repeating it sustains the outage (security review of #7098, finding F-003). No traffic is disclosed, but the exchange must not accept unauthenticated input. Seal every data-path message with an HMAC keyed from the PSK the two peers share; only a peer that holds it can rotate. A reply is sealed under the key that verified the request, because the responder has already committed the new PSK when it answers. The previous PSK stays valid for verification so an offer retransmitted after that commit is still answered. Bootstraps are refused on the data path entirely: they start without a shared PSK and only signalling can carry them. A rotation between peers whose configured secrets differ is now refused at authentication, so the configured-secret test checks that instead. --- client/internal/pqkem/convergence_test.go | 8 +- client/internal/pqkem/datapath_auth.go | 48 +++++++ client/internal/pqkem/datapath_auth_test.go | 137 ++++++++++++++++++++ client/internal/pqkem/manager.go | 76 +++++++++-- client/internal/pqkem/secret_test.go | 11 +- 5 files changed, 268 insertions(+), 12 deletions(-) create mode 100644 client/internal/pqkem/datapath_auth.go create mode 100644 client/internal/pqkem/datapath_auth_test.go diff --git a/client/internal/pqkem/convergence_test.go b/client/internal/pqkem/convergence_test.go index c2bef61ac03..14c4d2532e8 100644 --- a/client/internal/pqkem/convergence_test.go +++ b/client/internal/pqkem/convergence_test.go @@ -87,9 +87,13 @@ func TestManager_InitiatorRejectsOfferFromResponderRole(t *testing.T) { psk1 := wgB.psk("aaaa") require.NotEqual(t, PSK{}, psk1, "bootstrap established a PSK") - rogue, err := dA.startExchangeTest("bbbb", false, ExchangeID{}) + // Authenticated (sealed under the shared PSK) and shaped as a rotation, so only the + // role guard can stop it. + ack, err := newExchangeID() require.NoError(t, err) - require.NoError(t, dB.OnDataPathMessage("aaaa", rogue)) + rogue, err := dA.startExchangeTest("bbbb", false, ack) + require.NoError(t, err) + require.NoError(t, dB.OnDataPathMessage("aaaa", sealForPeer(t, dA, "bbbb", rogue))) assert.Equal(t, psk1, wgB.psk("aaaa"), "the initiator must not adopt a PSK from an offer sent by the role-responder") diff --git a/client/internal/pqkem/datapath_auth.go b/client/internal/pqkem/datapath_auth.go new file mode 100644 index 00000000000..fdd0fad8e2f --- /dev/null +++ b/client/internal/pqkem/datapath_auth.go @@ -0,0 +1,48 @@ +package pqkem + +import ( + "crypto/hkdf" + "crypto/hmac" + "crypto/sha256" + "fmt" +) + +// dataPathMACLabel separates the data-path MAC key from the PSK the consumer programs +// into its channel, so the same secret never serves two purposes directly. +const dataPathMACLabel = "netbird-pq-datapath-mac-v1" + +// dataPathTagSize is the length of the HMAC-SHA256 tag appended to every data-path frame. +const dataPathTagSize = sha256.Size + +// dataPathMACKey derives the key that authenticates data-path messages from a PSK both +// peers hold. +func dataPathMACKey(psk PSK) ([]byte, error) { + key, err := hkdf.Key(sha256.New, psk[:], nil, dataPathMACLabel, sha256.Size) + if err != nil { + return nil, fmt.Errorf("derive data-path mac key: %w", err) + } + return key, nil +} + +// sealDataPath appends the frame's HMAC-SHA256 tag under key. +func sealDataPath(key, frame []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write(frame) + sealed := make([]byte, 0, len(frame)+dataPathTagSize) + sealed = append(sealed, frame...) + return mac.Sum(sealed) +} + +// openDataPath verifies a sealed frame under key and returns the frame without its tag. +func openDataPath(key, sealed []byte) ([]byte, bool) { + if len(sealed) < dataPathTagSize { + return nil, false + } + frame, tag := sealed[:len(sealed)-dataPathTagSize], sealed[len(sealed)-dataPathTagSize:] + mac := hmac.New(sha256.New, key) + mac.Write(frame) + if !hmac.Equal(tag, mac.Sum(nil)) { + return nil, false + } + return frame, true +} diff --git a/client/internal/pqkem/datapath_auth_test.go b/client/internal/pqkem/datapath_auth_test.go new file mode 100644 index 00000000000..b1cf42aa9ea --- /dev/null +++ b/client/internal/pqkem/datapath_auth_test.go @@ -0,0 +1,137 @@ +package pqkem + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sealForPeer seals a data-path frame under the PSK m currently shares with remoteID, as +// a genuine sender would. +func sealForPeer(t *testing.T, m *Manager, remoteID RemoteID, frame []byte) []byte { + t.Helper() + psk, ok := m.PSK(remoteID) + require.True(t, ok, "sealing needs an established PSK") + key, err := dataPathMACKey(psk) + require.NoError(t, err) + return sealDataPath(key, frame) +} + +func TestDataPathSealOpen(t *testing.T) { + key, err := dataPathMACKey(PSK{1}) + require.NoError(t, err) + otherKey, err := dataPathMACKey(PSK{2}) + require.NoError(t, err) + frame := []byte("kem frame") + + sealed := sealDataPath(key, frame) + got, ok := openDataPath(key, sealed) + require.True(t, ok, "a frame sealed under the key must open") + assert.Equal(t, frame, got, "opening must return the original frame") + + _, ok = openDataPath(otherKey, sealed) + assert.False(t, ok, "a different key must not open the frame") + + tampered := append([]byte(nil), sealed...) + tampered[0] ^= 0x01 + _, ok = openDataPath(key, tampered) + assert.False(t, ok, "a modified frame must not open") + + _, ok = openDataPath(key, sealed[:dataPathTagSize-1]) + assert.False(t, ok, "a datagram shorter than a tag must not open") +} + +// TestManager_DataPathRejectsAuthenticatedBootstrap verifies a bootstrap is never taken +// from the data path, even with a valid MAC: the data path only carries rotations of an +// exchange that signalling already established. +func TestManager_DataPathRejectsAuthenticatedBootstrap(t *testing.T) { + dA, dB, wgA, _, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + psk := wgA.psk("bbbb") + + require.NoError(t, dA.OnDataPathMessage("bbbb", sealForPeer(t, dA, "bbbb", forgedOffer(t, ExchangeID{})))) + + assert.Equal(t, psk, wgA.psk("bbbb"), "a bootstrap offer over the data path must be dropped") +} + +// TestManager_DataPathRetransmitAfterCommitIsAnswered covers a lost answer. The responder +// commits the rotated PSK when it answers, so the initiator's retransmission, still sealed +// under the previous PSK, must be accepted under that key and answered, or the two peers +// never converge. +func TestManager_DataPathRetransmitAfterCommitIsAnswered(t *testing.T) { + sw := newSwitch() + wgA, wgB := newFakeWG(), newFakeWG() + dA := NewManager("aaaa", wgA, nil) // responder + dB := NewManager("bbbb", wgB, nil) // initiator + dB.retryInterval = 20 * time.Millisecond + lbA := &loopback{ep: epA, sw: sw} + dA.Start(lbA) + dB.Start(&loopback{ep: epB, sw: sw}) + dA.AddPeer("bbbb", epB) + dB.AddPeer("aaaa", epA) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + boot := wgB.psk("aaaa") + + lbA.drop.Store(true) // lose the responder's answers + dB.OnDataPathRekeyed("aaaa", 0) + require.Eventually(t, func() bool { return wgA.psk("bbbb") != boot }, time.Second, 5*time.Millisecond, + "the responder must commit the rotated PSK") + lbA.drop.Store(false) + + require.Eventually(t, func() bool { return wgB.psk("aaaa") == wgA.psk("bbbb") }, 2*time.Second, 5*time.Millisecond, + "the retransmitted offer must be answered so both peers converge") +} + +// forgedOffer builds a well-formed KEM offer that no peer authenticated: fresh KEM +// material, the given AckID, and no data-path MAC. +func forgedOffer(t *testing.T, ack ExchangeID) []byte { + t.Helper() + ini, err := NewInitiator() + require.NoError(t, err) + id, err := newExchangeID() + require.NoError(t, err) + raw, err := (&OfferMsg{ExchangeID: id, AckID: ack, KEMOffer: ini.Offer()}).Encode() + require.NoError(t, err) + return raw +} + +// TestManager_DataPathRejectsForgedOffer covers spoofed data-path datagrams. The data-path +// socket only knows the sender by source IP:port, and on a host-bound socket a datagram +// with a spoofed source can arrive from a physical interface rather than the tunnel. An +// unauthenticated offer must not re-key the responder: a forged bootstrap (zero AckID) +// used to be adopted, desynchronising the peers' PSKs. A guessed AckID is covered too. +func TestManager_DataPathRejectsForgedOffer(t *testing.T) { + guessed, err := newExchangeID() + require.NoError(t, err) + cases := map[string]ExchangeID{ + "forged bootstrap (zero AckID)": {}, + "forged rotation (guessed AckID)": guessed, + } + for name, ack := range cases { + t.Run(name, func(t *testing.T) { + dA, dB, wgA, _, _ := pair(t) // dB ("bbbb") initiator, dA ("aaaa") responder + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + psk := wgA.psk("bbbb") + require.NotEqual(t, PSK{}, psk, "bootstrap must establish a PSK") + + // Spoofed: it arrives from the initiator's registered data-path endpoint. + dA.onDataPathInbound(epB, forgedOffer(t, ack)) + + assert.Equal(t, psk, wgA.psk("bbbb"), "a forged data-path offer must not re-key the responder") + current, ok := dA.PSK("bbbb") + require.True(t, ok) + assert.Equal(t, psk, current, "the responder's stored PSK must be unchanged") + }) + } +} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index a779d044302..7efbef267bc 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -114,6 +114,7 @@ type Manager struct { established map[RemoteID]bool // peer has completed at least one exchange failures map[RemoteID]int // consecutive rekey failures per peer psks map[RemoteID]PSK // latest derived PSK per peer (pulled at WG peer-config time) + prevPSKs map[RemoteID]PSK // PSK before the latest commit; still authenticates data-path messages sealed under it capable map[RemoteID]bool // peer runs the KEM (advertised a PQ port); false = known non-capable peerAddrs map[RemoteID]netip.AddrPort // remoteID -> data-path endpoint (send routing) peersByAddr map[netip.AddrPort]RemoteID // reverse: source endpoint -> remoteID (inbound) @@ -164,6 +165,7 @@ func NewManager(localID LocalID, h CallbackHandler, logger *slog.Logger, opts .. established: make(map[RemoteID]bool), failures: make(map[RemoteID]int), psks: make(map[RemoteID]PSK), + prevPSKs: make(map[RemoteID]PSK), capable: make(map[RemoteID]bool), peerAddrs: make(map[RemoteID]netip.AddrPort), peersByAddr: make(map[netip.AddrPort]RemoteID), @@ -297,6 +299,7 @@ func (m *Manager) RemovePeer(remoteID RemoteID) { delete(m.established, remoteID) delete(m.failures, remoteID) delete(m.psks, remoteID) + delete(m.prevPSKs, remoteID) delete(m.capable, remoteID) if ep, ok := m.peerAddrs[remoteID]; ok { delete(m.peersByAddr, ep) @@ -479,22 +482,37 @@ func (m *Manager) onDataPathInbound(src netip.AddrPort, msg []byte) { } // OnDataPathMessage handles a KEM message received over the data path from remoteID -// and pushes any reply back over the data path. +// and pushes any reply back over the data path. The sender is only known by its source +// address, which can be spoofed, so a message that does not carry a valid MAC under a PSK +// shared with remoteID is rejected. func (m *Manager) OnDataPathMessage(remoteID RemoteID, raw []byte) error { - typ, msg, err := Decode(raw) + frame, key, err := m.openDataPathFrom(remoteID, raw) + if err != nil { + return err + } + typ, msg, err := Decode(frame) if err != nil { return fmt.Errorf("decode data-path msg from %s: %w", remoteID, err) } switch typ { case MsgOffer: - answer, err := m.processOffer(remoteID, msg.(*OfferMsg), viaDataPathLabel) + o := msg.(*OfferMsg) + // The data path only carries rotations of an established exchange. A bootstrap + // starts from scratch, so it must come over the authenticated signalling channel. + if o.AckID == (ExchangeID{}) { + m.debug("pqkem: dropping a bootstrap offer received over the data path", "peer", remoteID, "exchange", idHex(o.ExchangeID)) + return nil + } + answer, err := m.processOffer(remoteID, o, viaDataPathLabel) if err != nil { return err } if answer == nil { return nil } - return m.pushDataPath(remoteID, answer) + // Seal the reply under the key that verified the offer. processOffer has just + // committed the new PSK, which the initiator only derives once it reads this answer. + return m.sendDataPath(remoteID, sealDataPath(key, answer)) case MsgAnswer: return m.processAnswer(remoteID, msg.(*AnswerMsg), viaDataPathLabel) case MsgError: @@ -567,15 +585,57 @@ func (m *Manager) OnDataPathDown(remoteID RemoteID) {} // ---- internals ---- -// commitPSKLocked records a freshly derived PSK as the peer's current one. Assumes m.mu -// is held. +// commitPSKLocked records a freshly derived PSK as the peer's current one, keeping the one +// it replaces: messages sealed under it may still be in flight (see openDataPathFrom). +// Assumes m.mu is held. func (m *Manager) commitPSKLocked(remoteID RemoteID, psk PSK) { + if old, ok := m.psks[remoteID]; ok { + m.prevPSKs[remoteID] = old + } m.psks[remoteID] = psk } -// pushDataPath sends a message to the peer over the data-path transport. +// pushDataPath sends a message to the peer over the data-path transport, sealed under +// the PSK the two peers currently share. func (m *Manager) pushDataPath(remoteID RemoteID, msg []byte) error { - return m.sendDataPath(remoteID, msg) + m.mu.Lock() + psk, ok := m.psks[remoteID] + m.mu.Unlock() + if !ok { + return fmt.Errorf("no PSK to authenticate a data-path message to %s", remoteID) + } + key, err := dataPathMACKey(psk) + if err != nil { + return err + } + return m.sendDataPath(remoteID, sealDataPath(key, msg)) +} + +// openDataPathFrom authenticates a data-path message from a peer under its current PSK, +// or failing that the previous one: an offer retransmitted after this side committed the +// PSK it derived was still sealed under the old key. It returns the frame and the key +// that verified it. +func (m *Manager) openDataPathFrom(remoteID RemoteID, raw []byte) ([]byte, []byte, error) { + m.mu.Lock() + candidates := make([]PSK, 0, 2) + if psk, ok := m.psks[remoteID]; ok { + candidates = append(candidates, psk) + } + if psk, ok := m.prevPSKs[remoteID]; ok { + candidates = append(candidates, psk) + } + m.mu.Unlock() + + for _, psk := range candidates { + key, err := dataPathMACKey(psk) + if err != nil { + return nil, nil, err + } + if frame, ok := openDataPath(key, raw); ok { + return frame, key, nil + } + } + return nil, nil, fmt.Errorf("unauthenticated data-path message from %s", remoteID) } // sendDataPath resolves the peer's endpoint and sends over the data-path transport, diff --git a/client/internal/pqkem/secret_test.go b/client/internal/pqkem/secret_test.go index 5b1c42c2d4d..610ff07ff16 100644 --- a/client/internal/pqkem/secret_test.go +++ b/client/internal/pqkem/secret_test.go @@ -55,12 +55,19 @@ func TestManager_PresharedSecretBindsPSK(t *testing.T) { require.NotEqual(t, PSK{}, bootB, "initiator must derive a bootstrap PSK") assert.Equal(t, tc.match, bootA == bootB, "bootstrap PSKs must match only when the secrets match") - // Rotation over the data path re-derives the PSK; it must honour the secret too. dB.OnDataPathRekeyed("aaaa", 0) + if !tc.match { + // The peers share no PSK, so the responder cannot authenticate the data-path + // rotation offer and keeps its key: a rotation cannot bridge the secret either. + assert.Never(t, func() bool { return wgA.psk("bbbb") != bootA }, 200*time.Millisecond, 10*time.Millisecond, + "a rotation between peers with different secrets must not re-key the responder") + return + } + // Rotation over the data path re-derives the PSK; it must honour the secret too. require.Eventually(t, func() bool { return wgA.psk("bbbb") != bootA && wgB.psk("aaaa") != bootB }, time.Second, 5*time.Millisecond, "the rotation must re-key both peers") - assert.Equal(t, tc.match, wgA.psk("bbbb") == wgB.psk("aaaa"), "rotated PSKs must match only when the secrets match") + assert.Equal(t, wgA.psk("bbbb"), wgB.psk("aaaa"), "rotated PSKs must match when the secrets match") }) } } From c1118f6a781b92d0ccc07590e190454186ee4123 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:43:00 +0200 Subject: [PATCH 097/119] [client] build the lazy peer config in one place Preparation for installing idle peers with ML-KEM strict-mode key material (security review of #7098, finding F-002). The lazy connection manager receives a peer description from two call sites that each spell out the same struct literal; the new field should be added once, not kept in sync in two places. Pure refactor, no behaviour change: both sites call newLazyPeerConfig. --- client/internal/conn_mgr.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/client/internal/conn_mgr.go b/client/internal/conn_mgr.go index 8b01eabcf5f..9a5d744afd5 100644 --- a/client/internal/conn_mgr.go +++ b/client/internal/conn_mgr.go @@ -163,13 +163,7 @@ func (e *ConnMgr) SetExcludeList(ctx context.Context, peerIDs map[string]bool) { continue } - lazyPeerCfg := lazyconn.PeerConfig{ - PublicKey: peerID, - AllowedIPs: peerConn.WgConfig().AllowedIps, - PeerConnID: peerConn.ConnID(), - Log: peerConn.Log, - } - excludedPeers = append(excludedPeers, lazyPeerCfg) + excludedPeers = append(excludedPeers, newLazyPeerConfig(peerID, peerConn)) } added := e.lazyConnMgr.ExcludePeer(excludedPeers) @@ -188,6 +182,16 @@ func (e *ConnMgr) SetExcludeList(ctx context.Context, peerIDs map[string]bool) { } } +// newLazyPeerConfig describes a peer connection to the lazy connection manager. +func newLazyPeerConfig(peerKey string, conn *peer.Conn) lazyconn.PeerConfig { + return lazyconn.PeerConfig{ + PublicKey: peerKey, + AllowedIPs: conn.WgConfig().AllowedIps, + PeerConnID: conn.ConnID(), + Log: conn.Log, + } +} + // AddPeerConn registers a peer connection. permanent requests an always-active connection // (the peer belongs to the exclude set: a forwarder, or a peer that is not lazy by policy). // Non-permanent peers are handed to the lazy manager. The subsequent SetExcludeList call @@ -212,13 +216,7 @@ func (e *ConnMgr) AddPeerConn(ctx context.Context, peerKey string, conn *peer.Co return } - lazyPeerCfg := lazyconn.PeerConfig{ - PublicKey: peerKey, - AllowedIPs: conn.WgConfig().AllowedIps, - PeerConnID: conn.ConnID(), - Log: conn.Log, - } - excluded, err := e.lazyConnMgr.AddPeer(lazyPeerCfg) + excluded, err := e.lazyConnMgr.AddPeer(newLazyPeerConfig(peerKey, conn)) if err != nil { conn.Log.Errorf("failed to add peer to lazyconn manager: %v", err) if err := conn.Open(ctx); err != nil { From 3cfc5c82a835c78f26f21bedde87515be332b8f4 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:45:35 +0200 Subject: [PATCH 098/119] [client] pqkem: install strict-mode keys on idle lazy peers ML-KEM strict mode promises that no traffic crosses on a non-PQ key. Under lazy connections an idle peer is still installed in WireGuard, so that traffic towards it can wake it, and it accepts a handshake the remote initiates directly. Both lazy listeners installed it without a preshared key, so a freshly created idle peer accepted a classical session before any KEM exchange, for example from a remote whose WireGuard still targets the victim's real endpoint (security review of #7098, finding F-002). The strict sentinel only reached WireGuard later, on the normal connection path. Let the lazy peer config carry a key provider that the listeners install with the idle peer. In strict mode it yields the PQ PSK once derived and the sentinel until then; it is read at install time so it is never stale. Outside strict mode it yields nothing and the idle peer's key is left unchanged, as before. --- client/internal/conn_mgr.go | 9 +- .../lazyconn/activity/listener_bind.go | 2 +- .../lazyconn/activity/listener_psk_test.go | 106 ++++++++++++++++++ .../lazyconn/activity/listener_udp.go | 2 +- client/internal/lazyconn/peercfg.go | 14 +++ client/internal/peer/conn.go | 15 +++ client/internal/peer/conn_pq_test.go | 32 ++++++ 7 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 client/internal/lazyconn/activity/listener_psk_test.go diff --git a/client/internal/conn_mgr.go b/client/internal/conn_mgr.go index 9a5d744afd5..a2f9a972c74 100644 --- a/client/internal/conn_mgr.go +++ b/client/internal/conn_mgr.go @@ -185,10 +185,11 @@ func (e *ConnMgr) SetExcludeList(ctx context.Context, peerIDs map[string]bool) { // newLazyPeerConfig describes a peer connection to the lazy connection manager. func newLazyPeerConfig(peerKey string, conn *peer.Conn) lazyconn.PeerConfig { return lazyconn.PeerConfig{ - PublicKey: peerKey, - AllowedIPs: conn.WgConfig().AllowedIps, - PeerConnID: conn.ConnID(), - Log: conn.Log, + PublicKey: peerKey, + AllowedIPs: conn.WgConfig().AllowedIps, + PeerConnID: conn.ConnID(), + Log: conn.Log, + PresharedKey: conn.IdlePresharedKey, } } diff --git a/client/internal/lazyconn/activity/listener_bind.go b/client/internal/lazyconn/activity/listener_bind.go index 72a0cfc7675..82ff16190d6 100644 --- a/client/internal/lazyconn/activity/listener_bind.go +++ b/client/internal/lazyconn/activity/listener_bind.go @@ -106,7 +106,7 @@ func (d *BindListener) setupLazyConn() error { IP: d.fakeIP.AsSlice(), Port: lazyBindPort, } - return d.wgIface.UpdatePeer(d.peerCfg.PublicKey, d.peerCfg.AllowedIPs, 0, endpoint, nil) + return d.wgIface.UpdatePeer(d.peerCfg.PublicKey, d.peerCfg.AllowedIPs, 0, endpoint, d.peerCfg.IdlePresharedKey()) } // ReadPackets blocks until activity is detected on the LazyConn or the listener is closed. diff --git a/client/internal/lazyconn/activity/listener_psk_test.go b/client/internal/lazyconn/activity/listener_psk_test.go new file mode 100644 index 00000000000..1cf3643cb33 --- /dev/null +++ b/client/internal/lazyconn/activity/listener_psk_test.go @@ -0,0 +1,106 @@ +package activity + +import ( + "net" + "net/netip" + "sync" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/internal/lazyconn" +) + +// pskRecordingIface records the preshared key each idle peer is installed with. +type pskRecordingIface struct { + MocWGIface + mu sync.Mutex + psks []*wgtypes.Key +} + +func (r *pskRecordingIface) UpdatePeer(_ string, _ []netip.Prefix, _ time.Duration, _ *net.UDPAddr, psk *wgtypes.Key) error { + r.mu.Lock() + defer r.mu.Unlock() + r.psks = append(r.psks, psk) + return nil +} + +func (r *pskRecordingIface) installed() []*wgtypes.Key { + r.mu.Lock() + defer r.mu.Unlock() + return append([]*wgtypes.Key(nil), r.psks...) +} + +type idleListener interface { + ReadPackets() + Close() +} + +func closeListener(t *testing.T, l idleListener) { + t.Helper() + done := make(chan struct{}) + go func() { + l.ReadPackets() + close(done) + }() + l.Close() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for ReadPackets to exit after Close") + } +} + +// TestListeners_InstallIdlePeerWithItsPresharedKey covers ML-KEM strict mode with lazy +// connections. An idle peer is installed in WireGuard so a handshake the remote initiates +// directly is accepted; installed without a key, a fresh peer accepted a classical session +// before any post-quantum exchange, breaking strict mode's promise. Both listeners must +// install the key the peer config provides, and leave it unchanged when there is none. +func TestListeners_InstallIdlePeerWithItsPresharedKey(t *testing.T) { + sentinel := wgtypes.Key{1, 2, 3} + newCfg := func(psk func() *wgtypes.Key) lazyconn.PeerConfig { + peer := &MocPeer{PeerID: "testPeer1"} + return lazyconn.PeerConfig{ + PublicKey: peer.PeerID, + PeerConnID: peer.ConnID(), + AllowedIPs: []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}, + Log: log.WithField("peer", "testPeer1"), + PresharedKey: psk, + } + } + listeners := map[string]func(WgInterface, lazyconn.PeerConfig) (idleListener, error){ + "udp": func(iface WgInterface, cfg lazyconn.PeerConfig) (idleListener, error) { + return NewUDPListener(iface, cfg) + }, + "bind": func(iface WgInterface, cfg lazyconn.PeerConfig) (idleListener, error) { + return NewBindListener(iface, newMockEndpointManager(), cfg) + }, + } + for name, newListener := range listeners { + t.Run(name+"/with key", func(t *testing.T) { + iface := &pskRecordingIface{} + l, err := newListener(iface, newCfg(func() *wgtypes.Key { return &sentinel })) + require.NoError(t, err) + closeListener(t, l) + + installed := iface.installed() + require.Len(t, installed, 1, "the idle peer must be installed once") + require.NotNil(t, installed[0], "the idle peer must be installed with the provided key") + assert.Equal(t, sentinel, *installed[0], "the idle peer must carry the provided key") + }) + t.Run(name+"/without key", func(t *testing.T) { + iface := &pskRecordingIface{} + l, err := newListener(iface, newCfg(nil)) + require.NoError(t, err) + closeListener(t, l) + + installed := iface.installed() + require.Len(t, installed, 1, "the idle peer must be installed once") + assert.Nil(t, installed[0], "without a key the idle peer's key must be left unchanged") + }) + } +} diff --git a/client/internal/lazyconn/activity/listener_udp.go b/client/internal/lazyconn/activity/listener_udp.go index 4b7e0ddf76b..ba7e154075a 100644 --- a/client/internal/lazyconn/activity/listener_udp.go +++ b/client/internal/lazyconn/activity/listener_udp.go @@ -101,7 +101,7 @@ func (d *UDPListener) Close() { func (d *UDPListener) createEndpoint() error { d.peerCfg.Log.Debugf("creating lazy endpoint: %s", d.endpoint.String()) - return d.wgIface.UpdatePeer(d.peerCfg.PublicKey, d.peerCfg.AllowedIPs, 0, d.endpoint, nil) + return d.wgIface.UpdatePeer(d.peerCfg.PublicKey, d.peerCfg.AllowedIPs, 0, d.endpoint, d.peerCfg.IdlePresharedKey()) } func (d *UDPListener) newConn() (*net.UDPConn, error) { diff --git a/client/internal/lazyconn/peercfg.go b/client/internal/lazyconn/peercfg.go index 987d06a3e1a..7847b19e0d3 100644 --- a/client/internal/lazyconn/peercfg.go +++ b/client/internal/lazyconn/peercfg.go @@ -4,6 +4,7 @@ import ( "net/netip" log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/internal/peer/id" ) @@ -13,4 +14,17 @@ type PeerConfig struct { AllowedIPs []netip.Prefix PeerConnID id.ConnID Log *log.Entry + // PresharedKey returns the key to install with the idle WireGuard peer, or nil to + // leave the peer's key as it is. It is read when the peer is installed, not when the + // config is built, so it reflects keys derived after the peer joined. A nil func + // leaves the key unchanged too. + PresharedKey func() *wgtypes.Key +} + +// IdlePresharedKey is the key to install with the idle WireGuard peer (nil: unchanged). +func (c PeerConfig) IdlePresharedKey() *wgtypes.Key { + if c.PresharedKey == nil { + return nil + } + return c.PresharedKey() } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 51a362a58db..c22b34bca11 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -1141,6 +1141,21 @@ func (conn *Conn) pqPSK() (*wgtypes.Key, bool) { return &psk, true } +// IdlePresharedKey is the preshared key an idle (lazy) WireGuard peer is installed with. +// An idle peer accepts a handshake the remote initiates directly, so in ML-KEM strict +// mode it must already carry the PQ PSK, or the strict sentinel until one exists; +// installed without a key, a fresh peer would accept a classical session before any +// exchange. Outside strict mode it returns nil and leaves the idle peer's key unchanged. +func (conn *Conn) IdlePresharedKey() *wgtypes.Key { + if conn.config.PQ == nil || !conn.config.PQStrict { + return nil + } + if psk, ok := conn.pqPSK(); ok { + return psk + } + return conn.pqStrictSentinelKey +} + // presharedKey resolves the WireGuard preshared key for the peer. pqPSK is the // post-quantum PSK looked up out of band via pqPSK (nil when none is derived yet), passed // in so the manager lock is never taken under conn.mu. diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index b813a44a074..f62d2d6e6d3 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -6,6 +6,7 @@ import ( "time" log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) @@ -88,3 +89,34 @@ func TestConn_presharedKey_PQ(t *testing.T) { require.NotEqual(t, nbPSK, *got, "the sentinel must not be the ordinary key") }) } + +// TestConn_IdlePresharedKey covers the key an idle (lazy) peer is installed with. Strict +// mode must never leave an idle peer without a key, since the remote can complete a +// classical handshake with it directly; everything else leaves the idle key unchanged. +func TestConn_IdlePresharedKey(t *testing.T) { + derivedPSK, err := wgtypes.GenerateKey() + require.NoError(t, err) + sentinel, err := wgtypes.GenerateKey() + require.NoError(t, err) + + newConn := func(pq PQHandshaker, strict bool) *Conn { + c := &Conn{Log: log.WithField("peer", "pq-idle-test")} + c.config.PQ = pq + c.config.PQStrict = strict + if strict { + c.pqStrictSentinelKey = &sentinel + } + return c + } + + got := newConn(fakePQ{ok: false}, true).IdlePresharedKey() + require.NotNil(t, got, "strict mode must install a key on an idle peer") + assert.Equal(t, sentinel, *got, "before a PQ PSK exists strict mode installs the sentinel") + + got = newConn(fakePQ{psk: derivedPSK, ok: true}, true).IdlePresharedKey() + require.NotNil(t, got, "strict mode must install a key on an idle peer") + assert.Equal(t, derivedPSK, *got, "once derived, strict mode installs the PQ PSK") + + assert.Nil(t, newConn(fakePQ{ok: false}, false).IdlePresharedKey(), "non-strict mode leaves the idle key unchanged") + assert.Nil(t, newConn(nil, false).IdlePresharedKey(), "without ML-KEM the idle key is left unchanged") +} From 5f3f5d56d9bcccd40a9e220d46de8c2fd30f8f54 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:47:51 +0200 Subject: [PATCH 099/119] [client] pqkem: re-bootstrap over Signal only while the peer is connected A failed KEM exchange makes the manager ask for a fresh signalling offer. For a peer that is registered but offline that offer can never be answered, so with lazy connections off (the default) each such peer drew a new offer at every exchange timeout, about every 22 seconds, forever. That runs on top of the reconnect guard, which already retries the peer with exponential backoff, and every guard offer starts a fresh KEM exchange anyway. With many offline peers it is a steady stream of useless Signal traffic. Send the recovery offer only while the peer is connected. That is where it is needed: the transport is up but the KEM did not converge (strict mode stuck before its first PSK, or a broken rotation). An unreachable peer is left to the guard's backoff. --- client/internal/peer/conn.go | 8 ++++- client/internal/peer/conn_pq_test.go | 44 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index c22b34bca11..d688f109655 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -760,14 +760,20 @@ func (conn *Conn) onGuardEvent() { // RequestReoffer sends a fresh signalling offer for the peer, re-running the // post-quantum bootstrap over Signal. Used to recover from a persistent data-path // rekey failure: a new exchange overwrites the stalled PSK on both sides. No-op if the -// connection is not open yet. +// connection is not open yet, or if the peer is not connected: then the reconnect guard +// already retries it with backoff, and each of its offers starts a fresh exchange. func (conn *Conn) RequestReoffer() { conn.mu.Lock() h := conn.handshaker + connected := conn.evalStatus() == StatusConnected conn.mu.Unlock() if h == nil { return } + if !connected { + conn.Log.Debugf("pqkem: peer not connected, leaving the KEM re-bootstrap to the reconnect guard") + return + } if err := h.SendOffer(); err != nil { conn.Log.Debugf("pqkem: recovery re-offer failed: %v", err) } diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index f62d2d6e6d3..a010ceabb64 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -2,13 +2,18 @@ package peer import ( "net/netip" + "strings" "testing" "time" log "github.com/sirupsen/logrus" + logtest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/internal/peer/worker" + signal "github.com/netbirdio/netbird/shared/signal/client" ) // fakePQ is a minimal PQHandshaker: only PSK is exercised by presharedKey, the rest @@ -120,3 +125,42 @@ func TestConn_IdlePresharedKey(t *testing.T) { assert.Nil(t, newConn(fakePQ{ok: false}, false).IdlePresharedKey(), "non-strict mode leaves the idle key unchanged") assert.Nil(t, newConn(nil, false).IdlePresharedKey(), "without ML-KEM the idle key is left unchanged") } + +// TestConn_RequestReofferOnlyWhenConnected covers the PQ recovery path. A failed KEM +// exchange makes the manager ask for a fresh signalling offer. For a peer with no +// transport (offline) that offer cannot be answered: with lazy connections off it was +// repeated at every exchange timeout, a fixed-rate stream on top of the reconnect guard's +// own backed-off offers. The re-offer must only be sent while the peer is connected. +func TestConn_RequestReofferOnlyWhenConnected(t *testing.T) { + newConn := func() (*Conn, *logtest.Hook) { + logger, hook := logtest.NewNullLogger() + logger.SetLevel(log.DebugLevel) + entry := logger.WithField("peer", "reoffer-test") + // The signal client is never ready, so an attempted offer fails fast and is logged. + signaler := NewSignaler(&signal.MockClient{}, wgtypes.Key{}) + c := &Conn{ + Log: entry, + statusRelay: worker.NewAtomicStatus(), + statusICE: worker.NewAtomicStatus(), + } + c.handshaker = NewHandshaker(entry, ConnConfig{}, signaler, nil, nil, nil) + return c, hook + } + offerAttempted := func(hook *logtest.Hook) bool { + for _, e := range hook.AllEntries() { + if strings.Contains(e.Message, "recovery re-offer failed") { + return true + } + } + return false + } + + c, hook := newConn() + c.RequestReoffer() + assert.False(t, offerAttempted(hook), "a disconnected peer must be left to the reconnect guard") + + c, hook = newConn() + c.statusRelay.SetConnected() + c.RequestReoffer() + assert.True(t, offerAttempted(hook), "a connected peer must be re-offered") +} From 92764d8bff67a936f3a7e4467725f19d51b966b7 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:49:14 +0200 Subject: [PATCH 100/119] [client] pqkem: keep rotating after a rotation times out When every offer of a data-path rotation was lost, the initiator dropped the exchange and logged that it would retry next cycle. It never did: a rekey only chains a new rotation from an exchange awaiting it, and none was left. Rotation stopped until the next Signal re-bootstrap (a lazy wake, or never on a long-lived connection), and because no further attempt ran, the consecutive-failure count could not reach the threshold that hands recovery to the host either (reported in the security review of #7098). On a rotation timeout, hand the chain back to the exchange the rotation acknowledged. The next rekey then starts a fresh rotation from it: if the offer was simply lost both peers still hold that PSK and the rotation converges, and if the responder had already moved on, the attempts keep failing until the threshold triggers a re-bootstrap. A timed-out bootstrap is still dropped, as it has nothing to fall back to. --- client/internal/pqkem/convergence.go | 7 +++ client/internal/pqkem/manager.go | 1 + client/internal/pqkem/rotation_test.go | 69 ++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 client/internal/pqkem/rotation_test.go diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index f3069542607..897c9718494 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -72,6 +72,7 @@ func (m *Manager) startExchangeLocked(remoteID RemoteID, viaSignal bool, ackID E lastSent: raw, initiator: init, viaSignal: viaSignal, + acks: ackID, } m.wait.Add(1) @@ -325,6 +326,12 @@ func (m *Manager) initiatorLoop(ctx context.Context, remoteID RemoteID, id Excha case stateAwaitingAnswer: if attempts >= m.maxRetries { delete(m.exchanges, remoteID) + // A timed-out rotation hands the chain back to the exchange it + // acknowledged, so the next data-path rekey starts a new rotation. + // Without it rotation stops until the next Signal re-bootstrap. + if ex.acks != (ExchangeID{}) { + m.exchanges[remoteID] = &exchangeCtl{id: ex.acks, state: stateAwaitingRekey} + } initial := !m.established[remoteID] fail := m.registerFailureLocked(remoteID) m.mu.Unlock() diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 7efbef267bc..1fee7da0e73 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -87,6 +87,7 @@ type exchangeCtl struct { lastSent []byte initiator *Initiator viaSignal bool + acks ExchangeID // exchange this one acknowledges; zero for a bootstrap } // Manager is the stateful orchestrator — the analogue of go-rosenpass's Server. It diff --git a/client/internal/pqkem/rotation_test.go b/client/internal/pqkem/rotation_test.go new file mode 100644 index 00000000000..70fb90601a2 --- /dev/null +++ b/client/internal/pqkem/rotation_test.go @@ -0,0 +1,69 @@ +package pqkem + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestManager_RotationResumesAfterTimeout covers a rotation whose offers are all lost. +// The timeout must leave the rotation chain usable: the next data-path rekey has to start +// a new rotation. Deleting the exchange on timeout stopped rotation for good, since a +// rekey only chains from an exchange awaiting it, and the failure count never reached the +// threshold that triggers a Signal re-bootstrap, despite the log promising a retry. +func TestManager_RotationResumesAfterTimeout(t *testing.T) { + dA, dB, wgA, wgB, lbB := pair(t) // dB ("bbbb") initiator + defer dA.Stop() + defer dB.Stop() + dB.retryInterval = 10 * time.Millisecond + dB.maxRetries = 2 + + bootstrap(t, dA, dB) + boot := wgB.psk("aaaa") + + // Lose every rotation offer until the exchange times out. + lbB.drop.Store(true) + dB.OnDataPathRekeyed("aaaa", 0) + require.Eventually(t, func() bool { + dB.mu.Lock() + defer dB.mu.Unlock() + ex := dB.exchanges["aaaa"] + return ex == nil || ex.state != stateAwaitingAnswer + }, time.Second, 5*time.Millisecond, "the lost rotation must time out") + lbB.drop.Store(false) + + dB.OnDataPathRekeyed("aaaa", 0) + require.Eventually(t, func() bool { + return wgB.psk("aaaa") != boot && wgA.psk("bbbb") == wgB.psk("aaaa") + }, time.Second, 5*time.Millisecond, "the next rekey must start a rotation that converges") + assert.Empty(t, wgB.failed, "a single lost rotation must not escalate to a re-bootstrap") +} + +// TestManager_RepeatedRotationTimeoutsEscalate verifies the failure policy can complete: +// consecutive lost rotations must reach the rekey-failure threshold and hand recovery to +// the host. With rotation stopping after the first timeout the threshold was unreachable. +func TestManager_RepeatedRotationTimeoutsEscalate(t *testing.T) { + dA, dB, _, wgB, lbB := pair(t) // dB ("bbbb") initiator + defer dA.Stop() + defer dB.Stop() + dB.retryInterval = 10 * time.Millisecond + dB.maxRetries = 2 + + bootstrap(t, dA, dB) + lbB.drop.Store(true) + + for i := 0; i < dB.maxRekeyFailures; i++ { + dB.OnDataPathRekeyed("aaaa", 0) + require.Eventually(t, func() bool { + dB.mu.Lock() + defer dB.mu.Unlock() + ex := dB.exchanges["aaaa"] + return ex == nil || ex.state != stateAwaitingAnswer + }, time.Second, 5*time.Millisecond, "rotation %d must time out", i+1) + } + + assert.Eventually(t, func() bool { return failedCount(wgB) == 1 }, time.Second, 5*time.Millisecond, + "consecutive lost rotations must escalate to a rekey failure") +} From e9a622c89538f3c32a2b981bf4384ed53ebe2394 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 16:50:42 +0200 Subject: [PATCH 101/119] [client] pqkem: apply PSK generations to WireGuard in order Two derivations for the same peer can overlap, for example a rotation and a Signal re-bootstrap. The generation guard checked and recorded the generation under a lock but called SetPresharedKey after releasing it, so an older apply already inside SetPresharedKey could finish after a newer one and leave WireGuard on the stale key, splitting the peers until the next exchange (reported in the security review of #7098). Hold the lock across the generation check and the SetPresharedKey call. The lock only serialises PSK applies, which are quick local calls, so this does not reach into the manager's own locking. --- client/internal/pqkem_adapter.go | 39 +++++++++----- client/internal/pqkem_adapter_test.go | 75 ++++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index b15370d90d5..ad65a895cf3 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -23,17 +23,20 @@ func newAppliedGenerations() *appliedGenerations { return &appliedGenerations{last: make(map[string]uint64)} } -// claim reports whether gen is newer than the last applied for the peer; when it is, it -// records gen as the newest and returns true. The check and record are atomic, so the -// slow SetPresharedKey call runs outside this lock. -func (a *appliedGenerations) claim(peer string, gen uint64) bool { +// apply runs set when gen is newer than the last generation applied for the peer, and +// reports whether it ran. The check and set run under one lock: with only the check +// atomic, an older apply already inside set could finish after a newer one and restore +// its stale key. The generation is recorded before set runs, so a failed newer apply +// still keeps an older one from landing afterwards. The lock is shared by all peers, +// which is cheap because set is a quick local interface call. +func (a *appliedGenerations) apply(peer string, gen uint64, set func() error) (bool, error) { a.mu.Lock() defer a.mu.Unlock() if gen <= a.last[peer] { - return false + return false, nil } a.last[peer] = gen - return true + return true, set() } // pqPresharedKeySetter is the subset of the WireGuard interface the ML-KEM callback @@ -68,16 +71,26 @@ type pqCallbackHandler struct { // newer than the last applied for the peer is dropped, so a reordered apply can't // restore an older PSK over a newer one. func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, gen uint64, psk pqkem.PSK) error { - if h.applied != nil && !h.applied.claim(string(remoteID), gen) { - log.Tracef("pqkem: dropping stale PSK apply for peer %s (gen %d)", remoteID, gen) - return nil - } // updateOnly: applies to an already-configured peer (rotation). At bootstrap the // peer is not configured yet, so this is a no-op there and the PSK is instead // pulled at peer-config time (pqHandshaker.PSK / conn.presharedKey). - log.Tracef("pqkem: programming PSK for peer %s (gen %d)", remoteID, gen) - if err := h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true); err != nil { - return err + set := func() error { + log.Tracef("pqkem: programming PSK for peer %s (gen %d)", remoteID, gen) + return h.wg.SetPresharedKey(string(remoteID), wgtypes.Key(psk), true) + } + if h.applied == nil { + if err := set(); err != nil { + return err + } + } else { + applied, err := h.applied.apply(string(remoteID), gen, set) + if err != nil { + return err + } + if !applied { + log.Tracef("pqkem: dropping stale PSK apply for peer %s (gen %d)", remoteID, gen) + return nil + } } // The status flag is computed when the connection comes up; a PSK landing afterwards // would otherwise leave the peer reported as not quantum resistant. diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index b2e23684e24..2deda11253f 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -2,7 +2,9 @@ package internal import ( "errors" + "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -76,25 +78,86 @@ func TestPQCallbackHandler_UnknownStatusPeerDoesNotFailApply(t *testing.T) { // reordered PSK callback cannot restore an older key over a newer one. func TestAppliedGenerations_DropsStale(t *testing.T) { a := newAppliedGenerations() + accept := func(peer string, gen uint64) bool { + ran, err := a.apply(peer, gen, func() error { return nil }) + require.NoError(t, err) + return ran + } - if !a.claim("peerA", 1) { + if !accept("peerA", 1) { t.Fatal("first generation must be accepted") } - if !a.claim("peerA", 2) { + if !accept("peerA", 2) { t.Fatal("a newer generation must be accepted") } - if a.claim("peerA", 2) { + if accept("peerA", 2) { t.Fatal("re-applying the same generation must be dropped") } - if a.claim("peerA", 1) { + if accept("peerA", 1) { t.Fatal("an older generation arriving late must be dropped") } - if !a.claim("peerA", 3) { + if !accept("peerA", 3) { t.Fatal("a newer generation after a dropped stale one must still be accepted") } // Generations are tracked independently per peer. - if !a.claim("peerB", 1) { + if !accept("peerB", 1) { t.Fatal("a different peer's first generation must be accepted") } } + +// blockingPSKSetter holds the first SetPresharedKey call until released, recording the +// key WireGuard ends up with. +type blockingPSKSetter struct { + mu sync.Mutex + calls int + last wgtypes.Key + entered chan struct{} + release chan struct{} +} + +func (s *blockingPSKSetter) SetPresharedKey(_ string, psk wgtypes.Key, _ bool) error { + s.mu.Lock() + s.calls++ + first := s.calls == 1 + s.mu.Unlock() + if first { + close(s.entered) + <-s.release + } + s.mu.Lock() + s.last = psk + s.mu.Unlock() + return nil +} + +// TestPQCallbackHandler_AppliesGenerationsInOrder covers two PSK callbacks for one peer +// overlapping. The newer generation must be the one WireGuard keeps: with the generation +// checked under a lock but the key set outside it, an older apply that was already +// writing could finish last and restore its stale key. +func TestPQCallbackHandler_AppliesGenerationsInOrder(t *testing.T) { + setter := &blockingPSKSetter{entered: make(chan struct{}), release: make(chan struct{})} + h := pqCallbackHandler{wg: setter, applied: newAppliedGenerations()} + older, newer := pqkem.PSK{3}, pqkem.PSK{4} + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + assert.NoError(t, h.OnNewPSKReady(pqkem.RemoteID(pqTestPeerKey), 3, older)) + }() + <-setter.entered // the older apply is now inside SetPresharedKey + + go func() { + defer wg.Done() + assert.NoError(t, h.OnNewPSKReady(pqkem.RemoteID(pqTestPeerKey), 4, newer)) + }() + // Give the newer apply the chance to overtake the blocked older one. + time.Sleep(50 * time.Millisecond) + close(setter.release) + wg.Wait() + + setter.mu.Lock() + defer setter.mu.Unlock() + assert.Equal(t, wgtypes.Key(newer), setter.last, "WireGuard must keep the newest generation's PSK") +} From d0e8c4ed33839cc9b026e4d84f4aba0f71cb0144 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 19 Oct 2026 17:15:32 +0200 Subject: [PATCH 102/119] [client] pqkem: name the post-quantum recovery path for what it does The recovery that re-runs the ML-KEM bootstrap after a failed exchange was called RequestReoffer (and "reoffer" along the callback chain). The name only says that another offer goes out, so it reads like routine renegotiation or key rotation, while it fires only when the exchange has failed: the initial exchange, or rotations failing repeatedly. Normal rotation never takes this path; it runs over the data path. Rename it RecoverPQKeyOverSignal, and the callback field, engine wiring, log line, comments and test with it. The controller re-offer in the handshaker is a different mechanism and keeps its name. Pure rename, no behaviour change; only the debug log wording follows the new name. --- client/internal/engine.go | 4 ++-- client/internal/peer/conn.go | 15 ++++++++------- client/internal/peer/conn_pq_test.go | 16 ++++++++-------- client/internal/pqkem/capability_test.go | 2 +- client/internal/pqkem/manager.go | 2 +- client/internal/pqkem_adapter.go | 12 ++++++------ 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 73fca0893ed..89768da54e5 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -613,9 +613,9 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { // On a persistent rekey failure, re-bootstrap the KEM over Signal: a fresh // signalling offer starts a new exchange that overwrites the stalled PSK on both // sides, recovering from a data-path desync. - reoffer: func(remoteKey string) { + recoverPQKeyOverSignal: func(remoteKey string) { if conn, ok := e.peerStore.PeerConn(remoteKey); ok { - conn.RequestReoffer() + conn.RecoverPQKeyOverSignal() } }, } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index d688f109655..ca341cc1109 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -757,12 +757,13 @@ func (conn *Conn) onGuardEvent() { } } -// RequestReoffer sends a fresh signalling offer for the peer, re-running the -// post-quantum bootstrap over Signal. Used to recover from a persistent data-path -// rekey failure: a new exchange overwrites the stalled PSK on both sides. No-op if the +// RecoverPQKeyOverSignal re-runs the post-quantum bootstrap over Signal after the ML-KEM +// exchange failed (the initial exchange, or repeated rotation failures): a fresh +// signalling offer starts a new exchange that overwrites the stalled PSK on both sides. +// It is not part of normal key rotation, which runs over the data path. No-op if the // connection is not open yet, or if the peer is not connected: then the reconnect guard // already retries it with backoff, and each of its offers starts a fresh exchange. -func (conn *Conn) RequestReoffer() { +func (conn *Conn) RecoverPQKeyOverSignal() { conn.mu.Lock() h := conn.handshaker connected := conn.evalStatus() == StatusConnected @@ -775,7 +776,7 @@ func (conn *Conn) RequestReoffer() { return } if err := h.SendOffer(); err != nil { - conn.Log.Debugf("pqkem: recovery re-offer failed: %v", err) + conn.Log.Debugf("pqkem: PQ key recovery over signal failed: %v", err) } } @@ -806,7 +807,7 @@ func (conn *Conn) onWGDisconnected(watcherCtx context.Context) { key := conn.config.Key conn.mu.Unlock() - // Signal the PQ manager outside conn.mu: it may re-enter Conn (reoffer) under + // Signal the PQ manager outside conn.mu: it may re-enter Conn (PQ key recovery) under // conn.mu, so calling it while holding the lock would invert the lock order. if pq != nil { pq.OnDataPathDown(key) @@ -1135,7 +1136,7 @@ func (conn *Conn) AgentVersionString() string { // pqPSK returns the post-quantum PSK derived for this peer, if the ML-KEM exchange has // produced one. It reads the manager WITHOUT conn.mu, so callers fetch it before taking // conn.mu: the lock order is always conn.mu -> manager, never the reverse (the manager's -// reoffer callback re-enters Conn under conn.mu). +// PQ key recovery callback re-enters Conn under conn.mu). func (conn *Conn) pqPSK() (*wgtypes.Key, bool) { if conn.config.PQ == nil { return nil, false diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index a010ceabb64..3aa7cfa46b3 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -126,16 +126,16 @@ func TestConn_IdlePresharedKey(t *testing.T) { assert.Nil(t, newConn(nil, false).IdlePresharedKey(), "without ML-KEM the idle key is left unchanged") } -// TestConn_RequestReofferOnlyWhenConnected covers the PQ recovery path. A failed KEM +// TestConn_RecoverPQKeyOverSignalOnlyWhenConnected covers the PQ recovery path. A failed KEM // exchange makes the manager ask for a fresh signalling offer. For a peer with no // transport (offline) that offer cannot be answered: with lazy connections off it was // repeated at every exchange timeout, a fixed-rate stream on top of the reconnect guard's -// own backed-off offers. The re-offer must only be sent while the peer is connected. -func TestConn_RequestReofferOnlyWhenConnected(t *testing.T) { +// own backed-off offers. The recovery offer must only be sent while the peer is connected. +func TestConn_RecoverPQKeyOverSignalOnlyWhenConnected(t *testing.T) { newConn := func() (*Conn, *logtest.Hook) { logger, hook := logtest.NewNullLogger() logger.SetLevel(log.DebugLevel) - entry := logger.WithField("peer", "reoffer-test") + entry := logger.WithField("peer", "pq-recovery-test") // The signal client is never ready, so an attempted offer fails fast and is logged. signaler := NewSignaler(&signal.MockClient{}, wgtypes.Key{}) c := &Conn{ @@ -148,7 +148,7 @@ func TestConn_RequestReofferOnlyWhenConnected(t *testing.T) { } offerAttempted := func(hook *logtest.Hook) bool { for _, e := range hook.AllEntries() { - if strings.Contains(e.Message, "recovery re-offer failed") { + if strings.Contains(e.Message, "PQ key recovery over signal failed") { return true } } @@ -156,11 +156,11 @@ func TestConn_RequestReofferOnlyWhenConnected(t *testing.T) { } c, hook := newConn() - c.RequestReoffer() + c.RecoverPQKeyOverSignal() assert.False(t, offerAttempted(hook), "a disconnected peer must be left to the reconnect guard") c, hook = newConn() c.statusRelay.SetConnected() - c.RequestReoffer() - assert.True(t, offerAttempted(hook), "a connected peer must be re-offered") + c.RecoverPQKeyOverSignal() + assert.True(t, offerAttempted(hook), "a connected peer must get a recovery offer") } diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index af30ff19e77..0471cdb3f61 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -9,7 +9,7 @@ import ( // TestManager_NonCapablePeerNotOffered: a peer known not to run the KEM (it answered our // offer with an empty KEM payload over signalling) is never offered an exchange, and no -// failure is raised — this is what stops the reoffer storm against non-PQ peers (e.g. +// failure is raised — this is what stops the recovery storm against non-PQ peers (e.g. // Rosenpass peers). func TestManager_NonCapablePeerNotOffered(t *testing.T) { wg := newFakeWG() diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 1fee7da0e73..f2ba97031bf 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -364,7 +364,7 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { m.mu.Lock() if capable, ok := m.capable[remoteID]; ok && !capable { m.mu.Unlock() - return nil, nil // peer does not run the KEM; do not offer (avoids a failure/reoffer loop) + return nil, nil // peer does not run the KEM; do not offer (avoids a failure/recovery loop) } // Idempotent ONLY while a signalling bootstrap is still in flight (awaitingAnswer): // a repeat call returns the SAME offer instead of starting a duplicate exchange. Once diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index ad65a895cf3..8ec4a326c88 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -61,9 +61,9 @@ type pqCallbackHandler struct { // applied drops an out-of-order PSK apply so a stale callback can't overwrite a newer // key. Nil disables the guard. applied *appliedGenerations - // reoffer re-bootstraps the KEM over Signal for a peer (a fresh signalling offer) - // to recover from a persistent data-path rekey failure. Nil disables recovery. - reoffer func(remoteKey string) + // recoverPQKeyOverSignal re-bootstraps the KEM over Signal for a peer (a fresh + // signalling offer) after its exchange failed. Nil disables recovery. + recoverPQKeyOverSignal func(remoteKey string) } // OnNewPSKReady programs the freshly derived PSK for the peer (updateOnly: a no-op @@ -109,8 +109,8 @@ func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, gen uint64, ps // broken data path). func (h pqCallbackHandler) OnRekeyFailed(remoteID pqkem.RemoteID) error { log.Warnf("pqkem: post-quantum rekey failed for peer %s, re-bootstrapping over signal", remoteID) - if h.reoffer != nil { - h.reoffer(string(remoteID)) + if h.recoverPQKeyOverSignal != nil { + h.recoverPQKeyOverSignal(string(remoteID)) } return nil } @@ -176,7 +176,7 @@ func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { if len(recvAnswer) == 0 { // Capability signal (initiator side): the KEM answer flows responder->initiator, // so an empty answer to our offer means the peer does not run the KEM — mark it - // non-capable to stop offering (no failure/reoffer storm). Only meaningful when + // non-capable to stop offering (no failure/recovery storm). Only meaningful when // we are the initiator: as the responder we also receive an (empty) answer to // our own non-KEM offer from a perfectly capable peer, which must not be flagged. if p.mgr.IsInitiator(pqkem.RemoteID(remoteKey)) { From 2853a162978795b9d6e111177b095527be3c825f Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 20 Oct 2026 16:38:00 +0200 Subject: [PATCH 103/119] [client] pqkem: answer every responder offer with a fresh KEM bootstrap Only the controller's offer carries KEM material, so when the responder starts a negotiation the controller is meant to reply with its own KEM offer. It only did so before the very first exchange: once it held a converged PSK it answered the responder's offer normally and kept that PSK, on the assumption that a stale key would heal through the WireGuard watcher reconnecting over Signal. That assumption fails when the reconnect comes from the responder. If every answer of a data-path rotation is lost, the responder commits the new PSK and the controller does not, so WireGuard cannot handshake and no further rotation runs to escalate. The watcher closes the connection after ~3.5 minutes and ICE reconnects in seconds, but the reconnect offer came from the responder, the controller kept its stale key, and the tunnel stayed down. In the lab it took 201 s and a lazy-connection idle/wake cycle to heal; with lazy connections off, the default, nothing would heal it. Treat a converged exchange as not in flight: a responder offer then always gets a fresh bootstrap, so any responder-started renegotiation re-aligns both PSKs. While the controller's own exchange is still in flight a crossing offer is answered normally as before, so offers cannot multiply. The cost is one extra Signal KEM round per responder-started reconnect. --- client/internal/peer/conn.go | 4 +- client/internal/peer/handshaker.go | 9 ++-- client/internal/pqkem/manager.go | 18 +++++--- client/internal/pqkem/split_test.go | 69 +++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 client/internal/pqkem/split_test.go diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index ca341cc1109..e0e303d8ab5 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -86,8 +86,8 @@ type PQHandshaker interface { // peer is not the KEM initiator) and the local PQ data-path port to announce. OfferPayload(remoteKey string) (payload []byte, port uint16) // ShouldSendBootstrapOffer reports whether, as the controller, we should reply to a - // received responder offer with our own KEM offer (true only when no exchange is - // already in flight — so we kick the KEM once and ignore further offers). + // received responder offer with our own KEM offer (true unless an exchange of ours is + // already in flight, so we kick the KEM once and ignore further offers meanwhile). ShouldSendBootstrapOffer(remoteKey string) bool // AnswerPayload processes a received KEM offer (nil if absent) and returns the KEM // answer to embed in the outgoing answer (nil if none) and the local PQ port. diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index e81fc4cf5fd..c7789847b38 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -208,11 +208,10 @@ func (h *Handshaker) handleRemoteAnswer(remoteOfferAnswer OfferAnswer) { // // Returns true ONLY when it actually sent a re-offer and thus took ownership of the peer's // offer (the caller must not answer it). When there is nothing to re-offer — the peer is -// non-capable, or an exchange is already in flight (e.g. every PQ peer past its bootstrap, -// which is the steady state under lazy connections) — it returns false so the caller keeps -// handling the offer normally (sendAnswer + notifyListeners). Returning true unconditionally -// would drop the ICE credentials and relay info the responder's offer carries, stalling a -// responder-initiated reconnect. +// non-capable, or our own exchange is already in flight (a crossing offer) — it returns +// false so the caller keeps handling the offer normally (sendAnswer + notifyListeners). +// Returning true without sending would drop the ICE credentials and relay info the +// responder's offer carries, stalling a responder-initiated reconnect. func (h *Handshaker) pqControllerReoffer() bool { if h.config.PQ == nil || !isController(h.config) { return false diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index f2ba97031bf..caa80bfabfe 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -374,7 +374,7 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { // still-awaitingRekey side would reject — a permanent desync. Starting a fresh exchange // re-bootstraps and both sides converge (the responder always adopts the latest). The // controller-double-offer case the idempotency once guarded is already covered by - // ShouldSendBootstrapOffer (false while any exchange exists). + // ShouldSendBootstrapOffer (false while an exchange is in flight). if ex := m.exchanges[remoteID]; ex != nil && ex.viaSignal && ex.state == stateAwaitingAnswer { last := ex.lastSent m.mu.Unlock() @@ -395,11 +395,14 @@ func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { } // ShouldSendBootstrapOffer reports whether we should emit a fresh KEM offer to kick a -// bootstrap for this peer. True only if we are the initiator, the peer is not known -// non-capable, and no exchange is already in flight. The host uses this when it (as the -// controller) receives the responder's offer: it replies with a KEM offer exactly once -// to start the exchange, and ignores further responder offers while one is in flight, -// avoiding an offer-per-offer runaway. +// bootstrap for this peer. True if we are the initiator, the peer is not known +// non-capable, and no exchange of ours is in flight. The host uses this when it (as the +// controller) receives the responder's offer, which never carries KEM material: it replies +// with a KEM offer, and ignores further responder offers while that one is in flight, +// avoiding an offer-per-offer runaway. A converged exchange does not count as in flight: a +// responder offer then means the responder is renegotiating (a reconnect or a wake), and +// our current PSK may be the reason, e.g. split by a rotation whose answers were lost. A +// fresh bootstrap re-aligns both sides, so that reconnect cannot come back on a stale key. func (m *Manager) ShouldSendBootstrapOffer(remoteID RemoteID) bool { if !m.IsInitiator(remoteID) { return false @@ -409,7 +412,8 @@ func (m *Manager) ShouldSendBootstrapOffer(remoteID RemoteID) bool { if capable, ok := m.capable[remoteID]; ok && !capable { return false } - return m.exchanges[remoteID] == nil + ex := m.exchanges[remoteID] + return ex == nil || ex.state == stateAwaitingRekey } // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and diff --git a/client/internal/pqkem/split_test.go b/client/internal/pqkem/split_test.go new file mode 100644 index 00000000000..5edd9764e01 --- /dev/null +++ b/client/internal/pqkem/split_test.go @@ -0,0 +1,69 @@ +package pqkem + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestManager_ResponderOfferRebootstrapsSplitKeys reproduces a split PSK and checks that a +// reconnect started by the responder heals it. When every answer of a rotation is lost, +// the responder has committed the new PSK and the initiator has not, so WireGuard can no +// longer handshake. The connection layer then reconnects, and the reconnect offer may come +// from the responder, carrying no KEM. The controller used to answer it normally whenever +// it held a converged exchange, keeping its stale key, so the tunnel stayed down. A +// responder offer must instead be answered with a fresh KEM bootstrap. +func TestManager_ResponderOfferRebootstrapsSplitKeys(t *testing.T) { + sw := newSwitch() + wgA, wgB := newFakeWG(), newFakeWG() + dA := NewManager("aaaa", wgA, nil) // responder + dB := NewManager("bbbb", wgB, nil) // initiator (controller) + dB.retryInterval = 10 * time.Millisecond + dB.maxRetries = 2 + lbA := &loopback{ep: epA, sw: sw} + dA.Start(lbA) + dB.Start(&loopback{ep: epB, sw: sw}) + dA.AddPeer("bbbb", epB) + dB.AddPeer("aaaa", epA) + defer dA.Stop() + defer dB.Stop() + + bootstrap(t, dA, dB) + + // Lose every answer of a rotation: the responder commits, the initiator times out. + lbA.drop.Store(true) + dB.OnDataPathRekeyed("aaaa", 0) + require.Eventually(t, func() bool { + dB.mu.Lock() + defer dB.mu.Unlock() + ex := dB.exchanges["aaaa"] + return ex == nil || ex.state != stateAwaitingAnswer + }, time.Second, 5*time.Millisecond, "the rotation must time out on the initiator") + lbA.drop.Store(false) + require.NotEqual(t, wgA.psk("bbbb"), wgB.psk("aaaa"), "precondition: the two PSKs are split") + + // The responder's reconnect offer carries no KEM: the controller must answer it with + // a fresh bootstrap, which brings both peers back to one PSK. + require.True(t, dB.ShouldSendBootstrapOffer("aaaa"), + "a controller holding a converged exchange must still re-bootstrap on a responder offer") + bootstrap(t, dA, dB) + assert.Equal(t, wgA.psk("bbbb"), wgB.psk("aaaa"), "the re-bootstrap must heal the split PSKs") +} + +// TestManager_NoRebootstrapWhileExchangeInFlight keeps the glare protection: while the +// controller's own exchange is waiting for its answer, a crossing responder offer must not +// start a second exchange, which would supersede the first and churn the keys. +func TestManager_NoRebootstrapWhileExchangeInFlight(t *testing.T) { + dA, dB, _, _, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer, "the controller's bootstrap must be in flight") + + assert.False(t, dB.ShouldSendBootstrapOffer("aaaa"), + "no second exchange while the controller's own offer awaits its answer") +} From aa9141c451799d71e621c29e1f651e997dbe6c05 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 20 Oct 2026 17:53:02 +0200 Subject: [PATCH 104/119] [client] pqkem: make weak tests fail on the regressions they name The recovery test could time out its own bootstraps under CI load, the non-capable test passed with the cancellation removed, and the decoder rejection test only exercised the encoder. The timing spike ran on every go test; it is a benchmark now. --- client/internal/pqkem/capability_test.go | 40 +++++++++++++++++++++++ client/internal/pqkem/concurrency_test.go | 23 ++++++++++--- client/internal/pqkem/kem_test.go | 35 +++++--------------- client/internal/pqkem/message_test.go | 22 ++++++++++++- 4 files changed, 88 insertions(+), 32 deletions(-) diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index 0471cdb3f61..aabf2edd3d2 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -1,6 +1,7 @@ package pqkem import ( + "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -39,9 +40,28 @@ func TestManager_MarkNonCapableCancelsInFlight(t *testing.T) { require.NoError(t, err) require.NotNil(t, offer) + // Record the cancellation of the in-flight exchange's loop. Without it the loop would + // time out and raise OnRekeyFailed, the re-bootstrap storm MarkNonCapable prevents. + var cancelled atomic.Bool + d.mu.Lock() + ex := d.exchanges["aaaa"] + require.NotNil(t, ex, "precondition: the bootstrap exchange is in flight") + loopCancel := ex.cancel + ex.cancel = func() { + cancelled.Store(true) + loopCancel() + } + d.mu.Unlock() + // Now we learn the peer is non-PQ: the exchange must be dropped. d.MarkNonCapable("aaaa") + assert.True(t, cancelled.Load(), "the in-flight exchange's loop must be cancelled") + d.mu.Lock() + _, stillTracked := d.exchanges["aaaa"] + d.mu.Unlock() + assert.False(t, stillTracked, "the in-flight exchange must be dropped") + next, err := d.SignalOffer("aaaa") require.NoError(t, err) assert.Nil(t, next, "after learning non-capability the peer is no longer offered") @@ -97,3 +117,23 @@ func TestManager_ErrorMarkerIsBenign(t *testing.T) { assert.Equal(t, wgB.psk("aaaa"), wgA.psk("bbbb"), "the exchange must still converge after a benign error marker") assert.NotEqual(t, PSK{}, wgB.psk("aaaa")) } + +// TestManager_OfferErrorMarkerAnsweredWithMarker covers the other direction: the initiator +// failed to build its offer and sent a marker. The responder must answer with a marker of +// its own, never an empty answer, which the initiator would read as "peer has no KEM". +func TestManager_OfferErrorMarkerAnsweredWithMarker(t *testing.T) { + dA, dB, _, _, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + reply, err := dA.SignalOnOffer("bbbb", (&ErrorMsg{}).Encode()) + require.NoError(t, err) + require.NotEmpty(t, reply, "an empty reply would mark the responder non-capable") + + typ, _, err := Decode(reply) + require.NoError(t, err) + assert.Equal(t, MsgError, typ, "the reply to an offer marker must be a marker") + + _, established := dA.PSK("bbbb") + assert.False(t, established, "a marker must not derive a PSK") +} diff --git a/client/internal/pqkem/concurrency_test.go b/client/internal/pqkem/concurrency_test.go index bcf389404db..e81265ee2e4 100644 --- a/client/internal/pqkem/concurrency_test.go +++ b/client/internal/pqkem/concurrency_test.go @@ -17,11 +17,12 @@ func TestConcurrency_RecoversViaResignalAfterDataPathBreak(t *testing.T) { defer dA.Stop() defer dB.Stop() - // Tighten B's timings and make a single rotation miss raise OnRekeyFailed. Set - // before any exchange loop spawns (the loop reads these fields). + // A fast retransmit tick, set before any exchange loop spawns because the loop reads it + // unguarded at start. The retry budget stays generous for the two bootstraps, which + // must succeed: with a tight budget the loop could time an exchange out while the test + // goroutine is descheduled between the offer and the answer. dB.retryInterval = 5 * time.Millisecond - dB.maxRetries = 2 - dB.maxRekeyFailures = 1 + setRetryBudget(dB, 1000, 1) bootstrap(t, dA, dB) dA.OnDataPathRekeyed("bbbb", 0) @@ -30,7 +31,9 @@ func TestConcurrency_RecoversViaResignalAfterDataPathBreak(t *testing.T) { require.NotEqual(t, PSK{}, psk1) require.Equal(t, psk1, wgA.psk("bbbb"), "converged on the same PSK after bootstrap+rotation") - // Data path breaks: the rotation can no longer converge -> OnRekeyFailed. + // Data path breaks: the rotation can no longer converge, and with a budget of two + // retransmits a single miss raises OnRekeyFailed. + setRetryBudget(dB, 2, 1) lbB.drop.Store(true) _, err := dB.startExchangeTest("aaaa", false, ExchangeID{}) require.NoError(t, err) @@ -38,12 +41,22 @@ func TestConcurrency_RecoversViaResignalAfterDataPathBreak(t *testing.T) { // Recovery: re-bootstrap over signalling with the data path STILL broken. It must // still converge (signal is independent of the data path) on a fresh PSK. + setRetryBudget(dB, 1000, 1) bootstrap(t, dA, dB) psk2 := wgB.psk("aaaa") require.NotEqual(t, psk1, psk2, "recovery derived a fresh PSK") require.Equal(t, psk2, wgA.psk("bbbb"), "both sides resync after recovery") } +// setRetryBudget changes the retry and failure budgets of a running manager. The exchange +// loops read both under the manager lock, so they can change between phases of a test. +func setRetryBudget(m *Manager, maxRetries, maxRekeyFailures int) { + m.mu.Lock() + defer m.mu.Unlock() + m.maxRetries = maxRetries + m.maxRekeyFailures = maxRekeyFailures +} + // TestConcurrency_ConcurrentRekeysNoRace hammers both managers with concurrent rotation // clocks from many goroutines. Its primary job (with -race) is to prove the single-lock // state machine has no data races or deadlocks under contention; a final deterministic diff --git a/client/internal/pqkem/kem_test.go b/client/internal/pqkem/kem_test.go index 9a2c9c8f9ae..da5e98c1ed2 100644 --- a/client/internal/pqkem/kem_test.go +++ b/client/internal/pqkem/kem_test.go @@ -2,7 +2,6 @@ package pqkem import ( "testing" - "time" "github.com/stretchr/testify/require" ) @@ -74,33 +73,17 @@ func TestExchange_RejectsMalformedMessages(t *testing.T) { require.Error(t, err) } -// TestExchange_ReportSizesAndTiming is a spike measurement, not a pass/fail gate. -// Run with: go test -run TestExchange_ReportSizesAndTiming -v ./client/internal/pqkem/ -func TestExchange_ReportSizesAndTiming(t *testing.T) { - const iters = 200 - - var tInit, tResp, tFinish time.Duration - for i := 0; i < iters; i++ { - s0 := time.Now() +// BenchmarkExchange measures a full hybrid exchange (keygen, encapsulation, decapsulation) +// and reports its on-wire size. +// Run with: go test -run '^$' -bench BenchmarkExchange ./client/internal/pqkem/ +func BenchmarkExchange(b *testing.B) { + for i := 0; i < b.N; i++ { init, err := NewInitiator() - require.NoError(t, err) - tInit += time.Since(s0) - - s1 := time.Now() + require.NoError(b, err) answer, _, err := Respond(init.Offer(), Binding{LocalID: wgB, RemoteID: wgA}) - require.NoError(t, err) - tResp += time.Since(s1) - - s2 := time.Now() + require.NoError(b, err) _, err = init.Finish(answer, Binding{LocalID: wgA, RemoteID: wgB}) - require.NoError(t, err) - tFinish += time.Since(s2) + require.NoError(b, err) } - - t.Logf("wire sizes: offer=%d B answer=%d B (Rosenpass static pubkey ~524160 B)", OfferSize, AnswerSize) - t.Logf("total on-wire per handshake: %d B (~%.0fx smaller than RP static key)", OfferSize+AnswerSize, 524160.0/float64(OfferSize+AnswerSize)) - t.Logf("avg NewInitiator (keygen): %s", tInit/iters) - t.Logf("avg Respond (encaps+dh): %s", tResp/iters) - t.Logf("avg Finish (decaps+dh): %s", tFinish/iters) - t.Logf("avg full handshake CPU: %s", (tInit+tResp+tFinish)/iters) + b.ReportMetric(float64(OfferSize+AnswerSize), "wire-B/exchange") } diff --git a/client/internal/pqkem/message_test.go b/client/internal/pqkem/message_test.go index 0ce9be69d84..d061b9528d9 100644 --- a/client/internal/pqkem/message_test.go +++ b/client/internal/pqkem/message_test.go @@ -60,7 +60,27 @@ func TestDecodeRejects(t *testing.T) { _, _, err = Decode(bad2) require.Error(t, err) - // offer with wrong payload size + // the encoder refuses an offer with the wrong payload size _, err = (&OfferMsg{KEMOffer: []byte{1, 2, 3}}).Encode() require.Error(t, err) + + // the decoder refuses a framed offer, answer or error marker whose payload has the + // wrong size + short := make([]byte, headerSize+ExchangeIDSize+OfferSize-1) + short[0] = byte(MsgOffer) + short[1] = ProtocolVersion + _, _, err = Decode(short) + require.Error(t, err, "a truncated offer must be rejected") + + long := make([]byte, headerSize+AnswerSize+1) + long[0] = byte(MsgAnswer) + long[1] = ProtocolVersion + _, _, err = Decode(long) + require.Error(t, err, "an oversized answer must be rejected") + + marker := make([]byte, headerSize+1) + marker[0] = byte(MsgError) + marker[1] = ProtocolVersion + _, _, err = Decode(marker) + require.Error(t, err, "an error marker with a payload must be rejected") } From fb6e955f0003e3bb6eb5b34a0d393718495badc0 Mon Sep 17 00:00:00 2001 From: riccardom Date: Tue, 20 Oct 2026 17:53:03 +0200 Subject: [PATCH 105/119] [client] pqkem: cover the env flags, signal port, offer marker and status flag These branches decide whether ML-KEM runs, which port a peer is reached on, and what the status reports, and none of them had a test. --- client/internal/peer/conn_pq_test.go | 31 +++++++++++++++ client/internal/pqkem/env_test.go | 58 ++++++++++++++++++++++++++++ client/internal/pqkem_signal_test.go | 35 +++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 client/internal/pqkem/env_test.go create mode 100644 client/internal/pqkem_signal_test.go diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index 3aa7cfa46b3..ae11f70e88f 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -164,3 +164,34 @@ func TestConn_RecoverPQKeyOverSignalOnlyWhenConnected(t *testing.T) { c.RecoverPQKeyOverSignal() assert.True(t, offerAttempted(hook), "a connected peer must get a recovery offer") } + +// TestConn_QuantumResistant covers the status flag: Rosenpass counts only when both sides +// run it, and an established ML-KEM PSK counts on its own. +func TestConn_QuantumResistant(t *testing.T) { + localRP := []byte("local-rosenpass-key") + remoteRP := []byte("remote-rosenpass-key") + + cases := []struct { + name string + localRP []byte + remoteRP []byte + pqEstablished bool + want bool + }{ + {"nothing", nil, nil, false, false}, + {"rosenpass on both sides", localRP, remoteRP, false, true}, + {"rosenpass only locally", localRP, nil, false, false}, + {"rosenpass only remotely", nil, remoteRP, false, false}, + {"ml-kem established", nil, nil, true, true}, + {"ml-kem established, rosenpass only remotely", nil, remoteRP, true, true}, + {"ml-kem established, rosenpass on both sides", localRP, remoteRP, true, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + conn := &Conn{config: ConnConfig{RosenpassConfig: RosenpassConfig{PubKey: c.localRP}}} + assert.Equal(t, c.want, conn.quantumResistant(c.remoteRP, c.pqEstablished), + "local rosenpass %t, remote rosenpass %t, ml-kem %t", + c.localRP != nil, c.remoteRP != nil, c.pqEstablished) + }) + } +} diff --git a/client/internal/pqkem/env_test.go b/client/internal/pqkem/env_test.go new file mode 100644 index 00000000000..6af943eb8e8 --- /dev/null +++ b/client/internal/pqkem/env_test.go @@ -0,0 +1,58 @@ +package pqkem + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnvFlags(t *testing.T) { + cases := []struct { + value string + want bool + }{ + {"", false}, + {"on", true}, + {"ON", true}, + {" on ", true}, + {"off", false}, + {"Off", false}, + {"true", true}, + {"TRUE", true}, + {"1", true}, + {"t", true}, + {"false", false}, + {"0", false}, + {" 1 ", true}, + {"yes", false}, + {"garbage", false}, + } + + flags := []struct { + env string + get func() bool + }{ + {EnvEnabled, Enabled}, + {EnvStrict, Strict}, + } + + for _, f := range flags { + for _, c := range cases { + t.Run(f.env+"="+c.value, func(t *testing.T) { + t.Setenv(f.env, c.value) + assert.Equal(t, c.want, f.get(), "%s=%q", f.env, c.value) + }) + } + } +} + +func TestEnvFlagsUnset(t *testing.T) { + for _, env := range []string{EnvEnabled, EnvStrict} { + t.Setenv(env, "") + require.NoError(t, os.Unsetenv(env)) + } + assert.False(t, Enabled(), "an unset %s must leave ML-KEM disabled", EnvEnabled) + assert.False(t, Strict(), "an unset %s must leave strict mode off", EnvStrict) +} diff --git a/client/internal/pqkem_signal_test.go b/client/internal/pqkem_signal_test.go new file mode 100644 index 00000000000..9118802e943 --- /dev/null +++ b/client/internal/pqkem_signal_test.go @@ -0,0 +1,35 @@ +package internal + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + sProto "github.com/netbirdio/netbird/shared/signal/proto" +) + +func signalOfferWithMlkemPort(port uint32) *sProto.Message { + return &sProto.Message{Body: &sProto.Body{ + Type: sProto.Body_OFFER, + Payload: "ufrag:pwd", + MlkemPort: &port, + }} +} + +// TestConvertToOfferAnswer_MlkemPort checks that the proto's 32-bit ML-KEM port is narrowed +// to uint16 only when it fits: a larger value is a malformed message, not a port to +// truncate. +func TestConvertToOfferAnswer_MlkemPort(t *testing.T) { + for _, port := range []uint32{0, 51833, math.MaxUint16} { + oa, err := convertToOfferAnswer(signalOfferWithMlkemPort(port)) + require.NoError(t, err, "port %d", port) + assert.Equal(t, uint16(port), oa.MlkemPort, "port %d must survive the conversion", port) + } + + for _, port := range []uint32{math.MaxUint16 + 1, math.MaxUint32} { + _, err := convertToOfferAnswer(signalOfferWithMlkemPort(port)) + assert.Error(t, err, "port %d does not fit in uint16 and must be rejected", port) + } +} From 4eaa57e8b5f843e03ed0ad71aea730e3f75b47b1 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 21 Oct 2026 08:44:13 +0200 Subject: [PATCH 106/119] [client] pqkem: wake status subscribers when a peer turns quantum resistant The PSK usually lands after the connection came up, so the flag flips with no other state change. Without a notification, UI and CLI subscribers kept showing the peer as unprotected until something unrelated changed. --- client/internal/peer/status.go | 5 +++-- client/internal/peer/status_test.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 96b6d930fa8..067a9b58bea 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -782,17 +782,18 @@ func (d *Status) UpdatePeerFQDN(peerPubKey, fqdn string) error { // to show up in the status. The mark persists until the peer is removed. func (d *Status) MarkPeerQuantumResistant(peerPubKey string) error { d.mux.Lock() - defer d.mux.Unlock() - peerState, ok := d.peers[peerPubKey] if !ok { + d.mux.Unlock() return errors.New("peer doesn't exist") } d.pqMarked[peerPubKey] = struct{}{} peerState.RosenpassEnabled = true d.peers[peerPubKey] = peerState + d.mux.Unlock() + d.notifyStateChange() return nil } diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index d0de41fb691..a7458f12cbc 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -616,3 +616,22 @@ func TestStatus_PQMarkClearedOnRemove(t *testing.T) { require.NoError(t, err) assert.False(t, st.RosenpassEnabled, "a re-added peer must not inherit the removed peer's PQ mark") } + +// TestStatus_PQMarkWakesSubscribers checks that a PSK landing after the connection came up +// reaches status subscribers, which otherwise keep showing the peer as not protected. +func TestStatus_PQMarkWakesSubscribers(t *testing.T) { + const key = "eDQmZp9jhQGX7XdYpHGaFSpAouldr3Sbf7MZRcGTGGQ=" + status := NewRecorder("https://mgm") + require.NoError(t, status.AddPeer(key, "peer.netbird.cloud", "100.64.0.2", "")) + + id, changes := status.SubscribeToStateChanges() + defer status.UnsubscribeFromStateChanges(id) + + require.NoError(t, status.MarkPeerQuantumResistant(key)) + + select { + case <-changes: + default: + assert.Fail(t, "marking a peer quantum resistant must notify status subscribers") + } +} From ebeebe017955415199e6359784225f70252e80aa Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 21 Oct 2026 08:44:13 +0200 Subject: [PATCH 107/119] [client] pqkem: drop the previous PSKs on Stop They are WireGuard key material like the current ones, which Stop already clears, and nothing reads them after shutdown. --- client/internal/pqkem/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index caa80bfabfe..fdfa856a0a0 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -335,6 +335,7 @@ func (m *Manager) Stop() { m.mu.Lock() m.exchanges = make(map[RemoteID]*exchangeCtl) m.psks = make(map[RemoteID]PSK) + m.prevPSKs = make(map[RemoteID]PSK) m.mu.Unlock() } From 2c2f68457151b3f2684ac57a90a28dccd19f40c5 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 21 Oct 2026 08:44:13 +0200 Subject: [PATCH 108/119] [client] pqkem: test the controller reoffer through the handshaker The split-key test only checked the manager's decision, so a broken handshaker branch would still pass. The capability test asserted while holding the manager lock, which hangs instead of failing. --- client/internal/peer/handshaker_pq_test.go | 74 ++++++++++++++++++++++ client/internal/pqkem/capability_test.go | 14 ++-- 2 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 client/internal/peer/handshaker_pq_test.go diff --git a/client/internal/peer/handshaker_pq_test.go b/client/internal/peer/handshaker_pq_test.go new file mode 100644 index 00000000000..ec28954aac7 --- /dev/null +++ b/client/internal/peer/handshaker_pq_test.go @@ -0,0 +1,74 @@ +package peer + +import ( + "sync" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + relayClient "github.com/netbirdio/netbird/shared/relay/client" + signal "github.com/netbirdio/netbird/shared/signal/client" + sProto "github.com/netbirdio/netbird/shared/signal/proto" +) + +// bootstrapPQ is a fakePQ whose controller decision is set by the test. +type bootstrapPQ struct { + fakePQ + bootstrap bool +} + +func (b bootstrapPQ) ShouldSendBootstrapOffer(string) bool { return b.bootstrap } + +// newControllerHandshaker builds a handshaker for the controller side of a peer pair and +// records the type of every signalling message it sends. +func newControllerHandshaker(t *testing.T, pq PQHandshaker) (*Handshaker, func() []sProto.Body_Type) { + t.Helper() + var mu sync.Mutex + var sent []sProto.Body_Type + client := &signal.MockClient{ + ReadyFunc: func() bool { return true }, + SendFunc: func(msg *sProto.Message) error { + mu.Lock() + defer mu.Unlock() + sent = append(sent, msg.GetBody().GetType()) + return nil + }, + } + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + config := ConnConfig{Key: "aaaa", LocalKey: "bbbb", PQ: pq} // LocalKey > Key: controller + relay := &WorkerRelay{relayManager: &relayClient.Manager{}} + h := NewHandshaker(log.WithField("test", t.Name()), config, NewSignaler(client, key), nil, relay, nil) + return h, func() []sProto.Body_Type { + mu.Lock() + defer mu.Unlock() + return append([]sProto.Body_Type(nil), sent...) + } +} + +// TestHandshaker_ControllerReoffersOnResponderOffer drives the responder-originated +// reconnect through the handshaker: when the KEM manager asks for a fresh bootstrap, the +// controller replies to the responder's KEM-less offer with its own offer and sends no +// answer, so the reconnect re-derives one PSK on both sides. +func TestHandshaker_ControllerReoffersOnResponderOffer(t *testing.T) { + h, sent := newControllerHandshaker(t, bootstrapPQ{bootstrap: true}) + + h.handleRemoteOffer(OfferAnswer{}) + + assert.Equal(t, []sProto.Body_Type{sProto.Body_OFFER}, sent(), + "the controller must reply with its own offer, not an answer") +} + +// TestHandshaker_ControllerAnswersWhileExchangeInFlight keeps the glare protection: with +// the controller's own exchange in flight, the responder's offer gets a normal answer. +func TestHandshaker_ControllerAnswersWhileExchangeInFlight(t *testing.T) { + h, sent := newControllerHandshaker(t, bootstrapPQ{bootstrap: false}) + + h.handleRemoteOffer(OfferAnswer{}) + + assert.Equal(t, []sProto.Body_Type{sProto.Body_ANSWER}, sent(), + "with an exchange in flight the controller must answer normally") +} diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index aabf2edd3d2..88ab7b2b91f 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -45,13 +45,17 @@ func TestManager_MarkNonCapableCancelsInFlight(t *testing.T) { var cancelled atomic.Bool d.mu.Lock() ex := d.exchanges["aaaa"] - require.NotNil(t, ex, "precondition: the bootstrap exchange is in flight") - loopCancel := ex.cancel - ex.cancel = func() { - cancelled.Store(true) - loopCancel() + if ex != nil { + loopCancel := ex.cancel + ex.cancel = func() { + cancelled.Store(true) + loopCancel() + } } d.mu.Unlock() + // Asserted after unlocking: a failing require stops the test, and the deferred Stop + // would then block on the lock. + require.NotNil(t, ex, "precondition: the bootstrap exchange is in flight") // Now we learn the peer is non-PQ: the exchange must be dropped. d.MarkNonCapable("aaaa") From e5764dd9167f78493c1907790fb29bf018490e26 Mon Sep 17 00:00:00 2001 From: riccardom Date: Wed, 21 Oct 2026 13:22:21 +0200 Subject: [PATCH 109/119] [client] Raise the wasm size limit to 64 MB The post-quantum exchange adds about 270 KB to the wasm client, pushing it past 60 MB. ML-KEM runs in the browser too, so the code stays in. --- .github/workflows/wasm-build-validation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wasm-build-validation.yml b/.github/workflows/wasm-build-validation.yml index 5f21472e5aa..bbaec52da09 100644 --- a/.github/workflows/wasm-build-validation.yml +++ b/.github/workflows/wasm-build-validation.yml @@ -66,7 +66,7 @@ jobs: echo "Size: ${SIZE} bytes (${SIZE_MB} MB)" - if [ ${SIZE} -gt 62914560 ]; then - echo "Wasm binary size (${SIZE_MB}MB) exceeds 60MB limit!" + if [ ${SIZE} -gt 67108864 ]; then + echo "Wasm binary size (${SIZE_MB}MB) exceeds 64MB limit!" exit 1 fi From 50afdff228fe47410c5decf87471d4169ef76e62 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 22 Oct 2026 12:14:21 +0200 Subject: [PATCH 110/119] [client] pqkem: answer an unreadable offer with an error marker An empty answer tells the initiator the peer does not run the KEM, and it stops offering for good. A decode or version error is not that: a newer client talking to an older one would silently stay classical. --- client/internal/pqkem/capability_test.go | 46 ++++++++++++++++++++++++ client/internal/pqkem/manager.go | 11 +++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index 88ab7b2b91f..13acba1bc3d 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -141,3 +141,49 @@ func TestManager_OfferErrorMarkerAnsweredWithMarker(t *testing.T) { _, established := dA.PSK("bbbb") assert.False(t, established, "a marker must not derive a PSK") } + +// TestManager_UnreadableOfferAnsweredWithMarker: an offer the responder cannot process, such +// as one from a newer protocol version, must get an error marker. An empty answer would make +// the initiator mark the responder non-capable for good and stay classical with it. +func TestManager_UnreadableOfferAnsweredWithMarker(t *testing.T) { + dA, dB, _, _, _ := pair(t) + defer dB.Stop() + + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) + + newerVersion := append([]byte(nil), offer...) + newerVersion[1] = ProtocolVersion + 1 + answerType := (&ErrorMsg{}).Encode() + answerType[0] = byte(MsgAnswer) + + cases := map[string][]byte{ + "garbage": {1, 2, 3}, + "newer version": newerVersion, + "not an offer": answerType, + "truncated body": offer[:len(offer)-1], + } + for name, in := range cases { + t.Run(name, func(t *testing.T) { + reply, err := dA.SignalOnOffer("bbbb", in) + assert.Error(t, err) + assertErrorMarker(t, reply) + }) + } + + t.Run("manager stopping", func(t *testing.T) { + dA.Stop() + reply, err := dA.SignalOnOffer("bbbb", offer) + assert.Error(t, err) + assertErrorMarker(t, reply) + }) +} + +func assertErrorMarker(t *testing.T, reply []byte) { + t.Helper() + require.NotEmpty(t, reply, "an empty reply would mark the responder non-capable") + typ, _, err := Decode(reply) + require.NoError(t, err) + assert.Equal(t, MsgError, typ, "the reply must be an error marker") +} diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index fdfa856a0a0..d05b45b3fe0 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -418,15 +418,18 @@ func (m *Manager) ShouldSendBootstrapOffer(remoteID RemoteID) bool { } // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and -// returns the KEM answer for the host to embed in its outgoing answer. +// returns the KEM answer for the host to embed in its outgoing answer. Every failure +// still returns an error marker: an empty answer means "this peer does not run the KEM" +// and would stop the initiator from ever offering again. func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) { if !m.beginCall() { - return nil, fmt.Errorf("manager stopping") + return (&ErrorMsg{}).Encode(), fmt.Errorf("manager stopping") } defer m.wait.Done() typ, msg, err := Decode(offer) if err != nil { - return nil, fmt.Errorf("decode signal offer from %s: %w", remoteID, err) + // Also covers an offer from a newer protocol version this peer cannot read. + return (&ErrorMsg{}).Encode(), fmt.Errorf("decode signal offer from %s: %w", remoteID, err) } if typ == MsgError { // The initiator failed to build its offer and sent a marker. Answer with our own @@ -435,7 +438,7 @@ func (m *Manager) SignalOnOffer(remoteID RemoteID, offer []byte) ([]byte, error) return (&ErrorMsg{}).Encode(), nil } if typ != MsgOffer { - return nil, fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) + return (&ErrorMsg{}).Encode(), fmt.Errorf("expected offer from %s, got type %d", remoteID, typ) } o := msg.(*OfferMsg) answer, err := m.processOffer(remoteID, o, viaSignalLabel) From 70e6ce1095bc3be9868ca1578cabc0268d98a571 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 22 Oct 2026 12:17:49 +0200 Subject: [PATCH 111/119] [client] pqkem: reprogram the latest PSK after a stale WireGuard configure The conn reads the PSK before configuring WireGuard. A rotation committed in between pushes its key first, and the configure then overwrites it with the old one, splitting the keys until the next reconnect. --- client/internal/engine.go | 10 ++- client/internal/peer/conn.go | 21 ++++++- client/internal/peer/conn_pq_test.go | 1 + client/internal/pqkem_adapter.go | 47 ++++++++++++++ client/internal/pqkem_adapter_test.go | 91 +++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 3 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 89768da54e5..e6e760cd021 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -220,6 +220,9 @@ type Engine struct { // pqkemManager runs the ML-KEM post-quantum PSK exchange (gated by NB_ENABLE_PQ_MLKEM). // It owns the data-path transport and peer endpoint routing. pqkemManager *pqkem.Manager + // pqPSKReapplier restores the latest PQ PSK after a conn configures WireGuard with a + // stale one. Set together with pqkemManager. + pqPSKReapplier *pqPSKReapplier // syncMsgMux is used to guarantee sequential Management Service message processing syncMsgMux *sync.Mutex @@ -606,10 +609,11 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { log.Errorf("pqkem: transport bind failed, exchange disabled: %v", err) return nil } + applied := newAppliedGenerations() cbHandler := pqCallbackHandler{ wg: e.wgInterface, status: e.statusRecorder, - applied: newAppliedGenerations(), + applied: applied, // On a persistent rekey failure, re-bootstrap the KEM over Signal: a fresh // signalling offer starts a new exchange that overwrites the stalled PSK on both // sides, recovering from a data-path desync. @@ -626,6 +630,8 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { opts = append(opts, pqkem.WithPresharedSecret(e.config.PreSharedKey[:])) } e.pqkemManager = pqkem.NewManager(pqkem.LocalID(publicKey.String()), cbHandler, pqkem.NewLogger(), opts...) + // Shares the apply lock with the rotation push, so a reapply and a push are ordered. + e.pqPSKReapplier = &pqPSKReapplier{psks: e.pqkemManager, wg: e.wgInterface, applied: applied} e.pqkemManager.Start(tr) log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) return nil @@ -2095,7 +2101,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV NetMgr: e.netMgr, } if e.pqkemManager != nil { - config.PQ = pqHandshaker{mgr: e.pqkemManager} + config.PQ = pqHandshaker{mgr: e.pqkemManager, reapplier: e.pqPSKReapplier} config.PQStrict = pqkem.Strict() } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index e0e303d8ab5..9de4c80001b 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -97,6 +97,10 @@ type PQHandshaker interface { // PSK returns the peer's latest derived post-quantum PSK to program at WG // peer-config time (the pull path). ok is false until one has been derived. PSK(remoteKey string) (wgtypes.Key, bool) + // ReapplyPSK runs after the conn configured WireGuard with programmed, a PSK it read + // beforehand. A rotation committed in between was overwritten by that stale value, so + // the latest PSK is programmed again when it differs. + ReapplyPSK(remoteKey string, programmed *wgtypes.Key) // SetRemoteAddr registers the peer's data-path endpoint learned from signalling: // its WG overlay IP with the announced pq UDP port (port 0 means the peer omitted // it and is on the default port). @@ -538,6 +542,7 @@ func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConn return } wgConfigWorkaround() + conn.reapplyPQPSK(presharedKey) if conn.wgProxyRelay != nil { conn.Log.Debugf("redirect packets from relayed conn to WireGuard") @@ -585,6 +590,8 @@ func (conn *Conn) onICEStateDisconnected(sessionChanged bool) { presharedKey := conn.presharedKey(conn.rosenpassRemoteKey, pqPSK) if err := conn.endpointUpdater.SwitchWGEndpoint(conn.wgProxyRelay.EndpointAddr(), presharedKey); err != nil { conn.Log.Errorf("failed to switch to relay conn: %v", err) + } else { + conn.reapplyPQPSK(presharedKey) } conn.currentConnPriority = conntype.Relay @@ -664,7 +671,8 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) { } updateTime := time.Now() conn.enableWgWatcherIfNeeded(updateTime) - if err := conn.endpointUpdater.ConfigureWGEndpoint(wgProxy.EndpointAddr(), conn.presharedKey(rci.rosenpassPubKey, pqPSK)); err != nil { + presharedKey := conn.presharedKey(rci.rosenpassPubKey, pqPSK) + if err := conn.endpointUpdater.ConfigureWGEndpoint(wgProxy.EndpointAddr(), presharedKey); err != nil { if err := wgProxy.CloseConn(); err != nil { conn.Log.Warnf("Failed to close relay connection: %v", err) } @@ -676,6 +684,7 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) { } wgConfigWorkaround() + conn.reapplyPQPSK(presharedKey) conn.injectPendingFirstPacket(wgProxy, nil) @@ -1148,6 +1157,16 @@ func (conn *Conn) pqPSK() (*wgtypes.Key, bool) { return &psk, true } +// reapplyPQPSK restores the latest PQ PSK after a WireGuard configure that used +// programmed. It runs under conn.mu; the lock order conn.mu -> PSK apply -> manager holds, +// since neither of the other two calls back into Conn while locked. +func (conn *Conn) reapplyPQPSK(programmed *wgtypes.Key) { + if conn.config.PQ == nil { + return + } + conn.config.PQ.ReapplyPSK(conn.config.Key, programmed) +} + // IdlePresharedKey is the preshared key an idle (lazy) WireGuard peer is installed with. // An idle peer accepts a handshake the remote initiates directly, so in ML-KEM strict // mode it must already carry the PQ PSK, or the strict sentinel until one exists; diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index ae11f70e88f..6c14fe6cdfa 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -28,6 +28,7 @@ func (f fakePQ) ShouldSendBootstrapOffer(string) bool { return false } func (f fakePQ) AnswerPayload(string, []byte) ([]byte, uint16) { return nil, 0 } func (f fakePQ) OnAnswer(string, []byte) {} func (f fakePQ) PSK(string) (wgtypes.Key, bool) { return f.psk, f.ok } +func (f fakePQ) ReapplyPSK(string, *wgtypes.Key) {} func (f fakePQ) SetRemoteAddr(string, netip.AddrPort) {} func (f fakePQ) OnDataPathRekeyed(string, time.Duration) {} func (f fakePQ) OnDataPathDown(string) {} diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 8ec4a326c88..8d7a66950de 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -39,6 +39,43 @@ func (a *appliedGenerations) apply(peer string, gen uint64, set func() error) (b return true, set() } +// locked runs fn under the apply lock, so it is ordered with every PSK push. +func (a *appliedGenerations) locked(fn func()) { + a.mu.Lock() + defer a.mu.Unlock() + fn() +} + +// pqPSKSource is the subset of the ML-KEM manager the reapply needs. *pqkem.Manager +// satisfies it. +type pqPSKSource interface { + PSK(remoteID pqkem.RemoteID) (pqkem.PSK, bool) +} + +// pqPSKReapplier programs the latest PQ PSK again after a WireGuard configure that may +// have used a stale one. The conn reads the PSK before configuring, and a rotation that +// commits in between pushes its key first, which the configure then overwrites. +type pqPSKReapplier struct { + psks pqPSKSource + wg pqPresharedKeySetter + applied *appliedGenerations +} + +// reapply reads the latest PSK under the apply lock: a push already done is visible in +// it, and a later push of a newer key runs after this one and wins. +func (r *pqPSKReapplier) reapply(peer string, programmed *wgtypes.Key) { + r.applied.locked(func() { + psk, ok := r.psks.PSK(pqkem.RemoteID(peer)) + if !ok || (programmed != nil && wgtypes.Key(psk) == *programmed) { + return + } + log.Debugf("pqkem: reprogramming the latest PSK for peer %s after a stale configure", peer) + if err := r.wg.SetPresharedKey(peer, wgtypes.Key(psk), true); err != nil { + log.Warnf("pqkem: reprogram PSK for peer %s: %v", peer, err) + } + }) +} + // pqPresharedKeySetter is the subset of the WireGuard interface the ML-KEM callback // needs: programming a peer's preshared key. *iface.WGIface satisfies it. type pqPresharedKeySetter interface { @@ -119,6 +156,8 @@ func (h pqCallbackHandler) OnRekeyFailed(remoteID pqkem.RemoteID) error { // wiring the host's signalling offers/answers to the KEM exchange. type pqHandshaker struct { mgr *pqkem.Manager + // reapplier restores the latest PSK after a stale WireGuard configure. Nil disables it. + reapplier *pqPSKReapplier } // announcedPort is the PQ data-path port to advertise to peers. It is omitted (0) when @@ -198,6 +237,14 @@ func (p pqHandshaker) PSK(remoteKey string) (wgtypes.Key, bool) { return wgtypes.Key(psk), true } +// ReapplyPSK programs the peer's latest PSK again when the conn configured WireGuard with +// a different one. +func (p pqHandshaker) ReapplyPSK(remoteKey string, programmed *wgtypes.Key) { + if p.reapplier != nil { + p.reapplier.reapply(remoteKey, programmed) + } +} + // SetRemoteAddr registers the peer's data-path endpoint learned from signalling. A // zero port means the peer omitted it (it is on DefaultPort), so we resolve it here — // DefaultPort lives in this package, not in peer. Sends only ever fire once the tunnel diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index 2deda11253f..0492f687f0d 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -161,3 +161,94 @@ func TestPQCallbackHandler_AppliesGenerationsInOrder(t *testing.T) { defer setter.mu.Unlock() assert.Equal(t, wgtypes.Key(newer), setter.last, "WireGuard must keep the newest generation's PSK") } + +// fakePSKSource returns a fixed PSK for every peer. +type fakePSKSource struct { + psk pqkem.PSK + ok bool +} + +func (f fakePSKSource) PSK(pqkem.RemoteID) (pqkem.PSK, bool) { return f.psk, f.ok } + +// recordingPSKSetter records the last key programmed. +type recordingPSKSetter struct { + calls int + last wgtypes.Key +} + +func (r *recordingPSKSetter) SetPresharedKey(_ string, psk wgtypes.Key, _ bool) error { + r.calls++ + r.last = psk + return nil +} + +// TestPQPSKReapplier_RestoresRotatedKey reproduces a configure that read the PSK before a +// rotation committed: WireGuard was configured with the old key while the manager already +// holds the new one, and the push of the new key was overwritten. The reapply must +// program the new key again. +func TestPQPSKReapplier_RestoresRotatedKey(t *testing.T) { + stale, latest := wgtypes.Key{1}, pqkem.PSK{2} + setter := &recordingPSKSetter{} + r := &pqPSKReapplier{psks: fakePSKSource{psk: latest, ok: true}, wg: setter, applied: newAppliedGenerations()} + + r.reapply(pqTestPeerKey, &stale) + + assert.Equal(t, 1, setter.calls, "a stale configure must be followed by one reprogram") + assert.Equal(t, wgtypes.Key(latest), setter.last, "WireGuard must end on the latest PSK") +} + +// TestPQPSKReapplier_StrictSentinelReplaced covers strict mode: the configure programmed +// the sentinel because no PSK existed yet, and the first PSK landed right after. +func TestPQPSKReapplier_StrictSentinelReplaced(t *testing.T) { + sentinel, first := wgtypes.Key{9}, pqkem.PSK{3} + setter := &recordingPSKSetter{} + r := &pqPSKReapplier{psks: fakePSKSource{psk: first, ok: true}, wg: setter, applied: newAppliedGenerations()} + + r.reapply(pqTestPeerKey, &sentinel) + + assert.Equal(t, wgtypes.Key(first), setter.last, "the sentinel must be replaced by the first PQ PSK") +} + +// TestPQPSKReapplier_NoWriteWhenCurrent: the common case, where the configure already used +// the latest PSK or none exists yet, must not touch WireGuard. +func TestPQPSKReapplier_NoWriteWhenCurrent(t *testing.T) { + current := pqkem.PSK{4} + programmed := wgtypes.Key(current) + + setter := &recordingPSKSetter{} + r := &pqPSKReapplier{psks: fakePSKSource{psk: current, ok: true}, wg: setter, applied: newAppliedGenerations()} + r.reapply(pqTestPeerKey, &programmed) + assert.Zero(t, setter.calls, "a configure with the latest PSK needs no reprogram") + + r = &pqPSKReapplier{psks: fakePSKSource{}, wg: setter, applied: newAppliedGenerations()} + r.reapply(pqTestPeerKey, nil) + assert.Zero(t, setter.calls, "with no PSK derived yet there is nothing to reprogram") +} + +// lockCheckingSource records whether the apply lock was held when the PSK was read. +type lockCheckingSource struct { + applied *appliedGenerations + lockedRead bool +} + +func (l *lockCheckingSource) PSK(pqkem.RemoteID) (pqkem.PSK, bool) { + if l.applied.mu.TryLock() { + l.applied.mu.Unlock() + } else { + l.lockedRead = true + } + return pqkem.PSK{5}, true +} + +// TestPQPSKReapplier_ReadsUnderApplyLock checks the reapply reads the PSK while holding the +// lock the rotation push takes. Reading it outside would let a push land between the read +// and the write, and the reapply would restore the key it was meant to replace. +func TestPQPSKReapplier_ReadsUnderApplyLock(t *testing.T) { + applied := newAppliedGenerations() + src := &lockCheckingSource{applied: applied} + r := &pqPSKReapplier{psks: src, wg: &recordingPSKSetter{}, applied: applied} + + r.reapply(pqTestPeerKey, &wgtypes.Key{1}) + + assert.True(t, src.lockedRead, "the PSK must be read under the apply lock") +} From 95a976db88bccfbc9f928f4851f4dacf00952849 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 22 Oct 2026 12:58:05 +0200 Subject: [PATCH 112/119] [client] pqkem: leave a responder offer unanswered during an offer glare When the controller's KEM offer and the responder's offer cross, the controller used to answer the responder's offer without KEM. If the responder read that answer before the KEM offer, it could bring WireGuard up on no PSK and open a classical session. The responder's details still reach the controller in its answer to the KEM offer. --- client/internal/peer/conn.go | 3 +++ client/internal/peer/conn_pq_test.go | 1 + client/internal/peer/handshaker.go | 16 +++++++----- client/internal/peer/handshaker_pq_test.go | 29 +++++++++++++++++----- client/internal/pqkem/manager.go | 13 ++++++++++ client/internal/pqkem/split_test.go | 26 +++++++++++++++++++ client/internal/pqkem_adapter.go | 6 +++++ 7 files changed, 82 insertions(+), 12 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 9de4c80001b..c74583e32f2 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -89,6 +89,9 @@ type PQHandshaker interface { // received responder offer with our own KEM offer (true unless an exchange of ours is // already in flight, so we kick the KEM once and ignore further offers meanwhile). ShouldSendBootstrapOffer(remoteKey string) bool + // SignalOfferInFlight reports whether, as the controller, our KEM offer over + // signalling is still waiting for its answer, so a crossing responder offer is a glare. + SignalOfferInFlight(remoteKey string) bool // AnswerPayload processes a received KEM offer (nil if absent) and returns the KEM // answer to embed in the outgoing answer (nil if none) and the local PQ port. AnswerPayload(remoteKey string, recvOffer []byte) (payload []byte, port uint16) diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index 6c14fe6cdfa..85e7d714cff 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -25,6 +25,7 @@ type fakePQ struct { func (f fakePQ) OfferPayload(string) ([]byte, uint16) { return nil, 0 } func (f fakePQ) ShouldSendBootstrapOffer(string) bool { return false } +func (f fakePQ) SignalOfferInFlight(string) bool { return false } func (f fakePQ) AnswerPayload(string, []byte) ([]byte, uint16) { return nil, 0 } func (f fakePQ) OnAnswer(string, []byte) {} func (f fakePQ) PSK(string) (wgtypes.Key, bool) { return f.psk, f.ok } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index c7789847b38..c168a46eb55 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -206,16 +206,20 @@ func (h *Handshaker) handleRemoteAnswer(remoteOfferAnswer OfferAnswer) { // one is in flight are ignored (re-sending on every responder offer would be a runaway). // The re-offer reuses our stable ICE session id, so the peer dedups repeats. // -// Returns true ONLY when it actually sent a re-offer and thus took ownership of the peer's -// offer (the caller must not answer it). When there is nothing to re-offer — the peer is -// non-capable, or our own exchange is already in flight (a crossing offer) — it returns -// false so the caller keeps handling the offer normally (sendAnswer + notifyListeners). -// Returning true without sending would drop the ICE credentials and relay info the -// responder's offer carries, stalling a responder-initiated reconnect. +// Returns true when the caller must not answer the peer's offer: either a re-offer went +// out, or our own KEM offer is already in flight (an offer glare). In the glare the +// peer's credentials reach us in its answer to our offer, and a KEM-less answer from us +// could let the peer bring WireGuard up before it reads our offer, on no PSK at all. +// Otherwise, as for a non-capable peer, it returns false and the caller answers normally: +// dropping the offer then would lose the only copy of the peer's ICE and relay details. func (h *Handshaker) pqControllerReoffer() bool { if h.config.PQ == nil || !isController(h.config) { return false } + if h.config.PQ.SignalOfferInFlight(h.config.Key) { + h.log.Debugf("pqkem: controller received a responder offer while its own KEM offer is in flight, leaving it unanswered") + return true + } if !h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { h.log.Debugf("pqkem: controller received a responder offer with no fresh KEM offer to send, handling it normally") return false diff --git a/client/internal/peer/handshaker_pq_test.go b/client/internal/peer/handshaker_pq_test.go index ec28954aac7..22da26599d0 100644 --- a/client/internal/peer/handshaker_pq_test.go +++ b/client/internal/peer/handshaker_pq_test.go @@ -14,13 +14,15 @@ import ( sProto "github.com/netbirdio/netbird/shared/signal/proto" ) -// bootstrapPQ is a fakePQ whose controller decision is set by the test. +// bootstrapPQ is a fakePQ whose controller decisions are set by the test. type bootstrapPQ struct { fakePQ bootstrap bool + inFlight bool } func (b bootstrapPQ) ShouldSendBootstrapOffer(string) bool { return b.bootstrap } +func (b bootstrapPQ) SignalOfferInFlight(string) bool { return b.inFlight } // newControllerHandshaker builds a handshaker for the controller side of a peer pair and // records the type of every signalling message it sends. @@ -62,13 +64,28 @@ func TestHandshaker_ControllerReoffersOnResponderOffer(t *testing.T) { "the controller must reply with its own offer, not an answer") } -// TestHandshaker_ControllerAnswersWhileExchangeInFlight keeps the glare protection: with -// the controller's own exchange in flight, the responder's offer gets a normal answer. -func TestHandshaker_ControllerAnswersWhileExchangeInFlight(t *testing.T) { - h, sent := newControllerHandshaker(t, bootstrapPQ{bootstrap: false}) +// TestHandshaker_ControllerIgnoresResponderOfferInGlare covers an offer glare: the +// controller's KEM offer and the responder's KEM-less offer cross. The controller must leave +// the responder's offer unanswered. A KEM-less answer could reach the responder before the +// KEM offer and let it bring WireGuard up with no PSK. +func TestHandshaker_ControllerIgnoresResponderOfferInGlare(t *testing.T) { + h, sent := newControllerHandshaker(t, bootstrapPQ{inFlight: true}) + notified := false + h.AddICEListener(func(*OfferAnswer) { notified = true }) // called synchronously + + h.handleRemoteOffer(OfferAnswer{IceCredentials: IceCredentials{UFrag: "ufrag", Pwd: "pwd"}}) + + assert.Empty(t, sent(), "in a glare the controller must send neither an answer nor a second offer") + assert.False(t, notified, "in a glare the controller must not bring the connection up from the responder's offer") +} + +// TestHandshaker_ControllerAnswersNonCapablePeer keeps the normal answer when there is no +// KEM to run with the peer: its offer is then the only copy of its connection details. +func TestHandshaker_ControllerAnswersNonCapablePeer(t *testing.T) { + h, sent := newControllerHandshaker(t, bootstrapPQ{}) h.handleRemoteOffer(OfferAnswer{}) assert.Equal(t, []sProto.Body_Type{sProto.Body_ANSWER}, sent(), - "with an exchange in flight the controller must answer normally") + "without a KEM to run the controller must answer normally") } diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index d05b45b3fe0..6d24731a097 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -417,6 +417,19 @@ func (m *Manager) ShouldSendBootstrapOffer(remoteID RemoteID) bool { return ex == nil || ex.state == stateAwaitingRekey } +// SignalOfferInFlight reports whether we are the initiator and our bootstrap offer over +// signalling still awaits its answer. A data-path rotation in flight does not count: no +// offer of ours is crossing the peer's on the signalling channel then. +func (m *Manager) SignalOfferInFlight(remoteID RemoteID) bool { + if !m.IsInitiator(remoteID) { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + ex := m.exchanges[remoteID] + return ex != nil && ex.viaSignal && ex.state == stateAwaitingAnswer +} + // SignalOnOffer processes a KEM offer the host extracted from an incoming offer and // returns the KEM answer for the host to embed in its outgoing answer. Every failure // still returns an error marker: an empty answer means "this peer does not run the KEM" diff --git a/client/internal/pqkem/split_test.go b/client/internal/pqkem/split_test.go index 5edd9764e01..bfc0e1b416d 100644 --- a/client/internal/pqkem/split_test.go +++ b/client/internal/pqkem/split_test.go @@ -67,3 +67,29 @@ func TestManager_NoRebootstrapWhileExchangeInFlight(t *testing.T) { assert.False(t, dB.ShouldSendBootstrapOffer("aaaa"), "no second exchange while the controller's own offer awaits its answer") } + +// TestManager_SignalOfferInFlight: only our own bootstrap offer over signalling, still +// awaiting its answer, makes a crossing responder offer a glare. A converged exchange, a +// data-path rotation, or the responder role must not. +func TestManager_SignalOfferInFlight(t *testing.T) { + dA, dB, _, _, _ := pair(t) // dB ("bbbb") initiator + defer dA.Stop() + defer dB.Stop() + + assert.False(t, dB.SignalOfferInFlight("aaaa"), "no exchange yet: nothing in flight") + + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer) + assert.True(t, dB.SignalOfferInFlight("aaaa"), "our bootstrap offer awaits its answer") + assert.False(t, dA.SignalOfferInFlight("bbbb"), "the responder never has a signalling offer in flight") + + answer, err := dA.SignalOnOffer("bbbb", offer) + require.NoError(t, err) + require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) + assert.False(t, dB.SignalOfferInFlight("aaaa"), "a converged exchange is not in flight") + + _, err = dB.startExchangeTest("aaaa", false, ExchangeID{}) + require.NoError(t, err) + assert.False(t, dB.SignalOfferInFlight("aaaa"), "a data-path rotation is not a signalling offer") +} diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 8d7a66950de..3f8e3992bc8 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -187,6 +187,12 @@ func (p pqHandshaker) ShouldSendBootstrapOffer(remoteKey string) bool { return p.mgr.ShouldSendBootstrapOffer(pqkem.RemoteID(remoteKey)) } +// SignalOfferInFlight reports whether our KEM offer over signalling to the peer still +// awaits its answer. +func (p pqHandshaker) SignalOfferInFlight(remoteKey string) bool { + return p.mgr.SignalOfferInFlight(pqkem.RemoteID(remoteKey)) +} + // AnswerPayload processes a received KEM offer (nil when absent) and returns the KEM // answer to attach to the outgoing signalling answer, plus the data-path port to announce // (0 when on DefaultPort). An empty offer is treated as a capability signal. From 3a6e634130b2f5aa5ebf474fc44ef8d850b4d553 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 22 Oct 2026 13:26:21 +0200 Subject: [PATCH 113/119] [client] pqkem: resend the in-flight KEM offer during an offer glare The offer exchange is installed before the Signal send, so a failed send leaves it in flight with nothing delivered. Ignoring the responder's offer then stalled the connection until the exchange timed out. Resending the same offer delivers it; the responder answers a duplicate from its cache. --- client/internal/peer/handshaker.go | 16 ++++++++++------ client/internal/peer/handshaker_pq_test.go | 13 +++++++------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index c168a46eb55..7842efa6aec 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -207,17 +207,21 @@ func (h *Handshaker) handleRemoteAnswer(remoteOfferAnswer OfferAnswer) { // The re-offer reuses our stable ICE session id, so the peer dedups repeats. // // Returns true when the caller must not answer the peer's offer: either a re-offer went -// out, or our own KEM offer is already in flight (an offer glare). In the glare the -// peer's credentials reach us in its answer to our offer, and a KEM-less answer from us -// could let the peer bring WireGuard up before it reads our offer, on no PSK at all. -// Otherwise, as for a non-capable peer, it returns false and the caller answers normally: -// dropping the offer then would lose the only copy of the peer's ICE and relay details. +// out, or our own KEM offer is already in flight (an offer glare). In the glare we send +// that same offer again instead of answering: a KEM-less answer could let the peer bring +// WireGuard up before it reads our offer, on no PSK at all. Resending covers an earlier +// send that failed after the exchange was installed, and the peer answers a duplicate +// from its cache. Otherwise, as for a non-capable peer, it returns false and the caller +// answers normally: the peer's offer is then the only copy of its ICE and relay details. func (h *Handshaker) pqControllerReoffer() bool { if h.config.PQ == nil || !isController(h.config) { return false } if h.config.PQ.SignalOfferInFlight(h.config.Key) { - h.log.Debugf("pqkem: controller received a responder offer while its own KEM offer is in flight, leaving it unanswered") + h.log.Debugf("pqkem: controller received a responder offer while its own KEM offer is in flight, resending that offer instead of answering") + if err := h.sendOffer(); err != nil { + h.log.Errorf("failed to resend KEM offer during an offer glare: %s", err) + } return true } if !h.config.PQ.ShouldSendBootstrapOffer(h.config.Key) { diff --git a/client/internal/peer/handshaker_pq_test.go b/client/internal/peer/handshaker_pq_test.go index 22da26599d0..4bbaa231f10 100644 --- a/client/internal/peer/handshaker_pq_test.go +++ b/client/internal/peer/handshaker_pq_test.go @@ -64,18 +64,19 @@ func TestHandshaker_ControllerReoffersOnResponderOffer(t *testing.T) { "the controller must reply with its own offer, not an answer") } -// TestHandshaker_ControllerIgnoresResponderOfferInGlare covers an offer glare: the -// controller's KEM offer and the responder's KEM-less offer cross. The controller must leave -// the responder's offer unanswered. A KEM-less answer could reach the responder before the -// KEM offer and let it bring WireGuard up with no PSK. -func TestHandshaker_ControllerIgnoresResponderOfferInGlare(t *testing.T) { +// TestHandshaker_ControllerResendsOfferInGlare covers an offer glare: the controller's KEM +// offer and the responder's KEM-less offer cross. The controller must not answer: a +// KEM-less answer could reach the responder before the KEM offer and let it bring WireGuard +// up with no PSK. It sends its in-flight offer again instead, in case the first send failed. +func TestHandshaker_ControllerResendsOfferInGlare(t *testing.T) { h, sent := newControllerHandshaker(t, bootstrapPQ{inFlight: true}) notified := false h.AddICEListener(func(*OfferAnswer) { notified = true }) // called synchronously h.handleRemoteOffer(OfferAnswer{IceCredentials: IceCredentials{UFrag: "ufrag", Pwd: "pwd"}}) - assert.Empty(t, sent(), "in a glare the controller must send neither an answer nor a second offer") + assert.Equal(t, []sProto.Body_Type{sProto.Body_OFFER}, sent(), + "in a glare the controller must resend its offer, not answer") assert.False(t, notified, "in a glare the controller must not bring the connection up from the responder's offer") } From 442b3b44b1761ef464956f4f295d649c4d918c0e Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 22 Oct 2026 13:55:38 +0200 Subject: [PATCH 114/119] [client] pqkem: take the netstack type from wgbackend Main now imports the wireguard-go fork directly instead of through a replace directive, so the upstream netstack path resolves to the original module, which no longer builds against the bumped gvisor. --- client/internal/pqkem_transport.go | 5 +++-- client/internal/pqkem_transport_test.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/client/internal/pqkem_transport.go b/client/internal/pqkem_transport.go index 5dfd088bd6e..825aeb09634 100644 --- a/client/internal/pqkem_transport.go +++ b/client/internal/pqkem_transport.go @@ -7,7 +7,8 @@ import ( "sync" log "github.com/sirupsen/logrus" - "golang.zx2c4.com/wireguard/tun/netstack" + + "github.com/netbirdio/netbird/client/iface/wgbackend" ) // DefaultPort is the preferred UDP port for the ML-KEM data-path service, bound on @@ -34,7 +35,7 @@ type pqTransport struct { // clients): the overlay IP then exists only inside that userspace stack, so the socket // is opened there and its datagrams ride the tunnel like any other overlay traffic. A // nil ns binds on the host. -func newPQTransport(overlayIP netip.Addr, ns *netstack.Net) (*pqTransport, error) { +func newPQTransport(overlayIP netip.Addr, ns *wgbackend.NetstackNet) (*pqTransport, error) { if !overlayIP.IsValid() { return nil, fmt.Errorf("invalid overlay IP for pqkem transport") } diff --git a/client/internal/pqkem_transport_test.go b/client/internal/pqkem_transport_test.go index ffaec70141e..0a963f6d649 100644 --- a/client/internal/pqkem_transport_test.go +++ b/client/internal/pqkem_transport_test.go @@ -7,7 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.zx2c4.com/wireguard/tun/netstack" + + "github.com/netbirdio/netbird/client/iface/wgbackend" ) type pqInbound struct { @@ -68,7 +69,7 @@ func TestPQTransport_HostRoundTrip(t *testing.T) { // starting); the transport must open its socket on the netstack instead. func TestPQTransport_NetstackRoundTrip(t *testing.T) { overlay := netip.MustParseAddr("192.0.2.1") - dev, ns, err := netstack.CreateNetTUN([]netip.Addr{overlay}, nil, 1280) + dev, ns, err := wgbackend.CreateNetTUN([]netip.Addr{overlay}, nil, 1280) require.NoError(t, err) t.Cleanup(func() { _ = dev.Close() }) From 2069329dd4d1f35f30389f8a6d6fa415ff5490f5 Mon Sep 17 00:00:00 2001 From: riccardom Date: Thu, 22 Oct 2026 13:56:23 +0200 Subject: [PATCH 115/119] [client] pqkem: check the glare resend carries the in-flight offer The handshaker test only saw that an offer went out, so a fresh or empty KEM offer would have passed. The manager test pins the part the resend relies on: the same bytes while in flight, and a duplicate answered from the responder's cache. --- client/internal/peer/handshaker_pq_test.go | 36 +++++++++++++++------- client/internal/pqkem/split_test.go | 27 ++++++++++++++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/client/internal/peer/handshaker_pq_test.go b/client/internal/peer/handshaker_pq_test.go index 4bbaa231f10..0ee1e6fc6e1 100644 --- a/client/internal/peer/handshaker_pq_test.go +++ b/client/internal/peer/handshaker_pq_test.go @@ -19,23 +19,26 @@ type bootstrapPQ struct { fakePQ bootstrap bool inFlight bool + offer []byte } +func (b bootstrapPQ) OfferPayload(string) ([]byte, uint16) { return b.offer, 0 } + func (b bootstrapPQ) ShouldSendBootstrapOffer(string) bool { return b.bootstrap } func (b bootstrapPQ) SignalOfferInFlight(string) bool { return b.inFlight } // newControllerHandshaker builds a handshaker for the controller side of a peer pair and -// records the type of every signalling message it sends. -func newControllerHandshaker(t *testing.T, pq PQHandshaker) (*Handshaker, func() []sProto.Body_Type) { +// records the body of every signalling message it sends. +func newControllerHandshaker(t *testing.T, pq PQHandshaker) (*Handshaker, func() []*sProto.Body) { t.Helper() var mu sync.Mutex - var sent []sProto.Body_Type + var sent []*sProto.Body client := &signal.MockClient{ ReadyFunc: func() bool { return true }, SendFunc: func(msg *sProto.Message) error { mu.Lock() defer mu.Unlock() - sent = append(sent, msg.GetBody().GetType()) + sent = append(sent, msg.GetBody()) return nil }, } @@ -44,11 +47,19 @@ func newControllerHandshaker(t *testing.T, pq PQHandshaker) (*Handshaker, func() config := ConnConfig{Key: "aaaa", LocalKey: "bbbb", PQ: pq} // LocalKey > Key: controller relay := &WorkerRelay{relayManager: &relayClient.Manager{}} h := NewHandshaker(log.WithField("test", t.Name()), config, NewSignaler(client, key), nil, relay, nil) - return h, func() []sProto.Body_Type { + return h, func() []*sProto.Body { mu.Lock() defer mu.Unlock() - return append([]sProto.Body_Type(nil), sent...) + return append([]*sProto.Body(nil), sent...) + } +} + +func bodyTypes(bodies []*sProto.Body) []sProto.Body_Type { + types := make([]sProto.Body_Type, 0, len(bodies)) + for _, b := range bodies { + types = append(types, b.GetType()) } + return types } // TestHandshaker_ControllerReoffersOnResponderOffer drives the responder-originated @@ -60,7 +71,7 @@ func TestHandshaker_ControllerReoffersOnResponderOffer(t *testing.T) { h.handleRemoteOffer(OfferAnswer{}) - assert.Equal(t, []sProto.Body_Type{sProto.Body_OFFER}, sent(), + assert.Equal(t, []sProto.Body_Type{sProto.Body_OFFER}, bodyTypes(sent()), "the controller must reply with its own offer, not an answer") } @@ -69,14 +80,17 @@ func TestHandshaker_ControllerReoffersOnResponderOffer(t *testing.T) { // KEM-less answer could reach the responder before the KEM offer and let it bring WireGuard // up with no PSK. It sends its in-flight offer again instead, in case the first send failed. func TestHandshaker_ControllerResendsOfferInGlare(t *testing.T) { - h, sent := newControllerHandshaker(t, bootstrapPQ{inFlight: true}) + inFlightOffer := []byte("in-flight KEM offer") + h, sent := newControllerHandshaker(t, bootstrapPQ{inFlight: true, offer: inFlightOffer}) notified := false h.AddICEListener(func(*OfferAnswer) { notified = true }) // called synchronously h.handleRemoteOffer(OfferAnswer{IceCredentials: IceCredentials{UFrag: "ufrag", Pwd: "pwd"}}) - assert.Equal(t, []sProto.Body_Type{sProto.Body_OFFER}, sent(), - "in a glare the controller must resend its offer, not answer") + bodies := sent() + require.Len(t, bodies, 1, "in a glare the controller must send exactly one message") + assert.Equal(t, sProto.Body_OFFER, bodies[0].GetType(), "in a glare the controller must resend its offer, not answer") + assert.Equal(t, inFlightOffer, bodies[0].GetMlkemPayload(), "the resend must carry the in-flight KEM offer") assert.False(t, notified, "in a glare the controller must not bring the connection up from the responder's offer") } @@ -87,6 +101,6 @@ func TestHandshaker_ControllerAnswersNonCapablePeer(t *testing.T) { h.handleRemoteOffer(OfferAnswer{}) - assert.Equal(t, []sProto.Body_Type{sProto.Body_ANSWER}, sent(), + assert.Equal(t, []sProto.Body_Type{sProto.Body_ANSWER}, bodyTypes(sent()), "without a KEM to run the controller must answer normally") } diff --git a/client/internal/pqkem/split_test.go b/client/internal/pqkem/split_test.go index bfc0e1b416d..412953266da 100644 --- a/client/internal/pqkem/split_test.go +++ b/client/internal/pqkem/split_test.go @@ -93,3 +93,30 @@ func TestManager_SignalOfferInFlight(t *testing.T) { require.NoError(t, err) assert.False(t, dB.SignalOfferInFlight("aaaa"), "a data-path rotation is not a signalling offer") } + +// TestManager_ResentSignalOfferIsTheSameExchange backs the glare resend: while our +// bootstrap offer awaits its answer, building the offer again must return the same bytes, +// not start a second exchange, and the responder must answer the duplicate from its cache +// so both sides still derive one PSK. +func TestManager_ResentSignalOfferIsTheSameExchange(t *testing.T) { + dA, dB, wgA, wgB, _ := pair(t) // dB ("bbbb") initiator + defer dA.Stop() + defer dB.Stop() + + first, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + resent, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + assert.Equal(t, first, resent, "a resend while in flight must carry the same offer") + + answer, err := dA.SignalOnOffer("bbbb", first) + require.NoError(t, err) + duplicateAnswer, err := dA.SignalOnOffer("bbbb", resent) + require.NoError(t, err) + assert.Equal(t, answer, duplicateAnswer, "the responder must answer a duplicate offer from its cache") + + require.NoError(t, dB.SignalOnAnswer("aaaa", answer)) + require.NoError(t, dB.SignalOnAnswer("aaaa", duplicateAnswer)) + assert.NotEqual(t, PSK{}, wgB.psk("aaaa")) + assert.Equal(t, wgB.psk("aaaa"), wgA.psk("bbbb"), "a resent offer must still converge on one PSK") +} From b7761579b558af0417092272a4960874b87b7faf Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 16 Nov 2026 10:14:09 +0100 Subject: [PATCH 116/119] [client] pqkem: follow a peer's ML-KEM capability in both directions A peer that disabled ML-KEM left us on its dropped PSK with the tunnel down, and one that enabled it stayed classical until our restart. --- client/internal/engine.go | 2 +- client/internal/peer/conn.go | 6 ++ client/internal/peer/conn_pq_test.go | 12 +++ client/internal/peer/status.go | 19 ++++ client/internal/pqkem/capability_test.go | 37 +++++-- client/internal/pqkem/manager.go | 70 +++++++++----- client/internal/pqkem_adapter.go | 56 +++++++---- client/internal/pqkem_adapter_test.go | 117 +++++++++++++++++++++++ 8 files changed, 263 insertions(+), 56 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 5037f577a05..4005dd26db6 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -2100,7 +2100,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV NetMgr: e.netMgr, } if e.pqkemManager != nil { - config.PQ = pqHandshaker{mgr: e.pqkemManager, reapplier: e.pqPSKReapplier} + config.PQ = pqHandshaker{mgr: e.pqkemManager, reapplier: e.pqPSKReapplier, status: e.statusRecorder} config.PQStrict = pqkem.Strict() } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index fda3e1b56dc..a30ba7398d4 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -1204,6 +1204,12 @@ func (conn *Conn) presharedKey(remoteRosenpassKey []byte, pqPSK *wgtypes.Key) *w conn.Log.Debugf("pqkem: strict mode — no PQ PSK yet, blocking peer traffic until the ML-KEM exchange converges") return conn.pqStrictSentinelKey } + // A nil key keeps the one WireGuard has, which after a downgrade is the dropped PQ + // PSK. Program the classical key explicitly; zero is WireGuard's "no PSK". + if conn.config.WgConfig.PreSharedKey != nil { + return conn.config.WgConfig.PreSharedKey + } + return &wgtypes.Key{} } if conn.config.RosenpassConfig.PubKey == nil { diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index 85e7d714cff..02976810082 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -82,6 +82,18 @@ func TestConn_presharedKey_PQ(t *testing.T) { require.Equal(t, nbPSK, *got, "non-strict falls through to the NetBird PSK, not a sentinel") }) + t.Run("non-strict with no NetBird PSK programs the zero key", func(t *testing.T) { + // A nil key leaves WireGuard's current one in place, so after a downgrade the stale + // PQ PSK would stay programmed and the classical handshake would never complete. + c := newConn() + c.config.WgConfig.PreSharedKey = nil + c.config.PQ = fakePQ{ok: false} + pqPSK, _ := c.pqPSK() + got := c.presharedKey(nil, pqPSK) + require.NotNil(t, got, "a nil key would keep a dropped PQ PSK programmed") + assert.Equal(t, wgtypes.Key{}, *got, "non-strict without a NetBird PSK must program the zero key") + }) + t.Run("strict blocks with the per-conn sentinel before a PSK exists", func(t *testing.T) { sentinel, err := wgtypes.GenerateKey() require.NoError(t, err) diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 067a9b58bea..13cde71887e 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -797,6 +797,25 @@ func (d *Status) MarkPeerQuantumResistant(peerPubKey string) error { return nil } +// UnmarkPeerQuantumResistant clears the post-quantum flag of a peer that stopped running +// the KEM, so the status stops claiming a protection its tunnel no longer has. +func (d *Status) UnmarkPeerQuantumResistant(peerPubKey string) error { + d.mux.Lock() + peerState, ok := d.peers[peerPubKey] + if !ok { + d.mux.Unlock() + return errors.New("peer doesn't exist") + } + + delete(d.pqMarked, peerPubKey) + peerState.RosenpassEnabled = false + d.peers[peerPubKey] = peerState + d.mux.Unlock() + + d.notifyStateChange() + return nil +} + // quantumResistantLocked merges a connection update's verdict with the PQ mark. The // connection path computes its verdict before configuring WireGuard, so it can write it // after the PSK was programmed; the mark keeps that stale false from clearing it. diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index 13acba1bc3d..1cf3533c4fa 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -72,9 +72,11 @@ func TestManager_MarkNonCapableCancelsInFlight(t *testing.T) { assert.Empty(t, wg.failed, "cancelling an in-flight exchange must not raise a failure") } -// TestManager_EstablishedPeerNotDowngraded: a stray zero-port observation must not tear -// down a peer we already have a working PQ session with. -func TestManager_EstablishedPeerNotDowngraded(t *testing.T) { +// TestManager_EstablishedPeerDowngraded: every failure sends an error marker, so a missing +// KEM from an established peer means it restarted without the KEM. Keeping the PSK would +// program a key the peer no longer has and the tunnel would never come up, so it is dropped +// and the peer is no longer offered. +func TestManager_EstablishedPeerDowngraded(t *testing.T) { dA, dB, _, wgB, _ := pair(t) defer dA.Stop() defer dB.Stop() @@ -82,12 +84,31 @@ func TestManager_EstablishedPeerNotDowngraded(t *testing.T) { bootstrap(t, dA, dB) require.NotEqual(t, PSK{}, wgB.psk("aaaa"), "established a PSK") - dB.MarkNonCapable("aaaa") // stray zero after establishment + assert.True(t, dB.MarkNonCapable("aaaa"), "dropping an established PSK must be reported") - // The peer keeps its derived PSK (MarkNonCapable is a no-op once established). - psk, ok := dB.PSK("aaaa") - assert.True(t, ok, "an established peer must keep its PSK despite a stray zero") - assert.NotEqual(t, PSK{}, psk) + _, ok := dB.PSK("aaaa") + assert.False(t, ok, "a downgraded peer must not keep its PSK") + offer, err := dB.SignalOffer("aaaa") + require.NoError(t, err) + assert.Nil(t, offer, "a downgraded peer must not be offered") + assert.False(t, dB.MarkNonCapable("aaaa"), "a peer already downgraded has no PSK left to drop") +} + +// TestManager_ClearNonCapableReoffers: a peer found non-capable is offered again once the +// host learns it runs the KEM, and the exchange converges. +func TestManager_ClearNonCapableReoffers(t *testing.T) { + dA, dB, wgA, wgB, _ := pair(t) + defer dA.Stop() + defer dB.Stop() + + assert.False(t, dB.MarkNonCapable("aaaa"), "a peer never established is not a downgrade") + require.False(t, dB.ShouldSendBootstrapOffer("aaaa"), "precondition: not offered") + + dB.ClearNonCapable("aaaa") + + assert.True(t, dB.ShouldSendBootstrapOffer("aaaa"), "a cleared peer must be offered again") + bootstrap(t, dA, dB) + assert.Equal(t, wgB.psk("aaaa"), wgA.psk("bbbb"), "the renewed exchange must converge") } // TestManager_ErrorMarkerIsBenign verifies finding F's wire half: a responder failure is diff --git a/client/internal/pqkem/manager.go b/client/internal/pqkem/manager.go index 6d24731a097..5b9e242af88 100644 --- a/client/internal/pqkem/manager.go +++ b/client/internal/pqkem/manager.go @@ -263,34 +263,56 @@ func (m *Manager) AddPeer(remoteID RemoteID, endpoint netip.AddrPort) { m.mu.Unlock() } -// MarkNonCapable records that a peer does not run the KEM: it answered our offer with -// no KEM material over signalling (the capability signal is the peer's payload, not its -// optional data-path port). Any in-flight exchange is cancelled and further offers are -// suppressed (see SignalOffer), so a non-PQ peer never drives the rekey-recovery storm. -// An already-established peer is left untouched — a stray empty answer must not tear -// down a working PQ session. -func (m *Manager) MarkNonCapable(remoteID RemoteID) { +// MarkNonCapable records that a peer does not run the KEM: its signalling message +// carried no KEM where one was due. Any in-flight exchange is cancelled and further +// offers are suppressed (see SignalOffer), so a non-PQ peer never drives the +// rekey-recovery storm. Every failure path sends an error marker instead of an empty +// payload, so for a peer we hold a PSK with this is a downgrade: it restarted without the +// KEM and lost the PSK. Ours is dropped too, and downgraded reports it, so the host stops +// programming a key the peer no longer has. +func (m *Manager) MarkNonCapable(remoteID RemoteID) (downgraded bool) { m.mu.Lock() defer m.mu.Unlock() - if m.established[remoteID] { - return - } - if prev, ok := m.capable[remoteID]; ok && !prev { - return // already known non-capable, nothing to do + _, downgraded = m.psks[remoteID] + if prev, ok := m.capable[remoteID]; ok && !prev && !downgraded { + return false // already known non-capable, nothing to do } + m.resetPeerLocked(remoteID) m.capable[remoteID] = false - if ex := m.exchanges[remoteID]; ex != nil { - if ex.cancel != nil { - ex.cancel() - } - delete(m.exchanges, remoteID) + if downgraded { + m.debug("pqkem: peer stopped running the KEM, dropping its PSK", "peer", remoteID) + return true } m.trace("pqkem: peer advertises no PQ service — treating as non-capable, no KEM attempted", "peer", remoteID) + return false +} + +// ClearNonCapable forgets that a peer was found non-capable, so the next signalling round +// offers it the KEM again. The host calls it when the peer announces a PQ port: the peer +// runs the KEM now, though only its next payload proves it. +func (m *Manager) ClearNonCapable(remoteID RemoteID) { + m.mu.Lock() + defer m.mu.Unlock() + if capable, ok := m.capable[remoteID]; ok && !capable { + delete(m.capable, remoteID) + } } // RemovePeer stops any in-flight exchange for a peer and drops its state and routing. func (m *Manager) RemovePeer(remoteID RemoteID) { m.mu.Lock() + m.resetPeerLocked(remoteID) + if ep, ok := m.peerAddrs[remoteID]; ok { + delete(m.peersByAddr, ep) + delete(m.peerAddrs, remoteID) + } + m.mu.Unlock() +} + +// resetPeerLocked cancels the peer's in-flight exchange and drops its keys and +// capability, leaving its routing in place. Generations keep counting, so the host never +// takes a later key for a stale one. Assumes m.mu is held. +func (m *Manager) resetPeerLocked(remoteID RemoteID) { if ex, ok := m.exchanges[remoteID]; ok { if ex.cancel != nil { ex.cancel() @@ -302,11 +324,6 @@ func (m *Manager) RemovePeer(remoteID RemoteID) { delete(m.psks, remoteID) delete(m.prevPSKs, remoteID) delete(m.capable, remoteID) - if ep, ok := m.peerAddrs[remoteID]; ok { - delete(m.peersByAddr, ep) - delete(m.peerAddrs, remoteID) - } - m.mu.Unlock() } // Stop cancels all in-flight exchanges, closes the transport, and waits for the @@ -355,13 +372,14 @@ func (m *Manager) Stop() { // PSK still survives idle in the manager (dropped only on account-level peer removal), // so a pure lazy wake with no re-negotiation reuses it via the conn's WG-config pull. func (m *Manager) SignalOffer(remoteID RemoteID) ([]byte, error) { - if !m.beginCall() { - return nil, fmt.Errorf("manager stopping") - } - defer m.wait.Done() if !m.IsInitiator(remoteID) { return nil, nil } + if !m.beginCall() { + // A marker, not nil: an empty offer would read as a downgrade on the responder. + return (&ErrorMsg{}).Encode(), fmt.Errorf("manager stopping") + } + defer m.wait.Done() m.mu.Lock() if capable, ok := m.capable[remoteID]; ok && !capable { m.mu.Unlock() diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 3f8e3992bc8..0e7e061809e 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -82,10 +82,12 @@ type pqPresharedKeySetter interface { SetPresharedKey(peerKey string, psk wgtypes.Key, updateOnly bool) error } -// pqStatusMarker is the subset of the status recorder the ML-KEM callback needs: flagging -// a peer as post-quantum protected once its PSK is programmed. *peer.Status satisfies it. +// pqStatusMarker is the subset of the status recorder ML-KEM needs: flagging a peer as +// post-quantum protected once its PSK is programmed, and clearing the flag when the peer +// downgrades. *peer.Status satisfies it. type pqStatusMarker interface { MarkPeerQuantumResistant(peerPubKey string) error + UnmarkPeerQuantumResistant(peerPubKey string) error } // pqCallbackHandler programs the derived PQ PSK onto the WireGuard peer. It is the @@ -158,16 +160,26 @@ type pqHandshaker struct { mgr *pqkem.Manager // reapplier restores the latest PSK after a stale WireGuard configure. Nil disables it. reapplier *pqPSKReapplier + // status clears the peer's "Quantum resistance" flag when it downgrades. Nil disables it. + status pqStatusMarker } -// announcedPort is the PQ data-path port to advertise to peers. It is omitted (0) when -// the manager is on DefaultPort, since peers assume the default when no port is sent; -// only a non-default (collision-forced) port is announced explicitly. +// announcedPort is the PQ data-path port to advertise to peers. It is sent even on +// DefaultPort: a non-zero port is how a peer that runs the KEM announces it, and a +// responder's offer carries nothing else that could (see SetRemoteAddr). func (p pqHandshaker) announcedPort() uint16 { - if port := p.mgr.LocalPort(); port != DefaultPort { - return uint16(port) + return uint16(p.mgr.LocalPort()) +} + +// markNonCapable records that the peer sent no KEM where one was due, and clears its +// status flag when that drops the PSK we held with it. +func (p pqHandshaker) markNonCapable(remoteKey string) { + if !p.mgr.MarkNonCapable(pqkem.RemoteID(remoteKey)) || p.status == nil { + return + } + if err := p.status.UnmarkPeerQuantumResistant(remoteKey); err != nil { + log.Debugf("pqkem: unmark peer %s quantum resistant in status: %v", remoteKey, err) } - return 0 } // OfferPayload builds the KEM offer to attach to an outgoing signalling offer for the @@ -204,7 +216,7 @@ func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, // offer is normal — the peer is the responder and puts its material in the // answer — so we must not flag it. if !p.mgr.IsInitiator(pqkem.RemoteID(remoteKey)) { - p.mgr.MarkNonCapable(pqkem.RemoteID(remoteKey)) + p.markNonCapable(remoteKey) } return nil, p.announcedPort() } @@ -219,13 +231,12 @@ func (p pqHandshaker) AnswerPayload(remoteKey string, recvOffer []byte) ([]byte, // answer to our offer is treated as a capability signal on the initiator side. func (p pqHandshaker) OnAnswer(remoteKey string, recvAnswer []byte) { if len(recvAnswer) == 0 { - // Capability signal (initiator side): the KEM answer flows responder->initiator, - // so an empty answer to our offer means the peer does not run the KEM — mark it - // non-capable to stop offering (no failure/recovery storm). Only meaningful when - // we are the initiator: as the responder we also receive an (empty) answer to - // our own non-KEM offer from a perfectly capable peer, which must not be flagged. - if p.mgr.IsInitiator(pqkem.RemoteID(remoteKey)) { - p.mgr.MarkNonCapable(pqkem.RemoteID(remoteKey)) + // Capability signal (initiator side): an empty answer to our KEM offer means the + // peer does not run the KEM, so mark it non-capable to stop offering (no + // failure/recovery storm). Answering an offer that carried no KEM, an empty answer + // says nothing, from either role. + if p.mgr.SignalOfferInFlight(pqkem.RemoteID(remoteKey)) { + p.markNonCapable(remoteKey) } return } @@ -251,11 +262,12 @@ func (p pqHandshaker) ReapplyPSK(remoteKey string, programmed *wgtypes.Key) { } } -// SetRemoteAddr registers the peer's data-path endpoint learned from signalling. A -// zero port means the peer omitted it (it is on DefaultPort), so we resolve it here — -// DefaultPort lives in this package, not in peer. Sends only ever fire once the tunnel -// is up (clocked by OnDataPathRekeyed), so registering here is safe even before -// connection-up. +// SetRemoteAddr registers the peer's data-path endpoint learned from signalling. Sends +// only ever fire once the tunnel is up (clocked by OnDataPathRekeyed), so registering here +// is safe even before connection-up. A non-zero port means the peer runs the KEM, so one +// found non-capable earlier is offered it again; this is how a peer that enabled ML-KEM +// after we met it gets upgraded. A zero port comes from a peer that does not run the KEM +// and is resolved to DefaultPort, harmlessly, since nothing is sent to such a peer. func (p pqHandshaker) SetRemoteAddr(remoteKey string, addr netip.AddrPort) { if !addr.Addr().IsValid() { return @@ -263,6 +275,8 @@ func (p pqHandshaker) SetRemoteAddr(remoteKey string, addr netip.AddrPort) { port := addr.Port() if port == 0 { port = DefaultPort + } else { + p.mgr.ClearNonCapable(pqkem.RemoteID(remoteKey)) } p.mgr.AddPeer(pqkem.RemoteID(remoteKey), netip.AddrPortFrom(addr.Addr(), port)) } diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index 0492f687f0d..8301b796085 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -2,6 +2,7 @@ package internal import ( "errors" + "net/netip" "sync" "testing" "time" @@ -252,3 +253,119 @@ func TestPQPSKReapplier_ReadsUnderApplyLock(t *testing.T) { assert.True(t, src.lockedRead, "the PSK must be read under the apply lock") } + +// pqInitiatorKey sorts above pqTestPeerKey, so its manager is the KEM initiator. +const pqInitiatorKey = "kXQmZp9jhQGX7XdYpHGaFSpAouldr3Sbf7MZRcGTGGQ=" + +var pqTestOverlay = netip.MustParseAddr("100.64.0.9") + +// pqStubTransport is a bound PQ data path that never delivers anything; signalling is all +// these tests exercise. +type pqStubTransport struct{} + +func (pqStubTransport) Send(netip.AddrPort, []byte) error { return nil } +func (pqStubTransport) LocalPort() int { return DefaultPort } +func (pqStubTransport) Run(func(netip.AddrPort, []byte)) {} +func (pqStubTransport) Close() error { return nil } + +// newPQHandshakerPair builds the initiator and responder handshakers over real managers. +// The tests play the host, carrying payloads between them as signalling would. +func newPQHandshakerPair(t *testing.T) (initiator, responder pqHandshaker) { + t.Helper() + mI := pqkem.NewManager(pqkem.LocalID(pqInitiatorKey), pqCallbackHandler{wg: &fakePSKSetter{}}, nil) + mR := pqkem.NewManager(pqkem.LocalID(pqTestPeerKey), pqCallbackHandler{wg: &fakePSKSetter{}}, nil) + mI.Start(pqStubTransport{}) + mR.Start(pqStubTransport{}) + t.Cleanup(mI.Stop) + t.Cleanup(mR.Stop) + require.True(t, mI.IsInitiator(pqkem.RemoteID(pqTestPeerKey)), "precondition: roles") + return pqHandshaker{mgr: mI}, pqHandshaker{mgr: mR} +} + +// pqSignalBootstrap runs one signalling offer/answer round that carries the KEM. +func pqSignalBootstrap(t *testing.T, hI, hR pqHandshaker) { + t.Helper() + offer, _ := hI.OfferPayload(pqTestPeerKey) + require.NotEmpty(t, offer, "the initiator must offer the KEM") + answer, _ := hR.AnswerPayload(pqInitiatorKey, offer) + require.NotEmpty(t, answer, "the responder must answer the KEM") + hI.OnAnswer(pqTestPeerKey, answer) + _, okI := hI.PSK(pqTestPeerKey) + _, okR := hR.PSK(pqInitiatorKey) + require.True(t, okI && okR, "precondition: both sides hold a PQ PSK") +} + +// TestPQHandshaker_InitiatorDowngrade: the initiator restarts without ML-KEM, so its next +// offer carries no KEM. The responder must drop its PSK, or it keeps programming a key the +// peer no longer has and the WireGuard handshake never completes. +func TestPQHandshaker_InitiatorDowngrade(t *testing.T) { + hI, hR := newPQHandshakerPair(t) + pqSignalBootstrap(t, hI, hR) + status := peer.NewRecorder("https://mgm") + require.NoError(t, status.AddPeer(pqInitiatorKey, "initiator.netbird.cloud", "100.64.0.3", "")) + require.NoError(t, status.MarkPeerQuantumResistant(pqInitiatorKey)) + hR.status = status + + hR.AnswerPayload(pqInitiatorKey, nil) + + _, ok := hR.PSK(pqInitiatorKey) + assert.False(t, ok, "a KEM-less offer from an established initiator must drop the PQ PSK") + st, err := status.GetPeer(pqInitiatorKey) + require.NoError(t, err) + assert.False(t, st.RosenpassEnabled, "a downgraded peer must not be reported quantum resistant") +} + +// TestPQHandshaker_ResponderDowngrade: the responder restarts without ML-KEM and answers +// our KEM offer with nothing. The initiator must drop its PSK. +func TestPQHandshaker_ResponderDowngrade(t *testing.T) { + hI, hR := newPQHandshakerPair(t) + pqSignalBootstrap(t, hI, hR) + status := newPQTestStatus(t) + require.NoError(t, status.MarkPeerQuantumResistant(pqTestPeerKey)) + hI.status = status + + offer, _ := hI.OfferPayload(pqTestPeerKey) + require.NotEmpty(t, offer, "an established initiator still offers on a new signalling round") + hI.OnAnswer(pqTestPeerKey, nil) + + _, ok := hI.PSK(pqTestPeerKey) + assert.False(t, ok, "a KEM-less answer to our KEM offer must drop the PQ PSK") + st, err := status.GetPeer(pqTestPeerKey) + require.NoError(t, err) + assert.False(t, st.RosenpassEnabled, "a downgraded peer must not be reported quantum resistant") +} + +// TestPQHandshaker_NonCapablePeerUpgrades: the responder ran without ML-KEM, so the +// initiator marked it non-capable. Once it restarts with ML-KEM it announces its PQ port, +// and the initiator must offer the KEM again instead of staying classical until restart. +func TestPQHandshaker_NonCapablePeerUpgrades(t *testing.T) { + hI, hR := newPQHandshakerPair(t) + + offer, _ := hI.OfferPayload(pqTestPeerKey) + require.NotEmpty(t, offer) + hI.OnAnswer(pqTestPeerKey, nil) + require.False(t, hI.ShouldSendBootstrapOffer(pqTestPeerKey), "precondition: marked non-capable") + + _, port := hR.AnswerPayload(pqInitiatorKey, nil) + require.NotZero(t, port, "a peer running ML-KEM must announce its PQ port") + hI.SetRemoteAddr(pqTestPeerKey, netip.AddrPortFrom(pqTestOverlay, port)) + + assert.True(t, hI.ShouldSendBootstrapOffer(pqTestPeerKey), "an announced PQ port must make the peer worth a KEM offer again") + pqSignalBootstrap(t, hI, hR) +} + +// TestPQHandshaker_EmptyAnswerWithoutOfferKeepsHint: an empty answer is only a capability +// signal when it answers our KEM offer. Answering an offer that carried no KEM, it says +// nothing, and must not undo the PQ port the same answer announced. +func TestPQHandshaker_EmptyAnswerWithoutOfferKeepsHint(t *testing.T) { + hI, _ := newPQHandshakerPair(t) + + offer, _ := hI.OfferPayload(pqTestPeerKey) + require.NotEmpty(t, offer) + hI.OnAnswer(pqTestPeerKey, nil) + + hI.SetRemoteAddr(pqTestPeerKey, netip.AddrPortFrom(pqTestOverlay, 1)) + hI.OnAnswer(pqTestPeerKey, nil) + + assert.True(t, hI.ShouldSendBootstrapOffer(pqTestPeerKey), "an empty answer with no KEM offer in flight must not mark the peer non-capable") +} From e825ebd0fc0e1dc76e5ddea7274874baecdd7204 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 16 Nov 2026 13:22:53 +0100 Subject: [PATCH 117/119] [client] pqkem: warn about a failed key exchange only for a connected peer With lazy connections off, every reconnect offer to an offline peer ended in two warnings. --- client/internal/peer/conn.go | 1 + client/internal/peer/conn_pq_test.go | 13 ++++++ client/internal/pqkem/capability_test.go | 57 ++++++++++++++++++++++++ client/internal/pqkem/convergence.go | 3 +- client/internal/pqkem_adapter.go | 2 +- client/internal/pqkem_adapter_test.go | 18 ++++++++ 6 files changed, 92 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index a30ba7398d4..665e6736186 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -787,6 +787,7 @@ func (conn *Conn) RecoverPQKeyOverSignal() { conn.Log.Debugf("pqkem: peer not connected, leaving the KEM re-bootstrap to the reconnect guard") return } + conn.Log.Warnf("pqkem: post-quantum key exchange failed with a connected peer, re-bootstrapping over signal") if err := h.SendOffer(); err != nil { conn.Log.Debugf("pqkem: PQ key recovery over signal failed: %v", err) } diff --git a/client/internal/peer/conn_pq_test.go b/client/internal/peer/conn_pq_test.go index 02976810082..2bf9ea5a0a8 100644 --- a/client/internal/peer/conn_pq_test.go +++ b/client/internal/peer/conn_pq_test.go @@ -169,14 +169,27 @@ func TestConn_RecoverPQKeyOverSignalOnlyWhenConnected(t *testing.T) { return false } + warned := func(hook *logtest.Hook) bool { + for _, e := range hook.AllEntries() { + if e.Level <= log.WarnLevel { + return true + } + } + return false + } + + // An offline peer's bootstrap can only time out; with lazy connections off that + // happens at every reconnect offer, so it must stay out of the warnings. c, hook := newConn() c.RecoverPQKeyOverSignal() assert.False(t, offerAttempted(hook), "a disconnected peer must be left to the reconnect guard") + assert.False(t, warned(hook), "a disconnected peer's failed exchange must not be warned about") c, hook = newConn() c.statusRelay.SetConnected() c.RecoverPQKeyOverSignal() assert.True(t, offerAttempted(hook), "a connected peer must get a recovery offer") + assert.True(t, warned(hook), "a failed exchange with a connected peer must be warned about") } // TestConn_QuantumResistant covers the status flag: Rosenpass counts only when both sides diff --git a/client/internal/pqkem/capability_test.go b/client/internal/pqkem/capability_test.go index 1cf3533c4fa..84d81196830 100644 --- a/client/internal/pqkem/capability_test.go +++ b/client/internal/pqkem/capability_test.go @@ -1,8 +1,12 @@ package pqkem import ( + "context" + "log/slog" + "sync" "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -208,3 +212,56 @@ func assertErrorMarker(t *testing.T, reply []byte) { require.NoError(t, err) assert.Equal(t, MsgError, typ, "the reply must be an error marker") } + +// levelRecorder keeps the warning-or-above records a manager logs. +type levelRecorder struct { + mu sync.Mutex + warns []slog.Record +} + +func (r *levelRecorder) Enabled(context.Context, slog.Level) bool { return true } +func (r *levelRecorder) Handle(_ context.Context, rec slog.Record) error { + r.mu.Lock() + defer r.mu.Unlock() + if rec.Level >= slog.LevelWarn { + r.warns = append(r.warns, rec) + } + return nil +} +func (r *levelRecorder) WithAttrs([]slog.Attr) slog.Handler { return r } +func (r *levelRecorder) WithGroup(string) slog.Handler { return r } + +func (r *levelRecorder) warnings() []string { + r.mu.Lock() + defer r.mu.Unlock() + var msgs []string + for _, rec := range r.warns { + msgs = append(msgs, rec.Message) + } + return msgs +} + +// TestManager_InitialTimeoutNotWarned: with lazy connections off, every reconnect offer to +// an offline peer starts a bootstrap that can only time out. The manager cannot tell that +// from a real failure, so it must report it without a warning and leave the severity to +// the host, which knows whether the peer is connected. +func TestManager_InitialTimeoutNotWarned(t *testing.T) { + wg := newFakeWG() + rec := &levelRecorder{} + d := NewManager("bbbb", wg, slog.New(rec)) // initiator vs "aaaa" + d.retryInterval = 5 * time.Millisecond + d.maxRetries = 1 + d.Start(&loopback{ep: epB, sw: newSwitch()}) + defer d.Stop() + + offer, err := d.SignalOffer("aaaa") + require.NoError(t, err) + require.NotNil(t, offer, "precondition: the bootstrap offer goes out") + + require.Eventually(t, func() bool { + wg.mu.Lock() + defer wg.mu.Unlock() + return len(wg.failed) == 1 + }, 2*time.Second, 5*time.Millisecond, "the unanswered bootstrap must still be reported to the host") + assert.Empty(t, rec.warnings(), "an unanswered bootstrap must not be logged as a warning") +} diff --git a/client/internal/pqkem/convergence.go b/client/internal/pqkem/convergence.go index 897c9718494..ca90889337a 100644 --- a/client/internal/pqkem/convergence.go +++ b/client/internal/pqkem/convergence.go @@ -385,7 +385,8 @@ func (m *Manager) raiseFailure(remoteID RemoteID, fail, initial bool) { return } if initial { - m.logger.Warn("pqkem: initial exchange failed — no PQ PSK established for peer (strict mode keeps the peer blocked until it converges)", "peer", remoteID) + // Debug: an offline peer's bootstrap ends here too. The host knows if it was reachable. + m.debug("pqkem: initial exchange failed, no PQ PSK established for peer", "peer", remoteID) } else { m.logger.Warn("pqkem: rekey failed after retries — staying on the previous PSK", "peer", remoteID) } diff --git a/client/internal/pqkem_adapter.go b/client/internal/pqkem_adapter.go index 0e7e061809e..3079fcbd178 100644 --- a/client/internal/pqkem_adapter.go +++ b/client/internal/pqkem_adapter.go @@ -147,7 +147,7 @@ func (h pqCallbackHandler) OnNewPSKReady(remoteID pqkem.RemoteID, gen uint64, ps // stays up on the previous PSK meanwhile (the Signal channel is independent of the // broken data path). func (h pqCallbackHandler) OnRekeyFailed(remoteID pqkem.RemoteID) error { - log.Warnf("pqkem: post-quantum rekey failed for peer %s, re-bootstrapping over signal", remoteID) + log.Debugf("pqkem: post-quantum key exchange failed for peer %s", remoteID) if h.recoverPQKeyOverSignal != nil { h.recoverPQKeyOverSignal(string(remoteID)) } diff --git a/client/internal/pqkem_adapter_test.go b/client/internal/pqkem_adapter_test.go index 8301b796085..a75aa9b9980 100644 --- a/client/internal/pqkem_adapter_test.go +++ b/client/internal/pqkem_adapter_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + log "github.com/sirupsen/logrus" + logtest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -369,3 +371,19 @@ func TestPQHandshaker_EmptyAnswerWithoutOfferKeepsHint(t *testing.T) { assert.True(t, hI.ShouldSendBootstrapOffer(pqTestPeerKey), "an empty answer with no KEM offer in flight must not mark the peer non-capable") } + +// TestPQCallbackHandler_RekeyFailedLeavesSeverityToConn: the handler cannot tell an +// offline peer from a real failure, so it must not warn; the connection decides. +func TestPQCallbackHandler_RekeyFailedLeavesSeverityToConn(t *testing.T) { + hook := logtest.NewGlobal() + defer hook.Reset() + + var recovered []string + h := pqCallbackHandler{recoverPQKeyOverSignal: func(k string) { recovered = append(recovered, k) }} + require.NoError(t, h.OnRekeyFailed(pqkem.RemoteID(pqTestPeerKey))) + + assert.Equal(t, []string{pqTestPeerKey}, recovered, "the failure must still be handed to the connection") + for _, e := range hook.AllEntries() { + assert.Greater(t, e.Level, log.WarnLevel, "unexpected %s: %s", e.Level, e.Message) + } +} From 0677e23dac46f5063fd2a7e42914914434c33a3f Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 16 Nov 2026 15:05:10 +0100 Subject: [PATCH 118/119] [client] Extract the overlay UDP accept rule from the Rosenpass firewall setup The ML-KEM transport needs the same rule for its own port. --- client/internal/engine.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 4005dd26db6..0acb66811a0 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -868,14 +868,19 @@ func (e *Engine) initFirewall() error { e.blockLanAccess() } - if e.rpManager == nil || !e.config.RosenpassEnabled { - return nil + if e.rpManager != nil && e.config.RosenpassEnabled { + e.allowOverlayUDPPort("rosenpass", uint16(e.rpManager.GetAddress().Port)) } - rosenpassPort := e.rpManager.GetAddress().Port - port := firewallManager.Port{Values: []uint16{uint16(rosenpassPort)}} + return nil +} + +// allowOverlayUDPPort accepts inbound UDP from any overlay peer on a port a local +// service listens on, so the peer ACLs do not drop it. Failure is logged, not fatal. +func (e *Engine) allowOverlayUDPPort(service string, portValue uint16) { + port := firewallManager.Port{Values: []uint16{portValue}} - // IPv4-only: rosenpass peers connect via AllowedIps[0] which is always v4. + // IPv4-only: the peers reach the service via AllowedIps[0], which is always v4. if _, err := e.firewall.AddFilterRule( nil, []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)}, @@ -885,13 +890,11 @@ func (e *Engine) initFirewall() error { &port, firewallManager.ActionAccept, ); err != nil { - log.Errorf("failed to allow rosenpass interface traffic: %v", err) - return nil + log.Errorf("failed to allow %s interface traffic: %v", service, err) + return } - log.Infof("rosenpass interface traffic allowed on port %d", rosenpassPort) - - return nil + log.Infof("%s interface traffic allowed on port %d", service, portValue) } func (e *Engine) blockLanAccess() { From 4afc71e8e3fcba6732423cfd908e3a126b1199b5 Mon Sep 17 00:00:00 2001 From: riccardom Date: Mon, 16 Nov 2026 15:06:18 +0100 Subject: [PATCH 119/119] [client] pqkem: accept the ML-KEM data-path port in the firewall Peer ACLs that do not open the port drop every rotation offer, so rotation never succeeds. --- client/internal/engine.go | 5 + client/internal/engine_pqkem_firewall_test.go | 93 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 client/internal/engine_pqkem_firewall_test.go diff --git a/client/internal/engine.go b/client/internal/engine.go index 0acb66811a0..b53b7e9c29c 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -632,6 +632,11 @@ func (e *Engine) startPQKEMManager(publicKey wgtypes.Key) error { // Shares the apply lock with the rotation push, so a reapply and a push are ordered. e.pqPSKReapplier = &pqPSKReapplier{psks: e.pqkemManager, wg: e.wgInterface, applied: applied} e.pqkemManager.Start(tr) + // Here rather than in initFirewall: the firewall comes up before the interface, and the + // transport can only bind once the interface is up. + if e.firewall != nil { + e.allowOverlayUDPPort("ML-KEM", uint16(e.pqkemManager.LocalPort())) + } log.Infof("pqkem: enabled (udp port %d on overlay %s)", e.pqkemManager.LocalPort(), e.config.WgAddr.IP) return nil } diff --git a/client/internal/engine_pqkem_firewall_test.go b/client/internal/engine_pqkem_firewall_test.go new file mode 100644 index 00000000000..03b20793bf6 --- /dev/null +++ b/client/internal/engine_pqkem_firewall_test.go @@ -0,0 +1,93 @@ +package internal + +import ( + "net" + "net/netip" + "testing" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + firewallManager "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/firewall/uspfilter" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/iface/wgbackend" + "github.com/netbirdio/netbird/client/internal/netflow" + "github.com/netbirdio/netbird/client/internal/pqkem" + "github.com/netbirdio/netbird/client/internal/routemanager" +) + +// pqFirewallIface is the minimal overlay interface the userspace firewall attaches to. +type pqFirewallIface struct{ addr wgaddr.Address } + +func (i pqFirewallIface) Name() string { return "wt-pq-test" } +func (i pqFirewallIface) Address() wgaddr.Address { return i.addr } +func (i pqFirewallIface) SetFilter(device.PacketFilter) error { return nil } +func (i pqFirewallIface) GetWGDevice() *wgbackend.Device { return nil } + +// firewallOnlyRouteManager accepts the firewall initFirewall hands to the route manager. +type firewallOnlyRouteManager struct{ *routemanager.MockManager } + +func (firewallOnlyRouteManager) SetFirewall(firewallManager.Manager) error { return nil } + +// pqInboundUDP builds an overlay UDP packet from a peer to the local PQ data-path port. +func pqInboundUDP(t *testing.T, src, dst netip.Addr, port uint16) []byte { + t.Helper() + ip := &layers.IPv4{Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP, SrcIP: net.IP(src.AsSlice()), DstIP: net.IP(dst.AsSlice())} + udp := &layers.UDP{SrcPort: layers.UDPPort(port), DstPort: layers.UDPPort(port)} + require.NoError(t, udp.SetNetworkLayerForChecksum(ip)) + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true} + require.NoError(t, gopacket.SerializeLayers(buf, opts, ip, udp, gopacket.Payload("pq-offer"))) + return buf.Bytes() +} + +// pqFirewallWGIface is an overlay interface without netstack, so the PQ transport binds a +// plain OS socket. startPQKEMManager calls nothing else on it. +type pqFirewallWGIface struct{ WGIface } + +func (pqFirewallWGIface) GetNet() *wgbackend.NetstackNet { return nil } + +// TestEngine_PQKEMStartAllowsItsPort: the PQ transport listens on the overlay, so with peer +// ACLs that do not open its port a rotation offer from a peer is dropped and rotation times +// out. The engine sets the firewall up first and starts the transport later, once the +// interface is up; the port must still end up accepted. +func TestEngine_PQKEMStartAllowsItsPort(t *testing.T) { + t.Setenv(pqkem.EnvEnabled, "true") + local := netip.MustParseAddr("100.64.0.1") + remote := netip.MustParseAddr("100.64.0.2") + fw, err := uspfilter.Create(uspfilter.Config{ + IFace: pqFirewallIface{addr: wgaddr.Address{IP: local, Network: netip.MustParsePrefix("100.64.0.0/10")}}, + FlowLogger: netflow.NewManager(nil, []byte{}, nil).GetLogger(), + MTU: iface.DefaultMTU, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, fw.Close(nil)) }) + require.NoError(t, fw.UpdateLocalIPs()) + + e := &Engine{ + // The transport binds on loopback: the filter only looks at the packet's port. + config: &EngineConfig{WgAddr: wgaddr.Address{IP: netip.MustParseAddr("127.0.0.1")}}, + firewall: fw, + routeManager: firewallOnlyRouteManager{&routemanager.MockManager{}}, + wgInterface: pqFirewallWGIface{}, + } + // Engine.Start order: firewall first, PQ transport after the interface is up. + require.NoError(t, e.initFirewall()) + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + require.NoError(t, e.startPQKEMManager(key.PublicKey())) + require.NotNil(t, e.pqkemManager, "precondition: the PQ manager started") + t.Cleanup(e.pqkemManager.Stop) + + packet := pqInboundUDP(t, remote, local, uint16(e.pqkemManager.LocalPort())) + assert.False(t, fw.FilterInbound(packet, len(packet)), "a peer's PQ data-path message must pass the firewall") + + other := pqInboundUDP(t, remote, local, uint16(e.pqkemManager.LocalPort())+1) + assert.True(t, fw.FilterInbound(other, len(other)), "only the PQ port may be opened") +}