Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions management/internals/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,9 @@ func (s *BaseServer) Stop() error {
if s.certListener != nil {
_ = s.certListener.Close()
}
if drainer, ok := s.AccountManager().(interface{ StartDraining() }); ok {
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
drainer.StartDraining()
}
s.ManagementGRPCServer().StartDraining()
s.stopGRPCServer()
deadline, _ := ctx.Deadline()
Expand Down
7 changes: 7 additions & 0 deletions management/server/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"time"

cacheStore "github.com/eko/gocache/lib/v4/store"
Expand Down Expand Up @@ -117,6 +118,11 @@ type DefaultAccountManager struct {

deletionHooksMu sync.RWMutex
deletionHooks []account.DeletionHook

// sessionEventsFrom is when peer session events start being recorded, set once at
// construction and read-only afterwards. draining is set when shutdown begins.
sessionEventsFrom time.Time
draining atomic.Bool
}

var _ account.Manager = (*DefaultAccountManager)(nil)
Expand Down Expand Up @@ -258,6 +264,7 @@ func BuildManager(
permissionsManager: permissionsManager,
disableDefaultPolicy: disableDefaultPolicy,
peerLiveness: peerLiveness,
sessionEventsFrom: start.Add(peerSessionEventsStartupGrace()),
}

am.certChallenges = newCertChallengeRefresher(am.refreshCertificateChallenges)
Expand Down
7 changes: 7 additions & 0 deletions management/server/activity/codes.go
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,11 @@ const (
// MCPSessionRevokedOnTokenReuse indicates that an MCP session was revoked because a refresh token or an authorization code was used twice
MCPSessionRevokedOnTokenReuse Activity = 148

// PeerConnected indicates that a peer opened its sync stream and is now connected
PeerConnected Activity = 149
// PeerDisconnected indicates that a peer's sync stream ended and the peer is now disconnected
PeerDisconnected Activity = 150

AccountDeleted Activity = 99999
)

Expand Down Expand Up @@ -489,6 +494,8 @@ var activityMap = map[Activity]Code{
MCPClientSignedIn: {"MCP client signed in", "mcp.session.start"},
MCPClientSignedOut: {"MCP client signed out", "mcp.session.revoke"},
MCPSessionRevokedOnTokenReuse: {"MCP session revoked after token reuse", "mcp.session.replay"},
PeerConnected: {"Peer connected", "peer.connect"},
PeerDisconnected: {"Peer disconnected", "peer.disconnect"},
}

// StringCode returns a string code of the activity
Expand Down
50 changes: 47 additions & 3 deletions management/server/peer.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ func (am *DefaultAccountManager) markPeerConnected(ctx context.Context, peerPubK

log.WithContext(ctx).Debugf("mark peer %s connected", peer.ID)

am.storePeerSessionEvent(ctx, accountID, peer, settings, activity.PeerConnected, nil)

if err = am.schedulePeerExpirations(ctx, accountID, peer, settings); err != nil {
return err
}
Expand Down Expand Up @@ -211,6 +213,9 @@ func (am *DefaultAccountManager) MarkPeerDisconnected(ctx context.Context, peerP

log.WithContext(ctx).Debugf("mark peer %s disconnected", peer.ID)

settings, settingsErr := am.Store.GetAccountSettings(ctx, store.LockingStrengthNone, accountID)
am.storePeerSessionEvent(ctx, accountID, peer, settings, activity.PeerDisconnected, peerDisconnectMeta(sessionStartedAt))

// Symmetric with MarkPeerConnected: when an embedded proxy peer goes
// offline, refresh the peers that had synthesized records pointing at
// it so they pull the stale entries instead of waiting out TTL.
Expand All @@ -223,9 +228,8 @@ func (am *DefaultAccountManager) MarkPeerDisconnected(ctx context.Context, peerP
}

if peer.AddedWithSSOLogin() && peer.InactivityExpirationEnabled {
settings, err := am.Store.GetAccountSettings(ctx, store.LockingStrengthNone, accountID)
if err != nil {
log.WithContext(ctx).Warnf("failed getting account settings to schedule inactivity expiration for peer %s: %v", peer.ID, err)
if settingsErr != nil {
log.WithContext(ctx).Warnf("failed getting account settings to schedule inactivity expiration for peer %s: %v", peer.ID, settingsErr)
} else if settings.PeerInactivityExpirationEnabled {
am.checkAndSchedulePeerInactivityExpiration(ctx, accountID)
}
Expand All @@ -234,6 +238,46 @@ func (am *DefaultAccountManager) MarkPeerDisconnected(ctx context.Context, peerP
return nil
}

// peerDisconnectMeta returns the extra event meta for a disconnect: how long the
// session that just ended lasted, in whole seconds. It is empty when the start is unknown.
func peerDisconnectMeta(sessionStartedAt int64) map[string]any {
if sessionStartedAt <= 0 {
return nil
}
duration := time.Since(time.Unix(0, sessionStartedAt)).Round(time.Second)
return map[string]any{"session_duration": int64(duration.Seconds())}
}

// storePeerSessionEvent records a peer connect or disconnect. A peer owned by a user is
// attributed to that user; otherwise the event is a system one, which keeps the event
// listing from looking up a user for every session change. Nil settings are loaded; if
// that fails the event is still stored, without the DNS domain in its FQDN. Nothing is
// stored while sessionEventsSuppressed.
func (am *DefaultAccountManager) storePeerSessionEvent(ctx context.Context, accountID string, peer *nbpeer.Peer, settings *types.Settings, code activity.Activity, extraMeta map[string]any) {
if am.sessionEventsSuppressed(code == activity.PeerDisconnected) {
return
}

dnsDomain := ""
settings, err := am.accountSettings(ctx, accountID, settings)
if err != nil {
log.WithContext(ctx).Debugf("get account settings for peer %s session event: %v", peer.ID, err)
} else {
dnsDomain = am.networkMapController.GetDNSDomain(settings)
}

meta := peer.EventMeta(dnsDomain)
for k, v := range extraMeta {
meta[k] = v
}

initiatorID := peer.UserID
if initiatorID == "" {
initiatorID = activity.SystemInitiator
}
am.StoreEvent(ctx, initiatorID, peer.ID, accountID, code, meta)
}

// resolvePeerLocation looks up the geo location for realIP, returning nil when
// there is nothing to apply: geo disabled, no real IP, the IP is unchanged from
// what the peer already has, or the lookup failed. Geo lookups are skipped on
Expand Down
50 changes: 50 additions & 0 deletions management/server/peer_session_events.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package server

import (
"os"
"time"

log "github.com/sirupsen/logrus"
)

const (
// EnvPeerSessionEventsStartupGrace is a Go duration, such as "2m" or "90s", during which
// a freshly started instance records no peer connect or disconnect events. "0" disables it.
EnvPeerSessionEventsStartupGrace = "NB_PEER_SESSION_EVENTS_STARTUP_GRACE"

defaultPeerSessionEventsStartupGrace = 2 * time.Minute
)

// peerSessionEventsStartupGrace reads the startup grace window once. A missing, negative or
// unparseable value falls back to the default, and the unparseable one is logged.
func peerSessionEventsStartupGrace() time.Duration {
val := os.Getenv(EnvPeerSessionEventsStartupGrace)
if val == "" {
return defaultPeerSessionEventsStartupGrace
}
grace, err := time.ParseDuration(val)
if err != nil {
log.Warnf("failed to parse %s, using %s: %v", EnvPeerSessionEventsStartupGrace, defaultPeerSessionEventsStartupGrace, err)
return defaultPeerSessionEventsStartupGrace
}
if grace < 0 {
log.Warnf("%s must not be negative, using %s", EnvPeerSessionEventsStartupGrace, defaultPeerSessionEventsStartupGrace)
return defaultPeerSessionEventsStartupGrace
}
return grace
}

// StartDraining marks the instance as shutting down, so peers disconnecting from here on
// leave no peer.disconnect event. It does not change what a disconnect writes.
func (am *DefaultAccountManager) StartDraining() {
am.draining.Store(true)
}

// sessionEventsSuppressed reports whether peer session events are skipped: during the
// startup window, when every peer reconnects to the new instance, and while draining.
func (am *DefaultAccountManager) sessionEventsSuppressed(disconnect bool) bool {
if time.Now().Before(am.sessionEventsFrom) {
return true
}
return disconnect && am.draining.Load()
}
Loading
Loading