Repository navigation
Expand file tree
/
Copy pathhermes_state_gateway.py
More file actions
992 lines (930 loc) · 56.6 KB
/
Copy pathhermes_state_gateway.py
File metadata and controls
992 lines (930 loc) · 56.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
"""Gateway-facing SessionDB persistence: routing index, peers, orphans, heartbeats, handoffs.
Mixin bound onto ``SessionDB`` via the MRO; built on its ``_read_ctx`` /
``_execute_write`` / ``_write_sql`` / ``_read_all`` primitives."""
from __future__ import annotations
import json
import logging
import sys
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple
from hermes_state_common import (
_RECOVERABLE_END_REASONS_SQL, _RESET_CHILD_SQL, _RESET_END_REASONS_SQL, _sql_json_extract,
_sql_session_last_active)
# Log-record parity with the origin module (caplog tests pin "hermes_state").
logger = logging.getLogger("hermes_state")
# Recursive CTE naming a session plus its compression ancestors (rows a
# resume must keep on one routing peer); branch/delegate/tool/reset rows stop it
# (same membership rule as get_compression_lineage / _CHAIN_STEP_SQL, #114271).
_COMPRESSION_LINEAGE_CTE = f"""
WITH RECURSIVE compression_lineage(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM compression_lineage lineage
JOIN sessions child ON child.id = lineage.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
AND {_sql_json_extract('child.model_config', '$._branched_from')} IS NULL
AND {_sql_json_extract('child.model_config', '$._delegate_from')} IS NULL
AND NOT ({_RESET_CHILD_SQL.format(a='child')})
AND COALESCE(child.source, '') != 'tool'
)
"""
# Projection shared by both peer-recovery queries (exact key, then peer tuple).
_PEER_SELECT_HEAD = """
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
(COALESCE(s.message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1
)) AS _has_messages
FROM sessions s
LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash
"""
# A row a *completed handoff* owns stays recoverable whatever end_reason it carries: the source
# interface's teardown runs AFTER ownership moved to the destination and can stamp a terminal reason
# (``cli_close``) before the destination's first reply — the row must survive that race or the
# handed-off leg is orphaned and a fresh empty session is minted in its place. A later DELIBERATE
# close is still fenced the ordinary way: a reset-boundary row ended after this row's last activity
# blocks recovery (the NOT EXISTS below). ``pending``/``failed``/NULL handoffs are untouched, so this
# is strictly "ownership was transferred", never "a handoff was attempted".
_HANDOFF_OWNED_ROW_SQL = "(s.handoff_state = 'completed')"
_PEER_BY_KEY_SQL = f"""{_PEER_SELECT_HEAD} WHERE s.session_key = ?
AND s.source = ?
AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})
OR {_HANDOFF_OWNED_ROW_SQL})
AND NOT EXISTS (
SELECT 1 FROM sessions b
WHERE b.session_key = s.session_key
AND b.source = s.source
AND b.ended_at IS NOT NULL
AND b.end_reason IN ({_RESET_END_REASONS_SQL})
AND b.ended_at
> COALESCE(s.last_activity_at, s.started_at)
)
ORDER BY _has_messages DESC,
COALESCE(s.last_activity_at, s.started_at) DESC
LIMIT 1
"""
_PEER_BY_TUPLE_SQL = f"""{_PEER_SELECT_HEAD} WHERE s.source = ?
AND COALESCE(s.user_id, '') = COALESCE(?, '')
AND COALESCE(s.chat_id, '') = COALESCE(?, '')
AND COALESCE(s.chat_type, '') = COALESCE(?, '')
AND COALESCE(s.thread_id, '') = COALESCE(?, '')
AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?)
AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})
OR {_HANDOFF_OWNED_ROW_SQL})
AND (COALESCE(s.message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1
))
AND NOT EXISTS (
SELECT 1 FROM sessions b
WHERE b.source = s.source
AND COALESCE(b.user_id, '') = COALESCE(s.user_id, '')
AND COALESCE(b.chat_id, '') = COALESCE(s.chat_id, '')
AND COALESCE(b.chat_type, '') = COALESCE(s.chat_type, '')
AND COALESCE(b.thread_id, '') = COALESCE(s.thread_id, '')
AND (? IS NULL OR COALESCE(b.profile_name, ?) = ?)
AND b.ended_at IS NOT NULL
AND b.end_reason IN ({_RESET_END_REASONS_SQL})
AND b.ended_at
> COALESCE(s.last_activity_at, s.started_at)
)
ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC
LIMIT 1
"""
_ORPHAN_DONOR_COLUMNS = (
"d.id, d.session_key, d.chat_id, d.chat_type, d.thread_id, "
"d.user_id, d.origin_json, d.display_name, d.end_reason"
)
_ORPHANS_SQL = f"""
SELECT o.id, o.source, o.user_id, o.started_at,
o.parent_session_id, o.profile_name,
{_sql_session_last_active("o")} AS last_active,
(SELECT COUNT(*) FROM messages m
WHERE m.session_id = o.id) AS message_count
FROM sessions o
WHERE o.session_key IS NULL
AND EXISTS (SELECT 1 FROM messages m
WHERE m.session_id = o.id)
AND COALESCE(o.source, '') != 'tool'
AND {_sql_json_extract('o.model_config', '$._branched_from')} IS NULL
AND {_sql_json_extract('o.model_config', '$._delegate_from')} IS NULL
ORDER BY o.started_at ASC
"""
# Donor profile fence: the donor's effective profile (NULL reads as the store's own, the same convention
# the peer-tuple recovery predicate uses) must equal the orphan's. A sibling profile's keyed row in a
# shared legacy store would otherwise stamp its ``agent:<other>:`` routing identity onto this profile's
# orphan. Stores outside the profile tree derive no owner and keep the historical unfenced behavior.
_ORPHAN_DONOR_PROFILE_FENCE_SQL = "AND (? IS NULL OR COALESCE(d.profile_name, ?) = ?)"
_ORPHAN_LINEAGE_DONOR_SQL = f"""
SELECT {_ORPHAN_DONOR_COLUMNS}
FROM sessions d
WHERE d.id = ?
AND d.session_key IS NOT NULL
AND COALESCE(d.source, '') = COALESCE(?, '')
{_ORPHAN_DONOR_PROFILE_FENCE_SQL}
"""
_ORPHAN_CONTIGUITY_DONORS_SQL = f"""
SELECT {_ORPHAN_DONOR_COLUMNS}, {_sql_session_last_active("d")} AS last_active
FROM sessions d
WHERE d.session_key IS NOT NULL
AND d.id != ?
AND COALESCE(d.source, '') = COALESCE(?, '')
AND (COALESCE(d.user_id, '') = ''
OR COALESCE(?, '') = ''
OR d.user_id = ?)
AND {_sql_session_last_active("d")} BETWEEN ? AND ?
AND {_sql_session_last_active("d")} < ?
{_ORPHAN_DONOR_PROFILE_FENCE_SQL}
ORDER BY last_active DESC
LIMIT 2
"""
_OPTIONAL_TABLE_NAMES = (
"telegram_dm_topic_mode", "telegram_dm_topic_bindings", "delivery_obligations")
def _optional_table_columns(conn) -> dict[str, set[str]]:
"""Live column sets of the lazily-created tables that exist (``{}`` when none do).
``apply_telegram_topic_migration`` runs only on explicit ``/topic`` opt-in, so a store
can hold supported v1/v2 tables without ``profile_name`` for its whole life; likewise the
delivery ledger adds ``delivery_obligations.adapter_profile`` only when a gateway opens
it. Identity settlement must gate its column SQL on the column actually being there
rather than on table existence, and must not force those migrations (#113757).
"""
existing = {row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name IN (?, ?, ?)",
_OPTIONAL_TABLE_NAMES)}
return {
table: {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")}
for table in _OPTIONAL_TABLE_NAMES if table in existing
}
_HANDOFF_FAIL_SQL = "UPDATE sessions SET handoff_state = 'failed', handoff_error = ? WHERE "
class SessionGatewayMixin:
"""Routing index, session peers/orphans, hygiene streaks, heartbeats, handoffs."""
def _reap_inactive_orphan_desktop_holders(
self, holders: list[tuple[int, str]], *, min_age_seconds: float) -> list[int]:
"""Terminate old PPID-1 Desktop ephemeral backends with no client.
Fails closed: anything whose parent, age, argv, or network connections
cannot be proved safe remains a repair-blocking holder."""
from hermes_state import psutil
from hermes_state_dbfile import _concrete_state_db_holder_pids, _is_inactive_orphan_desktop_holder
if not sys.platform.startswith("linux") or psutil is None:
return []
try:
from hermes_cli.dashboard_procs import _is_ephemeral_port_zero_backend
except Exception:
return []
now = time.time()
candidates = []
for pid in _concrete_state_db_holder_pids(self.db_path, holders):
try:
process = psutil.Process(pid)
statuses = [conn.status for conn in process.net_connections(kind="inet")]
if not _is_inactive_orphan_desktop_holder(
ppid=process.ppid(), age_seconds=now - process.create_time(),
min_age_seconds=min_age_seconds,
ephemeral_backend=_is_ephemeral_port_zero_backend(process.cmdline()),
connection_statuses=statuses):
continue
except Exception:
continue
candidates.append(process)
signalled: list[int] = []
for process in candidates:
try:
process.terminate()
signalled.append(process.pid)
except (psutil.Error, OSError):
continue
if not signalled:
return []
try:
_gone, alive = psutil.wait_procs(candidates, timeout=1.5)
except Exception:
alive = []
for process in alive:
try:
process.kill()
except (psutil.Error, OSError):
continue
if alive:
try:
psutil.wait_procs(alive, timeout=1.5)
except Exception:
pass
return signalled
def record_gateway_session_peer(
self, session_id: str, *, source: str, user_id: str | None = None, session_key: str | None = None,
chat_id: str | None = None, chat_type: str | None = None, thread_id: str | None = None, display_name: str | None = None,
origin_json: str | None = None, include_compression_ancestors: bool = False,
transport_profile: str | None = None) -> None:
"""Persist the gateway routing peer for an existing session row. ``display_name`` / ``origin_json``:
``None`` leaves the stored value untouched (consumers read routing data from state.db, not
sessions.json). ``include_compression_ancestors`` keeps a compression lineage on one routing peer
when an explicit resume moves its tip to another lane; per-turn refreshes update only the supplied
row. Self-healing: a missing target row (deferred ``create_session`` write, or crash between routing
publication and row creation) is INSERTed with full identity rather than no-opped, so a gateway row
is never first-created by the identity-less lazy writer (``update_token_counts``) and left
unroutable forever.
See #9006.
"""
if not session_id or not session_key:
return
identity = (
session_key, source, user_id, chat_id, chat_type, thread_id, display_name, origin_json,
transport_profile)
ancestors = include_compression_ancestors
query_params = [session_id, *identity] if ancestors else [*identity, session_id]
def _do(conn):
conn.execute(
f"""{_COMPRESSION_LINEAGE_CTE if ancestors else ""}
UPDATE sessions
SET session_key = ?, source = ?, user_id = ?, chat_id = ?,
chat_type = ?, thread_id = ?,
display_name = COALESCE(?, display_name),
origin_json = COALESCE(?, origin_json),
transport_profile = COALESCE(?, transport_profile)
{"WHERE id IN (SELECT id FROM compression_lineage)" if ancestors else "WHERE id = ?"}""",
query_params,
)
if ancestors:
return
# The UPDATE silently no-ops on a missing row — insert it with full identity.
if conn.execute("SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,)).fetchone() is None:
conn.execute(
"""INSERT INTO sessions (
id, source, created_source, user_id, session_key, chat_id,
chat_type, thread_id, display_name, origin_json,
profile_name, transport_profile, started_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
session_key = COALESCE(sessions.session_key, excluded.session_key),
chat_id = COALESCE(sessions.chat_id, excluded.chat_id),
chat_type = COALESCE(sessions.chat_type, excluded.chat_type),
thread_id = COALESCE(sessions.thread_id, excluded.thread_id),
display_name = COALESCE(sessions.display_name, excluded.display_name),
origin_json = COALESCE(sessions.origin_json, excluded.origin_json),
transport_profile = COALESCE(sessions.transport_profile, excluded.transport_profile),
created_source = COALESCE(sessions.created_source, excluded.created_source)""",
# Same ownership stamp as _insert_session_row: an unowned (NULL) row
# vanishes from profile-keyed consumers.
(session_id, source, source, user_id, session_key, chat_id, chat_type, thread_id, display_name,
origin_json, self._own_profile_name(), transport_profile, time.time()),
)
self._execute_write(_do)
def save_gateway_routing_entry(self, session_key: str, entry_json: str, *, scope: str = "") -> None:
"""Upsert one gateway routing entry (session_key -> SessionEntry JSON); ``scope``
namespaces the index per sessions_dir so two stores never share routing state."""
if not session_key or not entry_json:
return
self._write_sql(
"""INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at)
SELECT ?, ?, ?, ? WHERE NOT EXISTS (
SELECT 1 FROM state_meta WHERE key='gateway.retired_session.v1.' || json_extract(?, '$.session_id'))
ON CONFLICT(scope, session_key) DO UPDATE SET
entry_json = excluded.entry_json,
updated_at = excluded.updated_at""",
(scope, session_key, entry_json, time.time(), entry_json),
)
def replace_gateway_routing_entries(self, entries: dict[str, str], *, scope: str = "") -> None:
"""Atomically replace the routing index for *scope* (keys absent from *entries*
are removed); other scopes untouched."""
now = time.time()
def _do(conn):
conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,))
if entries:
conn.executemany(
"INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) "
"SELECT ?, ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM state_meta "
"WHERE key='gateway.retired_session.v1.' || json_extract(?, '$.session_id'))",
[(scope, k, v, now, v) for k, v in entries.items() if k and v])
self._execute_write(_do)
def load_gateway_routing_entries(self, *, scope: str = "") -> dict[str, str]:
"""Load routing entries for *scope* as {session_key: entry_json}."""
rows = self._read_all("SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?", (scope,))
return {r["session_key"]: r["entry_json"] for r in rows}
def list_never_active_keyed_sessions(self, *, older_than_days: float) -> list[dict[str, Any]]:
"""Keyed, still-open rows with no evidence of a single turn (no messages, tokens, tool/API calls,
activity, or title): leaked fixtures or chats routed but never answered. Safe to drop — the gateway
mints a fresh session on the next message. Needs its own selector because ``bulk prune``/``archive``
are pinned to ``ended_at IS NOT NULL``. ``pinned``/``archived`` = explicit keep intent.
That is exactly the shape of a leaked test fixture (#82770) — and also of a chat that was routed but
never answered.
"""
if older_than_days < 0:
raise ValueError(
f"older_than_days must be >= 0, got {older_than_days!r}: a negative "
"retention builds a future cutoff that matches every never-active keyed row.")
cutoff = time.time() - (float(older_than_days) * 86400.0)
rows = self._read_all(
"""
SELECT s.id, s.session_key, s.source, s.chat_id,
s.chat_type, s.user_id, s.started_at
FROM sessions s
WHERE s.session_key IS NOT NULL
AND s.ended_at IS NULL
AND s.title IS NULL
AND s.last_activity_at IS NULL
AND COALESCE(s.message_count, 0) = 0
AND COALESCE(s.tool_call_count, 0) = 0
AND COALESCE(s.api_call_count, 0) = 0
AND COALESCE(s.input_tokens, 0) = 0
AND COALESCE(s.output_tokens, 0) = 0
AND COALESCE(s.pinned, 0) = 0
AND COALESCE(s.archived, 0) = 0
AND s.started_at IS NOT NULL
AND s.started_at < ?
AND NOT EXISTS (
SELECT 1 FROM messages m WHERE m.session_id = s.id
)
ORDER BY s.started_at
""",
(cutoff,),
)
return [dict(r) for r in rows]
def gateway_routing_entry_for_session(self, session_id: str) -> Optional[dict[str, Any]]:
"""The routing entry (any scope) whose current owner is *session_id*, or None. The id lives
only inside ``entry_json``, so matching is done in Python; an archived/rotated row has none."""
for row in self._read_all("SELECT entry_json FROM gateway_routing"):
try:
entry = json.loads(row["entry_json"] or "{}")
except (ValueError, TypeError, RecursionError): # malformed / non-text / over-nested entry
continue
if isinstance(entry, dict) and entry.get("session_id") == session_id:
return entry
return None
def _routing_entries_for_sessions(self, session_ids: set[str]) -> list[tuple[str, str, str]]:
"""``(scope, session_key, session_id)`` of ``gateway_routing`` rows pointing at any of
*session_ids*; the target id lives only inside ``entry_json``, so matching is in Python."""
found: list[tuple[str, str, str]] = []
if not session_ids:
return found
for row in self._read_all("SELECT scope, session_key, entry_json FROM gateway_routing"):
try:
entry = json.loads(row["entry_json"] or "{}")
except (ValueError, TypeError, RecursionError): # malformed / non-text / over-nested entry
continue
if isinstance(entry, dict) and entry.get("session_id") in session_ids:
found.append((row["scope"], row["session_key"], entry["session_id"]))
return found
def _delete_routing_entries_for_sessions(self, session_ids: set[str]) -> int:
"""Drop ``gateway_routing`` rows pointing at any of *session_ids*."""
doomed = [(scope, key) for scope, key, _ in self._routing_entries_for_sessions(session_ids)]
if not doomed:
return 0
self._write_sql("DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", doomed, many=True)
return len(doomed)
def prune_never_active_keyed_sessions(
self, *, older_than_days: float, sessions_dir: Optional[Path] = None,
deleted_ids: Optional[list] = None) -> tuple[int, int, int]:
"""Delete never-active keyed rows and the routing entries naming them; returns
``(sessions_deleted, routing_entries_deleted, sessions_skipped)``. Deletion goes through
:meth:`delete_sessions` (one transaction; delegate cascade, FTS, transcripts).
Rows under a live turn lease or compression lock are skipped (#123583): the
never-active predicate reads committed state, so a keyed row whose first turn lease
is already held — its messages not yet flushed — would otherwise be deleted
mid-turn. Routing entries are dropped only after the rows commit, for every
candidate except the guarded ones: rows that vanished concurrently are gone too,
and a stale entry outliving its target would have the gateway resume a
nonexistent id. The row sweep is one transaction; routing entries are dropped in a
second write after it commits. ``sessions_skipped`` counts only the guarded rows.
``deleted_ids`` (the owning gateway) receives every candidate id that was not kept."""
candidates = self.list_never_active_keyed_sessions(older_than_days=older_than_days)
# A fresh local reset tip is its conversation's current target, not a never-active chat:
# the conversation holds the earlier segments' history and the owner's queued work.
from hermes_state_local import owned_lineage_ids
with self._read_ctx() as conn:
ids = {str(row["id"]) for row in candidates if owned_lineage_ids(conn, str(row["id"])) == [str(row["id"])]}
if not ids:
return (0, 0, 0)
skipped: list[str] = []
# ``delete_sessions`` retires the deleted rows' routing entries inside its own transaction
# (``retire_routes``), so count what pointed at the candidates before it commits; the
# follow-up sweep then only catches entries a concurrent writer re-added.
routed_before = self._routing_entries_for_sessions(ids)
deleted = self.delete_sessions(
list(ids), sessions_dir=sessions_dir, exclude_active_write_guards=True, skipped_ids=skipped)
kept = set(skipped)
self._delete_routing_entries_for_sessions(ids - kept)
routing_deleted = sum(1 for _, _, sid in routed_before if sid not in kept)
if deleted_ids is not None:
deleted_ids.extend(sorted(ids - kept))
return (deleted, routing_deleted, len(skipped))
def list_gateway_sessions(
self, *, platform: Optional[str] = None, active_only: bool = True) -> list[dict[str, Any]]:
"""List gateway sessions (rows with a session_key): newest row per key, one live
mapping per routing key. ``platform`` filters on ``source``."""
# Full rows carry token/cost totals — drain queued async accounting deltas first.
self.flush_token_counts()
query = f"""
SELECT sessions.*,
COALESCE(sp.prompt, sessions.system_prompt)
AS _system_prompt_resolved,
{_sql_session_last_active("sessions")} AS last_active
FROM sessions
LEFT JOIN system_prompts sp
ON sp.hash = sessions.system_prompt_hash
WHERE session_key IS NOT NULL
AND started_at = (
SELECT MAX(s2.started_at) FROM sessions s2
WHERE s2.session_key = sessions.session_key
)
"""
params: list = [platform] if platform else []
query += (" AND LOWER(source) = LOWER(?)" if platform else "") + (
" AND ended_at IS NULL" if active_only else "") + " ORDER BY last_active DESC"
return [self._session_row_dict(r) for r in self._read_all(query, params)]
def find_latest_gateway_session_for_peer(
self, *, source: str, user_id: Optional[str] = None, session_key: Optional[str] = None,
chat_id: Optional[str] = None, chat_type: Optional[str] = None, thread_id: Optional[str] = None,
) -> Optional[dict[str, Any]]:
"""Find the latest recoverable gateway session for a routing peer. The durable ``session_key`` on the row rebuilds a missing/pruned ``sessions.json`` mapping. Rows
ended only by the old ``agent_close`` bug or a mistaken TUI ``ws_orphan_reap`` are recoverable;
explicit boundaries (/new, /resume switches, compression splits) are not. Ranked by
``last_activity_at`` (fallback ``started_at`` — alone it resurrected days-old zombies); rows with
messages win, but an empty keyed row still beats ``None`` (which mints a new id while the transcript
may live under a compression child). Reset fence: a candidate is rejected when a peer boundary row
ended *after* its last activity, or the has-messages ranking could reach behind a /new. The
exact-key fallback requires the complete peer tuple (never cross chats/threads/users) plus a profile
fence: a Telegram DM's tuple is identical for every bot, so a sibling profile's legacy row would
otherwise be adopted. Ours = profile_name is the owner or NULL; stores outside the profile tree
derive no owner and stay unfenced.
See #60609.
Reset boundaries fence recovery (#68539): an intentional boundary such as ``session_reset`` (or any
explicit non-recoverable end_reason) must block fallback to an *older* row for the same peer. Each
candidate is therefore rejected when a boundary row for the peer ended *after* the candidate's last
activity — if the conversation's most recent event is an intentional reset, recovery returns nothing
rather than reaching behind it. The boundary row carries the same profile predicate as the
candidate: a sibling profile's reset must not suppress this profile's recovery.
"""
if not session_key:
return None
with self._read_ctx() as conn:
row = conn.execute(_PEER_BY_KEY_SQL, (session_key, source)).fetchone()
if row is not None:
return self._session_row_dict(row)
if chat_id is None or chat_type is None:
return None
# Profile fence (#74285): a Telegram DM's peer tuple is identical for every bot (chat_id ==
# user_id, no thread), so a sibling profile's row written into this store before the per-profile
# partition (legacy data) would otherwise be adopted here. Every profile-tree store has one
# owner; a row is ours when its profile_name is the owner or NULL (legacy rows this store
# minted). Stores outside the tree derive no owner and keep the historical unfenced behavior.
owner = self._own_profile_name()
row = conn.execute(
_PEER_BY_TUPLE_SQL,
(source, user_id, chat_id, chat_type, thread_id, owner, owner, owner, owner, owner, owner),
).fetchone()
return self._session_row_dict(row) if row else None
def find_orphaned_gateway_sessions(self, *, max_gap_s: Optional[float] = None) -> list[dict[str, Any]]:
"""Report message-bearing rows that lost their routing identity (messages, no ``session_key``).
Adoptable only when exactly one keyed predecessor can be named: ``lineage`` (``parent_session_id``
is a keyed row of the same source; no time window) or ``contiguity`` (exactly one keyed same-source
row with compatible ``user_id`` fell quiet within *max_gap_s* of the orphan's start and is older
than its last activity). Ambiguity is reported ``adoptable=False`` with a reason, never guessed —
mis-adopting splices one person's conversation into another's chat. The donor must share the
orphan's effective profile (NULL reads as this store's own): a sibling profile's keyed row in a
shared legacy store must never stamp its routing identity here. Branch/delegate/tool rows are
excluded: unkeyed by design, not damage."""
gap = self._ORPHAN_ADOPTION_MAX_GAP_S if max_gap_s is None else float(max_gap_s)
owner = self._own_profile_name()
records: list[dict[str, Any]] = []
with self._read_ctx() as conn:
for orphan in conn.execute(_ORPHANS_SQL).fetchall():
donor = None
reason = ""
orphan_owner = orphan["profile_name"] or owner
if orphan["parent_session_id"]:
evidence = "lineage"
donor = conn.execute(
_ORPHAN_LINEAGE_DONOR_SQL,
(orphan["parent_session_id"], orphan["source"], owner, owner, orphan_owner),
).fetchone()
if donor is None:
reason = "parent session carries no gateway identity of this source"
else:
evidence = "contiguity"
started = orphan["started_at"] or 0
candidates = conn.execute(
_ORPHAN_CONTIGUITY_DONORS_SQL,
(orphan["id"], orphan["source"], orphan["user_id"], orphan["user_id"],
started - gap, started + gap, orphan["last_active"],
owner, owner, orphan_owner),
).fetchall()
if not candidates:
reason = f"no keyed predecessor fell quiet within {gap:.0f}s of this session's start"
elif len(candidates) > 1:
reason = "ambiguous: more than one keyed predecessor matches this window"
else:
donor = candidates[0]
records.append({
"orphan_id": orphan["id"], "source": orphan["source"],
"message_count": orphan["message_count"], "started_at": orphan["started_at"],
"last_active": orphan["last_active"], "donor_id": donor["id"] if donor else None,
"session_key": donor["session_key"] if donor else None,
"evidence": evidence if donor else "", "adoptable": donor is not None, "reason": reason})
# Two unkeyed successors claiming one predecessor: at most one continues that chat.
contested = {
r["donor_id"] for r in records
if r["adoptable"] and sum(1 for x in records if x["donor_id"] == r["donor_id"]) > 1
}
for record in records:
if record["donor_id"] in contested:
record["adoptable"] = False
record["reason"] = "ambiguous: more than one unkeyed session claims this predecessor"
return records
def adopt_orphaned_gateway_session(self, orphan_id: str, donor_id: str) -> bool:
"""Stamp *orphan_id* with *donor_id*'s routing identity, retire *donor_id*.
Re-verifies the pair inside the write txn so a concurrent gateway that healed
either row makes this a no-op. Non-NULL orphan columns are preserved."""
if not orphan_id or not donor_id or orphan_id == donor_id:
return False
owner = self._own_profile_name()
def _do(conn):
donor = conn.execute(
"SELECT session_key, chat_id, chat_type, thread_id, user_id, "
"origin_json, display_name, source, profile_name FROM sessions WHERE id = ?",
(donor_id,),
).fetchone()
orphan = conn.execute(
"SELECT session_key, source, profile_name FROM sessions WHERE id = ?",
(orphan_id,)).fetchone()
if (donor is None or orphan is None or not donor["session_key"] or orphan["session_key"]
or (donor["source"] or "") != (orphan["source"] or "")
or (owner is not None
and (donor["profile_name"] or owner) != (orphan["profile_name"] or owner))):
return False
# Belt-and-suspenders for gateway routing metadata (#59527): the gateway re-records the peer on
# the child after rotation (d5b4879d4), but a hard crash between child creation and that write
# leaves the child row without origin columns, so ``find_latest_gateway_session_for_peer`` can't
# recover the mapping on restart. Inherit them from the parent at creation time — but ONLY for
# compression forks (parent already ended with end_reason='compression'). Delegate/subagent
# children are spawned while the parent is still live and must NOT inherit routing keys, or peer
# recovery could repoint gateway traffic into a subagent's session.
conn.execute(
"""UPDATE sessions
SET session_key = ?,
chat_id = COALESCE(chat_id, ?),
chat_type = COALESCE(chat_type, ?),
thread_id = COALESCE(thread_id, ?),
user_id = COALESCE(user_id, ?),
origin_json = COALESCE(origin_json, ?),
display_name = COALESCE(display_name, ?),
parent_session_id = COALESCE(parent_session_id, ?)
WHERE id = ? AND session_key IS NULL""",
(donor["session_key"], donor["chat_id"], donor["chat_type"], donor["thread_id"],
donor["user_id"], donor["origin_json"], donor["display_name"], donor_id, orphan_id),
)
# Retire under a reason recovery does NOT treat as resumable — 'agent_close' /
# 'ws_orphan_reap' would keep it in the running and the orphan could lose the chat again.
conn.execute(
"UPDATE sessions SET ended_at = COALESCE(ended_at, ?), "
"end_reason = 'superseded_by_repair' WHERE id = ?",
(time.time(), donor_id))
return True
return self._execute_write(_do)
def increment_hygiene_failure_streak(self, session_key: str) -> int:
"""Atomically increment the session-hygiene failure streak for one chat."""
if not session_key:
return 1
def _do(conn):
conn.execute(
"""INSERT INTO gateway_hygiene_state (session_key, failure_streak)
VALUES (?, 1)
ON CONFLICT(session_key) DO UPDATE SET
failure_streak = gateway_hygiene_state.failure_streak + 1""",
(session_key,),
)
row = conn.execute(
"SELECT failure_streak FROM gateway_hygiene_state WHERE session_key = ?", (session_key,),
).fetchone()
return int(row[0])
return self._execute_write(_do)
def reset_hygiene_failure_streak(self, session_key: str) -> None:
"""Clear the persisted session-hygiene failure streak for one chat."""
if not session_key:
return
self._write_sql("DELETE FROM gateway_hygiene_state WHERE session_key = ?", (session_key,))
def rekey_profile_state(self, old_name: str, new_name: str) -> dict[str, int]:
"""Atomically rewrite exact profile identity in this state database."""
old, new = (old_name or "").strip(), (new_name or "").strip()
counts: dict[str, int] = {}
if not old or not new or old == new:
return counts
old_ns, new_ns = f"agent:{old}:", f"agent:{new}:"
ns_len = len(old_ns)
def _do(conn):
existing = {row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
topic_columns = _optional_table_columns(conn)
collision = conn.execute(
"SELECT old.scope, ? || substr(old.session_key, ?) "
"FROM gateway_routing AS old JOIN gateway_routing AS target "
"ON target.scope = old.scope "
"AND target.session_key = ? || substr(old.session_key, ?) "
"WHERE substr(old.session_key, 1, ?) = ? LIMIT 1",
(new_ns, ns_len + 1, new_ns, ns_len + 1, ns_len, old_ns),
).fetchone()
if collision is not None:
raise ValueError(
f"profile routing collision in scope {collision[0]!r}: {collision[1]!r}")
for table, columns in (
("telegram_dm_topic_mode", ("chat_id",)),
("telegram_dm_topic_bindings", ("chat_id", "thread_id")),
):
if "profile_name" not in topic_columns.get(table, set()):
continue
equality = " AND ".join(
f"target.{column} = old.{column}" for column in columns)
collision = conn.execute(
f"SELECT 1 FROM {table} AS old JOIN {table} AS target "
f"ON target.profile_name = ? AND {equality} "
"WHERE old.profile_name = ? LIMIT 1", (new, old)).fetchone()
if collision is not None:
raise ValueError(f"profile identity collision in {table}")
counts["sessions_profile_name"] = conn.execute(
"UPDATE sessions SET profile_name = ? WHERE profile_name = ?", (new, old)).rowcount
counts["gateway_heartbeats_profile"] = conn.execute(
"UPDATE gateway_heartbeats SET profile = ? WHERE profile = ?", (new, old)).rowcount
counts["sessions_session_key"] = conn.execute(
"UPDATE sessions SET session_key = ? || substr(session_key, ?) "
"WHERE substr(session_key, 1, ?) = ?",
(new_ns, ns_len + 1, ns_len, old_ns)).rowcount
origin_count = 0
for session_id, origin_json in conn.execute(
"SELECT id, origin_json FROM sessions WHERE origin_json IS NOT NULL").fetchall():
try:
payload = json.loads(origin_json)
except (ValueError, TypeError):
continue
if isinstance(payload, dict) and payload.get("profile") == old:
payload["profile"] = new
conn.execute("UPDATE sessions SET origin_json = ? WHERE id = ?",
(json.dumps(payload, ensure_ascii=False), session_id))
origin_count += 1
counts["sessions_origin_json"] = origin_count
if "delivery_obligations" in existing:
if "adapter_profile" in topic_columns.get("delivery_obligations", set()):
counts["delivery_obligations_adapter_profile"] = conn.execute(
"UPDATE delivery_obligations SET adapter_profile = ? WHERE adapter_profile = ?",
(new, old)).rowcount
counts["delivery_obligations_session_key"] = conn.execute(
"UPDATE delivery_obligations SET session_key = ? || substr(session_key, ?) "
"WHERE substr(session_key, 1, ?) = ?",
(new_ns, ns_len + 1, ns_len, old_ns)).rowcount
for table in ("telegram_dm_topic_mode", "telegram_dm_topic_bindings"):
if "profile_name" in topic_columns.get(table, set()):
counts[f"{table}_profile_name"] = conn.execute(
f"UPDATE {table} SET profile_name = ? WHERE profile_name = ?",
(new, old)).rowcount
if "session_key" in topic_columns.get("telegram_dm_topic_bindings", set()):
counts["telegram_dm_topic_bindings_session_key"] = conn.execute(
"UPDATE telegram_dm_topic_bindings "
"SET session_key = ? || substr(session_key, ?) "
"WHERE substr(session_key, 1, ?) = ?",
(new_ns, ns_len + 1, ns_len, old_ns)).rowcount
routing = conn.execute(
"SELECT rowid, session_key, entry_json FROM gateway_routing "
"WHERE substr(session_key, 1, ?) = ?", (ns_len, old_ns)).fetchall()
for rowid, session_key, entry_json in routing:
new_session_key = new_ns + session_key[ns_len:]
new_json = entry_json
if entry_json:
try:
payload = json.loads(entry_json)
except (ValueError, TypeError):
payload = None
if isinstance(payload, dict):
if isinstance(payload.get("session_key"), str) and payload["session_key"].startswith(old_ns):
payload["session_key"] = new_ns + payload["session_key"][ns_len:]
origin = payload.get("origin")
if isinstance(origin, dict) and origin.get("profile") == old:
origin["profile"] = new
new_json = json.dumps(payload, ensure_ascii=False)
conn.execute(
"UPDATE gateway_routing SET session_key = ?, entry_json = ? WHERE rowid = ?",
(new_session_key, new_json, rowid))
counts["gateway_routing"] = len(routing)
self._execute_write(_do)
return counts
def purge_profile_state(self, profile: str) -> dict[str, int]:
"""Delete exact profile identity from this state database (#111926, delete side).
The mirror of :meth:`rekey_profile_state`: a rename must rekey a profile's identity, a
delete must purge it. ``agent:<name>:*`` routing keys, ``gateway_heartbeats.profile``,
``delivery_obligations`` and the telegram topic tables are bookkeeping for a profile that
no longer exists — left behind, every inbound event on a chat keyed to the deleted name
enters the routing index, resolves a profile whose directory is gone, and logs
``Profile '<name>' does not exist`` on each event for the life of the store.
What each store gets, and why:
* Routing keys and heartbeat rows are hard-deleted — pure bookkeeping for a dead name.
* ``delivery_obligations`` rows are **terminalized** (``state='abandoned'``), not deleted:
a pending obligation is delivery state that should not vanish silently, and the ledger's
own retention prunes abandoned rows. Delivered history is left as it was.
* ``sessions`` rows are not deleted here: this helper settles identity, not history, and it
does not decide what a delete leaves of a profile's conversation record — ``delete_profile``
removes the profile's own home, ``state.db`` included, with the directory. Rows in a shared
store keep whatever ownership they had; re-binding or archiving them belongs to the flow
that recreates the name, not to this purge.
Idempotent.
"""
name = (profile or "").strip()
counts: dict[str, int] = {}
if not name:
return counts
ns, ns_len = f"agent:{name}:", len(f"agent:{name}:")
def _do(conn):
existing = {row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
topic_columns = _optional_table_columns(conn)
if "gateway_routing" in existing:
counts["gateway_routing"] = conn.execute(
"DELETE FROM gateway_routing WHERE substr(session_key, 1, ?) = ?",
(ns_len, ns)).rowcount
if "gateway_heartbeats" in existing:
counts["gateway_heartbeats"] = conn.execute(
"DELETE FROM gateway_heartbeats WHERE profile = ?", (name,)).rowcount
if "delivery_obligations" in existing:
# Terminalize, never hard-delete: a pending obligation is delivery state someone may
# still care about, and the ledger's own retention prunes abandoned rows. Only
# non-terminal rows are touched — delivered history is left exactly as it was.
# A ledger created before ``adapter_profile`` existed matches on namespace alone.
by_profile = ("adapter_profile = ? OR "
if "adapter_profile" in topic_columns.get("delivery_obligations", set())
else "")
params = (time.time(), name, ns_len, ns) if by_profile else (time.time(), ns_len, ns)
counts["delivery_obligations"] = conn.execute(
"UPDATE delivery_obligations SET state='abandoned', updated_at=? "
f"WHERE ({by_profile}substr(session_key, 1, ?) = ?) "
"AND state NOT IN ('delivered', 'abandoned')", params).rowcount
if "profile_name" in topic_columns.get("telegram_dm_topic_mode", set()):
counts["telegram_dm_topic_mode"] = conn.execute(
"DELETE FROM telegram_dm_topic_mode WHERE profile_name = ?", (name,)).rowcount
binding_columns = topic_columns.get("telegram_dm_topic_bindings", set())
if "session_key" in binding_columns:
# A rename rewrites a binding's session_key namespace as well as its profile_name
# (:meth:`rekey_profile_state`), so matching on one alone leaves the other behind.
# Legacy v1/v2 bindings have no profile_name but keep the ``agent:<name>:`` namespace
# in session_key, so the namespace match alone is the exact cleanup there.
by_profile = "profile_name = ? OR " if "profile_name" in binding_columns else ""
params = (name, ns_len, ns) if by_profile else (ns_len, ns)
counts["telegram_dm_topic_bindings"] = conn.execute(
"DELETE FROM telegram_dm_topic_bindings "
f"WHERE {by_profile}substr(session_key, 1, ?) = ?", params).rowcount
self._execute_write(_do)
return counts
@staticmethod
def session_gateway_runtime(session_meta: Optional[dict[str, Any]]) -> dict[str, Any]:
"""Read the persisted runtime route off a session row dict (``model_config`` as
JSON string or parsed dict). Precedence: nested ``gateway_runtime`` (gateway sync /
CLI ``/model``), then top-level ``provider``/``base_url``/``api_mode`` (TUI), with
``billing_provider`` filling a missing provider so sessions that never ran ``/model``
still restore the provider that served them. Empty dict on parse failure — resume uses ambient config."""
from hermes_state import _BARE_BILLING_PROVIDERS
raw = (session_meta or {}).get("model_config")
if isinstance(raw, str):
try:
raw = json.loads(raw)
except Exception:
raw = {}
if not isinstance(raw, dict):
raw = {}
runtime = raw.get("gateway_runtime")
# Filter None: the persist path writes or-None to trigger deletion in the top-level
# merge, but gateway_runtime is replaced whole (not deep-merged), so None survives here.
if isinstance(runtime, dict) and runtime.get("provider"):
return {k: v for k, v in runtime.items() if v is not None}
top_level = {key: raw.get(key) for key in ("provider", "base_url", "api_mode") if raw.get(key)}
# billing_provider is COALESCE-written on the first accounted API call — the only durable
# record for sessions that never ran /model. Bare buckets ("auto"/"custom") are not
# routable identities; filter them so resume falls back to the ambient default.
billing_provider = str((session_meta or {}).get("billing_provider") or "").strip()
if billing_provider and billing_provider.lower() not in _BARE_BILLING_PROVIDERS:
top_level.setdefault("provider", billing_provider)
if top_level:
return top_level
if not isinstance(runtime, dict):
return {}
return {k: v for k, v in runtime.items() if v is not None}
def register_backend_heartbeat(
self, *, backend_id: str, pid: int, started_at: float, last_heartbeat: Optional[float] = None,
profile: str = "", host: str = "") -> None:
"""Upsert this backend's liveness row. ``backend_id`` MUST be stable for the process
lifetime (e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot inherit a dead
predecessor's heartbeat; ``started_at`` is when THIS process started, so a backend
whose previous run died is not mistaken for a freshly-spawned sibling."""
if not backend_id:
return
ts = time.time() if last_heartbeat is None else float(last_heartbeat)
self._write_sql(
"INSERT INTO gateway_heartbeats (backend_id, pid, started_at, last_heartbeat, profile, host)"
" VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(backend_id) DO UPDATE SET pid = excluded.pid,"
" started_at = excluded.started_at, last_heartbeat = excluded.last_heartbeat,"
" profile = excluded.profile, host = excluded.host",
(str(backend_id), int(pid), float(started_at), ts, str(profile), str(host)))
def clear_backend_heartbeat(self, backend_id: str) -> bool:
"""Remove this backend's heartbeat row (from ``atexit``); True if removed. A crashed
backend's row is reclaimed later by ``prune_stale_heartbeats``."""
if not backend_id:
return False
return self._write_rowcount(
"DELETE FROM gateway_heartbeats WHERE backend_id = ?", (str(backend_id),)) > 0
def prune_stale_heartbeats(self, *, max_age_seconds: float) -> list[str]:
"""Drop heartbeat rows older than the staleness window; return removed backend ids.
Safe from any process — only stale rows are touched."""
if max_age_seconds <= 0:
return []
cutoff = time.time() - max_age_seconds
def _do(conn):
cur = conn.execute(
"DELETE FROM gateway_heartbeats WHERE last_heartbeat < ? RETURNING backend_id",
(cutoff,))
return [str(r[0]) for r in cur.fetchall()]
return list(self._execute_write(_do) or [])
def list_backend_heartbeats(self) -> list[dict[str, Any]]:
"""Snapshot of every backend heartbeat (diagnostics/tests); fields mirror the table."""
rows = self._read_all(
"SELECT backend_id, pid, started_at, last_heartbeat, profile, host FROM gateway_heartbeats"
" ORDER BY last_heartbeat DESC")
return [dict(r) for r in rows]
def request_handoff(self, session_id: str, platform: str) -> bool:
"""Mark a session pending handoff to *platform*; False if a handoff is already in flight."""
return self._write_rowcount(
"UPDATE sessions SET handoff_state = 'pending', handoff_platform = ?, "
" handoff_error = NULL WHERE id = ? AND (handoff_state IS NULL "
" OR handoff_state IN ('completed', 'failed'))",
(platform, session_id),
) > 0
def get_handoff_state(self, session_id: str) -> Optional[dict[str, Any]]:
"""Return ``{"state", "platform", "error"}`` or None if the session has no handoff record."""
try:
row = self._read_one(
"SELECT handoff_state, handoff_platform, handoff_error FROM sessions WHERE id = ?",
(session_id,))
if not row:
return None
return {"state": row["handoff_state"], "platform": row["handoff_platform"],
"error": row["handoff_error"]}
except Exception:
return None
def list_pending_handoffs(self) -> list[dict[str, Any]]:
"""All sessions in handoff_state='pending', oldest first (gateway handoff watcher)."""
try:
rows = self._read_all(
"SELECT s.*, COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved FROM sessions s "
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
"WHERE s.handoff_state = 'pending' "
"ORDER BY s.started_at ASC")
return [self._session_row_dict(r) for r in rows]
except Exception:
return []
def claim_handoff(self, session_id: str) -> bool:
"""Atomically transition pending → running. Returns True if claimed."""
return self._write_rowcount(
"UPDATE sessions SET handoff_state = 'running' WHERE id = ? AND handoff_state = 'pending'",
(session_id,),
) > 0
def has_pending_handoffs(self) -> bool:
"""Bounded existence probe for the handoff watcher's idle gate (the gate fails open on error)."""
return self._read_one("SELECT 1 FROM sessions WHERE handoff_state = 'pending' LIMIT 1") is not None
def complete_handoff(self, session_id: str) -> None:
"""Mark a handoff as completed."""
self._write_sql(
"UPDATE sessions SET handoff_state = 'completed', handoff_error = NULL WHERE id = ?",
(session_id,))
def fail_handoff(
self, session_id: str, error: str, *, only_states: Optional[tuple[str, ...]] = None) -> bool:
"""Mark a handoff failed and record the reason; True when a row transitioned. ``only_states`` makes
the write a compare-and-swap on ``handoff_state``. Waiters that give up (CLI 60s poll, Desktop
bounded poll) MUST pass ``only_states=("pending",)``: once the watcher has claimed the row
(``running``) it owns the terminal state, and an unconditional waiter-side fail races the dispatch —
the gateway later overwrites ``failed`` → ``completed`` after the user was told the gateway is down
(split-brain: the handoff delivered and ``switch_session`` re-pointed the session). The watcher
fails its OWN claimed row unconditionally."""
states = tuple(only_states) if only_states else ()
sql = _HANDOFF_FAIL_SQL + "id = ?" + (
f" AND handoff_state IN ({', '.join('?' for _ in states)})" if states else "")
return self._write_rowcount(sql, (error[:500], session_id, *states)) > 0
def reclaim_stale_running_handoffs(self, error: str) -> list[str]:
"""Fail every handoff stuck in ``running``; returns the ids reclaimed. Only the gateway watcher sets
``running``, for one in-process dispatch — so a ``running`` row at watcher startup belongs to a
PREVIOUS gateway that died mid-dispatch. It is poisonous: ``request_handoff`` only accepts
NULL/``completed``/``failed``, so the session could never hand off again, with no error surfaced.
Failing rather than re-queueing is deliberate: the dead gateway may already have switched the
session key and dispatched the synthetic turn, so a blind retry risks double delivery; a clean
terminal state the user can retry from is right."""
def _do(conn):
cur = conn.execute("SELECT id FROM sessions WHERE handoff_state = 'running'")
ids = [r[0] for r in cur.fetchall()]
if ids:
conn.execute(_HANDOFF_FAIL_SQL + "handoff_state = 'running'", (error[:500],))
return ids
try:
return self._execute_write(_do) or []
except Exception:
# Swallow but never silently: a persistently failing reclaim leaves poisonous
# 'running' rows in place, so the operator needs a trace.
logger.warning(
"reclaim_stale_running_handoffs failed; stranded 'running' "
"handoff rows (if any) were left in place", exc_info=True)
return []