From de99893cb41f6e5abaab9bcb117a883fc9eef9a2 Mon Sep 17 00:00:00 2001 From: kairan Date: Wed, 11 Nov 2026 17:25:33 +0800 Subject: [PATCH 1/2] fix(codex): preserve native mixed history and tool records --- .../src/backends/codex/appServer/runtime.ts | 2 +- .../session/projectCodexNativeHistoryItem.ts | 7 +-- .../codex/appServer/streamEventBridge.test.ts | 35 ++++++++++++- .../codex/appServer/streamEventBridge.ts | 22 ++++++-- .../mapCodexRolloutLineToDirectMessages.ts | 1 + .../pageCodexNativeTranscript.ts | 5 +- .../pageCodexTranscript.nativeHistory.test.ts | 50 +++++++++++++++++-- .../codex/localControl/rolloutMapper.ts | 2 +- .../rollout/projectCodexRolloutActions.ts | 2 +- .../utils/readCodexUserMessageContent.test.ts | 41 +++++++++++++++ .../utils/readCodexUserMessageContent.ts | 36 +++++++++++++ docs/codex-feature-matrix.md | 2 +- 12 files changed, 183 insertions(+), 22 deletions(-) create mode 100644 apps/cli/src/backends/codex/utils/readCodexUserMessageContent.test.ts create mode 100644 apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts diff --git a/apps/cli/src/backends/codex/appServer/runtime.ts b/apps/cli/src/backends/codex/appServer/runtime.ts index 931b09d8319..092a467d274 100644 --- a/apps/cli/src/backends/codex/appServer/runtime.ts +++ b/apps/cli/src/backends/codex/appServer/runtime.ts @@ -2327,7 +2327,7 @@ export function createCodexAppServerRuntime(params: Readonly<{ await flushItemTranscriptBoundary(null); await params.session.sendUserTextMessageCommitted(update.text, { localId: `codex-app-server-user:${context.streamScopeId}:${update.itemId}`, - meta: { importedFrom: 'codex-app-server' }, + meta: { importedFrom: 'codex-app-server', ...(update.nativeContent ? { codexNativeContent: update.nativeContent } : {}) }, }); return; } diff --git a/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts b/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts index e924671f230..ca1a477fe82 100644 --- a/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts +++ b/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts @@ -3,11 +3,6 @@ import type { CodexRolloutAction } from '../../localControl/rolloutMapper'; /** The native history API returns the same completed items as live app-server notifications. */ export function projectCodexNativeHistoryItem(item: Readonly>): CodexRolloutAction[] { - if (item.type === 'userMessage' && (!Array.isArray(item.content) || !item.content.every((part: unknown) => - part !== null && typeof part === 'object' && 'type' in part && part.type === 'text' - && 'text' in part && typeof part.text === 'string'))) { - throw new Error('Codex native history contains unsupported user content'); - } const bridge = createCodexAppServerStreamEventBridge(); const updates = bridge.onNotification({ method: item.type === 'userMessage' ? 'item/started' : 'item/completed', params: { item }, @@ -15,7 +10,7 @@ export function projectCodexNativeHistoryItem(item: Readonly { + it('retains collaboration identities that arrive only when a live tool completes', () => { + const bridge = createCodexAppServerStreamEventBridge(); + bridge.onNotification({ method: 'item/started', params: { item: { + id: 'spawn', type: 'collabAgentToolCall', tool: 'spawnAgent', receiverThreadIds: [], + } } }); + const result = bridge.onNotification({ method: 'item/completed', params: { item: { + id: 'spawn', type: 'collabAgentToolCall', tool: 'spawnAgent', receiverThreadIds: ['child'], status: 'completed', + } } }); + expect(result).toEqual([expect.objectContaining({ type: 'tool-result', output: expect.objectContaining({ receiverThreadIds: ['child'] }) })]); + }); + it.each([ + { type: 'subAgentActivity', name: 'subAgentActivity', input: { kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/child', model: 'model', reasoningEffort: 'high' }, result: { completedAtMs: 20 } }, + { type: 'collabAgentToolCall', name: 'spawnAgent', input: { tool: 'spawnAgent', senderThreadId: 'parent', receiverThreadIds: ['child-1'], prompt: 'Inspect fixture', model: 'model' }, result: { status: 'completed', agentsStates: { 'child-1': { status: 'running' } } } }, + { type: 'dynamicToolCall', name: 'lookup', input: { namespace: 'catalog', tool: 'lookup', arguments: { query: 'fixture' } }, result: { status: 'completed', success: true, contentItems: [{ type: 'inputText', text: 'result' }], durationMs: 10 } }, + ])('preserves native $type inputs and results in both history and live projection', ({ type, name, input, result }) => { + const completed = { id: 'call-1', type, ...input, ...result }; + const history = createCodexAppServerStreamEventBridge().onNotification({ method: 'item/completed', params: { item: completed } }); + expect(history).toEqual([ + expect.objectContaining({ type: 'tool-call', callId: 'call-1', name, input }), + expect.objectContaining({ type: 'tool-result', callId: 'call-1', output: result }), + ]); + const live = createCodexAppServerStreamEventBridge(); + expect(live.onNotification({ method: 'item/started', params: { item: { id: 'call-1', type, ...input } } })) + .toEqual([expect.objectContaining({ type: 'tool-call', name, input })]); + const settled = live.onNotification({ method: 'item/completed', params: { item: completed } }); + expect(settled).toHaveLength(1); + expect(settled[0]).toMatchObject({ type: 'tool-result', callId: 'call-1', output: result }); + }); it('keeps native completed command output in the tool result when no started event was observed', () => { const updates = createCodexAppServerStreamEventBridge().onNotification({ method: 'item/completed', params: { item: { id: 'native-command', type: 'commandExecution', command: 'pwd', cwd: '/project', @@ -39,7 +67,12 @@ describe('createCodexAppServerStreamEventBridge', () => { type: 'provider-user-text', itemId: 'native_user_1', clientId: null, - text: 'hello from Codex TUI\nsecond paragraph', + text: 'hello from Codex TUI\n[Codex mention: README.md (/repo/README.md)]\nsecond paragraph', + nativeContent: [ + { type: 'text', text: 'hello from Codex TUI' }, + { type: 'mention', name: 'README.md', path: '/repo/README.md' }, + { type: 'text', text: 'second paragraph' }, + ], }]); expect( diff --git a/apps/cli/src/backends/codex/appServer/streamEventBridge.ts b/apps/cli/src/backends/codex/appServer/streamEventBridge.ts index acd6ecf1f48..b718628525f 100644 --- a/apps/cli/src/backends/codex/appServer/streamEventBridge.ts +++ b/apps/cli/src/backends/codex/appServer/streamEventBridge.ts @@ -1,13 +1,14 @@ import { readCodexReasoningText } from '../utils/readCodexReasoningText'; import { extractMcpToolCallResultOutput } from '../runtime/sessionTurnLifecycle'; import { readCodexMessageContentText } from '../utils/readCodexMessageContentText'; +import { readCodexUserMessageContent } from '../utils/readCodexUserMessageContent'; import { canonicalizeCodexMcpToolName } from '../utils/canonicalizeCodexMcpToolName'; import { extractCodexGeneratedMedia } from '../media/extractCodexGeneratedMedia'; import type { SessionMediaSource } from '@/agent/core/AgentMessage'; type RecordLike = Record; -type ToolKind = 'command' | 'mcp' | 'file-change' | 'web-search' | 'image-view' | 'sleep'; +type ToolKind = 'command' | 'mcp' | 'file-change' | 'web-search' | 'image-view' | 'sleep' | 'native-agent' | 'dynamic'; type PermissionRequestToolName = 'request_permissions'; type NotificationEnvelope = Readonly<{ @@ -27,7 +28,7 @@ type ToolContext = Readonly<{ }>; export type CodexAppServerStreamUpdate = - | Readonly<{ type: 'provider-user-text'; itemId: string; clientId: string | null; text: string }> + | Readonly<{ type: 'provider-user-text'; itemId: string; clientId: string | null; text: string; nativeContent?: readonly Record[] }> | Readonly<{ type: 'assistant-text-delta'; itemId: string; text: string }> | Readonly<{ type: 'assistant-text-final'; itemId: string; text: string }> | Readonly<{ type: 'async-user-input-request'; itemId: string; text: string; questions: unknown[] }> @@ -140,6 +141,17 @@ function omitKeys(value: RecordLike, keys: readonly string[]): RecordLike { function readToolContextFromItem(item: RecordLike): ToolContext | null { const itemType = readItemType(item); + // These native records describe an activity/tool occurrence, not completion + // of the child agent itself. Keep its identity and reported state intact. + if (itemType === 'subagentactivity' || itemType === 'collabagenttoolcall' || itemType === 'dynamictoolcall') { + const name = itemType === 'subagentactivity' ? 'subAgentActivity' : readString(item.tool); + if (!name) return null; + return { + toolKind: itemType === 'dynamictoolcall' ? 'dynamic' : 'native-agent', + name, + input: omitKeys(item, ['id', 'itemId', 'type', 'itemType', 'status', 'agentsStates', 'contentItems', 'success', 'durationMs', 'completedAtMs', 'error']), + }; + } if (itemType === 'commandexecution') { const command = readString(item.command); const cwd = readString(item.cwd); @@ -269,13 +281,13 @@ export function createCodexAppServerStreamEventBridge(): Readonly<{ if (item) { if (readItemType(item) === 'usermessage') { const itemId = readItemId(item); - const text = readCodexMessageContentText(item.content); - if (itemId && text) { + const content = readCodexUserMessageContent(item.content); + if (itemId && content) { return [{ type: 'provider-user-text', itemId, clientId: readString(item.clientId) ?? readString(item.client_id), - text, + ...content, }]; } } diff --git a/apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts b/apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts index e9f464fadcf..1e247cb3c1d 100644 --- a/apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts +++ b/apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts @@ -72,6 +72,7 @@ export function mapCodexProjectedEventsToDirectMessages(params: Readonly<{ raw: { role: 'user', content: { type: 'text', text: action.text }, + ...(action.nativeContent ? { meta: { codexNativeContent: action.nativeContent } } : {}), }, }); continue; diff --git a/apps/cli/src/backends/codex/directSessions/pageCodexNativeTranscript.ts b/apps/cli/src/backends/codex/directSessions/pageCodexNativeTranscript.ts index 1e321948925..dc103f3adad 100644 --- a/apps/cli/src/backends/codex/directSessions/pageCodexNativeTranscript.ts +++ b/apps/cli/src/backends/codex/directSessions/pageCodexNativeTranscript.ts @@ -108,8 +108,9 @@ export async function pageCodexNativeTranscript(params: Readonly<{ for (; offset < messages.length; offset += 1) { const item = messages[offset]!; const size = measureDirectTranscriptItemBytes(item); - if (items.length >= params.maxItems || bytes + size > params.maxBytes) { - if (items.length === 0) throw new Error('Codex native transcript item exceeds the requested byte budget'); + // Match rollout stream paging: the byte budget bounds aggregation, + // not a valid indivisible record. Return a larger record on its own. + if (items.length > 0 && (items.length >= params.maxItems || bytes + size > params.maxBytes)) { const nextCursor = encode({ ...state, skip: index, offset, pendingId: key, anchor: index ? identity(page.data[index - 1]!) : null, fingerprint: index ? fingerprint(page.data[index - 1]!) : null }); diff --git a/apps/cli/src/backends/codex/directSessions/pageCodexTranscript.nativeHistory.test.ts b/apps/cli/src/backends/codex/directSessions/pageCodexTranscript.nativeHistory.test.ts index 6f22e0b9b99..3abf93ed5f5 100644 --- a/apps/cli/src/backends/codex/directSessions/pageCodexTranscript.nativeHistory.test.ts +++ b/apps/cli/src/backends/codex/directSessions/pageCodexTranscript.nativeHistory.test.ts @@ -115,6 +115,21 @@ describe('Codex native paginated direct history (0.162.1 contract)', () => { expect(items[1]?.raw).toMatchObject({ content: { data: { output: { stdout: '/project', exitCode: 0 } } } }); }); + it('returns an oversized native record alone and continues without truncating its contents', async () => { + const text = 'large-result'.repeat(1000); + const f = await fixture([message('before', 'before', 'user'), message('large', text, 'agent'), message('after', 'after', 'agent')]); + const pages: number[] = []; + const items = await loadDirectSessionTranscriptItems({ readPage: async cursor => { + const page = await pageCodexTranscript({ ...f.params, maxBytes: 1024, direction: 'older', cursor }); + pages.push(page.items.length); + return page; + } }); + expect(items).toHaveLength(3); + expect(new Set(items.map(item => item.id)).size).toBe(3); + expect(items[1]?.raw.content).toMatchObject({ data: { message: text } }); + expect(pages).toEqual([1, 1, 1]); + }); + it('imports native web searches, image views, and sleeps as tool rows', async () => { const results = [{ type: 'text_result', ref_id: 'turn0search0', title: 'Fusion benchmark', url: 'https://example.invalid/bench', domain: 'example.invalid', snippet: 'Fused kernels compared.' }]; @@ -139,12 +154,39 @@ describe('Codex native paginated direct history (0.162.1 contract)', () => { ]); }); - it('rejects native mixed media instead of importing only its text as complete history', async () => { + it('preserves native mixed user content instead of dropping image and skill parts', async () => { + const content = [{ type: 'text', text: 'question with image' }, + { type: 'image', url: 'data:image/png;base64,aGVsbG8=', detail: 'original' }, + { type: 'skill', name: 'review', path: '/skills/review/SKILL.md' }]; const f = await fixture([{ ...message('mixed', 'question with image', 'user'), item: { id: 'mixed', type: 'userMessage', - content: [{ type: 'text', text: 'question with image' }, { type: 'image', url: 'https://example.invalid/image.png' }] } }]); - await expect(loadDirectSessionTranscriptItems({ readPage: cursor => pageCodexTranscript({ + content } }]); + const items = await loadDirectSessionTranscriptItems({ readPage: cursor => pageCodexTranscript({ ...f.params, direction: 'older', cursor, - }) })).rejects.toBeInstanceOf(DirectSessionsProviderUnavailableError); + }) }); + expect(items).toHaveLength(1); + expect(items[0]?.raw).toMatchObject({ role: 'user', meta: { codexNativeContent: content } }); + expect(items[0]?.raw.content).toMatchObject({ text: expect.stringContaining('question with image') }); + expect(JSON.stringify(items[0]?.raw.content)).not.toContain('aGVsbG8='); + }); + + it('imports collaboration and dynamic tool records across partial pages without losing results', async () => { + const native = [ + { id: 'activity', type: 'subAgentActivity', kind: 'interacted', agentThreadId: 'child', agentPath: '/root/child', completedAtMs: 20 }, + { id: 'collab', type: 'collabAgentToolCall', tool: 'spawnAgent', senderThreadId: 'parent', receiverThreadIds: ['child'], prompt: 'inspect', status: 'completed', agentsStates: { child: { status: 'running' } } }, + { id: 'dynamic', type: 'dynamicToolCall', namespace: 'catalog', tool: 'lookup', arguments: { query: 'fixture' }, success: true, contentItems: [{ type: 'inputText', text: 'result' }] }, + ]; + const f = await fixture(native.map(item => ({ turnId: 'turn', startedAtMs: 10, completedAtMs: 20, item }))); + const items = await loadDirectSessionTranscriptItems({ readPage: cursor => pageCodexTranscript({ ...f.params, maxItems: 1, direction: 'older', cursor }) }); + expect(items).toHaveLength(6); + expect(new Set(items.map(item => item.id)).size).toBe(6); + expect(items.map(item => item.raw.content)).toMatchObject([ + { data: { type: 'tool-call', name: 'subAgentActivity', input: { agentThreadId: 'child', kind: 'interacted' } } }, + { data: { type: 'tool-call-result', output: { completedAtMs: 20 } } }, + { data: { type: 'tool-call', name: 'spawnAgent', input: { prompt: 'inspect' } } }, + { data: { type: 'tool-call-result', output: { agentsStates: { child: { status: 'running' } } } } }, + { data: { type: 'tool-call', name: 'lookup', input: { namespace: 'catalog', arguments: { query: 'fixture' } } } }, + { data: { type: 'tool-call-result', output: { success: true, contentItems: [{ type: 'inputText', text: 'result' }] } } }, + ]); }); it('distinguishes confirmed empty native history from unavailable history', async () => { diff --git a/apps/cli/src/backends/codex/localControl/rolloutMapper.ts b/apps/cli/src/backends/codex/localControl/rolloutMapper.ts index f810e0a17af..5ab7d6ee1ef 100644 --- a/apps/cli/src/backends/codex/localControl/rolloutMapper.ts +++ b/apps/cli/src/backends/codex/localControl/rolloutMapper.ts @@ -10,7 +10,7 @@ export type CodexRolloutAction = | { type: 'codex-session-id'; id: string } | { type: 'turn-lifecycle'; event: LocalTurnLifecycleEvent } | { type: 'context-compaction'; phase: 'completed'; lifecycleId: string; source: 'provider-event' | 'transcript-inference'; providerEventId?: string } - | { type: 'user-text'; text: string } + | { type: 'user-text'; text: string; nativeContent?: readonly Record[] } | { type: 'assistant-text'; text: string } | { type: 'thinking'; text: string } | { type: 'tool-call'; callId: string; name: string; input: unknown } diff --git a/apps/cli/src/backends/codex/rollout/projectCodexRolloutActions.ts b/apps/cli/src/backends/codex/rollout/projectCodexRolloutActions.ts index 20043fe2507..3ef047297b9 100644 --- a/apps/cli/src/backends/codex/rollout/projectCodexRolloutActions.ts +++ b/apps/cli/src/backends/codex/rollout/projectCodexRolloutActions.ts @@ -7,7 +7,7 @@ import type { CodexRolloutAction } from '../localControl/rolloutMapper'; export type CodexProjectedRolloutEvent = | { type: 'codex-session-id'; id: string } | { type: 'context-compaction'; phase: 'completed'; lifecycleId: string; source: 'provider-event' | 'transcript-inference'; providerEventId?: string; sidechainId: string | null } - | { type: 'user-text'; text: string } + | { type: 'user-text'; text: string; nativeContent?: readonly Record[] } | { type: 'assistant-text'; text: string; sidechainId: string | null } | { type: 'thinking'; text: string; sidechainId: string | null } | { type: 'tool-call'; callId: string; name: string; input: unknown; sidechainId: string | null } diff --git a/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.test.ts b/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.test.ts new file mode 100644 index 00000000000..1906abd6073 --- /dev/null +++ b/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { projectCodexNativeHistoryItem } from '../appServer/session/projectCodexNativeHistoryItem'; +import { projectCodexRolloutActions } from '../rollout/projectCodexRolloutActions'; +import { mapCodexProjectedEventsToDirectMessages } from '../directSessions/mapCodexRolloutLineToDirectMessages'; +import { readCodexUserMessageContent } from './readCodexUserMessageContent'; + +describe('native Codex user content preservation', () => { + it('preserves native mentions, local image references and text annotations', () => { + const content = [ + { type: 'text', text: 'See README', text_elements: [{ byte_range: { start: 4, end: 10 }, placeholder: 'README' }] }, + { type: 'mention', name: 'README.md', path: '/repo/README.md' }, + { type: 'localImage', path: '/repo/image.png' }, + ]; + const result = readCodexUserMessageContent(content); + expect(result?.nativeContent).toEqual(content); + expect(result?.text).toContain('/repo/README.md'); + expect(result?.text).toContain('/repo/image.png'); + expect(readCodexUserMessageContent([content[0]])?.nativeContent).toEqual([content[0]]); + }); + it('retains skill references and image bytes through the direct transcript projection', () => { + const content = [ + { type: 'text', text: 'Inspect this', text_elements: [] }, + { type: 'skill', name: 'review', path: '/skills/review/SKILL.md' }, + { type: 'image', url: 'data:image/png;base64,cHJpdmF0ZQ==', detail: 'original' }, + ]; + const actions = projectCodexNativeHistoryItem({ type: 'userMessage', id: 'user-1', content }); + const messages = mapCodexProjectedEventsToDirectMessages({ + projected: projectCodexRolloutActions(actions, { sidechainId: null }), + createdAtMs: 1, idForIndex: index => String(index), + }); + expect(messages).toHaveLength(1); + expect(messages[0]?.raw).toMatchObject({ role: 'user', meta: { codexNativeContent: content } }); + expect(messages[0]?.raw.content).toMatchObject({ text: expect.stringContaining('/skills/review/SKILL.md') }); + expect(JSON.stringify(messages[0]?.raw.content)).not.toContain('cHJpdmF0ZQ=='); + expect(messages[0]?.raw.content).toMatchObject({ text: expect.stringContaining('preview unavailable') }); + }); + + it('does not silently accept unsupported future content', () => { + expect(() => projectCodexNativeHistoryItem({ type: 'userMessage', id: 'user-2', content: [{ type: 'future' }] })).toThrow(); + }); +}); diff --git a/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts b/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts new file mode 100644 index 00000000000..49eea10eb38 --- /dev/null +++ b/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts @@ -0,0 +1,36 @@ +/** Preserve native content alongside the text projection used by existing clients. */ +export function readCodexUserMessageContent(content: unknown): { + text: string; + nativeContent?: readonly Record[]; +} | null { + if (typeof content === 'string') return content ? { text: content } : null; + if (!Array.isArray(content)) throw new Error('Codex native history contains unsupported user content'); + const parts: Record[] = []; + const text: string[] = []; + let preserveContent = false; + for (const value of content) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Codex native history contains unsupported user content'); + } + const part: Record = value; + if (part.type === 'text' && typeof part.text === 'string') { + text.push(part.text); + if (Array.isArray(part.text_elements) && part.text_elements.length > 0) preserveContent = true; + } else if ((part.type === 'skill' || part.type === 'mention') && typeof part.name === 'string' && typeof part.path === 'string') { + text.push(`[Codex ${part.type}: ${part.name} (${part.path})]`); + preserveContent = true; + } else if (part.type === 'image' && typeof part.url === 'string') { + // Native data URLs can contain large private image bodies. Keep them out of display text. + text.push('[Codex image: original content retained; preview unavailable]'); + preserveContent = true; + } else if (part.type === 'localImage' && typeof part.path === 'string') { + text.push(`[Codex local image: ${part.path}; preview unavailable]`); + preserveContent = true; + } else { + throw new Error('Codex native history contains unsupported user content'); + } + parts.push(part); + } + if (text.length === 0) return null; + return { text: text.join('\n'), ...(preserveContent ? { nativeContent: parts } : {}) }; +} diff --git a/docs/codex-feature-matrix.md b/docs/codex-feature-matrix.md index 352e81173d6..16c3a2179a9 100644 --- a/docs/codex-feature-matrix.md +++ b/docs/codex-feature-matrix.md @@ -46,7 +46,7 @@ Connected-service continuation recognises both `.jsonl` and `.jsonl.zst` rollout | --- | --- | --- | --- | --- | | Direct session browse/list | `supported` | `apps/cli/src/backends/codex/directSessions/listCodexSessionCandidates.ts`, `apps/cli/src/backends/codex/appServer/session/listCodexDirectSessionCandidatesViaAppServer.ts`, `apps/ui/sources/agents/providers/codex/directSessions/resolveCodexBrowseSourceOptions.ts`, `apps/ui/sources/agents/providers/codex/uiBehavior.ts` | UI exposes user-home and `openai-codex` connected-service homes; listing prefers app-server `thread/list` and falls back to rollout file discovery; candidates are branded `codexBackendMode: 'appServer'`. | Promote direct-session source and transport affinity into one provider session source record shared by browse, wake, takeover, and handoff. | | Transcript source: persisted Happier transcript | `supported` | `apps/cli/src/backends/codex/appServer/runtime.ts`, `apps/cli/src/backends/codex/acp/runtime.ts`, `apps/cli/src/backends/codex/utils/codexSessionIdMetadata.ts`, `apps/cli/src/backends/codex/runCodex.ts` | App-server streaming writes assistant/thinking deltas into Happier; ACP also publishes vendor session identity into Happy metadata; persisted transcript is the default unless `HAPPIER_TRANSCRIPT_STORAGE=direct`. | Keep one transcript-ingestion pipeline and avoid transport-specific metadata branching for persisted sessions. | -| Transcript source: linked direct transcript | `supported` | `apps/cli/src/backends/codex/utils/codexSessionIdMetadata.ts`, `apps/cli/src/backends/codex/directSessions/pageCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/readAfterCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts`, `apps/cli/src/backends/codex/directSessions/codexDirectTranscriptSource.ts` | `directSessionV1` is only published when transcript storage is `direct`; direct paging is backward-only and tailing uses a provider-specific forward cursor. In development source, paging, follow and import share rollout-first selection; missing or paginated rollout storage uses read-only native `thread/items/list` pages. An empty native head alongside paginated rollout storage is treated as unavailable because Codex can recreate an empty SQL projection. Unavailable or unrepresentable history fails explicitly instead of treating a listing preview as a complete transcript. Native `webSearch`, `imageView`, and `sleep` items become `WebSearch`, `view_image`, and `sleep` tool rows through the app-server stream bridge that live sessions share; rollout projection still omits `web_search_call` items. | Replace `directSessionV1` and Codex-only cursor shapes with a single transcript-source envelope and generic direct-history cursor contract. | +| Transcript source: linked direct transcript | `supported` | `apps/cli/src/backends/codex/utils/codexSessionIdMetadata.ts`, `apps/cli/src/backends/codex/directSessions/pageCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/readAfterCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts`, `apps/cli/src/backends/codex/directSessions/codexDirectTranscriptSource.ts` | `directSessionV1` is only published when transcript storage is `direct`; direct paging is backward-only and tailing uses a provider-specific forward cursor. In development source, paging, follow and import share rollout-first selection; missing or paginated rollout storage uses read-only native `thread/items/list` pages. An empty native head alongside paginated rollout storage is treated as unavailable because Codex can recreate an empty SQL projection. Unavailable or unrepresentable history fails explicitly instead of treating a listing preview as a complete transcript. Native `webSearch`, `imageView`, and `sleep` items become `WebSearch`, `view_image`, and `sleep` tool rows through the app-server stream bridge that live sessions share; rollout projection still omits `web_search_call` items. The local native-history compatibility patch also projects subagent activity, collaboration tools, and dynamic tools without inventing child completion. Mixed user content retains its original parts in `meta.codexNativeContent`; skill/file references are visible, while images explicitly report unavailable previews and retain their original URL/data or local path (local files are not copied). An indivisible oversized record is returned alone, matching rollout paging. | Replace `directSessionV1` and Codex-only cursor shapes with a single transcript-source envelope and generic direct-history cursor contract. | | Direct import / takeover into a Happy session | `supported` | `apps/cli/src/api/directSessions/takeover/resolveDirectTakeoverSpawnOptions.ts`, `apps/cli/src/api/machine/rpcHandlers.directSessions.ts`, `apps/cli/src/backends/codex/directSessions/getCodexDirectSessionWorkingDirectory.ts` | Takeover re-spawns with `resume`, `transcriptStorage: 'direct'`, `CODEX_HOME`, and any linked `codexBackendMode`; current direct-session candidates effectively route this through app-server semantics. | Reframe takeover as a provider-session source transition instead of a direct-session special case layered on top of spawn options. | | Local/remote switching | `supported` | `apps/cli/src/agent/localControl/createSharedProviderLocalControl.ts`, `apps/cli/src/backends/codex/runCodex.ts`, `apps/cli/src/backends/codex/appServer/createCodexSharedAppServer.ts`, `apps/cli/src/backends/codex/localControl/**` | App Server with Codex CLI 0.131.0+ on macOS/Linux uses one shared socket server for Happier and the native TUI; remote writes and approvals remain available while attached. Windows, ACP, and older App Server versions retain the exclusive compatibility path; plain MCP has no local control. | Shared provider state and terminal supervision are generic owners reused by OpenCode; Codex owns only version gating, socket-server lifecycle, and native attach arguments. | | Dynamic controls: collaboration modes / models / config options | `supported` | `apps/cli/src/backends/codex/appServer/sessionControlsMetadata.ts`, `apps/cli/src/backends/codex/appServer/runtime.ts`, `apps/ui/sources/agents/providers/codex/core.ts`, `apps/ui/sources/agents/providers/codex/uiBehavior.ts` | App-server reads `collaborationMode/list` and `model/list`, publishes generic session-control metadata, and currently supports one config option (`speed`) gated to `gpt-5.4` plus compatible auth methods. Initial catalog and native-goal projection is best-effort after the core thread attaches, so unavailable discovery endpoints do not keep session open pending. | Keep the generic state keys and extend the same controls contract to ACP/MCP if they ever gain parity; avoid reintroducing ACP-branded metadata for Codex. | From 9308855b1cf55c7ca901fd6d7c1aa56d3ff38fab Mon Sep 17 00:00:00 2001 From: Leeroy Brun Date: Wed, 11 Nov 2026 18:27:09 +0100 Subject: [PATCH 2/2] fix(codex): preserve current native media and identify projection failures Accept Codex 0.167 audio and file-id media inputs at the shared native user content boundary. Preserve text annotations using either supported spelling. Report the projection phase and sanitized unsupported type without including native content bodies or paths in errors. --- .../session/projectCodexNativeHistoryItem.ts | 5 +- .../codex/utils/codexNativeProjectionError.ts | 5 ++ .../utils/readCodexUserMessageContent.test.ts | 49 ++++++++++++++++++- .../utils/readCodexUserMessageContent.ts | 22 +++++---- docs/codex-feature-matrix.md | 2 +- 5 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 apps/cli/src/backends/codex/utils/codexNativeProjectionError.ts diff --git a/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts b/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts index ca1a477fe82..7693c5b9caf 100644 --- a/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts +++ b/apps/cli/src/backends/codex/appServer/session/projectCodexNativeHistoryItem.ts @@ -1,5 +1,6 @@ import { createCodexAppServerStreamEventBridge } from '../streamEventBridge'; import type { CodexRolloutAction } from '../../localControl/rolloutMapper'; +import { codexNativeProjectionError } from '../../utils/codexNativeProjectionError'; /** The native history API returns the same completed items as live app-server notifications. */ export function projectCodexNativeHistoryItem(item: Readonly>): CodexRolloutAction[] { @@ -17,11 +18,11 @@ export function projectCodexNativeHistoryItem(item: Readonly { expect(messages[0]?.raw.content).toMatchObject({ text: expect.stringContaining('preview unavailable') }); }); + it('accepts the camelCase text annotation field emitted by the app-server protocol', () => { + const content = [{ + type: 'text', + text: 'See README', + textElements: [{ byteRange: { start: 4, end: 10 }, placeholder: 'README' }], + }]; + expect(readCodexUserMessageContent(content)?.nativeContent).toEqual(content); + }); + it('does not silently accept unsupported future content', () => { - expect(() => projectCodexNativeHistoryItem({ type: 'userMessage', id: 'user-2', content: [{ type: 'future' }] })).toThrow(); + expect(() => projectCodexNativeHistoryItem({ type: 'userMessage', id: 'user-2', content: [{ type: 'future' }] })) + .toThrow(/projection.*user content.*future/); + expect(() => projectCodexNativeHistoryItem({ type: 'future', id: 'item-1' })) + .toThrow(/projection.*transcript item.*future/); + }); + + it.each([ + { type: 'image', fileId: 'file-image' }, + { type: 'audio', url: 'data:audio/wav;base64,cHJpdmF0ZQ==' }, + { type: 'audio', fileId: 'file-audio' }, + { type: 'localAudio', path: '/repo/recording.wav' }, + ])('retains Codex 0.167 $type media without putting inline bytes in display text', part => { + const content = [{ type: 'text', text: 'Inspect this' }, part]; + const actions = projectCodexNativeHistoryItem({ type: 'userMessage', id: 'media', content }); + const messages = mapCodexProjectedEventsToDirectMessages({ + projected: projectCodexRolloutActions(actions, { sidechainId: null }), + createdAtMs: 1, idForIndex: index => String(index), + }); + expect(messages[0]?.raw).toMatchObject({ meta: { codexNativeContent: content } }); + expect(messages[0]?.raw.content).toMatchObject({ text: expect.stringContaining('unavailable') }); + expect(JSON.stringify(messages[0]?.raw.content)).not.toContain('cHJpdmF0ZQ=='); + }); + + it('does not expose content or malformed type strings in projection diagnostics', () => { + for (const type of ['private\ncontent', 'file:///private/history']) { + for (const item of [ + { type, id: 'bad', text: 'private payload' }, + { type: 'userMessage', id: 'bad', content: [{ type, text: 'private payload' }] }, + ]) { + try { + projectCodexNativeHistoryItem(item); + expect.fail('unsupported content must fail'); + } catch (error) { + expect(String(error)).toContain('unknown'); + expect(String(error)).not.toContain(type); + expect(String(error)).not.toContain('private payload'); + } + } + } }); }); diff --git a/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts b/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts index 49eea10eb38..0f3dd6f67bb 100644 --- a/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts +++ b/apps/cli/src/backends/codex/utils/readCodexUserMessageContent.ts @@ -1,33 +1,37 @@ +import { codexNativeProjectionError } from './codexNativeProjectionError'; + /** Preserve native content alongside the text projection used by existing clients. */ export function readCodexUserMessageContent(content: unknown): { text: string; nativeContent?: readonly Record[]; } | null { if (typeof content === 'string') return content ? { text: content } : null; - if (!Array.isArray(content)) throw new Error('Codex native history contains unsupported user content'); + if (!Array.isArray(content)) throw codexNativeProjectionError('user content', null); const parts: Record[] = []; const text: string[] = []; let preserveContent = false; for (const value of content) { if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error('Codex native history contains unsupported user content'); + throw codexNativeProjectionError('user content', null); } const part: Record = value; if (part.type === 'text' && typeof part.text === 'string') { text.push(part.text); - if (Array.isArray(part.text_elements) && part.text_elements.length > 0) preserveContent = true; + if ((Array.isArray(part.text_elements) && part.text_elements.length > 0) + || (Array.isArray(part.textElements) && part.textElements.length > 0)) preserveContent = true; } else if ((part.type === 'skill' || part.type === 'mention') && typeof part.name === 'string' && typeof part.path === 'string') { text.push(`[Codex ${part.type}: ${part.name} (${part.path})]`); preserveContent = true; - } else if (part.type === 'image' && typeof part.url === 'string') { - // Native data URLs can contain large private image bodies. Keep them out of display text. - text.push('[Codex image: original content retained; preview unavailable]'); + } else if ((part.type === 'image' || part.type === 'audio') + && (typeof part.url === 'string' || typeof part.fileId === 'string')) { + // Codex 0.167 UserInput supports URLs or file ids; inline bodies stay out of display text. + text.push(`[Codex ${part.type}: original content retained; preview unavailable]`); preserveContent = true; - } else if (part.type === 'localImage' && typeof part.path === 'string') { - text.push(`[Codex local image: ${part.path}; preview unavailable]`); + } else if ((part.type === 'localImage' || part.type === 'localAudio') && typeof part.path === 'string') { + text.push(`[Codex local ${part.type === 'localImage' ? 'image' : 'audio'}: ${part.path}; preview unavailable]`); preserveContent = true; } else { - throw new Error('Codex native history contains unsupported user content'); + throw codexNativeProjectionError('user content', part.type); } parts.push(part); } diff --git a/docs/codex-feature-matrix.md b/docs/codex-feature-matrix.md index 16c3a2179a9..1ae1179fed3 100644 --- a/docs/codex-feature-matrix.md +++ b/docs/codex-feature-matrix.md @@ -46,7 +46,7 @@ Connected-service continuation recognises both `.jsonl` and `.jsonl.zst` rollout | --- | --- | --- | --- | --- | | Direct session browse/list | `supported` | `apps/cli/src/backends/codex/directSessions/listCodexSessionCandidates.ts`, `apps/cli/src/backends/codex/appServer/session/listCodexDirectSessionCandidatesViaAppServer.ts`, `apps/ui/sources/agents/providers/codex/directSessions/resolveCodexBrowseSourceOptions.ts`, `apps/ui/sources/agents/providers/codex/uiBehavior.ts` | UI exposes user-home and `openai-codex` connected-service homes; listing prefers app-server `thread/list` and falls back to rollout file discovery; candidates are branded `codexBackendMode: 'appServer'`. | Promote direct-session source and transport affinity into one provider session source record shared by browse, wake, takeover, and handoff. | | Transcript source: persisted Happier transcript | `supported` | `apps/cli/src/backends/codex/appServer/runtime.ts`, `apps/cli/src/backends/codex/acp/runtime.ts`, `apps/cli/src/backends/codex/utils/codexSessionIdMetadata.ts`, `apps/cli/src/backends/codex/runCodex.ts` | App-server streaming writes assistant/thinking deltas into Happier; ACP also publishes vendor session identity into Happy metadata; persisted transcript is the default unless `HAPPIER_TRANSCRIPT_STORAGE=direct`. | Keep one transcript-ingestion pipeline and avoid transport-specific metadata branching for persisted sessions. | -| Transcript source: linked direct transcript | `supported` | `apps/cli/src/backends/codex/utils/codexSessionIdMetadata.ts`, `apps/cli/src/backends/codex/directSessions/pageCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/readAfterCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts`, `apps/cli/src/backends/codex/directSessions/codexDirectTranscriptSource.ts` | `directSessionV1` is only published when transcript storage is `direct`; direct paging is backward-only and tailing uses a provider-specific forward cursor. In development source, paging, follow and import share rollout-first selection; missing or paginated rollout storage uses read-only native `thread/items/list` pages. An empty native head alongside paginated rollout storage is treated as unavailable because Codex can recreate an empty SQL projection. Unavailable or unrepresentable history fails explicitly instead of treating a listing preview as a complete transcript. Native `webSearch`, `imageView`, and `sleep` items become `WebSearch`, `view_image`, and `sleep` tool rows through the app-server stream bridge that live sessions share; rollout projection still omits `web_search_call` items. The local native-history compatibility patch also projects subagent activity, collaboration tools, and dynamic tools without inventing child completion. Mixed user content retains its original parts in `meta.codexNativeContent`; skill/file references are visible, while images explicitly report unavailable previews and retain their original URL/data or local path (local files are not copied). An indivisible oversized record is returned alone, matching rollout paging. | Replace `directSessionV1` and Codex-only cursor shapes with a single transcript-source envelope and generic direct-history cursor contract. | +| Transcript source: linked direct transcript | `supported` | `apps/cli/src/backends/codex/utils/codexSessionIdMetadata.ts`, `apps/cli/src/backends/codex/directSessions/pageCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/readAfterCodexTranscript.ts`, `apps/cli/src/backends/codex/directSessions/mapCodexRolloutLineToDirectMessages.ts`, `apps/cli/src/backends/codex/directSessions/codexDirectTranscriptSource.ts` | `directSessionV1` is only published when transcript storage is `direct`; direct paging is backward-only and tailing uses a provider-specific forward cursor. In development source, paging, follow and import share rollout-first selection; missing or paginated rollout storage uses read-only native `thread/items/list` pages. An empty native head alongside paginated rollout storage is treated as unavailable because Codex can recreate an empty SQL projection. Unavailable or unrepresentable history fails explicitly instead of treating a listing preview as a complete transcript. Native `webSearch`, `imageView`, and `sleep` items become `WebSearch`, `view_image`, and `sleep` tool rows through the app-server stream bridge that live sessions share; rollout projection still omits `web_search_call` items. The development native-history projection also supports subagent activity, collaboration tools, and dynamic tools without inventing child completion. Mixed user content retains its original parts in `meta.codexNativeContent`; skill/file references are visible, while images and audio explicitly report unavailable previews and retain their original URL/data, file id, or local path (local files are not copied). An indivisible oversized record is returned alone, matching rollout paging. | Replace `directSessionV1` and Codex-only cursor shapes with a single transcript-source envelope and generic direct-history cursor contract. | | Direct import / takeover into a Happy session | `supported` | `apps/cli/src/api/directSessions/takeover/resolveDirectTakeoverSpawnOptions.ts`, `apps/cli/src/api/machine/rpcHandlers.directSessions.ts`, `apps/cli/src/backends/codex/directSessions/getCodexDirectSessionWorkingDirectory.ts` | Takeover re-spawns with `resume`, `transcriptStorage: 'direct'`, `CODEX_HOME`, and any linked `codexBackendMode`; current direct-session candidates effectively route this through app-server semantics. | Reframe takeover as a provider-session source transition instead of a direct-session special case layered on top of spawn options. | | Local/remote switching | `supported` | `apps/cli/src/agent/localControl/createSharedProviderLocalControl.ts`, `apps/cli/src/backends/codex/runCodex.ts`, `apps/cli/src/backends/codex/appServer/createCodexSharedAppServer.ts`, `apps/cli/src/backends/codex/localControl/**` | App Server with Codex CLI 0.131.0+ on macOS/Linux uses one shared socket server for Happier and the native TUI; remote writes and approvals remain available while attached. Windows, ACP, and older App Server versions retain the exclusive compatibility path; plain MCP has no local control. | Shared provider state and terminal supervision are generic owners reused by OpenCode; Codex owns only version gating, socket-server lifecycle, and native attach arguments. | | Dynamic controls: collaboration modes / models / config options | `supported` | `apps/cli/src/backends/codex/appServer/sessionControlsMetadata.ts`, `apps/cli/src/backends/codex/appServer/runtime.ts`, `apps/ui/sources/agents/providers/codex/core.ts`, `apps/ui/sources/agents/providers/codex/uiBehavior.ts` | App-server reads `collaborationMode/list` and `model/list`, publishes generic session-control metadata, and currently supports one config option (`speed`) gated to `gpt-5.4` plus compatible auth methods. Initial catalog and native-goal projection is best-effort after the core thread attaches, so unavailable discovery endpoints do not keep session open pending. | Keep the generic state keys and extend the same controls contract to ACP/MCP if they ever gain parity; avoid reintroducing ACP-branded metadata for Codex. |