shiyue/cc-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复历史边界与流式消息去重并更新发布包

Browse Source
This commit is contained in:
shiyue
2026-09-21 00:38:51 +08:00
parent db04e088bc
commit f8f5ef3a0b
14 changed files with 1733 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

398
lib/codex-rollouts.js
View File

@@ -22,23 +22,37 @@ function createCodexRolloutStore(deps) {
return crypto.createHash('sha256').update(String(value || '')).digest('hex').slice(0, 24);
}
function extractTurnKey(entry, payload, fallback) {
const context = payload?.turn_context || entry?.turn_context || {};
const value = entry?.turn_id
|| payload?.turn_id
|| context?.turn_id
|| context?.id
|| null;
return value ? String(value) : fallback;
function getRolloutTurnId(entry) {
const payload = entry?.payload || {};
const context = payload.turn_context || entry?.turn_context || {};
const value = payload.turn_id
|| payload.internal_chat_message_metadata_passthrough?.turn_id
|| entry?.turn_id
|| context.turn_id
|| context.id
|| (entry?.type === 'turn_context' ? payload.id : null);
return value ? String(value) : null;
}
function extractCcwebSourceConversation(text) {
const match = String(text || '').match(/^来自「([^」]+)」对话（ID:\s*([0-9a-fA-F-]{36})）的消息：/);
if (!match) return null;
return { title: match[1], id: match[2].toLowerCase() };
const value = String(text || '').replace(/\r\n/g, '\n');
const match = value.match(/^来自「([^」]+)」对话（ID:\s*([0-9a-fA-F-]{36})）的消息：/);
if (match) return { title: match[1], id: match[2].toLowerCase() };
// 自动继续包装必须同时满足固定开头、请求标识、目标会话和正文结构。
// 普通用户只是在正文中提到“子对话回传”不能被当作内部消息过滤。
const returned = value.match(/^子对话回传已返回，但已返回不等于已完成。请先检查返回内容是否完整满足原始请求，再决定继续推进、补问目标对话或向用户汇报。\n\nrequestId：([^\n]+)\n目标对话：「([^」]+)」（ID: ([0-9a-fA-F-]{36}|未知)）\n\n原始请求：\n[\s\S]*?\n\n返回正文：\n/);
if (!returned) return null;
return { title: returned[2], id: returned[3] === '未知' ? null : returned[3].toLowerCase() };
}
function parseCodexRolloutLines(lines) {
const entries = [];
for (const line of lines) {
try {
const entry = JSON.parse(line);
if (entry && typeof entry === 'object') entries.push(entry);
} catch {}
}
const messages = [];
const pendingToolCalls = new Map();
const meta = {
@@ -53,10 +67,14 @@ function createCodexRolloutStore(deps) {
};
const totalUsage = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
let currentAssistant = null;
let currentTurnId = null;
let currentTurnKey = null;
let currentSegment = null;
let implicitTurnSequence = 0;
let sawRealUserMessage = false;
const fallbackUserMessages = [];
let lastUser = null;
const userMessagesById = new Map();
const assistantsByTurn = new Map();
const originalAssistantIds = new WeakMap();
function rememberSourceConversation(text) {
if (meta.sourceConversationId) return;
@@ -66,42 +84,190 @@ function createCodexRolloutStore(deps) {
meta.sourceConversationTitle = sourceConversation.title;
}
function ensureAssistant(ts, turnKey = null) {
// 没有 turn_id 的旧 rollout 仍需在每次新助手输出时生成新的隐式轮次键。
// 否则不同轮次会共用同一个 nativeTurnKey，前端按稳定键去重时会误删后续回复。
const resolvedTurnKey = turnKey || currentTurnKey || `${meta.threadId || 'native'}:implicit:${implicitTurnSequence += 1}`;
if (currentAssistant && currentTurnKey !== resolvedTurnKey) flushAssistant();
currentTurnKey = resolvedTurnKey;
if (!currentAssistant) {
currentAssistant = {
role: 'assistant',
content: '',
toolCalls: [],
timestamp: ts || null,
nativeTurnKey: `native:${meta.threadId || 'unknown-thread'}:${currentTurnKey}`,
id: `native:${stableHash(`${meta.threadId || ''}:${currentTurnKey}`)}`,
};
} else if (!currentAssistant.timestamp && ts) {
currentAssistant.timestamp = ts;
function messageIdentity(entry) {
const payload = entry.payload || {};
const passthrough = payload.internal_chat_message_metadata_passthrough || {};
const clientMessageId = payload.clientMessageId || payload.client_user_message_id
|| passthrough.clientMessageId || passthrough.client_user_message_id || null;
const id = payload.id || payload.message_id || entry.id || entry.message_id || null;
return { id: id ? String(id) : null, clientMessageId: clientMessageId ? String(clientMessageId) : null };
}
// 有 user_message 事件时，它才是用户输入记录；response_item 中还混有
// AGENTS、environment 等运行上下文，只有配对成功的记录才可补充身份字段。
const userEventsByText = new Map();
const responseUsers = [];
const responseForUserEvent = new Map();
let scannedTurnId = null;
let hasUserEvents = false;
for (let index = 0; index < entries.length; index += 1) {
const entry = entries[index];
scannedTurnId = getRolloutTurnId(entry) || scannedTurnId;
if (entry.type === 'event_msg' && entry.payload?.type === 'user_message') {
hasUserEvents = true;
const text = String(entry.payload.message || '').trim();
const candidates = userEventsByText.get(text) || [];
candidates.push({ entry, index, turnId: scannedTurnId });
userEventsByText.set(text, candidates);
} else if (entry.type === 'response_item' && entry.payload?.type === 'message' && entry.payload.role === 'user') {
responseUsers.push({ entry, index, turnId: scannedTurnId });
}
return currentAssistant;
if (entry.type === 'event_msg' && ['task_complete', 'task_completed', 'turn_complete', 'turn_completed', 'turn_failed', 'turn_aborted'].includes(entry.payload?.type)) {
scannedTurnId = null;
}
}
for (const response of responseUsers) {
const text = extractCodexMessageText(response.entry.payload.content).trim();
let matched = null;
for (const candidate of userEventsByText.get(text) || []) {
if (responseForUserEvent.has(candidate.entry)) continue;
if (response.turnId && candidate.turnId && response.turnId !== candidate.turnId) continue;
if (!matched || Math.abs(candidate.index - response.index) < Math.abs(matched.index - response.index)) matched = candidate;
}
if (matched) responseForUserEvent.set(matched.entry, response.entry);
}
function assignAssistantIdentity(assistant) {
const segmentSuffix = currentSegment ? `:segment:${currentSegment}` : '';
assistant.turnId = currentTurnId;
assistant.nativeThreadId = meta.threadId;
assistant.nativeTurnKey = `native:${meta.threadId || 'unknown-thread'}:${currentTurnKey}${segmentSuffix}`;
assistant.id = originalAssistantIds.get(assistant)
|| `history-assistant:${stableHash(`${meta.threadId || ''}:${currentTurnKey}${segmentSuffix}`)}`;
if (currentSegment) assistant.nativeTurnSegment = currentSegment;
}
function flushAssistant() {
if (!currentAssistant) return;
if ((currentAssistant.content || '').trim() || currentAssistant.toolCalls.length > 0) {
if (currentAssistant && ((currentAssistant.content || '').trim() || currentAssistant.toolCalls.length > 0)) {
if (!currentAssistant.turnId && !originalAssistantIds.has(currentAssistant)) {
const content = currentAssistant.content || JSON.stringify(currentAssistant.toolCalls);
currentAssistant.id = `history-assistant:${stableHash(`${meta.threadId || ''}:${currentAssistant.timestamp || ''}:${content}`)}`;
}
messages.push(currentAssistant);
}
currentAssistant = null;
currentTurnKey = null;
pendingToolCalls.clear();
}
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
let entry;
try { entry = JSON.parse(trimmed); } catch { continue; }
function selectTurn(turnId) {
if (!turnId || turnId === currentTurnId) return;
const previousTurnKey = currentTurnKey;
if (currentTurnId) flushAssistant();
currentTurnId = turnId;
currentTurnKey = turnId;
currentSegment = null;
// 老格式可能在首段输出后才提供 turn_context，补齐身份而不凭空拆出气泡。
if (currentAssistant) {
assistantsByTurn.delete(previousTurnKey);
assistantsByTurn.set(currentTurnKey, [currentAssistant]);
assignAssistantIdentity(currentAssistant);
}
if (lastUser && !lastUser.assistantStarted && lastUser.turnKey === previousTurnKey) {
lastUser.turnKey = currentTurnKey;
lastUser.message.turnId = currentTurnId;
}
}
function ensureAssistant(entry) {
selectTurn(getRolloutTurnId(entry));
if (!currentTurnKey) currentTurnKey = `implicit:${implicitTurnSequence += 1}`;
if (!currentAssistant) {
currentAssistant = { role: 'assistant', content: '', toolCalls: [], timestamp: entry.timestamp || null };
assignAssistantIdentity(currentAssistant);
const assistants = assistantsByTurn.get(currentTurnKey) || [];
assistants.push(currentAssistant);
assistantsByTurn.set(currentTurnKey, assistants);
}
if (lastUser) lastUser.assistantStarted = true;
return currentAssistant;
}
function rememberAssistantMessageId(assistant, entry) {
const { id } = messageIdentity(entry);
if (!id) return;
if (!originalAssistantIds.has(assistant)) {
originalAssistantIds.set(assistant, id);
assistant.id = id;
}
if (!assistant.nativeMessageIds) assistant.nativeMessageIds = [];
if (!assistant.nativeMessageIds.includes(id)) assistant.nativeMessageIds.push(id);
}
function appendUser(entry, text) {
text = String(text || '').trim();
if (!text) return;
rememberSourceConversation(text);
// 内部回传只是 native 的输入包装，不能冒充用户，也不能打断正在聚合的助手。
if (extractCcwebSourceConversation(text)) return;
const identity = messageIdentity(entry);
const turnId = getRolloutTurnId(entry);
const ts = entry.timestamp || null;
const known = (identity.id && userMessagesById.get(identity.id))
|| (identity.clientMessageId && userMessagesById.get(identity.clientMessageId));
const existing = known;
if (existing) {
if (identity.clientMessageId) existing.message.clientMessageId = identity.clientMessageId;
if (identity.id) {
if (!existing.hasOriginalId) {
existing.message.id = identity.id;
existing.hasOriginalId = true;
}
userMessagesById.set(identity.id, existing);
}
if (identity.clientMessageId) userMessagesById.set(identity.clientMessageId, existing);
return;
}
selectTurn(turnId);
const hadAssistant = (assistantsByTurn.get(currentTurnKey) || []).length > 0;
flushAssistant();
if (!currentTurnId) {
currentTurnKey = `implicit:${implicitTurnSequence += 1}`;
currentSegment = null;
}
const id = identity.id || (identity.clientMessageId ? `client:${identity.clientMessageId}` : null)
|| `history-user:${stableHash(`${meta.threadId || ''}:${currentTurnId || ''}:${ts || ''}:${text}`)}`;
if (hadAssistant && currentTurnId) {
// steer 是真实用户输入：保持前后邻接，同时避免同 turn 的两个助手段互相去重。
for (const assistant of assistantsByTurn.get(currentTurnKey) || []) {
if (!assistant.nativeTurnSegment) assistant.nativeTurnSegment = 'initial';
}
currentSegment = stableHash(id);
}
const message = { role: 'user', content: text, timestamp: ts, id, turnId: currentTurnId, nativeThreadId: meta.threadId };
if (identity.clientMessageId) message.clientMessageId = identity.clientMessageId;
messages.push(message);
lastUser = { message, turnKey: currentTurnKey, assistantStarted: false, hasOriginalId: !!identity.id };
if (identity.id) userMessagesById.set(identity.id, lastUser);
if (identity.clientMessageId) userMessagesById.set(identity.clientMessageId, lastUser);
if (!meta.title) meta.title = text.slice(0, 80).replace(/\n/g, ' ');
}
function appendTool(entry, output = false) {
const payload = entry.payload || {};
const assistant = ensureAssistant(entry);
const fallbackName = payload.type.startsWith('custom_') ? 'CustomToolCall'
: payload.type.startsWith('mcp_') ? 'McpToolCall' : 'FunctionCall';
const toolUseId = payload.call_id || payload.id
|| `native-tool:${stableHash(`${assistant.nativeTurnKey}:${payload.type}:${entry.timestamp || ''}:${JSON.stringify(payload)}`)}`;
let tool = pendingToolCalls.get(toolUseId);
if (!tool) {
tool = { name: payload.name || payload.tool || fallbackName, id: toolUseId, input: null, done: false };
assistant.toolCalls.push(tool);
pendingToolCalls.set(toolUseId, tool);
}
if (!output) {
tool.name = payload.name || payload.tool || tool.name;
tool.input = sanitizeToolInput(tool.name, payload.input ?? payload.arguments ?? '');
}
if (output || payload.status === 'completed' || payload.status === 'failed' || payload.result !== undefined) {
tool.done = true;
const result = payload.output ?? payload.result;
if (result !== undefined) tool.result = (typeof result === 'string' ? result : JSON.stringify(result)).slice(0, 2000);
}
}
for (const entry of entries) {
const ts = entry.timestamp || null;
if (ts) meta.updatedAt = ts;
@@ -113,21 +279,22 @@ function createCodexRolloutStore(deps) {
continue;
}
if (entry.type === 'event_msg' && ['task_started', 'turn_started'].includes(entry.payload?.type)) {
flushAssistant();
currentTurnKey = extractTurnKey(entry, entry.payload, `${meta.threadId || 'native'}:turn:${implicitTurnSequence += 1}`);
if ((entry.type === 'event_msg' && ['task_started', 'turn_started'].includes(entry.payload?.type))
|| entry.type === 'turn_context') {
selectTurn(getRolloutTurnId(entry));
if (!currentTurnKey) currentTurnKey = `implicit:${implicitTurnSequence += 1}`;
continue;
}
if (entry.type === 'turn_context') {
const turnKey = extractTurnKey(entry, entry.payload, currentTurnKey || `${meta.threadId || 'native'}:turn:${implicitTurnSequence += 1}`);
if (currentAssistant && currentTurnKey !== turnKey) flushAssistant();
currentTurnKey = turnKey;
continue;
}
if (entry.type === 'event_msg' && ['task_complete', 'turn_completed', 'turn_failed', 'turn_aborted'].includes(entry.payload?.type)) {
flushAssistant();
if (entry.type === 'event_msg' && ['task_complete', 'task_completed', 'turn_complete', 'turn_completed', 'turn_failed', 'turn_aborted'].includes(entry.payload?.type)) {
const turnId = getRolloutTurnId(entry);
// 迟到的上一轮结束事件不能截断当前轮。
if (!turnId || !currentTurnId || turnId === currentTurnId) {
flushAssistant();
currentTurnId = null;
currentTurnKey = null;
currentSegment = null;
}
continue;
}
@@ -147,119 +314,41 @@ function createCodexRolloutStore(deps) {
}
if (entry.type === 'event_msg' && entry.payload?.type === 'user_message') {
const text = String(entry.payload?.message || '').trim();
if (text) {
sawRealUserMessage = true;
flushAssistant();
const sourceConversation = extractCcwebSourceConversation(text);
rememberSourceConversation(text);
// cc-web 跨对话投递在 native rollout 中会留下内部 user 输入。
// 目标会话已经保存了带来源的消息，这里不再伪造普通 user 气泡。
if (sourceConversation) continue;
if (!meta.title) meta.title = text.slice(0, 80).replace(/\n/g, ' ');
messages.push({
role: 'user',
content: text,
timestamp: ts,
id: `native-user:${stableHash(`${meta.threadId || ''}:${ts || ''}:${text}`)}`,
});
}
const response = responseForUserEvent.get(entry);
const responseIdentity = response ? messageIdentity(response) : {};
const eventIdentity = messageIdentity(entry);
const enrichedEntry = response ? {
...entry,
payload: {
...entry.payload,
id: eventIdentity.id || responseIdentity.id,
clientMessageId: eventIdentity.clientMessageId || responseIdentity.clientMessageId,
turn_id: getRolloutTurnId(entry) || getRolloutTurnId(response),
},
} : entry;
appendUser(enrichedEntry, entry.payload?.message);
continue;
}
if (entry.type !== 'response_item') continue;
const payload = entry.payload || {};
switch (payload.type) {
case 'message': {
if (payload.role === 'assistant') {
const text = extractCodexMessageText(payload.content);
if (text.trim()) {
appendAssistantContent(ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey)), text);
}
} else if (payload.role === 'user' && !sawRealUserMessage) {
const text = extractCodexMessageText(payload.content);
if (text.trim()) {
rememberSourceConversation(text);
if (!extractCcwebSourceConversation(text)) {
fallbackUserMessages.push({
role: 'user',
content: text,
timestamp: ts,
id: `native-user:${stableHash(`${meta.threadId || ''}:${ts || ''}:${text}`)}`,
});
}
}
if (payload.type === 'message') {
const text = extractCodexMessageText(payload.content);
if (payload.role === 'assistant' && text.trim()) {
const assistant = ensureAssistant(entry);
rememberAssistantMessageId(assistant, entry);
appendAssistantContent(assistant, text);
} else if (payload.role === 'user' && !hasUserEvents) {
appendUser(entry, text);
}
break;
}
case 'function_call': {
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
const toolUseId = payload.call_id || payload.id || crypto.randomUUID();
const tc = {
name: payload.name || 'FunctionCall',
id: toolUseId,
input: sanitizeToolInput(payload.name || 'FunctionCall', payload.arguments || ''),
done: false,
};
assistant.toolCalls.push(tc);
pendingToolCalls.set(toolUseId, tc);
break;
}
case 'function_call_output': {
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
const toolUseId = payload.call_id || crypto.randomUUID();
let tc = pendingToolCalls.get(toolUseId);
if (!tc) {
tc = { name: 'FunctionCall', id: toolUseId, input: null, done: false };
assistant.toolCalls.push(tc);
pendingToolCalls.set(toolUseId, tc);
}
tc.done = true;
tc.result = (typeof payload.output === 'string'
? payload.output
: JSON.stringify(payload.output || '')).slice(0, 2000);
break;
}
case 'custom_tool_call': {
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
const toolUseId = payload.call_id || payload.id || crypto.randomUUID();
const tc = {
name: payload.name || 'CustomToolCall',
id: toolUseId,
input: sanitizeToolInput(payload.name || 'CustomToolCall', payload.input || payload.arguments || ''),
done: payload.status === 'completed',
};
assistant.toolCalls.push(tc);
pendingToolCalls.set(toolUseId, tc);
break;
}
case 'custom_tool_call_output': {
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
const toolUseId = payload.call_id || crypto.randomUUID();
let tc = pendingToolCalls.get(toolUseId);
if (!tc) {
tc = { name: 'CustomToolCall', id: toolUseId, input: null, done: false };
assistant.toolCalls.push(tc);
pendingToolCalls.set(toolUseId, tc);
}
tc.done = true;
tc.result = (typeof payload.output === 'string'
? payload.output
: JSON.stringify(payload.output || '')).slice(0, 2000);
break;
}
default:
break;
} else if (['function_call', 'custom_tool_call', 'mcp_tool_call', 'mcp_call'].includes(payload.type)) {
appendTool(entry);
} else if (['function_call_output', 'custom_tool_call_output', 'mcp_tool_call_output', 'mcp_call_output'].includes(payload.type)) {
appendTool(entry, true);
}
}
flushAssistant();
if (!sawRealUserMessage && fallbackUserMessages.length > 0) {
const fallback = fallbackUserMessages[0];
if (!meta.title) meta.title = fallback.content.trim().slice(0, 80).replace(/\n/g, ' ');
return { meta, messages: fallbackUserMessages.concat(messages), totalUsage };
}
return { meta, messages, totalUsage };
}
@@ -309,6 +398,7 @@ function createCodexRolloutStore(deps) {
}
return {
getRolloutTurnId,
parseCodexRolloutLines,
getCodexRolloutFiles,
getImportedCodexThreadIds,