320 lines
12 KiB
JavaScript
320 lines
12 KiB
JavaScript
const fs = require('fs');
|
||
const path = require('path');
|
||
const crypto = require('crypto');
|
||
|
||
function createCodexRolloutStore(deps) {
|
||
const { codexSessionsDir, sessionsDir, normalizeSession, sanitizeToolInput } = deps;
|
||
|
||
function extractCodexMessageText(content) {
|
||
if (!Array.isArray(content)) return '';
|
||
return content
|
||
.filter((item) => item && (item.type === 'input_text' || item.type === 'output_text'))
|
||
.map((item) => item.text || '')
|
||
.join('');
|
||
}
|
||
|
||
function appendAssistantContent(turn, text) {
|
||
if (!turn || !text || !text.trim()) return;
|
||
turn.content = turn.content ? `${turn.content}\n\n${text}` : text;
|
||
}
|
||
|
||
function stableHash(value) {
|
||
return crypto.createHash('sha256').update(String(value || '')).digest('hex').slice(0, 24);
|
||
}
|
||
|
||
function extractTurnKey(entry, payload, fallback) {
|
||
const context = payload?.turn_context || entry?.turn_context || {};
|
||
const value = entry?.turn_id
|
||
|| payload?.turn_id
|
||
|| context?.turn_id
|
||
|| context?.id
|
||
|| null;
|
||
return value ? String(value) : fallback;
|
||
}
|
||
|
||
function extractCcwebSourceConversation(text) {
|
||
const match = String(text || '').match(/^来自「([^」]+)」对话（ID:\s*([0-9a-fA-F-]{36})）的消息：/);
|
||
if (!match) return null;
|
||
return { title: match[1], id: match[2].toLowerCase() };
|
||
}
|
||
|
||
function parseCodexRolloutLines(lines) {
|
||
const messages = [];
|
||
const pendingToolCalls = new Map();
|
||
const meta = {
|
||
threadId: null,
|
||
cwd: null,
|
||
title: '',
|
||
updatedAt: null,
|
||
cliVersion: null,
|
||
source: null,
|
||
sourceConversationId: null,
|
||
sourceConversationTitle: '',
|
||
};
|
||
const totalUsage = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
|
||
let currentAssistant = null;
|
||
let currentTurnKey = null;
|
||
let implicitTurnSequence = 0;
|
||
let sawRealUserMessage = false;
|
||
const fallbackUserMessages = [];
|
||
|
||
function rememberSourceConversation(text) {
|
||
if (meta.sourceConversationId) return;
|
||
const sourceConversation = extractCcwebSourceConversation(text);
|
||
if (!sourceConversation) return;
|
||
meta.sourceConversationId = sourceConversation.id;
|
||
meta.sourceConversationTitle = sourceConversation.title;
|
||
}
|
||
|
||
function ensureAssistant(ts, turnKey = null) {
|
||
// 没有 turn_id 的旧 rollout 仍需在每次新助手输出时生成新的隐式轮次键。
|
||
// 否则不同轮次会共用同一个 nativeTurnKey，前端按稳定键去重时会误删后续回复。
|
||
const resolvedTurnKey = turnKey || currentTurnKey || `${meta.threadId || 'native'}:implicit:${implicitTurnSequence += 1}`;
|
||
if (currentAssistant && currentTurnKey !== resolvedTurnKey) flushAssistant();
|
||
currentTurnKey = resolvedTurnKey;
|
||
if (!currentAssistant) {
|
||
currentAssistant = {
|
||
role: 'assistant',
|
||
content: '',
|
||
toolCalls: [],
|
||
timestamp: ts || null,
|
||
nativeTurnKey: `native:${meta.threadId || 'unknown-thread'}:${currentTurnKey}`,
|
||
id: `native:${stableHash(`${meta.threadId || ''}:${currentTurnKey}`)}`,
|
||
};
|
||
} else if (!currentAssistant.timestamp && ts) {
|
||
currentAssistant.timestamp = ts;
|
||
}
|
||
return currentAssistant;
|
||
}
|
||
|
||
function flushAssistant() {
|
||
if (!currentAssistant) return;
|
||
if ((currentAssistant.content || '').trim() || currentAssistant.toolCalls.length > 0) {
|
||
messages.push(currentAssistant);
|
||
}
|
||
currentAssistant = null;
|
||
currentTurnKey = null;
|
||
pendingToolCalls.clear();
|
||
}
|
||
|
||
for (const line of lines) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed) continue;
|
||
let entry;
|
||
try { entry = JSON.parse(trimmed); } catch { continue; }
|
||
const ts = entry.timestamp || null;
|
||
if (ts) meta.updatedAt = ts;
|
||
|
||
if (entry.type === 'session_meta') {
|
||
meta.threadId = entry.payload?.id || meta.threadId;
|
||
meta.cwd = entry.payload?.cwd || meta.cwd;
|
||
meta.cliVersion = entry.payload?.cli_version || meta.cliVersion;
|
||
meta.source = entry.payload?.source || meta.source;
|
||
continue;
|
||
}
|
||
|
||
if (entry.type === 'event_msg' && ['task_started', 'turn_started'].includes(entry.payload?.type)) {
|
||
flushAssistant();
|
||
currentTurnKey = extractTurnKey(entry, entry.payload, `${meta.threadId || 'native'}:turn:${implicitTurnSequence += 1}`);
|
||
continue;
|
||
}
|
||
|
||
if (entry.type === 'turn_context') {
|
||
const turnKey = extractTurnKey(entry, entry.payload, currentTurnKey || `${meta.threadId || 'native'}:turn:${implicitTurnSequence += 1}`);
|
||
if (currentAssistant && currentTurnKey !== turnKey) flushAssistant();
|
||
currentTurnKey = turnKey;
|
||
continue;
|
||
}
|
||
|
||
if (entry.type === 'event_msg' && ['task_complete', 'turn_completed', 'turn_failed', 'turn_aborted'].includes(entry.payload?.type)) {
|
||
flushAssistant();
|
||
continue;
|
||
}
|
||
|
||
if (entry.type === 'event_msg' && entry.payload?.type === 'token_count') {
|
||
const total = entry.payload?.info?.total_token_usage || null;
|
||
const usage = entry.payload?.info?.last_token_usage || null;
|
||
if (total) {
|
||
totalUsage.inputTokens = Math.max(totalUsage.inputTokens, total.input_tokens || 0);
|
||
totalUsage.cachedInputTokens = Math.max(totalUsage.cachedInputTokens, total.cached_input_tokens || 0);
|
||
totalUsage.outputTokens = Math.max(totalUsage.outputTokens, total.output_tokens || 0);
|
||
} else if (usage) {
|
||
totalUsage.inputTokens += usage.input_tokens || 0;
|
||
totalUsage.cachedInputTokens += usage.cached_input_tokens || 0;
|
||
totalUsage.outputTokens += usage.output_tokens || 0;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (entry.type === 'event_msg' && entry.payload?.type === 'user_message') {
|
||
const text = String(entry.payload?.message || '').trim();
|
||
if (text) {
|
||
sawRealUserMessage = true;
|
||
flushAssistant();
|
||
const sourceConversation = extractCcwebSourceConversation(text);
|
||
rememberSourceConversation(text);
|
||
// cc-web 跨对话投递在 native rollout 中会留下内部 user 输入。
|
||
// 目标会话已经保存了带来源的消息，这里不再伪造普通 user 气泡。
|
||
if (sourceConversation) continue;
|
||
if (!meta.title) meta.title = text.slice(0, 80).replace(/\n/g, ' ');
|
||
messages.push({
|
||
role: 'user',
|
||
content: text,
|
||
timestamp: ts,
|
||
id: `native-user:${stableHash(`${meta.threadId || ''}:${ts || ''}:${text}`)}`,
|
||
});
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (entry.type !== 'response_item') continue;
|
||
|
||
const payload = entry.payload || {};
|
||
switch (payload.type) {
|
||
case 'message': {
|
||
if (payload.role === 'assistant') {
|
||
const text = extractCodexMessageText(payload.content);
|
||
if (text.trim()) {
|
||
appendAssistantContent(ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey)), text);
|
||
}
|
||
} else if (payload.role === 'user' && !sawRealUserMessage) {
|
||
const text = extractCodexMessageText(payload.content);
|
||
if (text.trim()) {
|
||
rememberSourceConversation(text);
|
||
if (!extractCcwebSourceConversation(text)) {
|
||
fallbackUserMessages.push({
|
||
role: 'user',
|
||
content: text,
|
||
timestamp: ts,
|
||
id: `native-user:${stableHash(`${meta.threadId || ''}:${ts || ''}:${text}`)}`,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case 'function_call': {
|
||
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
|
||
const toolUseId = payload.call_id || payload.id || crypto.randomUUID();
|
||
const tc = {
|
||
name: payload.name || 'FunctionCall',
|
||
id: toolUseId,
|
||
input: sanitizeToolInput(payload.name || 'FunctionCall', payload.arguments || ''),
|
||
done: false,
|
||
};
|
||
assistant.toolCalls.push(tc);
|
||
pendingToolCalls.set(toolUseId, tc);
|
||
break;
|
||
}
|
||
case 'function_call_output': {
|
||
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
|
||
const toolUseId = payload.call_id || crypto.randomUUID();
|
||
let tc = pendingToolCalls.get(toolUseId);
|
||
if (!tc) {
|
||
tc = { name: 'FunctionCall', id: toolUseId, input: null, done: false };
|
||
assistant.toolCalls.push(tc);
|
||
pendingToolCalls.set(toolUseId, tc);
|
||
}
|
||
tc.done = true;
|
||
tc.result = (typeof payload.output === 'string'
|
||
? payload.output
|
||
: JSON.stringify(payload.output || '')).slice(0, 2000);
|
||
break;
|
||
}
|
||
case 'custom_tool_call': {
|
||
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
|
||
const toolUseId = payload.call_id || payload.id || crypto.randomUUID();
|
||
const tc = {
|
||
name: payload.name || 'CustomToolCall',
|
||
id: toolUseId,
|
||
input: sanitizeToolInput(payload.name || 'CustomToolCall', payload.input || payload.arguments || ''),
|
||
done: payload.status === 'completed',
|
||
};
|
||
assistant.toolCalls.push(tc);
|
||
pendingToolCalls.set(toolUseId, tc);
|
||
break;
|
||
}
|
||
case 'custom_tool_call_output': {
|
||
const assistant = ensureAssistant(ts, extractTurnKey(entry, payload, currentTurnKey));
|
||
const toolUseId = payload.call_id || crypto.randomUUID();
|
||
let tc = pendingToolCalls.get(toolUseId);
|
||
if (!tc) {
|
||
tc = { name: 'CustomToolCall', id: toolUseId, input: null, done: false };
|
||
assistant.toolCalls.push(tc);
|
||
pendingToolCalls.set(toolUseId, tc);
|
||
}
|
||
tc.done = true;
|
||
tc.result = (typeof payload.output === 'string'
|
||
? payload.output
|
||
: JSON.stringify(payload.output || '')).slice(0, 2000);
|
||
break;
|
||
}
|
||
default:
|
||
break;
|
||
}
|
||
}
|
||
|
||
flushAssistant();
|
||
if (!sawRealUserMessage && fallbackUserMessages.length > 0) {
|
||
const fallback = fallbackUserMessages[0];
|
||
if (!meta.title) meta.title = fallback.content.trim().slice(0, 80).replace(/\n/g, ' ');
|
||
return { meta, messages: fallbackUserMessages.concat(messages), totalUsage };
|
||
}
|
||
return { meta, messages, totalUsage };
|
||
}
|
||
|
||
function walkFiles(dir, files = []) {
|
||
let entries;
|
||
try {
|
||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||
} catch {
|
||
return files;
|
||
}
|
||
for (const entry of entries) {
|
||
const fullPath = path.join(dir, entry.name);
|
||
if (entry.isDirectory()) walkFiles(fullPath, files);
|
||
else if (entry.isFile()) files.push(fullPath);
|
||
}
|
||
return files;
|
||
}
|
||
|
||
function getCodexRolloutFiles() {
|
||
if (!fs.existsSync(codexSessionsDir)) return [];
|
||
return walkFiles(codexSessionsDir, []).filter((filePath) => filePath.endsWith('.jsonl')).sort().reverse();
|
||
}
|
||
|
||
function getImportedCodexThreadIds(agent = 'codex') {
|
||
const field = agent === 'codexapp' ? 'codexAppThreadId' : 'codexThreadId';
|
||
const imported = new Set();
|
||
try {
|
||
for (const f of fs.readdirSync(sessionsDir).filter((name) => name.endsWith('.json'))) {
|
||
try {
|
||
const session = normalizeSession(JSON.parse(fs.readFileSync(path.join(sessionsDir, f), 'utf8')));
|
||
if (session[field]) imported.add(session[field]);
|
||
} catch {}
|
||
}
|
||
} catch {}
|
||
return imported;
|
||
}
|
||
|
||
function parseCodexRolloutFile(filePath) {
|
||
try {
|
||
const content = fs.readFileSync(filePath, 'utf8');
|
||
const parsed = parseCodexRolloutLines(content.split('\n'));
|
||
parsed.filePath = filePath;
|
||
return parsed;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
return {
|
||
parseCodexRolloutLines,
|
||
getCodexRolloutFiles,
|
||
getImportedCodexThreadIds,
|
||
parseCodexRolloutFile,
|
||
};
|
||
}
|
||
|
||
module.exports = { createCodexRolloutStore };
|