diff --git a/.planning/2026-10-14-history-memory-fix/findings.md b/.planning/2026-10-14-history-memory-fix/findings.md new file mode 100644 index 0000000..8a9128f --- /dev/null +++ b/.planning/2026-10-14-history-memory-fix/findings.md @@ -0,0 +1,10 @@ +# 调查与实现依据 + +- 现有完整历史库:lib/session-history-store.js。 +- server.js 的 loadSession、saveSession、resolveSessionHistory、refreshSessionSnapshotCache 接入归档。 +- 原有契约要求原子提交、可信加载、旧修订合并、删除及清空不复活、完整运行态优先恢复。 +- 不能把分块存储只做在文件层:分页入口和快照刷新也须避免整份加载。 +- 采用不可变内容哈希块与原子 manifest 提交;可信基线只保留块引用,旧修订按单消息逐字段合并。 +- 服务端常用加载保留有界尾部,分页直接读区间,任务跟踪只读元数据;明确导入/全量维护才申请完整加载。 +- 集成回归创建 `_history/_turns` 后发现保留目录触发无效会话ID错误;列表枚举须显式排除该目录和同名旧文件,34项持久化回归已覆盖。 +- 最终合成夹具:70.5MiB完整旧归档在256MiB堆上限下完成流式迁移、增量保存和67页读取,RSS峰值145.9MiB;1128条正文和工具数据逐条摘要相符。完整集成回归通过;运行服务尚未重启,生产内存效果须在修复生效后另行观察。 diff --git a/.planning/2026-10-14-history-memory-fix/progress.md b/.planning/2026-10-14-history-memory-fix/progress.md new file mode 100644 index 0000000..a71232a --- /dev/null +++ b/.planning/2026-10-14-history-memory-fix/progress.md @@ -0,0 +1,15 @@ +# 执行记录 + +2026-10-14:初始化项目上下文,确认完整历史功能与归档库。用户要求继续推进修复。 +定位已完成,进入分块存储和服务端有界入口实现;测试代理负责隔离内存回归。未重启服务。 + +分块清单、增量保存、64KiB流式迁移、默认尾部和首屏/重连/分页入口已实现。 +补齐元数据保存、单消息修改窗口、跨窗口查重和附件逐块遍历,避免尾部加载遗漏早期消息。 +首轮70.5MiB合成归档在256MiB堆上限下通过,RSS峰值144.6MiB;全部1128条消息逐条摘要相符。 +当前补齐默认窗口、并发和失败重试回归,并验证运行完成及自动重试路径。测试保持隔离目录,未重启生产服务。 + +集成回归定位并修复 `_history/_turns` 保留目录被会话列表枚举的问题,补充真实运行态目录及同名旧文件回归。完整历史持久化单测34项全部通过,完整集成回归继续执行。 + +最终验证:`TMPDIR=/dev/shm timeout 300s node scripts/regression.js` 退出0,输出 Regression checks passed;包含完整历史存储、内存、首屏/分页、运行完成、重启恢复及自动重试回归。最终单独内存回归退出0:70.5MiB旧归档、256MiB堆上限、RSS峰值145.9MiB,1128条消息经67页逐条校验完整,6次增量保存复用37个原始块,单次旧文件读取最多64KiB。该数字为合成隔离夹具,不能当作生产进程测量。语法检查和 git diff --check 通过。 + +交付完成:所有计划步骤完成,临时TODO CSV已清理;源码与后端契约已更新,保留既有未提交文案改动及10-11任务。未修改生产历史,未打包、提交或重启服务,运行进程仍须后续安全重启后生效。 diff --git a/.planning/2026-10-14-history-memory-fix/task_plan.md b/.planning/2026-10-14-history-memory-fix/task_plan.md new file mode 100644 index 0000000..eaa66ff --- /dev/null +++ b/.planning/2026-10-14-history-memory-fix/task_plan.md @@ -0,0 +1,25 @@ +# 完整历史内存修复 + +目标:保留完整会话历史,消除完整归档加载、保存和基线的整份复制,增加分块存储、按页读取和安全迁移。 + +## 阶段 +1. 确认归档和分页入口,确定内存有界的实现:complete +2. 实现分块归档、按页读取和旧数据迁移:complete +3. 验证历史完整性、并发保护和内存峰值:complete +4. 完成代码复核并交付验证结果:complete + +## 边界 +- 保留正文、工具参数和结果,不用截断历史换取内存。 +- 不覆盖现有 server.js 未提交改动。 +- 测试使用隔离目录,不修改生产归档。 +- 重启前必须确认没有其他运行中的对话。 + +## 已知情况 +- 当前完整历史格式见 .trellis/spec/backend/session-history.md。 +- 用户要求继续推进,不再停留在审计流程。 +- 初始 git 状态:server.js 已修改;10-11-remove-compact-followup 为既有任务。 + +## 问题记录 +- 首次定位 todo-list-csv 技能不存在,采用项目规定的 CSV 与 update_plan 同步。 +- Codebase Memory 部分命中位置错位,须用实际文件交叉校验。 +- 本轮 Codebase Memory 连接已关闭,改用 FastCtx 精确读文件继续实现。 diff --git a/.planning/2026-10-14-runtime-json-fix/findings.md b/.planning/2026-10-14-runtime-json-fix/findings.md new file mode 100644 index 0000000..e0273a4 --- /dev/null +++ b/.planning/2026-10-14-runtime-json-fix/findings.md @@ -0,0 +1,9 @@ +# 实现依据 + +- `saveTurn`仍通过`writeAtomicJson`整份序列化,`loadTurn`仍整文件读取并JSON.parse,是本轮补齐点。 +- `preserveNewerTaskTrackingForSessionSave`已只读元数据,但每次保存重复读清单,改为归档内部合并。 +- 历史基线已经是不可变块引用,不需要改回整文件磁盘基线。 +- 流式JSON实现固定64KiB字节缓冲,字符串按8Ki字符段转义并保护代理对;解析按容器和值构造对象,保留原v1运行态格式。旧归档迁移共用新解析器,移除重复扫描实现。 +- 任务状态保护使用同一次提交取得的当前清单;可信同修订保存仅读256字节修订头,保存后才同步受保护的任务状态,失败不推进会话与基线。 +- 最终隔离集成回归退出0;JSON10项、历史持久化38项通过。80MiB运行态在256MiB堆参数下RSS峰值210.5MiB,读写最大64KiB;重开正文摘要完整,write/fsync/rename失败保留旧档且无tmp残留。 +- 本轮仍未重启服务或修改生产历史。运行态返回完整对象,峰值随单条正文和同时存活运行态增长;测得RSS只代表合成夹具。 diff --git a/.planning/2026-10-14-runtime-json-fix/progress.md b/.planning/2026-10-14-runtime-json-fix/progress.md new file mode 100644 index 0000000..868aa07 --- /dev/null +++ b/.planning/2026-10-14-runtime-json-fix/progress.md @@ -0,0 +1,15 @@ +# 执行记录 + +2026-10-14:用户要求按另一实例方案继续修改;限定为运行态流式读写、结构比较及单次提交中的任务状态保护,保留分块归档设计。 + +已实现通用流式JSON读写、仅容器克隆和结构比较,并接入完整运行态、原子清单写入及旧格式迁移。归档内部保护任务版本,同修订保存只读修订头。测试代理只新增JSON与大运行态回归,主代理补任务状态及失败边界。 + +首次持久化回归发现新增修订头调用与提交修订局部变量同名,命中暂存区错误;局部变量改为nextRevision后重新验证,避免同修订快速路径失效。 + +持久化回归38项通过,新增同修订零完整读取、并发任务版本、任务看板删除及失败不推进基线四类验证。JSON新测试10项通过;80MiB运行态在256MiB堆下读取和保存并注入写/fsync/rename失败,正文摘要完整、旧文件不变、无临时文件残留,RSS峰值210.1MiB、单次读写64KiB。继续完整集成回归。 + +最终完整回归:`TMPDIR=/dev/shm timeout 300s node scripts/regression.js`退出0,输出Regression checks passed。进一步修复toJSON getter只读取一次并补断言,最终JSON回归10项及运行态内存回归均退出0;运行态80MiB、256MiB堆参数下RSS峰值210.5MiB,单次读写64KiB。lib/json-persistence.js、lib/session-history-store.js与server.js语法检查通过,git diff --check通过。文档已补充格式、失败语义和峰值局限。 + +交付完成:最终测试日志与工作区状态已核对,全部计划项完成,移除临时CSV跟踪文件。保留历史分块归档和分页读取,未改写生产历史、未打包、未提交或重启服务;当前服务进程尚未加载本轮源码修改。 + +后续交付:按用户要求使用本地Bun 1.4.2重新生成CentOS 7 baseline单文件发布包,目标为bun-linux-x64-baseline。规定的8个文件语法检查及git diff --check通过;新二进制MCP initialize返回有效JSON-RPC结果并正常退出。压缩包138个归档项,二进制和前端资源与当前构建一致;大小42.91MiB,SHA-256为7bbe795e39f61fdc4e47d96a2083d92e9a2f59593e2e4c5397debdb67a84b7c9。提交范围包含工作区全部源码、回归脚本、文档、任务记录和更新后的发布包;未重启服务。 diff --git a/.planning/2026-10-14-runtime-json-fix/task_plan.md b/.planning/2026-10-14-runtime-json-fix/task_plan.md new file mode 100644 index 0000000..bc6f9c9 --- /dev/null +++ b/.planning/2026-10-14-runtime-json-fix/task_plan.md @@ -0,0 +1,12 @@ +# 运行态流式持久化补齐 + +目标:吸收另一实例的流式JSON读写和单次任务状态提交方案,同时保留当前分块归档、分页读取和清单基线。 + +1. 确认运行态持久化和任务状态合并的现有契约:complete +2. 实现流式JSON读写并整合任务状态保护:complete +3. 验证JSON语义、失败保护及大运行态低堆回归:complete +4. 更新契约与交付验证结果:complete + +边界:不覆盖此前已完成的历史分块实现和既有文案改动;不修改生产历史,不打包或重启服务;单测上限60秒。 + +已知:Codebase Memory连接此前已关闭,源码使用FastCtx读取及rg精确校验;todo-list-csv技能不存在,按AGENTS维护CSV和update_plan。 diff --git a/.trellis/spec/backend/quality-guidelines.md b/.trellis/spec/backend/quality-guidelines.md index 8777537..6b0c3b1 100644 --- a/.trellis/spec/backend/quality-guidelines.md +++ b/.trellis/spec/backend/quality-guidelines.md @@ -45,6 +45,8 @@ Questions to answer: - 完整档案先原子提交,随后更新有界快照;完整档案失败时不得用裁剪快照覆盖上次状态。档案提交成功而快照失败时,读取和会话列表仍须能找到已提交档案。 - 可信完整读取后的修改应保留清空、撤回和插入顺序;短预览及旧修订不能覆盖完整消息或丢掉后来追加的记录,正常并发保存不得静默漏记。 - 旧版截断历史只有在确认原始记录对应边界后才能补入完整档案;新建档案本身不能证明旧历史已完整。 +- 大型运行态JSON须使用分段编码和直接构造对象的流式读取,禁止完整原文、全量写出缓冲及序列化克隆同时驻留;不能把单次读写有界解释为返回对象总内存有界。 +- 任务状态版本保护在同一次归档提交内处理,失败不得提前推进可信基线或活跃任务状态;JSON读写需对照原生语义并覆盖跨段编码、短读短写及原子失败保护。 --- diff --git a/.trellis/spec/backend/session-history.md b/.trellis/spec/backend/session-history.md index 94c11f3..f1c8b65 100644 --- a/.trellis/spec/backend/session-history.md +++ b/.trellis/spec/backend/session-history.md @@ -9,7 +9,15 @@ ```js createSessionHistoryStore({ sessionsDir, fs? }) store.load(id) // 完整会话或 null;注册可信加载基线 -store.save(session) // { revision, complete, session, rebased };失败抛错 +store.loadTail(id, count = 180, { maxBytes? }) // 只加载可修改尾部,保留全局索引 +store.loadMetadata(id) // 只加载元数据,保存时保留全部消息 +store.loadWindow(id, start, end) // 可修改中间窗口,保存时保留前缀和后缀 +store.loadHistoryPage(id, before, count = 24) // { messages, total, start, end, complete, revision } +store.findMessage(id, predicate) // 按块查找,内存只保留少量块 +store.forEachMessage(id, callback) // 按块遍历,用于附件清理等全历史操作 +store.messageIndex(id, messageId) // 从轻量清单查询全局消息索引 +store.countMessages(id, role) // 从清单计数,不加载正文 +store.save(session, { taskBoardMetadata? }) // { revision, complete, session, rebased, taskTrackingPreserved };失败抛错 store.markLegacy(session) // 注册旧快照来源与完整性状态 store.markComplete(session, { reset? }) // 确认可靠补齐或主动清空;返回状态还原函数 store.state(session) // 来源、修订与完整性;不暴露消息基线 @@ -21,21 +29,26 @@ store.loadTurn(id) // 优先于有界运行预览 store.listTurnIds() store.removeTurn(id) // 完整消息已提交或主动清空后才清理 saveSession(session) // 完整档案提交成功为 true,否则 false -loadSession(id) // 优先完整档案;无档案才读取旧快照 -resolveSessionHistory(session) // { messages, source, available, recoverable } +loadSession(id, options?) // 默认尾部最多180条、2MiB;单条超大消息完整保留 +// options 支持 metadata、count、window、messageId、promptId;显式 full 才全量加载 +resolveSessionHistory(session, { count?, before? }) // 有界出口另含 total/start/end ``` ## 3. 存储与响应契约 -- `${SESSIONS_DIR}/_history/.json` 是持久提交点,格式为 `{ version: 1, revision, complete, resetRevision, session }`。`session` 包含完整可序列化字段,排除运行态句柄;正文、工具、数组和深度不使用预览限额。 +- `${SESSIONS_DIR}/_history//manifest.json` 是唯一持久提交点,格式为 `{ version: 2, revision, complete, resetRevision, session, entries }`。`session` 只有可序列化元数据,排除运行态句柄;`entries` 只保留稳定消息 ID、角色、摘要、块引用、块内索引和字节数。 +- 完整消息保存在 `chunks/.json` 不可变块中,默认每块最多 2 MiB 或 64 条;单条超大消息允许独占大块,正文、工具、数组和深度不使用预览限额。保存复用内容未变的块,只写修改的消息,不序列化或复制整份历史。 +- 旧 `_history/.json` 首次读取时使用 `lib/json-persistence.js` 按 64 KiB 扫描流式迁移,每次构造一个字段或消息;块全部落盘后才提交新清单。旧文件保留,迁移失败不破坏原文件。 - `.json` 是有界快照缓存。`historyArchiveRevision` 位于顶层首字段,用来判断缓存是否过期;不得从嵌套同名键推断修订。 - `CC_WEB_SESSION_PERSIST_MAX_MESSAGES`、`CC_WEB_SESSION_MESSAGE_CONTENT_MAX_CHARS` 等预览配置不能限制完整档案;真实完成和恢复路径也不得先裁剪消息再提交。 -- 可信来源保存在存储实例的 `WeakMap`,不能凭 JSON 的修订字段伪造。当前修订精确保存删除、清空和插入;旧修订根据加载基线合并互不冲突的修改,保持活跃消息及工具对象引用。 -- 归档提交顺序为临时文件写入、文件 `fsync`、原子 `rename`、推进基线,再更新快照。档案失败不得更新快照;档案已提交而快照失败仍返回成功。 -- 列表枚举快照与档案 ID 的并集。首次快照失败后,孤立档案仍须可读、可列出;缓存继续失败时使用内存有界预览。 -- Codex App 运行态写入 `_history/_turns/.json`,包含完整正文、工具及映射;有界运行态文件只作预览。重启枚举完整运行态,恢复消息提交成功后才清理,异常档案保留证据。 +- 可信来源保存在存储实例的 `WeakMap`,不能凭 JSON 的修订字段伪造。基线只引用清单,不保留整份正文字符串。当前修订精确保存窗口内删除和插入,未加载前后缀原样保留;跨未知前缀的重复 ID 保留已提交原文。旧修订按单消息三方合并,保持活跃消息及工具对象引用。主动清空须明确 `markComplete(session, { reset: true })`。 +- 归档提交顺序为修改块写入、文件 `fsync`、原子 `rename`,完成协调所需的块读取,再原子提交清单、推进基线、更新有界快照。清单失败不得推进基线或更新快照;已提交而快照失败仍返回成功。每32次提交回收无引用块,跨存储实例的活跃旧基线通过弱引用保护,清理失败不改变提交结果。 +- 可信同修订保存仅读取最多256字节的修订头并复用不可变清单;修订已变化才读取当前清单。任务状态版本保护使用同一次提交的清单,不为单个字段重复加载历史或清单。普通保存保留更高任务版本,任务看板通过 `taskBoardMetadata=true` 显式提交其负责的更新或删除;受保护状态仅在提交成功后同步到活跃对象。克隆只复制JSON容器,比较使用规范JSON值的结构比较,不为正文生成序列化副本。 +- 列表枚举快照与档案 ID 的并集,排除 `_turns` 保留目录及同名旧文件。首次快照失败后,孤立档案仍须可读、可列出;缓存继续失败时使用内存有界预览。 +- Codex App 运行态写入 `_history/_turns/.json`,包含完整正文、工具及映射,保留原 `version:1` 格式。读取直接构造对象,写入按小段转义正文并聚合写出,单次读写最多64KiB字节;不生成整份JSON原文或写出缓冲。运行态读取结果仍为完整对象,不能声称总内存与正文大小无关。有界运行态文件只作预览。重启枚举完整运行态,恢复消息提交成功后才清理,异常档案保留证据。 - 用户提问必须保存成功后才启动 runtime 或接受 steer;完成、重启恢复及跨会话返回必须检查保存结果,失败保留可重试内容。CLI 恢复标识须区分回合,同一已提交回合不可因清理失败再次追加。 - 确认原始边界后的旧前缀必须先进入自身档案。`complete=false` 保留旧缺口状态,不能因创建档案就宣称旧历史完整。 +- 首屏和重连只读取最近12条;向前分页每次读取24条,目标消息预取受最大页数限制。表单和撤回使用单消息窗口,历史查重和附件枚举按块执行;任务看板只读取和保存元数据。 - 响应的 `historyCursor` 是尚未展示的已有消息数;正数允许分页。`historyAvailable=false` 只说明旧来源待核实,不能阻断正文或用户目录。完整档案响应 `historySource='archive'`、`historyAvailable=true`。 ## 4. 校验与错误矩阵 @@ -62,6 +75,11 @@ resolveSessionHistory(session) // { messages, source, available, recoverable ## 6. 必需回归与断言 - `scripts/session-history-store-unit.js`:实际生产保存和读取入口、超限内容、存储重开、两类写失败、孤立档案、并发及活跃引用、删除/清空/撤回、确认前缀先归档。 +- 同一脚本还须覆盖默认尾部、中间窗口及元数据保存保留未加载历史,不同加载范围并发合并、清空防复活、失败后重试和未加载前缀 ID 去重。 +- 同一脚本还须覆盖同修订保存只读修订头、更高任务版本整体保护、任务看板显式删除以及清单失败后任务状态和可信基线不提前推进。 +- `scripts/json-persistence-unit.js`:编码与读取对照原生JSON及项目bigint/Map/Set转换,覆盖toJSON单次求值、对象键、UTF-8和代理对跨段、短读短写、非法JSON及编码中途异常的原子失败保护。 +- `scripts/runtime-state-memory-unit.js`:独立子进程在256MiB堆参数下读写80MiB旧v1完整运行态,正文与工具映射摘要逐项一致;检查单次64KiB字节读写及write/fsync/rename失败保留旧文件、无临时文件残留。合成峰值不能代表生产或任意并发量。 +- `scripts/session-history-memory-unit.js`:独立子进程在256MiB堆上限下流式迁移70.5MiB旧归档,执行尾部/元数据保存和完整分页;逐条摘要校验所有历史,监测RSS、旧文件读取块大小和块复用。合成夹具的结果不能当作生产进程测量。 - 真实完成与恢复路径使用超限正文和工具,断言归档原样读回;不能仅绕过生产入库函数直接构造消息数组。 - 真实 runtime 事件覆盖纯工具、纯文本与结构块混合、Claude user/tool_result;恢复覆盖图片内容块、提交失败保留以及重复恢复幂等性。任务看板的克隆适配器需重新可信加载并只更新元数据。 - `scripts/session-history-unit.js` 与 `scripts/history-frontend-unit.js`:57 条、首屏 12 条、剩余 45 条在来源待核实状态下仍全部可读,顺序正确、无重复,用户目录完成索引。 @@ -79,7 +97,8 @@ writeFileAtomicSync(sessionPath(session.id), buildSessionJsonForPersist(session) session.messages.push({ content: entry.fullText }); const committed = store.save(session); try { - const preview = buildSessionJsonForPersist({ ...committed.session, historyArchiveRevision: committed.revision }); + const tail = store.loadTail(session.id, 180, { maxBytes: 2 * 1024 * 1024 }); + const preview = buildSessionJsonForPersist({ ...tail, historyArchiveRevision: committed.revision }); writeFileAtomicSync(sessionPath(session.id), preview.json); } catch (error) { // 完整档案已提交,缓存失败不改成历史保存失败。 diff --git a/.trellis/tasks/10-11-remove-compact-followup/check.jsonl b/.trellis/tasks/10-11-remove-compact-followup/check.jsonl new file mode 100644 index 0000000..2887c74 --- /dev/null +++ b/.trellis/tasks/10-11-remove-compact-followup/check.jsonl @@ -0,0 +1,2 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} +{"file": ".trellis/spec/backend/index.md", "reason": "核对最小修改与文案验收"} diff --git a/.trellis/tasks/10-11-remove-compact-followup/implement.jsonl b/.trellis/tasks/10-11-remove-compact-followup/implement.jsonl new file mode 100644 index 0000000..d68527e --- /dev/null +++ b/.trellis/tasks/10-11-remove-compact-followup/implement.jsonl @@ -0,0 +1,2 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} +{"file": ".trellis/spec/backend/index.md", "reason": "遵守后端规范,仅修改提示文案"} diff --git a/.trellis/tasks/10-11-remove-compact-followup/prd.md b/.trellis/tasks/10-11-remove-compact-followup/prd.md new file mode 100644 index 0000000..6aae532 --- /dev/null +++ b/.trellis/tasks/10-11-remove-compact-followup/prd.md @@ -0,0 +1,18 @@ +# 删除多余会话提示 + +用户明确要求去掉“下次继续在同一会话发送即可。”这句话。 + +## 修改范围 + +仅修改 `server.js` 中 `compactDoneMessage` 的 Codex App、Codex、Claude Code 三个返回值,去掉该句及前面的逗号,并以句号结束原生 `/compact` 完成说明。 + +## 完成条件 + +- 三种引擎均不再输出该句。 +- 保留各引擎原有的压缩完成说明。 +- `node --check server.js` 通过,代码差异仅包含三处文案修改。 +- 不增加测试文件,不重启当前服务。 + +## 规范判断 + +此次仅调整提示文案,没有新增架构或实现约定,无需更新规范。 diff --git a/.trellis/tasks/10-11-remove-compact-followup/task.json b/.trellis/tasks/10-11-remove-compact-followup/task.json new file mode 100644 index 0000000..f0a9a4d --- /dev/null +++ b/.trellis/tasks/10-11-remove-compact-followup/task.json @@ -0,0 +1,26 @@ +{ + "id": "remove-compact-followup", + "name": "remove-compact-followup", + "title": "删除压缩完成提示中的多余句子", + "description": "", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "shiyue", + "assignee": "shiyue", + "createdAt": "2026-10-11", + "completedAt": "2026-10-11", + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "已删除三种引擎压缩完成提示中的多余句子;语法检查、差异检查和独立审查通过。", + "meta": {} +} diff --git a/.trellis/tasks/10-14-history-memory-audit/check.jsonl b/.trellis/tasks/10-14-history-memory-audit/check.jsonl new file mode 100644 index 0000000..cf677af --- /dev/null +++ b/.trellis/tasks/10-14-history-memory-audit/check.jsonl @@ -0,0 +1,2 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} +{"file": ".trellis/spec/backend/session-history.md", "reason": "完整性与隔离回归"} diff --git a/.trellis/tasks/10-14-history-memory-audit/implement.jsonl b/.trellis/tasks/10-14-history-memory-audit/implement.jsonl new file mode 100644 index 0000000..feff3ce --- /dev/null +++ b/.trellis/tasks/10-14-history-memory-audit/implement.jsonl @@ -0,0 +1,2 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} +{"file": ".trellis/spec/backend/session-history.md", "reason": "完整历史、可信修订与并发契约"} diff --git a/.trellis/tasks/10-14-history-memory-audit/prd.md b/.trellis/tasks/10-14-history-memory-audit/prd.md new file mode 100644 index 0000000..d796551 --- /dev/null +++ b/.trellis/tasks/10-14-history-memory-audit/prd.md @@ -0,0 +1,7 @@ +# 完整历史内存修复 + +保留已保存完整历史,修复整份 JSON 解析、复制、序列化基线导致的内存峰值。 +采用分块存储和按页读取,迁移旧归档时避免全量读取;保留原子提交与可信并发合并。 +验证生产入口、旧归档迁移、清空删除、运行态恢复和大归档内存。测试目录必须隔离。 + +具体计划与发现见 .planning/2026-10-14-history-memory-fix/。 diff --git a/.trellis/tasks/10-14-history-memory-audit/task.json b/.trellis/tasks/10-14-history-memory-audit/task.json new file mode 100644 index 0000000..ece9bee --- /dev/null +++ b/.trellis/tasks/10-14-history-memory-audit/task.json @@ -0,0 +1,36 @@ +{ + "id": "history-memory-audit", + "name": "history-memory-audit", + "title": "修复完整历史归档内存回归", + "description": "保留完整历史,使用不可变消息块、原子清单提交、分页读取和流式旧格式迁移,消除默认加载与保存的整份正文复制。", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "shiyue", + "assignee": "shiyue", + "createdAt": "2026-10-14", + "completedAt": "2026-10-14", + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [ + "lib/session-history-store.js", + "lib/json-persistence.js", + "server.js", + "scripts/session-history-store-unit.js", + "scripts/session-history-memory-unit.js", + "scripts/history-runtime-completion-unit.js", + "scripts/codexapp-retry-history-unit.js", + "scripts/regression.js", + ".trellis/spec/backend/session-history.md" + ], + "notes": "修复完成:34项完整历史持久化回归及完整隔离集成回归通过。70.5MiB合成旧归档在256MiB堆上限下流式迁移,RSS峰值145.9MiB;1128条消息经67页逐条摘要校验完整,6次尾部保存复用37个原始块,旧文件单次读取最多64KiB。语法与差异检查通过。未修改生产历史,未重启服务,运行进程尚未生效。", + "meta": {} +} diff --git a/.trellis/tasks/10-14-runtime-json-memory-fix/task.json b/.trellis/tasks/10-14-runtime-json-memory-fix/task.json new file mode 100644 index 0000000..fd9cdb3 --- /dev/null +++ b/.trellis/tasks/10-14-runtime-json-memory-fix/task.json @@ -0,0 +1,36 @@ +{ + "id": "runtime-json-memory-fix", + "name": "runtime-json-memory-fix", + "title": "补齐完整运行态流式JSON持久化与单次任务状态提交", + "description": "参考另一实例的修复,将完整运行态改为64KiB流式读写,取消序列化克隆与等值比较,归档提交内部保护更新的任务状态;保留已完成的历史分块存储和分页读取。", + "status": "completed", + "dev_type": "backend", + "scope": "persistence", + "package": null, + "priority": "P2", + "creator": "shiyue", + "assignee": "shiyue", + "createdAt": "2026-10-14", + "completedAt": "2026-10-14", + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [ + "lib/json-persistence.js", + "lib/session-history-store.js", + "server.js", + "scripts/json-persistence-unit.js", + "scripts/runtime-state-memory-unit.js", + "scripts/session-history-store-unit.js", + "scripts/regression.js", + ".trellis/spec/backend/session-history.md", + ".trellis/spec/backend/quality-guidelines.md" + ], + "notes": "完整隔离集成回归、JSON10项、持久化38项通过。80MiB旧v1运行态在256MiB堆参数下RSS峰值210.5MiB,单次读写64KiB,正文摘要完整、write/fsync/rename失败保留旧档。未修改生产历史,未重启服务。", + "meta": {} +} diff --git a/dist-exe/cc-web-bun-linux-x64-baseline.tar.gz b/dist-exe/cc-web-bun-linux-x64-baseline.tar.gz index 974c1a5..104e4eb 100644 Binary files a/dist-exe/cc-web-bun-linux-x64-baseline.tar.gz and b/dist-exe/cc-web-bun-linux-x64-baseline.tar.gz differ diff --git a/lib/json-persistence.js b/lib/json-persistence.js new file mode 100644 index 0000000..8c4ee46 --- /dev/null +++ b/lib/json-persistence.js @@ -0,0 +1,305 @@ +'use strict'; + +const { StringDecoder } = require('string_decoder'); +const { types } = require('util'); + +const BUFFER_BYTES = 64 * 1024; + +// 读取时直接构造容器和正文,不先拼接整份 JSON 原文。 +function createJsonReader(fs, filename) { + const fd = fs.openSync(filename, 'r'); + const buffer = Buffer.allocUnsafe(BUFFER_BYTES); + let index = 0; + let length = 0; + let eof = false; + let closed = false; + + function available() { + if (index < length) return true; + if (eof) return false; + length = fs.readSync(fd, buffer, 0, buffer.length, null); + index = 0; + eof = length === 0; + return !eof; + } + + function whitespace() { + while (available()) { + const ch = buffer[index]; + if (ch !== 32 && ch !== 9 && ch !== 10 && ch !== 13) break; + index += 1; + } + } + + function peek() { + whitespace(); + return available() ? buffer[index] : -1; + } + + function invalid() { + throw new SyntaxError('持久化 JSON 结构无效或意外结束'); + } + + function consume(expected) { + if (peek() !== expected.charCodeAt(0)) invalid(); + index += 1; + } + + function string() { + consume('"'); + const parts = []; + let pending = []; + let pendingChars = 0; + let decoder = new StringDecoder('utf8'); + function append(text) { + if (!text) return; + pending.push(text); + pendingChars += text.length; + if (pendingChars >= 8192) { + parts.push(pending.join('')); + pending = []; + pendingChars = 0; + } + } + while (available()) { + const start = index; + while (index < length && buffer[index] >= 32 && buffer[index] !== 34 && buffer[index] !== 92) index += 1; + if (index > start) append(decoder.write(buffer.subarray(start, index))); + if (index === length) continue; + const ch = buffer[index++]; + if (ch < 32) invalid(); + // 转义字符与引号都是 ASCII,先结束前一段 UTF-8 解码。 + append(decoder.end()); + if (ch === 34) { parts.push(pending.join('')); return parts.join(''); } + decoder = new StringDecoder('utf8'); + if (!available()) invalid(); + const escaped = buffer[index++]; + const simple = { 34: '"', 92: '\\', 47: '/', 98: '\b', 102: '\f', 110: '\n', 114: '\r', 116: '\t' }; + if (Object.prototype.hasOwnProperty.call(simple, escaped)) append(simple[escaped]); + else if (escaped === 117) { + let hex = ''; + for (let count = 0; count < 4; count += 1) { + if (!available()) invalid(); + hex += String.fromCharCode(buffer[index++]); + } + if (!/^[0-9a-fA-F]{4}$/.test(hex)) invalid(); + append(String.fromCharCode(Number.parseInt(hex, 16))); + } else invalid(); + } + invalid(); + } + + function object(field) { + consume('{'); + if (peek() === 125) { index += 1; return; } + while (true) { + if (peek() !== 34) invalid(); + const key = string(); + consume(':'); + field(key); + if (peek() === 125) { index += 1; return; } + consume(','); + } + } + + function array(item) { + consume('['); + if (peek() === 93) { index += 1; return; } + while (true) { + item(value()); + if (peek() === 93) { index += 1; return; } + consume(','); + } + } + + function value() { + const first = peek(); + if (first === 34) return string(); + if (first === 123) { + const result = {}; + object(key => Object.defineProperty(result, key, { + value: value(), enumerable: true, writable: true, configurable: true, + })); + return result; + } + if (first === 91) { + const result = []; + array(item => result.push(item)); + return result; + } + const parts = []; + while (available()) { + const start = index; + while (index < length && ![44, 125, 93, 32, 9, 10, 13].includes(buffer[index])) index += 1; + if (index > start) parts.push(buffer.toString('utf8', start, index)); + if (index < length) break; + } + const token = parts.join(''); + if (token === 'null') return null; + if (token === 'true') return true; + if (token === 'false') return false; + if (!/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(token)) invalid(); + return Number(token); + } + + return { + value, object, array, + end() { if (peek() !== -1) invalid(); }, + close() { if (!closed) { closed = true; fs.closeSync(fd); } }, + }; +} + +function readJsonSync(fs, filename) { + const reader = createJsonReader(fs, filename); + try { + const result = reader.value(); + reader.end(); + return result; + } finally { + reader.close(); + } +} + +// 与历史持久化既有 replacer 保持一致;toJSON 在项目类型转换之前执行。 +function jsonValue(value, key) { + if (value !== null && (typeof value === 'object' || typeof value === 'function' || typeof value === 'bigint')) { + const toJSON = value.toJSON; + if (typeof toJSON === 'function') value = toJSON.call(value, key); + } + if (typeof value === 'bigint') return String(value); + if (value instanceof Map) value = Object.fromEntries(value); + else if (value instanceof Set) value = [...value]; + if (value && typeof value === 'object') { + if (types.isNumberObject(value)) value = Number(value); + else if (types.isStringObject(value)) value = String(value); + else if (types.isBooleanObject(value)) value = Boolean.prototype.valueOf.call(value); + else if (types.isBigIntObject(value)) throw new TypeError('JSON 不支持包装的 bigint'); + } + if (typeof value === 'number' && !Number.isFinite(value)) return null; + if (value === 0) return 0; + if (value === undefined || typeof value === 'function' || typeof value === 'symbol') return undefined; + return value; +} + +function cloneJsonValue(value) { + const ancestors = new Set(); + function clone(item, key) { + item = jsonValue(item, key); + if (item === null || typeof item !== 'object') return item; + if (ancestors.has(item)) throw new TypeError('JSON 不支持循环引用'); + ancestors.add(item); + try { + if (Array.isArray(item)) { + const result = []; + const length = item.length; + for (let index = 0; index < length; index += 1) result.push(clone(item[index], String(index)) ?? null); + return result; + } + const result = {}; + for (const name of Object.keys(item)) { + const next = clone(item[name], name); + if (next !== undefined) Object.defineProperty(result, name, { + value: next, enumerable: true, writable: true, configurable: true, + }); + } + return result; + } finally { + ancestors.delete(item); + } + } + return clone(value, ''); +} + +function writeJsonSync(fs, fd, value) { + const ancestors = new Set(); + const output = Buffer.allocUnsafe(BUFFER_BYTES); + let used = 0; + function flush() { + let offset = 0; + while (offset < used) { + const written = fs.writeSync(fd, output, offset, used - offset, null); + if (!Number.isInteger(written) || written <= 0) throw new Error('持久化 JSON 写入未取得进展'); + offset += written; + } + used = 0; + } + function append(text) { + const bytes = Buffer.from(text, 'utf8'); + for (let offset = 0; offset < bytes.length;) { + const count = Math.min(output.length - used, bytes.length - offset); + bytes.copy(output, used, offset, offset + count); + offset += count; + used += count; + if (used === output.length) flush(); + } + } + function string(text) { + append('"'); + for (let start = 0; start < text.length;) { + let end = Math.min(text.length, start + 8192); + // 不能在分段转义处拆开 UTF-16 代理对,否则会改变实际 JSON 字节。 + if (end < text.length && text.charCodeAt(end - 1) >= 0xd800 && text.charCodeAt(end - 1) <= 0xdbff + && text.charCodeAt(end) >= 0xdc00 && text.charCodeAt(end) <= 0xdfff) end -= 1; + append(JSON.stringify(text.slice(start, end)).slice(1, -1)); + start = end; + } + append('"'); + } + function write(item, key, arrayItem = false, prepared = false) { + if (!prepared) item = jsonValue(item, key); + if (item === undefined) { + if (arrayItem) { append('null'); return true; } + return false; + } + if (item === null) append('null'); + else if (typeof item === 'string') string(item); + else if (typeof item !== 'object') append(String(item)); + else { + if (ancestors.has(item)) throw new TypeError('JSON 不支持循环引用'); + ancestors.add(item); + try { + if (Array.isArray(item)) { + append('['); + const length = item.length; + for (let index = 0; index < length; index += 1) { + if (index) append(','); + write(item[index], String(index), true); + } + append(']'); + } else { + append('{'); + let first = true; + for (const name of Object.keys(item)) { + // 每个字段只求值一次,避免 getter 或 toJSON 重复执行。 + const next = jsonValue(item[name], name); + if (next === undefined) continue; + if (!first) append(','); + first = false; + string(name); + append(':'); + write(next, name, false, true); + } + append('}'); + } + } finally { + ancestors.delete(item); + } + } + return true; + } + if (!write(value, '')) throw new TypeError('顶层值不可持久化为 JSON'); + flush(); +} + +// 合并输入已是规范 JSON 值;直接比较结构,避免为长正文制造序列化副本。 +function jsonValueEqual(left, right) { + if (left === right) return true; + if (left === null || right === null || typeof left !== 'object' || typeof right !== 'object') return false; + if (Array.isArray(left) !== Array.isArray(right)) return false; + if (Array.isArray(left)) return left.length === right.length && left.every((item, index) => jsonValueEqual(item, right[index])); + const keys = Object.keys(left); + return keys.length === Object.keys(right).length && keys.every(key => Object.prototype.hasOwnProperty.call(right, key) + && jsonValueEqual(left[key], right[key])); +} + +module.exports = { createJsonReader, readJsonSync, writeJsonSync, cloneJsonValue, jsonValueEqual }; diff --git a/lib/session-history-store.js b/lib/session-history-store.js index f92737b..daa9d13 100644 --- a/lib/session-history-store.js +++ b/lib/session-history-store.js @@ -3,6 +3,10 @@ const nativeFs = require('fs'); const path = require('path'); const crypto = require('crypto'); +const { createJsonReader, readJsonSync, writeJsonSync, cloneJsonValue, jsonValueEqual } = require('./json-persistence'); + +// 跨存储实例保护仍在使用的旧修订;弱引用不延长会话或消息的寿命。 +const liveBaselines = new Set(); const SESSION_RUNTIME_KEYS = new Set([ 'ws', 'tailer', 'codexAppStateTimer', 'codexAppStateDirty', 'codexAppStateCleaned', @@ -16,10 +20,13 @@ function createSessionHistoryStore(options = {}) { // 身份只留在当前存储实例,不能通过 JSON 快照伪造可信完整加载或修订。 const loadedStates = new WeakMap(); const removedIds = new Set(); + const chunkMaxBytes = options.chunkMaxBytes || 2 * 1024 * 1024; + const chunkMaxMessages = options.chunkMaxMessages || 64; + let commits = 0; function validId(value) { const id = String(value || ''); - if (!/^[a-zA-Z0-9_-]+$/.test(id)) throw Object.assign(new Error('会话归档 ID 无效'), { code: 'history_invalid_id' }); + if (!/^[a-zA-Z0-9_-]+$/.test(id) || id === '_turns') throw Object.assign(new Error('会话归档 ID 无效'), { code: 'history_invalid_id' }); return id; } @@ -27,18 +34,116 @@ function createSessionHistoryStore(options = {}) { return path.join(historyDir, `${validId(id)}.json`); } - function readRecord(id) { - let json; + function archiveDir(id) { + return path.join(historyDir, validId(id)); + } + + function manifestPath(id) { + return path.join(archiveDir(id), 'manifest.json'); + } + + function chunkPath(id, file) { + if (!/^[a-f0-9]{64}\.json$/.test(file)) throw new Error('完整会话归档块引用无效'); + return path.join(archiveDir(id), 'chunks', file); + } + + function stringify(value) { + return JSON.stringify(value, (key, item) => { + if (typeof item === 'bigint') return String(item); + if (item instanceof Map) return Object.fromEntries(item); + if (item instanceof Set) return [...item]; + return item; + }); + } + + function sessionMetadata(session) { + const metadata = Object.create(null); + for (const key of Object.keys(session)) { + if (key === 'messages' || key === 'historySnapshotBaseIndex' || key === 'historySnapshotCount' + || key === 'historyArchiveRevision' || SESSION_RUNTIME_KEYS.has(key)) continue; + Object.defineProperty(metadata, key, { value: session[key], enumerable: true }); + } + return cloneJsonValue(metadata); + } + + function chunkWriter(id) { + const entries = []; + let pending = []; + let bytes = 2; + function flush() { + if (!pending.length) return; + const json = `[${pending.map(item => item.json).join(',')}]`; + const file = `${crypto.createHash('sha256').update(json).digest('hex')}.json`; + const target = chunkPath(id, file); + if (!fs.existsSync(target)) writeAtomicJson(target, null, json); + pending.forEach((item, index) => { + Object.assign(item.entry, { id: item.id || null, role: item.role, digest: item.digest, file, index, bytes: item.bytes }); + entries.push(item.entry); + }); + pending = []; + bytes = 2; + } + function add(message) { + const json = stringify(message); + if (json === undefined) throw new Error('完整会话消息不可序列化'); + const size = Buffer.byteLength(json); + if (pending.length && (bytes + size + 1 > chunkMaxBytes || pending.length >= chunkMaxMessages)) flush(); + const entry = {}; + pending.push({ json, id: message?.id, role: message?.role || null, + digest: crypto.createHash('sha256').update(json).digest('hex'), bytes: size, entry }); + bytes += size + 1; + // 单条超大消息仍完整保存,不能按预览限额截断。 + if (bytes >= chunkMaxBytes || pending.length >= chunkMaxMessages) flush(); + return entry; + } + return { add, finish() { flush(); return entries; } }; + } + + function migrateLegacy(id) { + if (!fs.existsSync(filePath(id))) return null; + const reader = createJsonReader(fs, filePath(id)); + const record = Object.create(null); + const metadata = Object.create(null); + const writer = chunkWriter(id); + let hasMessages = false; try { - json = fs.readFileSync(filePath(id), 'utf8'); + reader.object(key => { + if (key !== 'session') { record[key] = reader.value(); return; } + reader.object(field => { + if (field === 'messages') { + hasMessages = true; + reader.array(message => { if (message?.ccwebPersistenceNotice !== true) writer.add(message); }); + } else Object.defineProperty(metadata, field, { value: reader.value(), enumerable: true, configurable: true }); + }); + }); + reader.end(); + } finally { + reader.close(); + } + if (record.version !== 1 || typeof record.revision !== 'string' || !record.revision + || metadata.id !== id || !hasMessages || typeof record.complete !== 'boolean') { + throw Object.assign(new Error('完整会话归档格式无效'), { code: 'history_invalid_archive' }); + } + const entries = writer.finish(); + const manifest = { version: 2, revision: record.revision, complete: record.complete, + resetRevision: record.resetRevision || null, session: sessionMetadata(metadata), entries }; + // 旧单文件保留为迁移来源;所有块成功落盘后,清单才成为唯一提交点。 + writeAtomicJson(manifestPath(id), manifest); + return manifest; + } + + function readRecord(id) { + let record; + try { + record = JSON.parse(fs.readFileSync(manifestPath(id), 'utf8')); } catch (error) { - if (error?.code === 'ENOENT') return null; + if (error?.code === 'ENOENT') return migrateLegacy(id); throw error; } - const record = JSON.parse(json); - if (record?.version !== 1 || typeof record.revision !== 'string' || !record.revision - || record.session?.id !== id || !Array.isArray(record.session.messages) - || typeof record.complete !== 'boolean') { + if (record?.version !== 2 || typeof record.revision !== 'string' || !record.revision + || record.session?.id !== id || !Array.isArray(record.entries) || typeof record.complete !== 'boolean' + || record.entries.some(entry => !/^[a-f0-9]{64}\.json$/.test(entry.file) + || !/^[a-f0-9]{64}$/.test(entry.digest) || !Number.isSafeInteger(entry.index) || entry.index < 0)) { throw Object.assign(new Error('完整会话归档格式无效'), { code: 'history_invalid_archive' }); } return record; @@ -47,18 +152,125 @@ function createSessionHistoryStore(options = {}) { function state(session) { const value = session && loadedStates.get(session); if (!value) return null; - const { baseline, ...metadata } = value; - return metadata; + return { id: value.id, revision: value.revision, complete: value.complete, source: value.source, + resetRevision: value.resetRevision || null, start: value.start || 0, + end: value.end, total: value.manifest?.entries.length, partial: value.partial === true }; + } + + function register(session, manifest, start, end, partial) { + const baseline = { id: session.id, revision: manifest.revision, complete: manifest.complete, + source: 'archive', resetRevision: manifest.resetRevision || null, manifest, start, end, partial, directory: archiveDir(session.id) }; + loadedStates.set(session, baseline); + liveBaselines.add(new WeakRef(baseline)); + } + + function entryReader(id) { + const cache = new Map(); + return entry => { + let messages = cache.get(entry.file); + if (!messages) { + const json = fs.readFileSync(chunkPath(id, entry.file), 'utf8'); + if (crypto.createHash('sha256').update(json).digest('hex') !== entry.file.slice(0, -5)) { + throw new Error('完整会话归档块校验失败'); + } + messages = JSON.parse(json); + if (!Array.isArray(messages)) throw new Error('完整会话归档块格式无效'); + cache.set(entry.file, messages); + if (cache.size > 2) cache.delete(cache.keys().next().value); + } + const message = messages[entry.index]; + if (message === undefined) throw new Error('完整会话归档消息引用无效'); + return message; + }; + } + + function loadRange(id, record, start, end, partial) { + const session = cloneJsonValue(record.session); + const read = entryReader(id); + session.messages = record.entries.slice(start, end).map(read); + session.historySnapshotBaseIndex = start; + session.historySnapshotCount = session.messages.length; + register(session, record, start, end, partial); + return session; } function load(id) { - const record = readRecord(validId(id)); + id = validId(id); + const record = readRecord(id); + return record ? loadRange(id, record, 0, record.entries.length, false) : null; + } + + function loadTail(id, count = 180, limits = {}) { + id = validId(id); + const record = readRecord(id); if (!record) return null; - loadedStates.set(record.session, { - id, revision: record.revision, complete: record.complete, source: 'archive', - resetRevision: record.resetRevision || null, baseline: JSON.stringify(record.session), - }); - return record.session; + const end = record.entries.length; + let start = Math.max(0, end - Math.max(0, Math.trunc(count))); + if (Number.isFinite(limits.maxBytes) && limits.maxBytes > 0) { + let bytes = 0; + for (let index = end - 1; index >= start; index -= 1) { + const size = record.entries[index].bytes || 0; + if (bytes && bytes + size > limits.maxBytes) { start = index + 1; break; } + bytes += size; + } + } + return loadRange(id, record, start, end, start > 0); + } + + function loadMetadata(id) { + return loadTail(id, 0); + } + + function loadWindow(id, start, end = start + 1) { + id = validId(id); + const record = readRecord(id); + if (!record) return null; + const total = record.entries.length; + start = Math.max(0, Math.min(total, Math.trunc(start))); + end = Math.max(start, Math.min(total, Math.trunc(end))); + return loadRange(id, record, start, end, start > 0 || end < total); + } + + function forEachMessage(id, callback) { + id = validId(id); + const record = readRecord(id); + if (!record) return; + const read = entryReader(id); + for (let index = 0; index < record.entries.length; index += 1) callback(read(record.entries[index]), index); + } + + function findMessage(id, predicate, before = Infinity) { + id = validId(id); + const record = readRecord(id); + if (!record) return null; + const read = entryReader(id); + for (let index = Math.min(record.entries.length, before) - 1; index >= 0; index -= 1) { + const message = read(record.entries[index]); + if (predicate(message, index)) return { message, index }; + } + return null; + } + + function loadHistoryPage(id, before, count = 24) { + id = validId(id); + const record = readRecord(id); + if (!record) return null; + const total = record.entries.length; + const end = Number.isFinite(before) ? Math.max(0, Math.min(total, Math.trunc(before))) : total; + const start = Math.max(0, end - Math.max(0, Math.trunc(count))); + const read = entryReader(id); + return { messages: record.entries.slice(start, end).map(read), total, start, end, + complete: record.complete, revision: record.revision }; + } + + function messageIndex(id, messageId) { + const record = readRecord(validId(id)); + return record ? record.entries.findIndex(entry => entry.id === messageId) : -1; + } + + function countMessages(id, role) { + const record = readRecord(validId(id)); + return record ? record.entries.filter(entry => !role || entry.role === role).length : null; } function markLegacy(session) { @@ -76,28 +288,16 @@ function createSessionHistoryStore(options = {}) { if (!previous || previous.id !== session.id) { throw Object.assign(new Error('只能确认可信加载会话的历史状态'), { code: 'history_untrusted_source' }); } - loadedStates.set(session, { ...previous, complete: true, reset: options.reset === true }); + const confirmed = { ...previous, complete: true, reset: options.reset === true, partial: false }; + loadedStates.set(session, confirmed); + if (confirmed.manifest) liveBaselines.add(new WeakRef(confirmed)); session.historySnapshotBaseIndex = 0; session.historySnapshotCount = session.messages.length; return () => loadedStates.set(session, previous); } - function serializeSession(session) { - // 完整档案没有文本、消息数、数组长度或深度裁剪;不可序列化时整次提交失败。 - const json = JSON.stringify(session, function replacer(key, value) { - if (this === session && SESSION_RUNTIME_KEYS.has(key)) return undefined; - if (typeof value === 'bigint') return String(value); - if (value instanceof Map) return Object.fromEntries(value); - if (value instanceof Set) return [...value]; - return value; - }); - const persisted = JSON.parse(json); - if (!Array.isArray(persisted.messages)) throw new Error('完整会话消息必须为数组'); - return persisted; - } - function equal(left, right) { - return JSON.stringify(left) === JSON.stringify(right); + return jsonValueEqual(left, right); } function plainObject(value) { @@ -187,13 +387,14 @@ function createSessionHistoryStore(options = {}) { return persisted; } - function writeAtomicJson(target, record) { + function writeAtomicJson(target, record, json) { fs.mkdirSync(path.dirname(target), { recursive: true }); const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`; let fd; try { fd = fs.openSync(temporary, 'wx', 0o600); - fs.writeFileSync(fd, JSON.stringify(record), 'utf8'); + if (json === undefined) writeJsonSync(fs, fd, record); + else fs.writeFileSync(fd, json, 'utf8'); fs.fsyncSync(fd); fs.closeSync(fd); fd = undefined; @@ -206,10 +407,6 @@ function createSessionHistoryStore(options = {}) { } } - function writeRecord(id, record) { - writeAtomicJson(filePath(id), record); - } - function turnPath(id) { return path.join(turnsDir, `${validId(id)}.json`); } @@ -220,7 +417,7 @@ function createSessionHistoryStore(options = {}) { function loadTurn(id) { try { - const state = JSON.parse(fs.readFileSync(turnPath(id), 'utf8')); + const state = readJsonSync(fs, turnPath(id)); if (state?.sessionId !== id || state.agent !== 'codexapp') throw new Error('完整运行态归档无效'); return state; } catch (error) { @@ -244,11 +441,16 @@ function createSessionHistoryStore(options = {}) { } } - function save(session) { + function save(session, options = {}) { const id = validId(session?.id); const previous = loadedStates.get(session); if (removedIds.has(id)) throw Object.assign(new Error('已删除会话不能由旧对象重新保存'), { code: 'history_removed' }); - const current = readRecord(id); + if (previous?.source === 'archive' && !fs.existsSync(manifestPath(id))) { + throw Object.assign(new Error('完整会话归档已移除,拒绝旧对象重新创建'), { code: 'history_removed' }); + } + // 同修订只核对文件头,可信清单已经是不可变基线;旧修订才重新读取当前清单。 + const current = previous?.source === 'archive' && revision(id) === previous.revision + ? previous.manifest : readRecord(id); if (current) { if (previous?.source !== 'archive' || previous.id !== id) { throw Object.assign(new Error('受限预览不能覆盖完整会话归档'), { code: 'history_untrusted_source' }); @@ -259,47 +461,177 @@ function createSessionHistoryStore(options = {}) { const complete = current?.complete || (previous ? previous.complete : !session.messages?.some(message => message?.ccwebPersistenceNotice === true) && !(Number(session.historySnapshotBaseIndex) > 0)); - let persisted = serializeSession(session); + if (!Array.isArray(session.messages)) throw new Error('完整会话消息必须为数组'); + let metadata = sessionMetadata(session); + let taskTrackingPreserved = null; + const currentTracking = current?.session.taskTracking; + if (!options.taskBoardMetadata && currentTracking && typeof currentTracking === 'object') { + const persistedVersion = Number.isSafeInteger(currentTracking.version) ? currentTracking.version : 0; + const incomingVersion = Number.isSafeInteger(metadata.taskTracking?.version) ? metadata.taskTracking.version : -1; + if (incomingVersion < persistedVersion) { + metadata.taskTracking = cloneJsonValue(currentTracking); + taskTrackingPreserved = { incomingVersion, persistedVersion }; + } + } const rebased = !!current && previous.revision !== current.revision; - if (rebased) { - const baseline = JSON.parse(previous.baseline); - if (!identifiedArray(baseline.messages) || !identifiedArray(persisted.messages) || !identifiedArray(current.session.messages)) { + const baseEntries = previous?.manifest?.entries || []; + const viewedEntries = previous?.partial ? baseEntries.slice(previous.start, previous.end) : baseEntries; + const descriptors = []; + const positions = new Map(); + for (const message of session.messages) { + if (message?.ccwebPersistenceNotice === true) continue; + const json = stringify(message); + if (json === undefined) throw new Error('完整会话消息不可序列化'); + const descriptor = { id: message?.id || null, digest: crypto.createHash('sha256').update(json).digest('hex'), message }; + // 同 ID 的最新片段保留最初位置,不同气泡即使同 turn 也分别保存。 + if (descriptor.id && positions.has(descriptor.id)) descriptors[positions.get(descriptor.id)] = descriptor; + else { + if (descriptor.id) positions.set(descriptor.id, descriptors.length); + descriptors.push(descriptor); + } + } + let ordered = descriptors; + let start = previous?.partial ? previous.start : 0; + let end; + const read = entryReader(id); + const currentEntries = current?.entries || []; + const resetChanged = rebased && (current.resetRevision || null) !== (previous.resetRevision || null); + if (rebased && !resetChanged) { + if ([viewedEntries, descriptors, currentEntries].some(entries => !entries.every(entry => typeof entry.id === 'string' && entry.id))) { throw Object.assign(new Error('旧修订缺少可靠消息 ID,拒绝覆盖新历史'), { code: 'history_stale_unidentified' }); } - if (baseline.messages.length > 0 && persisted.messages.length === 0 && !equal(current.session.messages, baseline.messages)) { + if (!previous.partial && baseEntries.length > 0 && descriptors.length === 0 + && !equal(currentEntries.map(entry => entry.digest), baseEntries.map(entry => entry.digest))) { throw Object.assign(new Error('清空前历史已经更新,请重新加载'), { code: 'history_stale_clear' }); } - if ((current.resetRevision || null) !== (previous.resetRevision || null)) { - // 清空提交后旧运行对象只能同步元数据,旧线程的历史和输出不能复活。 - persisted.messages = baseline.messages; - for (const key of ['claudeSessionId', 'codexThreadId', 'codexAppThreadId', 'historySnapshotBaseIndex', 'historySnapshotCount']) { - if (Object.prototype.hasOwnProperty.call(baseline, key)) persisted[key] = baseline[key]; - else delete persisted[key]; + const baseById = new Map(viewedEntries.map(entry => [entry.id, entry])); + const incomingById = new Map(descriptors.map(entry => [entry.id, entry])); + const currentIds = new Set(currentEntries.map(entry => entry.id)); + ordered = []; + for (const entry of currentEntries) { + const base = baseById.get(entry.id); + const next = incomingById.get(entry.id); + if (!base) { ordered.push(entry); continue; } + if (!next) { + if (entry.digest !== base.digest) ordered.push(entry); + continue; } + if (next.digest === base.digest || next.digest === entry.digest) ordered.push(entry); + else if (entry.digest === base.digest) ordered.push(next); + else ordered.push({ id: next.id, incoming: next.message, base, current: entry }); } - persisted = mergeValue(baseline, persisted, current.session); + const orderedIds = new Set(ordered.map(entry => entry.id)); + for (let index = 0; index < descriptors.length; index += 1) { + const item = descriptors[index]; + if (baseById.has(item.id) || currentIds.has(item.id) || orderedIds.has(item.id)) continue; + let position = -1; + for (let next = index + 1; next < descriptors.length; next += 1) { + position = ordered.findIndex(entry => entry.id === descriptors[next].id); + if (position >= 0) break; + } + if (position < 0) { + for (let anchor = index - 1; anchor >= 0; anchor -= 1) { + const found = ordered.findIndex(entry => entry.id === descriptors[anchor].id); + if (found >= 0) { position = found + 1; break; } + } + } + ordered.splice(position < 0 ? ordered.length : position, 0, item); + orderedIds.add(item.id); + } + if (previous.partial) { + // 只协调原加载窗口及窗口内的并发新增,不能把未加载的后缀变成内存基线。 + const prefixIds = new Set(baseEntries.slice(0, previous.start).map(entry => entry.id)); + const suffixIds = new Set(baseEntries.slice(previous.end).map(entry => entry.id)); + const lastPrefix = ordered.reduce((found, entry, index) => prefixIds.has(entry.id) ? index : found, -1); + start = lastPrefix >= 0 ? lastPrefix + 1 : Math.min(previous.start, ordered.length); + const firstSuffix = ordered.findIndex(entry => suffixIds.has(entry.id)); + end = firstSuffix >= 0 ? firstSuffix : ordered.length; + if (previous.start === previous.end && descriptors.length === 0) start = end; + } + } else if (resetChanged) { + // 清空后的旧对象只同步元数据,旧线程及输出不能复活。 + ordered = currentEntries; + start = previous.partial ? currentEntries.length : 0; + end = currentEntries.length; + for (const key of ['claudeSessionId', 'codexThreadId', 'codexAppThreadId']) { + if (Object.prototype.hasOwnProperty.call(previous.manifest.session, key)) metadata[key] = previous.manifest.session[key]; + else delete metadata[key]; + } + } else if (previous?.partial) { + const prefix = currentEntries.slice(0, previous.start); + const suffix = currentEntries.slice(previous.end); + const unseenIds = new Set([...prefix, ...suffix].filter(entry => entry.id).map(entry => entry.id)); + // 重试的稳定 ID 若已存在于未加载区间,保留已提交原文和位置。 + ordered = [...prefix, ...descriptors.filter(entry => !entry.id || !unseenIds.has(entry.id)), ...suffix]; + end = ordered.length - suffix.length; } - if (complete) { - persisted.historySnapshotBaseIndex = 0; - persisted.historySnapshotCount = persisted.messages.length; + if (rebased) metadata = mergeValue(previous.manifest.session, metadata, current.session); + const writer = chunkWriter(id); + const currentById = new Map(currentEntries.filter(entry => entry.id).map(entry => [entry.id, entry])); + const entries = ordered.map((item, index) => { + if (item.file) return item; + const same = item.id ? currentById.get(item.id) : currentEntries[index]; + if (same && same.digest === item.digest) return same; + const message = item.current + ? mergeValue(read(item.base), cloneJsonValue(item.incoming), read(item.current)) + : item.message; + return writer.add(message); + }); + writer.finish(); + const nextRevision = crypto.randomUUID(); + const resetRevision = previous?.reset ? nextRevision : current?.resetRevision || null; + const record = { version: 2, revision: nextRevision, complete, resetRevision, session: metadata, entries }; + end ??= entries.length; + let updated; + let synchronizedMetadata; + if (rebased || end - start !== session.messages.length) { + const byId = new Map(session.messages.filter(message => message?.id).map(message => [message.id, message])); + // 提交前完成所有可能失败的块读取;提交失败不修改活跃消息和可信基线。 + updated = entries.slice(start, end).map(entry => ({ target: byId.get(entry.id), persisted: read(entry) })); + // 活跃元数据可以原位更新,不能与不可变清单基线共用新增的对象引用。 + synchronizedMetadata = cloneJsonValue(metadata); } - const revision = crypto.randomUUID(); - const resetRevision = previous?.reset ? revision : current?.resetRevision || null; - const record = { version: 1, revision, complete, resetRevision, session: persisted }; - writeRecord(id, record); - // 只有原子提交后才推进修订,失败重试仍以原修订为基线。 - loadedStates.set(session, { id, revision, complete, resetRevision, source: 'archive', baseline: JSON.stringify(persisted) }); - if (rebased) { - // 流式处理闭包可能持有气泡和工具对象,合并时按 ID 原位协调,保持后续 delta 写入目标。 - synchronizeValue(session, persisted, true); + const synchronizedTracking = taskTrackingPreserved && !updated ? cloneJsonValue(metadata.taskTracking) : null; + writeAtomicJson(manifestPath(id), record); + if (updated) { + session.messages.length = updated.length; + updated.forEach((item, index) => { session.messages[index] = synchronizeValue(item.target, item.persisted); }); + synchronizeValue(session, { ...synchronizedMetadata, messages: session.messages }, true); + } + if (synchronizedTracking) session.taskTracking = synchronizeValue(session.taskTracking, synchronizedTracking); + session.historySnapshotBaseIndex = start; + session.historySnapshotCount = session.messages.length; + register(session, record, start, end, start > 0 || end < entries.length); + if (++commits % 32 === 0) { + // 只回收当前清单和活跃旧修订都不再引用的块,清理失败不改变已提交结果。 + try { collectChunks(id, record); } catch {} + } + return { revision: nextRevision, complete, session, rebased, taskTrackingPreserved }; + } + + function collectChunks(id, current) { + const files = new Set(current.entries.map(entry => entry.file)); + for (const reference of liveBaselines) { + const baseline = reference.deref(); + if (!baseline) { liveBaselines.delete(reference); continue; } + if (baseline.id === id && baseline.directory === archiveDir(id)) baseline.manifest.entries.forEach(entry => files.add(entry.file)); + } + const directory = path.join(archiveDir(id), 'chunks'); + for (const file of fs.readdirSync(directory)) { + if (/^[a-f0-9]{64}\.json$/.test(file) && !files.has(file)) fs.unlinkSync(path.join(directory, file)); } - return { revision, complete, session: persisted, rebased }; } function listIds() { try { - return fs.readdirSync(historyDir).filter(name => /^[a-zA-Z0-9_-]+\.json$/.test(name)) - .map(name => name.slice(0, -5)); + const ids = new Set(); + for (const name of fs.readdirSync(historyDir)) { + // 运行态归档属于保留目录,不参与会话归档枚举。 + if (name === '_turns' || name === '_turns.json') continue; + if (/^[a-zA-Z0-9_-]+\.json$/.test(name)) ids.add(name.slice(0, -5)); + else if (/^[a-zA-Z0-9_-]+$/.test(name) && fs.existsSync(manifestPath(name))) ids.add(name); + } + return [...ids]; } catch (error) { if (error?.code === 'ENOENT') return []; throw error; @@ -309,10 +641,11 @@ function createSessionHistoryStore(options = {}) { function revision(id) { let fd; try { - fd = fs.openSync(filePath(id), 'r'); + const recordPath = fs.existsSync(manifestPath(id)) ? manifestPath(id) : filePath(id); + fd = fs.openSync(recordPath, 'r'); const head = Buffer.alloc(256); const count = fs.readSync(fd, head, 0, head.length, 0); - const matched = /^\{"version":1,"revision":"([0-9a-f-]{36})",/.exec(head.toString('utf8', 0, count)); + const matched = /^\{"version":[12],"revision":"([0-9a-f-]{36})",/.exec(head.toString('utf8', 0, count)); if (!matched) throw new Error('完整会话归档修订无效'); return matched[1]; } catch (error) { @@ -328,11 +661,13 @@ function createSessionHistoryStore(options = {}) { try { fs.unlinkSync(filePath(id)); } catch (error) { if (error?.code !== 'ENOENT') throw error; } + fs.rmSync(archiveDir(id), { recursive: true, force: true }); removedIds.add(id); removeTurn(id); } - return { load, save, state, markLegacy, markComplete, listIds, revision, remove, saveTurn, loadTurn, removeTurn, listTurnIds, historyDir }; + return { load, loadTail, loadMetadata, loadWindow, forEachMessage, findMessage, loadHistoryPage, messageIndex, countMessages, save, state, markLegacy, markComplete, + listIds, revision, remove, saveTurn, loadTurn, removeTurn, listTurnIds, historyDir }; } module.exports = { createSessionHistoryStore }; diff --git a/scripts/codexapp-retry-history-unit.js b/scripts/codexapp-retry-history-unit.js index 39672c1..8df2943 100644 --- a/scripts/codexapp-retry-history-unit.js +++ b/scripts/codexapp-retry-history-unit.js @@ -73,7 +73,7 @@ function createApi(dir, options = {}) { 'textByteLength', 'truncateTextValue', 'sanitizePersistValue', 'sanitizeToolCallsForPersist', 'mapToPairs', 'codexAppStatePath', 'codexAppTurnKey', 'serializeCodexAppEntry', 'buildCodexAppStateJson', 'writeCodexAppTurnState', 'persistCodexAppTurnState', - 'cleanupCodexAppTurnState', 'loadCodexAppTurnState', 'hasCodexAppTurnMessage', + 'cleanupCodexAppTurnState', 'loadCodexAppTurnState', 'findSessionMessage', 'hasCodexAppTurnMessage', 'appendCodexAppRecoveryNotice', 'recoverCodexAppTurnState', 'hasRuntimeOutput', 'retryTailText', 'retryPreviewValue', 'buildCodexRetryToolSummary', 'shouldUseCodexAppContinuationRetry', 'buildCodexAppContinuationRetryText', diff --git a/scripts/history-runtime-completion-unit.js b/scripts/history-runtime-completion-unit.js index ca74b03..9e24404 100644 --- a/scripts/history-runtime-completion-unit.js +++ b/scripts/history-runtime-completion-unit.js @@ -135,6 +135,7 @@ function createCliCompletionApi(dir, options = {}) { 'hasRuntimeOutput', 'codexOutputContent', 'cliRunKey', + 'findSessionMessage', 'hasCliRunMessage', 'isContextLimitError', 'handleProcessComplete', @@ -248,6 +249,7 @@ function createCodexAppStateApi(dir, options = {}) { 'writeCodexAppTurnState', 'loadCodexAppTurnState', 'cleanupCodexAppTurnState', + 'findSessionMessage', 'hasCodexAppTurnMessage', 'appendCodexAppRecoveryNotice', 'recoverCodexAppTurnState', @@ -410,6 +412,7 @@ function createCrossConversationApi(dir, options = {}) { const sent = []; const pending = new Map(); const api = new Function('fs', 'path', 'crypto', 'dir', 'storeApi', 'pending', 'sent', 'options', 'CCWEB_REPLY_MODES', ` + const sessionHistoryStore = storeApi.sessionHistoryStore; const wsSend = (ws, payload) => sent.push(payload); const loadSession = storeApi.loadSession; const saveSession = session => (options.failSave ? false : storeApi.saveSession(session)); @@ -427,6 +430,7 @@ function createCrossConversationApi(dir, options = {}) { 'setPendingCrossConversationReply', 'updatePendingCrossConversationReply', 'deletePendingCrossConversationReply', + 'findSessionMessage', 'hasProcessedCrossConversationReply', 'extractCrossConversationReplyText', 'buildCrossConversationReplyContent', @@ -536,7 +540,7 @@ function runTests() { assert.equal(seedApi.saveSession(sessionFixture(sessionId, 'codex', [ { id: 'user-before-cli', role: 'user', content: '读取异常前已经保存的问题' }, ])), true); - const archivePath = path.join(dir, '_history', `${sessionId}.json`); + const archivePath = path.join(dir, '_history', sessionId, 'manifest.json'); const archiveBefore = fs.readFileSync(archivePath, 'utf8'); const fullText = 'CLI 完成时必须保留的唯一输出'; const runPath = writeRunArtifacts(dir, sessionId, { @@ -579,7 +583,7 @@ function runTests() { assert.equal(seedApi.saveSession(sessionFixture(sessionId, 'codexapp', [ { id: 'user-before-app', role: 'user', content: '读取异常前已经保存的 App 问题' }, ])), true); - const archivePath = path.join(dir, '_history', `${sessionId}.json`); + const archivePath = path.join(dir, '_history', sessionId, 'manifest.json'); const archiveBefore = fs.readFileSync(archivePath, 'utf8'); // 同 CLI 夹具:损坏真实档案制造“会话暂不可读”,恢复时先修复档案再补交。 const corruptedArchive = '{损坏的会话归档:Codex App 读取失败夹具'; diff --git a/scripts/json-persistence-unit.js b/scripts/json-persistence-unit.js new file mode 100644 index 0000000..180cf3f --- /dev/null +++ b/scripts/json-persistence-unit.js @@ -0,0 +1,262 @@ +#!/usr/bin/env node +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { readJsonSync, writeJsonSync, cloneJsonValue, jsonValueEqual } = require('../lib/json-persistence'); +const { createSessionHistoryStore } = require('../lib/session-history-store'); + +const BLOCK_BYTES = 64 * 1024; + +function nativeJson(value) { + return JSON.stringify(value, (key, item) => { + if (typeof item === 'bigint') return String(item); + if (item instanceof Map) return Object.fromEntries(item); + if (item instanceof Set) return [...item]; + return item; + }); +} + +function temporaryRoot() { + if (process.platform === 'linux' && typeof fs.statfsSync === 'function') { + try { + const stat = fs.statfsSync('/dev/shm'); + if (Number(stat.bavail) * Number(stat.bsize) >= 32 * 1024 * 1024) return '/dev/shm'; + } catch {} + } + return os.tmpdir(); +} + +function runTests() { + const dir = fs.mkdtempSync(path.join(temporaryRoot(), 'ccweb-json-persistence-')); + const failures = []; + let checks = 0; + let sequence = 0; + function check(name, callback) { + try { + callback(); + checks += 1; + console.log(`通过:${name}`); + } catch (error) { + failures.push(name); + console.error(`失败:${name}:${error.stack}`); + } + } + function rawFile(json) { + const filename = path.join(dir, `input-${++sequence}.json`); + fs.writeFileSync(filename, json); + return filename; + } + function encode(value, filesystem = fs) { + const filename = path.join(dir, `output-${++sequence}.json`); + const fd = fs.openSync(filename, 'wx', 0o600); + try { writeJsonSync(filesystem, fd, value); } finally { fs.closeSync(fd); } + return { filename, json: fs.readFileSync(filename, 'utf8') }; + } + function assertNative(value) { + const expectedJson = nativeJson(value); + const expected = JSON.parse(expectedJson); + const output = encode(value); + assert.equal(output.json, expectedJson); + assert.deepEqual(readJsonSync(fs, output.filename), expected); + assert.deepEqual(cloneJsonValue(value), expected); + } + + try { + check('JSON基本类型、数值边界与控制字符与原生持久化语义一致', () => { + const samples = [null, true, false, 0, -0, 1.25, 1e-100, 1e100, NaN, Infinity, -Infinity, + '', '中文🙂\u0000\b\f\n\r\t\\"/\u2028\u2029', '\ud800\udfff', 9007199254740993n]; + samples.forEach(assertNative); + assertNative({ values: samples, boxed: [new Number(4), new String('包装文字'), new Boolean(false)] }); + }); + + check('BigInt、Map、Set、日期与toJSON保持项目转换及键参数', () => { + function fixture() { + return { + bigint: 123456789012345678901234567890n, + map: new Map([['a', 1n], ['__proto__', { own: true }], ['set', new Set([2, 3])]]), + set: new Set(['x', 4n]), + date: new Date('2026-10-14T01:02:03.456Z'), + custom: { toJSON(key) { return { key, value: 9n }; } }, + array: [{ toJSON(key) { return `数组键 ${key}`; } }], + }; + } + assertNative(fixture()); + assertNative({ toJSON(key) { return { rootKey: key, value: new Map([['ok', true]]) }; } }); + for (const action of ['write', 'clone']) { + let reads = 0; + let calls = 0; + const custom = { + get toJSON() { + reads += 1; + return function (key) { calls += 1; assert.equal(this, custom); return { key, value: '原值' }; }; + }, + }; + const result = action === 'write' ? JSON.parse(encode({ custom }).json) : cloneJsonValue({ custom }); + assert.deepEqual(result, { custom: { key: 'custom', value: '原值' } }); + assert.equal(reads, 1, 'toJSON属性只能读取一次'); + assert.equal(calls, 1, 'toJSON只能执行一次'); + } + }); + + check('对象省略与数组空洞置null、重复引用及危险属性名完整保存', () => { + const shared = { stable: '原值', nested: ['完整内容'] }; + const object = JSON.parse('{"__proto__":{"value":"自己的属性"},"constructor":"构造键","prototype":"普通键"}'); + object.missing = undefined; + object.callback = () => 1; + object.symbol = Symbol('省略'); + object.array = [undefined, () => 1, Symbol('数组位置'), , shared, shared]; + object.left = shared; + object.right = shared; + assertNative(object); + const cloned = cloneJsonValue(object); + assert.equal(Object.hasOwn(cloned, '__proto__'), true); + assert.deepEqual(cloned.__proto__, { value: '自己的属性' }); + assert.equal(Object.hasOwn(cloned, 'missing'), false); + assert.equal(cloned.array.length, 6); + assert.deepEqual(cloned.array.slice(0, 4), [null, null, null, null]); + }); + + check('容器复制隔离原对象并保留不可变字符串值', () => { + const text = '必须保留完整正文🙂'.repeat(20000); + const source = { text, nested: { rows: [{ value: 1 }] } }; + const cloned = cloneJsonValue(source); + assert.notEqual(cloned, source); + assert.notEqual(cloned.nested, source.nested); + assert.notEqual(cloned.nested.rows, source.nested.rows); + assert.notEqual(cloned.nested.rows[0], source.nested.rows[0]); + assert.equal(cloned.text, text); + cloned.nested.rows[0].value = 2; + assert.equal(source.nested.rows[0].value, 1); + assert.equal(jsonValueEqual(source, cloneJsonValue(source)), true); + assert.equal(jsonValueEqual(source, cloned), false); + assert.equal(jsonValueEqual(cloneJsonValue({ value: NaN, omitted: undefined }), { value: null }), true); + assert.equal(jsonValueEqual(cloneJsonValue([undefined, , Infinity]), [null, null, null]), true); + assert.equal(jsonValueEqual(cloneJsonValue(10n), '10'), true); + assert.equal(jsonValueEqual(cloneJsonValue(new Map([['value', 1]])), { value: 1 }), true); + assert.equal(jsonValueEqual({ left: 1, right: { value: 2 } }, { right: { value: 2 }, left: 1 }), true); + assert.equal(jsonValueEqual(undefined, undefined), true); + assert.equal(jsonValueEqual(undefined, null), false); + }); + + check('顶层不可持久化值拒绝提交且循环引用失败', () => { + for (const value of [undefined, () => 1, Symbol('顶层')]) { + assert.equal(cloneJsonValue(value), undefined); + assert.throws(() => encode(value), TypeError); + } + const cyclic = { text: '已经写出的字段' }; + cyclic.self = cyclic; + assert.throws(() => encode(cyclic), TypeError); + assert.throws(() => cloneJsonValue(cyclic), TypeError); + }); + + check('64KiB边界跨UTF-8、JSON转义和长字符串时完整读取', () => { + const prefix = '{"text":"'; + const utf8Json = prefix + 'a'.repeat(BLOCK_BYTES - Buffer.byteLength(prefix) - 1) + '🙂中文","tail":{"value":true}}'; + const escapedJson = prefix + 'b'.repeat(BLOCK_BYTES - Buffer.byteLength(prefix) - 1) + '\\uD83D\\uDE42\\n\\"\\\\","tail":[1,2,3]}'; + for (const json of [utf8Json, escapedJson, '\r\n ' + '[1e-3,true,null,{"duplicate":1,"duplicate":2}]' + '\t ']) { + const filename = rawFile(json); + let maxRead = 0; + const filesystem = new Proxy(fs, { get(target, key) { + if (key === 'readFileSync') return () => { throw new Error('流式读取禁止整文件readFileSync'); }; + if (key === 'readSync') return (fd, buffer, offset, length, position) => { + maxRead = Math.max(maxRead, length); + assert(length <= BLOCK_BYTES); + return target.readSync(fd, buffer, offset, length, position); + }; + return target[key]; + } }); + assert.deepEqual(readJsonSync(filesystem, filename), JSON.parse(json)); + assert(maxRead > 0 && maxRead <= BLOCK_BYTES); + } + }); + + check('多次短读不丢UTF-8字节和转义,读取错误关闭文件', () => { + const json = '{"text":"中文🙂\\n\\uD83D\\uDE42\\\\\\\"","number":-2.5e+3,"rows":[true,null,{"own":1}]}'; + const filename = rawFile(json); + let opened = 0; + let closed = 0; + let failRead = false; + const filesystem = new Proxy(fs, { get(target, key) { + if (key === 'openSync') return (...args) => { opened += 1; return target.openSync(...args); }; + if (key === 'closeSync') return (...args) => { closed += 1; return target.closeSync(...args); }; + if (key === 'readSync') return (fd, buffer, offset, length, position) => { + if (failRead) throw new Error('模拟读取中断'); + return target.readSync(fd, buffer, offset, Math.min(length, 3), position); + }; + return target[key]; + } }); + assert.deepEqual(readJsonSync(filesystem, filename), JSON.parse(json)); + failRead = true; + assert.throws(() => readJsonSync(filesystem, filename), /模拟读取中断/); + assert.equal(closed, opened, '读取成功或失败均应关闭文件'); + }); + + check('非法JSON、截断值、尾随内容与控制字符均拒绝', () => { + const invalid = ['', ' ', '{', '[1,', '{"a":}', '{"a":1,}', '[1,]', 'true false', 'nullx', '+1', '01', '1.', '1e', + 'NaN', 'Infinity', 'undefined', '\ufeff{}', '{unquoted:1}', '"未闭合', '"\\x01"', '"\\u12"', '"\n"', '"\u0000"']; + for (const json of invalid) { + assert.throws(() => JSON.parse(json)); + assert.throws(() => readJsonSync(fs, rawFile(json)), `非法JSON必须拒绝:${JSON.stringify(json)}`); + } + for (const json of ['1e400', '-0', '[1,2,3]', '{"__proto__":{"own":true},"constructor":4}']) { + assert.deepEqual(readJsonSync(fs, rawFile(json)), JSON.parse(json)); + } + }); + + check('分段编码每次最多64KiB且正确处理短写', () => { + const value = { text: '中文🙂\\"\n\u0000'.repeat(25000), nested: { lines: ['保持原值'] } }; + let maxWrite = 0; + let calls = 0; + const filesystem = new Proxy(fs, { get(target, key) { + if (key === 'writeFileSync') return () => { throw new Error('流式编码禁止整文件writeFileSync'); }; + if (key === 'writeSync') return (fd, data, ...args) => { + calls += 1; + const bytes = typeof data === 'string' ? Buffer.byteLength(data, typeof args[1] === 'string' ? args[1] : 'utf8') : args[1] ?? data.length; + maxWrite = Math.max(maxWrite, bytes); + assert(bytes <= BLOCK_BYTES, `单次编码写入 ${bytes} 字节超出64KiB`); + // 固定短写发生在字节缓冲层,不能以UTF-16字符数量当作已写字节数。 + if (typeof data === 'string') { + const encoded = Buffer.from(data, typeof args[1] === 'string' ? args[1] : 'utf8'); + return target.writeSync(fd, encoded, 0, Math.min(11, encoded.length), args[0] ?? null); + } + return target.writeSync(fd, data, args[0] ?? 0, Math.min(11, bytes), args[2] ?? null); + }; + return target[key]; + } }); + const output = encode(value, filesystem); + assert.equal(output.json, nativeJson(value)); + assert(calls > 1); + assert(maxWrite <= BLOCK_BYTES); + }); + + check('JSON中途编码异常保留原子归档且无临时文件残留', () => { + const store = createSessionHistoryStore({ sessionsDir: dir }); + const id = 'json-atomic-failure'; + const original = { version: 1, agent: 'codexapp', sessionId: id, fullText: '上次完整运行态', toolCalls: [] }; + store.saveTurn(id, original); + const filename = path.join(dir, '_history', '_turns', `${id}.json`); + const oldBytes = fs.readFileSync(filename); + const cyclic = { ...original, fullText: '新正文'.repeat(100000) }; + cyclic.self = cyclic; + assert.throws(() => store.saveTurn(id, cyclic), TypeError); + const customFailure = { ...original, fullText: '已编码新正文'.repeat(100000), tail: { toJSON() { throw new Error('模拟toJSON异常'); } } }; + assert.throws(() => store.saveTurn(id, customFailure), /模拟toJSON异常/); + assert.deepEqual(fs.readFileSync(filename), oldBytes); + assert.deepEqual(store.loadTurn(id), original); + assert.equal(fs.readdirSync(path.dirname(filename)).some(name => name.endsWith('.tmp')), false); + }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + if (failures.length) { + console.error(`JSON持久化回归失败:${failures.length} 项,已通过 ${checks} 项`); + process.exitCode = 1; + } else console.log(`JSON持久化回归通过:${checks} 项`); + return { checks, failures }; +} + +if (require.main === module) runTests(); +module.exports = { runTests }; diff --git a/scripts/regression.js b/scripts/regression.js index 5cc1dfe..3aa0e80 100644 --- a/scripts/regression.js +++ b/scripts/regression.js @@ -4748,7 +4748,7 @@ function assertServerSessionHistoryRequestIdContract() { } function assertHistoryRecallContract() { - for (const script of ['session-history-store-unit.js', 'session-history-unit.js', 'history-runtime-unit.js', 'history-runtime-completion-unit.js', 'codex-rollouts-unit.js', 'history-frontend-unit.js', 'codexapp-retry-history-unit.js', 'streaming-retry-frontend-unit.js', 'assistant-bubble-controls-unit.js']) { + for (const script of ['json-persistence-unit.js', 'runtime-state-memory-unit.js', 'session-history-store-unit.js', 'session-history-memory-unit.js', 'session-history-unit.js', 'history-runtime-unit.js', 'history-runtime-completion-unit.js', 'codex-rollouts-unit.js', 'history-frontend-unit.js', 'codexapp-retry-history-unit.js', 'streaming-retry-frontend-unit.js', 'assistant-bubble-controls-unit.js']) { const result = spawnSync(process.execPath, [path.join(REPO_DIR, 'scripts', script)], { encoding: 'utf8', timeout: 60000 }); assert(result.status === 0, `${script}: ${result.stderr || result.stdout || result.error}`); } @@ -4828,7 +4828,8 @@ function assertHistoryRecallContract() { 'Paged history requests should use the resolved full history and echo requestId' ); assert( - /prefetchChunks: targetMessageIndex === null \? 0/.test(serverSource), + /const prefetchChunks = targetMessageIndex === null \? 0/.test(serverSource) + && /Math\.min\(HISTORY_MAX_CHUNKS_PER_LOAD,/.test(serverSource), 'Normal session loads should keep older history on demand instead of prefetching it all' ); diff --git a/scripts/runtime-state-memory-unit.js b/scripts/runtime-state-memory-unit.js new file mode 100644 index 0000000..88a3210 --- /dev/null +++ b/scripts/runtime-state-memory-unit.js @@ -0,0 +1,238 @@ +#!/usr/bin/env node +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const { createSessionHistoryStore } = require('../lib/session-history-store'); + +const ID = 'synthetic-runtime-memory'; +const TARGET_BYTES = 80 * 1024 * 1024; +const BLOCK_BYTES = 64 * 1024; +const LARGE_PIECE = 'a'.repeat(32 * 1024 - 16) + '\\引号"\n'; +const ASCII_PIECE = 'b'.repeat(32 * 1024 - 16) + '\\quote"\n'; +const SPECIAL = JSON.parse('{"text":"中文🙂\\n\\\\\\\"","__proto__":{"own":"保留值"},"constructor":"普通属性"}'); +const startedAt = Date.now(); + +function stage(message) { + process.stderr.write(`[运行态内存回归 +${((Date.now() - startedAt) / 1000).toFixed(2)} 秒] ${message}\n`); +} + +function temporaryRoot() { + // 此项测编码和解析的内存峰值;慢盘fsync会耗尽60秒预算,优先真实tmpfs。 + if (process.platform === 'linux' && typeof fs.statfsSync === 'function') { + try { + const stat = fs.statfsSync('/dev/shm'); + fs.accessSync('/dev/shm', fs.constants.W_OK); + if (Number(stat.bavail) * Number(stat.bsize) >= 512 * 1024 * 1024) return '/dev/shm'; + } catch {} + } + return os.tmpdir(); +} + +function textHash(value) { + const hash = crypto.createHash('sha256'); + for (let offset = 0; offset < value.length; offset += 32 * 1024) hash.update(value.slice(offset, offset + 32 * 1024)); + return hash.digest('hex'); +} + +// 资料由父进程直接写标准version:1 JSON,不通过待测编码器制造测试基线。 +function writeFixture(sessionsDir) { + const directory = path.join(sessionsDir, '_history', '_turns'); + fs.mkdirSync(directory, { recursive: true }); + const filename = path.join(directory, `${ID}.json`); + const header = JSON.stringify({ version: 1, agent: 'codexapp', sessionId: ID, turnId: 'synthetic-turn', turnStatus: 'running', special: SPECIAL }).slice(0, -1) + ',"fullText":"'; + const betweenTool = '","toolCalls":[{"id":"synthetic-tool","name":"run","input":{"text":"完整输入"},"result":"'; + const betweenDelta = '"}],"toolOutputDeltas":[["synthetic-tool","'; + const betweenItem = '"]],"agentMessageItems":[["synthetic-item","'; + const ending = '"]],"agentMessagePendingPrefixes":[],"lastUsage":{"inputTokens":1,"outputTokens":2}}'; + const fixedBytes = Buffer.byteLength(header + betweenTool + betweenDelta + betweenItem + ending); + const toolBytes = 12 * 1024 * 1024; + const deltaBytes = 4 * 1024 * 1024; + const itemBytes = 4 * 1024 * 1024; + const fullTextBytes = TARGET_BYTES - fixedBytes - toolBytes - deltaBytes - itemBytes; + const fd = fs.openSync(filename, 'wx', 0o600); + function writeField(encodedBytes, piece) { + const encoded = JSON.stringify(piece).slice(1, -1); + const size = Buffer.byteLength(encoded); + const hash = crypto.createHash('sha256'); + let length = 0; + let remaining = encodedBytes; + while (remaining >= size) { + fs.writeSync(fd, encoded); + hash.update(piece); + length += piece.length; + remaining -= size; + } + if (remaining) { + const final = 'z'.repeat(remaining); + fs.writeSync(fd, final); + hash.update(final); + length += final.length; + } + return { length, hash: hash.digest('hex') }; + } + const expected = {}; + try { + fs.writeSync(fd, header); + expected.fullText = writeField(fullTextBytes, ASCII_PIECE); + fs.writeSync(fd, betweenTool); + expected.result = writeField(toolBytes, LARGE_PIECE); + fs.writeSync(fd, betweenDelta); + expected.delta = writeField(deltaBytes, LARGE_PIECE); + fs.writeSync(fd, betweenItem); + expected.item = writeField(itemBytes, LARGE_PIECE); + fs.writeSync(fd, ending); + } finally { fs.closeSync(fd); } + assert.equal(fs.statSync(filename).size, TARGET_BYTES); + return { filename, expected }; +} + +function fileHash(filename) { + const hash = crypto.createHash('sha256'); + const buffer = Buffer.allocUnsafe(BLOCK_BYTES); + const fd = fs.openSync(filename, 'r'); + try { + let bytes; + while ((bytes = fs.readSync(fd, buffer, 0, buffer.length, null))) hash.update(buffer.subarray(0, bytes)); + } finally { fs.closeSync(fd); } + return hash.digest('hex'); +} + +function trackedFilesystem(filename) { + const opened = new Map(); + const metrics = { reads: 0, writes: 0, maxRead: 0, maxWrite: 0, readBytes: 0, writeBytes: 0 }; + let failure = null; + let bytesSinceFailureMode = 0; + function runtimeFile(value) { + const resolved = typeof value === 'number' ? opened.get(value) || '' : path.resolve(String(value)); + return resolved === filename || resolved.startsWith(`${filename}.`); + } + const filesystem = new Proxy(fs, { get(target, key) { + if (key === 'openSync') return (file, ...args) => { + const fd = target.openSync(file, ...args); + opened.set(fd, path.resolve(String(file))); + return fd; + }; + if (key === 'closeSync') return fd => { const result = target.closeSync(fd); opened.delete(fd); return result; }; + if (key === 'readFileSync') return (file, ...args) => { + assert.equal(runtimeFile(file), false, '完整运行态禁止整份readFileSync'); + return target.readFileSync(file, ...args); + }; + if (key === 'writeFileSync') return (file, ...args) => { + assert.equal(runtimeFile(file), false, '完整运行态禁止整份writeFileSync'); + return target.writeFileSync(file, ...args); + }; + if (key === 'readSync') return (fd, buffer, offset, length, position) => { + if (runtimeFile(fd)) { + assert(length <= BLOCK_BYTES, '单次运行态读取不能超过64KiB'); + metrics.maxRead = Math.max(metrics.maxRead, length); + metrics.reads += 1; + } + const bytes = target.readSync(fd, buffer, offset, length, position); + if (runtimeFile(fd)) metrics.readBytes += bytes; + return bytes; + }; + if (key === 'writeSync') return (fd, data, ...args) => { + if (runtimeFile(fd)) { + const bytes = typeof data === 'string' ? Buffer.byteLength(data, typeof args[1] === 'string' ? args[1] : 'utf8') : args[1] ?? data.length; + assert(bytes <= BLOCK_BYTES, '单次运行态编码写入不能超过64KiB'); + metrics.maxWrite = Math.max(metrics.maxWrite, bytes); + metrics.writes += 1; + if (failure === 'write' && bytesSinceFailureMode >= 1024 * 1024) throw new Error('模拟运行态中途write失败'); + const written = target.writeSync(fd, data, ...args); + metrics.writeBytes += written; + bytesSinceFailureMode += written; + return written; + } + return target.writeSync(fd, data, ...args); + }; + if (key === 'fsyncSync') return fd => { + if (failure === 'fsync' && runtimeFile(fd)) throw new Error('模拟运行态fsync失败'); + return target.fsyncSync(fd); + }; + if (key === 'renameSync') return (from, to) => { + if (failure === 'rename' && String(to) === filename) throw new Error('模拟运行态rename失败'); + return target.renameSync(from, to); + }; + return target[key]; + } }); + return { fs: filesystem, metrics, fail(mode) { failure = mode; bytesSinceFailureMode = 0; } }; +} + +function assertComplete(state, fixture) { + assert.equal(state.version, 1, '旧version:1运行态须兼容'); + assert.equal(state.agent, 'codexapp'); + assert.equal(state.sessionId, ID); + assert.equal(state.turnStatus, 'running'); + assert.deepEqual(state.special, SPECIAL); + for (const [name, value] of [['fullText', state.fullText], ['result', state.toolCalls[0].result], ['delta', state.toolOutputDeltas[0][1]], ['item', state.agentMessageItems[0][1]]]) { + assert.equal(value.length, fixture.expected[name].length, `${name} 长度发生变化`); + assert.equal(textHash(value), fixture.expected[name].hash, `${name} 正文摘要发生变化`); + } + assert.deepEqual(state.toolCalls[0].input, { text: '完整输入' }); + assert.deepEqual(state.lastUsage, { inputTokens: 1, outputTokens: 2 }); +} + +function runChild(sessionsDir, fixture) { + const tracked = trackedFilesystem(fixture.filename); + let store = createSessionHistoryStore({ sessionsDir, fs: tracked.fs }); + stage('开始读取旧version:1完整运行态'); + let state = store.loadTurn(ID); + assertComplete(state, fixture); + stage('旧运行态摘要校验通过,开始分段保存'); + store.saveTurn(ID, state); + const committedHash = fileHash(fixture.filename); + assert.equal(fs.statSync(fixture.filename).size, TARGET_BYTES, '完整持久化不能裁剪运行态'); + stage('分段保存完成,开始原子失败注入'); + state.turnStatus = 'new-uncommitted-status'; + for (const mode of ['write', 'fsync', 'rename']) { + tracked.fail(mode); + assert.throws(() => store.saveTurn(ID, state), /模拟运行态/); + assert.equal(fileHash(fixture.filename), committedHash, `${mode}失败必须保留上次完整文件`); + assert.equal(fs.readdirSync(path.dirname(fixture.filename)).some(name => name.endsWith('.tmp')), false, `${mode}失败不能遗留临时文件`); + stage(`${mode}失败保护通过`); + } + tracked.fail(null); + state = null; + store = null; + if (typeof global.gc === 'function') global.gc(); + const reopened = createSessionHistoryStore({ sessionsDir, fs: tracked.fs }); + state = reopened.loadTurn(ID); + assertComplete(state, fixture); + assert.equal(reopened.loadTurn('missing-runtime'), null); + assert(tracked.metrics.reads > 1 && tracked.metrics.writes > 1); + console.log(JSON.stringify({ archiveMiB: TARGET_BYTES / 1024 / 1024, peakRssMiB: process.resourceUsage().maxRSS / 1024, + maxReadKiB: tracked.metrics.maxRead / 1024, maxWriteKiB: tracked.metrics.maxWrite / 1024, + reads: tracked.metrics.reads, writes: tracked.metrics.writes, failureModes: 3 })); +} + +function runTests() { + const sessionsDir = fs.mkdtempSync(path.join(temporaryRoot(), 'ccweb-runtime-memory-')); + try { + stage(`开始生成80MiB合成完整运行态(${path.dirname(sessionsDir)})`); + const fixture = writeFixture(sessionsDir); + stage('合成旧运行态完成'); + const child = spawnSync(process.execPath, ['--expose-gc', '--max-old-space-size=256', __filename, '--child', sessionsDir, JSON.stringify(fixture)], { + encoding: 'utf8', timeout: 60000, maxBuffer: 512 * 1024, + stdio: ['ignore', 'pipe', process.env.CCWEB_RUNTIME_TEST_PROGRESS === '1' ? 'inherit' : 'pipe'], + }); + if (child.error || child.status !== 0) process.stderr.write(child.stderr || ''); + assert.ifError(child.error); + assert.equal(child.status, 0, `256MiB堆运行态子进程失败(信号:${child.signal || '无'}):${child.stderr || child.stdout}`); + const result = JSON.parse(child.stdout.trim()); + console.log(`通过:${result.archiveMiB.toFixed(1)} MiB完整运行态兼容读取、分段保存、重开后四项正文摘要一致`); + console.log(`通过:write/fsync/rename三类失败保留旧文件且无临时文件,单次读取/写入最大 ${result.maxReadKiB.toFixed(1)}/${result.maxWriteKiB.toFixed(1)} KiB`); + console.log(`内存:256 MiB堆限制下RSS峰值 ${result.peakRssMiB.toFixed(1)} MiB;读取仍持有完整运行态对象,分段处理限制临时编码副本`); + return result; + } finally { fs.rmSync(sessionsDir, { recursive: true, force: true }); } +} + +if (require.main === module) { + if (process.argv[2] === '--child') runChild(process.argv[3], JSON.parse(process.argv[4])); + else runTests(); +} +module.exports = { runTests }; diff --git a/scripts/session-history-memory-unit.js b/scripts/session-history-memory-unit.js new file mode 100644 index 0000000..7ceb3a0 --- /dev/null +++ b/scripts/session-history-memory-unit.js @@ -0,0 +1,314 @@ +#!/usr/bin/env node +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const { createSessionHistoryStore } = require('../lib/session-history-store'); + +const ID = 'synthetic-memory-history'; +const TARGET_BYTES = Math.round(70.5 * 1024 * 1024); +const TAIL_COUNT = 12; +const PAGE_COUNT = 17; +const SAVE_COUNT = 6; +const ORIGINAL_CONTENT = '完整历史:中文、引号"、反斜线\\、换行\n、表情🙂\n' + 'x'.repeat(62 * 1024); +const ORIGINAL_RESULT = '完整工具输出:' + 't'.repeat(2 * 1024); +const startedAt = Date.now(); + +function stage(message) { + process.stderr.write(`[历史内存回归 +${((Date.now() - startedAt) / 1000).toFixed(2)} 秒] ${message}\n`); +} + +function temporaryRoot() { + // 内存回归优先使用真实 tmpfs;慢盘逐块 fsync 可能耗尽 60 秒测试预算。 + // 文件写入、校验和 fsync 都保持真实调用,原子失败语义由存储单测覆盖。 + if (process.platform === 'linux' && typeof fs.statfsSync === 'function') { + try { + const stat = fs.statfsSync('/dev/shm'); + fs.accessSync('/dev/shm', fs.constants.W_OK); + if (Number(stat.bavail) * Number(stat.bsize) >= 512 * 1024 * 1024) return '/dev/shm'; + } catch {} + } + return os.tmpdir(); +} + +function originalMessage(index) { + return { + id: `original-${index}`, + role: index % 2 ? 'assistant' : 'user', + content: `${index}:${ORIGINAL_CONTENT}`, + timestamp: new Date(1800000000000 + index * 1000).toISOString(), + toolCalls: [{ + id: `tool-${index}`, + name: 'synthetic-tool', + input: { text: '输入完整保存', nested: { array: [index, { value: '\\"\n🙂' }] } }, + result: ORIGINAL_RESULT, + }], + }; +} + +function addedMessage(index) { + return { id: `added-${index}`, role: 'assistant', content: `新增消息 ${index}:` + 'a'.repeat(4096) }; +} + +function digest(value) { + return crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +// 测试资料由父进程逐条写入,避免测试自身先建立整份大对象。 +function writeLegacyFixture(sessionsDir) { + const historyDir = path.join(sessionsDir, '_history'); + fs.mkdirSync(historyDir, { recursive: true }); + const archivePath = path.join(historyDir, `${ID}.json`); + const revision = crypto.randomUUID(); + const metadata = JSON.parse('{"__proto__":{"saved":"原值"},"constructor":"保留键","deep":{"a":{"b":{"c":{"value":"完整值"}}}}}'); + const session = { id: ID, agent: 'codexapp', codexAppThreadId: 'synthetic-thread', title: '内存回归合成资料', metadata }; + const head = JSON.stringify({ version: 1, revision, complete: true, resetRevision: null, session }).slice(0, -2) + ',"messages":['; + const fd = fs.openSync(archivePath, 'wx', 0o600); + let bytes = Buffer.byteLength(head); + let count = 0; + const paddingHead = '],"padding":"'; + const ending = '"}}'; + const paddingOverhead = Buffer.byteLength(paddingHead + ending); + let padding; + let batch = []; + let batchBytes = 0; + function flushBatch() { + if (!batch.length) return; + fs.writeSync(fd, batch.join('')); + batch = []; + batchBytes = 0; + } + try { + fs.writeSync(fd, head); + while (true) { + const json = `${count ? ',' : ''}${JSON.stringify(originalMessage(count))}`; + if (bytes + Buffer.byteLength(json) + paddingOverhead > TARGET_BYTES) break; + batch.push(json); + batchBytes += Buffer.byteLength(json); + if (batchBytes >= 1024 * 1024) flushBatch(); + bytes += Buffer.byteLength(json); + count += 1; + } + padding = 'p'.repeat(TARGET_BYTES - bytes - paddingOverhead); + flushBatch(); + fs.writeSync(fd, paddingHead + padding + ending); + } finally { + fs.closeSync(fd); + } + return { archivePath, count, bytes: fs.statSync(archivePath).size, metadataHash: digest(metadata), paddingHash: digest(padding) }; +} + +function createTrackedFilesystem(archivePath) { + const opened = new Map(); + let metrics; + function reset(label) { + metrics = { label, chunkFiles: new Set(), chunkBytes: 0, legacyBytes: 0, legacyMaxRead: 0, + chunkWrites: 0, chunkFsyncMs: 0, chunkFsyncs: 0, nextLegacyProgress: 8 * 1024 * 1024 }; + } + reset('初始化'); + function resolveFile(value) { + if (typeof value === 'number') return opened.get(value) || ''; + return value ? path.resolve(String(value)) : ''; + } + function recordRead(file, bytes) { + if (file === archivePath) { + metrics.legacyBytes += bytes; + metrics.legacyMaxRead = Math.max(metrics.legacyMaxRead, bytes); + if (metrics.legacyBytes >= metrics.nextLegacyProgress) { + stage(`已扫描旧归档 ${(metrics.legacyBytes / 1024 / 1024).toFixed(1)} MiB,写块 ${metrics.chunkWrites} 次,块 fsync 累计 ${(metrics.chunkFsyncMs / 1000).toFixed(2)} 秒`); + metrics.nextLegacyProgress += 8 * 1024 * 1024; + } + } + if (file.includes(`${path.sep}chunks${path.sep}`)) { + metrics.chunkFiles.add(file); + metrics.chunkBytes += bytes; + } + } + const filesystem = new Proxy(fs, { + get(target, key) { + if (key === 'openSync') return (file, ...args) => { + const fd = target.openSync(file, ...args); + const resolved = resolveFile(file); + opened.set(fd, resolved); + if (resolved.includes(`${path.sep}chunks${path.sep}`) && String(args[0]).includes('w')) metrics.chunkWrites += 1; + return fd; + }; + if (key === 'closeSync') return fd => { + const result = target.closeSync(fd); + opened.delete(fd); + return result; + }; + if (key === 'readFileSync') return (file, ...args) => { + const resolved = resolveFile(file); + assert.notEqual(resolved, archivePath, '旧完整归档必须流式读取,禁止整文件 readFileSync'); + const value = target.readFileSync(file, ...args); + recordRead(resolved, typeof value === 'string' ? Buffer.byteLength(value) : value.length); + return value; + }; + if (key === 'readSync') return (fd, ...args) => { + const file = resolveFile(fd); + const requested = typeof args[1] === 'object' ? args[1].length ?? args[0].length : args[2] ?? args[0].length; + if (file === archivePath) assert(requested <= 256 * 1024, '迁移旧归档的单次读取必须有界'); + const bytes = target.readSync(fd, ...args); + recordRead(file, bytes); + return bytes; + }; + if (key === 'fsyncSync') return fd => { + const chunk = resolveFile(fd).includes(`${path.sep}chunks${path.sep}`); + const started = Date.now(); + const result = target.fsyncSync(fd); + if (chunk) { + metrics.chunkFsyncs += 1; + metrics.chunkFsyncMs += Date.now() - started; + if (metrics.chunkFsyncs % 16 === 0) stage(`块 fsync ${metrics.chunkFsyncs} 次,累计 ${(metrics.chunkFsyncMs / 1000).toFixed(2)} 秒`); + } + return result; + }; + return target[key]; + }, + }); + return { fs: filesystem, reset, snapshot: () => ({ ...metrics, chunkFiles: metrics.chunkFiles.size }) }; +} + +function runChild(sessionsDir, fixture) { + const tracked = createTrackedFilesystem(fixture.archivePath); + const store = createSessionHistoryStore({ sessionsDir, fs: tracked.fs }); + for (const name of ['loadTail', 'loadMetadata', 'loadHistoryPage']) { + assert.equal(typeof store[name], 'function', `分块归档缺少 ${name} 接口`); + } + const observations = []; + stage('开始迁移旧归档并加载首个尾页'); + let tail = store.loadTail(ID, TAIL_COUNT); + stage('旧归档迁移完成'); + assert(tail, '旧归档迁移后必须可加载尾部'); + assert.equal(tail.messages.length, TAIL_COUNT); + assert.equal(tail.messages[0].id, `original-${fixture.count - TAIL_COUNT}`); + assert.equal(digest(tail.messages.at(-1)), digest(originalMessage(fixture.count - 1))); + observations.push(tracked.snapshot()); + const manifestPath = path.join(sessionsDir, '_history', ID, 'manifest.json'); + const chunkDir = path.join(sessionsDir, '_history', ID, 'chunks'); + assert(fs.existsSync(manifestPath), '旧归档应发布版本 2 清单'); + assert.equal(JSON.parse(fs.readFileSync(manifestPath, 'utf8')).version, 2); + const initialChunkFiles = fs.readdirSync(chunkDir).filter(name => name.endsWith('.json')); + assert(initialChunkFiles.length > 2, '大归档应拆成多个独立消息块'); + + function assertBoundedRead(label, callback, metadataOnly = false) { + tracked.reset(label); + const result = callback(); + const observation = tracked.snapshot(); + assert.equal(observation.legacyBytes, 0, `${label} 不应重新扫描旧完整归档`); + assert(observation.chunkFiles < initialChunkFiles.length, `${label} 不应加载全部消息块`); + assert(observation.chunkWrites < initialChunkFiles.length, `${label} 不应重写全部消息块`); + if (metadataOnly) assert.equal(observation.chunkFiles, 0, `${label} 不应读取消息块`); + observations.push(observation); + return result; + } + + const metadata = assertBoundedRead('只加载元数据', () => store.loadMetadata(ID), true); + assert.equal(metadata.id, ID); + assert.equal(digest(metadata.metadata), fixture.metadataHash); + assert.equal(digest(metadata.padding), fixture.paddingHash); + assert(!metadata.messages || metadata.messages.length === 0, '元数据接口不应携带消息正文'); + + for (let index = 0; index < SAVE_COUNT; index += 1) { + stage(`开始第 ${index + 1} 次尾部加载和保存`); + tail = assertBoundedRead(`第 ${index + 1} 次加载尾部`, () => store.loadTail(ID, TAIL_COUNT)); + const lastOriginal = tail.messages.find(message => message.id === `original-${fixture.count - 1}`); + assert(lastOriginal, '尾页应该覆盖待更新的最后一条原始消息'); + lastOriginal.content = originalMessage(fixture.count - 1).content + `|第 ${index + 1} 次完整修改`; + lastOriginal.toolCalls[0].result = ORIGINAL_RESULT + `|第 ${index + 1} 次工具输出`; + tail.messages.push(addedMessage(index)); + tail.title = `已提交 ${index + 1} 次尾部更新`; + assertBoundedRead(`第 ${index + 1} 次保存尾部`, () => store.save(tail)); + stage(`第 ${index + 1} 次尾部保存完成`); + } + + const finalMetadata = assertBoundedRead('重新加载最新元数据', () => store.loadMetadata(ID), true); + assert.equal(finalMetadata.title, `已提交 ${SAVE_COUNT} 次尾部更新`); + assert.equal(digest(finalMetadata.metadata), fixture.metadataHash); + assert.equal(digest(finalMetadata.padding), fixture.paddingHash); + const reopened = createSessionHistoryStore({ sessionsDir, fs: tracked.fs }); + let before = fixture.count + SAVE_COUNT; + let checked = 0; + let pageCount = 0; + let revision; + stage('开始重开存储并逐页校验全部历史'); + while (before > 0) { + const page = assertBoundedRead(`分页 ${pageCount + 1}`, () => reopened.loadHistoryPage(ID, before, PAGE_COUNT)); + assert.equal(page.total, fixture.count + SAVE_COUNT, '尾页保存必须保留未加载的全部历史'); + assert.equal(page.end, before); + assert.equal(page.start, Math.max(0, before - PAGE_COUNT)); + assert.equal(page.messages.length, page.end - page.start); + assert.equal(page.complete, true); + if (revision) assert.equal(page.revision, revision, '分页期间应保持同一归档修订'); + revision = page.revision; + page.messages.forEach((message, offset) => { + const position = page.start + offset; + const expected = position < fixture.count ? originalMessage(position) : addedMessage(position - fixture.count); + if (position === fixture.count - 1) { + expected.content += `|第 ${SAVE_COUNT} 次完整修改`; + expected.toolCalls[0].result += `|第 ${SAVE_COUNT} 次工具输出`; + } + assert.equal(message.id, expected.id, `第 ${position} 条消息的顺序或稳定 ID 错误`); + assert.equal(digest(message), digest(expected), `第 ${position} 条正文、工具或元数据出现截断或变化`); + checked += 1; + }); + before = page.start; + pageCount += 1; + if (pageCount % 16 === 0) stage(`已校验 ${pageCount} 页、${checked} 条消息`); + } + assert.equal(checked, fixture.count + SAVE_COUNT); + const currentManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + const currentChunkFiles = new Set(currentManifest.entries.map(entry => entry.file)); + const reusedChunks = initialChunkFiles.filter(name => currentChunkFiles.has(name)).length; + assert(reusedChunks > 0, '尾部更新必须复用未变化的消息块'); + stage(`全部 ${pageCount} 页、${checked} 条历史校验完成`); + console.log(JSON.stringify({ + archiveMiB: fixture.bytes / 1024 / 1024, + originalMessages: fixture.count, + checkedMessages: checked, + pages: pageCount, + saves: SAVE_COUNT, + peakRssMiB: process.resourceUsage().maxRSS / 1024, + initialChunks: initialChunkFiles.length, + reusedChunks, + maxChunksPerBoundedOperation: Math.max(...observations.slice(1).map(item => item.chunkFiles)), + legacyMaxReadKiB: observations[0].legacyMaxRead / 1024, + migrationFsyncSeconds: observations[0].chunkFsyncMs / 1000, + })); +} + +function runTests() { + const sessionsDir = fs.mkdtempSync(path.join(temporaryRoot(), 'ccweb-history-memory-')); + try { + stage(`开始流式写入 70.5 MiB 合成夹具(${path.dirname(sessionsDir)})`); + const fixture = writeLegacyFixture(sessionsDir); + stage(`夹具写入完成,共 ${fixture.count} 条原始消息`); + assert.equal(fixture.bytes, TARGET_BYTES); + const child = spawnSync(process.execPath, ['--max-old-space-size=256', __filename, '--child', sessionsDir, JSON.stringify(fixture)], { + encoding: 'utf8', timeout: 60000, maxBuffer: 512 * 1024, + stdio: ['ignore', 'pipe', process.env.CCWEB_HISTORY_TEST_PROGRESS === '1' ? 'inherit' : 'pipe'], + }); + if (child.error || child.status !== 0) process.stderr.write(child.stderr || ''); + assert.ifError(child.error); + assert.equal(child.status, 0, `256 MiB 堆子进程失败(信号:${child.signal || '无'}):\n${child.stderr || child.stdout}`); + const result = JSON.parse(child.stdout.trim()); + console.log(`通过:${result.archiveMiB.toFixed(1)} MiB 旧归档流式迁移,256 MiB 堆内逐页校验 ${result.checkedMessages} 条完整历史`); + console.log(`通过:${result.saves} 次尾部保存、${result.pages} 页读取,单次最多触达 ${result.maxChunksPerBoundedOperation}/${result.initialChunks} 个原始块`); + console.log(`内存:子进程 RSS 峰值 ${result.peakRssMiB.toFixed(1)} MiB;旧归档单次读取最大 ${result.legacyMaxReadKiB.toFixed(1)} KiB;复用 ${result.reusedChunks} 个原始块`); + return result; + } finally { + fs.rmSync(sessionsDir, { recursive: true, force: true }); + } +} + +if (require.main === module) { + if (process.argv[2] === '--child') runChild(process.argv[3], JSON.parse(process.argv[4])); + else runTests(); +} + +module.exports = { runTests }; diff --git a/scripts/session-history-store-unit.js b/scripts/session-history-store-unit.js index b9438e0..c656cf1 100644 --- a/scripts/session-history-store-unit.js +++ b/scripts/session-history-store-unit.js @@ -36,11 +36,13 @@ function createServerHarness(sessionsDir, options = {}) { 'sanitizeToolCallsForPersist', 'sanitizeMessageForPersist', 'sanitizeMessagesForPersist', 'sanitizeSessionForPersist', 'buildSessionJsonForPersist', 'writeFileAtomicSync', 'safeReadSessionJson', 'loadSession', 'preserveNewerTaskTrackingForSessionSave', + 'findSessionMessage', 'saveSession', 'sessionHistoryCacheKey', 'cacheSessionHistory', 'loadNativeSessionHistory', 'resolveSessionHistory', 'listSessionFiles', 'refreshSessionSnapshotCache', 'prepareNativeHistoryImport', 'handleSlashCommand', 'handleDeleteSession', 'findCcwebPromptMessage', 'removeCcwebPromptMessage', 'rollbackPersistedCodexAppSteerMessage', 'handleLoadHistoryPage', + 'buildSessionInfoPayload', 'handleResumeSession', ]; const api = new Function('fs', 'path', 'crypto', 'SESSIONS_DIR', 'sessionHistoryStore', 'rollouts', 'options', 'logs', ` const SESSION_MAX_JSON_BYTES = options.snapshotMaxBytes || 512 * 1024; @@ -60,6 +62,8 @@ function createServerHarness(sessionsDir, options = {}) { const retainedCodexAppTurns = new Map(); const activeCodexAppGoalCommands = new Map(); const activeCodexAppCompactions = new Map(); + const pendingCodexCapacityRetries = new Map(); + const wsSessionMap = new Map(); const codexAppGoalStates = new Map(); const pendingSlashCommands = new Map(); const pendingCompactRetries = new Map(); @@ -83,6 +87,10 @@ function createServerHarness(sessionsDir, options = {}) { function wsSend(ws, payload) { responses.push(payload); } function attachClientRequestId(payload, source) { return { ...payload, requestId: source?.requestId }; } function taskTrackingSnapshotForSession() { return null; } + function crossConversationWaitState() { return {}; } + function detachWsFromActiveRuntimes() {} + function isSessionRunning() { return false; } + function attachActiveRuntimeToWs() {} function publicTitleMetadata() { return {}; } function sessionModelLabel(session) { return session.model; } function isCodexAppSession(session) { return session.agent === 'codexapp'; } @@ -105,9 +113,16 @@ function createServerHarness(sessionsDir, options = {}) { ${pureSource} ${names.map(name => extract(name)).join('\n')} return { - loadSession, saveSession, resolveSessionHistory, listSessionFiles, refreshSessionSnapshotCache, + loadSession: (id, loadOptions = {}) => { + const explicitWindow = ['metadata', 'count', 'window', 'messageId', 'promptId'].some(key => Object.prototype.hasOwnProperty.call(loadOptions, key)); + const full = Object.prototype.hasOwnProperty.call(loadOptions, 'full') + ? loadOptions.full : options.fullHistory !== false && !explicitWindow; + return loadSession(id, { ...loadOptions, full }); + }, + saveSession, resolveSessionHistory, listSessionFiles, refreshSessionSnapshotCache, findSessionMessage, prepareNativeHistoryImport, handleSlashCommand, handleDeleteSession, removeCcwebPromptMessage, rollbackPersistedCodexAppSteerMessage, handleLoadHistoryPage, responses, logs, sessionHistoryStore, + buildSessionInfoPayload, handleResumeSession, }; `)(filesystem, path, crypto, sessionsDir, sessionHistoryStore, rollouts, options, logs); return api; @@ -302,6 +317,16 @@ check('首次快照失败的孤立档案重开后仍可读取并参与列表且 assert(fs.existsSync(path.join(dir, 'orphan.json'))); }); +check('运行态保留目录和同名旧文件不会干扰完整归档与会话列表', dir => { + const api = createServerHarness(dir); + assert.equal(api.saveSession(fixture('listed-session', 3)), true); + api.sessionHistoryStore.saveTurn('listed-session', { sessionId: 'listed-session', agent: 'codexapp' }); + fs.writeFileSync(path.join(dir, '_history', '_turns.json'), '{}'); + assert.deepEqual(api.sessionHistoryStore.listTurnIds(), ['listed-session']); + assert.deepEqual(api.sessionHistoryStore.listIds(), ['listed-session']); + assert.deepEqual(api.listSessionFiles(), ['listed-session.json']); +}); + check('可信旧修订并发追加和插入保留锚点顺序以及特殊 JSON 键', dir => { const api = createServerHarness(dir); const original = fixture('concurrent', 4); @@ -389,7 +414,7 @@ check('档案异常禁止回退预览覆盖,删除档案后旧加载对象不 assert.equal(api.saveSession(fixture('corrupt', 220)), true); const stale = api.loadSession('corrupt'); const oldSnapshot = fs.readFileSync(path.join(dir, 'corrupt.json'), 'utf8'); - fs.writeFileSync(path.join(dir, '_history', 'corrupt.json'), '{损坏文件'); + fs.writeFileSync(path.join(dir, '_history', 'corrupt', 'manifest.json'), '{损坏文件'); assert.equal(api.loadSession('corrupt'), null); assert.equal(api.saveSession(stale), false); assert.equal(fs.readFileSync(path.join(dir, 'corrupt.json'), 'utf8'), oldSnapshot); @@ -445,7 +470,7 @@ check('旧快照的真实 rollout 前缀确认后先归档,移除 rollout 后 const api = createServerHarness(dir, { nativeDir, nativeFiles: [rolloutPath] }); const resolved = api.resolveSessionHistory(api.loadSession(id)); assert.equal(resolved.messages.length, 8); - assert(fs.existsSync(path.join(dir, '_history', `${id}.json`)), '确认前缀后应提交自身档案'); + assert(fs.existsSync(path.join(dir, '_history', id, 'manifest.json')), '确认前缀后应提交自身档案'); fs.unlinkSync(rolloutPath); const reopened = createServerHarness(dir); const reloaded = reopened.loadSession(id); @@ -462,7 +487,8 @@ check('原生日志不匹配和不存在时只返回已有旧快照并保留缺 assert.equal(history.messages.length, 57); assert.equal(history.available, false); assert.equal(api.saveSession(loaded), true); - assert.equal(createServerHarness(dir).resolveSessionHistory(createServerHarness(dir).loadSession('unconfirmed')).available, false); + const reopened = createServerHarness(dir); + assert.equal(reopened.resolveSessionHistory(reopened.loadSession('unconfirmed')).available, false); }); check('归档提交失败的旧前缀恢复只提供原快照而不宣称恢复成功', dir => { @@ -538,6 +564,402 @@ check('生产分页出口读取57条旧快照:首屏12以外45条完整有序 assert.equal(new Set(chunks.map(message => message.id)).size, 45); }); +check('真实默认加载650条历史只持有180条尾部且显式窗口不走完整加载', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('bounded-default', 650); + assert.equal(api.saveSession(original), true); + const loaded = api.loadSession(original.id); + assert.equal(loaded.messages.length, 180); + assert.equal(loaded.historySnapshotBaseIndex, 470); + assert.deepEqual(loaded.messages, original.messages.slice(470)); + const state = api.sessionHistoryStore.state(loaded); + assert.equal(state.partial, true); + assert.equal(state.start, 470); + assert.equal(state.total, 650); + assert.deepEqual(api.loadSession(original.id, { full: true }).messages, original.messages); + const fullDefault = createServerHarness(dir); + assert.equal(fullDefault.loadSession(original.id, { count: 7 }).messages.length, 7); + assert.equal(fullDefault.loadSession(original.id, { metadata: true }).messages.length, 0); + assert.equal(fullDefault.loadSession(original.id, { window: { start: 470, end: 650 } }).messages.length, 180); +}); + +check('有界尾部修改删除前插追加精确保存且未加载前缀原样保留', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('bounded-edit', 650); + assert.equal(api.saveSession(original), true); + const loaded = api.loadSession(original.id); + loaded.messages[0].content = '尾部同 ID 更新'; + loaded.messages[0].toolCalls = [{ id: 'changed-tool', input: { retained: true }, result: '完整工具结果' }]; + loaded.messages.splice(1, 1); + loaded.messages.splice(2, 0, { id: 'inserted-before-tail', role: 'assistant', content: '已有气泡前插入' }); + loaded.messages.push({ id: 'new-tail', role: 'user', content: '追加完整内容' }); + const expected = [...original.messages.slice(0, 470), ...loaded.messages]; + assert.equal(api.saveSession(loaded), true); + const restored = api.loadSession(original.id, { full: true }); + assert.deepEqual(restored.messages, expected); + assert.deepEqual(restored.messages.slice(0, 470), original.messages.slice(0, 470)); + assert.equal(restored.messages.some(message => message.id === 'message-471'), false); +}); + +check('中间加载窗口修改删除前插保存同时保留未加载前缀和后缀', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('middle-window', 650); + original.messages[12].ccwebPrompt = { id: 'early-prompt' }; + assert.equal(api.saveSession(original), true); + const tail = api.loadSession(original.id); + assert.deepEqual(api.findSessionMessage(tail, message => message?.ccwebPrompt?.id === 'early-prompt'), original.messages[12]); + const promptWindow = api.loadSession(original.id, { promptId: 'early-prompt' }); + assert.equal(promptWindow.messages.length, 1); + assert.equal(promptWindow.messages[0].id, 'message-12'); + const messageWindow = api.loadSession(original.id, { messageId: 'message-10' }); + assert.equal(messageWindow.messages.length, 1); + assert.equal(messageWindow.messages[0].id, 'message-10'); + const loaded = api.loadSession(original.id, { window: { start: 250, end: 255 } }); + assert.equal(loaded.messages.length, 5); + assert.equal(loaded.historySnapshotBaseIndex, 250); + loaded.messages[0].content = '中间窗口同 ID 更新'; + loaded.messages.splice(1, 1); + loaded.messages.splice(2, 0, { id: 'middle-insert', role: 'assistant', content: '中间窗口前插内容' }); + const expected = [...original.messages.slice(0, 250), ...loaded.messages, ...original.messages.slice(255)]; + assert.equal(api.saveSession(loaded), true); + const restored = api.loadSession(original.id, { full: true }); + assert.deepEqual(restored.messages, expected); + assert.deepEqual(restored.messages.slice(0, 250), original.messages.slice(0, 250)); + assert.deepEqual(restored.messages.slice(255), original.messages.slice(255)); + assert.equal(restored.messages.some(message => message.id === 'message-251'), false); +}); + +check('仅元数据加载更新标题及taskTracking不读取或覆盖未加载消息', dir => { + const api = createServerHarness(dir); + const original = fixture('metadata-only', 650); + original.taskTracking = { version: 2, tasks: [{ id: 'task-a', title: '原任务', status: 'pending' }] }; + assert.equal(api.saveSession(original), true); + const metadata = api.loadSession(original.id, { metadata: true }); + assert.equal(metadata.messages.length, 0); + assert.equal(api.sessionHistoryStore.state(metadata).partial, true); + metadata.title = '仅元数据更新的标题'; + metadata.taskTracking = { version: 3, tasks: [{ id: 'task-a', title: '新任务', status: 'completed' }] }; + assert.equal(api.saveSession(metadata), true); + const restored = api.loadSession(original.id); + assert.equal(restored.title, metadata.title); + assert.deepEqual(restored.taskTracking, metadata.taskTracking); + assert.deepEqual(restored.messages, original.messages); +}); + +check('同修订任务版本保护只读取修订头且提交后同步活跃对象', dir => { + let tracking = false; + let wholeReads = 0; + const filesystem = new Proxy(fs, { + get(target, key) { + if (key === 'readFileSync') return (...args) => { + if (tracking) wholeReads += 1; + return target.readFileSync(...args); + }; + return target[key]; + }, + }); + const api = createServerHarness(dir, { fs: filesystem, fullHistory: false }); + const original = fixture('same-task-version', 650); + original.taskTracking = { version: 5, statusId: 'waiting', summary: '已提交任务' }; + assert.equal(api.saveSession(original), true); + const loaded = api.loadSession(original.id); + const taskReference = loaded.taskTracking; + loaded.taskTracking.version = 1; + loaded.taskTracking.statusId = '旧任务'; + tracking = true; + const committed = api.sessionHistoryStore.save(loaded); + tracking = false; + assert.equal(wholeReads, 0, '同修订提交不得重新读取清单或消息正文'); + assert.deepEqual(committed.taskTrackingPreserved, { incomingVersion: 1, persistedVersion: 5 }); + assert.equal(loaded.taskTracking, taskReference, '任务对象仍原位协调'); + assert.deepEqual(loaded.taskTracking, original.taskTracking); + loaded.taskTracking.summary = '提交后修改'; + const disk = api.loadSession(original.id, { metadata: true }); + assert.equal(disk.taskTracking.summary, '已提交任务', '活跃对象不能污染可信清单'); + assert.equal(api.loadSession(original.id, { full: true }).messages.length, 650); +}); + +check('旧修订普通保存完整保留更高任务版本与流式消息引用', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('stale-task-version', 650); + original.taskTracking = { version: 2, statusId: 'initial', nested: { value: '原任务' } }; + assert.equal(api.saveSession(original), true); + const streaming = api.loadSession(original.id); + const message = streaming.messages.at(-1); + const metadata = api.loadSession(original.id, { metadata: true }); + metadata.taskTracking = { version: 6, statusId: 'newer', nested: { value: '新任务' } }; + assert.equal(api.saveSession(metadata, { taskBoardMetadata: true }), true); + streaming.taskTracking = { version: 4, statusId: 'stale', nested: { value: '旧对象修改' } }; + message.content += '|增量'; + assert.equal(api.saveSession(streaming), true); + assert.deepEqual(streaming.taskTracking, metadata.taskTracking); + assert.equal(streaming.messages.at(-1), message); + const restored = api.loadSession(original.id, { full: true }); + assert.equal(restored.messages.length, 650); + assert.equal(restored.messages.at(-1).content, message.content); + assert.deepEqual(restored.taskTracking, metadata.taskTracking); +}); + +check('任务看板显式提交可删除任务状态而普通保存保留已提交版本', dir => { + const api = createServerHarness(dir); + const original = fixture('task-metadata-removal', 3); + original.taskTracking = { version: 3, statusId: 'waiting' }; + assert.equal(api.saveSession(original), true); + const ordinary = api.loadSession(original.id); + delete ordinary.taskTracking; + assert.equal(api.saveSession(ordinary), true); + assert.deepEqual(ordinary.taskTracking, original.taskTracking); + const board = api.loadSession(original.id, { metadata: true }); + delete board.taskTracking; + assert.equal(api.saveSession(board, { taskBoardMetadata: true }), true); + const restored = api.loadSession(original.id); + assert.equal(Object.prototype.hasOwnProperty.call(restored, 'taskTracking'), false); + assert.deepEqual(restored.messages, original.messages); +}); + +check('任务状态保护遇清单提交失败不提前修改对象或推进基线', dir => { + let failManifest = false; + const filesystem = new Proxy(fs, { + get(target, key) { + if (key === 'renameSync') return (from, to) => { + if (failManifest && path.basename(to) === 'manifest.json') throw new Error('模拟任务清单提交失败'); + return target.renameSync(from, to); + }; + return target[key]; + }, + }); + const api = createServerHarness(dir, { fs: filesystem }); + const original = fixture('task-failed-commit', 4); + original.taskTracking = { version: 5, statusId: 'saved' }; + assert.equal(api.saveSession(original), true); + const loaded = api.loadSession(original.id); + loaded.taskTracking = { version: 1, statusId: 'stale' }; + const pendingTracking = loaded.taskTracking; + const revision = api.sessionHistoryStore.state(loaded).revision; + failManifest = true; + assert.equal(api.saveSession(loaded), false); + assert.equal(loaded.taskTracking, pendingTracking); + assert.equal(loaded.taskTracking.version, 1); + assert.equal(api.sessionHistoryStore.state(loaded).revision, revision); + failManifest = false; + assert.equal(api.saveSession(loaded), true); + assert.deepEqual(loaded.taskTracking, original.taskTracking); +}); + +check('不同尾部加载范围的旧修订合并修改删除及新增并保留各自顺序', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('partial-ranges', 650); + assert.equal(api.saveSession(original), true); + const narrow = api.sessionHistoryStore.loadTail(original.id, 5); + const wide = api.sessionHistoryStore.loadTail(original.id, 30); + wide.messages[0].content = '较宽窗口的修改'; + wide.messages.splice(wide.messages.findIndex(message => message.id === 'message-640'), 1); + wide.messages.push({ id: 'wide-new', role: 'user', content: '较宽窗口追加' }); + assert.equal(api.saveSession(wide), true); + narrow.messages[0].content = '旧修订较窄窗口的修改'; + narrow.messages.splice(1, 0, { id: 'narrow-insert', role: 'assistant', content: '较窄窗口前插' }); + narrow.messages.push({ id: 'narrow-new', role: 'assistant', content: '较窄窗口追加' }); + assert.equal(api.saveSession(narrow), true); + const restored = api.loadSession(original.id, { full: true }); + const expected = original.messages.filter(message => message.id !== 'message-640').map(message => ({ ...message })); + expected.find(message => message.id === 'message-620').content = '较宽窗口的修改'; + expected.find(message => message.id === 'message-645').content = '旧修订较窄窗口的修改'; + expected.splice(expected.findIndex(message => message.id === 'message-646'), 0, { id: 'narrow-insert', role: 'assistant', content: '较窄窗口前插' }); + expected.push({ id: 'narrow-new', role: 'assistant', content: '较窄窗口追加' }, { id: 'wide-new', role: 'user', content: '较宽窗口追加' }); + assert.deepEqual(restored.messages, expected); + assert.deepEqual(narrow.messages, expected.slice(644)); +}); + +check('部分加载明确reset清空全部历史后旧运行对象只保存元数据', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('partial-clear', 650); + assert.equal(api.saveSession(original), true); + const stale = api.loadSession(original.id); + const current = api.loadSession(original.id); + current.messages = []; + current.codexAppThreadId = null; + api.sessionHistoryStore.markComplete(current, { reset: true }); + assert.equal(api.saveSession(current), true); + assert.deepEqual(api.loadSession(original.id, { full: true }).messages, []); + stale.messages[0].content = '清空后旧轮才返回的输出'; + stale.messages.push({ id: 'stale-output', role: 'assistant', content: '不能复活旧历史' }); + stale.model = '旧对象本次修改的模型'; + assert.equal(api.saveSession(stale), true); + const restored = api.loadSession(original.id, { full: true }); + assert.deepEqual(restored.messages, []); + assert.equal(restored.codexAppThreadId, null); + assert.equal(restored.model, stale.model); +}); + +check('尾部manifest提交失败不推进基线且重试保留并发元数据与完整前缀', dir => { + let failManifest = false; + const filesystem = new Proxy(fs, { get(target, key) { + if (key === 'renameSync') return (from, to) => { + if (failManifest && path.basename(to) === 'manifest.json') throw new Error('模拟清单提交失败'); + return target.renameSync(from, to); + }; + return target[key]; + } }); + const api = createServerHarness(dir, { fullHistory: false, fs: filesystem }); + const original = fixture('partial-retry', 650); + assert.equal(api.saveSession(original), true); + const pending = api.loadSession(original.id); + const baselineRevision = api.sessionHistoryStore.state(pending).revision; + const oldSnapshot = fs.readFileSync(path.join(dir, `${original.id}.json`), 'utf8'); + pending.messages[0].content = '失败重试后应保存的正文'; + pending.messages.push({ id: 'retry-new', role: 'assistant', content: '只提交一次的新消息' }); + failManifest = true; + assert.equal(api.saveSession(pending), false); + assert.equal(api.sessionHistoryStore.state(pending).revision, baselineRevision); + assert.equal(fs.readFileSync(path.join(dir, `${original.id}.json`), 'utf8'), oldSnapshot); + assert.deepEqual(api.loadSession(original.id, { full: true }).messages, original.messages); + failManifest = false; + const metadata = api.loadSession(original.id, { metadata: true }); + metadata.title = '失败后其他对象提交的标题'; + assert.equal(api.saveSession(metadata), true); + assert.equal(api.saveSession(pending), true); + assert.notEqual(api.sessionHistoryStore.state(pending).revision, baselineRevision); + const restored = api.loadSession(original.id, { full: true }); + assert.equal(restored.title, metadata.title); + assert.deepEqual(restored.messages.slice(0, 470), original.messages.slice(0, 470)); + assert.equal(restored.messages[470].content, '失败重试后应保存的正文'); + assert.equal(restored.messages.length, 651); + assert.equal(restored.messages.filter(message => message.id === 'retry-new').length, 1); +}); + +check('不完整归档分页总量和消息不计入持久化裁剪提示', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('incomplete-pages', 57); + original.messages.unshift({ id: 'persistence-notice', role: 'system', ccwebPersistenceNotice: true, content: '仅显示的旧截断提示' }); + original.historySnapshotBaseIndex = 500; + const expectedMessages = original.messages.slice(1); + assert.equal(api.saveSession(original), true); + const directPage = api.sessionHistoryStore.loadHistoryPage(original.id, 57, 24); + assert.equal(directPage.total, 57); + assert.equal(directPage.complete, false); + const chunks = []; + let cursor = 57; + while (cursor > 0) { + api.handleLoadHistoryPage(null, { sessionId: original.id, before: cursor, requestId: `archive-page-${cursor}` }); + const page = api.responses.at(-1); + assert.equal(page.type, 'session_history_chunk'); + assert.equal(page.historyTotal, 57); + assert.equal(page.historyAvailable, false); + assert.equal(page.messages.some(message => message.ccwebPersistenceNotice === true), false); + chunks.unshift(...page.messages); + assert(page.historyCursor < cursor, '分页游标必须前进'); + cursor = page.historyCursor; + } + assert.deepEqual(chunks, expectedMessages); +}); + +check('尾部追加ID撞未加载前缀时保留原始前缀并去重', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('partial-collision', 650); + assert.equal(api.saveSession(original), true); + const loaded = api.loadSession(original.id); + loaded.messages.push({ id: 'message-5', role: 'assistant', content: '碰撞内容不能覆盖未知前缀' }); + loaded.messages.push({ id: 'unique-tail', role: 'assistant', content: '真正新增的尾部内容' }); + assert.equal(api.saveSession(loaded), true); + const restored = api.loadSession(original.id, { full: true }); + assert.deepEqual(restored.messages.slice(0, 650), original.messages); + assert.equal(restored.messages.length, 651); + assert.equal(restored.messages.filter(message => message.id === 'message-5').length, 1); + assert.equal(restored.messages.at(-1).id, 'unique-tail'); +}); + +check('真实归档首屏及恢复仅返回12条并逐页读取650条完整有序历史', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('bounded-first-screen', 650); + assert.equal(api.saveSession(original), true); + const loaded = api.loadSession(original.id); + assert.equal(loaded.messages.length, 180); + const info = api.buildSessionInfoPayload(loaded); + assert.equal(info.type, 'session_info'); + assert.equal(info.messages.length, 12); + assert.deepEqual(info.messages, original.messages.slice(638)); + assert.equal(info.historyTotal, 650); + assert.equal(info.historyBaseIndex, 638); + assert.equal(info.historyCursor, 638); + assert.equal(info.historyTruncated, true); + assert.equal(info.historyAvailable, true); + api.handleLoadHistoryPage(null, { sessionId: original.id, before: 638, requestId: 'first-earlier-page' }); + const firstPage = api.responses.at(-1); + assert.equal(firstPage.type, 'session_history_chunk'); + assert.equal(firstPage.messages.length, 24); + assert.deepEqual(firstPage.messages, original.messages.slice(614, 638)); + assert.equal(firstPage.historyTotal, 650); + assert.equal(firstPage.historyBaseIndex, 614); + assert.equal(firstPage.historyCursor, 614); + const older = [...firstPage.messages]; + let cursor = firstPage.historyCursor; + while (cursor > 0) { + api.handleLoadHistoryPage(null, { sessionId: original.id, before: cursor, requestId: `complete-page-${cursor}` }); + const page = api.responses.at(-1); + assert.equal(page.type, 'session_history_chunk'); + assert.equal(page.historyTotal, 650); + assert.equal(page.historyAvailable, true); + assert(page.messages.length <= 24, '每页消息数必须有界'); + assert(page.historyCursor < cursor, '历史游标必须前进'); + older.unshift(...page.messages); + cursor = page.historyCursor; + } + const complete = [...older, ...info.messages]; + assert.deepEqual(complete, original.messages); + assert.equal(new Set(complete.map(message => message.id)).size, 650); + api.handleResumeSession({}, { sessionId: original.id, requestId: 'resume-bounded-history' }); + const resumed = api.responses.at(-1); + assert.equal(resumed.type, 'resume_session_result'); + assert.equal(resumed.messages.length, 12); + assert.deepEqual(resumed.messages, info.messages); + assert.equal(resumed.historyTotal, info.historyTotal); + assert.equal(resumed.historyBaseIndex, info.historyBaseIndex); + assert.equal(resumed.historyAvailable, true); + assert.equal(resumed.isRunning, false); +}); + +check('旧尾部对象协调新增深层元数据后原位修改不污染不可变基线', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('partial-nested-baseline', 650); + assert.equal(api.saveSession(original), true); + const streaming = api.loadSession(original.id); + const metadata = api.loadSession(original.id, { metadata: true }); + metadata.nested = { left: 1, right: 1 }; + assert.equal(api.saveSession(metadata), true); + assert.equal(api.saveSession(streaming), true); + assert.deepEqual(streaming.nested, { left: 1, right: 1 }); + const laterMetadata = api.loadSession(original.id, { metadata: true }); + laterMetadata.nested.right = 2; + assert.equal(api.saveSession(laterMetadata), true); + streaming.nested.left = 2; + assert.equal(api.saveSession(streaming), true); + const restored = api.loadSession(original.id, { full: true }); + assert.deepEqual(restored.nested, { left: 2, right: 2 }); + assert.deepEqual(streaming.nested, { left: 2, right: 2 }); + assert.deepEqual(restored.messages, original.messages); +}); + +check('中间窗口协调并发尾部提交后再次保存仍保留前后缀', dir => { + const api = createServerHarness(dir, { fullHistory: false }); + const original = fixture('middle-window-rebase', 650); + assert.equal(api.saveSession(original), true); + const middle = api.loadSession(original.id, { window: { start: 30, end: 33 } }); + const tail = api.loadSession(original.id); + const addedTail = { id: 'concurrent-tail', role: 'assistant', content: '另一运行对象追加的尾部' }; + tail.messages.push(addedTail); + assert.equal(api.saveSession(tail), true); + middle.messages[0].content = '中间窗口第一次更新'; + middle.messages.splice(1, 1); + assert.equal(api.saveSession(middle), true); + assert.deepEqual(middle.messages.map(message => message.id), ['message-30', 'message-32']); + assert.equal(middle.historySnapshotBaseIndex, 30); + middle.messages[0].content = '中间窗口第二次更新'; + middle.messages.splice(1, 0, { id: 'middle-second-insert', role: 'assistant', content: '第二次保存前插内容' }); + const expected = [...original.messages.slice(0, 30), ...middle.messages, ...original.messages.slice(33), addedTail]; + assert.equal(api.saveSession(middle), true); + assert.deepEqual(api.loadSession(original.id, { full: true }).messages, expected); +}); + if (failures.length > 0) { console.error(`完整历史回归失败:${failures.length} 项,已通过 ${checks} 项`); process.exitCode = 1; diff --git a/server.js b/server.js index 30611b3..e04a568 100644 --- a/server.js +++ b/server.js @@ -2377,12 +2377,14 @@ function cleanupExpiredAttachments() { function collectSessionAttachmentIds(session) { const ids = new Set(); - for (const message of Array.isArray(session?.messages) ? session.messages : []) { + const collect = (message) => { for (const attachment of Array.isArray(message?.attachments) ? message.attachments : []) { const id = sanitizeId(attachment?.id || ''); if (id) ids.add(id); } - } + }; + if (sessionHistoryStore.state(session)?.partial) sessionHistoryStore.forEachMessage(session.id, collect); + else for (const message of Array.isArray(session?.messages) ? session.messages : []) collect(message); return Array.from(ids); } @@ -4808,11 +4810,21 @@ function loadNativeSessionHistory(session) { } } -function resolveSessionHistory(session) { +function resolveSessionHistory(session, options = {}) { const persisted = Array.isArray(session?.messages) ? session.messages : []; const persistedMessages = persisted.filter((message) => !isPersistenceNoticeMessage(message)); const notice = persisted.some(isPersistenceNoticeMessage); const historyState = sessionHistoryStore.state(session); + if (historyState?.source === 'archive' && (Number.isFinite(options.count) || historyState.partial)) { + if (Number.isFinite(options.count)) { + const page = sessionHistoryStore.loadHistoryPage(session.id, options.before, options.count); + if (!page) throw new Error('完整会话归档已移除'); + return { ...page, messages: page.messages.map(normalizeHistoryMessage), source: 'archive', + available: page.complete, recoverable: false }; + } + return { messages: normalizeArchivedHistory(persistedMessages), total: historyState.total, + start: historyState.start, source: 'archive', available: historyState.complete, recoverable: false }; + } if (historyState?.source === 'archive' && historyState.complete) { return { messages: normalizeArchivedHistory(persistedMessages), source: 'archive', available: true, recoverable: false }; } @@ -6153,7 +6165,7 @@ function refreshSessionSnapshotCache(id) { if (fd !== undefined) fs.closeSync(fd); } if (cachedRevision === revision) return null; - const session = loadSession(id); + const session = loadSession(id, { count: SESSION_PERSIST_MAX_MESSAGES }); if (!session) return null; const result = buildSessionJsonForPersist({ ...session, historyArchiveRevision: revision }); try { @@ -6223,12 +6235,24 @@ function loadSessionMetaFromFile(filePath) { } } -function loadSession(id) { +function loadSession(id, options = {}) { const normalizedId = sanitizeId(id || ''); if (!normalizedId) return null; try { const filePath = sessionPath(normalizedId); - let session = sessionHistoryStore.load(normalizedId); + // 运行对象仅持有可修改的尾部;完整历史的可信基线是磁盘块引用。 + let window = options.window; + if (options.messageId) { + const index = sessionHistoryStore.messageIndex(normalizedId, options.messageId); + if (index >= 0) window = { start: index, end: index + 1 }; + } else if (options.promptId) { + const found = sessionHistoryStore.findMessage(normalizedId, message => message?.ccwebPrompt?.id === options.promptId); + if (found) window = { start: found.index, end: found.index + 1 }; + } + let session = window ? sessionHistoryStore.loadWindow(normalizedId, window.start, window.end) + : options.full === true ? sessionHistoryStore.load(normalizedId) + : options.metadata === true ? sessionHistoryStore.loadMetadata(normalizedId) + : sessionHistoryStore.loadTail(normalizedId, options.count ?? SESSION_PERSIST_MAX_MESSAGES, { maxBytes: 2 * 1024 * 1024 }); if (!session) { if (!fs.existsSync(filePath)) return null; session = safeReadSessionJson(filePath, SESSION_LOAD_MAX_BYTES, { sessionId: normalizedId }); @@ -6248,11 +6272,25 @@ function loadSession(id) { } } +function findSessionMessage(session, predicate) { + const messages = Array.isArray(session?.messages) ? session.messages : []; + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (predicate(messages[index])) return messages[index]; + } + const historyState = sessionHistoryStore.state(session); + if (!historyState?.partial) return null; + const found = sessionHistoryStore.findMessage(session.id, (message, index) => ( + (index < historyState.start || index >= historyState.end) && predicate(message) + )); + return found?.message || null; +} + function preserveNewerTaskTrackingForSessionSave(session, targetPath) { + // 可信归档的版本保护由同一次存储提交处理,避免保存前重复读取清单。 + if (sessionHistoryStore.state(session)?.source === 'archive') return; let persisted; try { - persisted = sessionHistoryStore.load(session.id); - if (!persisted && fs.existsSync(targetPath)) { + if (fs.existsSync(targetPath)) { persisted = safeReadSessionJson(targetPath, SESSION_LOAD_MAX_BYTES, { sessionId: session.id }); } } catch { @@ -6280,9 +6318,13 @@ function saveSession(session, options = {}) { const targetPath = sessionPath(session.id); try { if (!options.taskBoardMetadata) preserveNewerTaskTrackingForSessionSave(session, targetPath); - const committed = sessionHistoryStore.save(session); + const committed = sessionHistoryStore.save(session, { taskBoardMetadata: options.taskBoardMetadata === true }); + if (committed.taskTrackingPreserved) plog('INFO', 'task_tracking_preserved_on_session_save', { + sessionId: String(session.id || '').slice(0, 8), ...committed.taskTrackingPreserved, + }); try { - const result = buildSessionJsonForPersist({ ...committed.session, historyArchiveRevision: committed.revision }); + const preview = sessionHistoryStore.loadTail(session.id, SESSION_PERSIST_MAX_MESSAGES, { maxBytes: 2 * 1024 * 1024 }); + const result = buildSessionJsonForPersist({ ...preview, historyArchiveRevision: committed.revision }); writeFileAtomicSync(targetPath, result.json); if (result.guarded) { const lastAttempt = result.attempts[result.attempts.length - 1] || {}; @@ -6344,7 +6386,7 @@ function listTaskBoardSessionIds() { function saveTaskBoardSessionMetadata(session) { // 任务服务会克隆会话;重新读取可信档案,只提交它负责的元数据。 - const completeSession = loadSession(session?.id); + const completeSession = loadSession(session?.id, { metadata: true }); if (!completeSession) return false; if (Object.prototype.hasOwnProperty.call(session, 'taskTracking')) { completeSession.taskTracking = session.taskTracking; @@ -6357,7 +6399,7 @@ function saveTaskBoardSessionMetadata(session) { function createProductionTaskBoardService() { const dependencies = { - loadSession, + loadSession: id => loadSession(id, { metadata: true }), saveSession: saveTaskBoardSessionMetadata, listSessions: listTaskBoardSessionIds, loadStatusDefinitions: loadTaskBoardStatusConfig, @@ -6990,8 +7032,7 @@ function appendCodexAppRecoveryNotice(sessionId, content) { const session = loadSession(sessionId); if (!session || !isCodexAppSession(session)) return false; // 状态文件按设计会保留证据,重启会再次进入恢复分支;提示只保留一条。 - if (Array.isArray(session.messages) - && session.messages.some(message => message?.codexAppRecoveredPartial === true && message?.content === content)) { + if (findSessionMessage(session, message => message?.codexAppRecoveredPartial === true && message?.content === content)) { return true; } session.messages.push({ @@ -7006,8 +7047,7 @@ function appendCodexAppRecoveryNotice(sessionId, content) { } function hasCodexAppTurnMessage(session, turnKey) { - return Array.isArray(session?.messages) - && session.messages.some((message) => message?.codexAppTurnKey === turnKey); + return !!findSessionMessage(session, message => message?.codexAppTurnKey === turnKey); } function recoverCodexAppTurnState(sessionId) { @@ -8125,7 +8165,7 @@ function startCrossConversationReplyAutoRun(sourceSession, targetSession, replyT function hasProcessedCrossConversationReply(session, requestId) { const normalizedRequestId = String(requestId || '').trim(); if (!normalizedRequestId || !Array.isArray(session?.messages)) return false; - return session.messages.some((message) => ( + return !!findSessionMessage(session, (message) => ( message?.crossConversation?.replyToRequestId === normalizedRequestId && message?.crossConversation?.processed === true )); @@ -8342,7 +8382,8 @@ function getPendingCrossConversationReply(args = {}, sourceSessionId = '') { if (!pending || pending.sourceConversationId !== sourceId) { const sourceSession = loadSession(sourceId); if (hasProcessedCrossConversationReply(sourceSession, requestId)) { - const message = sourceSession.messages.find((item) => item?.crossConversation?.replyToRequestId === requestId); + const message = findSessionMessage(sourceSession, item => item?.crossConversation?.replyToRequestId === requestId + && item?.crossConversation?.processed === true); return { ok: true, requestId, @@ -8491,17 +8532,15 @@ function completeCrossConversationReply(requestId, entry = {}, targetSession = n } function latestAssistantMessageText(session) { - const messages = Array.isArray(session?.messages) ? session.messages : []; - for (let index = messages.length - 1; index >= 0; index -= 1) { - const message = messages[index]; - if (message?.role !== 'assistant') continue; - const text = extractCrossConversationReplyText(message.content || ''); - if (text.trim()) return text.trim(); - } - return ''; + const message = findSessionMessage(session, item => item?.role === 'assistant' + && extractCrossConversationReplyText(item.content || '').trim()); + return message ? extractCrossConversationReplyText(message.content || '').trim() : ''; } function assistantMessageCount(session) { + if (sessionHistoryStore.state(session)?.source === 'archive') { + return sessionHistoryStore.countMessages(session.id, 'assistant'); + } return Array.isArray(session?.messages) ? session.messages.filter((message) => message?.role === 'assistant').length : 0; @@ -8682,7 +8721,7 @@ function scriptSelectSemanticBranchApi(args = {}, sourceConversationId = '') { function insertJavascriptRunFailureNotice(sourceConversationId, entry = {}) { const session = loadSession(sourceConversationId); if (!session) return false; - if (Array.isArray(session.messages) && session.messages.some((message) => message?.scriptNotification?.runId === entry.runId)) return false; + if (findSessionMessage(session, message => message?.scriptNotification?.runId === entry.runId)) return false; const reason = entry.terminationReason || entry.signal || `exitCode=${entry.exitCode ?? 'unknown'}`; const stderr = String(entry.stderrPreview || '').trim(); const lines = [ @@ -9218,10 +9257,10 @@ function compactStartMessage(agent) { } function compactDoneMessage(agent) { - if (agent === 'codexapp') return '上下文压缩完成。已执行 Codex App 原生 /compact,下次继续在同一会话发送即可。'; + if (agent === 'codexapp') return '上下文压缩完成。已执行 Codex App 原生 /compact。'; return agent === 'codex' - ? '上下文压缩完成。已执行 Codex /compact,下次继续在同一会话发送即可。' - : '上下文压缩完成。已按 Claude Code 原生策略执行 /compact,下次继续在同一会话发送即可。'; + ? '上下文压缩完成。已执行 Codex /compact。' + : '上下文压缩完成。已按 Claude Code 原生策略执行 /compact。'; } function initStartMessage(agent) { @@ -9339,8 +9378,7 @@ function cliRunKey(sessionId, entry = {}) { } function hasCliRunMessage(session, runKey) { - return Array.isArray(session?.messages) - && session.messages.some(message => message?.role === 'assistant' && message.runKey === runKey); + return !!findSessionMessage(session, message => message?.role === 'assistant' && message.runKey === runKey); } function retryTailText(value, maxChars) { @@ -11534,8 +11572,10 @@ function setCurrentConversationTitle(args = {}, sourceSessionId = '') { const changed = previousTitle !== normalizedTitle; let titleEvent = null; if (changed) { - const messageIndex = Array.isArray(session.messages) ? session.messages.length : 0; - const anchorMessageIndex = findLatestUserMessageIndex(session.messages); + const historyBaseIndex = Number(session.historySnapshotBaseIndex) || 0; + const messageIndex = historyBaseIndex + (Array.isArray(session.messages) ? session.messages.length : 0); + const localAnchor = findLatestUserMessageIndex(session.messages); + const anchorMessageIndex = localAnchor === null ? null : historyBaseIndex + localAnchor; titleEvent = { title: normalizedTitle, changedAt: new Date().toISOString(), @@ -11611,7 +11651,7 @@ function resolveBranchSource(args = {}) { const sourceSessionId = sanitizeId(args.branchSourceSessionId || args.sourceSessionId || ''); if (!sourceSessionId) return { ok: true, sourceSession: null }; - const sourceSession = loadSession(sourceSessionId); + const sourceSession = loadSession(sourceSessionId, { full: true }); if (!sourceSession) { return mcpToolError('branch_source_not_found', '来源会话不存在,无法创建分支。', { sourceSessionId }); } @@ -11721,7 +11761,7 @@ function createPersistentConversationSession(args = {}, options = {}) { function buildSessionInfoPayload(session) { const waitState = crossConversationWaitState(session.id); const history = typeof resolveSessionHistory === 'function' - ? resolveSessionHistory(session) + ? resolveSessionHistory(session, { count: INITIAL_HISTORY_COUNT }) : { messages: session.messages || [], source: 'snapshot', available: true }; const messages = sanitizeMessagesForTransport(history.messages || []); const taskTracking = taskTrackingSnapshotForSession(session); @@ -11740,8 +11780,10 @@ function buildSessionInfoPayload(session) { totalCost: session.totalCost || 0, totalUsage: session.totalUsage || { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }, updated: session.updated, - historyTotal: messages.length, - historyBaseIndex: 0, + historyTotal: history.total ?? messages.length, + historyBaseIndex: history.start || 0, + historyCursor: history.start || 0, + historyTruncated: (history.start || 0) > 0, hasUnread: false, historyPending: false, isRunning: false, @@ -11822,7 +11864,7 @@ function handleNewSession(ws, msg) { function handleLoadHistoryPage(ws, msg = {}) { const sessionId = sanitizeId(msg.sessionId || ''); - const session = loadSession(sessionId); + const session = loadSession(sessionId, { metadata: true }); if (!session) { return wsSend(ws, attachClientRequestId({ type: 'error', @@ -11831,11 +11873,22 @@ function handleLoadHistoryPage(ws, msg = {}) { message: 'Session not found', }, msg)); } + const requestedBefore = Number.parseInt(String(msg.before || ''), 10); + const archived = sessionHistoryStore.state(session); + if (archived?.source === 'archive') { + const before = Number.isFinite(requestedBefore) ? requestedBefore : Math.max(0, archived.total - INITIAL_HISTORY_COUNT); + const page = sessionHistoryStore.loadHistoryPage(session.id, before, HISTORY_CHUNK_SIZE); + return wsSend(ws, attachClientRequestId({ + type: 'session_history_chunk', sessionId: session.id, + messages: sanitizeMessagesForTransport(page.messages.map(normalizeHistoryMessage)), remaining: 0, + historyCursor: page.start, historyBaseIndex: page.start, historyTruncated: page.start > 0, + historyTotal: page.total, historyAvailable: page.complete, historySource: 'archive', + }, msg)); + } const history = typeof resolveSessionHistory === 'function' ? resolveSessionHistory(session) : { messages: session.messages || [], source: 'snapshot', available: true }; - const list = sanitizeMessagesForTransport(history.messages); - const requestedBefore = Number.parseInt(String(msg.before || ''), 10); + const list = history.messages; const before = Number.isFinite(requestedBefore) ? Math.max(0, Math.min(list.length, requestedBefore)) : Math.max(0, list.length - INITIAL_HISTORY_COUNT); @@ -11844,7 +11897,7 @@ function handleLoadHistoryPage(ws, msg = {}) { wsSend(ws, attachClientRequestId({ type: 'session_history_chunk', sessionId: session.id, - messages: list.slice(start, end), + messages: sanitizeMessagesForTransport(list.slice(start, end)), remaining: 0, historyCursor: start, historyBaseIndex: start, @@ -11866,9 +11919,9 @@ function handleMigrateSessionHistory(ws, msg = {}) { if (!sessionId || !fs.existsSync(filePath)) return respond({ ok: false, code: 'session_not_found' }); try { const original = fs.readFileSync(filePath, 'utf8'); - const session = loadSession(sessionId); + const session = loadSession(sessionId, { full: true }); if (!session) return respond({ ok: false, code: 'session_not_found' }); - if (sessionHistoryStore.state(session)?.complete === true || !session.messages.some(isPersistenceNoticeMessage)) { + if (sessionHistoryStore.state(session)?.complete === true) { return respond({ ok: true, changed: false, messageCount: session.messages.length }); } const snapshot = session.messages.filter((message) => !isPersistenceNoticeMessage(message)); @@ -11887,7 +11940,7 @@ function handleMigrateSessionHistory(ws, msg = {}) { session.messages = merged.messages; sessionHistoryStore.markComplete(session); if (!saveSession(session)) return respond({ ok: false, code: 'history_archive_save_failed', backupPath }); - const reloaded = loadSession(sessionId); + const reloaded = loadSession(sessionId, { full: true }); if (!reloaded || reloaded.messages.length !== merged.messages.length) { return respond({ ok: false, code: 'history_reload_failed', backupPath }); } @@ -11986,7 +12039,7 @@ function attachActiveRuntimeToWs(ws, sessionId, source = {}) { function handleResumeSession(ws, msg = {}) { const sessionId = sanitizeId(msg.sessionId || ''); - const session = loadSession(sessionId); + const session = loadSession(sessionId, { metadata: true }); if (!session) { return wsSend(ws, attachClientRequestId({ type: 'error', @@ -12001,7 +12054,7 @@ function handleResumeSession(ws, msg = {}) { // 重连只发送 resume_generating 会让前端无法补回断线期间丢掉的历史气泡; // 同时带上最近历史,前端可按消息索引幂等补齐而不会重绘整个会话。 const history = typeof resolveSessionHistory === 'function' - ? resolveSessionHistory(session) + ? resolveSessionHistory(session, { count: INITIAL_HISTORY_COUNT }) : { messages: session.messages || [], source: 'snapshot', available: true }; const historyMessages = sanitizeMessagesForTransport(history.messages || []); const recentMessages = historyMessages.slice(-INITIAL_HISTORY_COUNT); @@ -12016,8 +12069,8 @@ function handleResumeSession(ws, msg = {}) { isRunning: isSessionRunning(sessionId), attached: hasActiveRuntime, messages: recentMessages, - historyTotal: historyMessages.length, - historyBaseIndex: Math.max(0, historyMessages.length - recentMessages.length), + historyTotal: history.total ?? historyMessages.length, + historyBaseIndex: (history.start || 0) + Math.max(0, historyMessages.length - recentMessages.length), historyAvailable: history.available !== false, historySource: history.source || 'snapshot', }, msg)); @@ -12028,7 +12081,7 @@ function handleResumeSession(ws, msg = {}) { function handleLoadSession(ws, msg) { const sessionId = sanitizeId(typeof msg === 'string' ? msg : msg?.sessionId); reconcilePendingCrossConversationReplies(); - const session = loadSession(sessionId); + const session = loadSession(sessionId, { metadata: true }); if (!session) { return wsSend(ws, attachClientRequestId({ type: 'error', @@ -12038,7 +12091,7 @@ function handleLoadSession(ws, msg) { }, msg)); } flushPendingCrossConversationReplies(sessionId); - const refreshedSession = loadSession(sessionId) || session; + const refreshedSession = loadSession(sessionId, { metadata: true }) || session; if (getSessionAgent(refreshedSession) === 'claude' && !refreshedSession.cwd && refreshedSession.claudeSessionId) { const localMeta = resolveClaudeSessionLocalMeta(refreshedSession.claudeSessionId); if (localMeta?.cwd) { @@ -12050,20 +12103,27 @@ function handleLoadSession(ws, msg) { const requestedTargetMessageIndex = Number.parseInt(String( typeof msg === 'object' ? msg?.targetMessageIndex ?? '' : '', ), 10); - const history = typeof resolveSessionHistory === 'function' - ? resolveSessionHistory(refreshedSession) + let history = typeof resolveSessionHistory === 'function' + ? resolveSessionHistory(refreshedSession, { count: INITIAL_HISTORY_COUNT }) : { messages: refreshedSession.messages || [], source: 'snapshot', available: true }; + const historyTotal = history.total ?? history.messages.length; const targetMessageIndex = Number.isFinite(requestedTargetMessageIndex) - ? Math.max(0, Math.min(history.messages.length - 1, requestedTargetMessageIndex)) + ? Math.max(0, Math.min(historyTotal - 1, requestedTargetMessageIndex)) : null; - const recentHistoryBaseIndex = Math.max(0, history.messages.length - INITIAL_HISTORY_COUNT); + const recentHistoryBaseIndex = Math.max(0, historyTotal - INITIAL_HISTORY_COUNT); const targetPrefetchChunks = targetMessageIndex !== null && targetMessageIndex < recentHistoryBaseIndex ? Math.ceil((recentHistoryBaseIndex - targetMessageIndex) / HISTORY_CHUNK_SIZE) : 0; - const { recentMessages, olderChunks, historyRemaining, historyBuffered } = splitHistoryMessages( + const prefetchChunks = targetMessageIndex === null ? 0 : Math.min(HISTORY_MAX_CHUNKS_PER_LOAD, Math.max(HISTORY_PREFETCH_CHUNKS, targetPrefetchChunks)); + if (prefetchChunks > 0 && sessionHistoryStore.state(refreshedSession)?.source === 'archive') { + history = resolveSessionHistory(refreshedSession, { count: INITIAL_HISTORY_COUNT + prefetchChunks * HISTORY_CHUNK_SIZE }); + } + const split = splitHistoryMessages( history.messages, - { prefetchChunks: targetMessageIndex === null ? 0 : Math.max(HISTORY_PREFETCH_CHUNKS, targetPrefetchChunks) }, + { prefetchChunks }, ); + const { recentMessages, olderChunks, historyBuffered } = split; + const historyRemaining = (history.start || 0) + split.historyRemaining; const effectiveCwd = refreshedSession.cwd || activeProcesses.get(sessionId)?.cwd || activeCodexAppTurns.get(sessionId)?.cwd || null; const waitState = crossConversationWaitState(sessionId); const taskTracking = taskTrackingSnapshotForSession(refreshedSession); @@ -12095,10 +12155,10 @@ function handleLoadSession(ws, msg) { cwd: effectiveCwd, totalCost: session.totalCost || 0, totalUsage: session.totalUsage || null, - historyTotal: history.messages.length, + historyTotal, historyBuffered, historyCursor: historyRemaining, - historyBaseIndex: Math.max(0, history.messages.length - recentMessages.length), + historyBaseIndex: Math.max(0, historyTotal - recentMessages.length), historyTruncated: historyRemaining > 0, historyPending: olderChunks.length > 0, historyAvailable: history.available, @@ -12116,7 +12176,7 @@ function handleLoadSession(ws, msg) { }, msg)); if (olderChunks.length > 0) { - let chunkEnd = Math.max(0, history.messages.length - recentMessages.length); + let chunkEnd = Math.max(0, historyTotal - recentMessages.length); olderChunks.forEach((chunk, index) => { const chunkStart = Math.max(0, chunkEnd - chunk.length); wsSend(ws, attachClientRequestId({ @@ -12127,7 +12187,7 @@ function handleLoadSession(ws, msg) { historyCursor: index === olderChunks.length - 1 ? historyRemaining : null, historyBaseIndex: chunkStart, historyTruncated: historyRemaining > 0, - historyTotal: history.messages.length, + historyTotal, historyAvailable: history.available, historySource: history.source, }, msg)); @@ -12740,7 +12800,7 @@ function handleMessage(ws, msg, options = {}) { if (ws && (options.emitUserMessage || msg.codexAppSteer === true) && persistedUserMessage) { wsSend(ws, { type: 'session_message', sessionId: currentSessionId, message: persistedUserMessage, - messageIndex: session.messages.findIndex((message) => message.id === persistedUserMessage.id), + messageIndex: (Number(session.historySnapshotBaseIndex) || 0) + session.messages.findIndex((message) => message.id === persistedUserMessage.id), }); } sendSessionList(ws); @@ -12758,7 +12818,7 @@ function handleMessage(ws, msg, options = {}) { wsSend(ws, { type: 'codex_app_steer_status', sessionId: currentSessionId, clientMessageId: msg.clientMessageId, status: 'sent', message: '已转为新对话', - messageIndex: session.messages.findIndex((message) => message.id === persistedUserMessage?.id), + messageIndex: (Number(session.historySnapshotBaseIndex) || 0) + session.messages.findIndex((message) => message.id === persistedUserMessage?.id), }); wsSend(ws, buildCodexAppResumePayload(currentSessionId, activeCodexAppTurns.get(currentSessionId))); } @@ -14165,7 +14225,7 @@ function handleCcwebPromptUserResponse(ws, msg = {}) { return fail('session_running', '当前会话正在运行,暂不能提交 ccweb 表单答案。'); } - const session = loadSession(sessionId); + const session = loadSession(sessionId, { promptId }); if (!session) return fail('session_not_found', '会话不存在。'); const found = findCcwebPromptMessage(session, promptId); @@ -14227,7 +14287,7 @@ function handleCcwebPromptUserDismiss(ws, msg = {}) { if (!sessionId) return fail('missing_session_id', '缺少会话 ID。'); if (!promptId) return fail('missing_prompt_id', '缺少 promptId。'); - const session = loadSession(sessionId); + const session = loadSession(sessionId, { promptId }); if (!session) return fail('session_not_found', '会话不存在。'); const found = removeCcwebPromptMessage(session, promptId); if (!found) return fail('prompt_not_found', 'ccweb 表单不存在或已被清理。', { promptId }); @@ -15217,7 +15277,7 @@ function handleCodexAppTurnComplete(sessionId, options = {}) { } const assistantMessage = session?.messages?.find((message) => message.codexAppTurnKey === turnKey); const assistantMessageIndex = assistantMessage - ? resolveSessionHistory(session).messages.findIndex((message) => message.id === assistantMessage.id) + ? sessionHistoryStore.messageIndex(sessionId, assistantMessage.id) : -1; wsSend(entry.ws, { type: 'done', sessionId, costUsd: null, goalActive, @@ -15266,7 +15326,7 @@ function isCodexAppNoActiveTurnError(err) { function rollbackPersistedCodexAppSteerMessage(sessionId, userMessageId) { const id = String(userMessageId || '').trim(); if (!sessionId || !id) return false; - const session = loadSession(sessionId); + const session = loadSession(sessionId, { messageId: id }); if (!session || !Array.isArray(session.messages)) return false; const nextMessages = session.messages.filter((message) => !(message?.role === 'user' && message.id === id)); if (nextMessages.length === session.messages.length) return false; @@ -15294,7 +15354,8 @@ function reorderStaleCodexAppTurnMessages(sessionId, entry, userMessageId) { session.updated = new Date().toISOString(); if (!saveSession(session)) return false; wsSend(entry.ws, { - type: 'session_message', sessionId, message: sanitizeMessageForTransport(assistantMessage), messageIndex: userIndex, + type: 'session_message', sessionId, message: sanitizeMessageForTransport(assistantMessage), + messageIndex: (Number(session.historySnapshotBaseIndex) || 0) + userIndex, }); return true; } @@ -15318,7 +15379,8 @@ function handleCodexAppSteerMessage(ws, msg, options = {}) { if (clientMessageId) payload.clientMessageId = clientMessageId; if (status === 'sent') { const savedSession = loadSession(sessionId); - payload.messageIndex = savedSession?.messages?.findIndex((message) => message.id === userMessageId); + const index = savedSession?.messages?.findIndex((message) => message.id === userMessageId); + payload.messageIndex = index >= 0 ? (Number(savedSession.historySnapshotBaseIndex) || 0) + index : index; } wsSend(targetWs, payload); }; @@ -15886,6 +15948,10 @@ function handleListNativeSessions(ws) { function prepareNativeHistoryImport(existingSession, importedSession) { if (!existingSession) return normalizeSession(importedSession); + if (sessionHistoryStore.state(existingSession)?.partial) { + existingSession = loadSession(existingSession.id, { full: true }); + if (!existingSession) return null; + } const snapshot = existingSession.messages.filter(message => !isPersistenceNoticeMessage(message)); const merged = mergeNativeHistoryWithSnapshot(existingSession, snapshot, importedSession.messages); if (!merged.confirmed) return null;