修复完整历史与运行态内存放大并重建发布包
This commit is contained in:
shiyue
262
scripts/json-persistence-unit.js
Normal file
262
scripts/json-persistence-unit.js
Normal file
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { readJsonSync, writeJsonSync, cloneJsonValue, jsonValueEqual } = require('../lib/json-persistence');
|
||||
const { createSessionHistoryStore } = require('../lib/session-history-store');
|
||||
|
||||
const BLOCK_BYTES = 64 * 1024;
|
||||
|
||||
function nativeJson(value) {
|
||||
return JSON.stringify(value, (key, item) => {
|
||||
if (typeof item === 'bigint') return String(item);
|
||||
if (item instanceof Map) return Object.fromEntries(item);
|
||||
if (item instanceof Set) return [...item];
|
||||
return item;
|
||||
});
|
||||
}
|
||||
|
||||
function temporaryRoot() {
|
||||
if (process.platform === 'linux' && typeof fs.statfsSync === 'function') {
|
||||
try {
|
||||
const stat = fs.statfsSync('/dev/shm');
|
||||
if (Number(stat.bavail) * Number(stat.bsize) >= 32 * 1024 * 1024) return '/dev/shm';
|
||||
} catch {}
|
||||
}
|
||||
return os.tmpdir();
|
||||
}
|
||||
|
||||
function runTests() {
|
||||
const dir = fs.mkdtempSync(path.join(temporaryRoot(), 'ccweb-json-persistence-'));
|
||||
const failures = [];
|
||||
let checks = 0;
|
||||
let sequence = 0;
|
||||
function check(name, callback) {
|
||||
try {
|
||||
callback();
|
||||
checks += 1;
|
||||
console.log(`通过：${name}`);
|
||||
} catch (error) {
|
||||
failures.push(name);
|
||||
console.error(`失败：${name}：${error.stack}`);
|
||||
}
|
||||
}
|
||||
function rawFile(json) {
|
||||
const filename = path.join(dir, `input-${++sequence}.json`);
|
||||
fs.writeFileSync(filename, json);
|
||||
return filename;
|
||||
}
|
||||
function encode(value, filesystem = fs) {
|
||||
const filename = path.join(dir, `output-${++sequence}.json`);
|
||||
const fd = fs.openSync(filename, 'wx', 0o600);
|
||||
try { writeJsonSync(filesystem, fd, value); } finally { fs.closeSync(fd); }
|
||||
return { filename, json: fs.readFileSync(filename, 'utf8') };
|
||||
}
|
||||
function assertNative(value) {
|
||||
const expectedJson = nativeJson(value);
|
||||
const expected = JSON.parse(expectedJson);
|
||||
const output = encode(value);
|
||||
assert.equal(output.json, expectedJson);
|
||||
assert.deepEqual(readJsonSync(fs, output.filename), expected);
|
||||
assert.deepEqual(cloneJsonValue(value), expected);
|
||||
}
|
||||
|
||||
try {
|
||||
check('JSON基本类型、数值边界与控制字符与原生持久化语义一致', () => {
|
||||
const samples = [null, true, false, 0, -0, 1.25, 1e-100, 1e100, NaN, Infinity, -Infinity,
|
||||
'', '中文🙂\u0000\b\f\n\r\t\\"/\u2028\u2029', '\ud800\udfff', 9007199254740993n];
|
||||
samples.forEach(assertNative);
|
||||
assertNative({ values: samples, boxed: [new Number(4), new String('包装文字'), new Boolean(false)] });
|
||||
});
|
||||
|
||||
check('BigInt、Map、Set、日期与toJSON保持项目转换及键参数', () => {
|
||||
function fixture() {
|
||||
return {
|
||||
bigint: 123456789012345678901234567890n,
|
||||
map: new Map([['a', 1n], ['__proto__', { own: true }], ['set', new Set([2, 3])]]),
|
||||
set: new Set(['x', 4n]),
|
||||
date: new Date('2026-10-14T01:02:03.456Z'),
|
||||
custom: { toJSON(key) { return { key, value: 9n }; } },
|
||||
array: [{ toJSON(key) { return `数组键 ${key}`; } }],
|
||||
};
|
||||
}
|
||||
assertNative(fixture());
|
||||
assertNative({ toJSON(key) { return { rootKey: key, value: new Map([['ok', true]]) }; } });
|
||||
for (const action of ['write', 'clone']) {
|
||||
let reads = 0;
|
||||
let calls = 0;
|
||||
const custom = {
|
||||
get toJSON() {
|
||||
reads += 1;
|
||||
return function (key) { calls += 1; assert.equal(this, custom); return { key, value: '原值' }; };
|
||||
},
|
||||
};
|
||||
const result = action === 'write' ? JSON.parse(encode({ custom }).json) : cloneJsonValue({ custom });
|
||||
assert.deepEqual(result, { custom: { key: 'custom', value: '原值' } });
|
||||
assert.equal(reads, 1, 'toJSON属性只能读取一次');
|
||||
assert.equal(calls, 1, 'toJSON只能执行一次');
|
||||
}
|
||||
});
|
||||
|
||||
check('对象省略与数组空洞置null、重复引用及危险属性名完整保存', () => {
|
||||
const shared = { stable: '原值', nested: ['完整内容'] };
|
||||
const object = JSON.parse('{"__proto__":{"value":"自己的属性"},"constructor":"构造键","prototype":"普通键"}');
|
||||
object.missing = undefined;
|
||||
object.callback = () => 1;
|
||||
object.symbol = Symbol('省略');
|
||||
object.array = [undefined, () => 1, Symbol('数组位置'), , shared, shared];
|
||||
object.left = shared;
|
||||
object.right = shared;
|
||||
assertNative(object);
|
||||
const cloned = cloneJsonValue(object);
|
||||
assert.equal(Object.hasOwn(cloned, '__proto__'), true);
|
||||
assert.deepEqual(cloned.__proto__, { value: '自己的属性' });
|
||||
assert.equal(Object.hasOwn(cloned, 'missing'), false);
|
||||
assert.equal(cloned.array.length, 6);
|
||||
assert.deepEqual(cloned.array.slice(0, 4), [null, null, null, null]);
|
||||
});
|
||||
|
||||
check('容器复制隔离原对象并保留不可变字符串值', () => {
|
||||
const text = '必须保留完整正文🙂'.repeat(20000);
|
||||
const source = { text, nested: { rows: [{ value: 1 }] } };
|
||||
const cloned = cloneJsonValue(source);
|
||||
assert.notEqual(cloned, source);
|
||||
assert.notEqual(cloned.nested, source.nested);
|
||||
assert.notEqual(cloned.nested.rows, source.nested.rows);
|
||||
assert.notEqual(cloned.nested.rows[0], source.nested.rows[0]);
|
||||
assert.equal(cloned.text, text);
|
||||
cloned.nested.rows[0].value = 2;
|
||||
assert.equal(source.nested.rows[0].value, 1);
|
||||
assert.equal(jsonValueEqual(source, cloneJsonValue(source)), true);
|
||||
assert.equal(jsonValueEqual(source, cloned), false);
|
||||
assert.equal(jsonValueEqual(cloneJsonValue({ value: NaN, omitted: undefined }), { value: null }), true);
|
||||
assert.equal(jsonValueEqual(cloneJsonValue([undefined, , Infinity]), [null, null, null]), true);
|
||||
assert.equal(jsonValueEqual(cloneJsonValue(10n), '10'), true);
|
||||
assert.equal(jsonValueEqual(cloneJsonValue(new Map([['value', 1]])), { value: 1 }), true);
|
||||
assert.equal(jsonValueEqual({ left: 1, right: { value: 2 } }, { right: { value: 2 }, left: 1 }), true);
|
||||
assert.equal(jsonValueEqual(undefined, undefined), true);
|
||||
assert.equal(jsonValueEqual(undefined, null), false);
|
||||
});
|
||||
|
||||
check('顶层不可持久化值拒绝提交且循环引用失败', () => {
|
||||
for (const value of [undefined, () => 1, Symbol('顶层')]) {
|
||||
assert.equal(cloneJsonValue(value), undefined);
|
||||
assert.throws(() => encode(value), TypeError);
|
||||
}
|
||||
const cyclic = { text: '已经写出的字段' };
|
||||
cyclic.self = cyclic;
|
||||
assert.throws(() => encode(cyclic), TypeError);
|
||||
assert.throws(() => cloneJsonValue(cyclic), TypeError);
|
||||
});
|
||||
|
||||
check('64KiB边界跨UTF-8、JSON转义和长字符串时完整读取', () => {
|
||||
const prefix = '{"text":"';
|
||||
const utf8Json = prefix + 'a'.repeat(BLOCK_BYTES - Buffer.byteLength(prefix) - 1) + '🙂中文","tail":{"value":true}}';
|
||||
const escapedJson = prefix + 'b'.repeat(BLOCK_BYTES - Buffer.byteLength(prefix) - 1) + '\\uD83D\\uDE42\\n\\"\\\\","tail":[1,2,3]}';
|
||||
for (const json of [utf8Json, escapedJson, '\r\n ' + '[1e-3,true,null,{"duplicate":1,"duplicate":2}]' + '\t ']) {
|
||||
const filename = rawFile(json);
|
||||
let maxRead = 0;
|
||||
const filesystem = new Proxy(fs, { get(target, key) {
|
||||
if (key === 'readFileSync') return () => { throw new Error('流式读取禁止整文件readFileSync'); };
|
||||
if (key === 'readSync') return (fd, buffer, offset, length, position) => {
|
||||
maxRead = Math.max(maxRead, length);
|
||||
assert(length <= BLOCK_BYTES);
|
||||
return target.readSync(fd, buffer, offset, length, position);
|
||||
};
|
||||
return target[key];
|
||||
} });
|
||||
assert.deepEqual(readJsonSync(filesystem, filename), JSON.parse(json));
|
||||
assert(maxRead > 0 && maxRead <= BLOCK_BYTES);
|
||||
}
|
||||
});
|
||||
|
||||
check('多次短读不丢UTF-8字节和转义，读取错误关闭文件', () => {
|
||||
const json = '{"text":"中文🙂\\n\\uD83D\\uDE42\\\\\\\"","number":-2.5e+3,"rows":[true,null,{"own":1}]}';
|
||||
const filename = rawFile(json);
|
||||
let opened = 0;
|
||||
let closed = 0;
|
||||
let failRead = false;
|
||||
const filesystem = new Proxy(fs, { get(target, key) {
|
||||
if (key === 'openSync') return (...args) => { opened += 1; return target.openSync(...args); };
|
||||
if (key === 'closeSync') return (...args) => { closed += 1; return target.closeSync(...args); };
|
||||
if (key === 'readSync') return (fd, buffer, offset, length, position) => {
|
||||
if (failRead) throw new Error('模拟读取中断');
|
||||
return target.readSync(fd, buffer, offset, Math.min(length, 3), position);
|
||||
};
|
||||
return target[key];
|
||||
} });
|
||||
assert.deepEqual(readJsonSync(filesystem, filename), JSON.parse(json));
|
||||
failRead = true;
|
||||
assert.throws(() => readJsonSync(filesystem, filename), /模拟读取中断/);
|
||||
assert.equal(closed, opened, '读取成功或失败均应关闭文件');
|
||||
});
|
||||
|
||||
check('非法JSON、截断值、尾随内容与控制字符均拒绝', () => {
|
||||
const invalid = ['', ' ', '{', '[1,', '{"a":}', '{"a":1,}', '[1,]', 'true false', 'nullx', '+1', '01', '1.', '1e',
|
||||
'NaN', 'Infinity', 'undefined', '\ufeff{}', '{unquoted:1}', '"未闭合', '"\\x01"', '"\\u12"', '"\n"', '"\u0000"'];
|
||||
for (const json of invalid) {
|
||||
assert.throws(() => JSON.parse(json));
|
||||
assert.throws(() => readJsonSync(fs, rawFile(json)), `非法JSON必须拒绝：${JSON.stringify(json)}`);
|
||||
}
|
||||
for (const json of ['1e400', '-0', '[1,2,3]', '{"__proto__":{"own":true},"constructor":4}']) {
|
||||
assert.deepEqual(readJsonSync(fs, rawFile(json)), JSON.parse(json));
|
||||
}
|
||||
});
|
||||
|
||||
check('分段编码每次最多64KiB且正确处理短写', () => {
|
||||
const value = { text: '中文🙂\\"\n\u0000'.repeat(25000), nested: { lines: ['保持原值'] } };
|
||||
let maxWrite = 0;
|
||||
let calls = 0;
|
||||
const filesystem = new Proxy(fs, { get(target, key) {
|
||||
if (key === 'writeFileSync') return () => { throw new Error('流式编码禁止整文件writeFileSync'); };
|
||||
if (key === 'writeSync') return (fd, data, ...args) => {
|
||||
calls += 1;
|
||||
const bytes = typeof data === 'string' ? Buffer.byteLength(data, typeof args[1] === 'string' ? args[1] : 'utf8') : args[1] ?? data.length;
|
||||
maxWrite = Math.max(maxWrite, bytes);
|
||||
assert(bytes <= BLOCK_BYTES, `单次编码写入 ${bytes} 字节超出64KiB`);
|
||||
// 固定短写发生在字节缓冲层，不能以UTF-16字符数量当作已写字节数。
|
||||
if (typeof data === 'string') {
|
||||
const encoded = Buffer.from(data, typeof args[1] === 'string' ? args[1] : 'utf8');
|
||||
return target.writeSync(fd, encoded, 0, Math.min(11, encoded.length), args[0] ?? null);
|
||||
}
|
||||
return target.writeSync(fd, data, args[0] ?? 0, Math.min(11, bytes), args[2] ?? null);
|
||||
};
|
||||
return target[key];
|
||||
} });
|
||||
const output = encode(value, filesystem);
|
||||
assert.equal(output.json, nativeJson(value));
|
||||
assert(calls > 1);
|
||||
assert(maxWrite <= BLOCK_BYTES);
|
||||
});
|
||||
|
||||
check('JSON中途编码异常保留原子归档且无临时文件残留', () => {
|
||||
const store = createSessionHistoryStore({ sessionsDir: dir });
|
||||
const id = 'json-atomic-failure';
|
||||
const original = { version: 1, agent: 'codexapp', sessionId: id, fullText: '上次完整运行态', toolCalls: [] };
|
||||
store.saveTurn(id, original);
|
||||
const filename = path.join(dir, '_history', '_turns', `${id}.json`);
|
||||
const oldBytes = fs.readFileSync(filename);
|
||||
const cyclic = { ...original, fullText: '新正文'.repeat(100000) };
|
||||
cyclic.self = cyclic;
|
||||
assert.throws(() => store.saveTurn(id, cyclic), TypeError);
|
||||
const customFailure = { ...original, fullText: '已编码新正文'.repeat(100000), tail: { toJSON() { throw new Error('模拟toJSON异常'); } } };
|
||||
assert.throws(() => store.saveTurn(id, customFailure), /模拟toJSON异常/);
|
||||
assert.deepEqual(fs.readFileSync(filename), oldBytes);
|
||||
assert.deepEqual(store.loadTurn(id), original);
|
||||
assert.equal(fs.readdirSync(path.dirname(filename)).some(name => name.endsWith('.tmp')), false);
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
if (failures.length) {
|
||||
console.error(`JSON持久化回归失败：${failures.length} 项，已通过 ${checks} 项`);
|
||||
process.exitCode = 1;
|
||||
} else console.log(`JSON持久化回归通过：${checks} 项`);
|
||||
return { checks, failures };
|
||||
}
|
||||
|
||||
if (require.main === module) runTests();
|
||||
module.exports = { runTests };
|
||||
Reference in New Issue
Block a user