239 lines
11 KiB
JavaScript
239 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
||
'use strict';
|
||
|
||
const assert = require('node:assert/strict');
|
||
const fs = require('node:fs');
|
||
const os = require('node:os');
|
||
const path = require('node:path');
|
||
const crypto = require('node:crypto');
|
||
const { spawnSync } = require('node:child_process');
|
||
const { createSessionHistoryStore } = require('../lib/session-history-store');
|
||
|
||
const ID = 'synthetic-runtime-memory';
|
||
const TARGET_BYTES = 80 * 1024 * 1024;
|
||
const BLOCK_BYTES = 64 * 1024;
|
||
const LARGE_PIECE = 'a'.repeat(32 * 1024 - 16) + '\\引号"\n';
|
||
const ASCII_PIECE = 'b'.repeat(32 * 1024 - 16) + '\\quote"\n';
|
||
const SPECIAL = JSON.parse('{"text":"中文🙂\\n\\\\\\\"","__proto__":{"own":"保留值"},"constructor":"普通属性"}');
|
||
const startedAt = Date.now();
|
||
|
||
function stage(message) {
|
||
process.stderr.write(`[运行态内存回归 +${((Date.now() - startedAt) / 1000).toFixed(2)} 秒] ${message}\n`);
|
||
}
|
||
|
||
function temporaryRoot() {
|
||
// 此项测编码和解析的内存峰值；慢盘fsync会耗尽60秒预算，优先真实tmpfs。
|
||
if (process.platform === 'linux' && typeof fs.statfsSync === 'function') {
|
||
try {
|
||
const stat = fs.statfsSync('/dev/shm');
|
||
fs.accessSync('/dev/shm', fs.constants.W_OK);
|
||
if (Number(stat.bavail) * Number(stat.bsize) >= 512 * 1024 * 1024) return '/dev/shm';
|
||
} catch {}
|
||
}
|
||
return os.tmpdir();
|
||
}
|
||
|
||
function textHash(value) {
|
||
const hash = crypto.createHash('sha256');
|
||
for (let offset = 0; offset < value.length; offset += 32 * 1024) hash.update(value.slice(offset, offset + 32 * 1024));
|
||
return hash.digest('hex');
|
||
}
|
||
|
||
// 资料由父进程直接写标准version:1 JSON，不通过待测编码器制造测试基线。
|
||
function writeFixture(sessionsDir) {
|
||
const directory = path.join(sessionsDir, '_history', '_turns');
|
||
fs.mkdirSync(directory, { recursive: true });
|
||
const filename = path.join(directory, `${ID}.json`);
|
||
const header = JSON.stringify({ version: 1, agent: 'codexapp', sessionId: ID, turnId: 'synthetic-turn', turnStatus: 'running', special: SPECIAL }).slice(0, -1) + ',"fullText":"';
|
||
const betweenTool = '","toolCalls":[{"id":"synthetic-tool","name":"run","input":{"text":"完整输入"},"result":"';
|
||
const betweenDelta = '"}],"toolOutputDeltas":[["synthetic-tool","';
|
||
const betweenItem = '"]],"agentMessageItems":[["synthetic-item","';
|
||
const ending = '"]],"agentMessagePendingPrefixes":[],"lastUsage":{"inputTokens":1,"outputTokens":2}}';
|
||
const fixedBytes = Buffer.byteLength(header + betweenTool + betweenDelta + betweenItem + ending);
|
||
const toolBytes = 12 * 1024 * 1024;
|
||
const deltaBytes = 4 * 1024 * 1024;
|
||
const itemBytes = 4 * 1024 * 1024;
|
||
const fullTextBytes = TARGET_BYTES - fixedBytes - toolBytes - deltaBytes - itemBytes;
|
||
const fd = fs.openSync(filename, 'wx', 0o600);
|
||
function writeField(encodedBytes, piece) {
|
||
const encoded = JSON.stringify(piece).slice(1, -1);
|
||
const size = Buffer.byteLength(encoded);
|
||
const hash = crypto.createHash('sha256');
|
||
let length = 0;
|
||
let remaining = encodedBytes;
|
||
while (remaining >= size) {
|
||
fs.writeSync(fd, encoded);
|
||
hash.update(piece);
|
||
length += piece.length;
|
||
remaining -= size;
|
||
}
|
||
if (remaining) {
|
||
const final = 'z'.repeat(remaining);
|
||
fs.writeSync(fd, final);
|
||
hash.update(final);
|
||
length += final.length;
|
||
}
|
||
return { length, hash: hash.digest('hex') };
|
||
}
|
||
const expected = {};
|
||
try {
|
||
fs.writeSync(fd, header);
|
||
expected.fullText = writeField(fullTextBytes, ASCII_PIECE);
|
||
fs.writeSync(fd, betweenTool);
|
||
expected.result = writeField(toolBytes, LARGE_PIECE);
|
||
fs.writeSync(fd, betweenDelta);
|
||
expected.delta = writeField(deltaBytes, LARGE_PIECE);
|
||
fs.writeSync(fd, betweenItem);
|
||
expected.item = writeField(itemBytes, LARGE_PIECE);
|
||
fs.writeSync(fd, ending);
|
||
} finally { fs.closeSync(fd); }
|
||
assert.equal(fs.statSync(filename).size, TARGET_BYTES);
|
||
return { filename, expected };
|
||
}
|
||
|
||
function fileHash(filename) {
|
||
const hash = crypto.createHash('sha256');
|
||
const buffer = Buffer.allocUnsafe(BLOCK_BYTES);
|
||
const fd = fs.openSync(filename, 'r');
|
||
try {
|
||
let bytes;
|
||
while ((bytes = fs.readSync(fd, buffer, 0, buffer.length, null))) hash.update(buffer.subarray(0, bytes));
|
||
} finally { fs.closeSync(fd); }
|
||
return hash.digest('hex');
|
||
}
|
||
|
||
function trackedFilesystem(filename) {
|
||
const opened = new Map();
|
||
const metrics = { reads: 0, writes: 0, maxRead: 0, maxWrite: 0, readBytes: 0, writeBytes: 0 };
|
||
let failure = null;
|
||
let bytesSinceFailureMode = 0;
|
||
function runtimeFile(value) {
|
||
const resolved = typeof value === 'number' ? opened.get(value) || '' : path.resolve(String(value));
|
||
return resolved === filename || resolved.startsWith(`${filename}.`);
|
||
}
|
||
const filesystem = new Proxy(fs, { get(target, key) {
|
||
if (key === 'openSync') return (file, ...args) => {
|
||
const fd = target.openSync(file, ...args);
|
||
opened.set(fd, path.resolve(String(file)));
|
||
return fd;
|
||
};
|
||
if (key === 'closeSync') return fd => { const result = target.closeSync(fd); opened.delete(fd); return result; };
|
||
if (key === 'readFileSync') return (file, ...args) => {
|
||
assert.equal(runtimeFile(file), false, '完整运行态禁止整份readFileSync');
|
||
return target.readFileSync(file, ...args);
|
||
};
|
||
if (key === 'writeFileSync') return (file, ...args) => {
|
||
assert.equal(runtimeFile(file), false, '完整运行态禁止整份writeFileSync');
|
||
return target.writeFileSync(file, ...args);
|
||
};
|
||
if (key === 'readSync') return (fd, buffer, offset, length, position) => {
|
||
if (runtimeFile(fd)) {
|
||
assert(length <= BLOCK_BYTES, '单次运行态读取不能超过64KiB');
|
||
metrics.maxRead = Math.max(metrics.maxRead, length);
|
||
metrics.reads += 1;
|
||
}
|
||
const bytes = target.readSync(fd, buffer, offset, length, position);
|
||
if (runtimeFile(fd)) metrics.readBytes += bytes;
|
||
return bytes;
|
||
};
|
||
if (key === 'writeSync') return (fd, data, ...args) => {
|
||
if (runtimeFile(fd)) {
|
||
const bytes = typeof data === 'string' ? Buffer.byteLength(data, typeof args[1] === 'string' ? args[1] : 'utf8') : args[1] ?? data.length;
|
||
assert(bytes <= BLOCK_BYTES, '单次运行态编码写入不能超过64KiB');
|
||
metrics.maxWrite = Math.max(metrics.maxWrite, bytes);
|
||
metrics.writes += 1;
|
||
if (failure === 'write' && bytesSinceFailureMode >= 1024 * 1024) throw new Error('模拟运行态中途write失败');
|
||
const written = target.writeSync(fd, data, ...args);
|
||
metrics.writeBytes += written;
|
||
bytesSinceFailureMode += written;
|
||
return written;
|
||
}
|
||
return target.writeSync(fd, data, ...args);
|
||
};
|
||
if (key === 'fsyncSync') return fd => {
|
||
if (failure === 'fsync' && runtimeFile(fd)) throw new Error('模拟运行态fsync失败');
|
||
return target.fsyncSync(fd);
|
||
};
|
||
if (key === 'renameSync') return (from, to) => {
|
||
if (failure === 'rename' && String(to) === filename) throw new Error('模拟运行态rename失败');
|
||
return target.renameSync(from, to);
|
||
};
|
||
return target[key];
|
||
} });
|
||
return { fs: filesystem, metrics, fail(mode) { failure = mode; bytesSinceFailureMode = 0; } };
|
||
}
|
||
|
||
function assertComplete(state, fixture) {
|
||
assert.equal(state.version, 1, '旧version:1运行态须兼容');
|
||
assert.equal(state.agent, 'codexapp');
|
||
assert.equal(state.sessionId, ID);
|
||
assert.equal(state.turnStatus, 'running');
|
||
assert.deepEqual(state.special, SPECIAL);
|
||
for (const [name, value] of [['fullText', state.fullText], ['result', state.toolCalls[0].result], ['delta', state.toolOutputDeltas[0][1]], ['item', state.agentMessageItems[0][1]]]) {
|
||
assert.equal(value.length, fixture.expected[name].length, `${name} 长度发生变化`);
|
||
assert.equal(textHash(value), fixture.expected[name].hash, `${name} 正文摘要发生变化`);
|
||
}
|
||
assert.deepEqual(state.toolCalls[0].input, { text: '完整输入' });
|
||
assert.deepEqual(state.lastUsage, { inputTokens: 1, outputTokens: 2 });
|
||
}
|
||
|
||
function runChild(sessionsDir, fixture) {
|
||
const tracked = trackedFilesystem(fixture.filename);
|
||
let store = createSessionHistoryStore({ sessionsDir, fs: tracked.fs });
|
||
stage('开始读取旧version:1完整运行态');
|
||
let state = store.loadTurn(ID);
|
||
assertComplete(state, fixture);
|
||
stage('旧运行态摘要校验通过，开始分段保存');
|
||
store.saveTurn(ID, state);
|
||
const committedHash = fileHash(fixture.filename);
|
||
assert.equal(fs.statSync(fixture.filename).size, TARGET_BYTES, '完整持久化不能裁剪运行态');
|
||
stage('分段保存完成，开始原子失败注入');
|
||
state.turnStatus = 'new-uncommitted-status';
|
||
for (const mode of ['write', 'fsync', 'rename']) {
|
||
tracked.fail(mode);
|
||
assert.throws(() => store.saveTurn(ID, state), /模拟运行态/);
|
||
assert.equal(fileHash(fixture.filename), committedHash, `${mode}失败必须保留上次完整文件`);
|
||
assert.equal(fs.readdirSync(path.dirname(fixture.filename)).some(name => name.endsWith('.tmp')), false, `${mode}失败不能遗留临时文件`);
|
||
stage(`${mode}失败保护通过`);
|
||
}
|
||
tracked.fail(null);
|
||
state = null;
|
||
store = null;
|
||
if (typeof global.gc === 'function') global.gc();
|
||
const reopened = createSessionHistoryStore({ sessionsDir, fs: tracked.fs });
|
||
state = reopened.loadTurn(ID);
|
||
assertComplete(state, fixture);
|
||
assert.equal(reopened.loadTurn('missing-runtime'), null);
|
||
assert(tracked.metrics.reads > 1 && tracked.metrics.writes > 1);
|
||
console.log(JSON.stringify({ archiveMiB: TARGET_BYTES / 1024 / 1024, peakRssMiB: process.resourceUsage().maxRSS / 1024,
|
||
maxReadKiB: tracked.metrics.maxRead / 1024, maxWriteKiB: tracked.metrics.maxWrite / 1024,
|
||
reads: tracked.metrics.reads, writes: tracked.metrics.writes, failureModes: 3 }));
|
||
}
|
||
|
||
function runTests() {
|
||
const sessionsDir = fs.mkdtempSync(path.join(temporaryRoot(), 'ccweb-runtime-memory-'));
|
||
try {
|
||
stage(`开始生成80MiB合成完整运行态（${path.dirname(sessionsDir)}）`);
|
||
const fixture = writeFixture(sessionsDir);
|
||
stage('合成旧运行态完成');
|
||
const child = spawnSync(process.execPath, ['--expose-gc', '--max-old-space-size=256', __filename, '--child', sessionsDir, JSON.stringify(fixture)], {
|
||
encoding: 'utf8', timeout: 60000, maxBuffer: 512 * 1024,
|
||
stdio: ['ignore', 'pipe', process.env.CCWEB_RUNTIME_TEST_PROGRESS === '1' ? 'inherit' : 'pipe'],
|
||
});
|
||
if (child.error || child.status !== 0) process.stderr.write(child.stderr || '');
|
||
assert.ifError(child.error);
|
||
assert.equal(child.status, 0, `256MiB堆运行态子进程失败（信号：${child.signal || '无'}）：${child.stderr || child.stdout}`);
|
||
const result = JSON.parse(child.stdout.trim());
|
||
console.log(`通过：${result.archiveMiB.toFixed(1)} MiB完整运行态兼容读取、分段保存、重开后四项正文摘要一致`);
|
||
console.log(`通过：write/fsync/rename三类失败保留旧文件且无临时文件，单次读取/写入最大 ${result.maxReadKiB.toFixed(1)}/${result.maxWriteKiB.toFixed(1)} KiB`);
|
||
console.log(`内存：256 MiB堆限制下RSS峰值 ${result.peakRssMiB.toFixed(1)} MiB；读取仍持有完整运行态对象，分段处理限制临时编码副本`);
|
||
return result;
|
||
} finally { fs.rmSync(sessionsDir, { recursive: true, force: true }); }
|
||
}
|
||
|
||
if (require.main === module) {
|
||
if (process.argv[2] === '--child') runChild(process.argv[3], JSON.parse(process.argv[4]));
|
||
else runTests();
|
||
}
|
||
module.exports = { runTests };
|