feat: add conversation search and usage dashboard
This commit is contained in:
842
lib/session-search-index.js
Normal file
842
lib/session-search-index.js
Normal file
@@ -0,0 +1,842 @@
|
||||
'use strict';
|
||||
|
||||
const fsp = require('fs/promises');
|
||||
const path = require('path');
|
||||
|
||||
const CACHE_VERSION = 1;
|
||||
const DEFAULT_MAX_FILE_BYTES = 32 * 1024 * 1024;
|
||||
const DEFAULT_MAX_QUERY_CHARS = 200;
|
||||
const DEFAULT_MAX_RESULTS = 50;
|
||||
const DEFAULT_SNIPPET_CHARS = 220;
|
||||
const MAX_QUERY_CHARS_HARD_LIMIT = 200;
|
||||
const MAX_RESULTS_HARD_LIMIT = 50;
|
||||
const UPSERT_DEBOUNCE_MS = 250;
|
||||
const PERSIST_DEBOUNCE_MS = 500;
|
||||
const BUILD_BATCH_SIZE = 25;
|
||||
|
||||
const SESSION_ID_RE = /^[a-zA-Z0-9-]+$/;
|
||||
|
||||
function isObject(value) {
|
||||
return !!value && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function clampInteger(value, fallback, min, max) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return fallback;
|
||||
const integer = Math.floor(number);
|
||||
if (integer < min) return min;
|
||||
if (integer > max) return max;
|
||||
return integer;
|
||||
}
|
||||
|
||||
function cleanDisplayText(value) {
|
||||
if (typeof value !== 'string') return '';
|
||||
return value.replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function normalizeSearchText(value) {
|
||||
if (typeof value !== 'string') return '';
|
||||
return value
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[\u0000-\u001f\u007f]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function isWordChar(char) {
|
||||
return /[\p{L}\p{N}_]/u.test(char);
|
||||
}
|
||||
|
||||
function tokenizeSearchText(value) {
|
||||
const text = normalizeSearchText(value);
|
||||
if (!text) return [];
|
||||
const tokens = [];
|
||||
let current = '';
|
||||
for (const char of text) {
|
||||
if (isWordChar(char)) {
|
||||
current += char;
|
||||
} else if (current) {
|
||||
tokens.push(current);
|
||||
current = '';
|
||||
}
|
||||
}
|
||||
if (current) tokens.push(current);
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function uniqueList(values) {
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
for (const value of values) {
|
||||
if (!value || seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
result.push(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function containsQueryTerms(queryText) {
|
||||
if (!queryText) return [];
|
||||
const parts = queryText.split(/\s+/).filter(Boolean);
|
||||
return uniqueList(parts.length > 1 ? parts : [queryText]);
|
||||
}
|
||||
|
||||
function sanitizeSessionId(sessionId) {
|
||||
return String(sessionId || '').replace(/[^a-zA-Z0-9-]/g, '');
|
||||
}
|
||||
|
||||
function legalSessionId(sessionId) {
|
||||
const id = String(sessionId || '');
|
||||
return !!id && SESSION_ID_RE.test(id);
|
||||
}
|
||||
|
||||
function sessionIdFromFileName(fileName) {
|
||||
if (typeof fileName !== 'string' || !fileName.endsWith('.json')) return '';
|
||||
const id = fileName.slice(0, -5);
|
||||
return legalSessionId(id) ? id : '';
|
||||
}
|
||||
|
||||
function filePathInsideRoot(rootDir, fileName) {
|
||||
const root = path.resolve(rootDir);
|
||||
const target = path.resolve(root, fileName);
|
||||
const relative = path.relative(root, target);
|
||||
if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) return null;
|
||||
return target;
|
||||
}
|
||||
|
||||
function sessionFilePath(rootDir, sessionId) {
|
||||
const id = sanitizeSessionId(sessionId);
|
||||
if (!legalSessionId(id)) return null;
|
||||
return filePathInsideRoot(rootDir, `${id}.json`);
|
||||
}
|
||||
|
||||
function sessionIdForDocument(sessionId, fallbackId) {
|
||||
const fallback = String(fallbackId || '');
|
||||
if (legalSessionId(fallback)) return fallback;
|
||||
const id = String(sessionId || '');
|
||||
return legalSessionId(id) ? id : '';
|
||||
}
|
||||
|
||||
function fileFingerprint(stat) {
|
||||
return {
|
||||
dev: Number(stat.dev) || 0,
|
||||
ino: Number(stat.ino) || 0,
|
||||
size: Number(stat.size) || 0,
|
||||
mtimeMs: Number(stat.mtimeMs) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
function sameFingerprint(left, right) {
|
||||
return !!left
|
||||
&& !!right
|
||||
&& left.dev === right.dev
|
||||
&& left.ino === right.ino
|
||||
&& left.size === right.size
|
||||
&& left.mtimeMs === right.mtimeMs;
|
||||
}
|
||||
|
||||
function safeIsoFromValue(value, fallbackMs) {
|
||||
const date = value ? new Date(value) : new Date(fallbackMs || Date.now());
|
||||
if (Number.isNaN(date.getTime())) return new Date(fallbackMs || Date.now()).toISOString();
|
||||
return date.toISOString();
|
||||
}
|
||||
|
||||
function basenameFromCwd(cwd) {
|
||||
const text = cleanDisplayText(cwd);
|
||||
if (!text) return '';
|
||||
return path.basename(text.replace(/[\\/]+$/, ''));
|
||||
}
|
||||
|
||||
function normalizeAgent(agent) {
|
||||
const text = normalizeSearchText(String(agent || ''));
|
||||
return text || 'codex';
|
||||
}
|
||||
|
||||
// 只抽取 message.content 中的可见文本,显式排除工具结果和附件类内容。
|
||||
function messageContentToText(content) {
|
||||
if (typeof content === 'string') return content;
|
||||
if (Array.isArray(content)) {
|
||||
return content.map((part) => {
|
||||
if (typeof part === 'string') return part;
|
||||
if (!isObject(part)) return '';
|
||||
const type = normalizeSearchText(String(part.type || ''));
|
||||
if (type === 'tool_use' || type === 'tool_result' || type === 'image' || type === 'attachment') return '';
|
||||
return typeof part.text === 'string' ? part.text : '';
|
||||
}).filter(Boolean).join('\n');
|
||||
}
|
||||
if (isObject(content) && typeof content.text === 'string') return content.text;
|
||||
return '';
|
||||
}
|
||||
|
||||
function makeSearchField(value, weight) {
|
||||
const text = cleanDisplayText(value);
|
||||
const normalized = normalizeSearchText(text);
|
||||
return {
|
||||
text,
|
||||
normalized,
|
||||
words: tokenizeSearchText(normalized),
|
||||
weight,
|
||||
};
|
||||
}
|
||||
|
||||
function createSessionDocument(session, stat, fallbackId) {
|
||||
const safeSession = isObject(session) ? session : {};
|
||||
const sessionId = sessionIdForDocument(safeSession.id, fallbackId);
|
||||
if (!legalSessionId(sessionId)) return null;
|
||||
|
||||
const cwd = cleanDisplayText(safeSession.cwd || '');
|
||||
const projectName = cleanDisplayText(safeSession.projectName || safeSession.project || basenameFromCwd(cwd));
|
||||
const title = cleanDisplayText(safeSession.title || 'Untitled') || 'Untitled';
|
||||
const updated = safeIsoFromValue(safeSession.updated || safeSession.updatedAt || safeSession.created, stat?.mtimeMs);
|
||||
const created = safeSession.created ? safeIsoFromValue(safeSession.created, stat?.birthtimeMs || stat?.mtimeMs) : null;
|
||||
const agent = normalizeAgent(safeSession.agent);
|
||||
|
||||
const metaFields = [
|
||||
makeSearchField(title, 8),
|
||||
makeSearchField(projectName, 5),
|
||||
makeSearchField(cwd, 3),
|
||||
makeSearchField(sessionId, 4),
|
||||
].filter((field) => field.normalized);
|
||||
|
||||
const messages = [];
|
||||
const sourceMessages = Array.isArray(safeSession.messages) ? safeSession.messages : [];
|
||||
for (let index = 0; index < sourceMessages.length; index += 1) {
|
||||
const message = sourceMessages[index];
|
||||
if (!isObject(message)) continue;
|
||||
const role = normalizeSearchText(String(message.role || ''));
|
||||
if (role !== 'user' && role !== 'assistant') continue;
|
||||
const text = cleanDisplayText(messageContentToText(message.content));
|
||||
if (!text) continue;
|
||||
const normalized = normalizeSearchText(text);
|
||||
if (!normalized) continue;
|
||||
messages.push({
|
||||
messageIndex: index,
|
||||
role,
|
||||
timestamp: message.timestamp || message.created || message.createdAt || null,
|
||||
text,
|
||||
normalized,
|
||||
words: tokenizeSearchText(normalized),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
sessionId,
|
||||
agent,
|
||||
title,
|
||||
projectName,
|
||||
cwd,
|
||||
updated,
|
||||
created,
|
||||
metaFields,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
|
||||
function countContains(text, term) {
|
||||
if (!text || !term) return 0;
|
||||
let count = 0;
|
||||
let offset = 0;
|
||||
while (offset < text.length) {
|
||||
const index = text.indexOf(term, offset);
|
||||
if (index < 0) break;
|
||||
count += 1;
|
||||
offset = index + Math.max(term.length, 1);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function countWordHits(words, terms) {
|
||||
if (!Array.isArray(words) || !words.length || !terms.length) return 0;
|
||||
const counts = new Map();
|
||||
for (const word of words) counts.set(word, (counts.get(word) || 0) + 1);
|
||||
let hits = 0;
|
||||
for (const term of terms) hits += counts.get(term) || 0;
|
||||
return hits;
|
||||
}
|
||||
|
||||
function fieldMatchScore(field, terms, matchMode) {
|
||||
if (!field || !terms.length) return 0;
|
||||
if (matchMode === 'word') {
|
||||
const hits = countWordHits(field.words, terms);
|
||||
return hits > 0 ? hits * field.weight : 0;
|
||||
}
|
||||
let hits = 0;
|
||||
for (const term of terms) hits += countContains(field.normalized, term);
|
||||
return hits > 0 ? hits * field.weight : 0;
|
||||
}
|
||||
|
||||
function messageMatchScore(message, terms, matchMode) {
|
||||
if (!message || !terms.length) return 0;
|
||||
if (matchMode === 'word') return countWordHits(message.words, terms);
|
||||
let hits = 0;
|
||||
for (const term of terms) hits += countContains(message.normalized, term);
|
||||
return hits;
|
||||
}
|
||||
|
||||
function findSnippetIndex(normalizedText, terms, matchMode) {
|
||||
if (!normalizedText || !terms.length) return 0;
|
||||
if (matchMode === 'word') {
|
||||
let best = -1;
|
||||
for (const term of terms) {
|
||||
const index = normalizedText.indexOf(term);
|
||||
if (index >= 0 && (best < 0 || index < best)) best = index;
|
||||
}
|
||||
return best >= 0 ? best : 0;
|
||||
}
|
||||
let best = -1;
|
||||
for (const term of terms) {
|
||||
const index = normalizedText.indexOf(term);
|
||||
if (index >= 0 && (best < 0 || index < best)) best = index;
|
||||
}
|
||||
return best >= 0 ? best : 0;
|
||||
}
|
||||
|
||||
function makeSnippet(text, terms, matchMode, snippetChars) {
|
||||
const display = cleanDisplayText(text);
|
||||
if (!display) return '';
|
||||
const limit = clampInteger(snippetChars, DEFAULT_SNIPPET_CHARS, 80, 1000);
|
||||
if (display.length <= limit) return display;
|
||||
const normalized = normalizeSearchText(display);
|
||||
const matchIndex = findSnippetIndex(normalized, terms, matchMode);
|
||||
const half = Math.floor(limit / 2);
|
||||
let start = Math.max(0, matchIndex - half);
|
||||
let end = Math.min(display.length, start + limit);
|
||||
start = Math.max(0, end - limit);
|
||||
const prefix = start > 0 ? '...' : '';
|
||||
const suffix = end < display.length ? '...' : '';
|
||||
return `${prefix}${display.slice(start, end)}${suffix}`;
|
||||
}
|
||||
|
||||
function buildMetadataSnippet(doc) {
|
||||
return [
|
||||
doc.title,
|
||||
doc.projectName,
|
||||
doc.cwd,
|
||||
doc.sessionId,
|
||||
].filter(Boolean).join(' ');
|
||||
}
|
||||
|
||||
function updatedTime(doc) {
|
||||
const time = new Date(doc?.updated || 0).getTime();
|
||||
return Number.isFinite(time) ? time : 0;
|
||||
}
|
||||
|
||||
function buildQuery(rawQuery, options) {
|
||||
const maxQueryChars = clampInteger(
|
||||
options.maxQueryChars,
|
||||
DEFAULT_MAX_QUERY_CHARS,
|
||||
1,
|
||||
MAX_QUERY_CHARS_HARD_LIMIT,
|
||||
);
|
||||
const queryText = normalizeSearchText(String(rawQuery || '').slice(0, maxQueryChars));
|
||||
const matchMode = options.matchMode === 'word' ? 'word' : 'contains';
|
||||
const terms = matchMode === 'word' ? uniqueList(tokenizeSearchText(queryText)) : containsQueryTerms(queryText);
|
||||
return { queryText, terms, matchMode, maxQueryChars };
|
||||
}
|
||||
|
||||
function sleepImmediate() {
|
||||
return new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
function timerUnref(timer) {
|
||||
if (timer && typeof timer.unref === 'function') timer.unref();
|
||||
}
|
||||
|
||||
function safeError(err) {
|
||||
return String(err?.message || err || '').slice(0, 240);
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(filePath, data) {
|
||||
const dir = path.dirname(filePath);
|
||||
await fsp.mkdir(dir, { recursive: true });
|
||||
const tmpPath = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp`);
|
||||
const json = `${JSON.stringify(data)}\n`;
|
||||
try {
|
||||
// 缓存包含派生文本,按私有文件写入后再原子替换。
|
||||
await fsp.writeFile(tmpPath, json, { encoding: 'utf8', mode: 0o600 });
|
||||
await fsp.rename(tmpPath, filePath);
|
||||
await fsp.chmod(filePath, 0o600).catch(() => {});
|
||||
} catch (err) {
|
||||
await fsp.unlink(tmpPath).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function createSessionSearchIndex(options = {}) {
|
||||
const sessionsDir = path.resolve(options.sessionsDir || path.join(process.cwd(), 'sessions'));
|
||||
const cacheFile = path.resolve(
|
||||
options.cacheFile
|
||||
? (path.isAbsolute(options.cacheFile) ? options.cacheFile : path.join(sessionsDir, options.cacheFile))
|
||||
: path.join(sessionsDir, '_search', 'index-v1.json'),
|
||||
);
|
||||
const maxFileBytes = clampInteger(options.maxFileBytes, DEFAULT_MAX_FILE_BYTES, 1024, Number.MAX_SAFE_INTEGER);
|
||||
const maxQueryChars = clampInteger(options.maxQueryChars, DEFAULT_MAX_QUERY_CHARS, 1, MAX_QUERY_CHARS_HARD_LIMIT);
|
||||
const maxResults = clampInteger(options.maxResults, DEFAULT_MAX_RESULTS, 1, MAX_RESULTS_HARD_LIMIT);
|
||||
const snippetChars = clampInteger(options.snippetChars, DEFAULT_SNIPPET_CHARS, 80, 1000);
|
||||
const logger = options.logger || null;
|
||||
|
||||
const documents = new Map();
|
||||
const fileMeta = new Map();
|
||||
const upsertTimers = new Map();
|
||||
let state = 'ready';
|
||||
let lastError = null;
|
||||
let initialized = false;
|
||||
let buildPromise = null;
|
||||
let dirty = false;
|
||||
let persistTimer = null;
|
||||
let persistPromise = null;
|
||||
let lastPersistedAt = null;
|
||||
let lastBuiltAt = null;
|
||||
let skippedFiles = 0;
|
||||
let failedFiles = 0;
|
||||
|
||||
function log(level, event, details = {}) {
|
||||
if (!logger) return;
|
||||
try {
|
||||
const fn = logger[level] || logger.warn || logger.log;
|
||||
if (typeof fn === 'function') fn.call(logger, event, details);
|
||||
} catch {
|
||||
// 日志不能影响索引主流程。
|
||||
}
|
||||
}
|
||||
|
||||
function serializeCache() {
|
||||
return {
|
||||
version: CACHE_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
sessionsDir,
|
||||
files: Object.fromEntries(fileMeta.entries()),
|
||||
sessions: Object.fromEntries(documents.entries()),
|
||||
};
|
||||
}
|
||||
|
||||
function applyCache(cache) {
|
||||
if (!isObject(cache) || cache.version !== CACHE_VERSION || !isObject(cache.files) || !isObject(cache.sessions)) {
|
||||
return false;
|
||||
}
|
||||
documents.clear();
|
||||
fileMeta.clear();
|
||||
for (const [sessionId, meta] of Object.entries(cache.files)) {
|
||||
if (!legalSessionId(sessionId) || !isObject(meta)) continue;
|
||||
fileMeta.set(sessionId, {
|
||||
dev: Number(meta.dev) || 0,
|
||||
ino: Number(meta.ino) || 0,
|
||||
size: Number(meta.size) || 0,
|
||||
mtimeMs: Number(meta.mtimeMs) || 0,
|
||||
});
|
||||
}
|
||||
for (const [sessionId, doc] of Object.entries(cache.sessions)) {
|
||||
if (!legalSessionId(sessionId) || !isObject(doc) || doc.sessionId !== sessionId) continue;
|
||||
documents.set(sessionId, doc);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function loadCache() {
|
||||
try {
|
||||
const raw = await fsp.readFile(cacheFile, 'utf8');
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!applyCache(parsed)) {
|
||||
log('warn', 'session_search_cache_ignored', { reason: 'version_or_shape' });
|
||||
documents.clear();
|
||||
fileMeta.clear();
|
||||
dirty = true;
|
||||
}
|
||||
} catch (err) {
|
||||
if (err && err.code !== 'ENOENT') {
|
||||
log('warn', 'session_search_cache_load_failed', { error: safeError(err) });
|
||||
}
|
||||
documents.clear();
|
||||
fileMeta.clear();
|
||||
dirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
async function scanSessionFiles() {
|
||||
await fsp.mkdir(sessionsDir, { recursive: true });
|
||||
const entries = await fsp.readdir(sessionsDir, { withFileTypes: true });
|
||||
const files = new Map();
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
const sessionId = sessionIdFromFileName(entry.name);
|
||||
if (!sessionId) continue;
|
||||
const filePath = filePathInsideRoot(sessionsDir, entry.name);
|
||||
if (!filePath) continue;
|
||||
try {
|
||||
const stat = await fsp.stat(filePath);
|
||||
if (!stat.isFile()) continue;
|
||||
files.set(sessionId, { filePath, stat, fingerprint: fileFingerprint(stat) });
|
||||
} catch (err) {
|
||||
failedFiles += 1;
|
||||
log('warn', 'session_search_stat_failed', { sessionId: sessionId.slice(0, 8), error: safeError(err) });
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function readAndIndexFile(sessionId, filePath, stat, fingerprint) {
|
||||
if (stat.size > maxFileBytes) {
|
||||
skippedFiles += 1;
|
||||
documents.delete(sessionId);
|
||||
fileMeta.delete(sessionId);
|
||||
dirty = true;
|
||||
log('warn', 'session_search_file_skipped', { sessionId: sessionId.slice(0, 8), reason: 'max_file_bytes', size: stat.size });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = await fsp.readFile(filePath, 'utf8');
|
||||
const session = JSON.parse(raw);
|
||||
const doc = createSessionDocument(session, stat, sessionId);
|
||||
if (!doc) {
|
||||
documents.delete(sessionId);
|
||||
fileMeta.delete(sessionId);
|
||||
dirty = true;
|
||||
return;
|
||||
}
|
||||
documents.set(sessionId, doc);
|
||||
fileMeta.set(sessionId, fingerprint);
|
||||
dirty = true;
|
||||
} catch (err) {
|
||||
failedFiles += 1;
|
||||
documents.delete(sessionId);
|
||||
fileMeta.delete(sessionId);
|
||||
dirty = true;
|
||||
log('warn', 'session_search_file_index_failed', { sessionId: sessionId.slice(0, 8), error: safeError(err) });
|
||||
}
|
||||
}
|
||||
|
||||
function markDirtyAndSchedulePersist() {
|
||||
dirty = true;
|
||||
if (persistTimer) clearTimeout(persistTimer);
|
||||
persistTimer = setTimeout(() => {
|
||||
persistTimer = null;
|
||||
persistNow().catch((err) => {
|
||||
state = 'error';
|
||||
lastError = safeError(err);
|
||||
log('error', 'session_search_cache_persist_failed', { error: lastError });
|
||||
});
|
||||
}, PERSIST_DEBOUNCE_MS);
|
||||
timerUnref(persistTimer);
|
||||
}
|
||||
|
||||
async function persistNow() {
|
||||
if (persistPromise) return persistPromise;
|
||||
persistPromise = (async () => {
|
||||
while (dirty) {
|
||||
dirty = false;
|
||||
await writeJsonAtomic(cacheFile, serializeCache());
|
||||
lastPersistedAt = new Date().toISOString();
|
||||
}
|
||||
})().finally(() => {
|
||||
persistPromise = null;
|
||||
});
|
||||
return persistPromise;
|
||||
}
|
||||
|
||||
async function rebuildChangedFiles() {
|
||||
state = 'building';
|
||||
lastError = null;
|
||||
skippedFiles = 0;
|
||||
failedFiles = 0;
|
||||
await loadCache();
|
||||
const scanned = await scanSessionFiles();
|
||||
|
||||
// 以 sessions 根目录下当前合法 JSON 文件为准,删除派生缓存里的孤儿项。
|
||||
for (const sessionId of Array.from(documents.keys())) {
|
||||
if (!scanned.has(sessionId)) {
|
||||
documents.delete(sessionId);
|
||||
fileMeta.delete(sessionId);
|
||||
dirty = true;
|
||||
}
|
||||
}
|
||||
for (const sessionId of Array.from(fileMeta.keys())) {
|
||||
if (!scanned.has(sessionId)) {
|
||||
fileMeta.delete(sessionId);
|
||||
dirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
const changed = [];
|
||||
for (const [sessionId, entry] of scanned.entries()) {
|
||||
if (!sameFingerprint(fileMeta.get(sessionId), entry.fingerprint) || !documents.has(sessionId)) {
|
||||
changed.push([sessionId, entry]);
|
||||
}
|
||||
}
|
||||
|
||||
for (let index = 0; index < changed.length; index += 1) {
|
||||
const [sessionId, entry] = changed[index];
|
||||
await readAndIndexFile(sessionId, entry.filePath, entry.stat, entry.fingerprint);
|
||||
if ((index + 1) % BUILD_BATCH_SIZE === 0) await sleepImmediate();
|
||||
}
|
||||
|
||||
if (dirty) await persistNow();
|
||||
lastBuiltAt = new Date().toISOString();
|
||||
state = 'ready';
|
||||
initialized = true;
|
||||
return status();
|
||||
}
|
||||
|
||||
function initialize() {
|
||||
if (buildPromise) return buildPromise;
|
||||
buildPromise = rebuildChangedFiles()
|
||||
.catch((err) => {
|
||||
state = 'error';
|
||||
lastError = safeError(err);
|
||||
log('error', 'session_search_initialize_failed', { error: lastError });
|
||||
return status();
|
||||
})
|
||||
.finally(() => {
|
||||
buildPromise = null;
|
||||
});
|
||||
return buildPromise;
|
||||
}
|
||||
|
||||
async function upsertNow(sessionId) {
|
||||
const id = sanitizeSessionId(sessionId);
|
||||
if (!legalSessionId(id)) return false;
|
||||
const filePath = sessionFilePath(sessionsDir, id);
|
||||
if (!filePath) return false;
|
||||
try {
|
||||
const stat = await fsp.stat(filePath);
|
||||
if (!stat.isFile()) {
|
||||
documents.delete(id);
|
||||
fileMeta.delete(id);
|
||||
markDirtyAndSchedulePersist();
|
||||
return true;
|
||||
}
|
||||
await readAndIndexFile(id, filePath, stat, fileFingerprint(stat));
|
||||
markDirtyAndSchedulePersist();
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
documents.delete(id);
|
||||
fileMeta.delete(id);
|
||||
markDirtyAndSchedulePersist();
|
||||
return true;
|
||||
}
|
||||
failedFiles += 1;
|
||||
log('warn', 'session_search_upsert_failed', { sessionId: id.slice(0, 8), error: safeError(err) });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleUpsert(sessionId) {
|
||||
const id = sanitizeSessionId(sessionId);
|
||||
if (!legalSessionId(id)) return false;
|
||||
const existing = upsertTimers.get(id);
|
||||
if (existing) clearTimeout(existing);
|
||||
const timer = setTimeout(() => {
|
||||
upsertTimers.delete(id);
|
||||
upsertNow(id).catch((err) => {
|
||||
state = 'error';
|
||||
lastError = safeError(err);
|
||||
log('error', 'session_search_schedule_upsert_failed', { sessionId: id.slice(0, 8), error: lastError });
|
||||
});
|
||||
}, UPSERT_DEBOUNCE_MS);
|
||||
timerUnref(timer);
|
||||
upsertTimers.set(id, timer);
|
||||
return true;
|
||||
}
|
||||
|
||||
function remove(sessionId) {
|
||||
const id = sanitizeSessionId(sessionId);
|
||||
if (!legalSessionId(id)) return false;
|
||||
const timer = upsertTimers.get(id);
|
||||
if (timer) clearTimeout(timer);
|
||||
upsertTimers.delete(id);
|
||||
const hadDocument = documents.delete(id);
|
||||
const hadFileMeta = fileMeta.delete(id);
|
||||
const existed = hadDocument || hadFileMeta;
|
||||
if (existed) markDirtyAndSchedulePersist();
|
||||
return existed;
|
||||
}
|
||||
|
||||
function docContainsAllTerms(doc, terms, matchMode) {
|
||||
if (!terms.length) return false;
|
||||
if (matchMode === 'word') {
|
||||
// word 模式用受控 tokenizer,不用用户输入拼接正则。
|
||||
const seen = new Set();
|
||||
for (const field of doc.metaFields || []) {
|
||||
for (const word of field.words || []) seen.add(word);
|
||||
}
|
||||
for (const message of doc.messages || []) {
|
||||
for (const word of message.words || []) seen.add(word);
|
||||
}
|
||||
return terms.every((term) => seen.has(term));
|
||||
}
|
||||
|
||||
for (const term of terms) {
|
||||
let found = false;
|
||||
for (const field of doc.metaFields || []) {
|
||||
if (field.normalized.includes(term)) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
for (const message of doc.messages || []) {
|
||||
if (message.normalized.includes(term)) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!found) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function buildSearchResult(doc, terms, matchMode) {
|
||||
let score = 0;
|
||||
let metaMatched = false;
|
||||
for (const field of doc.metaFields || []) {
|
||||
const fieldScore = fieldMatchScore(field, terms, matchMode);
|
||||
if (fieldScore > 0) metaMatched = true;
|
||||
score += fieldScore;
|
||||
}
|
||||
|
||||
const matchedMessages = [];
|
||||
for (const message of doc.messages || []) {
|
||||
const messageScore = messageMatchScore(message, terms, matchMode);
|
||||
if (messageScore <= 0) continue;
|
||||
score += messageScore;
|
||||
matchedMessages.push({ message, score: messageScore });
|
||||
}
|
||||
|
||||
matchedMessages.sort((a, b) => b.score - a.score || a.message.messageIndex - b.message.messageIndex);
|
||||
const matches = [];
|
||||
for (const entry of matchedMessages.slice(0, 2)) {
|
||||
matches.push({
|
||||
sessionId: doc.sessionId,
|
||||
messageIndex: entry.message.messageIndex,
|
||||
role: entry.message.role,
|
||||
timestamp: entry.message.timestamp || null,
|
||||
snippet: makeSnippet(entry.message.text, terms, matchMode, snippetChars),
|
||||
score: entry.score,
|
||||
});
|
||||
}
|
||||
|
||||
if (!matches.length && metaMatched) {
|
||||
matches.push({
|
||||
sessionId: doc.sessionId,
|
||||
messageIndex: null,
|
||||
role: 'metadata',
|
||||
timestamp: doc.updated,
|
||||
snippet: makeSnippet(buildMetadataSnippet(doc), terms, matchMode, snippetChars),
|
||||
score,
|
||||
});
|
||||
}
|
||||
|
||||
const firstMatch = matches[0] || null;
|
||||
return {
|
||||
sessionId: doc.sessionId,
|
||||
messageIndex: firstMatch ? firstMatch.messageIndex : null,
|
||||
role: firstMatch ? firstMatch.role : null,
|
||||
timestamp: firstMatch ? firstMatch.timestamp : null,
|
||||
title: doc.title,
|
||||
projectName: doc.projectName,
|
||||
updated: doc.updated,
|
||||
score,
|
||||
matchedMessageCount: matchedMessages.length,
|
||||
matches,
|
||||
};
|
||||
}
|
||||
|
||||
function search(params = {}) {
|
||||
const started = process.hrtime.bigint();
|
||||
const query = buildQuery(params.query, {
|
||||
maxQueryChars: Math.min(maxQueryChars, MAX_QUERY_CHARS_HARD_LIMIT),
|
||||
matchMode: params.matchMode,
|
||||
});
|
||||
const limit = clampInteger(params.limit, maxResults, 1, Math.min(maxResults, MAX_RESULTS_HARD_LIMIT));
|
||||
const sort = params.sort === 'newest' ? 'newest' : 'relevance';
|
||||
const agentFilter = params.agent ? normalizeAgent(params.agent) : '';
|
||||
|
||||
if (!query.queryText || !query.terms.length) {
|
||||
return {
|
||||
total: 0,
|
||||
tookMs: Number((process.hrtime.bigint() - started) / 1000000n),
|
||||
indexState: state,
|
||||
results: [],
|
||||
};
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (const doc of documents.values()) {
|
||||
if (agentFilter && doc.agent !== agentFilter) continue;
|
||||
if (!docContainsAllTerms(doc, query.terms, query.matchMode)) continue;
|
||||
const result = buildSearchResult(doc, query.terms, query.matchMode);
|
||||
if (result.score <= 0) continue;
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
results.sort((a, b) => {
|
||||
if (sort === 'newest') return updatedTime(b) - updatedTime(a) || b.score - a.score || a.sessionId.localeCompare(b.sessionId);
|
||||
return b.score - a.score || updatedTime(b) - updatedTime(a) || a.sessionId.localeCompare(b.sessionId);
|
||||
});
|
||||
|
||||
return {
|
||||
total: results.length,
|
||||
tookMs: Number((process.hrtime.bigint() - started) / 1000000n),
|
||||
indexState: state,
|
||||
results: results.slice(0, limit),
|
||||
};
|
||||
}
|
||||
|
||||
async function flush() {
|
||||
if (buildPromise) await buildPromise;
|
||||
const pendingIds = Array.from(upsertTimers.keys());
|
||||
for (const id of pendingIds) {
|
||||
const timer = upsertTimers.get(id);
|
||||
if (timer) clearTimeout(timer);
|
||||
upsertTimers.delete(id);
|
||||
await upsertNow(id);
|
||||
}
|
||||
if (persistTimer) {
|
||||
clearTimeout(persistTimer);
|
||||
persistTimer = null;
|
||||
}
|
||||
if (dirty) await persistNow();
|
||||
if (persistPromise) await persistPromise;
|
||||
return status();
|
||||
}
|
||||
|
||||
function status() {
|
||||
return {
|
||||
state,
|
||||
building: state === 'building',
|
||||
ready: state === 'ready',
|
||||
initialized,
|
||||
error: lastError,
|
||||
sessionsDir,
|
||||
cacheFile,
|
||||
indexedSessions: documents.size,
|
||||
trackedFiles: fileMeta.size,
|
||||
pendingUpserts: upsertTimers.size,
|
||||
dirty,
|
||||
skippedFiles,
|
||||
failedFiles,
|
||||
lastBuiltAt,
|
||||
lastPersistedAt,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
initialize,
|
||||
scheduleUpsert,
|
||||
remove,
|
||||
search,
|
||||
status,
|
||||
flush,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createSessionSearchIndex,
|
||||
normalizeSearchText,
|
||||
tokenizeSearchText,
|
||||
sanitizeSessionId,
|
||||
createSessionDocument,
|
||||
};
|
||||
Reference in New Issue
Block a user