From 42ed5ef113eabd186990be2c971486d520296b71 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 31 Mar 2026 23:22:46 +0900 Subject: [PATCH] feat: add local SQLite memory store --- src/db.test.ts | 64 +++++++ src/db.ts | 301 ++++++++++++++++++++++++++++++++ src/sqlite-memory-store.test.ts | 103 +++++++++++ src/sqlite-memory-store.ts | 62 +++++++ 4 files changed, 530 insertions(+) create mode 100644 src/sqlite-memory-store.test.ts create mode 100644 src/sqlite-memory-store.ts diff --git a/src/db.test.ts b/src/db.test.ts index 1b34f2e..a0c880e 100644 --- a/src/db.test.ts +++ b/src/db.test.ts @@ -30,6 +30,7 @@ import { getNewMessagesBySeq, getOpenWorkItem, getPendingServiceHandoffs, + recallMemories, getRegisteredAgentTypesForJid, getMessagesSince, getNewMessages, @@ -46,6 +47,7 @@ import { setSession, setRouterStateForService, setExplicitRoomMode, + rememberMemory, storeChatMetadata, storeMessage, updateRegisteredGroupName, @@ -1280,6 +1282,68 @@ describe('message seq cursors', () => { }); }); +describe('memories', () => { + it('recalls scoped memories through FTS and exact keyword matching', () => { + rememberMemory({ + scopeKind: 'room', + scopeKey: 'room:test-group', + content: '세션 재시작 후에도 방 메모리를 주입한다.', + keywords: ['room:test-group', 'session-reset'], + sourceKind: 'compact', + sourceRef: 'compact:1', + }); + rememberMemory({ + scopeKind: 'room', + scopeKey: 'room:test-group', + content: '이 메모리는 다른 검색어다.', + keywords: ['room:test-group'], + sourceKind: 'compact', + sourceRef: 'compact:2', + }); + + const byText = recallMemories({ + scopeKind: 'room', + scopeKey: 'room:test-group', + text: 'session reset', + limit: 5, + }); + expect(byText).toHaveLength(1); + expect(byText[0].content).toContain('방 메모리를 주입한다'); + + const byKeyword = recallMemories({ + scopeKind: 'room', + scopeKey: 'room:test-group', + keywords: ['session-reset'], + limit: 5, + }); + expect(byKeyword).toHaveLength(1); + expect(byKeyword[0].content).toContain('방 메모리를 주입한다'); + }); + + it('archives old memories when a scope exceeds its bounded limit', () => { + for (let index = 0; index < 305; index += 1) { + rememberMemory({ + scopeKind: 'room', + scopeKey: 'room:bounded', + content: `memory-${index}`, + keywords: ['room:bounded'], + sourceKind: 'compact', + sourceRef: `compact:${index}`, + }); + } + + const recalled = recallMemories({ + scopeKind: 'room', + scopeKey: 'room:bounded', + limit: 500, + }); + + expect(recalled).toHaveLength(300); + expect(recalled.some((memory) => memory.content === 'memory-0')).toBe(false); + expect(recalled.some((memory) => memory.content === 'memory-304')).toBe(true); + }); +}); + describe('work items', () => { it('tracks produced, retry, and delivered states', () => { const item = createProducedWorkItem({ diff --git a/src/db.ts b/src/db.ts index 1f113f4..3a172e8 100644 --- a/src/db.ts +++ b/src/db.ts @@ -72,6 +72,31 @@ export interface AssignRoomInput { ownerAgentConfig?: RegisteredGroup['agentConfig']; } +export type MemoryScopeKind = 'room' | 'user' | 'project' | 'global'; +export type MemorySourceKind = 'compact' | 'explicit' | 'import' | 'system'; + +export interface MemoryRecord { + id: number; + scopeKind: MemoryScopeKind; + scopeKey: string; + content: string; + keywords: string[]; + memoryKind: string | null; + sourceKind: MemorySourceKind; + sourceRef: string | null; + createdAt: string; + lastUsedAt: string | null; + archivedAt: string | null; +} + +export interface RecallMemoryQuery { + scopeKind: MemoryScopeKind; + scopeKey: string; + text?: string; + keywords?: string[]; + limit?: number; +} + interface RoomRegistrationSnapshot { name: string; folder: string; @@ -391,6 +416,43 @@ function createSchema(database: Database): void { ); CREATE INDEX IF NOT EXISTS idx_service_handoffs_target ON service_handoffs(target_service_id, status, created_at); + CREATE TABLE IF NOT EXISTS memories ( + id INTEGER PRIMARY KEY, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + content TEXT NOT NULL, + keywords_json TEXT NOT NULL DEFAULT '[]', + memory_kind TEXT, + source_kind TEXT NOT NULL, + source_ref TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_used_at TEXT, + archived_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_memories_scope + ON memories(scope_kind, scope_key); + CREATE INDEX IF NOT EXISTS idx_memories_active + ON memories(scope_kind, scope_key, archived_at, created_at); + CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5( + content, + keywords, + content='', + tokenize='unicode61' + ); + CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN + INSERT INTO memories_fts(rowid, content, keywords) + VALUES (new.id, new.content, new.keywords_json); + END; + CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, content, keywords) + VALUES ('delete', old.id, old.content, old.keywords_json); + END; + CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, content, keywords) + VALUES ('delete', old.id, old.content, old.keywords_json); + INSERT INTO memories_fts(rowid, content, keywords) + VALUES (new.id, new.content, new.keywords_json); + END; `); // Add context_mode column if it doesn't exist (migration for existing DBs) @@ -955,6 +1017,245 @@ export function _setStoredRoomOwnerAgentTypeForTests( ).run(ownerAgentType, new Date().toISOString(), chatJid); } +const MEMORY_SCOPE_LIMITS: Record = { + room: 300, + user: 100, + project: 200, + global: 100, +}; + +interface MemoryDatabaseRow { + id: number; + scope_kind: string; + scope_key: string; + content: string; + keywords_json: string; + memory_kind: string | null; + source_kind: string; + source_ref: string | null; + created_at: string; + last_used_at: string | null; + archived_at: string | null; +} + +function normalizeMemoryKeywords(keywords?: string[]): string[] { + if (!Array.isArray(keywords)) return []; + return [ + ...new Set( + keywords + .map((keyword) => keyword.trim().toLowerCase()) + .filter(Boolean), + ), + ]; +} + +function parseMemoryKeywords(raw: string | null | undefined): string[] { + if (!raw) return []; + try { + const parsed = JSON.parse(raw); + return normalizeMemoryKeywords(Array.isArray(parsed) ? parsed : []); + } catch { + return []; + } +} + +function mapMemoryRow(row: MemoryDatabaseRow): MemoryRecord { + return { + id: row.id, + scopeKind: row.scope_kind as MemoryScopeKind, + scopeKey: row.scope_key, + content: row.content, + keywords: parseMemoryKeywords(row.keywords_json), + memoryKind: row.memory_kind, + sourceKind: row.source_kind as MemorySourceKind, + sourceRef: row.source_ref, + createdAt: row.created_at, + lastUsedAt: row.last_used_at, + archivedAt: row.archived_at, + }; +} + +function buildMemoryFtsQuery(query: RecallMemoryQuery): string | null { + const tokens = normalizeMemoryKeywords([ + ...(query.text?.match(/[\p{L}\p{N}_:-]+/gu) ?? []), + ...(query.keywords ?? []), + ]); + if (tokens.length === 0) return null; + return tokens.map((token) => `"${token.replaceAll('"', '""')}"`).join(' OR '); +} + +function getMemoryRowsForScope( + scopeKind: MemoryScopeKind, + scopeKey: string, +): MemoryDatabaseRow[] { + return db + .prepare( + `SELECT * + FROM memories + WHERE scope_kind = ? + AND scope_key = ? + AND archived_at IS NULL + ORDER BY COALESCE(last_used_at, created_at) DESC, id DESC`, + ) + .all(scopeKind, scopeKey) as MemoryDatabaseRow[]; +} + +function queryFtsRowOrder(query: RecallMemoryQuery): Map { + const ftsQuery = buildMemoryFtsQuery(query); + if (!ftsQuery) return new Map(); + try { + const rows = db + .prepare( + `SELECT memories.id AS id + FROM memories_fts + JOIN memories ON memories.id = memories_fts.rowid + WHERE memories_fts MATCH ? + AND memories.scope_kind = ? + AND memories.scope_key = ? + AND memories.archived_at IS NULL + ORDER BY bm25(memories_fts), memories.created_at DESC + LIMIT ?`, + ) + .all( + ftsQuery, + query.scopeKind, + query.scopeKey, + Math.max(25, (query.limit ?? 10) * 8), + ) as Array<{ id: number }>; + return new Map(rows.map((row, index) => [row.id, rows.length - index])); + } catch (error) { + logger.warn( + { query, error }, + 'Memory FTS query failed; falling back to scope-only recall', + ); + return new Map(); + } +} + +export function touchMemories(ids: number[]): void { + const uniqueIds = [...new Set(ids.filter((id) => Number.isInteger(id) && id > 0))]; + if (uniqueIds.length === 0) return; + const now = new Date().toISOString(); + const stmt = db.prepare( + `UPDATE memories + SET last_used_at = ? + WHERE id = ?`, + ); + const tx = db.transaction(() => { + for (const id of uniqueIds) stmt.run(now, id); + }); + tx(); +} + +export function archiveMemory(id: number): void { + db.prepare( + `UPDATE memories + SET archived_at = COALESCE(archived_at, ?) + WHERE id = ?`, + ).run(new Date().toISOString(), id); +} + +export function enforceMemoryBounds( + scopeKind: MemoryScopeKind, + scopeKey: string, +): void { + const limit = MEMORY_SCOPE_LIMITS[scopeKind]; + const rows = db + .prepare( + `SELECT id + FROM memories + WHERE scope_kind = ? + AND scope_key = ? + AND archived_at IS NULL + ORDER BY COALESCE(last_used_at, created_at) DESC, id DESC + LIMIT -1 OFFSET ?`, + ) + .all(scopeKind, scopeKey, limit) as Array<{ id: number }>; + if (rows.length === 0) return; + const now = new Date().toISOString(); + const stmt = db.prepare( + `UPDATE memories + SET archived_at = ? + WHERE id = ?`, + ); + const tx = db.transaction(() => { + for (const row of rows) stmt.run(now, row.id); + }); + tx(); +} + +export function rememberMemory(input: { + scopeKind: MemoryScopeKind; + scopeKey: string; + content: string; + keywords?: string[]; + memoryKind?: string | null; + sourceKind: MemorySourceKind; + sourceRef?: string | null; +}): number { + const normalizedContent = input.content.trim(); + if (!normalizedContent) { + throw new Error('Memory content cannot be empty'); + } + const normalizedKeywords = normalizeMemoryKeywords(input.keywords); + const createdAt = new Date().toISOString(); + db.prepare( + `INSERT INTO memories ( + scope_kind, + scope_key, + content, + keywords_json, + memory_kind, + source_kind, + source_ref, + created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + input.scopeKind, + input.scopeKey, + normalizedContent, + JSON.stringify(normalizedKeywords), + input.memoryKind ?? null, + input.sourceKind, + input.sourceRef ?? null, + createdAt, + ); + const row = db.prepare('SELECT last_insert_rowid() AS id').get() as { id: number }; + enforceMemoryBounds(input.scopeKind, input.scopeKey); + return row.id; +} + +export function recallMemories(query: RecallMemoryQuery): MemoryRecord[] { + const limit = Math.max(1, query.limit ?? 6); + const rows = getMemoryRowsForScope(query.scopeKind, query.scopeKey); + if (rows.length === 0) return []; + + const exactKeywords = new Set(normalizeMemoryKeywords(query.keywords)); + const ftsOrder = queryFtsRowOrder(query); + const useQueryScoring = exactKeywords.size > 0 || Boolean(query.text?.trim()); + + const scored = rows + .map((row, index) => { + const keywords = parseMemoryKeywords(row.keywords_json); + const exactMatches = keywords.filter((keyword) => exactKeywords.has(keyword)) + .length; + const ftsScore = ftsOrder.get(row.id) ?? 0; + const recencyScore = rows.length - index; + return { + row, + matched: exactMatches > 0 || ftsScore > 0, + score: exactMatches * 100 + ftsScore * 10 + recencyScore, + }; + }) + .filter((entry) => (useQueryScoring ? entry.matched : true)) + .sort((a, b) => b.score - a.score || b.row.id - a.row.id) + .slice(0, limit); + + const memories = scored.map((entry) => mapMemoryRow(entry.row)); + touchMemories(memories.map((memory) => memory.id)); + return memories; +} + /** * Store chat metadata only (no message content). * Used for all chats to enable group discovery without storing sensitive content. diff --git a/src/sqlite-memory-store.test.ts b/src/sqlite-memory-store.test.ts new file mode 100644 index 0000000..200e8fb --- /dev/null +++ b/src/sqlite-memory-store.test.ts @@ -0,0 +1,103 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { _initTestDatabase, rememberMemory } from './db.js'; +import { + buildRoomMemoryBriefing, + buildRoomMemoryKey, + formatRoomMemoryBriefing, +} from './sqlite-memory-store.js'; + +beforeEach(() => { + _initTestDatabase(); +}); + +describe('sqlite-memory-store helpers', () => { + it('builds a stable room memory key from the group folder', () => { + expect(buildRoomMemoryKey('ejclaw')).toBe('room:ejclaw'); + }); + + it('formats recalled memories into a compact session briefing', () => { + const briefing = formatRoomMemoryBriefing('room:ejclaw', [ + { + id: 1, + scopeKind: 'room', + scopeKey: 'room:ejclaw', + content: '사용자는 세션 리셋 후에도 방 맥락이 이어지길 원함.', + keywords: ['room:ejclaw'], + memoryKind: 'decision', + sourceKind: 'explicit', + sourceRef: 'msg:1', + createdAt: new Date().toISOString(), + lastUsedAt: null, + archivedAt: null, + }, + { + id: 2, + scopeKind: 'room', + scopeKey: 'room:ejclaw', + content: '자동 recall/compact persist를 호스트가 책임지는 방향으로 합의함.', + keywords: ['room:ejclaw'], + memoryKind: null, + sourceKind: 'compact', + sourceRef: 'compact:1', + createdAt: new Date().toISOString(), + lastUsedAt: null, + archivedAt: null, + }, + ]); + + expect(briefing).toContain('## Shared Room Memory'); + expect(briefing).toContain('room:ejclaw'); + expect(briefing).toContain( + '[decision] 사용자는 세션 리셋 후에도 방 맥락이 이어지길 원함.', + ); + expect(briefing).toContain( + '- 자동 recall/compact persist를 호스트가 책임지는 방향으로 합의함.', + ); + }); + + it('builds a room briefing from stored SQLite memories', async () => { + rememberMemory({ + scopeKind: 'room', + scopeKey: 'room:ejclaw', + content: '방 메모리는 새 세션 시작 시에만 주입한다.', + keywords: ['room:ejclaw', 'session-start'], + sourceKind: 'compact', + sourceRef: 'compact:test', + }); + + const briefing = await buildRoomMemoryBriefing({ + groupFolder: 'ejclaw', + groupName: 'EJClaw', + }); + + expect(briefing).toContain('## Shared Room Memory'); + expect(briefing).toContain('방 메모리는 새 세션 시작 시에만 주입한다.'); + }); + + it('trims overly long briefings to the configured max length', () => { + const briefing = formatRoomMemoryBriefing( + 'room:ejclaw', + [ + { + id: 1, + scopeKind: 'room', + scopeKey: 'room:ejclaw', + content: 'a'.repeat(300), + keywords: [], + memoryKind: null, + sourceKind: 'compact', + sourceRef: 'compact:1', + createdAt: new Date().toISOString(), + lastUsedAt: null, + archivedAt: null, + }, + ], + 120, + ); + + expect(briefing).toBeDefined(); + expect(briefing!.length).toBeLessThanOrEqual(120); + expect(briefing!.endsWith('…')).toBe(true); + }); +}); diff --git a/src/sqlite-memory-store.ts b/src/sqlite-memory-store.ts new file mode 100644 index 0000000..74cbaf2 --- /dev/null +++ b/src/sqlite-memory-store.ts @@ -0,0 +1,62 @@ +import { type MemoryRecord, recallMemories } from './db.js'; + +const DEFAULT_PAGE_SIZE = 6; +const DEFAULT_MAX_BRIEFING_CHARS = 2_000; + +function trimToMaxChars(text: string, maxChars: number): string { + if (text.length <= maxChars) return text; + return text.slice(0, Math.max(0, maxChars - 1)).trimEnd() + '…'; +} + +export function buildRoomMemoryKey(groupFolder: string): string { + return `room:${groupFolder}`; +} + +export function formatRoomMemoryBriefing( + roomKey: string, + memories: MemoryRecord[], + maxChars = DEFAULT_MAX_BRIEFING_CHARS, +): string | undefined { + if (memories.length === 0) return undefined; + + const lines = memories + .map((memory) => { + const content = memory.content.trim(); + if (!content) return null; + const prefix = memory.memoryKind ? `- [${memory.memoryKind}] ` : '- '; + return `${prefix}${content}`; + }) + .filter((line): line is string => Boolean(line)); + + if (lines.length === 0) return undefined; + + const text = [ + '## Shared Room Memory', + `Room key: \`${roomKey}\``, + 'Treat this as background context for a fresh session start. The current conversation always takes precedence.', + ...lines, + ].join('\n'); + + return trimToMaxChars(text, maxChars); +} + +export async function buildRoomMemoryBriefing(args: { + groupFolder: string; + groupName?: string; + maxChars?: number; + limit?: number; +}): Promise { + void args.groupName; + const roomKey = buildRoomMemoryKey(args.groupFolder); + const memories = recallMemories({ + scopeKind: 'room', + scopeKey: roomKey, + limit: args.limit ?? DEFAULT_PAGE_SIZE, + }); + + return formatRoomMemoryBriefing( + roomKey, + memories, + args.maxChars ?? DEFAULT_MAX_BRIEFING_CHARS, + ); +}