feat: add local SQLite memory store

This commit is contained in:
Eyejoker
2026-03-31 23:22:46 +09:00
parent 1c287f4ad0
commit 42ed5ef113
4 changed files with 530 additions and 0 deletions

View File

@@ -30,6 +30,7 @@ import {
getNewMessagesBySeq,
getOpenWorkItem,
getPendingServiceHandoffs,
recallMemories,
getRegisteredAgentTypesForJid,
getMessagesSince,
getNewMessages,
@@ -46,6 +47,7 @@ import {
setSession,
setRouterStateForService,
setExplicitRoomMode,
rememberMemory,
storeChatMetadata,
storeMessage,
updateRegisteredGroupName,
@@ -1280,6 +1282,68 @@ describe('message seq cursors', () => {
});
});
describe('memories', () => {
it('recalls scoped memories through FTS and exact keyword matching', () => {
rememberMemory({
scopeKind: 'room',
scopeKey: 'room:test-group',
content: '세션 재시작 후에도 방 메모리를 주입한다.',
keywords: ['room:test-group', 'session-reset'],
sourceKind: 'compact',
sourceRef: 'compact:1',
});
rememberMemory({
scopeKind: 'room',
scopeKey: 'room:test-group',
content: '이 메모리는 다른 검색어다.',
keywords: ['room:test-group'],
sourceKind: 'compact',
sourceRef: 'compact:2',
});
const byText = recallMemories({
scopeKind: 'room',
scopeKey: 'room:test-group',
text: 'session reset',
limit: 5,
});
expect(byText).toHaveLength(1);
expect(byText[0].content).toContain('방 메모리를 주입한다');
const byKeyword = recallMemories({
scopeKind: 'room',
scopeKey: 'room:test-group',
keywords: ['session-reset'],
limit: 5,
});
expect(byKeyword).toHaveLength(1);
expect(byKeyword[0].content).toContain('방 메모리를 주입한다');
});
it('archives old memories when a scope exceeds its bounded limit', () => {
for (let index = 0; index < 305; index += 1) {
rememberMemory({
scopeKind: 'room',
scopeKey: 'room:bounded',
content: `memory-${index}`,
keywords: ['room:bounded'],
sourceKind: 'compact',
sourceRef: `compact:${index}`,
});
}
const recalled = recallMemories({
scopeKind: 'room',
scopeKey: 'room:bounded',
limit: 500,
});
expect(recalled).toHaveLength(300);
expect(recalled.some((memory) => memory.content === 'memory-0')).toBe(false);
expect(recalled.some((memory) => memory.content === 'memory-304')).toBe(true);
});
});
describe('work items', () => {
it('tracks produced, retry, and delivered states', () => {
const item = createProducedWorkItem({

301
src/db.ts
View File

@@ -72,6 +72,31 @@ export interface AssignRoomInput {
ownerAgentConfig?: RegisteredGroup['agentConfig'];
}
export type MemoryScopeKind = 'room' | 'user' | 'project' | 'global';
export type MemorySourceKind = 'compact' | 'explicit' | 'import' | 'system';
export interface MemoryRecord {
id: number;
scopeKind: MemoryScopeKind;
scopeKey: string;
content: string;
keywords: string[];
memoryKind: string | null;
sourceKind: MemorySourceKind;
sourceRef: string | null;
createdAt: string;
lastUsedAt: string | null;
archivedAt: string | null;
}
export interface RecallMemoryQuery {
scopeKind: MemoryScopeKind;
scopeKey: string;
text?: string;
keywords?: string[];
limit?: number;
}
interface RoomRegistrationSnapshot {
name: string;
folder: string;
@@ -391,6 +416,43 @@ function createSchema(database: Database): void {
);
CREATE INDEX IF NOT EXISTS idx_service_handoffs_target
ON service_handoffs(target_service_id, status, created_at);
CREATE TABLE IF NOT EXISTS memories (
id INTEGER PRIMARY KEY,
scope_kind TEXT NOT NULL,
scope_key TEXT NOT NULL,
content TEXT NOT NULL,
keywords_json TEXT NOT NULL DEFAULT '[]',
memory_kind TEXT,
source_kind TEXT NOT NULL,
source_ref TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_used_at TEXT,
archived_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_memories_scope
ON memories(scope_kind, scope_key);
CREATE INDEX IF NOT EXISTS idx_memories_active
ON memories(scope_kind, scope_key, archived_at, created_at);
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
content,
keywords,
content='',
tokenize='unicode61'
);
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
INSERT INTO memories_fts(rowid, content, keywords)
VALUES (new.id, new.content, new.keywords_json);
END;
CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
INSERT INTO memories_fts(memories_fts, rowid, content, keywords)
VALUES ('delete', old.id, old.content, old.keywords_json);
END;
CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
INSERT INTO memories_fts(memories_fts, rowid, content, keywords)
VALUES ('delete', old.id, old.content, old.keywords_json);
INSERT INTO memories_fts(rowid, content, keywords)
VALUES (new.id, new.content, new.keywords_json);
END;
`);
// Add context_mode column if it doesn't exist (migration for existing DBs)
@@ -955,6 +1017,245 @@ export function _setStoredRoomOwnerAgentTypeForTests(
).run(ownerAgentType, new Date().toISOString(), chatJid);
}
const MEMORY_SCOPE_LIMITS: Record<MemoryScopeKind, number> = {
room: 300,
user: 100,
project: 200,
global: 100,
};
interface MemoryDatabaseRow {
id: number;
scope_kind: string;
scope_key: string;
content: string;
keywords_json: string;
memory_kind: string | null;
source_kind: string;
source_ref: string | null;
created_at: string;
last_used_at: string | null;
archived_at: string | null;
}
function normalizeMemoryKeywords(keywords?: string[]): string[] {
if (!Array.isArray(keywords)) return [];
return [
...new Set(
keywords
.map((keyword) => keyword.trim().toLowerCase())
.filter(Boolean),
),
];
}
function parseMemoryKeywords(raw: string | null | undefined): string[] {
if (!raw) return [];
try {
const parsed = JSON.parse(raw);
return normalizeMemoryKeywords(Array.isArray(parsed) ? parsed : []);
} catch {
return [];
}
}
function mapMemoryRow(row: MemoryDatabaseRow): MemoryRecord {
return {
id: row.id,
scopeKind: row.scope_kind as MemoryScopeKind,
scopeKey: row.scope_key,
content: row.content,
keywords: parseMemoryKeywords(row.keywords_json),
memoryKind: row.memory_kind,
sourceKind: row.source_kind as MemorySourceKind,
sourceRef: row.source_ref,
createdAt: row.created_at,
lastUsedAt: row.last_used_at,
archivedAt: row.archived_at,
};
}
function buildMemoryFtsQuery(query: RecallMemoryQuery): string | null {
const tokens = normalizeMemoryKeywords([
...(query.text?.match(/[\p{L}\p{N}_:-]+/gu) ?? []),
...(query.keywords ?? []),
]);
if (tokens.length === 0) return null;
return tokens.map((token) => `"${token.replaceAll('"', '""')}"`).join(' OR ');
}
function getMemoryRowsForScope(
scopeKind: MemoryScopeKind,
scopeKey: string,
): MemoryDatabaseRow[] {
return db
.prepare(
`SELECT *
FROM memories
WHERE scope_kind = ?
AND scope_key = ?
AND archived_at IS NULL
ORDER BY COALESCE(last_used_at, created_at) DESC, id DESC`,
)
.all(scopeKind, scopeKey) as MemoryDatabaseRow[];
}
function queryFtsRowOrder(query: RecallMemoryQuery): Map<number, number> {
const ftsQuery = buildMemoryFtsQuery(query);
if (!ftsQuery) return new Map();
try {
const rows = db
.prepare(
`SELECT memories.id AS id
FROM memories_fts
JOIN memories ON memories.id = memories_fts.rowid
WHERE memories_fts MATCH ?
AND memories.scope_kind = ?
AND memories.scope_key = ?
AND memories.archived_at IS NULL
ORDER BY bm25(memories_fts), memories.created_at DESC
LIMIT ?`,
)
.all(
ftsQuery,
query.scopeKind,
query.scopeKey,
Math.max(25, (query.limit ?? 10) * 8),
) as Array<{ id: number }>;
return new Map(rows.map((row, index) => [row.id, rows.length - index]));
} catch (error) {
logger.warn(
{ query, error },
'Memory FTS query failed; falling back to scope-only recall',
);
return new Map();
}
}
export function touchMemories(ids: number[]): void {
const uniqueIds = [...new Set(ids.filter((id) => Number.isInteger(id) && id > 0))];
if (uniqueIds.length === 0) return;
const now = new Date().toISOString();
const stmt = db.prepare(
`UPDATE memories
SET last_used_at = ?
WHERE id = ?`,
);
const tx = db.transaction(() => {
for (const id of uniqueIds) stmt.run(now, id);
});
tx();
}
export function archiveMemory(id: number): void {
db.prepare(
`UPDATE memories
SET archived_at = COALESCE(archived_at, ?)
WHERE id = ?`,
).run(new Date().toISOString(), id);
}
export function enforceMemoryBounds(
scopeKind: MemoryScopeKind,
scopeKey: string,
): void {
const limit = MEMORY_SCOPE_LIMITS[scopeKind];
const rows = db
.prepare(
`SELECT id
FROM memories
WHERE scope_kind = ?
AND scope_key = ?
AND archived_at IS NULL
ORDER BY COALESCE(last_used_at, created_at) DESC, id DESC
LIMIT -1 OFFSET ?`,
)
.all(scopeKind, scopeKey, limit) as Array<{ id: number }>;
if (rows.length === 0) return;
const now = new Date().toISOString();
const stmt = db.prepare(
`UPDATE memories
SET archived_at = ?
WHERE id = ?`,
);
const tx = db.transaction(() => {
for (const row of rows) stmt.run(now, row.id);
});
tx();
}
export function rememberMemory(input: {
scopeKind: MemoryScopeKind;
scopeKey: string;
content: string;
keywords?: string[];
memoryKind?: string | null;
sourceKind: MemorySourceKind;
sourceRef?: string | null;
}): number {
const normalizedContent = input.content.trim();
if (!normalizedContent) {
throw new Error('Memory content cannot be empty');
}
const normalizedKeywords = normalizeMemoryKeywords(input.keywords);
const createdAt = new Date().toISOString();
db.prepare(
`INSERT INTO memories (
scope_kind,
scope_key,
content,
keywords_json,
memory_kind,
source_kind,
source_ref,
created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
).run(
input.scopeKind,
input.scopeKey,
normalizedContent,
JSON.stringify(normalizedKeywords),
input.memoryKind ?? null,
input.sourceKind,
input.sourceRef ?? null,
createdAt,
);
const row = db.prepare('SELECT last_insert_rowid() AS id').get() as { id: number };
enforceMemoryBounds(input.scopeKind, input.scopeKey);
return row.id;
}
export function recallMemories(query: RecallMemoryQuery): MemoryRecord[] {
const limit = Math.max(1, query.limit ?? 6);
const rows = getMemoryRowsForScope(query.scopeKind, query.scopeKey);
if (rows.length === 0) return [];
const exactKeywords = new Set(normalizeMemoryKeywords(query.keywords));
const ftsOrder = queryFtsRowOrder(query);
const useQueryScoring = exactKeywords.size > 0 || Boolean(query.text?.trim());
const scored = rows
.map((row, index) => {
const keywords = parseMemoryKeywords(row.keywords_json);
const exactMatches = keywords.filter((keyword) => exactKeywords.has(keyword))
.length;
const ftsScore = ftsOrder.get(row.id) ?? 0;
const recencyScore = rows.length - index;
return {
row,
matched: exactMatches > 0 || ftsScore > 0,
score: exactMatches * 100 + ftsScore * 10 + recencyScore,
};
})
.filter((entry) => (useQueryScoring ? entry.matched : true))
.sort((a, b) => b.score - a.score || b.row.id - a.row.id)
.slice(0, limit);
const memories = scored.map((entry) => mapMemoryRow(entry.row));
touchMemories(memories.map((memory) => memory.id));
return memories;
}
/**
* Store chat metadata only (no message content).
* Used for all chats to enable group discovery without storing sensitive content.

View File

@@ -0,0 +1,103 @@
import { beforeEach, describe, expect, it } from 'vitest';
import { _initTestDatabase, rememberMemory } from './db.js';
import {
buildRoomMemoryBriefing,
buildRoomMemoryKey,
formatRoomMemoryBriefing,
} from './sqlite-memory-store.js';
beforeEach(() => {
_initTestDatabase();
});
describe('sqlite-memory-store helpers', () => {
it('builds a stable room memory key from the group folder', () => {
expect(buildRoomMemoryKey('ejclaw')).toBe('room:ejclaw');
});
it('formats recalled memories into a compact session briefing', () => {
const briefing = formatRoomMemoryBriefing('room:ejclaw', [
{
id: 1,
scopeKind: 'room',
scopeKey: 'room:ejclaw',
content: '사용자는 세션 리셋 후에도 방 맥락이 이어지길 원함.',
keywords: ['room:ejclaw'],
memoryKind: 'decision',
sourceKind: 'explicit',
sourceRef: 'msg:1',
createdAt: new Date().toISOString(),
lastUsedAt: null,
archivedAt: null,
},
{
id: 2,
scopeKind: 'room',
scopeKey: 'room:ejclaw',
content: '자동 recall/compact persist를 호스트가 책임지는 방향으로 합의함.',
keywords: ['room:ejclaw'],
memoryKind: null,
sourceKind: 'compact',
sourceRef: 'compact:1',
createdAt: new Date().toISOString(),
lastUsedAt: null,
archivedAt: null,
},
]);
expect(briefing).toContain('## Shared Room Memory');
expect(briefing).toContain('room:ejclaw');
expect(briefing).toContain(
'[decision] 사용자는 세션 리셋 후에도 방 맥락이 이어지길 원함.',
);
expect(briefing).toContain(
'- 자동 recall/compact persist를 호스트가 책임지는 방향으로 합의함.',
);
});
it('builds a room briefing from stored SQLite memories', async () => {
rememberMemory({
scopeKind: 'room',
scopeKey: 'room:ejclaw',
content: '방 메모리는 새 세션 시작 시에만 주입한다.',
keywords: ['room:ejclaw', 'session-start'],
sourceKind: 'compact',
sourceRef: 'compact:test',
});
const briefing = await buildRoomMemoryBriefing({
groupFolder: 'ejclaw',
groupName: 'EJClaw',
});
expect(briefing).toContain('## Shared Room Memory');
expect(briefing).toContain('방 메모리는 새 세션 시작 시에만 주입한다.');
});
it('trims overly long briefings to the configured max length', () => {
const briefing = formatRoomMemoryBriefing(
'room:ejclaw',
[
{
id: 1,
scopeKind: 'room',
scopeKey: 'room:ejclaw',
content: 'a'.repeat(300),
keywords: [],
memoryKind: null,
sourceKind: 'compact',
sourceRef: 'compact:1',
createdAt: new Date().toISOString(),
lastUsedAt: null,
archivedAt: null,
},
],
120,
);
expect(briefing).toBeDefined();
expect(briefing!.length).toBeLessThanOrEqual(120);
expect(briefing!.endsWith('…')).toBe(true);
});
});

View File

@@ -0,0 +1,62 @@
import { type MemoryRecord, recallMemories } from './db.js';
const DEFAULT_PAGE_SIZE = 6;
const DEFAULT_MAX_BRIEFING_CHARS = 2_000;
function trimToMaxChars(text: string, maxChars: number): string {
if (text.length <= maxChars) return text;
return text.slice(0, Math.max(0, maxChars - 1)).trimEnd() + '…';
}
export function buildRoomMemoryKey(groupFolder: string): string {
return `room:${groupFolder}`;
}
export function formatRoomMemoryBriefing(
roomKey: string,
memories: MemoryRecord[],
maxChars = DEFAULT_MAX_BRIEFING_CHARS,
): string | undefined {
if (memories.length === 0) return undefined;
const lines = memories
.map((memory) => {
const content = memory.content.trim();
if (!content) return null;
const prefix = memory.memoryKind ? `- [${memory.memoryKind}] ` : '- ';
return `${prefix}${content}`;
})
.filter((line): line is string => Boolean(line));
if (lines.length === 0) return undefined;
const text = [
'## Shared Room Memory',
`Room key: \`${roomKey}\``,
'Treat this as background context for a fresh session start. The current conversation always takes precedence.',
...lines,
].join('\n');
return trimToMaxChars(text, maxChars);
}
export async function buildRoomMemoryBriefing(args: {
groupFolder: string;
groupName?: string;
maxChars?: number;
limit?: number;
}): Promise<string | undefined> {
void args.groupName;
const roomKey = buildRoomMemoryKey(args.groupFolder);
const memories = recallMemories({
scopeKind: 'room',
scopeKey: roomKey,
limit: args.limit ?? DEFAULT_PAGE_SIZE,
});
return formatRoomMemoryBriefing(
roomKey,
memories,
args.maxChars ?? DEFAULT_MAX_BRIEFING_CHARS,
);
}