feat: add local SQLite memory store
This commit is contained in:
@@ -30,6 +30,7 @@ import {
|
||||
getNewMessagesBySeq,
|
||||
getOpenWorkItem,
|
||||
getPendingServiceHandoffs,
|
||||
recallMemories,
|
||||
getRegisteredAgentTypesForJid,
|
||||
getMessagesSince,
|
||||
getNewMessages,
|
||||
@@ -46,6 +47,7 @@ import {
|
||||
setSession,
|
||||
setRouterStateForService,
|
||||
setExplicitRoomMode,
|
||||
rememberMemory,
|
||||
storeChatMetadata,
|
||||
storeMessage,
|
||||
updateRegisteredGroupName,
|
||||
@@ -1280,6 +1282,68 @@ describe('message seq cursors', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('memories', () => {
|
||||
it('recalls scoped memories through FTS and exact keyword matching', () => {
|
||||
rememberMemory({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:test-group',
|
||||
content: '세션 재시작 후에도 방 메모리를 주입한다.',
|
||||
keywords: ['room:test-group', 'session-reset'],
|
||||
sourceKind: 'compact',
|
||||
sourceRef: 'compact:1',
|
||||
});
|
||||
rememberMemory({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:test-group',
|
||||
content: '이 메모리는 다른 검색어다.',
|
||||
keywords: ['room:test-group'],
|
||||
sourceKind: 'compact',
|
||||
sourceRef: 'compact:2',
|
||||
});
|
||||
|
||||
const byText = recallMemories({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:test-group',
|
||||
text: 'session reset',
|
||||
limit: 5,
|
||||
});
|
||||
expect(byText).toHaveLength(1);
|
||||
expect(byText[0].content).toContain('방 메모리를 주입한다');
|
||||
|
||||
const byKeyword = recallMemories({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:test-group',
|
||||
keywords: ['session-reset'],
|
||||
limit: 5,
|
||||
});
|
||||
expect(byKeyword).toHaveLength(1);
|
||||
expect(byKeyword[0].content).toContain('방 메모리를 주입한다');
|
||||
});
|
||||
|
||||
it('archives old memories when a scope exceeds its bounded limit', () => {
|
||||
for (let index = 0; index < 305; index += 1) {
|
||||
rememberMemory({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:bounded',
|
||||
content: `memory-${index}`,
|
||||
keywords: ['room:bounded'],
|
||||
sourceKind: 'compact',
|
||||
sourceRef: `compact:${index}`,
|
||||
});
|
||||
}
|
||||
|
||||
const recalled = recallMemories({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:bounded',
|
||||
limit: 500,
|
||||
});
|
||||
|
||||
expect(recalled).toHaveLength(300);
|
||||
expect(recalled.some((memory) => memory.content === 'memory-0')).toBe(false);
|
||||
expect(recalled.some((memory) => memory.content === 'memory-304')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('work items', () => {
|
||||
it('tracks produced, retry, and delivered states', () => {
|
||||
const item = createProducedWorkItem({
|
||||
|
||||
301
src/db.ts
301
src/db.ts
@@ -72,6 +72,31 @@ export interface AssignRoomInput {
|
||||
ownerAgentConfig?: RegisteredGroup['agentConfig'];
|
||||
}
|
||||
|
||||
export type MemoryScopeKind = 'room' | 'user' | 'project' | 'global';
|
||||
export type MemorySourceKind = 'compact' | 'explicit' | 'import' | 'system';
|
||||
|
||||
export interface MemoryRecord {
|
||||
id: number;
|
||||
scopeKind: MemoryScopeKind;
|
||||
scopeKey: string;
|
||||
content: string;
|
||||
keywords: string[];
|
||||
memoryKind: string | null;
|
||||
sourceKind: MemorySourceKind;
|
||||
sourceRef: string | null;
|
||||
createdAt: string;
|
||||
lastUsedAt: string | null;
|
||||
archivedAt: string | null;
|
||||
}
|
||||
|
||||
export interface RecallMemoryQuery {
|
||||
scopeKind: MemoryScopeKind;
|
||||
scopeKey: string;
|
||||
text?: string;
|
||||
keywords?: string[];
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
interface RoomRegistrationSnapshot {
|
||||
name: string;
|
||||
folder: string;
|
||||
@@ -391,6 +416,43 @@ function createSchema(database: Database): void {
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_service_handoffs_target
|
||||
ON service_handoffs(target_service_id, status, created_at);
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id INTEGER PRIMARY KEY,
|
||||
scope_kind TEXT NOT NULL,
|
||||
scope_key TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
keywords_json TEXT NOT NULL DEFAULT '[]',
|
||||
memory_kind TEXT,
|
||||
source_kind TEXT NOT NULL,
|
||||
source_ref TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_used_at TEXT,
|
||||
archived_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_scope
|
||||
ON memories(scope_kind, scope_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_active
|
||||
ON memories(scope_kind, scope_key, archived_at, created_at);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
||||
content,
|
||||
keywords,
|
||||
content='',
|
||||
tokenize='unicode61'
|
||||
);
|
||||
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
||||
INSERT INTO memories_fts(rowid, content, keywords)
|
||||
VALUES (new.id, new.content, new.keywords_json);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
|
||||
INSERT INTO memories_fts(memories_fts, rowid, content, keywords)
|
||||
VALUES ('delete', old.id, old.content, old.keywords_json);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
|
||||
INSERT INTO memories_fts(memories_fts, rowid, content, keywords)
|
||||
VALUES ('delete', old.id, old.content, old.keywords_json);
|
||||
INSERT INTO memories_fts(rowid, content, keywords)
|
||||
VALUES (new.id, new.content, new.keywords_json);
|
||||
END;
|
||||
`);
|
||||
|
||||
// Add context_mode column if it doesn't exist (migration for existing DBs)
|
||||
@@ -955,6 +1017,245 @@ export function _setStoredRoomOwnerAgentTypeForTests(
|
||||
).run(ownerAgentType, new Date().toISOString(), chatJid);
|
||||
}
|
||||
|
||||
const MEMORY_SCOPE_LIMITS: Record<MemoryScopeKind, number> = {
|
||||
room: 300,
|
||||
user: 100,
|
||||
project: 200,
|
||||
global: 100,
|
||||
};
|
||||
|
||||
interface MemoryDatabaseRow {
|
||||
id: number;
|
||||
scope_kind: string;
|
||||
scope_key: string;
|
||||
content: string;
|
||||
keywords_json: string;
|
||||
memory_kind: string | null;
|
||||
source_kind: string;
|
||||
source_ref: string | null;
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
archived_at: string | null;
|
||||
}
|
||||
|
||||
function normalizeMemoryKeywords(keywords?: string[]): string[] {
|
||||
if (!Array.isArray(keywords)) return [];
|
||||
return [
|
||||
...new Set(
|
||||
keywords
|
||||
.map((keyword) => keyword.trim().toLowerCase())
|
||||
.filter(Boolean),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function parseMemoryKeywords(raw: string | null | undefined): string[] {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return normalizeMemoryKeywords(Array.isArray(parsed) ? parsed : []);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function mapMemoryRow(row: MemoryDatabaseRow): MemoryRecord {
|
||||
return {
|
||||
id: row.id,
|
||||
scopeKind: row.scope_kind as MemoryScopeKind,
|
||||
scopeKey: row.scope_key,
|
||||
content: row.content,
|
||||
keywords: parseMemoryKeywords(row.keywords_json),
|
||||
memoryKind: row.memory_kind,
|
||||
sourceKind: row.source_kind as MemorySourceKind,
|
||||
sourceRef: row.source_ref,
|
||||
createdAt: row.created_at,
|
||||
lastUsedAt: row.last_used_at,
|
||||
archivedAt: row.archived_at,
|
||||
};
|
||||
}
|
||||
|
||||
function buildMemoryFtsQuery(query: RecallMemoryQuery): string | null {
|
||||
const tokens = normalizeMemoryKeywords([
|
||||
...(query.text?.match(/[\p{L}\p{N}_:-]+/gu) ?? []),
|
||||
...(query.keywords ?? []),
|
||||
]);
|
||||
if (tokens.length === 0) return null;
|
||||
return tokens.map((token) => `"${token.replaceAll('"', '""')}"`).join(' OR ');
|
||||
}
|
||||
|
||||
function getMemoryRowsForScope(
|
||||
scopeKind: MemoryScopeKind,
|
||||
scopeKey: string,
|
||||
): MemoryDatabaseRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT *
|
||||
FROM memories
|
||||
WHERE scope_kind = ?
|
||||
AND scope_key = ?
|
||||
AND archived_at IS NULL
|
||||
ORDER BY COALESCE(last_used_at, created_at) DESC, id DESC`,
|
||||
)
|
||||
.all(scopeKind, scopeKey) as MemoryDatabaseRow[];
|
||||
}
|
||||
|
||||
function queryFtsRowOrder(query: RecallMemoryQuery): Map<number, number> {
|
||||
const ftsQuery = buildMemoryFtsQuery(query);
|
||||
if (!ftsQuery) return new Map();
|
||||
try {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT memories.id AS id
|
||||
FROM memories_fts
|
||||
JOIN memories ON memories.id = memories_fts.rowid
|
||||
WHERE memories_fts MATCH ?
|
||||
AND memories.scope_kind = ?
|
||||
AND memories.scope_key = ?
|
||||
AND memories.archived_at IS NULL
|
||||
ORDER BY bm25(memories_fts), memories.created_at DESC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(
|
||||
ftsQuery,
|
||||
query.scopeKind,
|
||||
query.scopeKey,
|
||||
Math.max(25, (query.limit ?? 10) * 8),
|
||||
) as Array<{ id: number }>;
|
||||
return new Map(rows.map((row, index) => [row.id, rows.length - index]));
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
{ query, error },
|
||||
'Memory FTS query failed; falling back to scope-only recall',
|
||||
);
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
export function touchMemories(ids: number[]): void {
|
||||
const uniqueIds = [...new Set(ids.filter((id) => Number.isInteger(id) && id > 0))];
|
||||
if (uniqueIds.length === 0) return;
|
||||
const now = new Date().toISOString();
|
||||
const stmt = db.prepare(
|
||||
`UPDATE memories
|
||||
SET last_used_at = ?
|
||||
WHERE id = ?`,
|
||||
);
|
||||
const tx = db.transaction(() => {
|
||||
for (const id of uniqueIds) stmt.run(now, id);
|
||||
});
|
||||
tx();
|
||||
}
|
||||
|
||||
export function archiveMemory(id: number): void {
|
||||
db.prepare(
|
||||
`UPDATE memories
|
||||
SET archived_at = COALESCE(archived_at, ?)
|
||||
WHERE id = ?`,
|
||||
).run(new Date().toISOString(), id);
|
||||
}
|
||||
|
||||
export function enforceMemoryBounds(
|
||||
scopeKind: MemoryScopeKind,
|
||||
scopeKey: string,
|
||||
): void {
|
||||
const limit = MEMORY_SCOPE_LIMITS[scopeKind];
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT id
|
||||
FROM memories
|
||||
WHERE scope_kind = ?
|
||||
AND scope_key = ?
|
||||
AND archived_at IS NULL
|
||||
ORDER BY COALESCE(last_used_at, created_at) DESC, id DESC
|
||||
LIMIT -1 OFFSET ?`,
|
||||
)
|
||||
.all(scopeKind, scopeKey, limit) as Array<{ id: number }>;
|
||||
if (rows.length === 0) return;
|
||||
const now = new Date().toISOString();
|
||||
const stmt = db.prepare(
|
||||
`UPDATE memories
|
||||
SET archived_at = ?
|
||||
WHERE id = ?`,
|
||||
);
|
||||
const tx = db.transaction(() => {
|
||||
for (const row of rows) stmt.run(now, row.id);
|
||||
});
|
||||
tx();
|
||||
}
|
||||
|
||||
export function rememberMemory(input: {
|
||||
scopeKind: MemoryScopeKind;
|
||||
scopeKey: string;
|
||||
content: string;
|
||||
keywords?: string[];
|
||||
memoryKind?: string | null;
|
||||
sourceKind: MemorySourceKind;
|
||||
sourceRef?: string | null;
|
||||
}): number {
|
||||
const normalizedContent = input.content.trim();
|
||||
if (!normalizedContent) {
|
||||
throw new Error('Memory content cannot be empty');
|
||||
}
|
||||
const normalizedKeywords = normalizeMemoryKeywords(input.keywords);
|
||||
const createdAt = new Date().toISOString();
|
||||
db.prepare(
|
||||
`INSERT INTO memories (
|
||||
scope_kind,
|
||||
scope_key,
|
||||
content,
|
||||
keywords_json,
|
||||
memory_kind,
|
||||
source_kind,
|
||||
source_ref,
|
||||
created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
).run(
|
||||
input.scopeKind,
|
||||
input.scopeKey,
|
||||
normalizedContent,
|
||||
JSON.stringify(normalizedKeywords),
|
||||
input.memoryKind ?? null,
|
||||
input.sourceKind,
|
||||
input.sourceRef ?? null,
|
||||
createdAt,
|
||||
);
|
||||
const row = db.prepare('SELECT last_insert_rowid() AS id').get() as { id: number };
|
||||
enforceMemoryBounds(input.scopeKind, input.scopeKey);
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export function recallMemories(query: RecallMemoryQuery): MemoryRecord[] {
|
||||
const limit = Math.max(1, query.limit ?? 6);
|
||||
const rows = getMemoryRowsForScope(query.scopeKind, query.scopeKey);
|
||||
if (rows.length === 0) return [];
|
||||
|
||||
const exactKeywords = new Set(normalizeMemoryKeywords(query.keywords));
|
||||
const ftsOrder = queryFtsRowOrder(query);
|
||||
const useQueryScoring = exactKeywords.size > 0 || Boolean(query.text?.trim());
|
||||
|
||||
const scored = rows
|
||||
.map((row, index) => {
|
||||
const keywords = parseMemoryKeywords(row.keywords_json);
|
||||
const exactMatches = keywords.filter((keyword) => exactKeywords.has(keyword))
|
||||
.length;
|
||||
const ftsScore = ftsOrder.get(row.id) ?? 0;
|
||||
const recencyScore = rows.length - index;
|
||||
return {
|
||||
row,
|
||||
matched: exactMatches > 0 || ftsScore > 0,
|
||||
score: exactMatches * 100 + ftsScore * 10 + recencyScore,
|
||||
};
|
||||
})
|
||||
.filter((entry) => (useQueryScoring ? entry.matched : true))
|
||||
.sort((a, b) => b.score - a.score || b.row.id - a.row.id)
|
||||
.slice(0, limit);
|
||||
|
||||
const memories = scored.map((entry) => mapMemoryRow(entry.row));
|
||||
touchMemories(memories.map((memory) => memory.id));
|
||||
return memories;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store chat metadata only (no message content).
|
||||
* Used for all chats to enable group discovery without storing sensitive content.
|
||||
|
||||
103
src/sqlite-memory-store.test.ts
Normal file
103
src/sqlite-memory-store.test.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { _initTestDatabase, rememberMemory } from './db.js';
|
||||
import {
|
||||
buildRoomMemoryBriefing,
|
||||
buildRoomMemoryKey,
|
||||
formatRoomMemoryBriefing,
|
||||
} from './sqlite-memory-store.js';
|
||||
|
||||
beforeEach(() => {
|
||||
_initTestDatabase();
|
||||
});
|
||||
|
||||
describe('sqlite-memory-store helpers', () => {
|
||||
it('builds a stable room memory key from the group folder', () => {
|
||||
expect(buildRoomMemoryKey('ejclaw')).toBe('room:ejclaw');
|
||||
});
|
||||
|
||||
it('formats recalled memories into a compact session briefing', () => {
|
||||
const briefing = formatRoomMemoryBriefing('room:ejclaw', [
|
||||
{
|
||||
id: 1,
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:ejclaw',
|
||||
content: '사용자는 세션 리셋 후에도 방 맥락이 이어지길 원함.',
|
||||
keywords: ['room:ejclaw'],
|
||||
memoryKind: 'decision',
|
||||
sourceKind: 'explicit',
|
||||
sourceRef: 'msg:1',
|
||||
createdAt: new Date().toISOString(),
|
||||
lastUsedAt: null,
|
||||
archivedAt: null,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:ejclaw',
|
||||
content: '자동 recall/compact persist를 호스트가 책임지는 방향으로 합의함.',
|
||||
keywords: ['room:ejclaw'],
|
||||
memoryKind: null,
|
||||
sourceKind: 'compact',
|
||||
sourceRef: 'compact:1',
|
||||
createdAt: new Date().toISOString(),
|
||||
lastUsedAt: null,
|
||||
archivedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(briefing).toContain('## Shared Room Memory');
|
||||
expect(briefing).toContain('room:ejclaw');
|
||||
expect(briefing).toContain(
|
||||
'[decision] 사용자는 세션 리셋 후에도 방 맥락이 이어지길 원함.',
|
||||
);
|
||||
expect(briefing).toContain(
|
||||
'- 자동 recall/compact persist를 호스트가 책임지는 방향으로 합의함.',
|
||||
);
|
||||
});
|
||||
|
||||
it('builds a room briefing from stored SQLite memories', async () => {
|
||||
rememberMemory({
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:ejclaw',
|
||||
content: '방 메모리는 새 세션 시작 시에만 주입한다.',
|
||||
keywords: ['room:ejclaw', 'session-start'],
|
||||
sourceKind: 'compact',
|
||||
sourceRef: 'compact:test',
|
||||
});
|
||||
|
||||
const briefing = await buildRoomMemoryBriefing({
|
||||
groupFolder: 'ejclaw',
|
||||
groupName: 'EJClaw',
|
||||
});
|
||||
|
||||
expect(briefing).toContain('## Shared Room Memory');
|
||||
expect(briefing).toContain('방 메모리는 새 세션 시작 시에만 주입한다.');
|
||||
});
|
||||
|
||||
it('trims overly long briefings to the configured max length', () => {
|
||||
const briefing = formatRoomMemoryBriefing(
|
||||
'room:ejclaw',
|
||||
[
|
||||
{
|
||||
id: 1,
|
||||
scopeKind: 'room',
|
||||
scopeKey: 'room:ejclaw',
|
||||
content: 'a'.repeat(300),
|
||||
keywords: [],
|
||||
memoryKind: null,
|
||||
sourceKind: 'compact',
|
||||
sourceRef: 'compact:1',
|
||||
createdAt: new Date().toISOString(),
|
||||
lastUsedAt: null,
|
||||
archivedAt: null,
|
||||
},
|
||||
],
|
||||
120,
|
||||
);
|
||||
|
||||
expect(briefing).toBeDefined();
|
||||
expect(briefing!.length).toBeLessThanOrEqual(120);
|
||||
expect(briefing!.endsWith('…')).toBe(true);
|
||||
});
|
||||
});
|
||||
62
src/sqlite-memory-store.ts
Normal file
62
src/sqlite-memory-store.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import { type MemoryRecord, recallMemories } from './db.js';
|
||||
|
||||
const DEFAULT_PAGE_SIZE = 6;
|
||||
const DEFAULT_MAX_BRIEFING_CHARS = 2_000;
|
||||
|
||||
function trimToMaxChars(text: string, maxChars: number): string {
|
||||
if (text.length <= maxChars) return text;
|
||||
return text.slice(0, Math.max(0, maxChars - 1)).trimEnd() + '…';
|
||||
}
|
||||
|
||||
export function buildRoomMemoryKey(groupFolder: string): string {
|
||||
return `room:${groupFolder}`;
|
||||
}
|
||||
|
||||
export function formatRoomMemoryBriefing(
|
||||
roomKey: string,
|
||||
memories: MemoryRecord[],
|
||||
maxChars = DEFAULT_MAX_BRIEFING_CHARS,
|
||||
): string | undefined {
|
||||
if (memories.length === 0) return undefined;
|
||||
|
||||
const lines = memories
|
||||
.map((memory) => {
|
||||
const content = memory.content.trim();
|
||||
if (!content) return null;
|
||||
const prefix = memory.memoryKind ? `- [${memory.memoryKind}] ` : '- ';
|
||||
return `${prefix}${content}`;
|
||||
})
|
||||
.filter((line): line is string => Boolean(line));
|
||||
|
||||
if (lines.length === 0) return undefined;
|
||||
|
||||
const text = [
|
||||
'## Shared Room Memory',
|
||||
`Room key: \`${roomKey}\``,
|
||||
'Treat this as background context for a fresh session start. The current conversation always takes precedence.',
|
||||
...lines,
|
||||
].join('\n');
|
||||
|
||||
return trimToMaxChars(text, maxChars);
|
||||
}
|
||||
|
||||
export async function buildRoomMemoryBriefing(args: {
|
||||
groupFolder: string;
|
||||
groupName?: string;
|
||||
maxChars?: number;
|
||||
limit?: number;
|
||||
}): Promise<string | undefined> {
|
||||
void args.groupName;
|
||||
const roomKey = buildRoomMemoryKey(args.groupFolder);
|
||||
const memories = recallMemories({
|
||||
scopeKind: 'room',
|
||||
scopeKey: roomKey,
|
||||
limit: args.limit ?? DEFAULT_PAGE_SIZE,
|
||||
});
|
||||
|
||||
return formatRoomMemoryBriefing(
|
||||
roomKey,
|
||||
memories,
|
||||
args.maxChars ?? DEFAULT_MAX_BRIEFING_CHARS,
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user