feat(voice_worker): record history from the app and answer reply requests with the brain

Worker (voice_worker.spec.md "History and brain"):
- --history <file> keeps the conversation history store; --brain hosts the
  jarvis reply engine (JarvisBrain, settings point ollama_base_url at /v1).
- voice_join / voice_leave / member_join / member_leave keep presence; the
  session end closes it. Non-empty finals are stored under the speaker's name
  in the current voice channel.
- chat_message / chat_edit / chat_delete keep registered chat channels.
- reply builds the asker's scoped history context, runs the brain on a thread
  (one at a time), answers reply_text, stores a voice answer, and with
  speak=true says it under the same id. memory settings in hello and
  memory_config. brain_unavailable without --brain.

App (discord_core.spec.md "History reporting"):
- WorkerClient sends the new messages; the core reports the voice channel
  and its heard members after a join, a move and every worker reconnect,
  member joins and leaves, voice_leave when a session ends, and every message
  of a readable registered channel (its own too, with `to` on replies), edits
  and deletes.
- live_discord runs the worker with --history and checks the store at the
  end: 8 finals under the speaker's name, the edited chat message only,
  Jarvis's own messages, 5 presence intervals, none left open (passed, 107.8 s).

Eval: history recall gains a chat-record case and fails a reply that reads a
record stamp out (field run with qwen2.5:7b did once in four).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-11 13:16:24 +09:00
parent 7a288129de
commit 864b2e7fdb
13 changed files with 775 additions and 18 deletions

View File

@@ -14,8 +14,8 @@ use serenity::all::{
Cache, ChannelId, ChannelType, Client, CommandDataOptionValue, CommandInteraction, CommandOptionType, Context,
CreateAllowedMentions, CreateCommand, CreateCommandOption, CreateInteractionResponse,
CreateInteractionResponseMessage, CreateMessage, EditInteractionResponse, EventHandler, GatewayIntents, GetMessages,
GuildId, Http, InstallationContext, Interaction, InteractionContext, Message, Ready, ShardManager, UserId,
VoiceState,
GuildId, Http, InstallationContext, Interaction, InteractionContext, Message, MessageId, MessageUpdateEvent, Ready,
ShardManager, UserId, VoiceState,
};
use songbird::driver::DecodeMode;
use songbird::input::Input;
@@ -359,6 +359,7 @@ impl DiscordCore {
tokio::spawn(speaking_end_timer(Arc::downgrade(&self.inner), rx.clone(), alive.clone()));
*self.inner.session.lock().unwrap() = Some(Session { guild_id: gc.guild_id.get(), channel_id, rx, alive });
let _ = self.inner.player.send(PlayerCmd::SetCall(Some(call)));
report_voice_state(&self.inner);
debug_log!("voice", "joined voice {channel_id}");
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id });
let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice)
@@ -477,6 +478,7 @@ async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: b
inner.worker.speaker_leave(&user.to_string());
}
inner.gate.lock().unwrap().clear();
inner.worker.voice_leave();
if disconnect {
let _ = inner.songbird.remove(GuildId::new(s.guild_id)).await;
}
@@ -484,6 +486,27 @@ async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: b
let _ = inner.events.send(CoreEvent::Left { guild_id: s.guild_id, channel_id: s.channel_id, reason: reason.into() });
}
/// Tell the worker which voice channel Jarvis is in and who is there, so the history store knows
/// who heard what. After a join, a move, and every (re)connection of the worker.
fn report_voice_state(inner: &Inner) {
let Some((guild, channel)) = inner.session.lock().unwrap().as_ref().map(|s| (s.guild_id, s.channel_id)) else {
return;
};
inner.worker.voice_join(channel);
let users: Vec<u64> = inner.cache.get()
.and_then(|c| c.guild(GuildId::new(guild)).map(|g| g.voice_states.values()
.filter(|v| v.channel_id.map(|c| c.get()) == Some(channel))
.map(|v| v.user_id.get())
.collect()))
.unwrap_or_default();
let mut n = 0;
for user in users.into_iter().filter(|u| should_hear(inner, *u)) {
inner.worker.member_join(user, &name_of(inner, user));
n += 1;
}
debug_log!("voice", "reported voice {channel} with {n} members to the worker");
}
fn should_hear(inner: &Inner, user: u64) -> bool {
if Some(user) == inner.bot_id.get().copied() {
return false;
@@ -551,7 +574,8 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
WorkerEvent::Final(m) => on_final(&inner, m),
WorkerEvent::TtsAudio { id, seq, pcm } => with_playback(&inner, |p| p.audio(&id, seq, pcm)),
WorkerEvent::TtsDone(d) => with_playback(&inner, |p| p.done(&d)),
WorkerEvent::Ready(_) | WorkerEvent::Disconnected | WorkerEvent::Error(_) => {}
WorkerEvent::Ready(_) => report_voice_state(&inner),
WorkerEvent::Disconnected | WorkerEvent::Error(_) => {}
}
}
}
@@ -756,23 +780,65 @@ impl EventHandler for Handler {
if let Some(s) = inner.session.lock().unwrap().as_mut() {
s.channel_id = c;
}
report_voice_state(inner);
}
return;
}
let was_here = old.and_then(|o| o.channel_id).map(|c| c.get()) == Some(session_channel);
if was_here && new_channel != Some(session_channel) {
let is_here = new_channel == Some(session_channel);
if was_here && !is_here {
Receiver { core: Arc::downgrade(inner), rx }.left(inner, user);
inner.worker.member_leave(user);
} else if is_here && !was_here && should_hear(inner, user) {
// Also fires for a mute or deafen when the old state was not cached; the worker keeps
// one open presence per person, so that is harmless.
inner.worker.member_join(user, &name_of(inner, user));
}
}
async fn message(&self, _: Context, m: Message) {
let inner = &self.core.inner;
let ch = m.channel_id.get();
if !self.core.chat_content() || !inner.chat.can_read(ch) || Some(m.author.id.get()) == self.core.bot_id() {
if !self.core.chat_content() || !inner.chat.can_read(ch) {
return;
}
let c = to_chat(inner, &m);
// The history keeps Jarvis's own messages too; a reply to someone's message is an answer to them.
let to = c.from_self.then(|| m.referenced_message.as_ref().map(|r| r.author.id.get())).flatten();
inner.worker.chat_message(c.channel_id, c.message_id, c.author_id, &c.author_name, &c.content,
m.timestamp.unix_timestamp() as f64, to);
if c.from_self {
return;
}
debug_log!("chat", "chat message in {ch} from {} chars={}", m.author.id, m.content.chars().count());
let _ = inner.events.send(CoreEvent::Chat(to_chat(inner, &m)));
let _ = inner.events.send(CoreEvent::Chat(c));
}
async fn message_update(&self, _: Context, _: Option<Message>, _: Option<Message>, e: MessageUpdateEvent) {
let ch = e.channel_id.get();
if !self.core.chat_content() || !self.core.inner.chat.can_read(ch) {
return;
}
if let Some(text) = &e.content {
debug_log!("chat", "chat message {} edited in {ch}", e.id);
self.core.inner.worker.chat_edit(ch, e.id.get(), text);
}
}
async fn message_delete(&self, _: Context, channel: ChannelId, id: MessageId, _: Option<GuildId>) {
if self.core.inner.chat.can_read(channel.get()) {
debug_log!("chat", "chat message {id} deleted in {channel}");
self.core.inner.worker.chat_delete(channel.get(), id.get());
}
}
async fn message_delete_bulk(&self, _: Context, channel: ChannelId, ids: Vec<MessageId>, _: Option<GuildId>) {
if self.core.inner.chat.can_read(channel.get()) {
debug_log!("chat", "{} chat messages deleted in {channel}", ids.len());
for id in ids {
self.core.inner.worker.chat_delete(channel.get(), id.get());
}
}
}
async fn interaction_create(&self, _: Context, interaction: Interaction) {

View File

@@ -91,6 +91,22 @@ are dropped (not queued), and every TTS request in flight ends as failed, so pla
request the worker refuses (`tts_unavailable`) also ends as failed. TTS audio for request ids the client
did not open is ignored.
### History reporting (M3)
The core tells the worker what its history store needs (`voice_worker.spec.md`, "History and brain"):
- `voice_join` with the channel after every join or move (also when someone else moves the bot), and
`member_join` with the display name for everyone already in the channel that the bot hears (people;
bots only with `voice.listenToBots`). Again whenever the worker link becomes ready, because messages
sent while disconnected are dropped.
- `member_join` / `member_leave` as heard people enter and leave the channel.
- `voice_leave` whenever a session ends (leave, move, disconnect).
- For every message in a readable registered channel, the bot's own included: `chat_message` with
Discord's time; on the bot's own message that replies to someone, `to` is that person. Edits
(`chat_edit`), deletes and bulk deletes (`chat_delete`) follow. Chat sent while the worker is
disconnected is not recorded.
- Finals are recorded by the worker itself; the core does not echo them.
## Voice
One voice session at a time, in the configured server. Joining another channel ends the current
@@ -175,14 +191,18 @@ registered until it is removed (M5 setup).
voice channel with the real worker on the GPU. It refuses to run if a person is in the channel or
either bot is already in a voice channel, and cleans up even when a step fails. It checks command
registration, transcription of recorded sentences played into the channel, speech the other bot
hears and the STT reads back, stop, chat events, history, sending without pings, moving to a second
hears and the STT reads back, stop, chat events, history, an edit and a delete, sending without pings, moving to a second
voice channel and back (the session survives each move and only the two moves end one), the speaker
leaving and coming back three times (each a DAVE group change), and leaving. It needs a second voice
leaving and coming back three times (each a DAVE group change), and leaving. The worker runs with
`--history`; at the end the store must hold the speaker's finals under its display name and channel,
its chat message with the edit and without the deleted one, Jarvis's own messages, a presence
interval per join, and no presence left open. It needs a second voice
channel of the server (`JARVIS_LIVE_VOICE_CHANNEL_2`) and refuses if a person is in either.
Measured on this server's RTX 5050 (2026-11-11): five sentences all heard (character error rate 0%
for three, 5% and 12.5% for two), 0.71-0.90 s from the end of speech to the final, Jarvis's speech
read back at 3.4-6.9%, sound gone 0.03-0.08 s after stop, and the sentence heard exactly after each
of three rejoins.
of three rejoins. With history (2026-11-11): 8 voice records under the speaker's name, the edited
chat message only, 3 of Jarvis's own messages, 5 presence intervals, none open.
## Known risks

View File

@@ -221,6 +221,48 @@ impl WorkerClient {
self.send_json(json!({"type": "speaker_leave", "speaker": speaker}));
}
// --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the
// discord core sends the voice state again when the worker is ready. ----------------------
/// Jarvis is in (or moved to) this voice channel.
pub fn voice_join(&self, channel: u64) {
self.send_json(json!({"type": "voice_join", "channel": channel.to_string()}));
}
pub fn voice_leave(&self) {
self.send_json(json!({"type": "voice_leave"}));
}
pub fn member_join(&self, user: u64, name: &str) {
self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name}));
}
pub fn member_leave(&self, user: u64) {
self.send_json(json!({"type": "member_leave", "user": user.to_string()}));
}
/// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages.
#[allow(clippy::too_many_arguments)]
pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64,
to: Option<u64>) {
let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(),
"author": author.to_string(), "name": name, "text": text, "ts": ts});
if let Some(to) = to {
v["to"] = Value::String(to.to_string());
}
self.send_json(v);
}
pub fn chat_edit(&self, channel: u64, message: u64, text: &str) {
self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(),
"text": text}));
}
pub fn chat_delete(&self, channel: u64, message: u64) {
self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(),
"message_id": message.to_string()}));
}
/// Change STT settings now and for every later connection.
pub fn configure(&self, changes: Map<String, Value>) {
self.inner.state.lock().unwrap().stt.extend(changes.clone());

View File

@@ -163,9 +163,12 @@ async fn discord_voice_core_on_real_discord() {
}
// The real voice worker.
let mut args = vec![];
// A fresh history store: the worker records what this test says and who was there.
let history_db = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("live-history-{}.sqlite", std::process::id()));
let _ = std::fs::remove_file(&history_db);
let mut args = vec!["--history".to_string(), history_db.to_string_lossy().into_owned()];
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
args = vec!["--melo-python".to_string(), m];
args.extend(["--melo-python".to_string(), m]);
}
let wp = WorkerProcess::new(WorkerProcessOptions::new(env("JARVIS_VOICE_PYTHON"), root.join("../src"), args)).unwrap();
let mut pev = wp.subscribe();
@@ -325,6 +328,14 @@ async fn discord_voice_core_on_real_discord() {
let history = core.read_chat(voice, 20).await.unwrap();
assert!(history.iter().any(|m| m.content == marker && !m.from_self));
assert!(history.iter().any(|m| m.from_self && m.content.contains("받아 적습니다")), "join notice missing");
// An edit and a delete reach the history store.
let mut marker_msg = ChannelId::new(voice).messages(&s_http, serenity::all::GetMessages::new().limit(5)).await.unwrap()
.into_iter().find(|m| m.content == marker).unwrap();
let edited = format!("{marker} 수정");
marker_msg.edit(&s_http, serenity::all::EditMessage::new().content(&edited)).await.unwrap();
let doomed = ChannelId::new(voice).send_message(&s_http, CreateMessage::new().content("곧 지울 메시지")).await.unwrap();
tokio::time::sleep(Duration::from_millis(800)).await;
doomed.delete(&s_http).await.unwrap();
let reply = format!("자비스 답장 @everyone {}", stop_at.elapsed().as_nanos());
let ids = core.send_chat(voice, &reply).await.unwrap();
let sent = ChannelId::new(voice).message(&s_http, serenity::all::MessageId::new(ids[0])).await.unwrap();
@@ -387,6 +398,41 @@ async fn discord_voice_core_on_real_discord() {
assert_eq!(core.voice_channel_id(), None);
assert!(core.say("아무도 못 듣는 말", false).is_none());
// 9. The history store: the speaker's finals under its name, its chat with the edit and without
// the deleted message, Jarvis's own messages, and presence closed now that Jarvis left.
tokio::time::sleep(Duration::from_millis(500)).await;
let check = r#"
import json, sys
from jarvis.memory.history import HistoryStore
s = HistoryStore(sys.argv[1]); sp, jv = sys.argv[2], sys.argv[3]
own = s.records_of(sp)
q = lambda sql, *a: s._conn.execute(sql, a).fetchone()[0]
print(json.dumps({"voice": [[r.author_name, r.channel_id, r.text] for r in own if r.kind == "voice"],
"chat": [r.text for r in own if r.kind == "chat"], "jarvis_chat": [r.text for r in s.records_of(jv) if r.kind == "chat"],
"presence": q("select count(*) from presence where user_id = ?", sp), "open": q("select count(*) from presence where left_at is null")},
ensure_ascii=False))
"#;
let out = std::process::Command::new(env("JARVIS_VOICE_PYTHON")).current_dir(root.join("../src"))
.args(["-c", check, &history_db.to_string_lossy(), &speaker_id.to_string(), &jarvis.to_string()])
.output().unwrap();
assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
let h: Value = serde_json::from_slice(&out.stdout).unwrap();
let voice_recs = h["voice"].as_array().unwrap();
let speaker_name = s_cache.user(UserId::new(speaker_id)).map(|u| u.display_name().to_owned()).unwrap_or_default();
report.insert("history".into(), json!({"voice_records": voice_recs.len(), "presence_intervals": h["presence"],
"open_presence": h["open"], "chat": h["chat"], "jarvis_chat": h["jarvis_chat"].as_array().map(|a| a.len())}));
assert!(voice_recs.len() >= 8, "finals not stored: {h}");
assert!(voice_recs.iter().all(|r| r[0].as_str() != Some(&speaker_id.to_string()) && !r[2].as_str().unwrap().is_empty()),
"a final without the speaker's name: {h}");
assert!(voice_recs.iter().any(|r| r[0].as_str() == Some(speaker_name.as_str())), "{speaker_name} not in {h}");
assert!(voice_recs.iter().any(|r| r[1] == voice.to_string()), "{h}");
let chat: Vec<&str> = h["chat"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
assert_eq!(chat, [edited.as_str()], "speaker chat in the history");
let jarvis_chat: Vec<&str> = h["jarvis_chat"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
assert!(jarvis_chat.contains(&reply.as_str()) && jarvis_chat.iter().any(|t| t.contains("받아 적습니다")), "{h}");
assert!(h["presence"].as_u64().unwrap() >= 4, "joins and rejoins open presence: {h}");
assert_eq!(h["open"], 0, "presence left open after Jarvis left: {h}");
println!("live report {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
})).await;

View File

@@ -228,3 +228,34 @@ async fn tts_while_disconnected_fails_at_once() {
let e = next_matching(&mut rx, |_| true).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
}
#[tokio::test]
async fn history_messages_follow_the_worker_protocol() {
let w = FakeWorker::start(0).await;
let (c, _rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
c.voice_join(42);
c.member_join(7, "Alice");
c.member_leave(7);
c.chat_message(9, 100, 7, "Alice", "hi there", 1_800_000_000.5, None);
c.chat_message(9, 101, 1, "Jarvis", "hello", 1_800_000_001.0, Some(7));
c.chat_edit(9, 100, "hi, edited");
c.chat_delete(9, 100);
c.voice_leave();
until(|| w.got().len() >= 9).await;
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
assert_eq!(got, vec![
json!({"type": "voice_join", "channel": "42"}),
json!({"type": "member_join", "user": "7", "name": "Alice"}),
json!({"type": "member_leave", "user": "7"}),
json!({"type": "chat_message", "channel": "9", "message_id": "100", "author": "7", "name": "Alice",
"text": "hi there", "ts": 1_800_000_000.5}),
json!({"type": "chat_message", "channel": "9", "message_id": "101", "author": "1", "name": "Jarvis",
"text": "hello", "ts": 1_800_000_001.0, "to": "7"}),
json!({"type": "chat_edit", "channel": "9", "message_id": "100", "text": "hi, edited"}),
json!({"type": "chat_delete", "channel": "9", "message_id": "100"}),
json!({"type": "voice_leave"}),
]);
c.close();
}

View File

@@ -7,7 +7,7 @@
| M0 | 게임PC 사전 검증 | 측정 스크립트를 게임PC에서 직접 실행하고 결과 파일을 보내 주시는 방식입니다. 결과로 기본값을 확정합니다. 확인 항목은 아래 목록에 있습니다. 가상 모니터 확인을 뺀 모든 항목 측정 완료(2026-11-10). 가상 모니터 확인은 M6에서 합니다. 2026-11-10에 추가한 `desktop` 단계(agent-desktop)는 측정을 마쳤고, 자동 성능 조절용 `gamemode` 단계는 세 번 쟀고, `gamemode_stt`(LLM과 STT를 함께 켬, STT GPU·CPU)와 `gamemode_live`(실제 게임을 켬) 단계도 측정을 마쳤습니다(2026-11-10). 결과로 게임 중 최소 배치는 전문가 2GB로 정했고, 평소 배치에 STT 자리를 남기는 것과 게임 중 LLM이 답하는 동안 부분 전사 간격을 늘리는 것(게임이 없을 때는 그대로)도 정했습니다(사용자 결정, 2026-11-10). `gamemode_tune` 단계도 측정을 마쳤고(2026-11-11), 게임 중 LLM이 답하는 동안에는 부분 전사를 멈추기로 정했습니다. GPU 95% 제한은 측정에서 사용률을 낮추지 못해 기본값을 끔으로 바꿨고(사용자 결정, 2026-11-11), `video_drops` 단계(Jarvis를 돌리는 동안 유튜브 영상의 끊긴 프레임, 직접 실행)도 측정을 마쳤습니다(2026-11-11). Jarvis가 바쁠 때도 영상이 끊기지 않았고 제한이 끊김을 줄이지 못해 설정을 뺐습니다. 대신 게임 화면의 프레임을 직접 재는 `game_frames` 단계(게임을 켠 채로 Jarvis를 끈 때·쉴 때·바쁠 때와 우선순위를 낮춘 양보 상태, 직접 실행, 관리자 권한)를 넣었고 측정을 마쳤습니다(2026-11-11). Jarvis가 바쁠 때도 게임의 평균 fps와 1% low가 5% 넘게 떨어지지 않아 양보는 넣지 않기로 했습니다. 남은 M0 측정은 가상 모니터 확인(M6)입니다. 측정 키트: [scripts/m0/](../../scripts/m0/README.md) |
| M1 | 음성 워커 | 화자별 스트리밍 STT(공용 모델, 배치), 문장 단위 TTS를 WebSocket으로 제공. 녹음 파일 10개 동시 입력 테스트 통과. 구현과 테스트 완료(2026-11-11): `src/voice_worker/`, 스펙은 [voice_worker.spec.md](../../src/voice_worker/voice_worker.spec.md). 이 서버의 RTX 5050에서 10명이 겹쳐 말하는 실시간 입력(48kHz 스테레오, M0 한국어 샘플 8개를 10명에게 나눠 줌, 절반은 디스코드 speaking 종료, 절반은 무음으로 끝남)을 넣어 10명 모두 최종 전사를 받았고 문자 오류율은 평균 4%, 최대 19%("일곱 시"를 "7시"로 적은 것), 말 끝 판정부터 최종 전사 전송까지 중앙값 0.64초, 최대 1.56초였습니다. TTS는 첫 문장 소리가 0.2초에 나왔고, 그 소리를 다시 넣으면 같은 문장으로 받아 적혔습니다. 샘플이 MeloTTS 합성 음성이라 실제 사람 녹음과 게임PC 수치는 아직 없습니다 |
| M2 | 디스코드 음성 코어 이전 | v1 봇 기능을 새 앱 안으로 이전. 입장/퇴장, 화자별 수신, 재생, 정지 명령 동작. 채팅 채널 연결(읽기·보내기). 구현과 테스트 완료(2026-11-11): `app/`(Rust, serenity + songbird), 스펙은 [discord_core.spec.md](../../app/src/discord_core.spec.md). 처음 Electron + TypeScript로 만든 것을 사용자 요청으로 Rust로 다시 씀. 앱이 음성 워커를 띄우고(실행마다 새 토큰, 죽으면 다시 시작) 연결이 끊기면 다시 붙습니다. 음성 채널에 들어갈 때마다 전사 고지를 채널 채팅에 올립니다. songbird 0.6.1의 DAVE 수신 해독이 조각 3분의 1가량을 깨뜨려서(20ms마다 음이 바뀌는 시험 신호로 확인, 보내기는 정상) 받은 패킷의 순서 맞추기와 Opus 해독은 앱이 직접 합니다. 실제 디스코드에서 봇 두 개로 확인(이 서버의 RTX 5050): 다른 봇이 채널에 튼 녹음 문장 5개를 모두 받아 적었고(문자 오류율 3개 0%, "기록해"를 "비록해"로 적은 1개 5%, "일곱 시"를 "7시"로 적은 1개 12.5%), 말이 끝난 뒤 최종 전사까지 0.71~0.90초였습니다. 자비스가 말한 문장을 상대 봇이 받아 다시 전사하니 오류율 3.4%였고, 정지 명령 뒤 0.08초 만에 소리가 끊겼습니다. 채팅 채널은 새 글 받기, 최근 글 읽기, 보내기(2,000자 넘으면 나눔, 아무도 호출하지 않음)가 됩니다. 사람이 직접 말한 소리와 게임PC(Windows) 빌드는 아직 확인하지 않았고, 음성으로 "자비스, 멈춰"는 호출 판정이 필요해 M4에서 합니다 |
| M3 | 두뇌 연결 | 기존 jarvis를 Ollama 대신 Strata 주소로 연결. 대화 기록 저장소와 지난 대화 전체 검색(본인 기록 + 등록한 채팅 채널 + 같은 음성 채널의 다른 사람 말). 채팅 채널 도구(읽기·검색·보내기·수정·반응·삭제·고정). 메모리, 날씨, 웹 검색, 식단, MCP 기존 테스트와 평가(evals) 통과. 1단계 완료(2026-11-11): LLM 주소가 `/v1`로 끝나면 OpenAI 호환 방식(`/chat/completions`, `/embeddings`)으로 부름. 형식 변환은 `src/jarvis/llm.py` 한곳에서 하고 두뇌의 나머지 코드는 그대로 씀. 대화, 도구 호출, 스트리밍, 의도 판정, 받아쓰기 정리, 미리 불러오기가 모두 이 방식으로 감. 이 서버 Ollama의 `/v1` 주소와 qwen2.5:7b로 실제 두뇌 한 턴을 돌려 "서울 날씨 어때?"에 날씨 도구를 부르고 그 결과로 답하는 것을 확인함. Strata에는 임베딩 모델이 없어 지난 기록 검색은 낱말 검색으로 함. 2단계 완료(2026-11-11): 대화 기록 저장소 `src/jarvis/memory/history.py`(스펙 [history.spec.md](../../src/jarvis/memory/history.spec.md)). 음성 전사, 자비스의 답, 채팅 채널 글, 음성 채널 출입 시간을 저장하고, 묻는 사람이 볼 수 있는 기록(본인 기록, 채팅 채널, 함께 있던 동안의 다른 사람 말)만 찾음. 전체 기간에서 관련도 순으로 찾고, 기록 10만 개에서 검색 48~143ms. 두뇌는 찾은 기록을 받아 지시문에 참고 자료로 넣음(평가 `evals/test_history_context_recall.py`, qwen2.5:7b에서 3번 모두 통과). 남은 일: 앱이 음성 워커에 전사·채팅·출입을 알리고 워커가 두뇌를 불러 답하는 연결, 채팅 채널 도구, 기존 평가 통과 확인 |
| M3 | 두뇌 연결 | 기존 jarvis를 Ollama 대신 Strata 주소로 연결. 대화 기록 저장소와 지난 대화 전체 검색(본인 기록 + 등록한 채팅 채널 + 같은 음성 채널의 다른 사람 말). 채팅 채널 도구(읽기·검색·보내기·수정·반응·삭제·고정). 메모리, 날씨, 웹 검색, 식단, MCP 기존 테스트와 평가(evals) 통과. 1단계 완료(2026-11-11): LLM 주소가 `/v1`로 끝나면 OpenAI 호환 방식(`/chat/completions`, `/embeddings`)으로 부름. 형식 변환은 `src/jarvis/llm.py` 한곳에서 하고 두뇌의 나머지 코드는 그대로 씀. 대화, 도구 호출, 스트리밍, 의도 판정, 받아쓰기 정리, 미리 불러오기가 모두 이 방식으로 감. 이 서버 Ollama의 `/v1` 주소와 qwen2.5:7b로 실제 두뇌 한 턴을 돌려 "서울 날씨 어때?"에 날씨 도구를 부르고 그 결과로 답하는 것을 확인함. Strata에는 임베딩 모델이 없어 지난 기록 검색은 낱말 검색으로 함. 2단계 완료(2026-11-11): 대화 기록 저장소 `src/jarvis/memory/history.py`(스펙 [history.spec.md](../../src/jarvis/memory/history.spec.md)). 음성 전사, 자비스의 답, 채팅 채널 글, 음성 채널 출입 시간을 저장하고, 묻는 사람이 볼 수 있는 기록(본인 기록, 채팅 채널, 함께 있던 동안의 다른 사람 말)만 찾음. 전체 기간에서 관련도 순으로 찾고, 기록 10만 개에서 검색 48~143ms. 두뇌는 찾은 기록을 받아 지시문에 참고 자료로 넣음(평가 `evals/test_history_context_recall.py`, qwen2.5:7b에서 3번 모두 통과). 3단계 완료(2026-11-11): 앱이 음성 워커에 음성 채널 입장·퇴장, 사람 출입, 등록한 채팅 채널의 글·수정·삭제를 알리고, 워커가 최종 전사를 저장함. 워커는 `--history`로 저장소를, `--brain`으로 두뇌를 켜고, `reply` 요청에 묻는 사람의 범위로 찾은 기록을 넣어 답하고(`reply_text`) 음성이면 바로 말함. 실제 디스코드 시험(봇 두 개)에서 전사 8개가 이름과 함께 저장되고, 채팅 수정은 반영, 삭제한 글은 빠지고, 자비스가 나간 뒤 출입 기록이 모두 닫힘. 워커와 실제 두뇌(이 서버 Ollama `/v1`, qwen2.5:7b)로 다른 사람이 채팅에 남긴 약속을 묻자 맞게 답함. 언제 `reply`를 보낼지(호출 판정)와 음성 답 재생 연결은 M4. 남은 일: 채팅 채널 도구, 기존 평가 통과 확인 |
| M4 | 왕복 완성과 지연 튜닝 | 호출어·호출 방식·지정한 사람 설정, 동시 대화 응답 정책 4가지(기본 "처음 대화만")와 정책별 끼어들기 규칙. 말 끝부터 첫 소리까지 실측 1.5초 이내, 10명 채널 부하 테스트 |
| M5 | 관리 앱 기본 | 첫 실행 마법사, 데이터 폴더, 런타임·모델 자동 설치, 대시보드, 봇/채널·호출/채팅 채널(디스코드 채널 이름·주제 반영 포함)/대화 기억/모델 설정, 로그 화면(보기·복사·다운로드) |
| M6 | 기능 화면과 브라우저 | 기능 켜기/끄기, MCP 관리, 봇용 크롬(가상 모니터에 둠, Virtual Display Driver 포함), 예약 수집 틀(대상은 비움). 디스코드 "화면" 탭 공유, 커서, 모니터 해제 때 창 위치 확인 |

View File

@@ -8,13 +8,15 @@ model actually uses a matching earlier record to answer a question about a
past conversation, and that the block is what makes the difference (the same
question without the block cannot be answered from anything else).
The fact is a day of the week that the model cannot guess, so the check is a
The reply must answer in its own words, not read a record line out (TTS would
speak the stamp). The fact is a day of the week that the model cannot guess, so the check is a
plain substring on the reply (in either language, since the reply language is a
setting).
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_history_context_recall.py -v
"""
import re
from unittest.mock import patch
import pytest
@@ -41,8 +43,21 @@ CASES = [
"matches": ["[2026-11-08 20:10] Alice: the cat's vet appointment is on Thursday at 3pm"],
"expect": ["thursday"],
},
{
# Field run 2026-11-11 (qwen2.5:7b through the voice worker): a chat channel record from
# someone else, asked about by the user. The reply opened by reading the record line out,
# stamp and all, which TTS would speak.
"id": "korean-chat-record",
"query": "이번 주 모임 언제 어디라고 했지?",
"recent": [],
"matches": ["[2026-11-11 11:55] 민수: 이번 주 모임은 토요일 오후 네 시에 강남역이야"],
"expect": ["토요일", "saturday"],
},
]
# A record line read out: any "[YYYY-MM-DD HH:MM]" stamp in the reply.
_STAMP = re.compile(r"\[\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}\]")
def _run(mock_config, eval_db, eval_dialogue_memory, query, history):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
@@ -77,6 +92,9 @@ class TestHistoryContextRecall:
assert any(e in lowered for e in case["expect"]), (
f"[{case['id']}] reply did not use the earlier record: {response!r}"
)
assert not _STAMP.search(response or ""), (
f"[{case['id']}] reply read a record line out instead of answering: {response!r}"
)
@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
def test_without_the_block_the_fact_is_unknown(self, mock_config, eval_db, eval_dialogue_memory, case):

View File

@@ -21,7 +21,13 @@ def main() -> int:
ap.add_argument("--melo-device", default="cuda")
ap.add_argument("--melo-speed", type=float, default=1.5)
ap.add_argument("--melo-language", default="KR")
ap.add_argument("--history", default=None, help="conversation history SQLite file; no history without it")
ap.add_argument("--brain", action="store_true",
help="answer reply requests with the jarvis reply engine (needs --history)")
args = ap.parse_args()
if args.brain and not args.history:
print("--brain needs --history", file=sys.stderr)
return 2
token = os.environ.get("JARVIS_VOICE_TOKEN", "")
if not token:
@@ -44,10 +50,19 @@ def main() -> int:
from .tts_melo import MeloEngine
tts = MeloEngine(args.melo_python, args.melo_device, args.melo_speed, args.melo_language)
history = brain = None
if args.history:
from jarvis.memory.history import HistoryStore
history = HistoryStore(args.history)
if args.brain:
from .brain import JarvisBrain
brain = JarvisBrain()
async def run() -> None:
worker = VoiceWorker(token, transcriber, vad.stream, tts)
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain)
server = await worker.serve(args.host, args.port)
debug_log(f"voice worker listening on {args.host}:{args.port} tts={'on' if tts else 'off'}",
debug_log(f"voice worker listening on {args.host}:{args.port} tts={'on' if tts else 'off'} "
f"history={'on' if history else 'off'} brain={'on' if brain else 'off'}",
"voice_worker")
print(f"READY {args.host}:{args.port}", flush=True)
await server.serve_forever()
@@ -59,6 +74,8 @@ def main() -> int:
finally:
if tts:
tts.close()
if history:
history.close()
return 0

99
src/voice_worker/brain.py Normal file
View File

@@ -0,0 +1,99 @@
"""The brain the voice worker hosts: memory settings and the jarvis reply engine behind a small
interface, so the worker logic is tested with a fake (voice_worker.spec.md, "History and brain")."""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass
from typing import Optional, Protocol
from jarvis.debug import debug_log
from jarvis.memory.history import HistoryContext, Scope
class Brain(Protocol):
def reply(self, user: str, text: str, history: HistoryContext) -> Optional[str]:
"""Blocking; called on a worker thread, one call at a time."""
@dataclass
class MemorySettings:
chat: bool = True
others: bool = True
off: bool = False
since_days: Optional[float] = None
recent_minutes: float = 30
recent_records: int = 20
matches: int = 5
budget_tokens: float = 4000
_BOOLS = ("chat", "others", "off")
_NUMBERS = {"recent_minutes": (0, 24 * 60), "recent_records": (0, 200), "matches": (0, 20),
"budget_tokens": (0, 100_000)}
def update(self, changes: dict) -> None:
"""Apply ``changes`` atomically; raises ValueError naming the first bad field."""
new = dict(self.__dict__)
for key, value in changes.items():
if key in self._BOOLS:
if not isinstance(value, bool):
raise ValueError(f"{key} must be true or false")
elif key == "since_days":
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
or value <= 0):
raise ValueError("since_days must be a positive number or null")
elif key in self._NUMBERS:
lo, hi = self._NUMBERS[key]
if isinstance(value, bool) or not isinstance(value, (int, float)) or not lo <= value <= hi:
raise ValueError(f"{key} must be a number from {lo} to {hi}")
if key in ("recent_records", "matches"):
value = int(value)
else:
raise ValueError(f"unknown memory setting: {key}")
new[key] = value
self.__dict__.update(new)
def scope(self, now: float) -> Scope:
since = now - self.since_days * 86400 if self.since_days else None
return Scope(chat=self.chat, others=self.others, since=since, off=self.off)
class JarvisBrain:
"""The existing jarvis reply engine with the user's history context.
Uses the jarvis settings (``ollama_base_url`` is Strata's ``/v1`` in v2) and its database for
the long-term memory it already keeps. Each user has their own short-term dialogue memory so
tool carry-over follows the person who asked.
"""
def __init__(self, language: Optional[str] = None):
from jarvis.config import load_settings
from jarvis.memory.db import Database
self.cfg = load_settings()
self.db = Database(self.cfg.db_path, self.cfg.sqlite_vss_path)
self.language = language
self._memories: dict = {}
self._lock = threading.Lock()
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "voice_worker")
def _memory(self, user: str):
from jarvis.memory.conversation import DialogueMemory
with self._lock:
mem = self._memories.get(user)
if mem is None:
mem = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20)
self._memories[user] = mem
return mem
def reply(self, user: str, text: str, history: HistoryContext) -> Optional[str]:
from jarvis.reply.engine import run_reply_engine
t0 = time.perf_counter()
answer = run_reply_engine(self.db, self.cfg, None, text, self._memory(user),
language=self.language, history=history)
debug_log(f"brain reply user={user} ok={bool(answer)} ms={(time.perf_counter() - t0) * 1000:.0f}",
"voice_worker")
return answer

View File

@@ -10,3 +10,5 @@ numpy
python-dotenv==1.0.1
# MeloTTS runs in its own Python 3.11 venv (see the "melo" entry in scripts/m0/m0lib/envs.py) and is
# reached through melo_child.py; pass that venv's python with --melo-python.
# --history needs nothing more. --brain runs the jarvis reply engine and needs its dependencies too
# (the brain part of the repository's requirements.txt: requests, mcp, beautifulsoup4, ...).

View File

@@ -16,6 +16,7 @@ import numpy as np
from jarvis.debug import debug_log
from .audio import InputConverter, loudness_equalise
from .brain import Brain, MemorySettings
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
@@ -38,7 +39,8 @@ def is_loopback(host: str) -> bool:
class VoiceWorker:
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05):
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
history=None, brain: Brain | None = None):
if not token:
raise ValueError("a connection token is required")
self.token = token
@@ -47,6 +49,9 @@ class VoiceWorker:
self.tts_engine = tts_engine
self.hello_timeout_s = hello_timeout_s
self.tick_s = tick_s
self.history = history # jarvis.memory.history.HistoryStore or None
self.brain = brain
self.brain_lock = asyncio.Lock() # one reply at a time
self.scheduler: SttScheduler | None = None
self.tts: TtsService | None = None
self._ids = itertools.count(1)
@@ -129,6 +134,14 @@ class Session:
if not isinstance(stt, dict):
raise ValueError("stt must be an object")
self.settings.update(stt)
self.memory = MemorySettings()
memory = hello.get("memory")
if memory is not None:
if not isinstance(memory, dict):
raise ValueError("memory must be an object")
self.memory.update(memory)
self.voice_channel: str | None = None
self.names: dict[str, str] = {}
self.speakers: dict[str, _Speaker] = {}
self.tts_requests: dict[str, TtsRequest] = {}
self.out: asyncio.Queue = asyncio.Queue()
@@ -164,6 +177,7 @@ class Session:
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
finally:
self.closed = True
self._leave_voice()
ticker.cancel()
writer.cancel()
if w.scheduler:
@@ -231,6 +245,9 @@ class Session:
return cb
def _send_final(self, speaker: str, ev: Final, res: SttResult) -> None:
store = self.worker.history
if store is not None and self.voice_channel and res.text.strip():
store.add_voice(self.voice_channel, speaker, self.names.get(speaker, speaker), res.text, time.time())
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
@@ -247,7 +264,12 @@ class Session:
return
kind = msg.get("type")
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel}.get(kind)
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
"chat_delete": self._on_chat_delete, "reply": self._on_reply,
"memory_config": self._on_memory_config}.get(kind)
if handler is None:
self.error("bad_message", f"unknown type: {kind!r}")
return
@@ -304,6 +326,125 @@ class Session:
if req is not None and self.worker.tts:
self.worker.tts.cancel(req)
# --- history and brain ----------------------------------------------------------------------
def _strings(self, msg: dict, *keys: str) -> tuple | None:
values = tuple(msg.get(k) for k in keys)
if not all(isinstance(v, str) and v for v in values):
self.error("bad_message", f"{msg.get('type')} needs non-empty strings: {', '.join(keys)}",
**({"id": msg["id"]} if isinstance(msg.get("id"), str) else {}))
return None
return values
def _leave_voice(self) -> None:
if self.voice_channel and self.worker.history is not None:
self.worker.history.presence_close_channel(self.voice_channel, time.time())
if self.voice_channel:
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
self.voice_channel = None
def _on_voice_join(self, msg: dict) -> None:
got = self._strings(msg, "channel")
if got is None:
return
self._leave_voice()
self.voice_channel = got[0]
debug_log(f"session {self.sid} in voice channel {self.voice_channel}", "voice_worker")
def _on_member_join(self, msg: dict) -> None:
got = self._strings(msg, "user", "name")
if got is None:
return
user, name = got
self.names[user] = name
if self.voice_channel and self.worker.history is not None:
self.worker.history.presence_join(user, self.voice_channel, time.time())
def _on_member_leave(self, msg: dict) -> None:
got = self._strings(msg, "user")
if got is not None and self.voice_channel and self.worker.history is not None:
self.worker.history.presence_leave(got[0], self.voice_channel, time.time())
def _on_chat_message(self, msg: dict) -> None:
got = self._strings(msg, "channel", "message_id", "author", "name")
ts, to, text = msg.get("ts"), msg.get("to"), msg.get("text")
if got is None:
return
if not isinstance(text, str) or isinstance(ts, bool) or not isinstance(ts, (int, float)) \
or (to is not None and not isinstance(to, str)):
self.error("bad_message", "chat_message needs text, a numeric ts and an optional string to")
return
if self.worker.history is not None:
channel, message_id, author, name = got
self.worker.history.add_chat(channel, message_id, author, name, text, float(ts), to_id=to)
def _on_chat_edit(self, msg: dict) -> None:
got = self._strings(msg, "channel", "message_id")
if got is None:
return
if not isinstance(msg.get("text"), str):
self.error("bad_message", "chat_edit needs text")
return
if self.worker.history is not None:
self.worker.history.edit_chat(got[0], got[1], msg["text"])
def _on_chat_delete(self, msg: dict) -> None:
got = self._strings(msg, "channel", "message_id")
if got is not None and self.worker.history is not None:
self.worker.history.delete_chat(*got)
def _on_memory_config(self, msg: dict) -> None:
changes = {k: v for k, v in msg.items() if k != "type"}
try:
self.memory.update(changes)
except ValueError as e:
self.error("bad_message", str(e))
return
debug_log(f"session {self.sid} memory settings changed: {sorted(changes)}", "voice_worker")
def _on_reply(self, msg: dict) -> None:
got = self._strings(msg, "id", "user", "text")
if got is None:
return
rid, user, text = got
via, speak = msg.get("via"), msg.get("speak", False)
if via not in ("voice", "chat") or not isinstance(speak, bool) or len(rid.encode()) > 255:
self.error("bad_message", "reply needs via voice|chat, a boolean speak and an id under 256 bytes",
id=rid)
return
if self.worker.brain is None:
self.error("brain_unavailable", "the worker was started without --brain", id=rid)
return
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak))
async def _reply(self, rid: str, user: str, text: str, via: str, speak: bool) -> None:
w = self.worker
channel = self.voice_channel
async with w.brain_lock:
if self.closed:
return
now = time.time()
if w.history is not None:
m = self.memory
history = await asyncio.to_thread(
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
else:
from jarvis.memory.history import HistoryContext
history = HistoryContext()
try:
answer = await asyncio.to_thread(w.brain.reply, user, text, history)
except Exception as e: # the brain must never take the session down
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "voice_worker")
answer = None
answer = (answer or "").strip()
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
if not answer:
return
if via == "voice" and channel and w.history is not None:
w.history.add_reply(channel, user, answer, time.time())
if speak:
self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False})
def _tts_chunk_cb(self, rid: str):
def cb(seq: int, pcm: np.ndarray) -> None:
pcm = np.asarray(pcm, dtype="<i2")

View File

@@ -24,6 +24,7 @@ WebSocket. Design background lives in `docs/v2/voice-and-llm.md` and `docs/v2/ar
JARVIS_VOICE_TOKEN=<secret> python -m voice_worker --host 127.0.0.1 --port 8771 \
[--stt-model large-v3-turbo] [--stt-device cuda] [--stt-compute int8_float16] \
[--melo-python <path to the MeloTTS venv python>] [--melo-device cuda] [--melo-speed 1.5]
[--history <history.sqlite>] [--brain]
```
Run from `src/` (or with `src/` on `PYTHONPATH`) in a venv built from `voice_worker/requirements.txt`. MeloTTS needs its own Python 3.11 venv
@@ -176,10 +177,57 @@ is ready. Requests are served first come, first served by one TTS engine. `tts_c
request's unsent sentences, and `tts_done` follows with `cancelled: true`. A sentence that fails to
synthesise is reported with `error` (`code: "tts_failed"`) and the request ends with `failed: true`.
### History and brain (M3)
The worker also hosts the brain (the existing `jarvis` reply engine) and the conversation history
store (`src/jarvis/memory/history.spec.md`). Start it with `--history <file>` to keep history and
`--brain` to answer (`--brain` needs `--history`; it loads the jarvis settings, whose
`ollama_base_url` points at Strata's `/v1`). Without them the messages below are accepted and
ignored, except `reply`, which answers `error` `brain_unavailable`.
The app tells the worker where Jarvis is and who is there. Speaker IDs are Discord user IDs, the
same strings as in audio frames. Times are taken by the worker on arrival, except chat messages,
which carry Discord's own time.
```
-> {"type": "voice_join", "channel": "<voice channel id>"} // Jarvis joined (or moved to) this channel
-> {"type": "voice_leave"} // Jarvis left the voice channel
-> {"type": "member_join", "user": "<id>", "name": "<display name>"} // also for everyone already there
-> {"type": "member_leave", "user": "<id>"}
-> {"type": "chat_message", "channel": "...", "message_id": "...", "author": "<id>", "name": "...",
"text": "...", "ts": 1762830000.5, "to": "<id>"} // to: only on Jarvis's own answer to someone
-> {"type": "chat_edit", "channel": "...", "message_id": "...", "text": "..."}
-> {"type": "chat_delete", "channel": "...", "message_id": "..."}
-> {"type": "reply", "id": "r1", "user": "<id>", "text": "...", "via": "voice", "speak": true}
<- {"type": "reply_text", "id": "r1", "text": "...", "failed": false}
```
- `voice_join` closes the presence of everyone in the previous channel and starts a new one;
`voice_leave` and the end of the session close it, because nothing said after that is heard.
`member_join` opens the user's presence in the current channel and remembers their name;
`member_leave` closes it. Members outside a voice channel are ignored.
- Every non-empty `final` is stored as a `voice` record of the current channel under the speaker's
name (the user ID until `member_join` names them). A final outside a voice channel is not stored.
- Chat messages are stored only as given; the app sends only registered channels.
- `reply` asks the brain to answer `user`'s `text`. The worker builds the history context for that
user (their scope only) and runs the reply engine on a worker thread, one reply at a time.
`reply_text` carries the answer (`failed: true` with empty text when the brain gave none). A voice
answer (`via: "voice"`) is stored as a `reply` record to the user in the current channel; a chat
answer is stored when the app reports the sent message as `chat_message` with `to`. With
`speak: true` the answer is also spoken through the TTS path under the same id (audio frames and
`tts_done`), and `tts_cancel` with that id stops it. Each user keeps their own short-term dialogue
memory (tool carry-over) across replies.
- Memory settings, optional in `hello` as `"memory": {...}` and changeable with
`{"type": "memory_config", ...}`: `chat` (true), `others` (true), `off` (false), `since_days`
(null: whole history), `recent_minutes` (30), `recent_records` (20), `matches` (5),
`budget_tokens` (4000).
- Logs carry IDs, counts and timings only, never record, question or answer text.
### Errors
`{"type": "error", "code": "...", "message": "...", "id": "..."}`. Codes: `bad_message` (unknown type,
bad JSON, bad field), `bad_frame` (malformed binary frame), `tts_unavailable`, `tts_failed`. Errors
bad JSON, bad field), `bad_frame` (malformed binary frame), `tts_unavailable`, `tts_failed`,
`brain_unavailable`. Errors
never close the connection after the handshake.
## Debug logging

View File

@@ -0,0 +1,227 @@
"""Voice worker history and brain messages (voice_worker.spec.md, "History and brain").
Over a real local WebSocket with the fake STT/VAD/TTS engines of test_voice_worker_server and a
fake brain that records the history context it was given. Behaviours: presence and finals reach
the history store, chat events are kept, a reply answers with the asker's own scope, a voice answer
is stored and can be spoken, and the worker reports when it has no brain.
"""
from __future__ import annotations
import asyncio
import json
import pytest
from jarvis.memory.history import HistoryStore, Scope
from voice_worker.protocol import decode_tts_frame
from voice_worker.server import VoiceWorker
from tests.test_voice_worker_server import TOKEN, AmplitudeVad, FakeTts, LevelTranscriber, hello, recv_until, stream
websockets = pytest.importorskip("websockets")
from websockets.asyncio.client import connect # noqa: E402
class FakeBrain:
def __init__(self, answer=lambda user, text: f"answer for {text}"):
self.answer = answer
self.calls = []
def reply(self, user, text, history):
self.calls.append((user, text, history))
return self.answer(user, text)
@pytest.fixture
def store(tmp_path):
s = HistoryStore(str(tmp_path / "history.sqlite"))
yield s
s.close()
async def with_worker(body, store=None, brain=None, tts=None):
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, tts, history=store, brain=brain)
server = await worker.serve("127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
try:
return await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
finally:
server.close()
await server.wait_closed()
await worker.stop()
async def send(ws, **msg):
await ws.send(json.dumps(msg))
async def say(ws, speaker, level):
await stream(ws, speaker, level, 600)
await send(ws, type="speaking_end", speaker=speaker)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
def texts(records):
return [r.text for r in records]
@pytest.mark.unit
def test_finals_in_a_voice_channel_are_stored_under_the_speakers_name(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await say(ws, "100", 0.9) # before Jarvis is in a channel: not stored
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.6)
asyncio.run(with_worker(body, store=store))
recs = store.records_of("100")
assert texts(recs) == ["level 0.6"]
assert recs[0].author_name == "Alice" and recs[0].channel_id == "vc1" and recs[0].kind == "voice"
@pytest.mark.unit
def test_reply_answers_with_the_askers_own_scope_and_stores_the_voice_answer(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.9)
await send(ws, type="member_join", user="200", name="Bob")
await say(ws, "200", 0.6)
await send(ws, type="reply", id="r1", user="100", text="what did we say", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="r2", user="200", text="and me", via="voice", speak=False)
seen += await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return seen
seen = asyncio.run(with_worker(body, store=store, brain=brain))
answers = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
assert answers[0] == {"type": "reply_text", "id": "r1", "text": "answer for what did we say", "failed": False}
# Alice was there when Bob spoke, so she sees both; Bob joined after Alice spoke.
alice_ctx, bob_ctx = brain.calls[0][2], brain.calls[1][2]
assert any("Alice: level 0.9" in x for x in alice_ctx.recent)
assert any("Bob: level 0.6" in x for x in alice_ctx.recent)
assert not any("Alice: level 0.9" in x for x in bob_ctx.recent)
# The voice answer is Alice's own record now.
assert "answer for what did we say" in texts(store.records_of("100"))
@pytest.mark.unit
def test_spoken_reply_comes_back_as_tts_under_the_same_id(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r9", user="100", text="hello", via="voice", speak=True)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(), tts=FakeTts()))
frames = [decode_tts_frame(m) for m in seen if isinstance(m, bytes)]
assert frames and all(rid == "r9" for rid, _, _ in frames)
done = [m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"][0]
assert done["id"] == "r9" and not done["failed"]
@pytest.mark.unit
def test_chat_answers_are_not_stored_by_the_worker_but_reported_messages_are(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="chat_message", channel="ch1", message_id="m1", author="100", name="Alice",
text="first", ts=1_800_000_000.0)
await send(ws, type="chat_message", channel="ch1", message_id="m2", author="900", name="Jarvis",
text="answer for hi", ts=1_800_000_001.0, to="100")
await send(ws, type="chat_edit", channel="ch1", message_id="m1", text="first, edited")
await send(ws, type="chat_message", channel="ch1", message_id="m3", author="100", name="Alice",
text="gone soon", ts=1_800_000_002.0)
await send(ws, type="chat_delete", channel="ch1", message_id="m3")
# A round trip makes sure the worker handled everything above.
await send(ws, type="reply", id="c2", user="100", text="x", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c2")
asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert texts(store.records_of("100")) == ["first, edited", "answer for hi"]
@pytest.mark.unit
def test_leaving_the_voice_channel_stops_storing_and_closes_presence(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await send(ws, type="voice_leave")
await say(ws, "200", 0.6)
asyncio.run(with_worker(body, store=store))
assert store.records_of("200") == []
assert store.recent("100", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
@pytest.mark.unit
def test_memory_off_gives_the_brain_no_history(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False}, memory={"off": True})
await send(ws, type="voice_join", channel="vc1")
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="memory_config", off=False)
await send(ws, type="reply", id="r2", user="100", text="q", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
asyncio.run(with_worker(body, store=store, brain=brain))
off_ctx, on_ctx = brain.calls[0][2], brain.calls[1][2]
assert off_ctx.recent == [] and off_ctx.matches == []
assert any("level 0.9" in x for x in on_ctx.recent)
@pytest.mark.unit
@pytest.mark.parametrize("answer", [lambda u, t: None, lambda u, t: 1 / 0])
def test_a_failed_brain_answer_is_reported(store, answer):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=True)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer), tts=FakeTts()))
assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True}
@pytest.mark.unit
def test_without_a_brain_reply_is_an_error():
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
seen = asyncio.run(with_worker(body))
assert seen[-1]["code"] == "brain_unavailable" and seen[-1]["id"] == "r1"
@pytest.mark.unit
def test_malformed_history_messages_are_rejected(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="member_join", user=5)
await send(ws, type="reply", id="r1", user="100", via="radio")
return [await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
for _ in range(2)]
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert [s[-1]["code"] for s in seen] == ["bad_message", "bad_message"]