Compare commits
6 Commits
app-v0.1.4
...
app-v0.1.4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9a527aa470 | ||
|
|
628d89c4f1 | ||
|
|
478968949c | ||
|
|
caad09c613 | ||
|
|
97b827f424 | ||
|
|
045218b008 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.46"
|
||||
version = "0.1.48"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.46"
|
||||
version = "0.1.48"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.46"
|
||||
version = "0.1.48"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.46"
|
||||
version = "0.1.48"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.46"
|
||||
version = "0.1.48"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -31,6 +31,10 @@ pub struct CallSettings {
|
||||
/// After a wake word said alone, how long the speaker's next utterance may take to start and
|
||||
/// still be joined to it (the pause after "Jarvis," often ends an utterance).
|
||||
pub follow_up: Duration,
|
||||
/// A speaker who goes on talking before Jarvis's answer to them is heard, starting within this of
|
||||
/// their last utterance's end, is still asking: what they say is joined to the request
|
||||
/// (`call.continueMs`; zero turns it off).
|
||||
pub continue_window: Duration,
|
||||
/// Play the earcon when Jarvis takes a voice call (`call.earcon`).
|
||||
pub earcon: bool,
|
||||
}
|
||||
@@ -46,6 +50,7 @@ impl Default for CallSettings {
|
||||
designated_no_wake_word: false,
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
continue_window: Duration::from_millis(5000),
|
||||
earcon: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,6 +167,18 @@ pub struct Conversation<V: Voice> {
|
||||
wakes: HashMap<u64, (Request, Option<Instant>)>,
|
||||
/// Requests whose earcon already played (a wake word said alone was answered with it).
|
||||
chimed: HashSet<u64>,
|
||||
/// Each speaker's latest voice request whose answer has not been heard yet: what they say next
|
||||
/// may still be part of it ("Going on").
|
||||
turns: HashMap<u64, Turn>,
|
||||
}
|
||||
|
||||
/// A voice request whose answer is not heard yet.
|
||||
struct Turn {
|
||||
id: u64,
|
||||
text: String,
|
||||
ended_at: Instant,
|
||||
/// Still a "was it said to Jarvis?" question (its ask id), not yet a request of the policy.
|
||||
ask: Option<String>,
|
||||
}
|
||||
|
||||
impl<V: Voice> Conversation<V> {
|
||||
@@ -185,7 +197,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
|
||||
by_speech: HashMap::new(), asks: HashMap::new(), missed: MissedLog::default(), chat: ChatCalls::default(),
|
||||
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new(),
|
||||
chimed: HashSet::new() }
|
||||
chimed: HashSet::new(), turns: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Answer the registered chat channels too.
|
||||
@@ -225,6 +237,13 @@ impl<V: Voice> Conversation<V> {
|
||||
/// would be) lets the brain get ready while the speaker is still talking. Not while Jarvis is
|
||||
/// answering: the call may not be taken, and the brain is busy with the answer.
|
||||
fn on_partial(&mut self, speaker: u64, partial: &str) {
|
||||
// Going on with a request whose answer is not heard yet: it will be joined and taken.
|
||||
if let Some(t) = self.turns.get(&speaker) {
|
||||
if !self.calls.continue_window.is_zero() && !matches!(self.calls.judge(speaker, partial), Call::Stop) {
|
||||
self.voice.prepare(speaker, &format!("{} {}", t.text, partial));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if !self.answers.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -277,10 +296,15 @@ impl<V: Voice> Conversation<V> {
|
||||
self.started.remove(&speaker);
|
||||
}
|
||||
// What follows a wake word said alone is joined to it.
|
||||
let (text, chimed) = match self.wakes.remove(&speaker) {
|
||||
let (mut text, mut chimed) = match self.wakes.remove(&speaker) {
|
||||
Some((wake, _)) => (format!("{} {}", wake.text, f.text), self.chimed.remove(&wake.id)),
|
||||
None => (f.text.clone(), false),
|
||||
};
|
||||
// The speaker goes on before their answer was heard: still the same request.
|
||||
if let Some((earlier, earcon_played)) = self.going_on(speaker, f, now) {
|
||||
text = format!("{earlier} {text}");
|
||||
chimed |= earcon_played;
|
||||
}
|
||||
self.next_id += 1;
|
||||
if chimed {
|
||||
self.chimed.insert(self.next_id);
|
||||
@@ -316,21 +340,19 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
Call::Request { direct } => {
|
||||
let a = self.policy.request(Request { direct, ..req }, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
Call::Request { direct } => self.voice_request(Request { direct, ..req }, now),
|
||||
Call::Ask => {
|
||||
let room = self.room(speaker);
|
||||
if room.people <= 1 {
|
||||
// Alone with Jarvis: whatever the speaker says is said to it.
|
||||
debug_log!("conversation", "addressed req={} alone with jarvis", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
self.voice_request(req, now);
|
||||
return;
|
||||
}
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &text, room) {
|
||||
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
|
||||
ended_at: req.ended_at, ask: Some(id.clone()) });
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -344,12 +366,13 @@ impl<V: Voice> Conversation<V> {
|
||||
// An answer after the timeout counts as no, even before the timer ran.
|
||||
let yes = *addressed && now < asked + ADDRESSED_TIMEOUT;
|
||||
debug_log!("conversation", "addressed req={} yes={yes}", req.id);
|
||||
let a = if yes {
|
||||
self.policy.request(req, now)
|
||||
self.turns.retain(|_, t| t.id != req.id);
|
||||
if yes {
|
||||
self.voice_request(req, now);
|
||||
} else {
|
||||
self.policy.utterance_closed(req.speaker, now)
|
||||
};
|
||||
self.apply(a, now);
|
||||
let a = self.policy.utterance_closed(req.speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::Ranked { id, order } => {
|
||||
@@ -358,9 +381,16 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.ranked(rank_id, &order, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
// The answer is heard: what its asker says next is a new request.
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
self.turns.retain(|_, t| t.id != pid);
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
self.turns.retain(|_, t| t.id != pid);
|
||||
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
|
||||
e.played, e.cancelled, e.failed);
|
||||
let a = self.policy.answer_done(pid, now);
|
||||
@@ -378,6 +408,7 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
CoreEvent::WorkerLost => {
|
||||
self.started.clear();
|
||||
self.turns.clear();
|
||||
for (_, (req, _)) in self.wakes.drain() {
|
||||
self.chimed.remove(&req.id);
|
||||
}
|
||||
@@ -386,9 +417,42 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
|
||||
fn voice_request(&mut self, req: Request, now: Instant) {
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None });
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
|
||||
/// The final `f` continues the speaker's request whose answer is not heard yet (it started within
|
||||
/// `call.continueMs` of that request's end, and is not a stop): the request is taken back, its
|
||||
/// running answer stopped. Returns its text and whether its earcon already played.
|
||||
fn going_on(&mut self, speaker: u64, f: &crate::discord::HeardFinal, now: Instant) -> Option<(String, bool)> {
|
||||
let t = self.turns.get(&speaker)?;
|
||||
let started = now.checked_sub(Duration::from_millis(f.wait_ms + f.audio_ms)).unwrap_or(now);
|
||||
if self.calls.continue_window.is_zero() || started > t.ended_at + self.calls.continue_window
|
||||
|| matches!(self.calls.judge(speaker, &f.text), Call::Stop) {
|
||||
return None;
|
||||
}
|
||||
let t = self.turns.remove(&speaker).expect("checked");
|
||||
let earcon_played = self.chimed.remove(&t.id) || (self.calls.earcon && self.answers.contains_key(&t.id));
|
||||
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
|
||||
match t.ask {
|
||||
Some(q) => {
|
||||
self.asks.remove(&q);
|
||||
}
|
||||
None => {
|
||||
let a = self.policy.withdraw(t.id);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
Some((t.text, earcon_played))
|
||||
}
|
||||
|
||||
/// The policy's stop, and every pending "was it said to Jarvis?" question is void.
|
||||
fn stop_everything(&mut self, now: Instant) {
|
||||
self.wakes.clear();
|
||||
self.turns.clear();
|
||||
let a = self.policy.stop();
|
||||
self.apply(a, now);
|
||||
self.chimed.clear(); // nothing is waiting any more
|
||||
@@ -405,6 +469,7 @@ impl<V: Voice> Conversation<V> {
|
||||
for id in late {
|
||||
if let Some((req, _)) = self.asks.remove(&id) {
|
||||
debug_log!("conversation", "addressed req={} timed out", req.id);
|
||||
self.turns.retain(|_, t| t.id != req.id);
|
||||
let a = self.policy.utterance_closed(req.speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
@@ -415,8 +480,7 @@ impl<V: Voice> Conversation<V> {
|
||||
for speaker in due {
|
||||
if let Some((req, _)) = self.wakes.remove(&speaker) {
|
||||
debug_log!("conversation", "wake word alone req={} answered alone", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
self.voice_request(req, now);
|
||||
}
|
||||
}
|
||||
let a = self.policy.tick(now);
|
||||
@@ -457,6 +521,7 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
Action::Missed(r, reason) => {
|
||||
self.chimed.remove(&r.id);
|
||||
self.turns.retain(|_, t| t.id != r.id);
|
||||
debug_log!("conversation", "missed req={} speaker={} reason={reason:?}", r.id, r.speaker);
|
||||
let mut missed = self.missed.lock().unwrap();
|
||||
if missed.len() == MISSED_KEPT {
|
||||
@@ -590,6 +655,7 @@ mod tests {
|
||||
|
||||
const A: u64 = 11;
|
||||
const B: u64 = 22;
|
||||
const C: u64 = 33;
|
||||
|
||||
fn conv(kind: PolicyKind, mode: CallMode) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
@@ -613,7 +679,7 @@ mod tests {
|
||||
assert!(f.take().is_empty(), "a waiting request gets its earcon when its answer starts");
|
||||
c.on_event(&end("s1", false), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(B, "자비스 불 꺼 줘".into())]);
|
||||
c.on_event(&fin(B, "오늘 뭐 먹지", 100), ms(t, 3100));
|
||||
c.on_event(&fin(C, "오늘 뭐 먹지", 100), ms(t, 3100));
|
||||
assert!(f.take().is_empty(), "no call, no earcon");
|
||||
}
|
||||
|
||||
@@ -659,10 +725,10 @@ mod tests {
|
||||
c.on_event(&fin_utt(B, 1, "자비스"), ms(t, 100));
|
||||
c.on_event(&partial(B, "지금 몇 시"), ms(t, 300));
|
||||
assert_eq!(f.take(), vec![Did::Prepare(B, "자비스 지금 몇 시".into())]);
|
||||
// Not while an answer runs.
|
||||
// Not while an answer runs (to someone else: its asker going on is joined to it).
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 400));
|
||||
f.take();
|
||||
c.on_event(&partial(A, "자비스 내일은"), ms(t, 500));
|
||||
c.on_event(&partial(C, "자비스 내일은"), ms(t, 500));
|
||||
assert!(f.take().is_empty());
|
||||
}
|
||||
|
||||
@@ -904,7 +970,7 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed);
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0));
|
||||
c.on_event(&fin(B, "자비스 뉴스", 100), ms(t, 100));
|
||||
c.on_event(&fin(B, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed"
|
||||
c.on_event(&fin(C, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed"
|
||||
let did = f.take();
|
||||
let q = did.iter().find_map(|d| if let Did::Addressed(q, _) = d { Some(q.clone()) } else { None }).unwrap();
|
||||
// The slash command stops all speech: Stopped comes before the speech's end.
|
||||
@@ -1016,9 +1082,95 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 하나", 0), ms(t, 0));
|
||||
c.on_event(&fin(B, "자비스 둘", 0), ms(t, 1000));
|
||||
c.on_event(&fin(A, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘
|
||||
c.on_event(&fin(C, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘
|
||||
f.take();
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 셋".into())]);
|
||||
assert_eq!(f.take(), vec![Did::Answer(C, "자비스 셋".into())]);
|
||||
}
|
||||
|
||||
fn alone(kind: PolicyKind) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let (c, f, t) = conv(kind, CallMode::Addressed);
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(c, f, t)
|
||||
}
|
||||
fn speak_start(id: &str) -> CoreEvent {
|
||||
CoreEvent::SpeakStart(id.into())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_speaker_who_goes_on_before_the_answer_is_heard_gets_one_answer_for_all_of_it() {
|
||||
// Live 2026-11-19: each piece of one thought was answered on its own, one after another.
|
||||
for kind in [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important] {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into())], "{kind:?}");
|
||||
// The next piece starts 0.8 s after the first ended, while Jarvis is still thinking.
|
||||
c.on_event(&fin(A, "디코 화면 공유 창에 가려져 있잖아", 100), ms(t, 1800));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
c.on_event(&fin(A, "앞으로 오게 해 줘", 100), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s2".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아 앞으로 오게 해 줘".into())],
|
||||
"{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "a joined piece is not a missed request ({kind:?})");
|
||||
// The cancelled speech's end does not end the joined answer.
|
||||
c.on_event(&end("s1", true), ms(t, 4100));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_follows_a_wake_word_call_is_joined_even_without_the_wake_word() {
|
||||
let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 디코 화면 공유 보여", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "내 화면 말이야", 100), ms(t, 2000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 디코 화면 공유 보여".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "자비스 디코 화면 공유 보여 내 화면 말이야".into())]);
|
||||
// Someone else's words never join it.
|
||||
c.on_event(&fin(B, "나도 안 보여", 100), ms(t, 2500));
|
||||
assert!(f.take().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_the_answer_is_heard_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 2500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "고마워".into())]);
|
||||
// Still thinking, but the speaker paused longer than call.continueMs.
|
||||
let (mut c, f, t) = alone(PolicyKind::InOrder);
|
||||
let pause = test_calls().continue_window.as_millis() as u64;
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
// The next utterance (1 s long) starts `pause` + 100 ms after the first ended.
|
||||
c.on_event(&fin(A, "메모장도", 100), ms(t, pause + 1100));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())], "queued as its own request");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_said_while_the_answer_is_prepared_still_stops_it() {
|
||||
let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 날씨 알려 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "자비스 멈춰", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 알려 줘".into()), Did::StopAll, Did::Cancel("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&partial(A, "제일 위로"), ms(t, 1200));
|
||||
assert_eq!(f.take(), vec![Did::Prepare(A, "계산기 켜 줘 제일 위로".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_joined_request_gets_no_second_earcon() {
|
||||
let (mut c, f, t) = conv_earcon(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "부산", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스 날씨".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "자비스 날씨 부산".into())]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer is not heard yet. 0 turns joining off. 0-15000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
@@ -78,6 +79,27 @@ connection (`WorkerLost`): the final it waited for will not come, and the restar
|
||||
utterances are not continuations of it. Chat messages are not joined:
|
||||
a message is complete as sent.
|
||||
|
||||
**Going on.** People say one thing in pieces, with pauses of a fraction of a second to several
|
||||
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
|
||||
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
|
||||
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
|
||||
So a speaker's voice request stays open until its answer is heard (the first audio of its speech
|
||||
plays, `SpeakStart`), and while it is open their next final is joined to it when that utterance
|
||||
started within `call.continueMs` of the request's end (start = the final's arrival minus its
|
||||
`wait_ms` and `audio_ms`):
|
||||
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
|
||||
speech `cancel`led, which also stops the brain, `voice_worker.spec.md`), a waiting one leaves the
|
||||
queue, a pending "addressed" question is void. It is not a missed request.
|
||||
- The joined text (the earlier text, a space, the new final) is judged and answered as one request,
|
||||
as if it had been said in one go; it ends when the new final ended. It may be joined again.
|
||||
- The new final is joined whatever it is on its own (a piece needs no wake word of its own), except
|
||||
a stop: a stop stops, as always.
|
||||
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
|
||||
its answer is being written (it is the request that will be answered).
|
||||
|
||||
**Earcon.** A short two-tone "띠링" (about 0.3 s, made by the app, `app/src/earcon.rs`) tells the
|
||||
speaker Jarvis heard the call, long before the answer is written: it is the first sound after a call.
|
||||
It plays right before each voice answer starts (the policy's `Answer`), and at once when a wake word
|
||||
@@ -116,6 +138,9 @@ Rules shared by every policy:
|
||||
than the request's `ended_at` (in `important`: any ending utterance, since it competes on score), the
|
||||
policy waits for that final, at most `settleMs` from the first waiting request's arrival. A final that
|
||||
is not a call ends the wait for it. `latest` does not settle: an earlier utterance would lose anyway.
|
||||
- **Taking back.** A request whose speaker went on (Going on) is withdrawn: stopped when it is the
|
||||
running answer, removed when it waits, dropped from a pending ranking. Not missed; the joined
|
||||
request that follows goes through the policy as any new request.
|
||||
- **Stop.** A stop decision stops the current answer and drops every waiting request, in every policy.
|
||||
The stopped answer and the dropped requests are missed (`stopped`).
|
||||
- An answer that is already done is never started again.
|
||||
@@ -181,7 +206,8 @@ event is passed on unchanged). It talks to the core through a small `Voice` inte
|
||||
one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glue is tested with a fake.
|
||||
|
||||
- `Chat` and `ReplyText`: chat calls, above.
|
||||
- `Partial` (getting ready): a partial transcript that call detection already reads as a request
|
||||
- `Partial` (getting ready): a partial of a speaker whose request is open (Going on) is sent as the
|
||||
joined text. Otherwise a partial transcript that call detection already reads as a request
|
||||
(joined to the speaker's waiting wake word alone, as its final would be; not the wake word alone,
|
||||
not a stop) is sent to the brain as `prepare(user, text)`, so the brain judges what the call needs
|
||||
while the speaker is still talking (`voice_worker.spec.md`, `prepare`). Not while an answer runs:
|
||||
@@ -198,6 +224,7 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
@@ -218,7 +245,10 @@ fake voice side (answering and queueing, a busy drop recorded without text, voic
|
||||
with Jarvis answered without asking, the room counts sent with the question, addressed yes,
|
||||
no and timeout, the earcon before each answer, once for a wake word alone, none while busy or when
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
|
||||
stopping, no second earcon), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
|
||||
@@ -254,6 +254,21 @@ impl Policy {
|
||||
}
|
||||
}
|
||||
|
||||
/// The request is taken back before its answer was heard: its speaker went on talking and it is
|
||||
/// joined to what they said next (conversation.spec.md, "Going on"). Not a missed request. When
|
||||
/// it is the running answer it is stopped; nothing else starts until the joined request comes.
|
||||
pub fn withdraw(&mut self, id: u64) -> Vec<Action> {
|
||||
if self.current.as_ref().is_some_and(|r| r.id == id) {
|
||||
self.current = None;
|
||||
return vec![Action::StopAnswer(id)];
|
||||
}
|
||||
self.waiting.retain(|r| r.id != id);
|
||||
if self.ranking.as_ref().is_some_and(|r| r.ids.contains(&id)) {
|
||||
self.ranking = None;
|
||||
}
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// A stop command: stop the answer and drop everything waiting.
|
||||
pub fn stop(&mut self) -> Vec<Action> {
|
||||
let mut out = vec![];
|
||||
|
||||
@@ -453,6 +453,7 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
|
||||
designated_no_wake_word: get_bool(call, "designatedNoWakeWord", "call", d.designated_no_wake_word)?,
|
||||
stop_words,
|
||||
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
|
||||
continue_window: get_ms(call, "continueMs", "call", d.continue_window, 15000)?,
|
||||
earcon: get_bool(call, "earcon", "call", d.earcon)?,
|
||||
},
|
||||
chat_wake_words,
|
||||
|
||||
@@ -95,6 +95,7 @@ pub fn fields() -> Vec<Field> {
|
||||
f("call.stopWords", Call, TextList { max_items: 20, nullable: false },
|
||||
json!(["멈춰", "그만", "정지", "stop"]), "정지 명령"),
|
||||
f("call.followUpMs", Call, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.continueMs", Call, Int { min: 0, max: 15000 }, json!(5000), "끊어 말한 문장 합치기(ms)"),
|
||||
f("call.earcon", Call, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Response, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
json!("latest"), "동시 대화 응답 정책"),
|
||||
|
||||
@@ -20,6 +20,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i
|
||||
- **Operator instructions** (two sources, both framed "Additional instructions from the operator:" and appended near the end of the guidance list): the settings-UI `llm_instructions` config field, and every `*.md` file in `AGENTS_DIR` (default `/app/agents`, bind-mounted from `./agents`). The file-based set is read once per turn by `load_agent_instructions()` in [src/jarvis/system_prompt.py](src/jarvis/system_prompt.py) and concatenated in filename order, so dropping/editing a `.md` applies on the next reply with no rebuild/restart; fail-open to `""` when the folder is absent/empty/unreadable.
|
||||
- **Warm profile block** (query-agnostic User + Directives excerpt from the knowledge graph, composed by `build_warm_profile()` / `format_warm_profile_block()` in [src/jarvis/memory/graph_ops.py](src/jarvis/memory/graph_ops.py) at Step 3.5 of `reply()`; no LLM call, pure SQLite read; injected unconditionally so personalisation is the default; result cached in `DialogueMemory._hot_cache` under `DialogueMemory.WARM_PROFILE_CACHE_KEY` for the lifetime of the active conversation. Invalidated on `stop`, on new-conversation entry, AND on User/Directives graph mutations via the listener registered in [src/jarvis/daemon.py](src/jarvis/daemon.py) against `register_graph_mutation_listener` in [src/jarvis/memory/graph.py](src/jarvis/memory/graph.py); World-branch writes are ignored)
|
||||
- Digested memory enrichment (optional, see #4)
|
||||
- **v2 cancel** (`turn_state.set_cancel`, set by the voice worker's brain): once the app takes a voice request back (its asker went on talking, a newer request or a stop), the loop stops before its next chat-model call or tool call and the reply returns no answer, so a dropped request never holds up the one that replaces it (`voice_worker.spec.md`, "History and brain").
|
||||
- **v2 history context** (optional, caller-supplied `history=` on `run_reply_engine`): the recent window and search matches from the history store ([src/jarvis/memory/history.py](src/jarvis/memory/history.py), [spec](src/jarvis/memory/history.spec.md)), scoped to the asker and capped by the store's size budget (default about 4,000 tokens, matches at most half). No LLM call; not gated by the planner. Injected in a `<conversation_records>` fence labelled "Information shared in previous conversations"; the lines are data, not instructions. Eval: `evals/test_history_context_recall.py`. With `legacy_memory=False` (the v2 brain in the voice worker) the warm profile and the whole memory enrichment chain (#3 extractor, #3b recall gate, #4 digest and the diary/graph reads) are skipped for the turn, because v1's long-term memory belongs to a single user.
|
||||
- Time + location context (re-injected each turn)
|
||||
- **v2 chat channel tool**: when the voice worker's brain sets a chat backend (`turn_state.set_chat`), `chatChannels` is added to the turn's tools after routing (like `setBroadcast`) and its read/search results enter the loop inside an UNTRUSTED CHAT MESSAGES fence; without a backend it is not in the catalogue at all. Eval: `evals/test_chat_channels_tool.py`.
|
||||
|
||||
File diff suppressed because one or more lines are too long
90
docs/v2/stt-accuracy-plan.md
Normal file
90
docs/v2/stt-accuracy-plan.md
Normal file
@@ -0,0 +1,90 @@
|
||||
# STT 정확도 개선 계획 (2026-11-19)
|
||||
|
||||
사용자 요청(2026-11-19): 받아 적기가 문장을 제대로 맞히는 일이 드물고, 끊어 말한 문장을 하나씩 따로 처리해서 느림.
|
||||
음성 인식을 고칠 때마다 다른 곳이 어긋났으니, 이번에는 원인과 판정 기준을 먼저 정하고 측정은 한 번에 끝낸 뒤 코드는 한 번만 바꿈.
|
||||
모델이 약하면 바꾸되 다른 모델과 같은 자료로 비교함.
|
||||
|
||||
## 실제로 틀린 예 (2026-11-19 사용 중, 사용자가 준 정답)
|
||||
|
||||
| 들은 것 | 실제로 한 말 |
|
||||
|---|---|
|
||||
| 기사야 디코 화면군이 보여 | 비서야 디코 화면공유 보여? |
|
||||
| 내 화면 공이 안 보여? | 내 화면 공유 안보여? |
|
||||
| 실행하면 제일 위로와야지 가면 | 실행하면 제일 위로와야지 화면 |
|
||||
| 지금 화면 봐봐 지금 어 이게 계산지가 제일 위에 있는 거야 | 지금 화면 봐봐 지금 어 이게 계산기가 화면 제일 위에 있는 거야? |
|
||||
| 기구 어디 갔어? | 디코 어디 갔어? |
|
||||
|
||||
11-18 새벽 녹음 11개에서도 문장 첫머리의 호출어가 4개 틀림(아비스, 자비, 다비스, 사비스). 틀린 곳은 대부분 첫소리 자음(ㅂ→ㄱ, ㄷ→ㄱ, ㅎ→ㄱ, ㅈ→ㅇ/ㄷ/ㅅ)과 문장 끝의 짧은 낱말.
|
||||
|
||||
## 지금 구조 (바꿀 수 있는 곳)
|
||||
|
||||
- 앱이 디스코드 소리를 받아 사람마다 워커로 보냄. 100ms 동안 패킷이 없으면 `speaking_end`(디스코드는 말을 멈추면 보내기를 멈춤).
|
||||
- 워커(`segmenter.py`)는 음성 확률이 0.35 아래로 `end_silence_ms`(300ms) 이어지거나 `speaking_end`가 오면 말 하나를 끝냄. 그 조각만 따로 받아 적음.
|
||||
- 받아 적기: faster-whisper large-v3-turbo, int8_float16, beam 2, 언어 ko, 호출어 힌트(힌트 없는 결과로 검사).
|
||||
- 앱(`conversation.rs`)은 조각마다 호출 판정. 호출어만 말한 조각만 다음 조각과 합침. 그 밖의 조각은 각각 따로 요청이 되거나 버려짐.
|
||||
|
||||
## 원인 가설과 확인 방법
|
||||
|
||||
| | 가설 | 지금까지의 근거 | 확인 방법 |
|
||||
|---|---|---|---|
|
||||
| H1 | 너무 짧게 자름. 0.3초만 쉬어도, 디스코드가 0.1초 보내기를 멈춰도 말이 끝나서, 짧은 조각을 문맥 없이 받아 적음. 사용자 불만 2(한 문장씩 처리)의 원인이기도 함 | 예 3·5처럼 짧은 조각이 틀림. 조각마다 따로 답함 | 오늘 워커 로그의 끝난 이유(`reason`) 비율과 같은 사람 조각 사이 간격 분포. 같은 소리를 조각마다 받아 적기 대 이어 붙여 한 번에 받아 적기 |
|
||||
| H2 | 말 첫머리가 잘려 들어옴(디스코드 쪽 음성 감지가 늦게 보내기 시작) | 11-18 클립 1개(012257, "아비스")는 클립 시작 20ms에 이미 말소리. 나머지 틀린 호출어 3개는 70~180ms 여유가 있어 잘림이 아님. 일부만 설명함 | 모든 클립의 첫 소리 앞 여유와 첫 낱말 오류의 관계 |
|
||||
| H3 | 모델 한계. turbo는 디코더가 4층(large-v3는 32층)이고 int8, beam 2라 짧은 구어(디코, 화면공유)에 약함 | 측정 없음 | 같은 소리로 모델 비교(아래) |
|
||||
| H4 | 앞 대화 문맥이 없음 | 측정 없음 | 직전 받아 적은 문장을 프롬프트로 준 변형. 없는 말을 지어내는지 잡음 108개로 함께 확인 |
|
||||
|
||||
이미 확인해서 빼는 것:
|
||||
- 음량: 11-16 측정, 키워도 나아지지 않음(`groups/javis_bot/logs/stt-level-2026-11-16`).
|
||||
- 고음 강조: 11-18 측정, 오히려 나빠져서 0.1.36에서 되돌림.
|
||||
- 소리 손상: 11-18 클립 11개의 4~7kHz 성분이 Jarvis TTS와 같은 수준(300~1000Hz 대비 -27~-32dB)이고 7.2kHz 위는 워커 필터가 자른 것. 대역이 좁아진 소리는 아님. 오늘 클립에서 20ms 구멍(패킷 손실) 수도 함께 셈.
|
||||
|
||||
## 시험 자료 (한 번 만들고 모든 비교에 같이 씀)
|
||||
|
||||
- A. 사용자 실제 녹음: 게임PC `C:\JarvisM0\Jarvis\Jarvis\recordings`의 11-18, 11-19 `said` 클립과 서버에 있는 11-18 새벽 11개. 정답은 사용자가 준 5개 말고는 후보 모델 결과를 owner와 reviewer가 따로 맞춰 보고, 둘이 다르거나 모델끼리 갈리는 것만 사용자에게 짧은 목록으로 확인.
|
||||
- A-turn. A에서 같은 사람이 이어 말한 조각을 실제 시간 간격대로 이어 붙인 것(H1용).
|
||||
- B. 악조건: A를 오프라인으로 바꾼 것. 소리 작게(-15dB), 생활 잡음(ESC-50) SNR 10·5dB, 싼 헤드셋 대역(300~3400Hz), 낮은 비트레이트 Opus(16kbps) 재인코딩, 방 울림. 다른 사람들은 마이크가 나쁘고 작게, 흐리게 말한다는 조건.
|
||||
- C. 다른 목소리: 공개 한국어 음성(FLEURS ko, Common Voice ko, Zeroth, 가능하면 전화 음성) 200개 정도와 그 B 처리본.
|
||||
- 잡음 108개(M1에서 쓴 ESC-50): 없는 말이나 호출어를 지어내는지.
|
||||
|
||||
## 후보 모델
|
||||
|
||||
| 후보 | 엔진 | 확인할 것 |
|
||||
|---|---|---|
|
||||
| 지금: large-v3-turbo int8_float16 beam 2 | faster-whisper | 기준 |
|
||||
| large-v3 (fp16, int8) | faster-whisper | 디코더 32층. VRAM과 속도 |
|
||||
| 한국어 미세조정 Whisper: BuzzASR/korean(large-v3 기반), batisay-ko-turbo, ghost613/whisper-large-v3-turbo-korean | faster-whisper로 변환 | 엔진을 그대로 쓰므로 바꾸기 쉬움. 라이선스, 영어 낱말 섞인 말, 지어내기 |
|
||||
| Qwen3-ASR-1.7B와 한국어 미세조정판(TeamUNIVA/qwen3_asr_1.7b_ko_beta) | transformers | 엔진이 달라 Windows 설치, 10명 묶음 처리, VRAM을 따로 확인 |
|
||||
|
||||
한국어 전용 모델은 다른 언어를 못 받아 적을 수 있음. 바꾸게 되면 언어 설정이 그 모델의 언어일 때만 쓰고, 다른 언어는 지금 모델을 씀(코드에 언어별 규칙은 넣지 않음).
|
||||
|
||||
## 판정 기준 (측정 전에 정함)
|
||||
|
||||
1. A와 B의 문자 오류율(띄어쓰기·문장부호 빼고, 숫자는 읽는 대로), 호출어 놓침, 잡음에서 지어낸 호출어.
|
||||
2. 조건: RTX 5070 Ti에서 말 한 마디의 최종 전사 p95 0.4초 이하(지금 0.11~0.19초, pc-features.md). 10명 동시 전사 VRAM이 지금 STT 몫(최대 약 2.7GB) 안. 넘으면 LLM 배치 식을 다시 재야 하므로 사용자에게 물음.
|
||||
3. 기준보다 A+B 오류율이 상대 20% 넘게 낮을 때만 모델을 바꿈. 자료가 적어 그보다 작은 차이는 우연과 구분할 수 없음. C는 동률일 때만 봄.
|
||||
4. 말 차례 합치기(아래)는 A-turn에서 한 번에 받아 적은 쪽 오류율이 낮고, 늘어나는 대기가 사용자가 정한 범위 안일 때 넣음.
|
||||
|
||||
## 끊어 말한 문장 합치기 (불만 2, 스펙 변경이라 사용자 확인 필요)
|
||||
|
||||
제안: 같은 사람이 짧게 쉬었다 다시 말하면 같은 "말 차례"로 봄. 차례가 끝날 때(쉰 시간이 `end_silence_ms`를 넘을 때) 그 소리 전체를 한 번에 받아 적어 요청 하나로 처리. 디스코드가 보내기를 멈춘 것(`speaking_end`)은 바로 끝내지 않고 쉰 시간으로 셈.
|
||||
- 좋아지는 것: 여러 문장을 한 질문으로 받아 답 한 번. 받아 적기가 문맥을 가짐(H1).
|
||||
- 대가: 한 문장만 말해도 끝을 알기까지 그만큼 더 기다림. 말하는 동안의 부분 전사와 답 준비(`prepare`)는 지금처럼 돌아서 일부는 겹쳐짐.
|
||||
- 기다릴 시간은 A의 실제 간격 분포를 보고 정함(문장 사이 쉼의 대부분을 덮는 값). 1초를 넘으면 사용자에게 다시 물음.
|
||||
- 바뀌는 스펙: `voice_worker.spec.md`(말 끝내기), `conversation.spec.md`(호출어만 말한 경우 합치기는 대부분 필요 없어짐), `discord_core.spec.md`(`speaking_end`의 뜻).
|
||||
|
||||
## 순서
|
||||
|
||||
1. 자료 모으기: 게임PC에서 녹음과 워커 로그 복사(읽기만, 사용자가 정한 시간에).
|
||||
2. 진단: H1~H4 측정, 결과를 이 문서에 적고 reviewer가 교차 검토.
|
||||
3. 모델 비교: 게임PC GPU에서 한 번(Jarvis 서비스를 끄고), 고른 모델은 Strata와 함께 켠 상태로 VRAM·지연 한 번.
|
||||
4. 결정한 것만 한 번에 구현(모델, 말 차례), 테스트, 스펙 수정.
|
||||
5. 같은 시험 자료로 최종 확인 후 릴리스 한 번, 사용자 실사용 확인 한 번.
|
||||
|
||||
## 진행 기록
|
||||
|
||||
- 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능.
|
||||
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
|
||||
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
|
||||
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
|
||||
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR은 VRAM(Strata와 함께 남는 약 2.5GB)에 들어가지 않아 서버 CPU에서 따로 봄.
|
||||
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
|
||||
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.
|
||||
@@ -2376,6 +2376,9 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
_one_step_nudged = False
|
||||
|
||||
while turn < max_turns:
|
||||
if _turn_state.cancelled():
|
||||
debug_log("reply cancelled: no longer wanted", "planning")
|
||||
return None
|
||||
turn += 1
|
||||
debug_log(f"🔁 messages loop turn {turn}", "planning")
|
||||
print(f" 🔁 Turn {turn}/{max_turns}", flush=True)
|
||||
@@ -2840,7 +2843,10 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
messages.append({"role": "tool", "tool_call_id": tool_call_id, "content": f"You have already called {tool_name} {duplicate_tool_count} times. Please use the results from those calls to answer the user's question."})
|
||||
continue
|
||||
|
||||
# Execute tool
|
||||
# Execute tool (not for an answer that is no longer wanted: no actions on its behalf)
|
||||
if _turn_state.cancelled():
|
||||
debug_log("reply cancelled before a tool call", "planning")
|
||||
return None
|
||||
tools_started += 1
|
||||
result = run_tool_with_retries(
|
||||
db=db,
|
||||
|
||||
@@ -37,6 +37,9 @@ pc : Optional[object]
|
||||
trace : Optional[list]
|
||||
v2: where the engine writes down how it made this answer (history.spec.md, "Traces"): the
|
||||
tools it was given, the plan, the model's thinking and each tool call. None records nothing.
|
||||
cancel : Optional[callable]
|
||||
v2: true once the answer is no longer wanted (the asker went on talking, or a newer request or a
|
||||
stop took its place). The engine checks it before each model call and each tool call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -58,6 +61,17 @@ def reset() -> None:
|
||||
_state.screen = None
|
||||
_state.pc = None
|
||||
_state.trace = None
|
||||
_state.cancel = None
|
||||
|
||||
|
||||
def set_cancel(check) -> None:
|
||||
_state.cancel = check
|
||||
|
||||
|
||||
def cancelled() -> bool:
|
||||
"""Whether this turn's answer is no longer wanted."""
|
||||
check = getattr(_state, "cancel", None)
|
||||
return bool(check and check())
|
||||
|
||||
|
||||
def set_trace(steps: Optional[list]) -> None:
|
||||
|
||||
@@ -10,7 +10,7 @@ import unicodedata
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Protocol
|
||||
from typing import Callable, Optional, Protocol
|
||||
|
||||
from .log import debug_log, notice
|
||||
from jarvis.memory.history import HistoryContext, Scope
|
||||
@@ -19,7 +19,8 @@ from jarvis.memory.history import HistoryContext, Scope
|
||||
class Brain(Protocol):
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None, pc=None, trace: Optional[list] = None) -> Optional[str]:
|
||||
screen=None, pc=None, trace: Optional[list] = None,
|
||||
cancelled: Optional[Callable[[], bool]] = None) -> Optional[str]:
|
||||
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
|
||||
reply for different people may overlap). ``chat``: the chat backend for
|
||||
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
|
||||
@@ -28,7 +29,8 @@ class Brain(Protocol):
|
||||
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
|
||||
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
|
||||
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
|
||||
this answer to (history.spec.md, "Traces")."""
|
||||
this answer to (history.spec.md, "Traces"); ``cancelled``: true once the answer is no longer
|
||||
wanted, so the engine stops at its next model or tool call and None is returned."""
|
||||
|
||||
def addressed(self, user: str, text: str, names: list[str], people: Optional[int] = None,
|
||||
callers: Optional[int] = None) -> Optional[bool]:
|
||||
@@ -310,7 +312,8 @@ class JarvisBrain:
|
||||
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None, pc=None, trace: Optional[list] = None) -> Optional[str]:
|
||||
screen=None, pc=None, trace: Optional[list] = None,
|
||||
cancelled: Optional[Callable[[], bool]] = None) -> Optional[str]:
|
||||
from jarvis.memory.conversation import DialogueMemory
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
@@ -334,6 +337,7 @@ class JarvisBrain:
|
||||
turn_state.set_screen(None if task else screen)
|
||||
turn_state.set_pc(None if task else pc)
|
||||
turn_state.set_trace(trace)
|
||||
turn_state.set_cancel(cancelled)
|
||||
try:
|
||||
return run_reply_engine(self.db, cfg, None, text, memory,
|
||||
language=self.language, history=history, legacy_memory=False,
|
||||
|
||||
@@ -32,7 +32,8 @@ class SttResult:
|
||||
|
||||
|
||||
class Transcriber(Protocol):
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ...
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]: ...
|
||||
|
||||
|
||||
Callback = Callable[[SttResult], None]
|
||||
@@ -44,6 +45,7 @@ class _Job:
|
||||
audio: np.ndarray
|
||||
opts: DecodeOptions
|
||||
callback: Callback
|
||||
context: str = "" # finals only: the recent conversation, as the model's prompt
|
||||
|
||||
|
||||
class SttScheduler:
|
||||
@@ -74,8 +76,9 @@ class SttScheduler:
|
||||
pass
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback))
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback,
|
||||
context: str = "") -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback, context))
|
||||
self._wake.set()
|
||||
|
||||
def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
@@ -121,7 +124,7 @@ class SttScheduler:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips,
|
||||
batch[0].opts)
|
||||
batch[0].opts, [j.context for j in batch])
|
||||
error = None
|
||||
except Exception as e: # the model failing must not stop every speaker's STT
|
||||
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"
|
||||
|
||||
@@ -10,6 +10,7 @@ import itertools
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
@@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService
|
||||
CLOSE_BAD_HELLO = 4400
|
||||
CLOSE_BAD_TOKEN = 4401
|
||||
MAX_SPEAKERS = 32
|
||||
CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches
|
||||
|
||||
|
||||
def is_loopback(host: str) -> bool:
|
||||
@@ -274,6 +276,9 @@ class Session:
|
||||
self.chat_waiting: dict[str, asyncio.Future] = {}
|
||||
self._chat_ids = itertools.count(1)
|
||||
self.names: dict[str, str] = {}
|
||||
# What was said and answered lately in Jarvis's voice channel: (time, text), the STT's
|
||||
# conversation context (voice_worker.spec.md, "Conversation context"). Never logged.
|
||||
self.lines: deque[tuple[float, str]] = deque(maxlen=50)
|
||||
self.speakers: dict[str, _Speaker] = {}
|
||||
self.tts_requests: dict[str, TtsRequest] = {}
|
||||
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
|
||||
@@ -281,7 +286,7 @@ class Session:
|
||||
self.tts_recorded: dict[str, dict] = {}
|
||||
# Spoken replies whose speech was cancelled while the brain was still answering.
|
||||
self.cancelled_replies: set[str] = set()
|
||||
self.thinking_replies: set[str] = set()
|
||||
self.thinking_replies: dict[str, str] = {} # reply id -> asker
|
||||
# user -> the newest call text waiting for that user's running prepare to finish (or None).
|
||||
self.preparing: dict[str, str | None] = {}
|
||||
self.last_prepared: dict[str, str] = {} # user -> the text of their last prepare
|
||||
@@ -364,6 +369,15 @@ class Session:
|
||||
samples = st.converter.push(pcm)
|
||||
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
||||
|
||||
def _context(self) -> str:
|
||||
"""The recent conversation for a final's prompt: the channel's lines of the last
|
||||
CONTEXT_SECONDS, at most `context_chars` characters from the end."""
|
||||
n = self.settings.context_chars
|
||||
if not n or not self.voice_channel:
|
||||
return ""
|
||||
since = time.monotonic() - CONTEXT_SECONDS
|
||||
return " ".join(t for at, t in self.lines if at >= since)[-n:].strip()
|
||||
|
||||
def _opts(self, beam: int) -> DecodeOptions:
|
||||
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
||||
|
||||
@@ -388,7 +402,8 @@ class Session:
|
||||
else:
|
||||
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
||||
self._opts(self.settings.final_beam),
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said),
|
||||
context=self._context())
|
||||
|
||||
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
||||
def cb(res: SttResult) -> None:
|
||||
@@ -402,6 +417,8 @@ class Session:
|
||||
store = self.worker.history
|
||||
if store is not None and channel and res.text.strip():
|
||||
store.add_voice(channel, speaker, name, res.text, ts)
|
||||
if channel and channel == self.voice_channel and res.text.strip():
|
||||
self.lines.append((time.monotonic(), res.text.strip()))
|
||||
if self.worker.recorder is not None:
|
||||
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
|
||||
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
||||
@@ -499,6 +516,8 @@ class Session:
|
||||
rec["text"] += text
|
||||
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
|
||||
self.tts_recorded.pop(next(iter(self.tts_recorded)))
|
||||
if self.voice_channel and text.strip():
|
||||
self.lines.append((time.monotonic(), text.strip()))
|
||||
tts.feed(req, text, more)
|
||||
|
||||
def _on_tts_cancel(self, msg: dict) -> None:
|
||||
@@ -523,6 +542,7 @@ class Session:
|
||||
if self.voice_channel:
|
||||
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
||||
self.voice_channel = None
|
||||
self.lines.clear()
|
||||
|
||||
def _on_voice_join(self, msg: dict) -> None:
|
||||
got = self._strings(msg, "channel")
|
||||
@@ -681,12 +701,15 @@ class Session:
|
||||
asyncio.get_running_loop().create_task(self._task(rid, user, text, notes))
|
||||
return
|
||||
if speak:
|
||||
self.thinking_replies.add(rid)
|
||||
self.thinking_replies[rid] = user
|
||||
if user in self.preparing:
|
||||
self.preparing[user] = None # the call was answered: no newer text to prepare
|
||||
self.last_prepared.pop(user, None)
|
||||
waiting = self.worker.replies_unanswered
|
||||
early = waiting.get(user, 0) > 0 # sent before their previous answer: not a reply to it
|
||||
# Sent before their previous answer: not a reply to it. An answer taken back (cancelled while
|
||||
# the brain thinks) does not count: what comes now replaces it.
|
||||
taken_back = sum(1 for r in self.cancelled_replies if self.thinking_replies.get(r) == user)
|
||||
early = waiting.get(user, 0) - taken_back > 0
|
||||
waiting[user] = waiting.get(user, 0) + 1
|
||||
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes))
|
||||
|
||||
@@ -918,7 +941,7 @@ class Session:
|
||||
await self._reply_inner(rid, user, text, via, speak, early, channel, notes or [], trace=trace)
|
||||
finally:
|
||||
trace.finish("") # only when it did not get to its answer
|
||||
self.thinking_replies.discard(rid)
|
||||
self.thinking_replies.pop(rid, None)
|
||||
self.cancelled_replies.discard(rid)
|
||||
if not self._handed_over.pop(rid, False): # else _delivered counts it
|
||||
self._reached(user)
|
||||
@@ -986,23 +1009,39 @@ class Session:
|
||||
pc = AppBridge(self, asyncio.get_running_loop(), "pc", *origin, turn=turn,
|
||||
state=w.pc_state.setdefault(user, {}))
|
||||
trace.append(self._records_step(history, notes))
|
||||
try:
|
||||
async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
||||
schedule=schedule, notes=notes or [],
|
||||
notes_backend=notes_backend, screen=screen, pc=pc,
|
||||
trace=trace)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
w.scheduler.wake() # held partials go on
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = None
|
||||
answer = None
|
||||
if speak and rid in self.cancelled_replies:
|
||||
# Taken back while it waited for the asker's previous answer: never started.
|
||||
debug_log(f"brain reply {rid} not started: cancelled while waiting", "llm")
|
||||
else:
|
||||
try:
|
||||
async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(
|
||||
w.brain.reply, user, text, history, chat=chat, schedule=schedule,
|
||||
notes=notes or [], notes_backend=notes_backend, screen=screen, pc=pc,
|
||||
trace=trace, cancelled=lambda: rid in self.cancelled_replies)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
w.scheduler.wake() # held partials go on
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = None
|
||||
answer = (answer or "").strip()
|
||||
trace.finish(answer)
|
||||
if not answer and speak and rid in self.cancelled_replies:
|
||||
# Taken back before it was written: as if never asked, so the request that replaces
|
||||
# it is still the very next turn (a confirmation it carries does not lapse).
|
||||
w.reply_turns[user] = turn - (2 if early else 1)
|
||||
debug_log(f"brain reply {rid} taken back", "llm")
|
||||
self.send({"type": "reply_text", "id": rid, "text": "", "failed": True})
|
||||
if rid in self.tts_requests:
|
||||
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False})
|
||||
else:
|
||||
self.send({"type": "tts_done", "id": rid, "chunks": 0, "cancelled": True, "failed": False})
|
||||
return
|
||||
if not answer and rid in self.tts_requests:
|
||||
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech
|
||||
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
||||
|
||||
@@ -13,6 +13,7 @@ class SttSettings:
|
||||
final_beam: int = 2
|
||||
language: str | None = "ko"
|
||||
hotwords: tuple[str, ...] = ()
|
||||
context_chars: int = 200
|
||||
|
||||
def update(self, changes: dict) -> None:
|
||||
"""Apply ``changes``; raises ValueError and changes nothing if any value is bad."""
|
||||
@@ -52,6 +53,8 @@ def _check(key: str, value):
|
||||
if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8):
|
||||
raise ValueError("language must be a language code or null")
|
||||
return value
|
||||
if key == "context_chars":
|
||||
return _int_in(key, value, 0, 500)
|
||||
if key == "hotwords":
|
||||
if (not isinstance(value, (list, tuple)) or len(value) > 50
|
||||
or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)):
|
||||
|
||||
@@ -73,6 +73,25 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
|
||||
return plain_text
|
||||
|
||||
|
||||
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str:
|
||||
"""Pick between the decode without the conversation context (``heard``, already through the hint
|
||||
choice) and the one that had it as Whisper's previous-text prompt (``prompted``).
|
||||
|
||||
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
|
||||
words of the prompt, so the decode without it decides, as for the hint: nothing heard stays
|
||||
nothing, and the prompted text is used only when it stays close to what was heard and holds every
|
||||
hotword exactly as often.
|
||||
"""
|
||||
if not heard or not prompted:
|
||||
return heard
|
||||
h, p = _letters(heard), _letters(prompted)
|
||||
if _edit_distance(p, h) > len(h) // 2:
|
||||
return heard
|
||||
if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords):
|
||||
return heard
|
||||
return prompted
|
||||
|
||||
|
||||
def add_cuda_dll_dirs() -> None:
|
||||
"""On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin."""
|
||||
if not hasattr(os, "add_dll_directory"):
|
||||
@@ -104,8 +123,25 @@ class WhisperTranscriber:
|
||||
def warm_up(self) -> None:
|
||||
self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=()))
|
||||
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session."""
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
|
||||
conversation to prompt with ("" for none); see ``choose_with_context``."""
|
||||
if contexts and any(contexts):
|
||||
# Prompts of different lengths cannot share one generate call (the start token must sit
|
||||
# at the same position): one clip at a time. The encoder costs the same per clip anyway.
|
||||
return [self._with_context(c, opts, ctx) for c, ctx in zip(clips, contexts)]
|
||||
enc, langs = self._encode(clips, opts)
|
||||
return self._choose(enc, langs, opts, None)
|
||||
|
||||
def _with_context(self, clip: np.ndarray, opts: DecodeOptions, context: str) -> str:
|
||||
enc, langs = self._encode([clip], opts)
|
||||
heard = self._choose(enc, langs, opts, None)[0]
|
||||
if not context:
|
||||
return heard
|
||||
return choose_with_context(heard, self._choose(enc, langs, opts, [context])[0], opts.hotwords)
|
||||
|
||||
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
|
||||
m = self.model
|
||||
n_frames = m.feature_extractor.nb_max_frames
|
||||
feats = []
|
||||
@@ -118,16 +154,22 @@ class WhisperTranscriber:
|
||||
langs = [opts.language] * len(clips)
|
||||
else:
|
||||
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
return enc, langs
|
||||
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, contexts: list[str] | None) -> list[str]:
|
||||
plain = self._decode(enc, langs, opts.beam, None, contexts)
|
||||
if not opts.hotwords:
|
||||
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords))
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), contexts)
|
||||
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
|
||||
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]:
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
|
||||
contexts: list[str] | None = None) -> list[tuple[str, float]]:
|
||||
m = self.model
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords)
|
||||
for lang in langs]
|
||||
prompts = [m.get_prompt(self._tokenizer(lang),
|
||||
self._tokenizer(lang).encode(" " + ctx.strip()) if ctx else [],
|
||||
without_timestamps=True, hotwords=hotwords)
|
||||
for lang, ctx in zip(langs, contexts or [""] * len(langs))]
|
||||
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
|
||||
suppress_tokens=[-1], return_scores=True)
|
||||
# ``scores`` is the sequence log-probability divided by its length: the mean per token.
|
||||
|
||||
@@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`):
|
||||
| `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. |
|
||||
| `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. |
|
||||
| `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. |
|
||||
| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. |
|
||||
| `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). |
|
||||
|
||||
### Utterance rules
|
||||
@@ -152,6 +153,35 @@ text is empty:
|
||||
|
||||
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
|
||||
### Conversation context
|
||||
|
||||
People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기")
|
||||
were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이"
|
||||
and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made
|
||||
the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also
|
||||
decoded with the recent conversation as its previous-text prompt:
|
||||
|
||||
- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said
|
||||
(the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined
|
||||
by spaces, at most `context_chars` characters from the end. Joining another channel or leaving
|
||||
clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt.
|
||||
- Partials have none (they stay one cheap decode).
|
||||
- The clip is decoded as before (with the hotword choice above) and again with the context (the
|
||||
same choice, each decode prompted). The prompted text is used only when the one without context
|
||||
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
|
||||
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
|
||||
Whisper write words of the conversation; this rule keeps that out, as for the hint.
|
||||
- Prompts of different lengths cannot share one decoder call, so finals with a context are decoded
|
||||
one clip at a time (the encoder costs the same per clip).
|
||||
|
||||
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
|
||||
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
|
||||
text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more
|
||||
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
|
||||
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
|
||||
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
|
||||
whole sentences of the prompt. Evidence and the model comparison: `docs/v2/stt-accuracy-plan.md`.
|
||||
|
||||
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
|
||||
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
|
||||
clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD
|
||||
@@ -257,8 +287,15 @@ which carry Discord's own time.
|
||||
answer is stored when the app reports the sent message as `chat_message` with `to`. With
|
||||
`speak: true` the answer is also spoken through the TTS path under the same id (audio frames and
|
||||
`tts_done`), and `tts_cancel` with that id stops it. A `tts_cancel` that arrives while the brain is
|
||||
still answering means the answer is not spoken: `reply_text` still comes, then `tts_done` with
|
||||
`cancelled: true` and no audio. Every reply, a task's too and one that failed, leaves a trace of
|
||||
still answering means the answer is no longer wanted (the app took the request back: its asker
|
||||
went on talking, a newer request or a stop took its place). A reply still waiting for the asker's
|
||||
previous answer is never started; a running one stops at the engine's next model call or tool
|
||||
call (`turn_state.cancel`: no tool runs on its behalf) and returns no answer. Such a reply is as if
|
||||
never asked: `reply_text` comes with empty text and `failed: true`, then `tts_done` with
|
||||
`cancelled: true` and no audio, and it uses up no turn of the asker's (the request that replaces it
|
||||
is still the very next turn, so a confirmation it carries does not lapse; nor is that request
|
||||
"sent before the previous answer"). A reply whose answer was already written when the cancel
|
||||
came still sends its text, then `tts_done` with `cancelled: true` and no audio. Every reply, a task's too and one that failed, leaves a trace of
|
||||
how it was made (`history.spec.md`, "Traces"): the history lines and number of notes it was given,
|
||||
then what the engine wrote down. It is stored as soon as the request arrives (before it waits for
|
||||
the asker's previous answer) and again as each step is added, then ends with the answer, or with
|
||||
|
||||
66
tests/test_engine_cancel.py
Normal file
66
tests/test_engine_cancel.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""An answer that is no longer wanted stops at the engine's next model or tool call (turn_state.cancel).
|
||||
|
||||
The voice worker sets it when the app takes the request back (the asker went on talking and the
|
||||
request is joined to what they said next, a newer request took its place, or a stop). The engine
|
||||
then returns no answer and runs no tool on the request's behalf.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _native_call(name: str, args: dict) -> dict:
|
||||
return {"message": {"role": "assistant", "content": "", "tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": name, "arguments": args}}]}}
|
||||
|
||||
|
||||
def _run(mock_config, db, dialogue_memory, cancel_after_calls: int):
|
||||
from jarvis.memory.history import HistoryContext
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.tools.types import ToolExecutionResult
|
||||
|
||||
mock_config.ollama_chat_model = "gpt-oss:20b"
|
||||
mock_config.evaluator_enabled = False
|
||||
mock_config.stream_browser = False
|
||||
calls = []
|
||||
invoked = []
|
||||
|
||||
def fake_chat(*args, **kwargs):
|
||||
calls.append(1)
|
||||
return _native_call("getWeather", {"location": "Seoul"})
|
||||
|
||||
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
|
||||
invoked.append(tool_name)
|
||||
return ToolExecutionResult(success=True, reply_text="Seoul: 13C", error_message=None)
|
||||
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: len(calls) >= cancel_after_calls)
|
||||
try:
|
||||
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
|
||||
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
|
||||
patch.object(engine_mod, "select_tools", return_value=["getWeather", "stop"]), \
|
||||
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]):
|
||||
reply = engine_mod.run_reply_engine(
|
||||
db=db, cfg=mock_config, tts=None, text="서울 날씨 어때?",
|
||||
dialogue_memory=dialogue_memory, history=HistoryContext(), legacy_memory=False,
|
||||
intent_judge=True)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
return reply, len(calls), invoked
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_answer_cancelled_before_it_starts_makes_no_model_call(mock_config, db, dialogue_memory):
|
||||
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=0)
|
||||
assert reply is None and calls == 0 and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_answer_cancelled_during_a_model_call_runs_no_tool_for_it(mock_config, db, dialogue_memory):
|
||||
# The model asks for a tool, but the answer was taken back meanwhile: the tool is not run.
|
||||
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=1)
|
||||
assert reply is None and calls == 1 and invoked == []
|
||||
@@ -29,8 +29,9 @@ class FakeBrain:
|
||||
self.calls = []
|
||||
|
||||
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
|
||||
screen=None, pc=None, trace=None):
|
||||
screen=None, pc=None, trace=None, cancelled=None):
|
||||
self.calls.append((user, text, history))
|
||||
self.cancelled = cancelled
|
||||
if trace is not None:
|
||||
trace.append({"kind": "tools", "names": ["getWeather"]})
|
||||
self.chat, self.schedule, self.task = chat, schedule, task
|
||||
@@ -302,7 +303,7 @@ class SlowTranscriber(LevelTranscriber):
|
||||
super().__init__()
|
||||
self.delay = delay
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
import time as _t
|
||||
_t.sleep(self.delay)
|
||||
return super().transcribe(clips, opts)
|
||||
@@ -730,6 +731,83 @@ def test_a_reply_cancelled_while_the_brain_thinks_is_not_spoken(store):
|
||||
assert replies == ["r1", "r2"], "the text still comes back; only the speech is skipped"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_cancelled_reply_stops_the_brain_and_one_still_waiting_never_reaches_it(store):
|
||||
"""A request taken back (the asker went on talking) must not hold up the next one: the brain is
|
||||
told to stop the running answer, and a reply still waiting for it is never started."""
|
||||
import time
|
||||
started = []
|
||||
|
||||
def answer(user, text):
|
||||
started.append(text)
|
||||
if text == "first":
|
||||
t0 = time.monotonic()
|
||||
while not brain.cancelled() and time.monotonic() - t0 < 10:
|
||||
time.sleep(0.01)
|
||||
return None if brain.cancelled() else "late answer"
|
||||
return f"answer for {text}"
|
||||
|
||||
brain = FakeBrain(answer)
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="voice_join", channel="vc1")
|
||||
await send(ws, type="reply", id="r1", user="100", text="first", via="voice", speak=True)
|
||||
await asyncio.sleep(0.1)
|
||||
await send(ws, type="reply", id="r2", user="100", text="second", via="voice", speak=True)
|
||||
await send(ws, type="tts_cancel", id="r1")
|
||||
await send(ws, type="tts_cancel", id="r2")
|
||||
await send(ws, type="reply", id="r3", user="100", text="first second", via="voice", speak=True)
|
||||
t0 = time.monotonic()
|
||||
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done"
|
||||
and m["id"] == "r3")
|
||||
return seen, time.monotonic() - t0
|
||||
|
||||
seen, waited = asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
|
||||
assert waited < 3, "the joined request did not wait for the cancelled answer to run out"
|
||||
assert started == ["first", "first second"]
|
||||
done = {m["id"]: m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"}
|
||||
assert done["r1"]["cancelled"] and done["r2"]["cancelled"] and not done["r3"]["cancelled"]
|
||||
frames = {decode_tts_frame(m)[0] for m in seen if isinstance(m, bytes)}
|
||||
assert frames == {"r3"}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_request_taken_back_does_not_use_up_the_turn_a_confirmation_needs(store):
|
||||
"""Jarvis asked something; the asker says "yes" and goes on ("yes, quickly"). The app takes the
|
||||
"yes" back and sends the joined text: it must still be the very next turn, or a confirmation
|
||||
(pcControl, chat delete) would lapse."""
|
||||
import time
|
||||
turns = {}
|
||||
|
||||
def answer(user, text):
|
||||
turns[text] = brain.pc.turn
|
||||
if text == "yes":
|
||||
t0 = time.monotonic()
|
||||
while not brain.cancelled() and time.monotonic() - t0 < 10:
|
||||
time.sleep(0.01)
|
||||
return None
|
||||
return f"answer for {text}"
|
||||
|
||||
brain = FakeBrain(answer)
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="voice_join", channel="vc1")
|
||||
await send(ws, type="reply", id="r1", user="100", text="open the calculator", via="voice", speak=False)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
await send(ws, type="reply", id="r2", user="100", text="yes", via="voice", speak=True)
|
||||
await asyncio.sleep(0.1)
|
||||
await send(ws, type="tts_cancel", id="r2")
|
||||
await send(ws, type="reply", id="r3", user="100", text="yes quickly", via="voice", speak=True)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done" and m["id"] == "r3")
|
||||
|
||||
asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
|
||||
assert turns["yes quickly"] == turns["open the calculator"] + 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_brains_judgements_use_its_model_and_map_back_to_ids(monkeypatch):
|
||||
import voice_worker.judges as judges
|
||||
|
||||
@@ -96,3 +96,39 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
||||
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
||||
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
||||
== "자비스 오늘 서울 날씨 어때"
|
||||
|
||||
|
||||
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
|
||||
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
|
||||
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
|
||||
from voice_worker.stt_whisper import choose_with_context # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_may_fix_words_of_what_was_said():
|
||||
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_nothing_heard_without_the_context_stays_nothing():
|
||||
# Noise: the prompted decode wrote words of the conversation.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
|
||||
heard = "가상화면을 못 봐"
|
||||
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
|
||||
assert choose_with_context(heard, copied, HINT) == heard
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_never_changes_how_many_wake_words_were_heard():
|
||||
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
|
||||
== "라비스 지금 영상 재생되고 있어?"
|
||||
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_empty_prompted_decode_keeps_what_was_heard():
|
||||
assert choose_with_context("네.", "", HINT) == "네."
|
||||
|
||||
@@ -25,7 +25,7 @@ class FakeTranscriber:
|
||||
self.gate.set()
|
||||
self.fail_on_beam = fail_on_beam
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.entered.set()
|
||||
self.gate.wait(5)
|
||||
self.calls.append((opts, [len(c) for c in clips]))
|
||||
@@ -218,12 +218,45 @@ class StepTranscriber:
|
||||
def step(self):
|
||||
self._steps.release()
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append([len(c) for c in clips])
|
||||
self._steps.acquire(timeout=5)
|
||||
return ["" for _ in clips]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_reaches_the_model_with_its_own_context():
|
||||
class Ctx:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append(list(contexts or []))
|
||||
return [f"n{len(c)}" for c in clips]
|
||||
|
||||
eng = Ctx()
|
||||
|
||||
async def body():
|
||||
s = SttScheduler(eng)
|
||||
s.start()
|
||||
got = []
|
||||
done = asyncio.Event()
|
||||
|
||||
def cb(r):
|
||||
got.append(r.text)
|
||||
if len(got) == 3:
|
||||
done.set()
|
||||
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
|
||||
s.submit_final("b", clip(200), FINAL, cb)
|
||||
s.submit_partial("c", clip(300), PARTIAL, cb)
|
||||
await asyncio.wait_for(done.wait(), 5)
|
||||
await s.stop()
|
||||
|
||||
asyncio.run(body())
|
||||
assert eng.calls[0] == ["화면 공유", ""]
|
||||
assert eng.calls[1] == [""], "partials go without context"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
|
||||
t = StepTranscriber()
|
||||
|
||||
@@ -32,7 +32,7 @@ class LevelTranscriber:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append((opts, len(clips)))
|
||||
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
||||
|
||||
@@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k():
|
||||
lengths = []
|
||||
orig = t.transcribe
|
||||
|
||||
def spy(clips, opts):
|
||||
def spy(clips, opts, contexts=None):
|
||||
lengths.extend(len(c) for c in clips)
|
||||
return orig(clips, opts)
|
||||
|
||||
@@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path
|
||||
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
|
||||
r.close()
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]
|
||||
|
||||
|
||||
class ContextTranscriber(LevelTranscriber):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.contexts = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
if opts.beam > 1: # finals
|
||||
self.contexts += list(contexts or [""] * len(clips))
|
||||
return super().transcribe(clips, opts)
|
||||
|
||||
|
||||
async def _said(ws, speaker, level):
|
||||
await stream(ws, speaker, level, 600)
|
||||
await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel():
|
||||
"""voice_worker.spec.md, "Conversation context": what was said and answered just before, in the
|
||||
channel Jarvis is in, is the model's prompt for the next final."""
|
||||
t = ContextTranscriber()
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"}))
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.",
|
||||
"more": False}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
||||
await _said(ws, "200", 0.6)
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "config", "context_chars": 0}))
|
||||
await _said(ws, "100", 0.6)
|
||||
|
||||
asyncio.run(with_worker(body, transcriber=t, tts=FakeTts()))
|
||||
assert t.contexts[0] == ""
|
||||
assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다."
|
||||
assert t.contexts[2] == "", "a new channel starts without the old channel's conversation"
|
||||
assert t.contexts[3] == "", "context_chars 0 turns it off"
|
||||
|
||||
Reference in New Issue
Block a user