Compare commits
17 Commits
app-v0.1.4
...
codex/owne
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b39aabd368 | ||
|
|
d7dc62c202 | ||
|
|
b64097ef0d | ||
|
|
81bd8bd573 | ||
|
|
e716065538 | ||
|
|
41f539dd48 | ||
|
|
b30aa4a0db | ||
|
|
835b547ca2 | ||
|
|
09bdf05762 | ||
|
|
251f5e116d | ||
|
|
c1b561d66d | ||
|
|
9a527aa470 | ||
|
|
628d89c4f1 | ||
|
|
478968949c | ||
|
|
caad09c613 | ||
|
|
97b827f424 | ||
|
|
045218b008 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.46"
|
||||
version = "0.1.52"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.46"
|
||||
version = "0.1.52"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.46"
|
||||
version = "0.1.52"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.46"
|
||||
version = "0.1.52"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.46"
|
||||
version = "0.1.52"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -31,6 +31,13 @@ pub struct CallSettings {
|
||||
/// After a wake word said alone, how long the speaker's next utterance may take to start and
|
||||
/// still be joined to it (the pause after "Jarvis," often ends an utterance).
|
||||
pub follow_up: Duration,
|
||||
/// A speaker who goes on talking before Jarvis's answer to them is heard, starting within this of
|
||||
/// their last utterance's end, is still asking: what they say is joined to the request
|
||||
/// (`call.continueMs`; zero turns it off).
|
||||
pub continue_window: Duration,
|
||||
/// A voice answer starts playing only once its asker has been quiet this long after their request
|
||||
/// ended (they may go on); it is written meanwhile (`call.answerWaitMs`; zero: as soon as written).
|
||||
pub answer_wait: Duration,
|
||||
/// Play the earcon when Jarvis takes a voice call (`call.earcon`).
|
||||
pub earcon: bool,
|
||||
}
|
||||
@@ -46,6 +53,8 @@ impl Default for CallSettings {
|
||||
designated_no_wake_word: false,
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
continue_window: Duration::from_millis(5000),
|
||||
answer_wait: Duration::from_millis(3000),
|
||||
earcon: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,10 @@ pub struct Room {
|
||||
pub trait Voice {
|
||||
/// Have the brain answer and speak it; the speech request id, or None when it cannot. `heard`:
|
||||
/// when the request's speech ended and its final arrived (the dashboard times the answer by it).
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String>;
|
||||
/// `hold`: the answer is written but does not start playing until `release`.
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String>;
|
||||
/// Let a held answer play.
|
||||
fn release(&self, id: &str);
|
||||
/// Stop one speech request.
|
||||
fn cancel(&self, id: &str);
|
||||
/// Stop all speech at once.
|
||||
@@ -57,8 +60,11 @@ pub trait Voice {
|
||||
}
|
||||
|
||||
impl Voice for DiscordCore {
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard)
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard, hold)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
DiscordCore::release(self, id)
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
DiscordCore::cancel(self, id)
|
||||
@@ -167,6 +173,25 @@ pub struct Conversation<V: Voice> {
|
||||
wakes: HashMap<u64, (Request, Option<Instant>)>,
|
||||
/// Requests whose earcon already played (a wake word said alone was answered with it).
|
||||
chimed: HashSet<u64>,
|
||||
/// Each speaker's latest voice request whose answer has not been heard yet: what they say next
|
||||
/// may still be part of it ("Going on").
|
||||
turns: HashMap<u64, Turn>,
|
||||
/// Held answers (speech id): their asker, and when they will have been quiet for `call.answerWaitMs`.
|
||||
waiting: HashMap<String, (u64, Instant)>,
|
||||
}
|
||||
|
||||
/// A voice request whose answer is not heard yet.
|
||||
struct Turn {
|
||||
id: u64,
|
||||
text: String,
|
||||
ended_at: Instant,
|
||||
/// Still a "was it said to Jarvis?" question (its ask id), not yet a request of the policy.
|
||||
ask: Option<String>,
|
||||
/// The speaker started their next utterance within `call.continueMs` while the answer was not
|
||||
/// heard: the request stays open for that utterance's final, also if the answer starts meanwhile.
|
||||
held: bool,
|
||||
/// Its answer started playing (kept only while `held`).
|
||||
heard: bool,
|
||||
}
|
||||
|
||||
impl<V: Voice> Conversation<V> {
|
||||
@@ -185,7 +210,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
|
||||
by_speech: HashMap::new(), asks: HashMap::new(), missed: MissedLog::default(), chat: ChatCalls::default(),
|
||||
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new(),
|
||||
chimed: HashSet::new() }
|
||||
chimed: HashSet::new(), turns: HashMap::new(), waiting: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Answer the registered chat channels too.
|
||||
@@ -225,6 +250,13 @@ impl<V: Voice> Conversation<V> {
|
||||
/// would be) lets the brain get ready while the speaker is still talking. Not while Jarvis is
|
||||
/// answering: the call may not be taken, and the brain is busy with the answer.
|
||||
fn on_partial(&mut self, speaker: u64, partial: &str) {
|
||||
// Going on with a request whose answer is not heard yet: it will be joined and taken.
|
||||
if let Some(t) = self.turns.get(&speaker) {
|
||||
if !self.calls.continue_window.is_zero() && !matches!(self.calls.judge(speaker, partial), Call::Stop) {
|
||||
self.voice.prepare(speaker, &format!("{} {}", t.text, partial));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if !self.answers.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -248,6 +280,24 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
self.handle(ev, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
/// Held answers whose asker has been quiet for `call.answerWaitMs` start playing; while the asker
|
||||
/// talks, their answer waits for what they say (it may join the request).
|
||||
fn release_due(&mut self, now: Instant) {
|
||||
let due: Vec<String> = self.waiting.iter()
|
||||
.filter(|(_, (speaker, at))| now >= *at && !self.started.contains_key(speaker))
|
||||
.map(|(sid, _)| sid.clone()).collect();
|
||||
for sid in due {
|
||||
self.waiting.remove(&sid);
|
||||
debug_log!("conversation", "release speech={sid}");
|
||||
self.voice.release(&sid);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::SpeechStart(h) => {
|
||||
self.policy.utterance_open(h.speaker_id);
|
||||
@@ -255,6 +305,14 @@ impl<V: Voice> Conversation<V> {
|
||||
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
|
||||
*deadline = None; // they go on: wait for this utterance's final
|
||||
}
|
||||
let window = self.calls.continue_window;
|
||||
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
|
||||
t.held = !window.is_zero() && now <= t.ended_at + window;
|
||||
// Too late to go on with a request whose answer was heard: what comes is new.
|
||||
if !t.held && t.heard {
|
||||
self.turns.remove(&h.speaker_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::Chat(m) => self.on_chat(m, now),
|
||||
CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial),
|
||||
@@ -277,10 +335,15 @@ impl<V: Voice> Conversation<V> {
|
||||
self.started.remove(&speaker);
|
||||
}
|
||||
// What follows a wake word said alone is joined to it.
|
||||
let (text, chimed) = match self.wakes.remove(&speaker) {
|
||||
let (mut text, mut chimed) = match self.wakes.remove(&speaker) {
|
||||
Some((wake, _)) => (format!("{} {}", wake.text, f.text), self.chimed.remove(&wake.id)),
|
||||
None => (f.text.clone(), false),
|
||||
};
|
||||
// The speaker goes on before their answer was heard: still the same request.
|
||||
if let Some((earlier, earcon_played)) = self.going_on(speaker, f, now) {
|
||||
text = format!("{earlier} {text}");
|
||||
chimed |= earcon_played;
|
||||
}
|
||||
self.next_id += 1;
|
||||
if chimed {
|
||||
self.chimed.insert(self.next_id);
|
||||
@@ -316,21 +379,19 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
Call::Request { direct } => {
|
||||
let a = self.policy.request(Request { direct, ..req }, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
Call::Request { direct } => self.voice_request(Request { direct, ..req }, now),
|
||||
Call::Ask => {
|
||||
let room = self.room(speaker);
|
||||
if room.people <= 1 {
|
||||
// Alone with Jarvis: whatever the speaker says is said to it.
|
||||
debug_log!("conversation", "addressed req={} alone with jarvis", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
self.voice_request(req, now);
|
||||
return;
|
||||
}
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &text, room) {
|
||||
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
|
||||
ended_at: req.ended_at, ask: Some(id.clone()), held: self.holds(speaker), heard: false });
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -344,12 +405,13 @@ impl<V: Voice> Conversation<V> {
|
||||
// An answer after the timeout counts as no, even before the timer ran.
|
||||
let yes = *addressed && now < asked + ADDRESSED_TIMEOUT;
|
||||
debug_log!("conversation", "addressed req={} yes={yes}", req.id);
|
||||
let a = if yes {
|
||||
self.policy.request(req, now)
|
||||
self.turns.retain(|_, t| t.id != req.id);
|
||||
if yes {
|
||||
self.voice_request(req, now);
|
||||
} else {
|
||||
self.policy.utterance_closed(req.speaker, now)
|
||||
};
|
||||
self.apply(a, now);
|
||||
let a = self.policy.utterance_closed(req.speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::Ranked { id, order } => {
|
||||
@@ -358,9 +420,25 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.ranked(rank_id, &order, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
// The answer is heard; while it plays, its asker may still go on (within call.continueMs).
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
self.waiting.remove(&e.id);
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
// The speaker may still go on within call.continueMs of the request's end (a short
|
||||
// answer ends inside a pause), and a held request waits for its next final anyway.
|
||||
let window = self.calls.continue_window;
|
||||
self.turns.retain(|_, t| t.id != pid || t.held || now < t.ended_at + window);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
|
||||
e.played, e.cancelled, e.failed);
|
||||
let a = self.policy.answer_done(pid, now);
|
||||
@@ -378,6 +456,7 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
CoreEvent::WorkerLost => {
|
||||
self.started.clear();
|
||||
self.turns.clear();
|
||||
for (_, (req, _)) in self.wakes.drain() {
|
||||
self.chimed.remove(&req.id);
|
||||
}
|
||||
@@ -386,9 +465,58 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The speaker's next utterance already started (a pause shorter than the first final's
|
||||
/// transcription): their new request is held open for it from the start.
|
||||
fn holds(&self, speaker: u64) -> bool {
|
||||
!self.calls.continue_window.is_zero() && self.started.contains_key(&speaker)
|
||||
}
|
||||
|
||||
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
|
||||
fn voice_request(&mut self, req: Request, now: Instant) {
|
||||
let held = self.holds(req.speaker);
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
|
||||
held, heard: false });
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
|
||||
/// The final `f` continues the speaker's request whose answer is not heard yet (it started within
|
||||
/// `call.continueMs` of that request's end, and is not a stop): the request is taken back, its
|
||||
/// running answer stopped. Returns its text and whether its earcon already played.
|
||||
fn going_on(&mut self, speaker: u64, f: &crate::discord::HeardFinal, now: Instant) -> Option<(String, bool)> {
|
||||
let t = self.turns.get_mut(&speaker)?;
|
||||
let started = now.checked_sub(Duration::from_millis(f.wait_ms + f.audio_ms)).unwrap_or(now);
|
||||
// An empty final (noise the worker rejected) adds nothing and asks nothing again.
|
||||
if crate::calls::normalise(&f.text).is_empty() || self.calls.continue_window.is_zero()
|
||||
|| started > t.ended_at + self.calls.continue_window
|
||||
|| matches!(self.calls.judge(speaker, &f.text), Call::Stop) {
|
||||
t.held = false;
|
||||
if t.heard {
|
||||
self.turns.remove(&speaker);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
let t = self.turns.remove(&speaker).expect("checked");
|
||||
let earcon_played = self.chimed.remove(&t.id)
|
||||
|| (self.calls.earcon && (self.answers.contains_key(&t.id) || t.heard));
|
||||
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
|
||||
match t.ask {
|
||||
Some(q) => {
|
||||
self.asks.remove(&q);
|
||||
}
|
||||
None => {
|
||||
let a = self.policy.withdraw(t.id);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
Some((t.text, earcon_played))
|
||||
}
|
||||
|
||||
/// The policy's stop, and every pending "was it said to Jarvis?" question is void.
|
||||
fn stop_everything(&mut self, now: Instant) {
|
||||
self.wakes.clear();
|
||||
self.turns.clear();
|
||||
self.waiting.clear();
|
||||
let a = self.policy.stop();
|
||||
self.apply(a, now);
|
||||
self.chimed.clear(); // nothing is waiting any more
|
||||
@@ -405,6 +533,7 @@ impl<V: Voice> Conversation<V> {
|
||||
for id in late {
|
||||
if let Some((req, _)) = self.asks.remove(&id) {
|
||||
debug_log!("conversation", "addressed req={} timed out", req.id);
|
||||
self.turns.retain(|_, t| t.id != req.id);
|
||||
let a = self.policy.utterance_closed(req.speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
@@ -415,32 +544,38 @@ impl<V: Voice> Conversation<V> {
|
||||
for speaker in due {
|
||||
if let Some((req, _)) = self.wakes.remove(&speaker) {
|
||||
debug_log!("conversation", "wake word alone req={} answered alone", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
self.voice_request(req, now);
|
||||
}
|
||||
}
|
||||
let a = self.policy.tick(now);
|
||||
self.apply(a, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
pub fn next_deadline(&self) -> Option<Instant> {
|
||||
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT);
|
||||
let wakes = self.wakes.values().filter_map(|(_, d)| *d);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min()
|
||||
// Not while the asker talks: their final decides.
|
||||
let held = self.waiting.values().filter(|(s, _)| !self.started.contains_key(s)).map(|(_, at)| *at);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).chain(held).min()
|
||||
}
|
||||
|
||||
fn apply(&mut self, actions: Vec<Action>, now: Instant) {
|
||||
for action in actions {
|
||||
match action {
|
||||
Action::Answer(r) => match {
|
||||
// The earcon plays first, then the answer as soon as it is written.
|
||||
// The earcon plays first, then the answer once it is written and its asker has
|
||||
// been quiet for call.answerWaitMs.
|
||||
if self.calls.earcon && !self.chimed.remove(&r.id) {
|
||||
self.voice.earcon();
|
||||
}
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at))
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at), !self.calls.answer_wait.is_zero())
|
||||
} {
|
||||
Some(sid) => {
|
||||
debug_log!("conversation", "answer req={} speaker={} speech={sid}", r.id, r.speaker);
|
||||
if !self.calls.answer_wait.is_zero() {
|
||||
self.waiting.insert(sid.clone(), (r.speaker, r.ended_at + self.calls.answer_wait));
|
||||
}
|
||||
self.by_speech.insert(sid.clone(), r.id);
|
||||
self.answers.insert(r.id, sid);
|
||||
}
|
||||
@@ -452,11 +587,13 @@ impl<V: Voice> Conversation<V> {
|
||||
Action::StopAnswer(pid) => {
|
||||
if let Some(sid) = self.answers.remove(&pid) {
|
||||
self.by_speech.remove(&sid);
|
||||
self.waiting.remove(&sid);
|
||||
self.voice.cancel(&sid);
|
||||
}
|
||||
}
|
||||
Action::Missed(r, reason) => {
|
||||
self.chimed.remove(&r.id);
|
||||
self.turns.retain(|_, t| t.id != r.id);
|
||||
debug_log!("conversation", "missed req={} speaker={} reason={reason:?}", r.id, r.speaker);
|
||||
let mut missed = self.missed.lock().unwrap();
|
||||
if missed.len() == MISSED_KEPT {
|
||||
@@ -505,7 +642,7 @@ mod tests {
|
||||
|
||||
/// The tests call Jarvis by one wake word of their own, whatever the default is.
|
||||
fn test_calls() -> CallSettings {
|
||||
CallSettings { wake_words: vec!["자비스".into()], ..CallSettings::default() }
|
||||
CallSettings { wake_words: vec!["자비스".into()], answer_wait: Duration::ZERO, ..CallSettings::default() }
|
||||
}
|
||||
use crate::discord::{Heard, HeardFinal};
|
||||
use crate::playback::PlaybackEnd;
|
||||
@@ -517,6 +654,9 @@ mod tests {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Did {
|
||||
Answer(u64, String),
|
||||
/// The answer just asked for is held until released.
|
||||
Held(String),
|
||||
Release(String),
|
||||
Cancel(String),
|
||||
StopAll,
|
||||
Earcon,
|
||||
@@ -541,10 +681,17 @@ mod tests {
|
||||
people: Rc<RefCell<Vec<u64>>>,
|
||||
}
|
||||
impl Voice for Fake {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant)) -> Option<String> {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
*self.n.borrow_mut() += 1;
|
||||
let id = format!("s{}", self.n.borrow());
|
||||
self.did.borrow_mut().push(Did::Answer(user, text.into()));
|
||||
Some(format!("s{}", self.n.borrow()))
|
||||
if hold {
|
||||
self.did.borrow_mut().push(Did::Held(id.clone()));
|
||||
}
|
||||
Some(id)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Release(id.into()));
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Cancel(id.into()));
|
||||
@@ -590,6 +737,7 @@ mod tests {
|
||||
|
||||
const A: u64 = 11;
|
||||
const B: u64 = 22;
|
||||
const C: u64 = 33;
|
||||
|
||||
fn conv(kind: PolicyKind, mode: CallMode) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
@@ -613,7 +761,7 @@ mod tests {
|
||||
assert!(f.take().is_empty(), "a waiting request gets its earcon when its answer starts");
|
||||
c.on_event(&end("s1", false), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(B, "자비스 불 꺼 줘".into())]);
|
||||
c.on_event(&fin(B, "오늘 뭐 먹지", 100), ms(t, 3100));
|
||||
c.on_event(&fin(C, "오늘 뭐 먹지", 100), ms(t, 3100));
|
||||
assert!(f.take().is_empty(), "no call, no earcon");
|
||||
}
|
||||
|
||||
@@ -626,10 +774,11 @@ mod tests {
|
||||
c.on_event(&start(A, 2), ms(t, 1200));
|
||||
c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 지금 몇 시야?".into())]);
|
||||
// Nothing followed the next one: it is answered alone, still with one earcon.
|
||||
// Nothing followed the next one (said after the going-on window): it is answered alone,
|
||||
// still with one earcon.
|
||||
c.on_event(&end("s1", false), ms(t, 5000));
|
||||
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 6000));
|
||||
c.tick(ms(t, 6000) + test_calls().follow_up);
|
||||
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 9500));
|
||||
c.tick(ms(t, 9500) + test_calls().follow_up);
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스".into())]);
|
||||
}
|
||||
|
||||
@@ -659,10 +808,10 @@ mod tests {
|
||||
c.on_event(&fin_utt(B, 1, "자비스"), ms(t, 100));
|
||||
c.on_event(&partial(B, "지금 몇 시"), ms(t, 300));
|
||||
assert_eq!(f.take(), vec![Did::Prepare(B, "자비스 지금 몇 시".into())]);
|
||||
// Not while an answer runs.
|
||||
// Not while an answer runs (to someone else: its asker going on is joined to it).
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 400));
|
||||
f.take();
|
||||
c.on_event(&partial(A, "자비스 내일은"), ms(t, 500));
|
||||
c.on_event(&partial(C, "자비스 내일은"), ms(t, 500));
|
||||
assert!(f.take().is_empty());
|
||||
}
|
||||
|
||||
@@ -730,12 +879,12 @@ mod tests {
|
||||
assert!(f.take().is_empty());
|
||||
c.tick(ms(t, 900) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
// Another speaker's words do not join it.
|
||||
// Another speaker's words do not join it (A's next call comes after the going-on window).
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
c.on_event(&start(A, 2), ms(t, 4100));
|
||||
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500));
|
||||
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600));
|
||||
c.tick(ms(t, 4500) + wait);
|
||||
c.on_event(&start(A, 2), ms(t, 6000));
|
||||
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 6400));
|
||||
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 6500));
|
||||
c.tick(ms(t, 6400) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
}
|
||||
|
||||
@@ -817,7 +966,7 @@ mod tests {
|
||||
// Not yet in the channel list (it can lag): the speaker is there all the same.
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
*f.people.borrow_mut() = vec![];
|
||||
c.on_event(&fin(A, "오늘 부산에 비 온대", 100), ms(t, 2100));
|
||||
c.on_event(&fin(A, "오늘 부산에 비 온대", 100), ms(t, 6200)); // after the going-on window
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "오늘 부산에 비 온대".into())]);
|
||||
}
|
||||
|
||||
@@ -904,7 +1053,7 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed);
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0));
|
||||
c.on_event(&fin(B, "자비스 뉴스", 100), ms(t, 100));
|
||||
c.on_event(&fin(B, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed"
|
||||
c.on_event(&fin(C, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed"
|
||||
let did = f.take();
|
||||
let q = did.iter().find_map(|d| if let Did::Addressed(q, _) = d { Some(q.clone()) } else { None }).unwrap();
|
||||
// The slash command stops all speech: Stopped comes before the speech's end.
|
||||
@@ -1016,9 +1165,318 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 하나", 0), ms(t, 0));
|
||||
c.on_event(&fin(B, "자비스 둘", 0), ms(t, 1000));
|
||||
c.on_event(&fin(A, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘
|
||||
c.on_event(&fin(C, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘
|
||||
f.take();
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 셋".into())]);
|
||||
assert_eq!(f.take(), vec![Did::Answer(C, "자비스 셋".into())]);
|
||||
}
|
||||
|
||||
fn alone(kind: PolicyKind) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let (c, f, t) = conv(kind, CallMode::Addressed);
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(c, f, t)
|
||||
}
|
||||
fn speak_start(id: &str) -> CoreEvent {
|
||||
CoreEvent::SpeakStart(id.into())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_speaker_who_goes_on_before_the_answer_is_heard_gets_one_answer_for_all_of_it() {
|
||||
// Live 2026-11-19: each piece of one thought was answered on its own, one after another.
|
||||
for kind in [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important] {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into())], "{kind:?}");
|
||||
// The next piece starts 0.8 s after the first ended, while Jarvis is still thinking.
|
||||
c.on_event(&fin(A, "디코 화면 공유 창에 가려져 있잖아", 100), ms(t, 1800));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
c.on_event(&fin(A, "앞으로 오게 해 줘", 100), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s2".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아 앞으로 오게 해 줘".into())],
|
||||
"{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "a joined piece is not a missed request ({kind:?})");
|
||||
// The cancelled speech's end does not end the joined answer.
|
||||
c.on_event(&end("s1", true), ms(t, 4100));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_follows_a_wake_word_call_is_joined_even_without_the_wake_word() {
|
||||
let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 디코 화면 공유 보여", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "내 화면 말이야", 100), ms(t, 2000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 디코 화면 공유 보여".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "자비스 디코 화면 공유 보여 내 화면 말이야".into())]);
|
||||
// Someone else's words never join it.
|
||||
c.on_event(&fin(B, "나도 안 보여", 100), ms(t, 2500));
|
||||
assert!(f.take().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_the_pause_is_longer_than_the_window_the_next_words_are_a_new_request() {
|
||||
// The answer ended; the next utterance starts 5.3 s after the request ended (window 5 s).
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
c.on_event(&start(A, 2), ms(t, 5200));
|
||||
c.on_event(&fin_utt(A, 2, "고마워"), ms(t, 6300)); // started at 5200
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Answer(A, "고마워".into())]);
|
||||
// Still thinking, but the speaker paused longer than call.continueMs.
|
||||
let (mut c, f, t) = alone(PolicyKind::InOrder);
|
||||
let pause = test_calls().continue_window.as_millis() as u64;
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
// The next utterance (1 s long) starts `pause` + 100 ms after the first ended.
|
||||
c.on_event(&fin(A, "메모장도", 100), ms(t, pause + 1100));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())], "queued as its own request");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_said_while_the_answer_is_prepared_still_stops_it() {
|
||||
let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 날씨 알려 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "자비스 멈춰", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 알려 줘".into()), Did::StopAll, Did::Cancel("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_answer_played_is_still_joined() {
|
||||
// The next utterance starts while Jarvis thinks; the answer starts playing before its final.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_final_leaves_the_request_as_it_is() {
|
||||
// Noise the worker rejected comes as an empty final: nothing to join, nothing asked again.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())]);
|
||||
// An empty final after the answer played ends the hold: what follows is a new request.
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3200));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 4500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
|
||||
}
|
||||
|
||||
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
|
||||
|
||||
/// Alone with Jarvis, answers held until the asker has been quiet for `wait_ms`.
|
||||
fn alone_waiting(kind: PolicyKind, wait_ms: u64) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
let calls = CallSettings { mode: CallMode::Addressed, earcon: false, answer_wait: Duration::from_millis(wait_ms),
|
||||
..test_calls() };
|
||||
let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(Conversation::new(f.clone(), calls, policy), f, Instant::now())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_starts_only_after_its_asker_was_quiet_for_the_wait() {
|
||||
// Live 2026-11-19: pauses inside one thought were up to 4.7 s, longer than writing an answer.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Held("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 2900)), "{kind:?}");
|
||||
c.tick(ms(t, 2800));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
c.tick(ms(t, 2900));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), None, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_while_its_asker_talks_and_their_words_join_it() {
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
assert!(f.take().is_empty(), "still talking ({kind:?})");
|
||||
c.on_event(&fin_utt(A, 2, "메모장도"), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 메모장도".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
c.tick(ms(t, 6400)); // the joined request ended at 3400
|
||||
assert_eq!(f.take(), vec![Did::Release("s2".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_after_a_short_answer_ended_within_the_window_is_joined() {
|
||||
// Request ends at 0, its answer plays from 3 s to 4 s, the speaker goes on at 4.7 s.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 0), ms(t, 0));
|
||||
c.tick(ms(t, 3000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
f.take();
|
||||
c.on_event(&start(A, 2), ms(t, 4700));
|
||||
c.on_event(&fin_utt(A, 2, "제일 앞으로"), ms(t, 5800)); // started at 4700
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()), Did::Held("s2".into())],
|
||||
"{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn after_the_answer_ended_a_late_start_noise_a_stop_or_someone_else_join_nothing() {
|
||||
for kind in ALL {
|
||||
let after = |c: &mut Conversation<Fake>, f: &Fake, t: Instant| {
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 0), ms(t, 0));
|
||||
c.tick(ms(t, 3000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
f.take();
|
||||
};
|
||||
// Started at 5.3 s: a new request.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
c.on_event(&start(A, 2), ms(t, 5300));
|
||||
c.on_event(&fin_utt(A, 2, "메모장 켜 줘"), ms(t, 6400));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "메모장 켜 줘".into()), Did::Held("s2".into())], "{kind:?}");
|
||||
// Someone else (B, alone with A in the channel) never joins it.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
*f.people.borrow_mut() = vec![A, B]; // B joined the channel
|
||||
c.on_event(&fin(B, "자비스 나도", 100), ms(t, 4800));
|
||||
assert_eq!(f.take(), vec![Did::Answer(B, "자비스 나도".into()), Did::Held("s2".into())], "{kind:?}");
|
||||
// Noise: nothing joined, nothing asked again.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
c.on_event(&start(A, 2), ms(t, 4700));
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 5800));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
// A stop stops, and asks nothing again.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
c.on_event(&start(A, 2), ms(t, 4700));
|
||||
c.on_event(&fin_utt(A, 2, "자비스 멈춰"), ms(t, 5800));
|
||||
assert_eq!(f.take(), vec![Did::StopAll], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_is_released_when_what_followed_was_noise() {
|
||||
let (mut c, f, t) = alone_waiting(PolicyKind::Latest, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
f.take();
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_while_the_answer_plays_stops_it_and_answers_all_of_it() {
|
||||
// A pause longer than the wait: the answer started, then the speaker went on within
|
||||
// call.continueMs of their request's end.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.tick(ms(t, 2900));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
f.take();
|
||||
c.on_event(&fin(A, "제일 앞으로", 100), ms(t, 5000)); // started at 3900, 4 s after the end
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
|
||||
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_first_final_is_joined() {
|
||||
// A very short pause: the next utterance starts before the first one's final arrives.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_request_whose_next_final_is_empty_or_a_stop_is_not_asked_again() {
|
||||
for kind in ALL {
|
||||
for last in ["", "자비스 멈춰"] {
|
||||
// (a) the answer already ended.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 4000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
// What follows is a new request, not joined to the old one.
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 5000));
|
||||
assert_eq!(f.take().into_iter().filter(|d| matches!(d, Did::Answer(..))).collect::<Vec<_>>(),
|
||||
vec![Did::Answer(A, "고마워".into())], "{kind:?} {last:?}");
|
||||
// (b) the next utterance started before the first final, the answer is playing.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 3000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
if last.is_empty() {
|
||||
assert!(!did.contains(&Did::Cancel("s1".into())), "noise does not stop the answer ({kind:?})");
|
||||
} else {
|
||||
assert!(did.contains(&Did::StopAll), "{kind:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&partial(A, "제일 위로"), ms(t, 1200));
|
||||
assert_eq!(f.take(), vec![Did::Prepare(A, "계산기 켜 줘 제일 위로".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_joined_request_gets_no_second_earcon() {
|
||||
let (mut c, f, t) = conv_earcon(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "부산", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스 날씨".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "자비스 날씨 부산".into())]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, also when its answer already played or ended. 0 turns joining off. 0-15000. |
|
||||
| `call.answerWaitMs` | `3000` | Waiting before answering (below): a voice answer starts playing only once its asker has been quiet this long after their request ended. 0: as soon as it is written. 0-10000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
@@ -78,6 +80,44 @@ connection (`WorkerLost`): the final it waited for will not come, and the restar
|
||||
utterances are not continuations of it. Chat messages are not joined:
|
||||
a message is complete as sent.
|
||||
|
||||
**Going on.** People say one thing in pieces, with pauses of a fraction of a second to several
|
||||
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
|
||||
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
|
||||
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
|
||||
So a speaker's voice request stays open for `call.continueMs` after it ended, whatever its answer
|
||||
does meanwhile (live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer: it started,
|
||||
and a short one even ended, in the pause), and their next final is joined to it when that utterance
|
||||
started within `call.continueMs` of the request's end (start = the final's arrival minus its
|
||||
`wait_ms` and `audio_ms`):
|
||||
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
|
||||
speech `cancel`led, which also stops the brain, `voice_worker.spec.md`), a waiting one leaves the
|
||||
queue, a pending "addressed" question is void. It is not a missed request.
|
||||
- The joined text (the earlier text, a space, the new final) is judged and answered as one request,
|
||||
as if it had been said in one go; it ends when the new final ended. It may be joined again.
|
||||
- The new final is joined whatever it is on its own (a piece needs no wake word of its own), except
|
||||
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
|
||||
or asked again, the request goes on as it was).
|
||||
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
|
||||
end, or had already started when the request's final came (a pause shorter than the
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. The window passing (once the answer ended; a held
|
||||
request waits for its final), a missed request, a stop, leaving and a lost worker end it; then
|
||||
what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
|
||||
its answer is being written (it is the request that will be answered).
|
||||
|
||||
**Waiting before answering.** So that a pause inside one thought does not get an answer, a voice
|
||||
answer is written at once but starts playing only when its asker has been quiet for
|
||||
`call.answerWaitMs` after the request ended (default 3 s: 8 of the 10 pauses inside one thought
|
||||
measured live 2026-11-19 were shorter). While the asker is talking when that time comes, the answer
|
||||
waits for that utterance's final: it joins the request (Going on, the held answer is cancelled and
|
||||
the joined one waits again from its own end), or, when it is empty or not joined, the answer plays.
|
||||
A stop stops it as always. The earcon is not held: it still tells the asker at once that Jarvis
|
||||
heard them. Answers queued behind a held one wait their turn.
|
||||
|
||||
**Earcon.** A short two-tone "띠링" (about 0.3 s, made by the app, `app/src/earcon.rs`) tells the
|
||||
speaker Jarvis heard the call, long before the answer is written: it is the first sound after a call.
|
||||
It plays right before each voice answer starts (the policy's `Answer`), and at once when a wake word
|
||||
@@ -116,6 +156,9 @@ Rules shared by every policy:
|
||||
than the request's `ended_at` (in `important`: any ending utterance, since it competes on score), the
|
||||
policy waits for that final, at most `settleMs` from the first waiting request's arrival. A final that
|
||||
is not a call ends the wait for it. `latest` does not settle: an earlier utterance would lose anyway.
|
||||
- **Taking back.** A request whose speaker went on (Going on) is withdrawn: stopped when it is the
|
||||
running answer, removed when it waits, dropped from a pending ranking. Not missed; the joined
|
||||
request that follows goes through the policy as any new request.
|
||||
- **Stop.** A stop decision stops the current answer and drops every waiting request, in every policy.
|
||||
The stopped answer and the dropped requests are missed (`stopped`).
|
||||
- An answer that is already done is never started again.
|
||||
@@ -181,7 +224,8 @@ event is passed on unchanged). It talks to the core through a small `Voice` inte
|
||||
one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glue is tested with a fake.
|
||||
|
||||
- `Chat` and `ReplyText`: chat calls, above.
|
||||
- `Partial` (getting ready): a partial transcript that call detection already reads as a request
|
||||
- `Partial` (getting ready): a partial of a speaker whose request is open (Going on) is sent as the
|
||||
joined text. Otherwise a partial transcript that call detection already reads as a request
|
||||
(joined to the speaker's waiting wake word alone, as its final would be; not the wake word alone,
|
||||
not a stop) is sent to the brain as `prepare(user, text)`, so the brain judges what the call needs
|
||||
while the speaker is still talking (`voice_worker.spec.md`, `prepare`). Not while an answer runs:
|
||||
@@ -194,12 +238,17 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
judgement) closes the utterance, and no answer within 5 s counts as no, also when the answer arrives
|
||||
after that but before the timer ran.
|
||||
- Policy actions: `Answer` → the earcon (unless it already played for this request), then
|
||||
`answer(user, text)` (the brain answers and the answer is spoken; when it
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`answer(user, text, hold)` (the brain answers and the answer is spoken; held, with a non-zero
|
||||
`call.answerWaitMs`, until `release` when its asker has been quiet for it; when it cannot be
|
||||
queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `SpeakStart` of an answer's speech: the answer is heard (no second earcon when it is joined); its
|
||||
request stays open (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done; its request stays open while
|
||||
`call.continueMs` from the request's end has not passed (or while held), and a `SpeechStart` of
|
||||
its speaker after that window drops it. The end of a speech the loop already cancelled starts
|
||||
nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
`Left`: the same stop. The core sends `Stopped` before the `SpeakEnd`s of the speech it stopped, so a
|
||||
cancelled answer's end never starts the next waiting request.
|
||||
@@ -218,7 +267,14 @@ fake voice side (answering and queueing, a busy drop recorded without text, voic
|
||||
with Jarvis answered without asking, the room counts sent with the question, addressed yes,
|
||||
no and timeout, the earcon before each answer, once for a wake word alone, none while busy or when
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the pause is longer than `call.continueMs`, joined while the answer plays and after
|
||||
a short answer ended within that window (not another speaker, noise or a stop), answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
|
||||
and joined by what they say, released after noise, in every policy, a stop still
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
|
||||
@@ -563,15 +563,26 @@ impl DiscordCore {
|
||||
/// `SpeakStart` and `SpeakEnd` follow as for `say`), or None when not in a voice channel. It ends as
|
||||
/// failed when the brain gives no answer. `heard`: when the request's speech ended and its final
|
||||
/// arrived, passed on in `AnswerAsked`.
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
self.voice_channel_id()?;
|
||||
let id = with_playback(&self.inner, |p| p.adopt());
|
||||
let id = with_playback(&self.inner, |p| {
|
||||
let id = p.adopt();
|
||||
if hold {
|
||||
p.hold(&id);
|
||||
}
|
||||
id
|
||||
});
|
||||
let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user, ended_at: heard.0,
|
||||
final_at: heard.1 });
|
||||
self.inner.worker.reply_in(&id, user, text, "voice", true, None, ¬es_of(&self.inner, user));
|
||||
Some(id)
|
||||
}
|
||||
|
||||
/// Let an answer held by `answer(.., hold)` play.
|
||||
pub fn release(&self, id: &str) {
|
||||
with_playback(&self.inner, |p| p.release(id));
|
||||
}
|
||||
|
||||
/// Play the earcon in the voice channel, queued like speech. The request id, or None when not in a
|
||||
/// voice channel.
|
||||
pub fn earcon(&self) -> Option<String> {
|
||||
|
||||
@@ -171,7 +171,9 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
long the clip playing when it was stopped had played, timed from when it was handed to the player)
|
||||
so its voice recordings keep only what was heard (`voice_worker.spec.md`, "Recordings").
|
||||
- **Answering** (M4): `answer(user, text)` queues a request like `say` whose speech the worker makes
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. No answer
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. With
|
||||
`hold` it is written but does not start playing until `release(id)` (an answer that fails while held
|
||||
still ends at once and lets the queue go on). No answer
|
||||
from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed.
|
||||
`cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of
|
||||
an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and
|
||||
|
||||
@@ -48,6 +48,8 @@ struct Request {
|
||||
failed: bool,
|
||||
played: u32,
|
||||
started: bool,
|
||||
/// Not to start playing until released (`hold`).
|
||||
held: bool,
|
||||
}
|
||||
|
||||
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
||||
@@ -81,7 +83,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
||||
self.backend.tts(&id, text, more);
|
||||
id
|
||||
@@ -93,7 +95,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
||||
id
|
||||
}
|
||||
@@ -103,7 +105,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
||||
failed: false, played: 0, started: false });
|
||||
failed: false, played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
||||
self.pump();
|
||||
id
|
||||
@@ -118,6 +120,22 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep a request that has not started from starting until `release` (the asker may go on).
|
||||
pub fn hold(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.started) {
|
||||
r.held = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Let a held request play.
|
||||
pub fn release(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && r.held) {
|
||||
r.held = false;
|
||||
crate::debug_log!("play", "release id={id}");
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
||||
/// queue goes on.
|
||||
pub fn cancel(&mut self, id: &str) {
|
||||
@@ -207,6 +225,9 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
head.next = first;
|
||||
}
|
||||
}
|
||||
if head.held && head.chunks.contains_key(&head.next) {
|
||||
return;
|
||||
}
|
||||
if let Some(pcm) = head.chunks.remove(&head.next) {
|
||||
head.next += 1;
|
||||
head.played += 1;
|
||||
@@ -480,4 +501,28 @@ mod tests {
|
||||
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
||||
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_for_its_release_and_a_failed_one_does_not_block() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.adopt();
|
||||
p.hold(&id);
|
||||
p.audio(&id, 0, b("a0"));
|
||||
p.done(&done(&id, false));
|
||||
assert!(f.0.borrow().played.is_empty(), "written, but held");
|
||||
assert!(p.take_events().is_empty());
|
||||
p.release(&id);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
||||
// An answer that failed while held ends at once, and the next request plays.
|
||||
p.clip_ended();
|
||||
p.take_events();
|
||||
let bad = p.adopt();
|
||||
p.hold(&bad);
|
||||
let next = p.say("b", false);
|
||||
p.audio(&next, 0, b("b0"));
|
||||
p.fail(&bad);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
||||
assert_eq!(p.take_events()[0], end(&bad, 0, false, true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +254,21 @@ impl Policy {
|
||||
}
|
||||
}
|
||||
|
||||
/// The request is taken back before its answer was heard: its speaker went on talking and it is
|
||||
/// joined to what they said next (conversation.spec.md, "Going on"). Not a missed request. When
|
||||
/// it is the running answer it is stopped; nothing else starts until the joined request comes.
|
||||
pub fn withdraw(&mut self, id: u64) -> Vec<Action> {
|
||||
if self.current.as_ref().is_some_and(|r| r.id == id) {
|
||||
self.current = None;
|
||||
return vec![Action::StopAnswer(id)];
|
||||
}
|
||||
self.waiting.retain(|r| r.id != id);
|
||||
if self.ranking.as_ref().is_some_and(|r| r.ids.contains(&id)) {
|
||||
self.ranking = None;
|
||||
}
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// A stop command: stop the answer and drop everything waiting.
|
||||
pub fn stop(&mut self) -> Vec<Action> {
|
||||
let mut out = vec![];
|
||||
|
||||
@@ -453,6 +453,8 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
|
||||
designated_no_wake_word: get_bool(call, "designatedNoWakeWord", "call", d.designated_no_wake_word)?,
|
||||
stop_words,
|
||||
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
|
||||
continue_window: get_ms(call, "continueMs", "call", d.continue_window, 15000)?,
|
||||
answer_wait: get_ms(call, "answerWaitMs", "call", d.answer_wait, 10000)?,
|
||||
earcon: get_bool(call, "earcon", "call", d.earcon)?,
|
||||
},
|
||||
chat_wake_words,
|
||||
|
||||
@@ -95,6 +95,8 @@ pub fn fields() -> Vec<Field> {
|
||||
f("call.stopWords", Call, TextList { max_items: 20, nullable: false },
|
||||
json!(["멈춰", "그만", "정지", "stop"]), "정지 명령"),
|
||||
f("call.followUpMs", Call, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.continueMs", Call, Int { min: 0, max: 15000 }, json!(5000), "끊어 말한 문장 합치기(ms)"),
|
||||
f("call.answerWaitMs", Call, Int { min: 0, max: 10000 }, json!(3000), "말이 끝나고 답하기 전 기다림(ms)"),
|
||||
f("call.earcon", Call, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Response, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
json!("latest"), "동시 대화 응답 정책"),
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! cargo test --release --test live_round_trip -- --nocapture
|
||||
//!
|
||||
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
||||
//! JARVIS_LIVE_WORKER_ARGS adds worker arguments (e.g. "--stt-device cpu --stt-compute int8").
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -165,16 +166,23 @@ fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: A
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
/// Both bots in the voice channel: the speaker bot (the "person") and Jarvis running the app's services
|
||||
/// from a fresh data folder, with `call` as the settings' call section.
|
||||
struct Live {
|
||||
speaker_id: u64,
|
||||
speaker_songbird: Arc<Songbird>,
|
||||
s_shards: Arc<serenity::gateway::ShardManager>,
|
||||
services: jarvis_app::services::Services,
|
||||
core: jarvis_app::discord::DiscordCore,
|
||||
events: Arc<Mutex<Vec<(Instant, CoreEvent)>>>,
|
||||
call: Arc<tokio::sync::Mutex<songbird::Call>>,
|
||||
ears: Ears,
|
||||
guild: u64,
|
||||
}
|
||||
|
||||
async fn start_live(call_settings: impl FnOnce(u64) -> Value) -> Live {
|
||||
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
||||
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
env("JARVIS_LIVE_LLM_URL"); // the services point the brain at it (llm.url)
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
|
||||
@@ -208,21 +216,26 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
||||
args.extend(["--melo-python".to_string(), m]);
|
||||
}
|
||||
// e.g. "--stt-device cpu --stt-compute int8 --melo-device cpu" on a host without a GPU
|
||||
if let Ok(extra) = std::env::var("JARVIS_LIVE_WORKER_ARGS") {
|
||||
args.extend(extra.split_whitespace().map(String::from));
|
||||
}
|
||||
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
|
||||
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
|
||||
// The "person" here is a bot, so bots must be heard; the recorded questions call "자비스".
|
||||
"voice": {"listenToBots": true, "wakeWords": ["자비스"]},
|
||||
"call": call_settings(speaker_id),
|
||||
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
|
||||
"llm": {"url": env("JARVIS_LIVE_LLM_URL")},
|
||||
})).unwrap()).unwrap();
|
||||
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
|
||||
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
let mut services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
.expect("services start in time").expect("services start");
|
||||
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
|
||||
let core = services.core.clone();
|
||||
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
|
||||
let ev2 = events.clone();
|
||||
let mut rx = services.events;
|
||||
let mut rx = std::mem::replace(&mut services.events, mpsc::unbounded_channel().1);
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
ev2.lock().unwrap().push((Instant::now(), e));
|
||||
@@ -240,6 +253,31 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
|
||||
|
||||
Live { speaker_id, speaker_songbird, s_shards, services, core, events, call, ears, guild }
|
||||
}
|
||||
|
||||
impl Live {
|
||||
async fn stop(self) {
|
||||
let _ = self.speaker_songbird.remove(GuildId::new(self.guild)).await;
|
||||
self.s_shards.shutdown_all().await;
|
||||
self.services.core.leave("test").await;
|
||||
self.services.core.close().await;
|
||||
self.services.worker.close();
|
||||
self.services.worker_process.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
let live = start_live(|_| json!({})).await;
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let Live { speaker_id, ref services, ref core, ref events, ref call, ref ears, .. } = live;
|
||||
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
|
||||
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
|
||||
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
|
||||
@@ -355,12 +393,7 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
||||
})).await;
|
||||
|
||||
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
||||
s_shards.shutdown_all().await;
|
||||
services.core.leave("test").await;
|
||||
services.core.close().await;
|
||||
services.worker.close();
|
||||
services.worker_process.stop().await;
|
||||
live.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
@@ -371,3 +404,76 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Going on (conversation.spec.md), live: the person's own recordings of one thing said in pieces,
|
||||
/// played with the pauses they really made, get one answer. Settings as the person's own app: they
|
||||
/// are the only designated person, `addressed` mode, alone with Jarvis.
|
||||
///
|
||||
/// JARVIS_LIVE_GOING_ON=1 (the env of conversation_round_trip_on_real_discord, without the questions)
|
||||
/// JARVIS_LIVE_PIECES=<dir with pieces.json: [{"name", "file"}], each file one 16 kHz mono wav of
|
||||
/// the pieces and the silence between them> cargo test --release --test live_round_trip going_on -- --nocapture
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn going_on_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_GOING_ON").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_GOING_ON=1");
|
||||
return;
|
||||
}
|
||||
let dir = PathBuf::from(env("JARVIS_LIVE_PIECES"));
|
||||
let seqs: Value = serde_json::from_str(&std::fs::read_to_string(dir.join("pieces.json")).unwrap()).unwrap();
|
||||
let live = start_live(|speaker| json!({"designated": [speaker.to_string()], "designatedOnly": true, "mode": "addressed"})).await;
|
||||
let mut rows = vec![];
|
||||
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
||||
for s in seqs.as_array().unwrap() {
|
||||
let wav = std::fs::read(dir.join(s["file"].as_str().unwrap())).unwrap();
|
||||
until("Jarvis to finish speaking", 60_000, || !live.core.speaking()).await;
|
||||
let before = live.events.lock().unwrap().len();
|
||||
live.call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
tokio::time::sleep(Duration::from_secs_f64(speech_end_seconds(&wav))).await;
|
||||
// Done when nothing is playing and nothing new came for 10 s (at most 60 s).
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
let (mut seen, mut quiet_since) = (0, Instant::now());
|
||||
while Instant::now() < deadline {
|
||||
let n = live.events.lock().unwrap().len();
|
||||
if n != seen || live.core.speaking() {
|
||||
(seen, quiet_since) = (n, Instant::now());
|
||||
} else if quiet_since.elapsed() > Duration::from_secs(10) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
let ev = live.events.lock().unwrap()[before..].to_vec();
|
||||
let heard: Vec<String> = ev.iter().filter_map(|(_, e)| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == live.speaker_id => Some(f.text.clone()),
|
||||
_ => None,
|
||||
}).collect();
|
||||
let answers: Vec<Value> = ev.iter().filter_map(|(_, e)| match e {
|
||||
CoreEvent::ReplyText { id, text, failed } => {
|
||||
let end = ev.iter().find_map(|(_, e)| match e {
|
||||
CoreEvent::SpeakEnd(x) if &x.id == id => Some(x.clone()),
|
||||
_ => None,
|
||||
});
|
||||
Some(json!({"text": text, "failed": failed, "played": end.as_ref().map(|x| x.played),
|
||||
"cancelled": end.as_ref().map(|x| x.cancelled)}))
|
||||
}
|
||||
_ => None,
|
||||
}).collect();
|
||||
let asked = ev.iter().filter(|(_, e)| matches!(e, CoreEvent::AnswerAsked { .. })).count();
|
||||
rows.push(json!({"name": s["name"], "heard": heard, "asked": asked, "answers": answers}));
|
||||
println!("going on {}", rows.last().unwrap());
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
}
|
||||
})).await;
|
||||
live.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// One thing said in pieces: one answer heard to its end, and no other answer heard.
|
||||
for r in &rows {
|
||||
let heard = |a: &&Value| a["played"].as_u64().is_some_and(|p| p > 0);
|
||||
let whole: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard)
|
||||
.filter(|a| a["cancelled"] == json!(false)).collect();
|
||||
let cut: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard)
|
||||
.filter(|a| a["cancelled"] == json!(true)).collect();
|
||||
assert!(whole.len() == 1 && cut.is_empty(), "{}: not one answer: {r}", r["name"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i
|
||||
- **Operator instructions** (two sources, both framed "Additional instructions from the operator:" and appended near the end of the guidance list): the settings-UI `llm_instructions` config field, and every `*.md` file in `AGENTS_DIR` (default `/app/agents`, bind-mounted from `./agents`). The file-based set is read once per turn by `load_agent_instructions()` in [src/jarvis/system_prompt.py](src/jarvis/system_prompt.py) and concatenated in filename order, so dropping/editing a `.md` applies on the next reply with no rebuild/restart; fail-open to `""` when the folder is absent/empty/unreadable.
|
||||
- **Warm profile block** (query-agnostic User + Directives excerpt from the knowledge graph, composed by `build_warm_profile()` / `format_warm_profile_block()` in [src/jarvis/memory/graph_ops.py](src/jarvis/memory/graph_ops.py) at Step 3.5 of `reply()`; no LLM call, pure SQLite read; injected unconditionally so personalisation is the default; result cached in `DialogueMemory._hot_cache` under `DialogueMemory.WARM_PROFILE_CACHE_KEY` for the lifetime of the active conversation. Invalidated on `stop`, on new-conversation entry, AND on User/Directives graph mutations via the listener registered in [src/jarvis/daemon.py](src/jarvis/daemon.py) against `register_graph_mutation_listener` in [src/jarvis/memory/graph.py](src/jarvis/memory/graph.py); World-branch writes are ignored)
|
||||
- Digested memory enrichment (optional, see #4)
|
||||
- **v2 cancel** (`turn_state.set_cancel`, set by the voice worker's brain): once the app takes a voice request back (its asker went on talking, a newer request or a stop), the loop stops before its next chat-model call or tool call and the reply returns no answer, so a dropped request never holds up the one that replaces it (`voice_worker.spec.md`, "History and brain").
|
||||
- **v2 history context** (optional, caller-supplied `history=` on `run_reply_engine`): the recent window and search matches from the history store ([src/jarvis/memory/history.py](src/jarvis/memory/history.py), [spec](src/jarvis/memory/history.spec.md)), scoped to the asker and capped by the store's size budget (default about 4,000 tokens, matches at most half). No LLM call; not gated by the planner. Injected in a `<conversation_records>` fence labelled "Information shared in previous conversations"; the lines are data, not instructions. Eval: `evals/test_history_context_recall.py`. With `legacy_memory=False` (the v2 brain in the voice worker) the warm profile and the whole memory enrichment chain (#3 extractor, #3b recall gate, #4 digest and the diary/graph reads) are skipped for the turn, because v1's long-term memory belongs to a single user.
|
||||
- Time + location context (re-injected each turn)
|
||||
- **v2 chat channel tool**: when the voice worker's brain sets a chat backend (`turn_state.set_chat`), `chatChannels` is added to the turn's tools after routing (like `setBroadcast`) and its read/search results enter the loop inside an UNTRUSTED CHAT MESSAGES fence; without a backend it is not in the catalogue at all. Eval: `evals/test_chat_channels_tool.py`.
|
||||
|
||||
File diff suppressed because one or more lines are too long
108
docs/v2/stt-accuracy-plan.md
Normal file
108
docs/v2/stt-accuracy-plan.md
Normal file
@@ -0,0 +1,108 @@
|
||||
# STT 정확도 개선 계획 (2026-11-19)
|
||||
|
||||
사용자 요청(2026-11-19): 받아 적기가 문장을 제대로 맞히는 일이 드물고, 끊어 말한 문장을 하나씩 따로 처리해서 느림.
|
||||
음성 인식을 고칠 때마다 다른 곳이 어긋났으니, 이번에는 원인과 판정 기준을 먼저 정하고 측정은 한 번에 끝낸 뒤 코드는 한 번만 바꿈.
|
||||
모델이 약하면 바꾸되 다른 모델과 같은 자료로 비교함.
|
||||
|
||||
## 실제로 틀린 예 (2026-11-19 사용 중, 사용자가 준 정답)
|
||||
|
||||
| 들은 것 | 실제로 한 말 |
|
||||
|---|---|
|
||||
| 기사야 디코 화면군이 보여 | 비서야 디코 화면공유 보여? |
|
||||
| 내 화면 공이 안 보여? | 내 화면 공유 안보여? |
|
||||
| 실행하면 제일 위로와야지 가면 | 실행하면 제일 위로와야지 화면 |
|
||||
| 지금 화면 봐봐 지금 어 이게 계산지가 제일 위에 있는 거야 | 지금 화면 봐봐 지금 어 이게 계산기가 화면 제일 위에 있는 거야? |
|
||||
| 기구 어디 갔어? | 디코 어디 갔어? |
|
||||
|
||||
11-18 새벽 녹음 11개에서도 문장 첫머리의 호출어가 4개 틀림(아비스, 자비, 다비스, 사비스). 틀린 곳은 대부분 첫소리 자음(ㅂ→ㄱ, ㄷ→ㄱ, ㅎ→ㄱ, ㅈ→ㅇ/ㄷ/ㅅ)과 문장 끝의 짧은 낱말.
|
||||
|
||||
## 지금 구조 (바꿀 수 있는 곳)
|
||||
|
||||
- 앱이 디스코드 소리를 받아 사람마다 워커로 보냄. 100ms 동안 패킷이 없으면 `speaking_end`(디스코드는 말을 멈추면 보내기를 멈춤).
|
||||
- 워커(`segmenter.py`)는 음성 확률이 0.35 아래로 `end_silence_ms`(300ms) 이어지거나 `speaking_end`가 오면 말 하나를 끝냄. 그 조각만 따로 받아 적음.
|
||||
- 받아 적기: faster-whisper large-v3-turbo, int8_float16, beam 2, 언어 ko, 호출어 힌트(힌트 없는 결과로 검사).
|
||||
- 앱(`conversation.rs`)은 조각마다 호출 판정. 호출어만 말한 조각만 다음 조각과 합침. 그 밖의 조각은 각각 따로 요청이 되거나 버려짐.
|
||||
|
||||
## 원인 가설과 확인 방법
|
||||
|
||||
| | 가설 | 지금까지의 근거 | 확인 방법 |
|
||||
|---|---|---|---|
|
||||
| H1 | 너무 짧게 자름. 0.3초만 쉬어도, 디스코드가 0.1초 보내기를 멈춰도 말이 끝나서, 짧은 조각을 문맥 없이 받아 적음. 사용자 불만 2(한 문장씩 처리)의 원인이기도 함 | 예 3·5처럼 짧은 조각이 틀림. 조각마다 따로 답함 | 오늘 워커 로그의 끝난 이유(`reason`) 비율과 같은 사람 조각 사이 간격 분포. 같은 소리를 조각마다 받아 적기 대 이어 붙여 한 번에 받아 적기 |
|
||||
| H2 | 말 첫머리가 잘려 들어옴(디스코드 쪽 음성 감지가 늦게 보내기 시작) | 11-18 클립 1개(012257, "아비스")는 클립 시작 20ms에 이미 말소리. 나머지 틀린 호출어 3개는 70~180ms 여유가 있어 잘림이 아님. 일부만 설명함 | 모든 클립의 첫 소리 앞 여유와 첫 낱말 오류의 관계 |
|
||||
| H3 | 모델 한계. turbo는 디코더가 4층(large-v3는 32층)이고 int8, beam 2라 짧은 구어(디코, 화면공유)에 약함 | 측정 없음 | 같은 소리로 모델 비교(아래) |
|
||||
| H4 | 앞 대화 문맥이 없음 | 측정 없음 | 직전 받아 적은 문장을 프롬프트로 준 변형. 없는 말을 지어내는지 잡음 108개로 함께 확인 |
|
||||
|
||||
이미 확인해서 빼는 것:
|
||||
- 음량: 11-16 측정, 키워도 나아지지 않음(`groups/javis_bot/logs/stt-level-2026-11-16`).
|
||||
- 고음 강조: 11-18 측정, 오히려 나빠져서 0.1.36에서 되돌림.
|
||||
- 소리 손상: 11-18 클립 11개의 4~7kHz 성분이 Jarvis TTS와 같은 수준(300~1000Hz 대비 -27~-32dB)이고 7.2kHz 위는 워커 필터가 자른 것. 대역이 좁아진 소리는 아님. 오늘 클립에서 20ms 구멍(패킷 손실) 수도 함께 셈.
|
||||
|
||||
## 시험 자료 (한 번 만들고 모든 비교에 같이 씀)
|
||||
|
||||
- A. 사용자 실제 녹음: 게임PC `C:\JarvisM0\Jarvis\Jarvis\recordings`의 11-18, 11-19 `said` 클립과 서버에 있는 11-18 새벽 11개. 정답은 사용자가 준 5개 말고는 후보 모델 결과를 owner와 reviewer가 따로 맞춰 보고, 둘이 다르거나 모델끼리 갈리는 것만 사용자에게 짧은 목록으로 확인.
|
||||
- A-turn. A에서 같은 사람이 이어 말한 조각을 실제 시간 간격대로 이어 붙인 것(H1용).
|
||||
- B. 악조건: A를 오프라인으로 바꾼 것. 소리 작게(-15dB), 생활 잡음(ESC-50) SNR 10·5dB, 싼 헤드셋 대역(300~3400Hz), 낮은 비트레이트 Opus(16kbps) 재인코딩, 방 울림. 다른 사람들은 마이크가 나쁘고 작게, 흐리게 말한다는 조건.
|
||||
- C. 다른 목소리: 공개 한국어 음성(FLEURS ko, Common Voice ko, Zeroth, 가능하면 전화 음성) 200개 정도와 그 B 처리본.
|
||||
- 잡음 108개(M1에서 쓴 ESC-50): 없는 말이나 호출어를 지어내는지.
|
||||
|
||||
## 후보 모델
|
||||
|
||||
| 후보 | 엔진 | 확인할 것 |
|
||||
|---|---|---|
|
||||
| 지금: large-v3-turbo int8_float16 beam 2 | faster-whisper | 기준 |
|
||||
| large-v3 (fp16, int8) | faster-whisper | 디코더 32층. VRAM과 속도 |
|
||||
| 한국어 미세조정 Whisper: BuzzASR/korean(large-v3 기반), batisay-ko-turbo, ghost613/whisper-large-v3-turbo-korean | faster-whisper로 변환 | 엔진을 그대로 쓰므로 바꾸기 쉬움. 라이선스, 영어 낱말 섞인 말, 지어내기 |
|
||||
| Qwen3-ASR-1.7B와 한국어 미세조정판(TeamUNIVA/qwen3_asr_1.7b_ko_beta) | transformers | 엔진이 달라 Windows 설치, 10명 묶음 처리, VRAM을 따로 확인 |
|
||||
|
||||
한국어 전용 모델은 다른 언어를 못 받아 적을 수 있음. 바꾸게 되면 언어 설정이 그 모델의 언어일 때만 쓰고, 다른 언어는 지금 모델을 씀(코드에 언어별 규칙은 넣지 않음).
|
||||
|
||||
## 판정 기준 (측정 전에 정함)
|
||||
|
||||
1. A와 B의 문자 오류율(띄어쓰기·문장부호 빼고, 숫자는 읽는 대로), 호출어 놓침, 잡음에서 지어낸 호출어.
|
||||
2. 조건: RTX 5070 Ti에서 말 한 마디의 최종 전사 p95 0.4초 이하(지금 0.11~0.19초, pc-features.md). 10명 동시 전사 VRAM이 지금 STT 몫(최대 약 2.7GB) 안. 넘으면 LLM 배치 식을 다시 재야 하므로 사용자에게 물음.
|
||||
3. 기준보다 A+B 오류율이 상대 20% 넘게 낮을 때만 모델을 바꿈. 자료가 적어 그보다 작은 차이는 우연과 구분할 수 없음. C는 동률일 때만 봄.
|
||||
4. 말 차례 합치기(아래)는 A-turn에서 한 번에 받아 적은 쪽 오류율이 낮고, 늘어나는 대기가 사용자가 정한 범위 안일 때 넣음.
|
||||
|
||||
## 끊어 말한 문장 합치기 (불만 2, 스펙 변경이라 사용자 확인 필요)
|
||||
|
||||
제안: 같은 사람이 짧게 쉬었다 다시 말하면 같은 "말 차례"로 봄. 차례가 끝날 때(쉰 시간이 `end_silence_ms`를 넘을 때) 그 소리 전체를 한 번에 받아 적어 요청 하나로 처리. 디스코드가 보내기를 멈춘 것(`speaking_end`)은 바로 끝내지 않고 쉰 시간으로 셈.
|
||||
- 좋아지는 것: 여러 문장을 한 질문으로 받아 답 한 번. 받아 적기가 문맥을 가짐(H1).
|
||||
- 대가: 한 문장만 말해도 끝을 알기까지 그만큼 더 기다림. 말하는 동안의 부분 전사와 답 준비(`prepare`)는 지금처럼 돌아서 일부는 겹쳐짐.
|
||||
- 기다릴 시간은 A의 실제 간격 분포를 보고 정함(문장 사이 쉼의 대부분을 덮는 값). 1초를 넘으면 사용자에게 다시 물음.
|
||||
- 바뀌는 스펙: `voice_worker.spec.md`(말 끝내기), `conversation.spec.md`(호출어만 말한 경우 합치기는 대부분 필요 없어짐), `discord_core.spec.md`(`speaking_end`의 뜻).
|
||||
|
||||
## 순서
|
||||
|
||||
1. 자료 모으기: 게임PC에서 녹음과 워커 로그 복사(읽기만, 사용자가 정한 시간에).
|
||||
2. 진단: H1~H4 측정, 결과를 이 문서에 적고 reviewer가 교차 검토.
|
||||
3. 모델 비교: 게임PC GPU에서 한 번(Jarvis 서비스를 끄고), 고른 모델은 Strata와 함께 켠 상태로 VRAM·지연 한 번.
|
||||
4. 결정한 것만 한 번에 구현(모델, 말 차례), 테스트, 스펙 수정.
|
||||
5. 같은 시험 자료로 최종 확인 후 릴리스 한 번, 사용자 실사용 확인 한 번.
|
||||
|
||||
## 진행 기록
|
||||
|
||||
- 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능.
|
||||
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
|
||||
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
|
||||
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
|
||||
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR-1.7B(Apache-2.0)는 서버 CPU에서 정답 16개만: 9.4%(한국어판 TeamUNIVA beta 11.7%), 같은 곳을 틀림("기사야", "공리", "가면", "기고"). 앞 대화 문맥을 준 turbo(7.5%)보다 못하고 bf16로 약 5GB가 필요해 Strata와 함께 남는 VRAM(약 2.5GB)에 들어가지 않아 바꾸지 않음.
|
||||
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
|
||||
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.
|
||||
- 2026-11-19 밤 (검토 뒤): 0.1.49에서 고침
|
||||
- 이어 말하기: 다음 말이 시작된 뒤 답이 나오기 시작해도 그 말의 최종 결과까지 앞 요청을 붙잡아 둠. 빈 결과(잡음)는 합치지 않고 앞 요청을 다시 묻지 않음.
|
||||
- 취소: 답 시작, 도구 고르기, 계획, 계획한 도구 실행 직전에도 확인.
|
||||
- 문맥을 쓸 때도 여러 사람의 최종 전사를 한 번에 묶음(문맥을 묶음 안에서 가장 짧은 길이로 끝에서 맞춤). 10명 동시: 묶음 1.3초(문맥 없이 1.6~2.0초, Strata가 돌던 게임PC), VRAM 최대 15.7GB/16.3GB(시작 14.6GB).
|
||||
- B. 악조건(사용자 녹음 100개, 정답 16개, 문맥 없음 → 있음 문자 오류율): 원본 11.9→7.0%, 작은 목소리+마이크 잡음 52.8→51.4%, 잡음 SNR 10dB 18.8→18.9%, 5dB 34.2→33.9%, 싼 헤드셋 14.9→13.8%, Opus 12kbps 12.8→9.8%, 방 울림 13.0→8.8%. 문맥은 어느 조건에서도 나쁘게 하지 않음. 하지만 악조건 자체가 나쁨: 빈 결과가 17개에서 20~49개로 늘어남.
|
||||
- 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음.
|
||||
- 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증.
|
||||
- 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON).
|
||||
- 2026-11-19 밤 (0.1.50): 악조건 두 갈래를 같은 자료(녹음 100개 × 7조건, 정답 16개)로 비교. 근거: evidence/2026-11-19/stt-plan/harsh2/.
|
||||
- 앞단 잡음 제거 3종(DeepFilterNet3, RNNoise, noisereduce)은 전부 나빠서 넣지 않음. 원본 7.0% → 13.8/9.6/11.0%, 싼 헤드셋 13.8% → 21.8/11.4/19.5%, 잡음 10dB 18.9% → 44.9/48.3/31.6%. DeepFilterNet은 작은 목소리에서 없는 말을 길게 지어냈고(108.6%), VAD를 지난 잡음 5개를 모두 글자로 만듦. 디스코드가 이미 보내는 쪽에서 잡음 제거(Krisp)를 하고, Whisper는 잡음 섞인 소리로 학습되어 제거 흔적이 더 해로움.
|
||||
- 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로.
|
||||
- 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초.
|
||||
- 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내.
|
||||
- 2026-11-19 밤 (0.1.50 뒤): 근거 evidence/2026-11-19/stt-plan/wake/.
|
||||
- 첫마디 호출어: 힌트를 준 결과는 "비서야"를 맞게 들었는데(기사야·기서야·라비스·아비스 → 비서야·자비스), 힌트 결과는 힌트 없는 결과와 호출어 수가 같아야만 써서 버려졌음. 규칙을 바꿈: 힌트 없는 결과에 그 호출어의 비슷한 말이 있으면(글 전체가 호출어 글자 수의 3분의 1 이내, 한글은 자모로 셈) 호출어 하나를 더할 수 있음. 녹음 100개에서 5개를 바로잡고 3개는 그대로(그중 "기구"→"김비서"는 실제로 "디코"라 막은 것이 맞음), 잡음 216번(108개 × 힌트 2종)에서 더한 호출어 0. 소리로 다시 매기기(align 확률)는 맞는 쪽과 틀린 쪽을 가르지 못해 쓰지 않음.
|
||||
- 실제 디스코드(테스트 봇이 사용자 녹음을 실제 쉰 간격대로 틀음, app/tests/live_round_trip.rs going_on, 서버 CPU, LLM은 질문을 되돌려 주는 대역): 부산 날씨 3조각은 한 질문, 한 답. 계산기 6조각(쉼 2.3~4.7초)은 쉼이 답 만드는 시간보다 길어서 답이 먼저 들리기 시작했고, 들린 뒤의 말은 새 질문이 되어 중간 답 2개가 끊김. 서버 CPU라 받아 적기가 10~25초 걸려 시간은 실제와 다름(정확한 시간은 게임PC GPU에서 다시 재야 함).
|
||||
- 그래서(사용자 결정 전, 추천안 3으로 구현, 설정으로 끌 수 있음) 답은 바로 만들되 묻는 사람이 요청 끝에서 `call.answerWaitMs`(기본 3초) 동안 조용해야 말하기 시작하고, 그 사이 말하면 그 말의 결과를 기다려 합침. 답이 나오는 중에도 요청 끝에서 `call.continueMs`(5초) 안에 이어 말하면 답을 멈추고 합침(답이 끝난 뒤의 말은 새 질문). 실제 디스코드 다시 시험(같은 말과 같은 쉼을 MeloTTS로, 받아 적기 small CPU, 대역 LLM 0.8초×3): 세 묶음 모두 한 답, 중간에 들린 답 0(live3-answer-wait.log).
|
||||
- 0.1.52: 짧은 답이 쉼 안에서 끝나 버리는 경우(답 3~4초, 이어 말하기 4.7초)도 합침. 요청은 끝나고 `call.continueMs` 동안 답이 끝났어도 열려 있음. 대조: 5.3초에 시작한 말, 다른 사람, 잡음, "멈춰"는 합치지 않음(네 정책 모두 단위 테스트).
|
||||
@@ -68,6 +68,14 @@ TRACE_RESULT_CHARS = 2000
|
||||
TRACE_THINKING_CHARS = 8000
|
||||
|
||||
|
||||
def _cancelled(before: str) -> bool:
|
||||
"""The answer is no longer wanted (turn_state.cancel): stop before the next model or tool call."""
|
||||
if _turn_state.cancelled():
|
||||
debug_log(f"reply cancelled before {before}", "planning")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _clip(text: str, limit: int) -> str:
|
||||
return text if len(text) <= limit else text[:limit] + "…"
|
||||
|
||||
@@ -1253,6 +1261,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
# Step 0.5: Deterministic on-screen site search. If the user named a site and
|
||||
# asked to search/open it while broadcasting, do it directly — the small chat
|
||||
# model otherwise just narrates without calling the browser tool.
|
||||
if _cancelled("it started"):
|
||||
return None
|
||||
_site_search_reply = _maybe_deterministic_site_search(text, db, cfg, language)
|
||||
if _site_search_reply is not None:
|
||||
return _site_search_reply
|
||||
@@ -1390,6 +1400,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
routed_tools = list(_cached_routed)
|
||||
debug_log("tool router served from hot-window cache", "planning")
|
||||
else:
|
||||
if _cancelled("the tool router"):
|
||||
return None
|
||||
routed_tools = _route_tools(cfg, redacted, _builtin_tools, mcp_tools, recent_messages)
|
||||
# Don't cache the router's "fall open to all tools" fallback. That
|
||||
# path fires when the LLM router times out, returns empty, or emits
|
||||
@@ -1532,6 +1544,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
if _one_step_tool:
|
||||
debug_log(f"intent judge: one-step tool {_one_step_tool}, no plan", "planning")
|
||||
elif not intent_judge or _turn_needs_tools:
|
||||
if _cancelled("the planner"):
|
||||
return None
|
||||
action_plan = plan_query(
|
||||
cfg=cfg,
|
||||
query=redacted,
|
||||
@@ -2376,6 +2390,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
_one_step_nudged = False
|
||||
|
||||
while turn < max_turns:
|
||||
if _cancelled("a model call"):
|
||||
return None
|
||||
turn += 1
|
||||
debug_log(f"🔁 messages loop turn {turn}", "planning")
|
||||
print(f" 🔁 Turn {turn}/{max_turns}", flush=True)
|
||||
@@ -2493,6 +2509,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
}
|
||||
],
|
||||
})
|
||||
if _cancelled("a planned tool call"):
|
||||
return None
|
||||
tools_started += 1
|
||||
_plan_result = run_tool_with_retries(
|
||||
db=db,
|
||||
@@ -2840,7 +2858,9 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
messages.append({"role": "tool", "tool_call_id": tool_call_id, "content": f"You have already called {tool_name} {duplicate_tool_count} times. Please use the results from those calls to answer the user's question."})
|
||||
continue
|
||||
|
||||
# Execute tool
|
||||
# Execute tool (not for an answer that is no longer wanted: no actions on its behalf)
|
||||
if _cancelled("a tool call"):
|
||||
return None
|
||||
tools_started += 1
|
||||
result = run_tool_with_retries(
|
||||
db=db,
|
||||
|
||||
@@ -37,6 +37,9 @@ pc : Optional[object]
|
||||
trace : Optional[list]
|
||||
v2: where the engine writes down how it made this answer (history.spec.md, "Traces"): the
|
||||
tools it was given, the plan, the model's thinking and each tool call. None records nothing.
|
||||
cancel : Optional[callable]
|
||||
v2: true once the answer is no longer wanted (the asker went on talking, or a newer request or a
|
||||
stop took its place). The engine checks it before each model call and each tool call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -58,6 +61,17 @@ def reset() -> None:
|
||||
_state.screen = None
|
||||
_state.pc = None
|
||||
_state.trace = None
|
||||
_state.cancel = None
|
||||
|
||||
|
||||
def set_cancel(check) -> None:
|
||||
_state.cancel = check
|
||||
|
||||
|
||||
def cancelled() -> bool:
|
||||
"""Whether this turn's answer is no longer wanted."""
|
||||
check = getattr(_state, "cancel", None)
|
||||
return bool(check and check())
|
||||
|
||||
|
||||
def set_trace(steps: Optional[list]) -> None:
|
||||
|
||||
@@ -10,7 +10,7 @@ import unicodedata
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Protocol
|
||||
from typing import Callable, Optional, Protocol
|
||||
|
||||
from .log import debug_log, notice
|
||||
from jarvis.memory.history import HistoryContext, Scope
|
||||
@@ -19,7 +19,8 @@ from jarvis.memory.history import HistoryContext, Scope
|
||||
class Brain(Protocol):
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None, pc=None, trace: Optional[list] = None) -> Optional[str]:
|
||||
screen=None, pc=None, trace: Optional[list] = None,
|
||||
cancelled: Optional[Callable[[], bool]] = None) -> Optional[str]:
|
||||
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
|
||||
reply for different people may overlap). ``chat``: the chat backend for
|
||||
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
|
||||
@@ -28,7 +29,8 @@ class Brain(Protocol):
|
||||
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
|
||||
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
|
||||
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
|
||||
this answer to (history.spec.md, "Traces")."""
|
||||
this answer to (history.spec.md, "Traces"); ``cancelled``: true once the answer is no longer
|
||||
wanted, so the engine stops at its next model or tool call and None is returned."""
|
||||
|
||||
def addressed(self, user: str, text: str, names: list[str], people: Optional[int] = None,
|
||||
callers: Optional[int] = None) -> Optional[bool]:
|
||||
@@ -310,7 +312,8 @@ class JarvisBrain:
|
||||
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None, pc=None, trace: Optional[list] = None) -> Optional[str]:
|
||||
screen=None, pc=None, trace: Optional[list] = None,
|
||||
cancelled: Optional[Callable[[], bool]] = None) -> Optional[str]:
|
||||
from jarvis.memory.conversation import DialogueMemory
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
@@ -334,6 +337,7 @@ class JarvisBrain:
|
||||
turn_state.set_screen(None if task else screen)
|
||||
turn_state.set_pc(None if task else pc)
|
||||
turn_state.set_trace(trace)
|
||||
turn_state.set_cancel(cancelled)
|
||||
try:
|
||||
return run_reply_engine(self.db, cfg, None, text, memory,
|
||||
language=self.language, history=history, legacy_memory=False,
|
||||
|
||||
@@ -32,7 +32,8 @@ class SttResult:
|
||||
|
||||
|
||||
class Transcriber(Protocol):
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ...
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]: ...
|
||||
|
||||
|
||||
Callback = Callable[[SttResult], None]
|
||||
@@ -44,6 +45,7 @@ class _Job:
|
||||
audio: np.ndarray
|
||||
opts: DecodeOptions
|
||||
callback: Callback
|
||||
context: str = "" # finals only: the recent conversation, as the model's prompt
|
||||
|
||||
|
||||
class SttScheduler:
|
||||
@@ -74,8 +76,9 @@ class SttScheduler:
|
||||
pass
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback))
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback,
|
||||
context: str = "") -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback, context))
|
||||
self._wake.set()
|
||||
|
||||
def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
@@ -121,7 +124,7 @@ class SttScheduler:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips,
|
||||
batch[0].opts)
|
||||
batch[0].opts, [j.context for j in batch])
|
||||
error = None
|
||||
except Exception as e: # the model failing must not stop every speaker's STT
|
||||
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"
|
||||
|
||||
@@ -10,6 +10,7 @@ import itertools
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
@@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService
|
||||
CLOSE_BAD_HELLO = 4400
|
||||
CLOSE_BAD_TOKEN = 4401
|
||||
MAX_SPEAKERS = 32
|
||||
CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches
|
||||
|
||||
|
||||
def is_loopback(host: str) -> bool:
|
||||
@@ -274,6 +276,9 @@ class Session:
|
||||
self.chat_waiting: dict[str, asyncio.Future] = {}
|
||||
self._chat_ids = itertools.count(1)
|
||||
self.names: dict[str, str] = {}
|
||||
# What was said and answered lately in Jarvis's voice channel: (time, text), the STT's
|
||||
# conversation context (voice_worker.spec.md, "Conversation context"). Never logged.
|
||||
self.lines: deque[tuple[float, str]] = deque(maxlen=50)
|
||||
self.speakers: dict[str, _Speaker] = {}
|
||||
self.tts_requests: dict[str, TtsRequest] = {}
|
||||
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
|
||||
@@ -281,7 +286,7 @@ class Session:
|
||||
self.tts_recorded: dict[str, dict] = {}
|
||||
# Spoken replies whose speech was cancelled while the brain was still answering.
|
||||
self.cancelled_replies: set[str] = set()
|
||||
self.thinking_replies: set[str] = set()
|
||||
self.thinking_replies: dict[str, str] = {} # reply id -> asker
|
||||
# user -> the newest call text waiting for that user's running prepare to finish (or None).
|
||||
self.preparing: dict[str, str | None] = {}
|
||||
self.last_prepared: dict[str, str] = {} # user -> the text of their last prepare
|
||||
@@ -364,6 +369,15 @@ class Session:
|
||||
samples = st.converter.push(pcm)
|
||||
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
||||
|
||||
def _context(self) -> str:
|
||||
"""The recent conversation for a final's prompt: the channel's lines of the last
|
||||
CONTEXT_SECONDS, at most `context_chars` characters from the end."""
|
||||
n = self.settings.context_chars
|
||||
if not n or not self.voice_channel:
|
||||
return ""
|
||||
since = time.monotonic() - CONTEXT_SECONDS
|
||||
return " ".join(t for at, t in self.lines if at >= since)[-n:].strip()
|
||||
|
||||
def _opts(self, beam: int) -> DecodeOptions:
|
||||
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
||||
|
||||
@@ -388,7 +402,8 @@ class Session:
|
||||
else:
|
||||
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
||||
self._opts(self.settings.final_beam),
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said),
|
||||
context=self._context())
|
||||
|
||||
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
||||
def cb(res: SttResult) -> None:
|
||||
@@ -402,6 +417,8 @@ class Session:
|
||||
store = self.worker.history
|
||||
if store is not None and channel and res.text.strip():
|
||||
store.add_voice(channel, speaker, name, res.text, ts)
|
||||
if channel and channel == self.voice_channel and res.text.strip():
|
||||
self.lines.append((time.monotonic(), res.text.strip()))
|
||||
if self.worker.recorder is not None:
|
||||
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
|
||||
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
||||
@@ -499,6 +516,8 @@ class Session:
|
||||
rec["text"] += text
|
||||
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
|
||||
self.tts_recorded.pop(next(iter(self.tts_recorded)))
|
||||
if self.voice_channel and text.strip():
|
||||
self.lines.append((time.monotonic(), text.strip()))
|
||||
tts.feed(req, text, more)
|
||||
|
||||
def _on_tts_cancel(self, msg: dict) -> None:
|
||||
@@ -523,6 +542,7 @@ class Session:
|
||||
if self.voice_channel:
|
||||
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
||||
self.voice_channel = None
|
||||
self.lines.clear()
|
||||
|
||||
def _on_voice_join(self, msg: dict) -> None:
|
||||
got = self._strings(msg, "channel")
|
||||
@@ -681,12 +701,15 @@ class Session:
|
||||
asyncio.get_running_loop().create_task(self._task(rid, user, text, notes))
|
||||
return
|
||||
if speak:
|
||||
self.thinking_replies.add(rid)
|
||||
self.thinking_replies[rid] = user
|
||||
if user in self.preparing:
|
||||
self.preparing[user] = None # the call was answered: no newer text to prepare
|
||||
self.last_prepared.pop(user, None)
|
||||
waiting = self.worker.replies_unanswered
|
||||
early = waiting.get(user, 0) > 0 # sent before their previous answer: not a reply to it
|
||||
# Sent before their previous answer: not a reply to it. An answer taken back (cancelled while
|
||||
# the brain thinks) does not count: what comes now replaces it.
|
||||
taken_back = sum(1 for r in self.cancelled_replies if self.thinking_replies.get(r) == user)
|
||||
early = waiting.get(user, 0) - taken_back > 0
|
||||
waiting[user] = waiting.get(user, 0) + 1
|
||||
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes))
|
||||
|
||||
@@ -918,7 +941,7 @@ class Session:
|
||||
await self._reply_inner(rid, user, text, via, speak, early, channel, notes or [], trace=trace)
|
||||
finally:
|
||||
trace.finish("") # only when it did not get to its answer
|
||||
self.thinking_replies.discard(rid)
|
||||
self.thinking_replies.pop(rid, None)
|
||||
self.cancelled_replies.discard(rid)
|
||||
if not self._handed_over.pop(rid, False): # else _delivered counts it
|
||||
self._reached(user)
|
||||
@@ -986,23 +1009,39 @@ class Session:
|
||||
pc = AppBridge(self, asyncio.get_running_loop(), "pc", *origin, turn=turn,
|
||||
state=w.pc_state.setdefault(user, {}))
|
||||
trace.append(self._records_step(history, notes))
|
||||
try:
|
||||
async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
||||
schedule=schedule, notes=notes or [],
|
||||
notes_backend=notes_backend, screen=screen, pc=pc,
|
||||
trace=trace)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
w.scheduler.wake() # held partials go on
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = None
|
||||
answer = None
|
||||
if speak and rid in self.cancelled_replies:
|
||||
# Taken back while it waited for the asker's previous answer: never started.
|
||||
debug_log(f"brain reply {rid} not started: cancelled while waiting", "llm")
|
||||
else:
|
||||
try:
|
||||
async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(
|
||||
w.brain.reply, user, text, history, chat=chat, schedule=schedule,
|
||||
notes=notes or [], notes_backend=notes_backend, screen=screen, pc=pc,
|
||||
trace=trace, cancelled=lambda: rid in self.cancelled_replies)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
w.scheduler.wake() # held partials go on
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = None
|
||||
answer = (answer or "").strip()
|
||||
trace.finish(answer)
|
||||
if not answer and speak and rid in self.cancelled_replies:
|
||||
# Taken back before it was written: as if never asked, so the request that replaces
|
||||
# it is still the very next turn (a confirmation it carries does not lapse).
|
||||
w.reply_turns[user] = turn - (2 if early else 1)
|
||||
debug_log(f"brain reply {rid} taken back", "llm")
|
||||
self.send({"type": "reply_text", "id": rid, "text": "", "failed": True})
|
||||
if rid in self.tts_requests:
|
||||
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False})
|
||||
else:
|
||||
self.send({"type": "tts_done", "id": rid, "chunks": 0, "cancelled": True, "failed": False})
|
||||
return
|
||||
if not answer and rid in self.tts_requests:
|
||||
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech
|
||||
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
||||
|
||||
@@ -13,6 +13,7 @@ class SttSettings:
|
||||
final_beam: int = 2
|
||||
language: str | None = "ko"
|
||||
hotwords: tuple[str, ...] = ()
|
||||
context_chars: int = 200
|
||||
|
||||
def update(self, changes: dict) -> None:
|
||||
"""Apply ``changes``; raises ValueError and changes nothing if any value is bad."""
|
||||
@@ -52,6 +53,8 @@ def _check(key: str, value):
|
||||
if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8):
|
||||
raise ValueError("language must be a language code or null")
|
||||
return value
|
||||
if key == "context_chars":
|
||||
return _int_in(key, value, 0, 500)
|
||||
if key == "hotwords":
|
||||
if (not isinstance(value, (list, tuple)) or len(value) > 50
|
||||
or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)):
|
||||
|
||||
@@ -6,6 +6,7 @@ import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
import zlib
|
||||
|
||||
import numpy as np
|
||||
@@ -20,6 +21,12 @@ SR = 16000
|
||||
# Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
MIN_MEAN_LOGPROB = -0.7
|
||||
MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech
|
||||
# faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1).
|
||||
MAX_PREVIOUS_TOKENS = 223
|
||||
# A plain decode at or above this, though below MIN_MEAN_LOGPROB, may be rescued by the context
|
||||
# (choose_with_context). Measured 2026-11-19 on harsh copies of the user's speech: down to -0.8 no
|
||||
# more noise came out as text (VAD-cut noise 0/5, raw noise 16/108 as without); -0.9 let 3 more through.
|
||||
UNSURE_LOGPROB = -0.8
|
||||
|
||||
|
||||
def _letters(text: str) -> str:
|
||||
@@ -55,24 +62,81 @@ def _mentions(text: str, word: str) -> int:
|
||||
return _letters(text).count(w) if w else 0
|
||||
|
||||
|
||||
def _calls(text: str, hotwords: tuple[str, ...]) -> list[str]:
|
||||
"""The wake words in ``text``, longest first at each place, without overlaps ("김비서야" is one)."""
|
||||
s = _letters(text)
|
||||
words = sorted({_letters(w) for w in hotwords if _letters(w)}, key=len, reverse=True)
|
||||
found, i = [], 0
|
||||
while i < len(s):
|
||||
w = next((w for w in words if s.startswith(w, i)), None)
|
||||
if w:
|
||||
found.append(w)
|
||||
i += len(w)
|
||||
else:
|
||||
i += 1
|
||||
return found
|
||||
|
||||
|
||||
def keeps_wake_words(text: str, base: str, hotwords: tuple[str, ...]) -> bool:
|
||||
"""Whether ``text`` (a hinted or prompted decode) holds the wake words of ``base`` (the decode it is
|
||||
checked against): every hotword exactly as often, or one wake word more where ``base`` has a near
|
||||
miss of it. Near: the whole text is at most a third of that wake word's letters from ``base``,
|
||||
compared as Unicode NFKD letters (Hangul by its letters: "기사야" is two of six off "비서야")."""
|
||||
if all(_mentions(text, w) == _mentions(base, w) for w in hotwords):
|
||||
return True
|
||||
have, had = _calls(text, hotwords), _calls(base, hotwords)
|
||||
if len(have) != len(had) + 1 or any(_mentions(text, w) < _mentions(base, w) for w in hotwords):
|
||||
return False
|
||||
nfkd = lambda t: unicodedata.normalize("NFKD", _letters(t))
|
||||
return _edit_distance(nfkd(text), nfkd(base)) <= min(len(nfkd(w)) for w in have) // 3
|
||||
|
||||
|
||||
def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp: float,
|
||||
hotwords: tuple[str, ...]) -> str:
|
||||
"""Pick between the decode that had the hotwords as a hint and the plain decode of the same audio.
|
||||
|
||||
The hint can make Whisper write a wake word that nobody said (on laughter, tacked onto a real
|
||||
sentence, or in place of a similar word), so the plain decode decides: no plain text means nothing
|
||||
was said, and the hint's text is used only when it stays close to the plain one and contains every
|
||||
hotword exactly as often. It may change other words, never how many wake words were heard.
|
||||
sentence, or in place of another word), so the plain decode decides: no plain text means nothing
|
||||
was said, and the hint's text is used only when it stays close to the plain one and keeps its wake
|
||||
words (``keeps_wake_words``): it may change other words, and turn a near miss of a wake word into it.
|
||||
"""
|
||||
if not keep_transcript(plain_text, plain_lp):
|
||||
return ""
|
||||
close = _edit_distance(_letters(hint_text), _letters(plain_text)) <= len(_letters(plain_text)) // 2
|
||||
if close and keep_transcript(hint_text, hint_lp) and all(
|
||||
_mentions(hint_text, w) == _mentions(plain_text, w) for w in hotwords):
|
||||
if close and keep_transcript(hint_text, hint_lp) and keeps_wake_words(hint_text, plain_text, hotwords):
|
||||
if len(_calls(hint_text, hotwords)) != len(_calls(plain_text, hotwords)):
|
||||
debug_log("hint turned a near miss into a wake word", "stt")
|
||||
return hint_text
|
||||
return plain_text
|
||||
|
||||
|
||||
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...],
|
||||
plain: tuple[str, float] | None = None) -> str:
|
||||
"""Pick between the decode without the conversation context (``heard``, already through the hint
|
||||
choice) and the one that had it as Whisper's previous-text prompt (``prompted``, through the same
|
||||
choice, so it passed the gate itself).
|
||||
|
||||
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
|
||||
words of the prompt, so the decode without it decides, as for the hint: the prompted text is used
|
||||
only when it stays close to what was heard and holds every hotword exactly as often. When nothing
|
||||
was heard, ``plain`` (the plain decode's text and mean log-probability) may still be just short of
|
||||
the gate (at least UNSURE_LOGPROB): then the prompted text is used if it is that close to it.
|
||||
"""
|
||||
if not heard:
|
||||
if not prompted or plain is None or plain[1] < UNSURE_LOGPROB \
|
||||
or compression_ratio(plain[0]) > MAX_COMPRESSION_RATIO:
|
||||
return ""
|
||||
return prompted if choose_with_context(plain[0], prompted, hotwords) == prompted else ""
|
||||
if not prompted:
|
||||
return heard
|
||||
h, p = _letters(heard), _letters(prompted)
|
||||
if _edit_distance(p, h) > len(h) // 2:
|
||||
return heard
|
||||
if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords):
|
||||
return heard
|
||||
return prompted
|
||||
|
||||
|
||||
def add_cuda_dll_dirs() -> None:
|
||||
"""On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin."""
|
||||
if not hasattr(os, "add_dll_directory"):
|
||||
@@ -104,8 +168,34 @@ class WhisperTranscriber:
|
||||
def warm_up(self) -> None:
|
||||
self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=()))
|
||||
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session."""
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
|
||||
conversation to prompt with ("" for none); see ``choose_with_context``."""
|
||||
enc, langs = self._encode(clips, opts)
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
heard = self._choose(enc, langs, opts, None, plain)
|
||||
prev = self._previous(langs, contexts)
|
||||
if prev is None:
|
||||
return heard
|
||||
prompted = self._choose(enc, langs, opts, prev, self._decode(enc, langs, opts.beam, None, prev))
|
||||
return [choose_with_context(h, p, opts.hotwords, pl) if c else h
|
||||
for h, p, pl, c in zip(heard, prompted, plain, contexts)]
|
||||
|
||||
def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None:
|
||||
"""The contexts as previous-text tokens of one length for the whole batch: the decoder needs
|
||||
the start token at the same position in every prompt. Each is cut from its end to the
|
||||
shortest one (a channel's clips share most of their context); a clip without one borrows
|
||||
another's, and its prompted result is not used."""
|
||||
if not contexts or not any(contexts):
|
||||
return None
|
||||
toks = [self._tokenizer(lang).encode(" " + ctx.strip())[-MAX_PREVIOUS_TOKENS:] if ctx else None
|
||||
for lang, ctx in zip(langs, contexts)]
|
||||
n = min(len(t) for t in toks if t is not None)
|
||||
filler = next(t for t in toks if t is not None)[-n:]
|
||||
return [t[-n:] if t is not None else filler for t in toks]
|
||||
|
||||
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
|
||||
m = self.model
|
||||
n_frames = m.feature_extractor.nb_max_frames
|
||||
feats = []
|
||||
@@ -118,16 +208,20 @@ class WhisperTranscriber:
|
||||
langs = [opts.language] * len(clips)
|
||||
else:
|
||||
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
return enc, langs
|
||||
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None,
|
||||
plain: list[tuple[str, float]]) -> list[str]:
|
||||
if not opts.hotwords:
|
||||
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords))
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev)
|
||||
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
|
||||
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]:
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
|
||||
prev: list[list[int]] | None = None) -> list[tuple[str, float]]:
|
||||
m = self.model
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords)
|
||||
for lang in langs]
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), p, without_timestamps=True, hotwords=hotwords)
|
||||
for lang, p in zip(langs, prev or [[]] * len(langs))]
|
||||
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
|
||||
suppress_tokens=[-1], return_scores=True)
|
||||
# ``scores`` is the sequence log-probability divided by its length: the mean per token.
|
||||
|
||||
@@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`):
|
||||
| `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. |
|
||||
| `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. |
|
||||
| `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. |
|
||||
| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. |
|
||||
| `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). |
|
||||
|
||||
### Utterance rules
|
||||
@@ -144,14 +145,65 @@ text is empty:
|
||||
- Its compression ratio (UTF-8 bytes over zlib bytes) is at most 2.4, which catches repetition loops.
|
||||
- With hotwords: the clip is decoded once with them and once without, from the same encoder pass.
|
||||
The plain decode must pass the checks above, otherwise nothing was said. The hinted text is used
|
||||
only when it stays within half the plain text's letters of it and contains every hotword exactly
|
||||
as often as the plain one (ignoring case, spacing and punctuation, no fuzzy matching), so the hint
|
||||
can never add, drop or create a wake word: not from laughter, not after a real sentence, not from
|
||||
a one-letter wake word, and not by turning a similar word ("서비스", or the near miss "자비수")
|
||||
into it. Otherwise the plain text is used; matching near misses is left to call detection.
|
||||
only when it stays within half the plain text's letters of it and keeps the plain one's wake
|
||||
words (ignoring case, spacing and punctuation): every hotword exactly as often, or exactly one
|
||||
wake word more where the plain text has a near miss of it, that is when the whole hinted text is
|
||||
at most a third of that wake word's letters from the plain one, compared as Unicode NFKD letters
|
||||
(Hangul by its letters). So the hint never drops a wake word, and never adds one from laughter,
|
||||
after a real sentence, from a one-letter wake word or in place of a different word ("기구" heard
|
||||
for "디코" became "김비서" with the hint: five letters off). It does turn a near miss the plain
|
||||
decode wrote into the wake word the hinted decode heard: live 2026-11-19 "비서야" was decoded as
|
||||
"기사야" (two of six letters off, more than call detection allows), and the hinted decode of the
|
||||
same audio gave "비서야". Otherwise the plain text is used; other near misses are left to call
|
||||
detection. Measured on the user's 100 recordings and 108 noise clips (CPU int8, 2026-11-19): the
|
||||
hint added a wake word to 8 recordings and 15 noise clips; this rule takes 5 (all said: 자비스 or
|
||||
비서야 heard as 라비스, 아비스, 기사야, 기서야) and rejects the 3 others (two said, one "디코") and all
|
||||
15 noise clips. Of 15 synthetic sentences with words near the wake words, the hint added none
|
||||
where the word was heard as said (서비스, 기사, 이사야, 비상약, 미소야, 시사); two where the voice
|
||||
itself said "비사(야)" for "기사(야)" became "비서(야)".
|
||||
|
||||
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
|
||||
### Conversation context
|
||||
|
||||
People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기")
|
||||
were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이"
|
||||
and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made
|
||||
the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also
|
||||
decoded with the recent conversation as its previous-text prompt:
|
||||
|
||||
- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said
|
||||
(the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined
|
||||
by spaces, at most `context_chars` characters from the end. Joining another channel or leaving
|
||||
clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt.
|
||||
- Partials have none (they stay one cheap decode).
|
||||
- The clip is decoded as before (with the hotword choice above) and again with the context (the
|
||||
same choice, each decode prompted). The prompted text is used only when the one without context
|
||||
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
|
||||
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
|
||||
Whisper write words of the conversation; this rule keeps that out, as for the hint.
|
||||
- One rescue: when the decode without context was dropped by the gate but its mean log-probability
|
||||
is at least -0.8 (just short of -0.7), the prompted text (which passed the gate itself) is used if
|
||||
it is that close to it and keeps every hotword count. The unsure text itself is never used.
|
||||
Measured on harsh copies of the user's speech: a cheap headset 13.8% to 7.5% error, empty results
|
||||
fewer by 1-2 per 100 in most conditions, and no more noise with text (VAD-cut noise 0 of 5, noise
|
||||
decoded straight 16 of 108, as before); -0.9 let 3 more noise clips through.
|
||||
- Finals stay batched. The decoder needs the start token at the same position in every prompt of a
|
||||
batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's
|
||||
clips share most of it); a clip without a context borrows another's and keeps its plain result.
|
||||
|
||||
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
|
||||
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
|
||||
text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more
|
||||
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
|
||||
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
|
||||
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
|
||||
whole sentences of the prompt. Under harsh copies of the same recordings (a weak voice over a mic
|
||||
noise floor, ESC-50 noise at SNR 10 and 5 dB, a 300-3400 Hz headset, Opus 12 kbps, reverb) the
|
||||
context never made the error rate worse (it fell most with Opus and reverb). Ten finals at once stay
|
||||
one batch: 1.3 s with contexts, 1.6-2.0 s without (Strata running). Evidence and the model comparison:
|
||||
`docs/v2/stt-accuracy-plan.md`.
|
||||
|
||||
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
|
||||
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
|
||||
clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD
|
||||
@@ -257,8 +309,15 @@ which carry Discord's own time.
|
||||
answer is stored when the app reports the sent message as `chat_message` with `to`. With
|
||||
`speak: true` the answer is also spoken through the TTS path under the same id (audio frames and
|
||||
`tts_done`), and `tts_cancel` with that id stops it. A `tts_cancel` that arrives while the brain is
|
||||
still answering means the answer is not spoken: `reply_text` still comes, then `tts_done` with
|
||||
`cancelled: true` and no audio. Every reply, a task's too and one that failed, leaves a trace of
|
||||
still answering means the answer is no longer wanted (the app took the request back: its asker
|
||||
went on talking, a newer request or a stop took its place). A reply still waiting for the asker's
|
||||
previous answer is never started; a running one stops at the engine's next model call or tool
|
||||
call (`turn_state.cancel`: no tool runs on its behalf) and returns no answer. Such a reply is as if
|
||||
never asked: `reply_text` comes with empty text and `failed: true`, then `tts_done` with
|
||||
`cancelled: true` and no audio, and it uses up no turn of the asker's (the request that replaces it
|
||||
is still the very next turn, so a confirmation it carries does not lapse; nor is that request
|
||||
"sent before the previous answer"). A reply whose answer was already written when the cancel
|
||||
came still sends its text, then `tts_done` with `cancelled: true` and no audio. Every reply, a task's too and one that failed, leaves a trace of
|
||||
how it was made (`history.spec.md`, "Traces"): the history lines and number of notes it was given,
|
||||
then what the engine wrote down. It is stored as soon as the request arrives (before it waits for
|
||||
the asker's previous answer) and again as each step is added, then ends with the answer, or with
|
||||
|
||||
137
tests/test_engine_cancel.py
Normal file
137
tests/test_engine_cancel.py
Normal file
@@ -0,0 +1,137 @@
|
||||
"""An answer that is no longer wanted stops at the engine's next model or tool call (turn_state.cancel).
|
||||
|
||||
The voice worker sets it when the app takes the request back (the asker went on talking and the
|
||||
request is joined to what they said next, a newer request took its place, or a stop). The engine
|
||||
then returns no answer and runs no tool on the request's behalf.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _native_call(name: str, args: dict) -> dict:
|
||||
return {"message": {"role": "assistant", "content": "", "tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": name, "arguments": args}}]}}
|
||||
|
||||
|
||||
def _run(mock_config, db, dialogue_memory, cancel_after_calls: int):
|
||||
from jarvis.memory.history import HistoryContext
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.tools.types import ToolExecutionResult
|
||||
|
||||
mock_config.ollama_chat_model = "gpt-oss:20b"
|
||||
mock_config.evaluator_enabled = False
|
||||
mock_config.stream_browser = False
|
||||
calls = []
|
||||
invoked = []
|
||||
|
||||
def fake_chat(*args, **kwargs):
|
||||
calls.append(1)
|
||||
return _native_call("getWeather", {"location": "Seoul"})
|
||||
|
||||
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
|
||||
invoked.append(tool_name)
|
||||
return ToolExecutionResult(success=True, reply_text="Seoul: 13C", error_message=None)
|
||||
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: len(calls) >= cancel_after_calls)
|
||||
try:
|
||||
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
|
||||
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
|
||||
patch.object(engine_mod, "select_tools", return_value=["getWeather", "stop"]), \
|
||||
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]):
|
||||
reply = engine_mod.run_reply_engine(
|
||||
db=db, cfg=mock_config, tts=None, text="서울 날씨 어때?",
|
||||
dialogue_memory=dialogue_memory, history=HistoryContext(), legacy_memory=False,
|
||||
intent_judge=True)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
return reply, len(calls), invoked
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_answer_cancelled_before_it_starts_makes_no_model_call(mock_config, db, dialogue_memory):
|
||||
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=0)
|
||||
assert reply is None and calls == 0 and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_answer_cancelled_during_a_model_call_runs_no_tool_for_it(mock_config, db, dialogue_memory):
|
||||
# The model asks for a tool, but the answer was taken back meanwhile: the tool is not run.
|
||||
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=1)
|
||||
assert reply is None and calls == 1 and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_planned_tool_is_not_run_when_the_answer_is_cancelled_while_its_step_is_resolved(
|
||||
mock_config, db, dialogue_memory):
|
||||
# Small models run planned steps directly: the step resolver (a model call) returns a tool call,
|
||||
# but the answer was taken back meanwhile.
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
|
||||
mock_config.ollama_chat_model = "gemma4:e2b"
|
||||
mock_config.evaluator_enabled = False
|
||||
cancelled = [False]
|
||||
invoked = []
|
||||
|
||||
def fake_resolve(*args, **kwargs):
|
||||
cancelled[0] = True
|
||||
return ("webSearch", {"query": "x"})
|
||||
|
||||
plan = ["webSearch query='a'", "webSearch query='b from step 1'", "Reply to the user."]
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: cancelled[0])
|
||||
try:
|
||||
with patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: invoked.append(1)), \
|
||||
patch.object(engine_mod, "chat_with_messages", side_effect=AssertionError("no model call")), \
|
||||
patch.object(engine_mod, "select_tools", return_value=["webSearch", "stop"]), \
|
||||
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
|
||||
patch.object(engine_mod, "plan_query", return_value=plan), \
|
||||
patch.object(engine_mod, "_resolve_plan_step", side_effect=fake_resolve):
|
||||
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="a then b",
|
||||
dialogue_memory=dialogue_memory)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
assert reply is None and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("cancel_in", ["select_tools", "plan_query"])
|
||||
def test_a_cancel_during_routing_or_planning_stops_before_the_next_call(mock_config, db, dialogue_memory,
|
||||
cancel_in):
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
|
||||
mock_config.ollama_chat_model = "gpt-oss:20b"
|
||||
mock_config.evaluator_enabled = False
|
||||
cancelled = [False]
|
||||
calls = []
|
||||
|
||||
def route(*a, **k):
|
||||
calls.append("route")
|
||||
cancelled[0] = cancel_in == "select_tools"
|
||||
return ["webSearch", "stop"]
|
||||
|
||||
def plan(*a, **k):
|
||||
calls.append("plan")
|
||||
cancelled[0] = True
|
||||
return ["webSearch query='a'", "Reply to the user."]
|
||||
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: cancelled[0])
|
||||
try:
|
||||
with patch.object(engine_mod, "chat_with_messages", side_effect=lambda *a, **k: calls.append("chat")), \
|
||||
patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: calls.append("tool")), \
|
||||
patch.object(engine_mod, "select_tools", side_effect=route), \
|
||||
patch.object(engine_mod, "plan_query", side_effect=plan):
|
||||
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="날씨 찾아 줘",
|
||||
dialogue_memory=dialogue_memory)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
assert reply is None
|
||||
assert calls == (["route"] if cancel_in == "select_tools" else ["route", "plan"])
|
||||
@@ -29,8 +29,9 @@ class FakeBrain:
|
||||
self.calls = []
|
||||
|
||||
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
|
||||
screen=None, pc=None, trace=None):
|
||||
screen=None, pc=None, trace=None, cancelled=None):
|
||||
self.calls.append((user, text, history))
|
||||
self.cancelled = cancelled
|
||||
if trace is not None:
|
||||
trace.append({"kind": "tools", "names": ["getWeather"]})
|
||||
self.chat, self.schedule, self.task = chat, schedule, task
|
||||
@@ -302,7 +303,7 @@ class SlowTranscriber(LevelTranscriber):
|
||||
super().__init__()
|
||||
self.delay = delay
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
import time as _t
|
||||
_t.sleep(self.delay)
|
||||
return super().transcribe(clips, opts)
|
||||
@@ -730,6 +731,83 @@ def test_a_reply_cancelled_while_the_brain_thinks_is_not_spoken(store):
|
||||
assert replies == ["r1", "r2"], "the text still comes back; only the speech is skipped"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_cancelled_reply_stops_the_brain_and_one_still_waiting_never_reaches_it(store):
|
||||
"""A request taken back (the asker went on talking) must not hold up the next one: the brain is
|
||||
told to stop the running answer, and a reply still waiting for it is never started."""
|
||||
import time
|
||||
started = []
|
||||
|
||||
def answer(user, text):
|
||||
started.append(text)
|
||||
if text == "first":
|
||||
t0 = time.monotonic()
|
||||
while not brain.cancelled() and time.monotonic() - t0 < 10:
|
||||
time.sleep(0.01)
|
||||
return None if brain.cancelled() else "late answer"
|
||||
return f"answer for {text}"
|
||||
|
||||
brain = FakeBrain(answer)
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="voice_join", channel="vc1")
|
||||
await send(ws, type="reply", id="r1", user="100", text="first", via="voice", speak=True)
|
||||
await asyncio.sleep(0.1)
|
||||
await send(ws, type="reply", id="r2", user="100", text="second", via="voice", speak=True)
|
||||
await send(ws, type="tts_cancel", id="r1")
|
||||
await send(ws, type="tts_cancel", id="r2")
|
||||
await send(ws, type="reply", id="r3", user="100", text="first second", via="voice", speak=True)
|
||||
t0 = time.monotonic()
|
||||
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done"
|
||||
and m["id"] == "r3")
|
||||
return seen, time.monotonic() - t0
|
||||
|
||||
seen, waited = asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
|
||||
assert waited < 3, "the joined request did not wait for the cancelled answer to run out"
|
||||
assert started == ["first", "first second"]
|
||||
done = {m["id"]: m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"}
|
||||
assert done["r1"]["cancelled"] and done["r2"]["cancelled"] and not done["r3"]["cancelled"]
|
||||
frames = {decode_tts_frame(m)[0] for m in seen if isinstance(m, bytes)}
|
||||
assert frames == {"r3"}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_request_taken_back_does_not_use_up_the_turn_a_confirmation_needs(store):
|
||||
"""Jarvis asked something; the asker says "yes" and goes on ("yes, quickly"). The app takes the
|
||||
"yes" back and sends the joined text: it must still be the very next turn, or a confirmation
|
||||
(pcControl, chat delete) would lapse."""
|
||||
import time
|
||||
turns = {}
|
||||
|
||||
def answer(user, text):
|
||||
turns[text] = brain.pc.turn
|
||||
if text == "yes":
|
||||
t0 = time.monotonic()
|
||||
while not brain.cancelled() and time.monotonic() - t0 < 10:
|
||||
time.sleep(0.01)
|
||||
return None
|
||||
return f"answer for {text}"
|
||||
|
||||
brain = FakeBrain(answer)
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="voice_join", channel="vc1")
|
||||
await send(ws, type="reply", id="r1", user="100", text="open the calculator", via="voice", speak=False)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
await send(ws, type="reply", id="r2", user="100", text="yes", via="voice", speak=True)
|
||||
await asyncio.sleep(0.1)
|
||||
await send(ws, type="tts_cancel", id="r2")
|
||||
await send(ws, type="reply", id="r3", user="100", text="yes quickly", via="voice", speak=True)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done" and m["id"] == "r3")
|
||||
|
||||
asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
|
||||
assert turns["yes quickly"] == turns["open the calculator"] + 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_brains_judgements_use_its_model_and_map_back_to_ids(monkeypatch):
|
||||
import voice_worker.judges as judges
|
||||
|
||||
@@ -56,16 +56,29 @@ def test_a_one_letter_wake_word_the_hint_adds_is_dropped():
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_hint_may_not_turn_a_similar_real_word_into_the_wake_word():
|
||||
assert choose_with_hint("자비스 신청해 줘", -0.1, "서비스 신청해 줘", -0.1, HINT) == "서비스 신청해 줘"
|
||||
def test_the_hint_may_not_turn_a_different_word_into_the_wake_word():
|
||||
# Live 2026-11-19: "디코 어디 갔어?" decoded as "기구 어디 갔어?", and with the hint as "김비서 어디 갔어?".
|
||||
secretary = ("비서", "비서야", "김비서")
|
||||
assert choose_with_hint("김비서 어디 갔어?", -0.49, "기구 어디 갔어?", -0.59, secretary) == "기구 어디 갔어?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_near_miss_spelling_is_left_as_heard_rather_than_guessed():
|
||||
# "자비수" may be the wake word or another word; text alone cannot tell, so the plain decode stands
|
||||
# and fuzzy wake word matching is left to the call detection that reads it.
|
||||
plain = "자비수, 오늘 서울 날씨 어때?"
|
||||
assert choose_with_hint("자비스, 오늘 서울 날씨 어때?", -0.1, plain, -0.2, HINT) == plain
|
||||
def test_a_hinted_wake_word_replaces_a_near_miss_of_it():
|
||||
# Live 2026-11-19: "비서야 디코 화면공유 보여?" decoded as "기사야 ...", two letters off "비서야"
|
||||
# (more than call detection allows); decoded with the hint, the same audio gave "비서야".
|
||||
secretary = ("비서", "비서야", "김비서")
|
||||
hinted = "비서야 디코 화면군이 보여"
|
||||
assert choose_with_hint(hinted, -0.19, "기사야 디코 화면군이 보여.", -0.25, secretary) == hinted
|
||||
hinted = "자비스, 오늘 서울 날씨 어때?"
|
||||
assert choose_with_hint(hinted, -0.1, "자비수, 오늘 서울 날씨 어때?", -0.2, HINT) == hinted
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_near_miss_is_replaced_only_by_one_wake_word():
|
||||
assert choose_with_hint("자비스 자비스 날씨", -0.1, "자비수 자비수 날씨", -0.1, HINT) == "자비수 자비수 날씨"
|
||||
# Nor when the hint also changes much else: the budget is the whole text's difference.
|
||||
assert choose_with_hint("자비스 오늘 날씨 알려 줘", -0.1, "자비수 어제 날씨 알려 줘", -0.1, HINT) \
|
||||
== "자비수 어제 날씨 알려 줘"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -96,3 +109,68 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
||||
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
||||
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
||||
== "자비스 오늘 서울 날씨 어때"
|
||||
|
||||
|
||||
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
|
||||
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
|
||||
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
|
||||
from voice_worker.stt_whisper import choose_with_context # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_may_fix_words_of_what_was_said():
|
||||
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_nothing_heard_without_the_context_stays_nothing():
|
||||
# Noise: the prompted decode wrote words of the conversation.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
|
||||
heard = "가상화면을 못 봐"
|
||||
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
|
||||
assert choose_with_context(heard, copied, HINT) == heard
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_never_changes_how_many_wake_words_were_heard():
|
||||
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
|
||||
== "라비스 지금 영상 재생되고 있어?"
|
||||
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_empty_prompted_decode_keeps_what_was_heard():
|
||||
assert choose_with_context("네.", "", HINT) == "네."
|
||||
|
||||
|
||||
# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for
|
||||
# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes.
|
||||
from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one():
|
||||
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \
|
||||
== "디코 어디 갔어?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_unsure_decode_well_below_the_gate_is_not_rescued():
|
||||
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_noise_with_a_prompt_far_from_it_is_not_rescued():
|
||||
# The prompted decode wrote words of the conversation over laughter.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_rescue_never_adds_a_wake_word():
|
||||
assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT,
|
||||
("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
|
||||
@@ -25,7 +25,7 @@ class FakeTranscriber:
|
||||
self.gate.set()
|
||||
self.fail_on_beam = fail_on_beam
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.entered.set()
|
||||
self.gate.wait(5)
|
||||
self.calls.append((opts, [len(c) for c in clips]))
|
||||
@@ -218,12 +218,45 @@ class StepTranscriber:
|
||||
def step(self):
|
||||
self._steps.release()
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append([len(c) for c in clips])
|
||||
self._steps.acquire(timeout=5)
|
||||
return ["" for _ in clips]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_reaches_the_model_with_its_own_context():
|
||||
class Ctx:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append(list(contexts or []))
|
||||
return [f"n{len(c)}" for c in clips]
|
||||
|
||||
eng = Ctx()
|
||||
|
||||
async def body():
|
||||
s = SttScheduler(eng)
|
||||
s.start()
|
||||
got = []
|
||||
done = asyncio.Event()
|
||||
|
||||
def cb(r):
|
||||
got.append(r.text)
|
||||
if len(got) == 3:
|
||||
done.set()
|
||||
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
|
||||
s.submit_final("b", clip(200), FINAL, cb)
|
||||
s.submit_partial("c", clip(300), PARTIAL, cb)
|
||||
await asyncio.wait_for(done.wait(), 5)
|
||||
await s.stop()
|
||||
|
||||
asyncio.run(body())
|
||||
assert eng.calls[0] == ["화면 공유", ""]
|
||||
assert eng.calls[1] == [""], "partials go without context"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
|
||||
t = StepTranscriber()
|
||||
|
||||
@@ -32,7 +32,7 @@ class LevelTranscriber:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append((opts, len(clips)))
|
||||
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
||||
|
||||
@@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k():
|
||||
lengths = []
|
||||
orig = t.transcribe
|
||||
|
||||
def spy(clips, opts):
|
||||
def spy(clips, opts, contexts=None):
|
||||
lengths.extend(len(c) for c in clips)
|
||||
return orig(clips, opts)
|
||||
|
||||
@@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path
|
||||
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
|
||||
r.close()
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]
|
||||
|
||||
|
||||
class ContextTranscriber(LevelTranscriber):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.contexts = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
if opts.beam > 1: # finals
|
||||
self.contexts += list(contexts or [""] * len(clips))
|
||||
return super().transcribe(clips, opts)
|
||||
|
||||
|
||||
async def _said(ws, speaker, level):
|
||||
await stream(ws, speaker, level, 600)
|
||||
await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel():
|
||||
"""voice_worker.spec.md, "Conversation context": what was said and answered just before, in the
|
||||
channel Jarvis is in, is the model's prompt for the next final."""
|
||||
t = ContextTranscriber()
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"}))
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.",
|
||||
"more": False}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
||||
await _said(ws, "200", 0.6)
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "config", "context_chars": 0}))
|
||||
await _said(ws, "100", 0.6)
|
||||
|
||||
asyncio.run(with_worker(body, transcriber=t, tts=FakeTts()))
|
||||
assert t.contexts[0] == ""
|
||||
assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다."
|
||||
assert t.contexts[2] == "", "a new channel starts without the old channel's conversation"
|
||||
assert t.contexts[3] == "", "context_chars 0 turns it off"
|
||||
|
||||
Reference in New Issue
Block a user