Compare commits
3 Commits
app-v0.1.5
...
app-v0.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b64097ef0d | ||
|
|
81bd8bd573 | ||
|
|
e716065538 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.50"
|
||||
version = "0.1.51"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.50"
|
||||
version = "0.1.51"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.50"
|
||||
version = "0.1.51"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.50"
|
||||
version = "0.1.51"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.50"
|
||||
version = "0.1.51"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -35,6 +35,9 @@ pub struct CallSettings {
|
||||
/// their last utterance's end, is still asking: what they say is joined to the request
|
||||
/// (`call.continueMs`; zero turns it off).
|
||||
pub continue_window: Duration,
|
||||
/// A voice answer starts playing only once its asker has been quiet this long after their request
|
||||
/// ended (they may go on); it is written meanwhile (`call.answerWaitMs`; zero: as soon as written).
|
||||
pub answer_wait: Duration,
|
||||
/// Play the earcon when Jarvis takes a voice call (`call.earcon`).
|
||||
pub earcon: bool,
|
||||
}
|
||||
@@ -51,6 +54,7 @@ impl Default for CallSettings {
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
continue_window: Duration::from_millis(5000),
|
||||
answer_wait: Duration::from_millis(3000),
|
||||
earcon: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,10 @@ pub struct Room {
|
||||
pub trait Voice {
|
||||
/// Have the brain answer and speak it; the speech request id, or None when it cannot. `heard`:
|
||||
/// when the request's speech ended and its final arrived (the dashboard times the answer by it).
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String>;
|
||||
/// `hold`: the answer is written but does not start playing until `release`.
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String>;
|
||||
/// Let a held answer play.
|
||||
fn release(&self, id: &str);
|
||||
/// Stop one speech request.
|
||||
fn cancel(&self, id: &str);
|
||||
/// Stop all speech at once.
|
||||
@@ -57,8 +60,11 @@ pub trait Voice {
|
||||
}
|
||||
|
||||
impl Voice for DiscordCore {
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard)
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard, hold)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
DiscordCore::release(self, id)
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
DiscordCore::cancel(self, id)
|
||||
@@ -170,6 +176,8 @@ pub struct Conversation<V: Voice> {
|
||||
/// Each speaker's latest voice request whose answer has not been heard yet: what they say next
|
||||
/// may still be part of it ("Going on").
|
||||
turns: HashMap<u64, Turn>,
|
||||
/// Held answers (speech id): their asker, and when they will have been quiet for `call.answerWaitMs`.
|
||||
waiting: HashMap<String, (u64, Instant)>,
|
||||
}
|
||||
|
||||
/// A voice request whose answer is not heard yet.
|
||||
@@ -202,7 +210,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
|
||||
by_speech: HashMap::new(), asks: HashMap::new(), missed: MissedLog::default(), chat: ChatCalls::default(),
|
||||
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new(),
|
||||
chimed: HashSet::new(), turns: HashMap::new() }
|
||||
chimed: HashSet::new(), turns: HashMap::new(), waiting: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Answer the registered chat channels too.
|
||||
@@ -272,6 +280,24 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
self.handle(ev, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
/// Held answers whose asker has been quiet for `call.answerWaitMs` start playing; while the asker
|
||||
/// talks, their answer waits for what they say (it may join the request).
|
||||
fn release_due(&mut self, now: Instant) {
|
||||
let due: Vec<String> = self.waiting.iter()
|
||||
.filter(|(_, (speaker, at))| now >= *at && !self.started.contains_key(speaker))
|
||||
.map(|(sid, _)| sid.clone()).collect();
|
||||
for sid in due {
|
||||
self.waiting.remove(&sid);
|
||||
debug_log!("conversation", "release speech={sid}");
|
||||
self.voice.release(&sid);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::SpeechStart(h) => {
|
||||
self.policy.utterance_open(h.speaker_id);
|
||||
@@ -390,16 +416,16 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.ranked(rank_id, &order, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
// The answer is heard: what its asker says next is a new request.
|
||||
// The answer is heard; while it plays, its asker may still go on (within call.continueMs).
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
self.waiting.remove(&e.id);
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
// A held request waits for its speaker's next final even after its answer ended.
|
||||
@@ -484,6 +510,7 @@ impl<V: Voice> Conversation<V> {
|
||||
fn stop_everything(&mut self, now: Instant) {
|
||||
self.wakes.clear();
|
||||
self.turns.clear();
|
||||
self.waiting.clear();
|
||||
let a = self.policy.stop();
|
||||
self.apply(a, now);
|
||||
self.chimed.clear(); // nothing is waiting any more
|
||||
@@ -516,26 +543,33 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
let a = self.policy.tick(now);
|
||||
self.apply(a, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
pub fn next_deadline(&self) -> Option<Instant> {
|
||||
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT);
|
||||
let wakes = self.wakes.values().filter_map(|(_, d)| *d);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min()
|
||||
// Not while the asker talks: their final decides.
|
||||
let held = self.waiting.values().filter(|(s, _)| !self.started.contains_key(s)).map(|(_, at)| *at);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).chain(held).min()
|
||||
}
|
||||
|
||||
fn apply(&mut self, actions: Vec<Action>, now: Instant) {
|
||||
for action in actions {
|
||||
match action {
|
||||
Action::Answer(r) => match {
|
||||
// The earcon plays first, then the answer as soon as it is written.
|
||||
// The earcon plays first, then the answer once it is written and its asker has
|
||||
// been quiet for call.answerWaitMs.
|
||||
if self.calls.earcon && !self.chimed.remove(&r.id) {
|
||||
self.voice.earcon();
|
||||
}
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at))
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at), !self.calls.answer_wait.is_zero())
|
||||
} {
|
||||
Some(sid) => {
|
||||
debug_log!("conversation", "answer req={} speaker={} speech={sid}", r.id, r.speaker);
|
||||
if !self.calls.answer_wait.is_zero() {
|
||||
self.waiting.insert(sid.clone(), (r.speaker, r.ended_at + self.calls.answer_wait));
|
||||
}
|
||||
self.by_speech.insert(sid.clone(), r.id);
|
||||
self.answers.insert(r.id, sid);
|
||||
}
|
||||
@@ -547,6 +581,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Action::StopAnswer(pid) => {
|
||||
if let Some(sid) = self.answers.remove(&pid) {
|
||||
self.by_speech.remove(&sid);
|
||||
self.waiting.remove(&sid);
|
||||
self.voice.cancel(&sid);
|
||||
}
|
||||
}
|
||||
@@ -601,7 +636,7 @@ mod tests {
|
||||
|
||||
/// The tests call Jarvis by one wake word of their own, whatever the default is.
|
||||
fn test_calls() -> CallSettings {
|
||||
CallSettings { wake_words: vec!["자비스".into()], ..CallSettings::default() }
|
||||
CallSettings { wake_words: vec!["자비스".into()], answer_wait: Duration::ZERO, ..CallSettings::default() }
|
||||
}
|
||||
use crate::discord::{Heard, HeardFinal};
|
||||
use crate::playback::PlaybackEnd;
|
||||
@@ -613,6 +648,9 @@ mod tests {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Did {
|
||||
Answer(u64, String),
|
||||
/// The answer just asked for is held until released.
|
||||
Held(String),
|
||||
Release(String),
|
||||
Cancel(String),
|
||||
StopAll,
|
||||
Earcon,
|
||||
@@ -637,10 +675,17 @@ mod tests {
|
||||
people: Rc<RefCell<Vec<u64>>>,
|
||||
}
|
||||
impl Voice for Fake {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant)) -> Option<String> {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
*self.n.borrow_mut() += 1;
|
||||
let id = format!("s{}", self.n.borrow());
|
||||
self.did.borrow_mut().push(Did::Answer(user, text.into()));
|
||||
Some(format!("s{}", self.n.borrow()))
|
||||
if hold {
|
||||
self.did.borrow_mut().push(Did::Held(id.clone()));
|
||||
}
|
||||
Some(id)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Release(id.into()));
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Cancel(id.into()));
|
||||
@@ -1163,13 +1208,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_the_answer_is_heard_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
fn once_the_answer_ended_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 2500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "고마워".into())]);
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Answer(A, "고마워".into())]);
|
||||
// Still thinking, but the speaker paused longer than call.continueMs.
|
||||
let (mut c, f, t) = alone(PolicyKind::InOrder);
|
||||
let pause = test_calls().continue_window.as_millis() as u64;
|
||||
@@ -1216,6 +1261,77 @@ mod tests {
|
||||
|
||||
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
|
||||
|
||||
/// Alone with Jarvis, answers held until the asker has been quiet for `wait_ms`.
|
||||
fn alone_waiting(kind: PolicyKind, wait_ms: u64) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
let calls = CallSettings { mode: CallMode::Addressed, earcon: false, answer_wait: Duration::from_millis(wait_ms),
|
||||
..test_calls() };
|
||||
let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(Conversation::new(f.clone(), calls, policy), f, Instant::now())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_starts_only_after_its_asker_was_quiet_for_the_wait() {
|
||||
// Live 2026-11-19: pauses inside one thought were up to 4.7 s, longer than writing an answer.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Held("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 2900)), "{kind:?}");
|
||||
c.tick(ms(t, 2800));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
c.tick(ms(t, 2900));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), None, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_while_its_asker_talks_and_their_words_join_it() {
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
assert!(f.take().is_empty(), "still talking ({kind:?})");
|
||||
c.on_event(&fin_utt(A, 2, "메모장도"), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 메모장도".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
c.tick(ms(t, 6400)); // the joined request ended at 3400
|
||||
assert_eq!(f.take(), vec![Did::Release("s2".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_is_released_when_what_followed_was_noise() {
|
||||
let (mut c, f, t) = alone_waiting(PolicyKind::Latest, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
f.take();
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_while_the_answer_plays_stops_it_and_answers_all_of_it() {
|
||||
// A pause longer than the wait: the answer started, then the speaker went on within
|
||||
// call.continueMs of their request's end.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.tick(ms(t, 2900));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
f.take();
|
||||
c.on_event(&fin(A, "제일 앞으로", 100), ms(t, 5000)); // started at 3900, 4 s after the end
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
|
||||
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
|
||||
|
||||
@@ -34,7 +34,8 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer is not heard yet. 0 turns joining off. 0-15000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer has not ended. 0 turns joining off. 0-15000. |
|
||||
| `call.answerWaitMs` | `3000` | Waiting before answering (below): a voice answer starts playing only once its asker has been quiet this long after their request ended. 0: as soon as it is written. 0-10000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
@@ -83,8 +84,9 @@ a message is complete as sent.
|
||||
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
|
||||
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
|
||||
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
|
||||
So a speaker's voice request stays open until its answer is heard (the first audio of its speech
|
||||
plays, `SpeakStart`), and while it is open their next final is joined to it when that utterance
|
||||
So a speaker's voice request stays open until its answer has ended (`SpeakEnd`; also while it
|
||||
plays, since live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer and the answer
|
||||
started in the pause), and while it is open their next final is joined to it when that utterance
|
||||
started within `call.continueMs` of the request's end (start = the final's arrival minus its
|
||||
`wait_ms` and `audio_ms`):
|
||||
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
|
||||
@@ -100,12 +102,21 @@ started within `call.continueMs` of the request's end (start = the final's arriv
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
|
||||
- Other speakers' utterances never join it. An answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
|
||||
its answer is being written (it is the request that will be answered).
|
||||
|
||||
**Waiting before answering.** So that a pause inside one thought does not get an answer, a voice
|
||||
answer is written at once but starts playing only when its asker has been quiet for
|
||||
`call.answerWaitMs` after the request ended (default 3 s: 8 of the 10 pauses inside one thought
|
||||
measured live 2026-11-19 were shorter). While the asker is talking when that time comes, the answer
|
||||
waits for that utterance's final: it joins the request (Going on, the held answer is cancelled and
|
||||
the joined one waits again from its own end), or, when it is empty or not joined, the answer plays.
|
||||
A stop stops it as always. The earcon is not held: it still tells the asker at once that Jarvis
|
||||
heard them. Answers queued behind a held one wait their turn.
|
||||
|
||||
**Earcon.** A short two-tone "띠링" (about 0.3 s, made by the app, `app/src/earcon.rs`) tells the
|
||||
speaker Jarvis heard the call, long before the answer is written: it is the first sound after a call.
|
||||
It plays right before each voice answer starts (the policy's `Answer`), and at once when a wake word
|
||||
@@ -226,12 +237,13 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
judgement) closes the utterance, and no answer within 5 s counts as no, also when the answer arrives
|
||||
after that but before the timer ran.
|
||||
- Policy actions: `Answer` → the earcon (unless it already played for this request), then
|
||||
`answer(user, text)` (the brain answers and the answer is spoken; when it
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`answer(user, text, hold)` (the brain answers and the answer is spoken; held, with a non-zero
|
||||
`call.answerWaitMs`, until `release` when its asker has been quiet for it; when it cannot be
|
||||
queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on),
|
||||
unless its speaker already started their next utterance (held).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard (no second earcon when it is joined); its
|
||||
request stays open until the answer ends (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
@@ -254,7 +266,9 @@ no and timeout, the earcon before each answer, once for a wake word alone, none
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
|
||||
new request once the answer ended or the pause is longer than `call.continueMs`, joined while the
|
||||
answer plays, answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
|
||||
and joined by what they say, released after noise, in every policy, a stop still
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
|
||||
@@ -563,15 +563,26 @@ impl DiscordCore {
|
||||
/// `SpeakStart` and `SpeakEnd` follow as for `say`), or None when not in a voice channel. It ends as
|
||||
/// failed when the brain gives no answer. `heard`: when the request's speech ended and its final
|
||||
/// arrived, passed on in `AnswerAsked`.
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
self.voice_channel_id()?;
|
||||
let id = with_playback(&self.inner, |p| p.adopt());
|
||||
let id = with_playback(&self.inner, |p| {
|
||||
let id = p.adopt();
|
||||
if hold {
|
||||
p.hold(&id);
|
||||
}
|
||||
id
|
||||
});
|
||||
let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user, ended_at: heard.0,
|
||||
final_at: heard.1 });
|
||||
self.inner.worker.reply_in(&id, user, text, "voice", true, None, ¬es_of(&self.inner, user));
|
||||
Some(id)
|
||||
}
|
||||
|
||||
/// Let an answer held by `answer(.., hold)` play.
|
||||
pub fn release(&self, id: &str) {
|
||||
with_playback(&self.inner, |p| p.release(id));
|
||||
}
|
||||
|
||||
/// Play the earcon in the voice channel, queued like speech. The request id, or None when not in a
|
||||
/// voice channel.
|
||||
pub fn earcon(&self) -> Option<String> {
|
||||
|
||||
@@ -171,7 +171,9 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
long the clip playing when it was stopped had played, timed from when it was handed to the player)
|
||||
so its voice recordings keep only what was heard (`voice_worker.spec.md`, "Recordings").
|
||||
- **Answering** (M4): `answer(user, text)` queues a request like `say` whose speech the worker makes
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. No answer
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. With
|
||||
`hold` it is written but does not start playing until `release(id)` (an answer that fails while held
|
||||
still ends at once and lets the queue go on). No answer
|
||||
from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed.
|
||||
`cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of
|
||||
an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and
|
||||
|
||||
@@ -48,6 +48,8 @@ struct Request {
|
||||
failed: bool,
|
||||
played: u32,
|
||||
started: bool,
|
||||
/// Not to start playing until released (`hold`).
|
||||
held: bool,
|
||||
}
|
||||
|
||||
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
||||
@@ -81,7 +83,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
||||
self.backend.tts(&id, text, more);
|
||||
id
|
||||
@@ -93,7 +95,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
||||
id
|
||||
}
|
||||
@@ -103,7 +105,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
||||
failed: false, played: 0, started: false });
|
||||
failed: false, played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
||||
self.pump();
|
||||
id
|
||||
@@ -118,6 +120,22 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep a request that has not started from starting until `release` (the asker may go on).
|
||||
pub fn hold(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.started) {
|
||||
r.held = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Let a held request play.
|
||||
pub fn release(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && r.held) {
|
||||
r.held = false;
|
||||
crate::debug_log!("play", "release id={id}");
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
||||
/// queue goes on.
|
||||
pub fn cancel(&mut self, id: &str) {
|
||||
@@ -207,6 +225,9 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
head.next = first;
|
||||
}
|
||||
}
|
||||
if head.held && head.chunks.contains_key(&head.next) {
|
||||
return;
|
||||
}
|
||||
if let Some(pcm) = head.chunks.remove(&head.next) {
|
||||
head.next += 1;
|
||||
head.played += 1;
|
||||
@@ -480,4 +501,28 @@ mod tests {
|
||||
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
||||
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_for_its_release_and_a_failed_one_does_not_block() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.adopt();
|
||||
p.hold(&id);
|
||||
p.audio(&id, 0, b("a0"));
|
||||
p.done(&done(&id, false));
|
||||
assert!(f.0.borrow().played.is_empty(), "written, but held");
|
||||
assert!(p.take_events().is_empty());
|
||||
p.release(&id);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
||||
// An answer that failed while held ends at once, and the next request plays.
|
||||
p.clip_ended();
|
||||
p.take_events();
|
||||
let bad = p.adopt();
|
||||
p.hold(&bad);
|
||||
let next = p.say("b", false);
|
||||
p.audio(&next, 0, b("b0"));
|
||||
p.fail(&bad);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
||||
assert_eq!(p.take_events()[0], end(&bad, 0, false, true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -454,6 +454,7 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
|
||||
stop_words,
|
||||
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
|
||||
continue_window: get_ms(call, "continueMs", "call", d.continue_window, 15000)?,
|
||||
answer_wait: get_ms(call, "answerWaitMs", "call", d.answer_wait, 10000)?,
|
||||
earcon: get_bool(call, "earcon", "call", d.earcon)?,
|
||||
},
|
||||
chat_wake_words,
|
||||
|
||||
@@ -96,6 +96,7 @@ pub fn fields() -> Vec<Field> {
|
||||
json!(["멈춰", "그만", "정지", "stop"]), "정지 명령"),
|
||||
f("call.followUpMs", Call, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.continueMs", Call, Int { min: 0, max: 15000 }, json!(5000), "끊어 말한 문장 합치기(ms)"),
|
||||
f("call.answerWaitMs", Call, Int { min: 0, max: 10000 }, json!(3000), "말이 끝나고 답하기 전 기다림(ms)"),
|
||||
f("call.earcon", Call, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Response, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
json!("latest"), "동시 대화 응답 정책"),
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! cargo test --release --test live_round_trip -- --nocapture
|
||||
//!
|
||||
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
||||
//! JARVIS_LIVE_WORKER_ARGS adds worker arguments (e.g. "--stt-device cpu --stt-compute int8").
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -165,16 +166,23 @@ fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: A
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
/// Both bots in the voice channel: the speaker bot (the "person") and Jarvis running the app's services
|
||||
/// from a fresh data folder, with `call` as the settings' call section.
|
||||
struct Live {
|
||||
speaker_id: u64,
|
||||
speaker_songbird: Arc<Songbird>,
|
||||
s_shards: Arc<serenity::gateway::ShardManager>,
|
||||
services: jarvis_app::services::Services,
|
||||
core: jarvis_app::discord::DiscordCore,
|
||||
events: Arc<Mutex<Vec<(Instant, CoreEvent)>>>,
|
||||
call: Arc<tokio::sync::Mutex<songbird::Call>>,
|
||||
ears: Ears,
|
||||
guild: u64,
|
||||
}
|
||||
|
||||
async fn start_live(call_settings: impl FnOnce(u64) -> Value) -> Live {
|
||||
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
||||
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
env("JARVIS_LIVE_LLM_URL"); // the services point the brain at it (llm.url)
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
|
||||
@@ -208,21 +216,26 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
||||
args.extend(["--melo-python".to_string(), m]);
|
||||
}
|
||||
// e.g. "--stt-device cpu --stt-compute int8 --melo-device cpu" on a host without a GPU
|
||||
if let Ok(extra) = std::env::var("JARVIS_LIVE_WORKER_ARGS") {
|
||||
args.extend(extra.split_whitespace().map(String::from));
|
||||
}
|
||||
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
|
||||
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
|
||||
// The "person" here is a bot, so bots must be heard; the recorded questions call "자비스".
|
||||
"voice": {"listenToBots": true, "wakeWords": ["자비스"]},
|
||||
"call": call_settings(speaker_id),
|
||||
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
|
||||
"llm": {"url": env("JARVIS_LIVE_LLM_URL")},
|
||||
})).unwrap()).unwrap();
|
||||
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
|
||||
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
let mut services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
.expect("services start in time").expect("services start");
|
||||
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
|
||||
let core = services.core.clone();
|
||||
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
|
||||
let ev2 = events.clone();
|
||||
let mut rx = services.events;
|
||||
let mut rx = std::mem::replace(&mut services.events, mpsc::unbounded_channel().1);
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
ev2.lock().unwrap().push((Instant::now(), e));
|
||||
@@ -240,6 +253,31 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
|
||||
|
||||
Live { speaker_id, speaker_songbird, s_shards, services, core, events, call, ears, guild }
|
||||
}
|
||||
|
||||
impl Live {
|
||||
async fn stop(self) {
|
||||
let _ = self.speaker_songbird.remove(GuildId::new(self.guild)).await;
|
||||
self.s_shards.shutdown_all().await;
|
||||
self.services.core.leave("test").await;
|
||||
self.services.core.close().await;
|
||||
self.services.worker.close();
|
||||
self.services.worker_process.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
let live = start_live(|_| json!({})).await;
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let Live { speaker_id, ref services, ref core, ref events, ref call, ref ears, .. } = live;
|
||||
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
|
||||
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
|
||||
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
|
||||
@@ -355,12 +393,7 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
||||
})).await;
|
||||
|
||||
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
||||
s_shards.shutdown_all().await;
|
||||
services.core.leave("test").await;
|
||||
services.core.close().await;
|
||||
services.worker.close();
|
||||
services.worker_process.stop().await;
|
||||
live.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
@@ -371,3 +404,76 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Going on (conversation.spec.md), live: the person's own recordings of one thing said in pieces,
|
||||
/// played with the pauses they really made, get one answer. Settings as the person's own app: they
|
||||
/// are the only designated person, `addressed` mode, alone with Jarvis.
|
||||
///
|
||||
/// JARVIS_LIVE_GOING_ON=1 (the env of conversation_round_trip_on_real_discord, without the questions)
|
||||
/// JARVIS_LIVE_PIECES=<dir with pieces.json: [{"name", "file"}], each file one 16 kHz mono wav of
|
||||
/// the pieces and the silence between them> cargo test --release --test live_round_trip going_on -- --nocapture
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn going_on_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_GOING_ON").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_GOING_ON=1");
|
||||
return;
|
||||
}
|
||||
let dir = PathBuf::from(env("JARVIS_LIVE_PIECES"));
|
||||
let seqs: Value = serde_json::from_str(&std::fs::read_to_string(dir.join("pieces.json")).unwrap()).unwrap();
|
||||
let live = start_live(|speaker| json!({"designated": [speaker.to_string()], "designatedOnly": true, "mode": "addressed"})).await;
|
||||
let mut rows = vec![];
|
||||
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
||||
for s in seqs.as_array().unwrap() {
|
||||
let wav = std::fs::read(dir.join(s["file"].as_str().unwrap())).unwrap();
|
||||
until("Jarvis to finish speaking", 60_000, || !live.core.speaking()).await;
|
||||
let before = live.events.lock().unwrap().len();
|
||||
live.call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
tokio::time::sleep(Duration::from_secs_f64(speech_end_seconds(&wav))).await;
|
||||
// Done when nothing is playing and nothing new came for 10 s (at most 60 s).
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
let (mut seen, mut quiet_since) = (0, Instant::now());
|
||||
while Instant::now() < deadline {
|
||||
let n = live.events.lock().unwrap().len();
|
||||
if n != seen || live.core.speaking() {
|
||||
(seen, quiet_since) = (n, Instant::now());
|
||||
} else if quiet_since.elapsed() > Duration::from_secs(10) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
let ev = live.events.lock().unwrap()[before..].to_vec();
|
||||
let heard: Vec<String> = ev.iter().filter_map(|(_, e)| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == live.speaker_id => Some(f.text.clone()),
|
||||
_ => None,
|
||||
}).collect();
|
||||
let answers: Vec<Value> = ev.iter().filter_map(|(_, e)| match e {
|
||||
CoreEvent::ReplyText { id, text, failed } => {
|
||||
let end = ev.iter().find_map(|(_, e)| match e {
|
||||
CoreEvent::SpeakEnd(x) if &x.id == id => Some(x.clone()),
|
||||
_ => None,
|
||||
});
|
||||
Some(json!({"text": text, "failed": failed, "played": end.as_ref().map(|x| x.played),
|
||||
"cancelled": end.as_ref().map(|x| x.cancelled)}))
|
||||
}
|
||||
_ => None,
|
||||
}).collect();
|
||||
let asked = ev.iter().filter(|(_, e)| matches!(e, CoreEvent::AnswerAsked { .. })).count();
|
||||
rows.push(json!({"name": s["name"], "heard": heard, "asked": asked, "answers": answers}));
|
||||
println!("going on {}", rows.last().unwrap());
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
}
|
||||
})).await;
|
||||
live.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// One thing said in pieces: one answer heard to its end, and no other answer heard.
|
||||
for r in &rows {
|
||||
let heard = |a: &&Value| a["played"].as_u64().is_some_and(|p| p > 0);
|
||||
let whole: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard)
|
||||
.filter(|a| a["cancelled"] == json!(false)).collect();
|
||||
let cut: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard)
|
||||
.filter(|a| a["cancelled"] == json!(true)).collect();
|
||||
assert!(whole.len() == 1 && cut.is_empty(), "{}: not one answer: {r}", r["name"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,3 +101,7 @@
|
||||
- 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로.
|
||||
- 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초.
|
||||
- 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내.
|
||||
- 2026-11-19 밤 (0.1.50 뒤): 근거 evidence/2026-11-19/stt-plan/wake/.
|
||||
- 첫마디 호출어: 힌트를 준 결과는 "비서야"를 맞게 들었는데(기사야·기서야·라비스·아비스 → 비서야·자비스), 힌트 결과는 힌트 없는 결과와 호출어 수가 같아야만 써서 버려졌음. 규칙을 바꿈: 힌트 없는 결과에 그 호출어의 비슷한 말이 있으면(글 전체가 호출어 글자 수의 3분의 1 이내, 한글은 자모로 셈) 호출어 하나를 더할 수 있음. 녹음 100개에서 5개를 바로잡고 3개는 그대로(그중 "기구"→"김비서"는 실제로 "디코"라 막은 것이 맞음), 잡음 216번(108개 × 힌트 2종)에서 더한 호출어 0. 소리로 다시 매기기(align 확률)는 맞는 쪽과 틀린 쪽을 가르지 못해 쓰지 않음.
|
||||
- 실제 디스코드(테스트 봇이 사용자 녹음을 실제 쉰 간격대로 틀음, app/tests/live_round_trip.rs going_on, 서버 CPU, LLM은 질문을 되돌려 주는 대역): 부산 날씨 3조각은 한 질문, 한 답. 계산기 6조각(쉼 2.3~4.7초)은 쉼이 답 만드는 시간보다 길어서 답이 먼저 들리기 시작했고, 들린 뒤의 말은 새 질문이 되어 중간 답 2개가 끊김. 서버 CPU라 받아 적기가 10~25초 걸려 시간은 실제와 다름(정확한 시간은 게임PC GPU에서 다시 재야 함).
|
||||
- 그래서(사용자 결정 전, 추천안 3으로 구현, 설정으로 끌 수 있음) 답은 바로 만들되 묻는 사람이 요청 끝에서 `call.answerWaitMs`(기본 3초) 동안 조용해야 말하기 시작하고, 그 사이 말하면 그 말의 결과를 기다려 합침. 답이 나오는 중에도 요청 끝에서 `call.continueMs`(5초) 안에 이어 말하면 답을 멈추고 합침(답이 끝난 뒤의 말은 새 질문). 실제 디스코드 다시 시험(같은 말과 같은 쉼을 MeloTTS로, 받아 적기 small CPU, 대역 LLM 0.8초×3): 세 묶음 모두 한 답, 중간에 들린 답 0(live3-answer-wait.log).
|
||||
|
||||
@@ -6,6 +6,7 @@ import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
import zlib
|
||||
|
||||
import numpy as np
|
||||
@@ -61,20 +62,50 @@ def _mentions(text: str, word: str) -> int:
|
||||
return _letters(text).count(w) if w else 0
|
||||
|
||||
|
||||
def _calls(text: str, hotwords: tuple[str, ...]) -> list[str]:
|
||||
"""The wake words in ``text``, longest first at each place, without overlaps ("김비서야" is one)."""
|
||||
s = _letters(text)
|
||||
words = sorted({_letters(w) for w in hotwords if _letters(w)}, key=len, reverse=True)
|
||||
found, i = [], 0
|
||||
while i < len(s):
|
||||
w = next((w for w in words if s.startswith(w, i)), None)
|
||||
if w:
|
||||
found.append(w)
|
||||
i += len(w)
|
||||
else:
|
||||
i += 1
|
||||
return found
|
||||
|
||||
|
||||
def keeps_wake_words(text: str, base: str, hotwords: tuple[str, ...]) -> bool:
|
||||
"""Whether ``text`` (a hinted or prompted decode) holds the wake words of ``base`` (the decode it is
|
||||
checked against): every hotword exactly as often, or one wake word more where ``base`` has a near
|
||||
miss of it. Near: the whole text is at most a third of that wake word's letters from ``base``,
|
||||
compared as Unicode NFKD letters (Hangul by its letters: "기사야" is two of six off "비서야")."""
|
||||
if all(_mentions(text, w) == _mentions(base, w) for w in hotwords):
|
||||
return True
|
||||
have, had = _calls(text, hotwords), _calls(base, hotwords)
|
||||
if len(have) != len(had) + 1 or any(_mentions(text, w) < _mentions(base, w) for w in hotwords):
|
||||
return False
|
||||
nfkd = lambda t: unicodedata.normalize("NFKD", _letters(t))
|
||||
return _edit_distance(nfkd(text), nfkd(base)) <= min(len(nfkd(w)) for w in have) // 3
|
||||
|
||||
|
||||
def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp: float,
|
||||
hotwords: tuple[str, ...]) -> str:
|
||||
"""Pick between the decode that had the hotwords as a hint and the plain decode of the same audio.
|
||||
|
||||
The hint can make Whisper write a wake word that nobody said (on laughter, tacked onto a real
|
||||
sentence, or in place of a similar word), so the plain decode decides: no plain text means nothing
|
||||
was said, and the hint's text is used only when it stays close to the plain one and contains every
|
||||
hotword exactly as often. It may change other words, never how many wake words were heard.
|
||||
sentence, or in place of another word), so the plain decode decides: no plain text means nothing
|
||||
was said, and the hint's text is used only when it stays close to the plain one and keeps its wake
|
||||
words (``keeps_wake_words``): it may change other words, and turn a near miss of a wake word into it.
|
||||
"""
|
||||
if not keep_transcript(plain_text, plain_lp):
|
||||
return ""
|
||||
close = _edit_distance(_letters(hint_text), _letters(plain_text)) <= len(_letters(plain_text)) // 2
|
||||
if close and keep_transcript(hint_text, hint_lp) and all(
|
||||
_mentions(hint_text, w) == _mentions(plain_text, w) for w in hotwords):
|
||||
if close and keep_transcript(hint_text, hint_lp) and keeps_wake_words(hint_text, plain_text, hotwords):
|
||||
if len(_calls(hint_text, hotwords)) != len(_calls(plain_text, hotwords)):
|
||||
debug_log("hint turned a near miss into a wake word", "stt")
|
||||
return hint_text
|
||||
return plain_text
|
||||
|
||||
|
||||
@@ -145,11 +145,22 @@ text is empty:
|
||||
- Its compression ratio (UTF-8 bytes over zlib bytes) is at most 2.4, which catches repetition loops.
|
||||
- With hotwords: the clip is decoded once with them and once without, from the same encoder pass.
|
||||
The plain decode must pass the checks above, otherwise nothing was said. The hinted text is used
|
||||
only when it stays within half the plain text's letters of it and contains every hotword exactly
|
||||
as often as the plain one (ignoring case, spacing and punctuation, no fuzzy matching), so the hint
|
||||
can never add, drop or create a wake word: not from laughter, not after a real sentence, not from
|
||||
a one-letter wake word, and not by turning a similar word ("서비스", or the near miss "자비수")
|
||||
into it. Otherwise the plain text is used; matching near misses is left to call detection.
|
||||
only when it stays within half the plain text's letters of it and keeps the plain one's wake
|
||||
words (ignoring case, spacing and punctuation): every hotword exactly as often, or exactly one
|
||||
wake word more where the plain text has a near miss of it, that is when the whole hinted text is
|
||||
at most a third of that wake word's letters from the plain one, compared as Unicode NFKD letters
|
||||
(Hangul by its letters). So the hint never drops a wake word, and never adds one from laughter,
|
||||
after a real sentence, from a one-letter wake word or in place of a different word ("기구" heard
|
||||
for "디코" became "김비서" with the hint: five letters off). It does turn a near miss the plain
|
||||
decode wrote into the wake word the hinted decode heard: live 2026-11-19 "비서야" was decoded as
|
||||
"기사야" (two of six letters off, more than call detection allows), and the hinted decode of the
|
||||
same audio gave "비서야". Otherwise the plain text is used; other near misses are left to call
|
||||
detection. Measured on the user's 100 recordings and 108 noise clips (CPU int8, 2026-11-19): the
|
||||
hint added a wake word to 8 recordings and 15 noise clips; this rule takes 5 (all said: 자비스 or
|
||||
비서야 heard as 라비스, 아비스, 기사야, 기서야) and rejects the 3 others (two said, one "디코") and all
|
||||
15 noise clips. Of 15 synthetic sentences with words near the wake words, the hint added none
|
||||
where the word was heard as said (서비스, 기사, 이사야, 비상약, 미소야, 시사); two where the voice
|
||||
itself said "비사(야)" for "기사(야)" became "비서(야)".
|
||||
|
||||
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
|
||||
|
||||
@@ -56,16 +56,29 @@ def test_a_one_letter_wake_word_the_hint_adds_is_dropped():
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_hint_may_not_turn_a_similar_real_word_into_the_wake_word():
|
||||
assert choose_with_hint("자비스 신청해 줘", -0.1, "서비스 신청해 줘", -0.1, HINT) == "서비스 신청해 줘"
|
||||
def test_the_hint_may_not_turn_a_different_word_into_the_wake_word():
|
||||
# Live 2026-11-19: "디코 어디 갔어?" decoded as "기구 어디 갔어?", and with the hint as "김비서 어디 갔어?".
|
||||
secretary = ("비서", "비서야", "김비서")
|
||||
assert choose_with_hint("김비서 어디 갔어?", -0.49, "기구 어디 갔어?", -0.59, secretary) == "기구 어디 갔어?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_near_miss_spelling_is_left_as_heard_rather_than_guessed():
|
||||
# "자비수" may be the wake word or another word; text alone cannot tell, so the plain decode stands
|
||||
# and fuzzy wake word matching is left to the call detection that reads it.
|
||||
plain = "자비수, 오늘 서울 날씨 어때?"
|
||||
assert choose_with_hint("자비스, 오늘 서울 날씨 어때?", -0.1, plain, -0.2, HINT) == plain
|
||||
def test_a_hinted_wake_word_replaces_a_near_miss_of_it():
|
||||
# Live 2026-11-19: "비서야 디코 화면공유 보여?" decoded as "기사야 ...", two letters off "비서야"
|
||||
# (more than call detection allows); decoded with the hint, the same audio gave "비서야".
|
||||
secretary = ("비서", "비서야", "김비서")
|
||||
hinted = "비서야 디코 화면군이 보여"
|
||||
assert choose_with_hint(hinted, -0.19, "기사야 디코 화면군이 보여.", -0.25, secretary) == hinted
|
||||
hinted = "자비스, 오늘 서울 날씨 어때?"
|
||||
assert choose_with_hint(hinted, -0.1, "자비수, 오늘 서울 날씨 어때?", -0.2, HINT) == hinted
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_near_miss_is_replaced_only_by_one_wake_word():
|
||||
assert choose_with_hint("자비스 자비스 날씨", -0.1, "자비수 자비수 날씨", -0.1, HINT) == "자비수 자비수 날씨"
|
||||
# Nor when the hint also changes much else: the budget is the whole text's difference.
|
||||
assert choose_with_hint("자비스 오늘 날씨 알려 줘", -0.1, "자비수 어제 날씨 알려 줘", -0.1, HINT) \
|
||||
== "자비수 어제 날씨 알려 줘"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
|
||||
Reference in New Issue
Block a user