feat(conversation): answers wait until the asker is quiet, and going on joins while the answer plays
Live 2026-11-19 (the user's own pieced requests replayed into Discord) the pauses inside one thought, 2.3-4.7 s, were longer than writing the answer: the answer started in the pause, and what followed became a new request, so answers were cut off mid-sentence. - A voice answer is written at once but starts playing only after its asker has been quiet for call.answerWaitMs (default 3000, 0 = off, settings "말이 끝나고 답하기 전 기다림"); while they talk it waits for their final, which joins the request. - A request stays open (Going on) until its answer ended, not until it is first heard: going on within call.continueMs while it plays stops it and answers all of it. Replayed live: the three pieced requests each get one answer, none cut off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -35,6 +35,9 @@ pub struct CallSettings {
|
||||
/// their last utterance's end, is still asking: what they say is joined to the request
|
||||
/// (`call.continueMs`; zero turns it off).
|
||||
pub continue_window: Duration,
|
||||
/// A voice answer starts playing only once its asker has been quiet this long after their request
|
||||
/// ended (they may go on); it is written meanwhile (`call.answerWaitMs`; zero: as soon as written).
|
||||
pub answer_wait: Duration,
|
||||
/// Play the earcon when Jarvis takes a voice call (`call.earcon`).
|
||||
pub earcon: bool,
|
||||
}
|
||||
@@ -51,6 +54,7 @@ impl Default for CallSettings {
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
continue_window: Duration::from_millis(5000),
|
||||
answer_wait: Duration::from_millis(3000),
|
||||
earcon: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,10 @@ pub struct Room {
|
||||
pub trait Voice {
|
||||
/// Have the brain answer and speak it; the speech request id, or None when it cannot. `heard`:
|
||||
/// when the request's speech ended and its final arrived (the dashboard times the answer by it).
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String>;
|
||||
/// `hold`: the answer is written but does not start playing until `release`.
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String>;
|
||||
/// Let a held answer play.
|
||||
fn release(&self, id: &str);
|
||||
/// Stop one speech request.
|
||||
fn cancel(&self, id: &str);
|
||||
/// Stop all speech at once.
|
||||
@@ -57,8 +60,11 @@ pub trait Voice {
|
||||
}
|
||||
|
||||
impl Voice for DiscordCore {
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard)
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard, hold)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
DiscordCore::release(self, id)
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
DiscordCore::cancel(self, id)
|
||||
@@ -170,6 +176,8 @@ pub struct Conversation<V: Voice> {
|
||||
/// Each speaker's latest voice request whose answer has not been heard yet: what they say next
|
||||
/// may still be part of it ("Going on").
|
||||
turns: HashMap<u64, Turn>,
|
||||
/// Held answers (speech id): their asker, and when they will have been quiet for `call.answerWaitMs`.
|
||||
waiting: HashMap<String, (u64, Instant)>,
|
||||
}
|
||||
|
||||
/// A voice request whose answer is not heard yet.
|
||||
@@ -202,7 +210,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
|
||||
by_speech: HashMap::new(), asks: HashMap::new(), missed: MissedLog::default(), chat: ChatCalls::default(),
|
||||
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new(),
|
||||
chimed: HashSet::new(), turns: HashMap::new() }
|
||||
chimed: HashSet::new(), turns: HashMap::new(), waiting: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Answer the registered chat channels too.
|
||||
@@ -272,6 +280,24 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
self.handle(ev, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
/// Held answers whose asker has been quiet for `call.answerWaitMs` start playing; while the asker
|
||||
/// talks, their answer waits for what they say (it may join the request).
|
||||
fn release_due(&mut self, now: Instant) {
|
||||
let due: Vec<String> = self.waiting.iter()
|
||||
.filter(|(_, (speaker, at))| now >= *at && !self.started.contains_key(speaker))
|
||||
.map(|(sid, _)| sid.clone()).collect();
|
||||
for sid in due {
|
||||
self.waiting.remove(&sid);
|
||||
debug_log!("conversation", "release speech={sid}");
|
||||
self.voice.release(&sid);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::SpeechStart(h) => {
|
||||
self.policy.utterance_open(h.speaker_id);
|
||||
@@ -390,16 +416,16 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.ranked(rank_id, &order, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
// The answer is heard: what its asker says next is a new request.
|
||||
// The answer is heard; while it plays, its asker may still go on (within call.continueMs).
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
self.waiting.remove(&e.id);
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
// A held request waits for its speaker's next final even after its answer ended.
|
||||
@@ -484,6 +510,7 @@ impl<V: Voice> Conversation<V> {
|
||||
fn stop_everything(&mut self, now: Instant) {
|
||||
self.wakes.clear();
|
||||
self.turns.clear();
|
||||
self.waiting.clear();
|
||||
let a = self.policy.stop();
|
||||
self.apply(a, now);
|
||||
self.chimed.clear(); // nothing is waiting any more
|
||||
@@ -516,26 +543,33 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
let a = self.policy.tick(now);
|
||||
self.apply(a, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
pub fn next_deadline(&self) -> Option<Instant> {
|
||||
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT);
|
||||
let wakes = self.wakes.values().filter_map(|(_, d)| *d);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min()
|
||||
// Not while the asker talks: their final decides.
|
||||
let held = self.waiting.values().filter(|(s, _)| !self.started.contains_key(s)).map(|(_, at)| *at);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).chain(held).min()
|
||||
}
|
||||
|
||||
fn apply(&mut self, actions: Vec<Action>, now: Instant) {
|
||||
for action in actions {
|
||||
match action {
|
||||
Action::Answer(r) => match {
|
||||
// The earcon plays first, then the answer as soon as it is written.
|
||||
// The earcon plays first, then the answer once it is written and its asker has
|
||||
// been quiet for call.answerWaitMs.
|
||||
if self.calls.earcon && !self.chimed.remove(&r.id) {
|
||||
self.voice.earcon();
|
||||
}
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at))
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at), !self.calls.answer_wait.is_zero())
|
||||
} {
|
||||
Some(sid) => {
|
||||
debug_log!("conversation", "answer req={} speaker={} speech={sid}", r.id, r.speaker);
|
||||
if !self.calls.answer_wait.is_zero() {
|
||||
self.waiting.insert(sid.clone(), (r.speaker, r.ended_at + self.calls.answer_wait));
|
||||
}
|
||||
self.by_speech.insert(sid.clone(), r.id);
|
||||
self.answers.insert(r.id, sid);
|
||||
}
|
||||
@@ -547,6 +581,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Action::StopAnswer(pid) => {
|
||||
if let Some(sid) = self.answers.remove(&pid) {
|
||||
self.by_speech.remove(&sid);
|
||||
self.waiting.remove(&sid);
|
||||
self.voice.cancel(&sid);
|
||||
}
|
||||
}
|
||||
@@ -601,7 +636,7 @@ mod tests {
|
||||
|
||||
/// The tests call Jarvis by one wake word of their own, whatever the default is.
|
||||
fn test_calls() -> CallSettings {
|
||||
CallSettings { wake_words: vec!["자비스".into()], ..CallSettings::default() }
|
||||
CallSettings { wake_words: vec!["자비스".into()], answer_wait: Duration::ZERO, ..CallSettings::default() }
|
||||
}
|
||||
use crate::discord::{Heard, HeardFinal};
|
||||
use crate::playback::PlaybackEnd;
|
||||
@@ -613,6 +648,9 @@ mod tests {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Did {
|
||||
Answer(u64, String),
|
||||
/// The answer just asked for is held until released.
|
||||
Held(String),
|
||||
Release(String),
|
||||
Cancel(String),
|
||||
StopAll,
|
||||
Earcon,
|
||||
@@ -637,10 +675,17 @@ mod tests {
|
||||
people: Rc<RefCell<Vec<u64>>>,
|
||||
}
|
||||
impl Voice for Fake {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant)) -> Option<String> {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
*self.n.borrow_mut() += 1;
|
||||
let id = format!("s{}", self.n.borrow());
|
||||
self.did.borrow_mut().push(Did::Answer(user, text.into()));
|
||||
Some(format!("s{}", self.n.borrow()))
|
||||
if hold {
|
||||
self.did.borrow_mut().push(Did::Held(id.clone()));
|
||||
}
|
||||
Some(id)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Release(id.into()));
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Cancel(id.into()));
|
||||
@@ -1163,13 +1208,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_the_answer_is_heard_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
fn once_the_answer_ended_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 2500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "고마워".into())]);
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Answer(A, "고마워".into())]);
|
||||
// Still thinking, but the speaker paused longer than call.continueMs.
|
||||
let (mut c, f, t) = alone(PolicyKind::InOrder);
|
||||
let pause = test_calls().continue_window.as_millis() as u64;
|
||||
@@ -1216,6 +1261,77 @@ mod tests {
|
||||
|
||||
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
|
||||
|
||||
/// Alone with Jarvis, answers held until the asker has been quiet for `wait_ms`.
|
||||
fn alone_waiting(kind: PolicyKind, wait_ms: u64) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
let calls = CallSettings { mode: CallMode::Addressed, earcon: false, answer_wait: Duration::from_millis(wait_ms),
|
||||
..test_calls() };
|
||||
let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(Conversation::new(f.clone(), calls, policy), f, Instant::now())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_starts_only_after_its_asker_was_quiet_for_the_wait() {
|
||||
// Live 2026-11-19: pauses inside one thought were up to 4.7 s, longer than writing an answer.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Held("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 2900)), "{kind:?}");
|
||||
c.tick(ms(t, 2800));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
c.tick(ms(t, 2900));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), None, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_while_its_asker_talks_and_their_words_join_it() {
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
assert!(f.take().is_empty(), "still talking ({kind:?})");
|
||||
c.on_event(&fin_utt(A, 2, "메모장도"), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 메모장도".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
c.tick(ms(t, 6400)); // the joined request ended at 3400
|
||||
assert_eq!(f.take(), vec![Did::Release("s2".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_is_released_when_what_followed_was_noise() {
|
||||
let (mut c, f, t) = alone_waiting(PolicyKind::Latest, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
f.take();
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_while_the_answer_plays_stops_it_and_answers_all_of_it() {
|
||||
// A pause longer than the wait: the answer started, then the speaker went on within
|
||||
// call.continueMs of their request's end.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.tick(ms(t, 2900));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
f.take();
|
||||
c.on_event(&fin(A, "제일 앞으로", 100), ms(t, 5000)); // started at 3900, 4 s after the end
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
|
||||
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
|
||||
|
||||
@@ -34,7 +34,8 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer is not heard yet. 0 turns joining off. 0-15000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer has not ended. 0 turns joining off. 0-15000. |
|
||||
| `call.answerWaitMs` | `3000` | Waiting before answering (below): a voice answer starts playing only once its asker has been quiet this long after their request ended. 0: as soon as it is written. 0-10000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
@@ -83,8 +84,9 @@ a message is complete as sent.
|
||||
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
|
||||
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
|
||||
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
|
||||
So a speaker's voice request stays open until its answer is heard (the first audio of its speech
|
||||
plays, `SpeakStart`), and while it is open their next final is joined to it when that utterance
|
||||
So a speaker's voice request stays open until its answer has ended (`SpeakEnd`; also while it
|
||||
plays, since live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer and the answer
|
||||
started in the pause), and while it is open their next final is joined to it when that utterance
|
||||
started within `call.continueMs` of the request's end (start = the final's arrival minus its
|
||||
`wait_ms` and `audio_ms`):
|
||||
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
|
||||
@@ -100,12 +102,21 @@ started within `call.continueMs` of the request's end (start = the final's arriv
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
|
||||
- Other speakers' utterances never join it. An answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
|
||||
its answer is being written (it is the request that will be answered).
|
||||
|
||||
**Waiting before answering.** So that a pause inside one thought does not get an answer, a voice
|
||||
answer is written at once but starts playing only when its asker has been quiet for
|
||||
`call.answerWaitMs` after the request ended (default 3 s: 8 of the 10 pauses inside one thought
|
||||
measured live 2026-11-19 were shorter). While the asker is talking when that time comes, the answer
|
||||
waits for that utterance's final: it joins the request (Going on, the held answer is cancelled and
|
||||
the joined one waits again from its own end), or, when it is empty or not joined, the answer plays.
|
||||
A stop stops it as always. The earcon is not held: it still tells the asker at once that Jarvis
|
||||
heard them. Answers queued behind a held one wait their turn.
|
||||
|
||||
**Earcon.** A short two-tone "띠링" (about 0.3 s, made by the app, `app/src/earcon.rs`) tells the
|
||||
speaker Jarvis heard the call, long before the answer is written: it is the first sound after a call.
|
||||
It plays right before each voice answer starts (the policy's `Answer`), and at once when a wake word
|
||||
@@ -226,12 +237,13 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
judgement) closes the utterance, and no answer within 5 s counts as no, also when the answer arrives
|
||||
after that but before the timer ran.
|
||||
- Policy actions: `Answer` → the earcon (unless it already played for this request), then
|
||||
`answer(user, text)` (the brain answers and the answer is spoken; when it
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`answer(user, text, hold)` (the brain answers and the answer is spoken; held, with a non-zero
|
||||
`call.answerWaitMs`, until `release` when its asker has been quiet for it; when it cannot be
|
||||
queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on),
|
||||
unless its speaker already started their next utterance (held).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard (no second earcon when it is joined); its
|
||||
request stays open until the answer ends (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
@@ -254,7 +266,9 @@ no and timeout, the earcon before each answer, once for a wake word alone, none
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
|
||||
new request once the answer ended or the pause is longer than `call.continueMs`, joined while the
|
||||
answer plays, answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
|
||||
and joined by what they say, released after noise, in every policy, a stop still
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
|
||||
@@ -563,15 +563,26 @@ impl DiscordCore {
|
||||
/// `SpeakStart` and `SpeakEnd` follow as for `say`), or None when not in a voice channel. It ends as
|
||||
/// failed when the brain gives no answer. `heard`: when the request's speech ended and its final
|
||||
/// arrived, passed on in `AnswerAsked`.
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
self.voice_channel_id()?;
|
||||
let id = with_playback(&self.inner, |p| p.adopt());
|
||||
let id = with_playback(&self.inner, |p| {
|
||||
let id = p.adopt();
|
||||
if hold {
|
||||
p.hold(&id);
|
||||
}
|
||||
id
|
||||
});
|
||||
let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user, ended_at: heard.0,
|
||||
final_at: heard.1 });
|
||||
self.inner.worker.reply_in(&id, user, text, "voice", true, None, ¬es_of(&self.inner, user));
|
||||
Some(id)
|
||||
}
|
||||
|
||||
/// Let an answer held by `answer(.., hold)` play.
|
||||
pub fn release(&self, id: &str) {
|
||||
with_playback(&self.inner, |p| p.release(id));
|
||||
}
|
||||
|
||||
/// Play the earcon in the voice channel, queued like speech. The request id, or None when not in a
|
||||
/// voice channel.
|
||||
pub fn earcon(&self) -> Option<String> {
|
||||
|
||||
@@ -171,7 +171,9 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
long the clip playing when it was stopped had played, timed from when it was handed to the player)
|
||||
so its voice recordings keep only what was heard (`voice_worker.spec.md`, "Recordings").
|
||||
- **Answering** (M4): `answer(user, text)` queues a request like `say` whose speech the worker makes
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. No answer
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. With
|
||||
`hold` it is written but does not start playing until `release(id)` (an answer that fails while held
|
||||
still ends at once and lets the queue go on). No answer
|
||||
from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed.
|
||||
`cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of
|
||||
an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and
|
||||
|
||||
@@ -48,6 +48,8 @@ struct Request {
|
||||
failed: bool,
|
||||
played: u32,
|
||||
started: bool,
|
||||
/// Not to start playing until released (`hold`).
|
||||
held: bool,
|
||||
}
|
||||
|
||||
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
||||
@@ -81,7 +83,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
||||
self.backend.tts(&id, text, more);
|
||||
id
|
||||
@@ -93,7 +95,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
||||
id
|
||||
}
|
||||
@@ -103,7 +105,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
||||
failed: false, played: 0, started: false });
|
||||
failed: false, played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
||||
self.pump();
|
||||
id
|
||||
@@ -118,6 +120,22 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep a request that has not started from starting until `release` (the asker may go on).
|
||||
pub fn hold(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.started) {
|
||||
r.held = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Let a held request play.
|
||||
pub fn release(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && r.held) {
|
||||
r.held = false;
|
||||
crate::debug_log!("play", "release id={id}");
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
||||
/// queue goes on.
|
||||
pub fn cancel(&mut self, id: &str) {
|
||||
@@ -207,6 +225,9 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
head.next = first;
|
||||
}
|
||||
}
|
||||
if head.held && head.chunks.contains_key(&head.next) {
|
||||
return;
|
||||
}
|
||||
if let Some(pcm) = head.chunks.remove(&head.next) {
|
||||
head.next += 1;
|
||||
head.played += 1;
|
||||
@@ -480,4 +501,28 @@ mod tests {
|
||||
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
||||
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_for_its_release_and_a_failed_one_does_not_block() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.adopt();
|
||||
p.hold(&id);
|
||||
p.audio(&id, 0, b("a0"));
|
||||
p.done(&done(&id, false));
|
||||
assert!(f.0.borrow().played.is_empty(), "written, but held");
|
||||
assert!(p.take_events().is_empty());
|
||||
p.release(&id);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
||||
// An answer that failed while held ends at once, and the next request plays.
|
||||
p.clip_ended();
|
||||
p.take_events();
|
||||
let bad = p.adopt();
|
||||
p.hold(&bad);
|
||||
let next = p.say("b", false);
|
||||
p.audio(&next, 0, b("b0"));
|
||||
p.fail(&bad);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
||||
assert_eq!(p.take_events()[0], end(&bad, 0, false, true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -454,6 +454,7 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
|
||||
stop_words,
|
||||
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
|
||||
continue_window: get_ms(call, "continueMs", "call", d.continue_window, 15000)?,
|
||||
answer_wait: get_ms(call, "answerWaitMs", "call", d.answer_wait, 10000)?,
|
||||
earcon: get_bool(call, "earcon", "call", d.earcon)?,
|
||||
},
|
||||
chat_wake_words,
|
||||
|
||||
@@ -96,6 +96,7 @@ pub fn fields() -> Vec<Field> {
|
||||
json!(["멈춰", "그만", "정지", "stop"]), "정지 명령"),
|
||||
f("call.followUpMs", Call, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.continueMs", Call, Int { min: 0, max: 15000 }, json!(5000), "끊어 말한 문장 합치기(ms)"),
|
||||
f("call.answerWaitMs", Call, Int { min: 0, max: 10000 }, json!(3000), "말이 끝나고 답하기 전 기다림(ms)"),
|
||||
f("call.earcon", Call, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Response, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
json!("latest"), "동시 대화 응답 정책"),
|
||||
|
||||
@@ -104,3 +104,4 @@
|
||||
- 2026-11-19 밤 (0.1.50 뒤): 근거 evidence/2026-11-19/stt-plan/wake/.
|
||||
- 첫마디 호출어: 힌트를 준 결과는 "비서야"를 맞게 들었는데(기사야·기서야·라비스·아비스 → 비서야·자비스), 힌트 결과는 힌트 없는 결과와 호출어 수가 같아야만 써서 버려졌음. 규칙을 바꿈: 힌트 없는 결과에 그 호출어의 비슷한 말이 있으면(글 전체가 호출어 글자 수의 3분의 1 이내, 한글은 자모로 셈) 호출어 하나를 더할 수 있음. 녹음 100개에서 5개를 바로잡고 3개는 그대로(그중 "기구"→"김비서"는 실제로 "디코"라 막은 것이 맞음), 잡음 216번(108개 × 힌트 2종)에서 더한 호출어 0. 소리로 다시 매기기(align 확률)는 맞는 쪽과 틀린 쪽을 가르지 못해 쓰지 않음.
|
||||
- 실제 디스코드(테스트 봇이 사용자 녹음을 실제 쉰 간격대로 틀음, app/tests/live_round_trip.rs going_on, 서버 CPU, LLM은 질문을 되돌려 주는 대역): 부산 날씨 3조각은 한 질문, 한 답. 계산기 6조각(쉼 2.3~4.7초)은 쉼이 답 만드는 시간보다 길어서 답이 먼저 들리기 시작했고, 들린 뒤의 말은 새 질문이 되어 중간 답 2개가 끊김. 서버 CPU라 받아 적기가 10~25초 걸려 시간은 실제와 다름(정확한 시간은 게임PC GPU에서 다시 재야 함).
|
||||
- 그래서(사용자 결정 전, 추천안 3으로 구현, 설정으로 끌 수 있음) 답은 바로 만들되 묻는 사람이 요청 끝에서 `call.answerWaitMs`(기본 3초) 동안 조용해야 말하기 시작하고, 그 사이 말하면 그 말의 결과를 기다려 합침. 답이 나오는 중에도 요청 끝에서 `call.continueMs`(5초) 안에 이어 말하면 답을 멈추고 합침(답이 끝난 뒤의 말은 새 질문). 실제 디스코드 다시 시험(같은 말과 같은 쉼을 MeloTTS로, 받아 적기 small CPU, 대역 LLM 0.8초×3): 세 묶음 모두 한 답, 중간에 들린 답 0(live3-answer-wait.log).
|
||||
|
||||
Reference in New Issue
Block a user