Compare commits

...

2 Commits

Author SHA1 Message Date
javis-bot
b39aabd368 chore(app): version 0.1.52
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 22:20:33 +09:00
javis-bot
d7dc62c202 fix(conversation): going on joins within the window even after a short answer ended
With the 3 s wait a short answer can play and end inside the speaker's pause (answer
3-4 s, next piece at 4.7 s): the request was dropped at SpeakEnd and the next piece became
a new request. A request now stays open for call.continueMs after it ended whatever its
answer does; a SpeechStart after the window drops it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 22:20:33 +09:00
7 changed files with 98 additions and 30 deletions

2
app/Cargo.lock generated
View File

@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.51"
version = "0.1.52"
dependencies = [
"async-trait",
"base64 0.22.1",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.1.51"
version = "0.1.52"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"

4
app/shell/Cargo.lock generated
View File

@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.51"
version = "0.1.52"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2666,7 +2666,7 @@ dependencies = [
[[package]]
name = "jarvis-shell"
version = "0.1.51"
version = "0.1.52"
dependencies = [
"jarvis-app",
"serde_json",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-shell"
version = "0.1.51"
version = "0.1.52"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"

View File

@@ -308,6 +308,10 @@ impl<V: Voice> Conversation<V> {
let window = self.calls.continue_window;
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
t.held = !window.is_zero() && now <= t.ended_at + window;
// Too late to go on with a request whose answer was heard: what comes is new.
if !t.held && t.heard {
self.turns.remove(&h.speaker_id);
}
}
}
CoreEvent::Chat(m) => self.on_chat(m, now),
@@ -428,8 +432,10 @@ impl<V: Voice> Conversation<V> {
self.waiting.remove(&e.id);
if let Some(pid) = self.by_speech.remove(&e.id) {
self.answers.remove(&pid);
// A held request waits for its speaker's next final even after its answer ended.
self.turns.retain(|_, t| t.id != pid || t.held);
// The speaker may still go on within call.continueMs of the request's end (a short
// answer ends inside a pause), and a held request waits for its next final anyway.
let window = self.calls.continue_window;
self.turns.retain(|_, t| t.id != pid || t.held || now < t.ended_at + window);
for t in self.turns.values_mut().filter(|t| t.id == pid) {
t.heard = true;
}
@@ -768,10 +774,11 @@ mod tests {
c.on_event(&start(A, 2), ms(t, 1200));
c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 3000));
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 지금 몇 시야?".into())]);
// Nothing followed the next one: it is answered alone, still with one earcon.
// Nothing followed the next one (said after the going-on window): it is answered alone,
// still with one earcon.
c.on_event(&end("s1", false), ms(t, 5000));
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 6000));
c.tick(ms(t, 6000) + test_calls().follow_up);
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 9500));
c.tick(ms(t, 9500) + test_calls().follow_up);
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스".into())]);
}
@@ -872,12 +879,12 @@ mod tests {
assert!(f.take().is_empty());
c.tick(ms(t, 900) + wait);
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
// Another speaker's words do not join it.
// Another speaker's words do not join it (A's next call comes after the going-on window).
c.on_event(&end("s1", false), ms(t, 4000));
c.on_event(&start(A, 2), ms(t, 4100));
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500));
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600));
c.tick(ms(t, 4500) + wait);
c.on_event(&start(A, 2), ms(t, 6000));
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 6400));
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 6500));
c.tick(ms(t, 6400) + wait);
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
}
@@ -959,7 +966,7 @@ mod tests {
// Not yet in the channel list (it can lag): the speaker is there all the same.
c.on_event(&end("s1", false), ms(t, 2000));
*f.people.borrow_mut() = vec![];
c.on_event(&fin(A, "오늘 부산에 비 온대", 100), ms(t, 2100));
c.on_event(&fin(A, "오늘 부산에 비 온대", 100), ms(t, 6200)); // after the going-on window
assert_eq!(f.take(), vec![Did::Answer(A, "오늘 부산에 비 온대".into())]);
}
@@ -1208,12 +1215,14 @@ mod tests {
}
#[test]
fn once_the_answer_ended_or_the_pause_is_long_the_next_words_are_a_new_request() {
fn once_the_pause_is_longer_than_the_window_the_next_words_are_a_new_request() {
// The answer ended; the next utterance starts 5.3 s after the request ended (window 5 s).
let (mut c, f, t) = alone(PolicyKind::Latest);
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
c.on_event(&speak_start("s1"), ms(t, 1500));
c.on_event(&end("s1", false), ms(t, 2000));
c.on_event(&fin(A, "고마워", 100), ms(t, 3500));
c.on_event(&start(A, 2), ms(t, 5200));
c.on_event(&fin_utt(A, 2, "고마워"), ms(t, 6300)); // started at 5200
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Answer(A, "고마워".into())]);
// Still thinking, but the speaker paused longer than call.continueMs.
let (mut c, f, t) = alone(PolicyKind::InOrder);
@@ -1304,6 +1313,61 @@ mod tests {
}
}
#[test]
fn going_on_after_a_short_answer_ended_within_the_window_is_joined() {
// Request ends at 0, its answer plays from 3 s to 4 s, the speaker goes on at 4.7 s.
for kind in ALL {
let (mut c, f, t) = alone_waiting(kind, 3000);
c.on_event(&fin(A, "계산기 켜 줘", 0), ms(t, 0));
c.tick(ms(t, 3000));
c.on_event(&speak_start("s1"), ms(t, 3000));
c.on_event(&end("s1", false), ms(t, 4000));
f.take();
c.on_event(&start(A, 2), ms(t, 4700));
c.on_event(&fin_utt(A, 2, "제일 앞으로"), ms(t, 5800)); // started at 4700
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()), Did::Held("s2".into())],
"{kind:?}");
assert_eq!(c.missed().count(), 0, "{kind:?}");
}
}
#[test]
fn after_the_answer_ended_a_late_start_noise_a_stop_or_someone_else_join_nothing() {
for kind in ALL {
let after = |c: &mut Conversation<Fake>, f: &Fake, t: Instant| {
c.on_event(&fin(A, "계산기 켜 줘", 0), ms(t, 0));
c.tick(ms(t, 3000));
c.on_event(&speak_start("s1"), ms(t, 3000));
c.on_event(&end("s1", false), ms(t, 4000));
f.take();
};
// Started at 5.3 s: a new request.
let (mut c, f, t) = alone_waiting(kind, 3000);
after(&mut c, &f, t);
c.on_event(&start(A, 2), ms(t, 5300));
c.on_event(&fin_utt(A, 2, "메모장 켜 줘"), ms(t, 6400));
assert_eq!(f.take(), vec![Did::Answer(A, "메모장 켜 줘".into()), Did::Held("s2".into())], "{kind:?}");
// Someone else (B, alone with A in the channel) never joins it.
let (mut c, f, t) = alone_waiting(kind, 3000);
after(&mut c, &f, t);
*f.people.borrow_mut() = vec![A, B]; // B joined the channel
c.on_event(&fin(B, "자비스 나도", 100), ms(t, 4800));
assert_eq!(f.take(), vec![Did::Answer(B, "자비스 나도".into()), Did::Held("s2".into())], "{kind:?}");
// Noise: nothing joined, nothing asked again.
let (mut c, f, t) = alone_waiting(kind, 3000);
after(&mut c, &f, t);
c.on_event(&start(A, 2), ms(t, 4700));
c.on_event(&fin_utt(A, 2, " "), ms(t, 5800));
assert!(f.take().is_empty(), "{kind:?}");
// A stop stops, and asks nothing again.
let (mut c, f, t) = alone_waiting(kind, 3000);
after(&mut c, &f, t);
c.on_event(&start(A, 2), ms(t, 4700));
c.on_event(&fin_utt(A, 2, "자비스 멈춰"), ms(t, 5800));
assert_eq!(f.take(), vec![Did::StopAll], "{kind:?}");
}
}
#[test]
fn a_held_answer_is_released_when_what_followed_was_noise() {
let (mut c, f, t) = alone_waiting(PolicyKind::Latest, 3000);

View File

@@ -34,7 +34,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer has not ended. 0 turns joining off. 0-15000. |
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, also when its answer already played or ended. 0 turns joining off. 0-15000. |
| `call.answerWaitMs` | `3000` | Waiting before answering (below): a voice answer starts playing only once its asker has been quiet this long after their request ended. 0: as soon as it is written. 0-10000. |
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
@@ -84,9 +84,9 @@ a message is complete as sent.
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
So a speaker's voice request stays open until its answer has ended (`SpeakEnd`; also while it
plays, since live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer and the answer
started in the pause), and while it is open their next final is joined to it when that utterance
So a speaker's voice request stays open for `call.continueMs` after it ended, whatever its answer
does meanwhile (live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer: it started,
and a short one even ended, in the pause), and their next final is joined to it when that utterance
started within `call.continueMs` of the request's end (start = the final's arrival minus its
`wait_ms` and `audio_ms`):
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
@@ -102,8 +102,9 @@ started within `call.continueMs` of the request's end (start = the final's arriv
transcription), the request is held open for that utterance's final, also if its answer starts
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
empty final or a stop ends the hold, and nothing is asked again.
- Other speakers' utterances never join it. An answer that ended, a missed
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
- Other speakers' utterances never join it. The window passing (once the answer ended; a held
request waits for its final), a missed request, a stop, leaving and a lost worker end it; then
what the speaker says is a new request.
- The joined request gets no second earcon when the first one's already played.
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
its answer is being written (it is the request that will be answered).
@@ -243,9 +244,11 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
the missed list (the last 100: speaker, reason, time and length, never the text).
- `SpeakStart` of an answer's speech: the answer is heard (no second earcon when it is joined); its
request stays open until the answer ends (Going on).
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
cancelled starts nothing.
request stays open (Going on).
- `SpeakEnd` of an answer's speech is that answer's done; its request stays open while
`call.continueMs` from the request's end has not passed (or while held), and a `SpeechStart` of
its speaker after that window drops it. The end of a speech the loop already cancelled starts
nothing.
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
`Left`: the same stop. The core sends `Stopped` before the `SpeakEnd`s of the speech it stopped, so a
cancelled answer's end never starts the next waiting request.
@@ -266,8 +269,8 @@ no and timeout, the earcon before each answer, once for a wake word alone, none
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
answering someone else, the joined text while going on), going on (pieces joined into one answer in
every policy without a missed record, a piece without a wake word joined, other speakers never, a
new request once the answer ended or the pause is longer than `call.continueMs`, joined while the
answer plays, answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
new request once the pause is longer than `call.continueMs`, joined while the answer plays and after
a short answer ended within that window (not another speaker, noise or a stop), answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
and joined by what they say, released after noise, in every policy, a stop still
stopping, no second earcon, an utterance started before the answer played or before the first final
came still joined, also after a short answer ended, in every policy; an empty final or a stop

View File

@@ -105,3 +105,4 @@
- 첫마디 호출어: 힌트를 준 결과는 "비서야"를 맞게 들었는데(기사야·기서야·라비스·아비스 → 비서야·자비스), 힌트 결과는 힌트 없는 결과와 호출어 수가 같아야만 써서 버려졌음. 규칙을 바꿈: 힌트 없는 결과에 그 호출어의 비슷한 말이 있으면(글 전체가 호출어 글자 수의 3분의 1 이내, 한글은 자모로 셈) 호출어 하나를 더할 수 있음. 녹음 100개에서 5개를 바로잡고 3개는 그대로(그중 "기구"→"김비서"는 실제로 "디코"라 막은 것이 맞음), 잡음 216번(108개 × 힌트 2종)에서 더한 호출어 0. 소리로 다시 매기기(align 확률)는 맞는 쪽과 틀린 쪽을 가르지 못해 쓰지 않음.
- 실제 디스코드(테스트 봇이 사용자 녹음을 실제 쉰 간격대로 틀음, app/tests/live_round_trip.rs going_on, 서버 CPU, LLM은 질문을 되돌려 주는 대역): 부산 날씨 3조각은 한 질문, 한 답. 계산기 6조각(쉼 2.3~4.7초)은 쉼이 답 만드는 시간보다 길어서 답이 먼저 들리기 시작했고, 들린 뒤의 말은 새 질문이 되어 중간 답 2개가 끊김. 서버 CPU라 받아 적기가 10~25초 걸려 시간은 실제와 다름(정확한 시간은 게임PC GPU에서 다시 재야 함).
- 그래서(사용자 결정 전, 추천안 3으로 구현, 설정으로 끌 수 있음) 답은 바로 만들되 묻는 사람이 요청 끝에서 `call.answerWaitMs`(기본 3초) 동안 조용해야 말하기 시작하고, 그 사이 말하면 그 말의 결과를 기다려 합침. 답이 나오는 중에도 요청 끝에서 `call.continueMs`(5초) 안에 이어 말하면 답을 멈추고 합침(답이 끝난 뒤의 말은 새 질문). 실제 디스코드 다시 시험(같은 말과 같은 쉼을 MeloTTS로, 받아 적기 small CPU, 대역 LLM 0.8초×3): 세 묶음 모두 한 답, 중간에 들린 답 0(live3-answer-wait.log).
- 0.1.52: 짧은 답이 쉼 안에서 끝나 버리는 경우(답 3~4초, 이어 말하기 4.7초)도 합침. 요청은 끝나고 `call.continueMs` 동안 답이 끝났어도 열려 있음. 대조: 5.3초에 시작한 말, 다른 사람, 잡음, "멈춰"는 합치지 않음(네 정책 모두 단위 테스트).