Compare commits
2 Commits
app-v0.1.5
...
app-v0.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b39aabd368 | ||
|
|
d7dc62c202 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.51"
|
||||
version = "0.1.52"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.51"
|
||||
version = "0.1.52"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.51"
|
||||
version = "0.1.52"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.51"
|
||||
version = "0.1.52"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.51"
|
||||
version = "0.1.52"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -308,6 +308,10 @@ impl<V: Voice> Conversation<V> {
|
||||
let window = self.calls.continue_window;
|
||||
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
|
||||
t.held = !window.is_zero() && now <= t.ended_at + window;
|
||||
// Too late to go on with a request whose answer was heard: what comes is new.
|
||||
if !t.held && t.heard {
|
||||
self.turns.remove(&h.speaker_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::Chat(m) => self.on_chat(m, now),
|
||||
@@ -428,8 +432,10 @@ impl<V: Voice> Conversation<V> {
|
||||
self.waiting.remove(&e.id);
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
// A held request waits for its speaker's next final even after its answer ended.
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
// The speaker may still go on within call.continueMs of the request's end (a short
|
||||
// answer ends inside a pause), and a held request waits for its next final anyway.
|
||||
let window = self.calls.continue_window;
|
||||
self.turns.retain(|_, t| t.id != pid || t.held || now < t.ended_at + window);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
@@ -768,10 +774,11 @@ mod tests {
|
||||
c.on_event(&start(A, 2), ms(t, 1200));
|
||||
c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 지금 몇 시야?".into())]);
|
||||
// Nothing followed the next one: it is answered alone, still with one earcon.
|
||||
// Nothing followed the next one (said after the going-on window): it is answered alone,
|
||||
// still with one earcon.
|
||||
c.on_event(&end("s1", false), ms(t, 5000));
|
||||
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 6000));
|
||||
c.tick(ms(t, 6000) + test_calls().follow_up);
|
||||
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 9500));
|
||||
c.tick(ms(t, 9500) + test_calls().follow_up);
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스".into())]);
|
||||
}
|
||||
|
||||
@@ -872,12 +879,12 @@ mod tests {
|
||||
assert!(f.take().is_empty());
|
||||
c.tick(ms(t, 900) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
// Another speaker's words do not join it.
|
||||
// Another speaker's words do not join it (A's next call comes after the going-on window).
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
c.on_event(&start(A, 2), ms(t, 4100));
|
||||
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500));
|
||||
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600));
|
||||
c.tick(ms(t, 4500) + wait);
|
||||
c.on_event(&start(A, 2), ms(t, 6000));
|
||||
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 6400));
|
||||
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 6500));
|
||||
c.tick(ms(t, 6400) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
}
|
||||
|
||||
@@ -959,7 +966,7 @@ mod tests {
|
||||
// Not yet in the channel list (it can lag): the speaker is there all the same.
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
*f.people.borrow_mut() = vec![];
|
||||
c.on_event(&fin(A, "오늘 부산에 비 온대", 100), ms(t, 2100));
|
||||
c.on_event(&fin(A, "오늘 부산에 비 온대", 100), ms(t, 6200)); // after the going-on window
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "오늘 부산에 비 온대".into())]);
|
||||
}
|
||||
|
||||
@@ -1208,12 +1215,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_the_answer_ended_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
fn once_the_pause_is_longer_than_the_window_the_next_words_are_a_new_request() {
|
||||
// The answer ended; the next utterance starts 5.3 s after the request ended (window 5 s).
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 3500));
|
||||
c.on_event(&start(A, 2), ms(t, 5200));
|
||||
c.on_event(&fin_utt(A, 2, "고마워"), ms(t, 6300)); // started at 5200
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Answer(A, "고마워".into())]);
|
||||
// Still thinking, but the speaker paused longer than call.continueMs.
|
||||
let (mut c, f, t) = alone(PolicyKind::InOrder);
|
||||
@@ -1304,6 +1313,61 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_after_a_short_answer_ended_within_the_window_is_joined() {
|
||||
// Request ends at 0, its answer plays from 3 s to 4 s, the speaker goes on at 4.7 s.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 0), ms(t, 0));
|
||||
c.tick(ms(t, 3000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
f.take();
|
||||
c.on_event(&start(A, 2), ms(t, 4700));
|
||||
c.on_event(&fin_utt(A, 2, "제일 앞으로"), ms(t, 5800)); // started at 4700
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()), Did::Held("s2".into())],
|
||||
"{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn after_the_answer_ended_a_late_start_noise_a_stop_or_someone_else_join_nothing() {
|
||||
for kind in ALL {
|
||||
let after = |c: &mut Conversation<Fake>, f: &Fake, t: Instant| {
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 0), ms(t, 0));
|
||||
c.tick(ms(t, 3000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
f.take();
|
||||
};
|
||||
// Started at 5.3 s: a new request.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
c.on_event(&start(A, 2), ms(t, 5300));
|
||||
c.on_event(&fin_utt(A, 2, "메모장 켜 줘"), ms(t, 6400));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "메모장 켜 줘".into()), Did::Held("s2".into())], "{kind:?}");
|
||||
// Someone else (B, alone with A in the channel) never joins it.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
*f.people.borrow_mut() = vec![A, B]; // B joined the channel
|
||||
c.on_event(&fin(B, "자비스 나도", 100), ms(t, 4800));
|
||||
assert_eq!(f.take(), vec![Did::Answer(B, "자비스 나도".into()), Did::Held("s2".into())], "{kind:?}");
|
||||
// Noise: nothing joined, nothing asked again.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
c.on_event(&start(A, 2), ms(t, 4700));
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 5800));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
// A stop stops, and asks nothing again.
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
after(&mut c, &f, t);
|
||||
c.on_event(&start(A, 2), ms(t, 4700));
|
||||
c.on_event(&fin_utt(A, 2, "자비스 멈춰"), ms(t, 5800));
|
||||
assert_eq!(f.take(), vec![Did::StopAll], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_is_released_when_what_followed_was_noise() {
|
||||
let (mut c, f, t) = alone_waiting(PolicyKind::Latest, 3000);
|
||||
|
||||
@@ -34,7 +34,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer has not ended. 0 turns joining off. 0-15000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, also when its answer already played or ended. 0 turns joining off. 0-15000. |
|
||||
| `call.answerWaitMs` | `3000` | Waiting before answering (below): a voice answer starts playing only once its asker has been quiet this long after their request ended. 0: as soon as it is written. 0-10000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
@@ -84,9 +84,9 @@ a message is complete as sent.
|
||||
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
|
||||
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
|
||||
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
|
||||
So a speaker's voice request stays open until its answer has ended (`SpeakEnd`; also while it
|
||||
plays, since live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer and the answer
|
||||
started in the pause), and while it is open their next final is joined to it when that utterance
|
||||
So a speaker's voice request stays open for `call.continueMs` after it ended, whatever its answer
|
||||
does meanwhile (live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer: it started,
|
||||
and a short one even ended, in the pause), and their next final is joined to it when that utterance
|
||||
started within `call.continueMs` of the request's end (start = the final's arrival minus its
|
||||
`wait_ms` and `audio_ms`):
|
||||
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
|
||||
@@ -102,8 +102,9 @@ started within `call.continueMs` of the request's end (start = the final's arriv
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. An answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- Other speakers' utterances never join it. The window passing (once the answer ended; a held
|
||||
request waits for its final), a missed request, a stop, leaving and a lost worker end it; then
|
||||
what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
|
||||
its answer is being written (it is the request that will be answered).
|
||||
@@ -243,9 +244,11 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard (no second earcon when it is joined); its
|
||||
request stays open until the answer ends (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
request stays open (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done; its request stays open while
|
||||
`call.continueMs` from the request's end has not passed (or while held), and a `SpeechStart` of
|
||||
its speaker after that window drops it. The end of a speech the loop already cancelled starts
|
||||
nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
`Left`: the same stop. The core sends `Stopped` before the `SpeakEnd`s of the speech it stopped, so a
|
||||
cancelled answer's end never starts the next waiting request.
|
||||
@@ -266,8 +269,8 @@ no and timeout, the earcon before each answer, once for a wake word alone, none
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer ended or the pause is longer than `call.continueMs`, joined while the
|
||||
answer plays, answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
|
||||
new request once the pause is longer than `call.continueMs`, joined while the answer plays and after
|
||||
a short answer ended within that window (not another speaker, noise or a stop), answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
|
||||
and joined by what they say, released after noise, in every policy, a stop still
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
|
||||
@@ -105,3 +105,4 @@
|
||||
- 첫마디 호출어: 힌트를 준 결과는 "비서야"를 맞게 들었는데(기사야·기서야·라비스·아비스 → 비서야·자비스), 힌트 결과는 힌트 없는 결과와 호출어 수가 같아야만 써서 버려졌음. 규칙을 바꿈: 힌트 없는 결과에 그 호출어의 비슷한 말이 있으면(글 전체가 호출어 글자 수의 3분의 1 이내, 한글은 자모로 셈) 호출어 하나를 더할 수 있음. 녹음 100개에서 5개를 바로잡고 3개는 그대로(그중 "기구"→"김비서"는 실제로 "디코"라 막은 것이 맞음), 잡음 216번(108개 × 힌트 2종)에서 더한 호출어 0. 소리로 다시 매기기(align 확률)는 맞는 쪽과 틀린 쪽을 가르지 못해 쓰지 않음.
|
||||
- 실제 디스코드(테스트 봇이 사용자 녹음을 실제 쉰 간격대로 틀음, app/tests/live_round_trip.rs going_on, 서버 CPU, LLM은 질문을 되돌려 주는 대역): 부산 날씨 3조각은 한 질문, 한 답. 계산기 6조각(쉼 2.3~4.7초)은 쉼이 답 만드는 시간보다 길어서 답이 먼저 들리기 시작했고, 들린 뒤의 말은 새 질문이 되어 중간 답 2개가 끊김. 서버 CPU라 받아 적기가 10~25초 걸려 시간은 실제와 다름(정확한 시간은 게임PC GPU에서 다시 재야 함).
|
||||
- 그래서(사용자 결정 전, 추천안 3으로 구현, 설정으로 끌 수 있음) 답은 바로 만들되 묻는 사람이 요청 끝에서 `call.answerWaitMs`(기본 3초) 동안 조용해야 말하기 시작하고, 그 사이 말하면 그 말의 결과를 기다려 합침. 답이 나오는 중에도 요청 끝에서 `call.continueMs`(5초) 안에 이어 말하면 답을 멈추고 합침(답이 끝난 뒤의 말은 새 질문). 실제 디스코드 다시 시험(같은 말과 같은 쉼을 MeloTTS로, 받아 적기 small CPU, 대역 LLM 0.8초×3): 세 묶음 모두 한 답, 중간에 들린 답 0(live3-answer-wait.log).
|
||||
- 0.1.52: 짧은 답이 쉼 안에서 끝나 버리는 경우(답 3~4초, 이어 말하기 4.7초)도 합침. 요청은 끝나고 `call.continueMs` 동안 답이 끝났어도 열려 있음. 대조: 5.3초에 시작한 말, 다른 사람, 잡음, "멈춰"는 합치지 않음(네 정책 모두 단위 테스트).
|
||||
|
||||
Reference in New Issue
Block a user