fix: recordings keep only what was played, newest waiting request kept, video pages opened play

- Playback tells the worker how much of each request was played (tts_played); the worker keeps a
  spoken answer's recording only that far (an answer stopped mid-way no longer keeps its unheard
  rest, and is marked stopped); nothing played or never reported: no file.
- Old recording day folders are also removed with the first recording of each new day.
- response.interrupt off, latest: a request whose final comes after a newer one's is replaced
  instead of replacing it.
- botBrowser presses play on a page's paused main player also after open, not only after a click.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-16 15:56:40 +09:00
parent fdcaae4820
commit ec2672cd4a
12 changed files with 176 additions and 33 deletions

View File

@@ -94,7 +94,7 @@ Settings (section `response`):
| `response.collectMs` | `700` | `important`: how long candidates are collected before choosing. 0-3000. |
| `response.settleMs` | `800` | Longest wait for an earlier-ended utterance that is still being transcribed. 0-3000. |
| `response.priorities` | `{}` | `important`: user ID → priority (higher first, default 0). |
| `response.interrupt` | `true` | `latest` and `important`: a new request may stop the running answer (below). `false`: the running answer is always spoken to its end and the new request waits for it (`latest`: the newest waiting request is answered next and older waiting ones are missed, `replaced`; `important`: the best candidate collected meanwhile is answered next, no ranking against the running answer). A voice stop still stops it. `first` and `inOrder` never interrupt. |
| `response.interrupt` | `true` | `latest` and `important`: a new request may stop the running answer (below). `false`: the running answer is always spoken to its end and the new request waits for it (`latest`: the newest waiting request is answered next and older waiting ones are missed, `replaced`, also a request whose final arrives after a newer one's; `important`: the best candidate collected meanwhile is answered next, no ranking against the running answer). A voice stop still stops it. `first` and `inOrder` never interrupt. |
Terms:

View File

@@ -167,7 +167,8 @@ and the late "left" update cancelled the new join (seen on real Discord).
sends becomes one clip (a 48 kHz stereo WAV played through songbird without resampling); clips play
in sentence order as soon as they arrive, and requests play first come, first served. A sentence that
failed is skipped and the rest still play. `SpeakEnd` reports how many sentences played and whether
the request was cancelled or failed.
the request was cancelled or failed; the worker is told the same (`tts_played`) so its voice
recordings keep only what was heard (`voice_worker.spec.md`, "Recordings").
- **Answering** (M4): `answer(user, text)` queues a request like `say` whose speech the worker makes
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. No answer
from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed.

View File

@@ -13,6 +13,9 @@ pub trait TtsBackend {
/// Send text for request `id`; false when the worker is not connected (the request then fails).
fn tts(&self, id: &str, text: &str, more: bool) -> bool;
fn tts_cancel(&self, id: &str);
/// Request `id` ended after `played` of its sentences started playing (the worker's recording
/// keeps that much, voice_worker.spec.md, "Recordings").
fn tts_played(&self, id: &str, played: u32, cancelled: bool);
}
/// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes.
@@ -125,8 +128,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
self.sink.stop();
}
crate::debug_log!("play", "cancel id={} played={}", r.id, r.played);
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: true,
failed: false }));
self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: true, failed: false });
self.pump();
}
@@ -172,13 +174,18 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
self.sink.stop();
}
for r in &stopped {
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true,
failed: false }));
self.ended(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true, failed: false });
}
crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len());
stopped.len()
}
/// A request ended: tell the worker how much of it was played, then the owner.
fn ended(&mut self, end: PlaybackEnd) {
self.backend.tts_played(&end.id, end.played, end.cancelled);
self.events.push(PlaybackEvent::End(end));
}
fn pump(&mut self) {
while !self.playing {
let Some(head) = self.queue.front_mut() else { return };
@@ -204,8 +211,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
}
let r = self.queue.pop_front().expect("head exists");
crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed);
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: false,
failed: r.failed }));
self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: false, failed: r.failed });
}
}
}
@@ -221,6 +227,7 @@ mod tests {
cancelled: Vec<String>,
played: Vec<String>,
stops: usize,
reported: Vec<(String, u32, bool)>,
}
#[derive(Clone, Default)]
struct Fake(Rc<RefCell<Log>>);
@@ -232,6 +239,9 @@ mod tests {
fn tts_cancel(&self, id: &str) {
self.0.borrow_mut().cancelled.push(id.into());
}
fn tts_played(&self, id: &str, played: u32, cancelled: bool) {
self.0.borrow_mut().reported.push((id.into(), played, cancelled));
}
}
impl ClipSink for Fake {
fn play(&mut self, pcm: Vec<u8>) {
@@ -274,6 +284,25 @@ mod tests {
assert!(!p.busy());
}
#[test]
fn the_worker_is_told_how_much_of_each_request_was_played() {
let (mut p, f) = setup();
let a = p.say("하나. 둘.", false);
p.audio(&a, 0, b("a0"));
p.audio(&a, 1, b("a1"));
p.done(&done(&a, false));
p.clip_ended();
p.clip_ended();
let c = p.say("셋. 넷.", false);
p.audio(&c, 0, b("c0"));
p.audio(&c, 1, b("c1"));
p.done(&done(&c, false));
p.cancel(&c); // stopped while its first sentence played
let d = p.say("다섯.", false);
p.stop(); // stopped before any sound
assert_eq!(f.0.borrow().reported, vec![(a, 2, false), (c, 1, true), (d, 0, true)]);
}
#[test]
fn requests_play_in_the_order_they_were_made() {
let (mut p, f) = setup();

View File

@@ -166,7 +166,10 @@ impl Policy {
PolicyKind::First => out.push(Action::Missed(req, MissReason::Busy)),
PolicyKind::InOrder => self.enqueue(req, &mut out),
PolicyKind::Latest => {
if req.ended_at > cur.ended_at {
// Newer than the running answer and than every waiting request (only possible
// without interrupting: otherwise the newest is already the running one).
let newest = self.waiting.iter().map(|r| r.ended_at).max().map_or(cur.ended_at, |w| w.max(cur.ended_at));
if req.ended_at > newest {
if self.s.interrupt {
self.interrupt(&mut out);
}
@@ -567,6 +570,17 @@ mod tests {
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![c]);
}
#[test]
fn without_interrupting_latest_keeps_the_newest_waiting_request_whatever_order_finals_arrive_in() {
let mut t = T::with(PolicySettings { kind: PolicyKind::Latest, interrupt: false, ..PolicySettings::default() });
let (a, _) = t.call(A, true, 0, 100);
let (b, _) = t.call(B, true, 700, 800);
// C ended before B but its final came later: B stays, C is replaced.
let (c, out) = t.call(C, true, 500, 900);
assert_eq!(missed(&out), vec![(c, MissReason::Replaced)]);
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![b]);
}
#[test]
fn without_interrupting_important_waits_for_the_answer_even_for_a_better_request() {
let s = PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO, interrupt: false,

View File

@@ -564,6 +564,10 @@ impl TtsBackend for WorkerClient {
fn tts_cancel(&self, id: &str) {
self.send_json(json!({"type": "tts_cancel", "id": id}));
}
fn tts_played(&self, id: &str, played: u32, cancelled: bool) {
self.send_json(json!({"type": "tts_played", "id": id, "played": played, "cancelled": cancelled}));
}
}
fn mark_disconnected(inner: &Inner) {

View File

@@ -224,6 +224,7 @@ class BrowserSession:
raise BrowserError(f"the page did not open ({r['errorText']})")
self._settle(started=True)
self._wait_media()
self._press_play()
return self.read()
def click(self, target: str) -> Dict[str, Any]:
@@ -232,15 +233,19 @@ class BrowserSession:
raise BrowserError("no link or button with that label")
self._settle()
self._wait_media()
# Clicked through to a video page whose player did not start: look again and press play.
if self._eval("location.href") != before:
self._press_play()
return self.read()
def _press_play(self) -> None:
"""On a page just reached whose main player did not start: look again and press play."""
try:
if self._eval("location.href") != before and self._eval(_PLAY, gesture=True):
if self._eval(_PLAY, gesture=True):
debug_log("bot browser: pressed play on the page's paused player", "browser")
self._wait_media()
except BrowserError as e:
if "too long" in str(e):
raise
return self.read()
def type(self, text: str, field: Optional[str] = None, submit: bool = False) -> Dict[str, Any]:
if not self._eval(_FOCUS % json.dumps(field or "")):

View File

@@ -54,10 +54,12 @@ when muted; no line without any), text, and the numbered links and buttons, all
After `open` and `click`, a page whose sound or video has not started yet (none plays and one is at
0:00) gets up to 3 s to start before it is read, so the answer says truly whether it plays (a
video page's player starts a moment after the page loads; the bot browser lets pages start sound,
admin.spec.md, "Bot browser"). When a `click` led to another address and the page's main player (a
video at least half the window wide, with a source) is still paused at 0:00 after that wait, the tool
looks again and starts it (`play()` with a user gesture, as the user's own click on the video would
have), then waits for it once more; `open` and `read` leave a page's player as the page has it.
admin.spec.md, "Bot browser"). When an `open` (or a `read` with an address), or a `click` that led to
another address, reached a page whose main player (a video at least half the window wide, with a
source) is still paused at 0:00 after that wait, the tool looks again and starts it (`play()` with a
user gesture, as the user's own click on the video would have), then waits for it once more; the
Media line then says whether it truly plays. Sound or video that is not the main player (an audio
element, a small video) and a click that stays on the same page leave it as the page has it.
(Live, 2026-11-16: a video page clicked for "play X" stayed paused behind an ad and the answer said
it played.) Failures come back as `Error: ...` with `success=False`.

View File

@@ -3,7 +3,7 @@
Off unless the app sets JARVIS_RECORDINGS_DIR (setting `voice.record`). Each finished utterance and
each spoken answer becomes a WAV file in a folder per local day, with one JSON line per file in that
day's `index.jsonl` (time, who, text). Day folders older than KEEP_DAYS are removed when the worker
starts. Writing happens on a thread of its own and a failure never stops the voice path.
starts and with the first recording of each new day. Writing happens on a thread of its own and a failure never stops the voice path.
"""
from __future__ import annotations
@@ -36,7 +36,9 @@ class Recorder:
self.keep_days = keep_days
self.root.mkdir(parents=True, exist_ok=True)
self._pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="recordings")
self.purge(time.time() if now is None else now)
now = time.time() if now is None else now
self._purged_day = date.fromtimestamp(now)
self.purge(now)
def purge(self, now: float) -> int:
"""Remove day folders older than keep_days (by their name, the local day)."""
@@ -58,7 +60,7 @@ class Recorder:
{"kind": "said", "who": speaker, "name": name, "channel": channel, "text": text})
def spoke(self, ts: float, rid: str, text: str, pcm: np.ndarray, rate: int, cancelled: bool) -> None:
"""A spoken answer: int16 mono samples at `rate`, as far as it was made."""
"""A spoken answer: int16 mono samples at `rate`, as far as the app played it."""
self._submit(ts, "jarvis", np.asarray(pcm, dtype="<i2"), rate,
{"kind": "spoke", "who": "jarvis", "id": rid, "text": text, "cancelled": cancelled})
@@ -73,6 +75,9 @@ class Recorder:
def _write(self, ts: float, tag: str, pcm: np.ndarray, rate: int, entry: dict) -> None:
try:
if date.fromtimestamp(ts) != self._purged_day: # a new day while running
self._purged_day = date.fromtimestamp(ts)
self.purge(ts)
lt = time.localtime(ts)
day = self.root / time.strftime("%Y-%m-%d", lt)
day.mkdir(parents=True, exist_ok=True)

View File

@@ -246,7 +246,8 @@ class Session:
self.names: dict[str, str] = {}
self.speakers: dict[str, _Speaker] = {}
self.tts_requests: dict[str, TtsRequest] = {}
# With recordings on: each spoken answer's text and mono samples until it is done.
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
# app said how much of it was played (`tts_played`).
self.tts_recorded: dict[str, dict] = {}
# Spoken replies whose speech was cancelled while the brain was still answering.
self.cancelled_replies: set[str] = set()
@@ -390,7 +391,7 @@ class Session:
kind = msg.get("type")
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
"tts_speed": self._on_tts_speed,
"tts_speed": self._on_tts_speed, "tts_played": self._on_tts_played,
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
@@ -462,8 +463,11 @@ class Session:
self.error("bad_message", "this tts id is already closed", id=rid)
return
if self.worker.recorder is not None:
rec = self.tts_recorded.setdefault(rid, {"text": "", "pcm": [], "at": None})
rec = self.tts_recorded.setdefault(rid, {"text": "", "pcm": [], "at": None, "rate": req.rate,
"made": False, "played": None})
rec["text"] += text
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
self.tts_recorded.pop(next(iter(self.tts_recorded)))
tts.feed(req, text, more)
def _on_tts_cancel(self, msg: dict) -> None:
@@ -959,13 +963,34 @@ class Session:
def _tts_done(self, req: TtsRequest) -> None:
self.tts_requests.pop(req.id, None)
rec = self.tts_recorded.pop(req.id, None)
if rec is not None and rec["pcm"] and self.worker.recorder is not None:
self.worker.recorder.spoke(rec["at"], req.id, rec["text"], np.concatenate(rec["pcm"]), req.rate,
req.cancelled or req.failed)
rec = self.tts_recorded.get(req.id)
if rec is not None:
rec["made"] = True
self._record_spoken(req.id)
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
"failed": req.failed})
self._delivered(req.id, not (req.cancelled or req.failed))
def _on_tts_played(self, msg: dict) -> None:
"""How much of a spoken answer the app played: the recording keeps that much."""
rec = self.tts_recorded.get(msg.get("id"))
played = msg.get("played")
if rec is None or isinstance(played, bool) or not isinstance(played, int) or played < 0:
return
rec["played"], rec["cancelled"] = played, bool(msg.get("cancelled"))
self._record_spoken(msg["id"])
def _record_spoken(self, rid: str) -> None:
"""Keep a spoken answer once it is made and the app said how much of it was heard (whole
sentences, the last one maybe cut off by a stop)."""
rec = self.tts_recorded.get(rid)
if rec is None or not rec["made"] or rec["played"] is None:
return
self.tts_recorded.pop(rid)
heard = rec["pcm"][:rec["played"]]
if heard and self.worker.recorder is not None:
self.worker.recorder.spoke(rec["at"], rid, rec["text"], np.concatenate(heard), rec["rate"],
rec["cancelled"])
def _tts_error(self, req: TtsRequest, message: str) -> None:
self.error("tts_failed", message, id=req.id)

View File

@@ -165,6 +165,7 @@ line (-0.6) would lose more real words than it saves.
-> {"type": "tts", "id": "r1", "text": "...", "more": false} // more=true: more text follows for r1
-> {"type": "tts_cancel", "id": "r1"}
-> {"type": "tts_speed", "speed": 1.8} // 0.5-2.0, from the next sentence on
-> {"type": "tts_played", "id": "r1", "played": 1, "cancelled": true} // how much the app played (Recordings)
<- binary 0x02 | u8 n | id (n bytes UTF-8) | u32 LE seq | PCM16LE in the hello "tts" format
<- {"type": "tts_done", "id": "r1", "chunks": 3, "cancelled": false, "failed": false}
```
@@ -481,15 +482,20 @@ sets `JARVIS_RECORDINGS_DIR` (setting `voice.record`, off by default; the app pa
- each finished utterance (every final, also one with no text) as a 16 kHz mono WAV of the clip as it
was cut for transcription, before loudness levelling: `<day>/<HHMMSS>-<ms>-said-<speaker>.wav`;
- each spoken answer (every `tts` request that made sound) as a mono WAV at the session's TTS rate,
as far as it was made (also when cancelled or failed): `<day>/<HHMMSS>-<ms>-jarvis.wav`, timed at
its first sentence;
- each spoken answer as a mono WAV at the session's TTS rate, as far as the app played it:
`<day>/<HHMMSS>-<ms>-jarvis.wav`, timed at its first sentence. The app sends `tts_played` (`played`:
how many of its sentences started playing, `cancelled`: whether it was stopped) when each request
ends; the worker keeps the request's sentences until it is made and that report came, then writes
the first `played` sentences (the last one maybe cut off by the stop; it may have been made in full)
with `cancelled`. Nothing played (0), never reported (the session ends, an older app; at most 50
are held) or ids it does not know (the earcon): no file;
- one JSON line per file in `<day>/index.jsonl`: `{file, at, ms, kind: said|spoke, who, name,
channel, text, id, cancelled}` (the transcript, or the text sent to `tts`).
`<day>` is the local day. Files are written on a thread of their own, so the voice path never waits,
and a failed write is a debug line (no text) that changes nothing else. Day folders older than 7 days
are removed when the worker starts. Nothing is recorded without the variable.
are removed when the worker starts and with the first recording of each new day while it runs.
Nothing is recorded without the variable.
### Errors

View File

@@ -378,14 +378,17 @@ def test_the_page_says_whether_its_sound_plays_and_opening_waits_for_it_to_start
assert s.open(site + "/second").get("media") == []
def test_clicking_through_to_a_video_page_whose_player_stays_paused_presses_play(chrome):
def test_a_video_page_whose_player_stays_paused_gets_play_pressed_when_clicked_to_or_opened(chrome):
cdp, site, _ = chrome
s = BrowserSession(cdp, time.monotonic() + 15)
s.open(site + "/results")
page = s.click("The song")
assert page["url"].endswith("/watch") and [m["playing"] for m in page["media"]] == [True], page
# Opening the page by its address is not a click on it: the player is left as the page has it.
page = s.open(site + "/watch")
s.open(site + "/results")
page = s.open(site + "/watch") # straight to the video page by its address
assert [m["playing"] for m in page["media"]] == [True], page
# Sound that is not the page's main player (an audio element, no big video) is left as it is.
page = s.open(site + "/button")
assert [m["playing"] for m in page["media"]] == [False]

View File

@@ -402,6 +402,9 @@ def test_with_recordings_on_each_utterance_and_spoken_answer_is_kept_as_a_wav_wi
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
# The app played both sentences to the end.
await ws.send(json.dumps({"type": "tts_played", "id": "r1", "played": 2, "cancelled": False}))
await asyncio.sleep(0.2)
asyncio.run(with_worker(body, tts=FakeTts(), recorder=recorder))
recorder.close()
@@ -446,3 +449,49 @@ def test_recordings_older_than_the_keep_days_are_removed_at_start(tmp_path):
(tmp_path / "notes").mkdir() # not a day folder: left alone
Recorder(tmp_path, now=datetime(2026, 11, 16, 12).timestamp()).close()
assert sorted(p.name for p in tmp_path.iterdir()) == sorted([d.isoformat() for d in keep] + ["notes"])
@pytest.mark.unit
def test_a_spoken_answer_is_kept_only_as_far_as_the_app_played_it(tmp_path):
import wave
from voice_worker.recordings import Recorder
recorder = Recorder(tmp_path)
async def body(url):
async with connect(url) as ws:
await hello(ws)
# Made in full, then stopped while its first sentence played.
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await ws.send(json.dumps({"type": "tts_played", "id": "r1", "played": 1, "cancelled": True}))
# Stopped before any of it was heard: nothing to keep.
await ws.send(json.dumps({"type": "tts", "id": "r2", "text": "두 번째 답이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await ws.send(json.dumps({"type": "tts_played", "id": "r2", "played": 0, "cancelled": True}))
# Never reported as played (an older app): not kept as if it had been heard.
await ws.send(json.dumps({"type": "tts", "id": "r3", "text": "세 번째 답이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await asyncio.sleep(0.2)
asyncio.run(with_worker(body, tts=FakeTts(), recorder=recorder))
recorder.close()
day, lines = _index(tmp_path)
assert [(l["id"], l["cancelled"]) for l in lines] == [("r1", True)]
with wave.open(str(day / lines[0]["file"])) as w:
assert w.getnframes() == len("오늘은 맑아요.") * 10, "only the sentence that was played"
@pytest.mark.unit
def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path):
from datetime import datetime, timedelta
import numpy as np
from voice_worker.recordings import KEEP_DAYS, Recorder
start = datetime(2026, 11, 16, 12)
r = Recorder(tmp_path, now=start.timestamp())
r.said(start.timestamp(), "1", "A", "vc", "첫날", np.zeros(1600))
later = start + timedelta(days=KEEP_DAYS) # the first recording of a day past the keep days
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
r.close()
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]