perf(bridge): stream TTS per sentence to cut voice reply latency

The /converse turn synthesised the entire reply before any audio played, so
time-to-first-audio grew with reply length. Add a streaming /converse_stream
endpoint that emits the transcript/reply first, then one audio clip per
sentence as each finishes synthesising. The Discord voice layer enqueues each
clip on arrival via the existing FIFO playQueue, so the first sentence starts
speaking while the rest are still being synthesised.

STT and the reply engine still run to completion before the first clip; only
TTS is pipelined. The non-streaming /converse and /text endpoints are
unchanged.

- bridge: language-agnostic sentence splitter (bridge/text_utils.py) + NDJSON
  streaming route
- bot: ndjson() reader + converseStream() client; voice.ts plays clips
  progressively
- tests: splitter unit tests + bot ndjson/converseStream tests

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-13 01:09:39 +09:00
parent 989a4f3e98
commit 5c295420ea
6 changed files with 363 additions and 18 deletions

View File

@@ -37,13 +37,22 @@ import wave
from pathlib import Path
from typing import Optional
# Ensure repo-root/src is importable (jarvis package lives in src/jarvis)
# Ensure repo-root/src is importable (jarvis package lives in src/jarvis) and
# the repo root itself (so ``bridge.text_utils`` resolves whether this module is
# launched as ``python -m bridge.server`` or ``python bridge/server.py``).
_REPO_ROOT = Path(__file__).resolve().parent.parent
_SRC = _REPO_ROOT / "src"
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
from flask import Flask, request, jsonify
from flask import Flask, request, jsonify, Response, stream_with_context
try: # package-relative when imported as ``bridge.server``
from bridge.text_utils import split_sentences
except ImportError: # script-relative when run as ``bridge/server.py``
from text_utils import split_sentences
app = Flask(__name__)
@@ -372,6 +381,59 @@ def http_converse():
)
@app.post("/converse_stream")
def http_converse_stream():
"""Streaming full turn: speech in -> transcript -> reply -> speech out.
Reduces perceived latency by synthesising the reply one sentence at a time
and emitting each clip as soon as it is ready, so the Discord layer can play
the first sentence while the rest are still being spoken. The response is
newline-delimited JSON (NDJSON):
{"type":"meta","transcript":..,"language":..,"reply":..,"error":..,"broadcast_action":..}
{"type":"audio","seq":0,"audio_b64":..}
{"type":"audio","seq":1,"audio_b64":..}
{"type":"end"}
STT and the reply engine still run to completion before the meta line; only
TTS is pipelined. The non-streaming /converse endpoint is unchanged.
"""
raw = request.get_data()
if not raw:
return jsonify({"error": "empty body; send a WAV blob"}), 400
broadcasting = _coerce_bool(request.args.get("broadcasting"))
def gen():
stt = transcribe(raw)
transcript = stt.get("text", "")
if not transcript:
yield json.dumps({"type": "meta", "transcript": "", "language": stt.get("language"),
"reply": "", "error": stt.get("error"), "broadcast_action": None}) + "\n"
yield json.dumps({"type": "end"}) + "\n"
return
result = think(transcript, stt.get("language"), broadcasting)
reply = result.get("reply", "")
yield json.dumps({
"type": "meta",
"transcript": transcript,
"language": stt.get("language"),
"reply": reply,
"error": result.get("error"),
"broadcast_action": result.get("broadcast_action"),
}) + "\n"
for seq, sentence in enumerate(split_sentences(reply)):
audio = synthesize(sentence)
if audio:
yield json.dumps({
"type": "audio",
"seq": seq,
"audio_b64": base64.b64encode(audio).decode("ascii"),
}) + "\n"
yield json.dumps({"type": "end"}) + "\n"
return Response(stream_with_context(gen()), mimetype="application/x-ndjson")
def main():
print(f"[bridge] listening on http://{BRIDGE_HOST}:{BRIDGE_PORT}", flush=True)
# threaded=True so STT (slow) on one request doesn't block /health, etc.