perf(bridge): lock STT to Korean + add per-stage turn timing
- transcribe() now passes language="ko" (STT_LANGUAGE env, default ko): skips Whisper auto-detect, fixing occasional Korean->Chinese mis-detection and shaving latency. LLM is already locked via OUTPUT_LANGUAGE=Korean; MeloTTS is Korean-only — so STT/LLM/TTS are all Korean now. - converse_stream logs "⏱️ turn stt=.. think(LLM)=.. tts=.. total=.." so the ~30s voice-reply latency can be attributed to the real bottleneck stage. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -75,6 +75,11 @@ VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False")
|
|||||||
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
|
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
|
||||||
VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
||||||
|
|
||||||
|
# Lock STT to a single language (this deployment is Korean-only). Skipping
|
||||||
|
# Whisper's language auto-detect both fixes occasional mis-detection (e.g. a
|
||||||
|
# Korean phrase decoded as Chinese) and shaves a little latency. Empty = auto.
|
||||||
|
STT_LANGUAGE = os.environ.get("STT_LANGUAGE", "ko").strip() or None
|
||||||
|
|
||||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||||
@@ -208,7 +213,7 @@ def transcribe(wav_bytes: bytes) -> dict:
|
|||||||
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
|
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
|
||||||
return {"text": "", "language": None}
|
return {"text": "", "language": None}
|
||||||
|
|
||||||
segments, info = _whisper.transcribe(audio, beam_size=1)
|
segments, info = _whisper.transcribe(audio, beam_size=1, language=STT_LANGUAGE)
|
||||||
# Second line of defence: drop non-speech / hallucinated segments by
|
# Second line of defence: drop non-speech / hallucinated segments by
|
||||||
# Whisper's own no_speech_prob. The no_speech_prob hard cutoff (plus the VAD
|
# Whisper's own no_speech_prob. The no_speech_prob hard cutoff (plus the VAD
|
||||||
# pre-gate above) is what rejects noise/hallucinations. The avg_logprob
|
# pre-gate above) is what rejects noise/hallucinations. The avg_logprob
|
||||||
@@ -446,7 +451,10 @@ def http_converse_stream():
|
|||||||
broadcasting = _coerce_bool(request.args.get("broadcasting"))
|
broadcasting = _coerce_bool(request.args.get("broadcasting"))
|
||||||
|
|
||||||
def gen():
|
def gen():
|
||||||
|
import time
|
||||||
|
t0 = time.monotonic()
|
||||||
stt = transcribe(raw)
|
stt = transcribe(raw)
|
||||||
|
t_stt = time.monotonic()
|
||||||
transcript = stt.get("text", "")
|
transcript = stt.get("text", "")
|
||||||
if not transcript:
|
if not transcript:
|
||||||
yield json.dumps({"type": "meta", "transcript": "", "language": stt.get("language"),
|
yield json.dumps({"type": "meta", "transcript": "", "language": stt.get("language"),
|
||||||
@@ -454,6 +462,7 @@ def http_converse_stream():
|
|||||||
yield json.dumps({"type": "end"}) + "\n"
|
yield json.dumps({"type": "end"}) + "\n"
|
||||||
return
|
return
|
||||||
result = think(transcript, stt.get("language"), broadcasting)
|
result = think(transcript, stt.get("language"), broadcasting)
|
||||||
|
t_think = time.monotonic()
|
||||||
reply = result.get("reply", "")
|
reply = result.get("reply", "")
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "meta",
|
"type": "meta",
|
||||||
@@ -463,8 +472,11 @@ def http_converse_stream():
|
|||||||
"error": result.get("error"),
|
"error": result.get("error"),
|
||||||
"broadcast_action": result.get("broadcast_action"),
|
"broadcast_action": result.get("broadcast_action"),
|
||||||
}) + "\n"
|
}) + "\n"
|
||||||
|
tts_total = 0.0
|
||||||
for seq, sentence in enumerate(split_sentences(reply)):
|
for seq, sentence in enumerate(split_sentences(reply)):
|
||||||
|
ts = time.monotonic()
|
||||||
audio = synthesize(sentence)
|
audio = synthesize(sentence)
|
||||||
|
tts_total += time.monotonic() - ts
|
||||||
if audio:
|
if audio:
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "audio",
|
"type": "audio",
|
||||||
@@ -472,6 +484,12 @@ def http_converse_stream():
|
|||||||
"audio_b64": base64.b64encode(audio).decode("ascii"),
|
"audio_b64": base64.b64encode(audio).decode("ascii"),
|
||||||
}) + "\n"
|
}) + "\n"
|
||||||
yield json.dumps({"type": "end"}) + "\n"
|
yield json.dumps({"type": "end"}) + "\n"
|
||||||
|
print(
|
||||||
|
f"[bridge] ⏱️ turn stt={t_stt - t0:.1f}s think(LLM)={t_think - t_stt:.1f}s "
|
||||||
|
f"tts={tts_total:.1f}s total={time.monotonic() - t0:.1f}s replylen={len(reply)} "
|
||||||
|
f"transcript={transcript[:40]!r}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
return Response(stream_with_context(gen()), mimetype="application/x-ndjson")
|
return Response(stream_with_context(gen()), mimetype="application/x-ndjson")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user