feat(dashboard): browser STT recognition test on the GPU
The status page was view-only, so there was no way to actually verify Korean recognition end-to-end. Add a live test: record from the mic (localhost/https) or upload an audio file (works over LAN http, where browsers block getUserMedia), POST it to a new /api/stt endpoint that ffmpeg-normalises the blob to 16 kHz mono and runs the real GPU faster-whisper, then shows the recognised text + latency + device. Results also land in the live turn feed. The dashboard now optionally holds a WhisperSTT and drives it from a private asyncio loop thread. New `python -m wsai --stt-test` serves the page with STT enabled and pre-warms the GPU worker so the first recognition is instant. WhisperSTT.resolved_device is exposed for the UI. Verified: wav and browser-style webm/opus uploads both return the correct Korean text on device=cuda in ~240-280ms. 12 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
@@ -26,7 +26,9 @@ from .monitor import Monitor
|
||||
log = logging.getLogger("wsai.dashboard")
|
||||
|
||||
|
||||
def _make_handler(monitor: Monitor):
|
||||
def _make_handler(dash: "Dashboard"):
|
||||
monitor = dash.monitor
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
# Quiet: don't spam the console with one line per request.
|
||||
def log_message(self, *args) -> None: # noqa: D401
|
||||
@@ -52,6 +54,39 @@ def _make_handler(monitor: Monitor):
|
||||
else:
|
||||
self._send(404, b"not found", "text/plain; charset=utf-8")
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
path = self.path.split("?", 1)[0]
|
||||
if path == "/api/stt":
|
||||
self._handle_stt()
|
||||
else:
|
||||
self._send(404, b"not found", "text/plain; charset=utf-8")
|
||||
|
||||
def _handle_stt(self) -> None:
|
||||
"""Accept an uploaded audio blob (mic recording or file), run it
|
||||
through the real GPU STT, and return the recognised text."""
|
||||
if dash.stt is None:
|
||||
self._send(503, json.dumps({"ok": False, "error": "STT not enabled"}).encode(),
|
||||
"application/json; charset=utf-8")
|
||||
return
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
length = 0
|
||||
if length <= 0:
|
||||
self._send(400, json.dumps({"ok": False, "error": "empty upload"}).encode(),
|
||||
"application/json; charset=utf-8")
|
||||
return
|
||||
raw = self.rfile.read(length)
|
||||
try:
|
||||
result = dash.transcribe_upload(raw)
|
||||
body = json.dumps({"ok": True, **result}, ensure_ascii=False).encode("utf-8")
|
||||
self._send(200, body, "application/json; charset=utf-8")
|
||||
except Exception as exc: # noqa: BLE001 — surface the reason to the page
|
||||
log.exception("STT upload failed")
|
||||
body = json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"},
|
||||
ensure_ascii=False).encode("utf-8")
|
||||
self._send(500, body, "application/json; charset=utf-8")
|
||||
|
||||
def _stream_events(self) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
||||
@@ -86,17 +121,29 @@ def _make_handler(monitor: Monitor):
|
||||
|
||||
|
||||
class Dashboard:
|
||||
"""Owns the HTTP server thread."""
|
||||
"""Owns the HTTP server thread.
|
||||
|
||||
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787) -> None:
|
||||
Optionally holds a real STT backend so the page can offer a live
|
||||
recognition test (upload/record audio -> GPU whisper -> text). The STT
|
||||
backend is async, so the dashboard runs its own asyncio loop in a
|
||||
background thread and bridges the synchronous HTTP handlers onto it.
|
||||
"""
|
||||
|
||||
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
|
||||
stt=None) -> None:
|
||||
self.monitor = monitor
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.stt = stt
|
||||
self._server: ThreadingHTTPServer | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._loop = None
|
||||
self._loop_thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
handler = _make_handler(self.monitor)
|
||||
if self.stt is not None:
|
||||
self._start_loop()
|
||||
handler = _make_handler(self)
|
||||
self._server = ThreadingHTTPServer((self.host, self.port), handler)
|
||||
self._server.daemon_threads = True
|
||||
self._thread = threading.Thread(
|
||||
@@ -110,6 +157,79 @@ class Dashboard:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
self._server = None
|
||||
if self._loop is not None:
|
||||
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||
self._loop = None
|
||||
|
||||
# -- async bridge (STT test) ----------------------------------------- #
|
||||
def _start_loop(self) -> None:
|
||||
import asyncio
|
||||
|
||||
self._loop = asyncio.new_event_loop()
|
||||
self._loop_thread = threading.Thread(
|
||||
target=self._loop.run_forever, name="wsai-dashboard-loop", daemon=True
|
||||
)
|
||||
self._loop_thread.start()
|
||||
|
||||
def _submit(self, coro, timeout: float = 120.0):
|
||||
import asyncio
|
||||
|
||||
fut = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
||||
return fut.result(timeout=timeout)
|
||||
|
||||
def warm(self) -> None:
|
||||
"""Pre-start the STT worker (loads + warms the GPU) so the first web
|
||||
recognition is instant instead of paying model-load + CUDA autotune."""
|
||||
if self.stt is not None:
|
||||
self._submit(self.stt._ensure())
|
||||
|
||||
def transcribe_upload(self, audio_bytes: bytes) -> dict:
|
||||
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
|
||||
on the GPU, and record the result as a monitor turn so it also shows in
|
||||
the live feed. Returns {text, ms, device}."""
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
updir = os.path.expanduser("~/.cache/wsai/uploads")
|
||||
os.makedirs(updir, exist_ok=True)
|
||||
stem = os.path.join(updir, uuid.uuid4().hex)
|
||||
src, wav = stem + ".bin", stem + ".wav"
|
||||
with open(src, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
turn = self.monitor.turn(source="web")
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
# Decode whatever the browser sent (webm/opus, ogg, mp4, wav) to the
|
||||
# 16 kHz mono wav faster-whisper expects.
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
text = self._submit(self.stt.transcribe(wav))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
turn.heard(text or "(빈 결과)")
|
||||
step = turn.step("STT(GPU)")
|
||||
step.ok, step.ms = True, float(ms)
|
||||
turn._steps.append(step)
|
||||
turn.finish()
|
||||
return {"text": text, "ms": ms,
|
||||
"device": getattr(self.stt, "resolved_device", None) or "?"}
|
||||
except subprocess.CalledProcessError as exc:
|
||||
turn.finish(error="ffmpeg decode failed")
|
||||
err = exc.stderr.decode("utf-8", "replace")[-300:] if exc.stderr else str(exc)
|
||||
raise RuntimeError(f"ffmpeg: {err}") from exc
|
||||
except Exception as exc:
|
||||
turn.finish(error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
for p in (src, wav):
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -182,6 +302,16 @@ PAGE = r"""<!DOCTYPE html>
|
||||
.demobar{background:#2a2210;border:1px solid #6b5417;color:var(--warn);border-radius:12px;
|
||||
padding:11px 15px;margin:0 0 16px;font-size:13px;line-height:1.55}
|
||||
.demobar b{color:#ffe08a}
|
||||
.sttbox{background:var(--panel);border:1px solid var(--line);border-radius:14px;padding:14px 16px;margin:0 0 16px}
|
||||
.sttbox h2{font-size:13px;margin:0 0 10px;font-weight:600;color:var(--fg)}
|
||||
.sttrow{display:flex;gap:10px;align-items:center;flex-wrap:wrap}
|
||||
.btn{background:#173042;border:1px solid #234a63;color:var(--fg);border-radius:10px;padding:8px 14px;font-size:13px;cursor:pointer}
|
||||
.btn:hover{background:#1d3d54}
|
||||
.btn.rec{background:#4a1f27;border-color:#7a2531;color:#ffb3bb}
|
||||
.sttstat{color:var(--muted);font-size:12.5px}
|
||||
.sttres{margin-top:12px;font-size:15px;min-height:1px}
|
||||
.sttres .txt{color:var(--heard);font-weight:600;line-height:1.5}
|
||||
.sttres .meta{color:var(--muted);font-size:12px;margin-top:5px}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -199,6 +329,16 @@ PAGE = r"""<!DOCTYPE html>
|
||||
</div>
|
||||
</header>
|
||||
<main>
|
||||
<section class="sttbox" id="sttbox" style="display:none">
|
||||
<h2>🎤 음성 인식(STT) 테스트 · GPU</h2>
|
||||
<div class="sttrow">
|
||||
<button id="recbtn" class="btn">🎤 녹음 시작</button>
|
||||
<label class="btn" for="fileinp">📁 오디오 파일 올리기</label>
|
||||
<input id="fileinp" type="file" accept="audio/*" hidden>
|
||||
<span id="ststat" class="sttstat">녹음하거나 오디오 파일을 올리면 GPU로 인식합니다.</span>
|
||||
</div>
|
||||
<div id="sttres" class="sttres"></div>
|
||||
</section>
|
||||
<div class="demobar" id="demobar" style="display:none"></div>
|
||||
<div class="comp" id="comp"></div>
|
||||
<div id="turns"></div>
|
||||
@@ -238,6 +378,8 @@ function renderStatus(s){
|
||||
const comps = s.components || {};
|
||||
// Demo banner: if the ears/brain/mouth are still mock, everything below is
|
||||
// replayed sample data, not a real conversation. Say so loudly.
|
||||
const sttReal = comps.stt && comps.stt!=='mock' && comps.stt!=='none';
|
||||
$('sttbox').style.display = sttReal ? 'block' : 'none';
|
||||
const mockParts = ['stt','brain','tts'].filter(k => comps[k]==='mock');
|
||||
const bar = $('demobar');
|
||||
if(mockParts.length){
|
||||
@@ -335,6 +477,50 @@ function connect(){
|
||||
else if(ev.type==='log') { addEvent(ev); if(statusData){ statusData.errors_total=(statusData.errors_total||0)+(ev.level==='error'?1:0); $('s-errors').textContent=statusData.errors_total; } }
|
||||
};
|
||||
}
|
||||
// --- STT recognition test (upload / mic record -> GPU whisper) ----------- #
|
||||
let mediaRec=null, chunks=[];
|
||||
async function sendBlob(blob){
|
||||
$('ststat').textContent='인식 중… (GPU)';
|
||||
$('sttres').innerHTML='';
|
||||
try{
|
||||
const r=await fetch('/api/stt',{method:'POST',
|
||||
headers:{'Content-Type':blob.type||'application/octet-stream'},body:blob});
|
||||
const j=await r.json();
|
||||
if(j.ok){
|
||||
$('sttres').innerHTML='<div class="txt">'+esc(j.text||'(빈 결과)')+'</div>'
|
||||
+'<div class="meta">인식 '+j.ms+' ms · '+esc(j.device)+'</div>';
|
||||
$('ststat').textContent='완료. 다시 녹음하거나 파일을 올릴 수 있습니다.';
|
||||
}else{
|
||||
$('sttres').innerHTML='<div class="meta" style="color:var(--err)">오류: '+esc(j.error)+'</div>';
|
||||
$('ststat').textContent='실패.';
|
||||
}
|
||||
}catch(e){ $('ststat').textContent='요청 실패: '+e; }
|
||||
}
|
||||
(function(){
|
||||
const fi=$('fileinp'); if(fi) fi.onchange=()=>{ if(fi.files[0]) sendBlob(fi.files[0]); };
|
||||
const rb=$('recbtn'); if(!rb) return;
|
||||
rb.onclick=async()=>{
|
||||
if(mediaRec && mediaRec.state==='recording'){ mediaRec.stop(); return; }
|
||||
if(!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia){
|
||||
$('ststat').textContent='이 주소(원격 http)에서는 브라우저 마이크가 막혀 있습니다. 파일 업로드를 사용하세요.';
|
||||
return;
|
||||
}
|
||||
try{
|
||||
const stream=await navigator.mediaDevices.getUserMedia({audio:true});
|
||||
chunks=[]; mediaRec=new MediaRecorder(stream);
|
||||
mediaRec.ondataavailable=(e)=>{ if(e.data.size) chunks.push(e.data); };
|
||||
mediaRec.onstop=()=>{
|
||||
stream.getTracks().forEach(t=>t.stop());
|
||||
rb.textContent='🎤 녹음 시작'; rb.classList.remove('rec');
|
||||
sendBlob(new Blob(chunks,{type:mediaRec.mimeType||'audio/webm'}));
|
||||
};
|
||||
mediaRec.start();
|
||||
rb.textContent='⏹ 녹음 중지'; rb.classList.add('rec');
|
||||
$('ststat').textContent='녹음 중… 말한 뒤 중지를 누르세요.';
|
||||
}catch(e){ $('ststat').textContent='마이크 접근 실패: '+e; }
|
||||
};
|
||||
})();
|
||||
|
||||
connect();
|
||||
// Refresh uptime label every second from the last known status.
|
||||
setInterval(()=>{ if(statusData){ statusData.uptime_s=(statusData.uptime_s||0)+1; $('s-up').textContent=fmtUptime(statusData.uptime_s);} }, 1000);
|
||||
|
||||
Reference in New Issue
Block a user