From f64d76e7372d3ff92911f6f16a581ac0f4b926aa Mon Sep 17 00:00:00 2001 From: javis-bot Date: Tue, 23 Jun 2026 03:44:15 +0900 Subject: [PATCH] feat: use Edge TTS (Korean Hyunsu voice @ +45%) as the default voice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user chose Microsoft Edge TTS, voice ko-KR-HyunsuMultilingualNeural at rate +45% (~1.45x), as the natural Korean voice. Wire it into the bridge and make it the default engine. - bridge/server.py: _edge_synthesize() calls edge-tts and transcodes the MP3 to PCM16 mono WAV with the system ffmpeg (temp file for a correct header); TTS_ENGINE default -> edge; EDGE_TTS_VOICE / EDGE_TTS_RATE env-driven - requirements-bridge.txt: add edge-tts (lightweight; httpx) - compose/.env.example/README: TTS_ENGINE=edge + EDGE_TTS_* knobs; note the online/privacy trade-off (reply text is sent to Microsoft, needs internet) - drop the now-unused MeloTTS build layer (Dockerfile) and melo-worker (supervisord) — edge synthesises in-process, no model/worker baked, slimmer and faster image; settings UI engine list -> edge/piper, restart only bridge Verified on host: edge-tts -> ffmpeg yields a valid 16-bit mono 24kHz WAV; envsubst renders tts_engine=edge; docker build --check + 26 tests pass. Co-Authored-By: Claude Opus 4.7 --- .env.example | 22 ++++----- Dockerfile | 18 +++---- README.md | 8 +-- bridge/requirements-bridge.txt | 6 ++- bridge/server.py | 89 +++++++++++++++++++++++++++------- bridge/settings_web.py | 24 +++------ docker-compose.yml | 17 +++---- docker/supervisord.conf | 27 +---------- 8 files changed, 115 insertions(+), 96 deletions(-) diff --git a/.env.example b/.env.example index a488413..86eaaf4 100644 --- a/.env.example +++ b/.env.example @@ -34,18 +34,18 @@ WHISPER_DEVICE=cuda WHISPER_COMPUTE_TYPE=float16 # Optional explicit Piper voice model (.onnx). If empty, the jarvis default is used. TTS_PIPER_MODEL_PATH= -# TTS engine: "melo" (default) uses the MeloTTS Korean voice served by the warm -# melo-worker (Korean speaker, speed 1.5). Set to "piper" to use Piper directly. -TTS_ENGINE=melo -# Melo-only by default: if MeloTTS synthesis fails the bridge returns no audio -# rather than speaking Korean through the English Piper voice (which mangles it). -# Set to 1 only if you explicitly want the Piper fallback. +# TTS engine: "edge" (default) uses Microsoft Edge TTS, a natural Korean neural +# voice. Set to "piper" for the offline English voice. NOTE: edge is ONLINE — +# reply text is sent to Microsoft's servers and needs internet. +TTS_ENGINE=edge +# Edge voice + speaking rate. Rate is a percentage (+45% ≈ 1.45×). Korean voices: +# ko-KR-HyunsuMultilingualNeural (M), ko-KR-InJoonNeural (M), ko-KR-SunHiNeural (F). +EDGE_TTS_VOICE=ko-KR-HyunsuMultilingualNeural +EDGE_TTS_RATE=+45% +# Neural-only by default: if synthesis fails the bridge returns no audio rather +# than speaking Korean through the English Piper voice. Set to 1 to allow the +# Piper fallback. MELO_FALLBACK_PIPER=0 -# Where the bridge reaches the in-container MeloTTS worker, and how long it -# waits for a synthesis. Speaking rate is set on the worker via MELO_SPEED. -MELO_WORKER_URL=http://127.0.0.1:8770 -MELO_TIMEOUT=30 -MELO_SPEED=1.5 # --------------------------------------------------------------------------- # Jarvis brain (Ollama-backed). In Docker these populate the rendered diff --git a/Dockerfile b/Dockerfile index 0f5c837..fd19fdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,20 +65,14 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \ > /etc/ld.so.conf.d/nvidia-cu12.conf 2>/dev/null \ && /sbin/ldconfig || true -# --- MeloTTS Korean voice (separate /opt/melo py3.11 venv; see setup-melo.sh). -# Heavy layer (torch CPU + transformers + MeCab); placed before the app -# COPY so it stays cached across source-only changes. --- -COPY docker/setup-melo.sh /app/docker/setup-melo.sh -# Strip CR before running: a Windows checkout (autocrlf) yields CRLF, which makes -# bash read line 18 as `set -euxo pipefail\r` and abort with -# "set: pipefail: invalid option name". .gitattributes pins *.sh to LF, but this -# keeps the build working even on a not-yet-renormalised working tree. -RUN sed -i 's/\r$//' /app/docker/setup-melo.sh && bash /app/docker/setup-melo.sh +# --- Korean voice: Microsoft Edge TTS (online neural). No model is baked — the +# `edge-tts` pip package (in requirements-bridge.txt) calls the MS service at +# runtime and the bridge transcodes the MP3 to PCM16 with ffmpeg. No heavy +# TTS build layer is needed. --- # --- Human input + window management for the on-screen Chrome control tool. -# Placed AFTER the heavy melo layer so it doesn't bust that cache. xdotool -# injects real X pointer/keyboard events (visible cursor, char-by-char -# typing) into the broadcast; wmctrl lists/moves windows. --- +# xdotool injects real X pointer/keyboard events (visible cursor, +# char-by-char typing) into the broadcast; wmctrl lists/moves windows. --- RUN apt-get update && apt-get install -y --no-install-recommends \ xdotool wmctrl \ && rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index f57095a..22b961d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ docker compose -f docker-compose.yml -f docker-compose.gpu-linux.yml up -d --bui docker compose -f docker-compose.yml -f docker-compose.gpu-windows.yml up -d --build # ── GPU 없이 (CPU 전용 호스트) ── -# .env 에 WHISPER_DEVICE=cpu, MELO_DEVICE=cpu 를 넣고 베이스만 사용 +# .env 에 WHISPER_DEVICE=cpu 를 넣고 베이스만 사용 docker compose up -d --build ``` @@ -113,7 +113,7 @@ docker compose up -d # 유저봇이 로그인해 지정 음성채널에 ### GPU 가속 (OS별) -LLM(Ollama), Whisper STT, MeloTTS가 GPU에서 돕니다(env 기본 `WHISPER_DEVICE=cuda`, `MELO_DEVICE=cuda`). NVIDIA Blackwell(sm_120, 예: RTX 5050/5070Ti)에서 검증: 컨테이너 내 torch cu128 CUDA 동작, Ollama GPU 오프로드, faster-whisper float16, MeloTTS GPU 합성 모두 확인. +LLM(Ollama)과 Whisper STT가 GPU에서 돕니다(env 기본 `WHISPER_DEVICE=cuda`). TTS는 기본값이 Edge TTS(온라인 한국어 음성)라 GPU를 쓰지 않습니다. NVIDIA Blackwell(sm_120, 예: RTX 5050/5070Ti)에서 검증: 컨테이너 내 torch cu128 CUDA 동작, Ollama GPU 오프로드, faster-whisper float16 모두 확인. GPU는 위 "실행 — Docker"의 OS별 override 파일로 켜집니다. 호스트 사전 준비는 OS마다 다릅니다: @@ -137,7 +137,7 @@ docker run --rm --device nvidia.com/gpu=all ubuntu nvidia-smi -L # GPU 보이 **공통:** - 모델: 베이스 compose 기본은 `qwen2.5:3b`(8GB VRAM에서 도구호출 안정적). 더 무겁게(`qwen2.5:7b`, `qwen3:8b` 등) 쓰려면 `.env`의 `OLLAMA_CHAT_MODEL` 변경. -- **GPU가 없거나 인식 실패 시 자동으로 CPU 폴백**(Whisper)하므로 안전합니다. 명시적으로 CPU만 쓰려면 override 파일 없이 베이스만 올리고 `.env`에 `WHISPER_DEVICE=cpu`, `MELO_DEVICE=cpu`를 두세요. +- **GPU가 없거나 인식 실패 시 자동으로 CPU 폴백**(Whisper)하므로 안전합니다. 명시적으로 CPU만 쓰려면 override 파일 없이 베이스만 올리고 `.env`에 `WHISPER_DEVICE=cpu`를 두세요. - 데이터(메모리 DB), Whisper 캐시, Piper 음성은 named volume에 영속됩니다. - 셀프봇 영상 송출 의존성은 이미지에 기본 포함하지 않습니다. 쓰려면 컨테이너에서 `cd /app/bot && bun add discord.js-selfbot-v13 @dank074/discord-video-stream` 후 재시작(또는 Dockerfile에 추가). @@ -243,7 +243,7 @@ cd bot; bun run register; bun run start # 창 2: (일반 봇이면) 슬래시 - `BRIDGE_URL` — 봇이 호출할 브릿지 주소 (기본 `http://127.0.0.1:8765`) - `STREAM_BACKEND`, `DISCORD_SELFBOT_TOKEN`, `NOVNC_URL` — 화면 송출 - `VNC_DISPLAY=:1`, `VNC_RESOLUTION`, `VNC_FRAMERATE`, `VNC_BITRATE_KBPS` — 캡처 -- `WHISPER_DEVICE/COMPUTE_TYPE`, `MELO_DEVICE` — GPU 호스트면 `cuda`/`float16`, CPU 전용이면 `cpu`(GPU 자체는 OS별 override compose 파일로 켬) +- `WHISPER_DEVICE/COMPUTE_TYPE` — GPU 호스트면 `cuda`/`float16`, CPU 전용이면 `cpu`(GPU 자체는 OS별 override compose 파일로 켬) - `OLLAMA_CHAT_MODEL` — 두뇌 LLM (기본 `qwen2.5:3b`) - `COMPOSE_FILE` — OS별 GPU override를 매번 `-f`로 안 치고 자동 적용 (위 "실행 — Docker" 참고) - `output_language` — 출력 언어 고정(비우면 사용자 언어). 설정 웹 UI(`/settings`)에서 바꾸면 env 기본값보다 우선하며 컨테이너 재생성 후에도 유지됩니다. diff --git a/bridge/requirements-bridge.txt b/bridge/requirements-bridge.txt index a0f9531..fc76a60 100644 --- a/bridge/requirements-bridge.txt +++ b/bridge/requirements-bridge.txt @@ -21,7 +21,11 @@ nvidia-cudnn-cu12 # --- Bridge HTTP service --- flask>=3.0.0 -# --- Text-to-speech (Piper) --- +# --- Text-to-speech --- +# Edge TTS: the primary Korean voice (online MS neural). Lightweight (httpx); +# emits MP3, transcoded to PCM16 by the system ffmpeg in the bridge. +edge-tts>=6.1.0 +# Piper: offline English fallback. piper-tts>=1.3.0 # --- Built-in tools (lazily imported; needed for full functionality) --- diff --git a/bridge/server.py b/bridge/server.py index 285c868..721984e 100644 --- a/bridge/server.py +++ b/bridge/server.py @@ -87,12 +87,11 @@ VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200")) # Korean phrase decoded as Chinese) and shaves a little latency. Empty = auto. STT_LANGUAGE = os.environ.get("STT_LANGUAGE", "ko").strip() or None -# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary -# voice; Piper is kept as a fallback if the worker is unreachable. Set -# TTS_ENGINE=piper to disable MeloTTS entirely. +# TTS engine: "edge" (Microsoft Edge TTS, natural Korean neural voice) is the +# primary voice. "melo" (a warm MeloTTS worker) and "piper" remain selectable. def _tts_engine_setting() -> str: """TTS engine: settings-UI value (runtime config JSON) wins, else env, else - melo. Read at startup; the settings UI restarts the bridge on apply.""" + edge. Read at startup; the settings UI restarts the bridge on apply.""" try: _cp = os.environ.get("JARVIS_CONFIG_PATH", "/app/config/jarvis.json") _v = json.loads(open(_cp, encoding="utf-8").read()).get("tts_engine") @@ -100,16 +99,22 @@ def _tts_engine_setting() -> str: return str(_v).strip().lower() except Exception: pass - return os.environ.get("TTS_ENGINE", "melo").strip().lower() + return os.environ.get("TTS_ENGINE", "edge").strip().lower() TTS_ENGINE = _tts_engine_setting() +# Edge TTS (online MS neural voice). Voice + rate are env-driven so they can be +# changed without code. Default: Korean "Hyunsu" multilingual voice at +45% +# (≈1.45×), the chosen settings. NOTE: edge synthesis sends the reply TEXT to +# Microsoft's servers and needs internet — an intentional privacy trade-off for +# the more natural voice. +EDGE_TTS_VOICE = os.environ.get("EDGE_TTS_VOICE", "ko-KR-HyunsuMultilingualNeural").strip() +EDGE_TTS_RATE = os.environ.get("EDGE_TTS_RATE", "+45%").strip() MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770") MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30")) -# When MeloTTS is the engine, do NOT silently fall back to the English Piper -# voice on failure: speaking Korean text through an English voice produces -# mangled audio. Default is melo-only (return no audio on failure); set -# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback. +# Do NOT silently fall back to the English Piper voice on a neural-voice failure: +# speaking Korean through an English voice produces mangled audio. Default is +# neural-only (return no audio on failure); set MELO_FALLBACK_PIPER=1 to opt in. MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on") # --------------------------------------------------------------------------- @@ -302,6 +307,54 @@ def _coerce_bool(value) -> Optional[bool]: return str(value).strip().lower() in ("1", "true", "yes", "on") +def _edge_synthesize(text: str) -> Optional[bytes]: + """Synthesise via Microsoft Edge TTS (online neural voice) and return a + 16-bit PCM WAV, or None on any failure. Edge emits MP3; we transcode to + PCM16 mono with the system ffmpeg, writing to a temp file (seekable) so the + WAV header carries a correct length. Needs internet.""" + import asyncio + import subprocess + import tempfile + + try: + import edge_tts # type: ignore + + async def _gen() -> bytes: + comm = edge_tts.Communicate(text, EDGE_TTS_VOICE, rate=EDGE_TTS_RATE) + buf = bytearray() + async for chunk in comm.stream(): + if chunk.get("type") == "audio": + buf.extend(chunk["data"]) + return bytes(buf) + + mp3 = asyncio.run(_gen()) + if not mp3: + print("[bridge] edge TTS returned no audio", flush=True) + return None + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as t: + out_path = t.name + try: + proc = subprocess.run( + ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-i", "pipe:0", "-ac", "1", "-ar", "24000", + "-acodec", "pcm_s16le", out_path], + input=mp3, capture_output=True, + ) + if proc.returncode != 0: + print(f"[bridge] edge ffmpeg transcode failed: {proc.stderr.decode('utf-8','ignore')[:200]}", flush=True) + return None + with open(out_path, "rb") as f: + return f.read() + finally: + try: + os.unlink(out_path) + except OSError: + pass + except Exception as e: # pragma: no cover - network / dep dependent + print(f"[bridge] edge synth failed: {e}", flush=True) + return None + + def _melo_synthesize(text: str) -> Optional[bytes]: """Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so @@ -361,20 +414,22 @@ def _tts_ready() -> bool: def synthesize(text: str) -> Optional[bytes]: - """Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS - (Korean speaker, speed 1.5) served by the warm melo worker; Piper is a - fallback if the worker is unavailable. Returns None if TTS is off.""" + """Synthesize text to a 16-bit PCM WAV. The primary voice is Edge TTS (a + natural Korean neural voice); "melo" uses the warm MeloTTS worker. For a + neural engine, Piper (English) is only used if explicitly enabled, since + speaking Korean through an English voice mangles it. Returns None if off.""" if not TTS_ENABLED or not text.strip(): return None - if TTS_ENGINE == "melo": - audio = _melo_synthesize(text) + _neural = {"edge": _edge_synthesize, "melo": _melo_synthesize}.get(TTS_ENGINE) + if _neural is not None: + audio = _neural(text) if audio: return audio if not MELO_FALLBACK_PIPER: - # Melo-only: better silent than mangled English for Korean text. - print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True) + # Neural-only: better silent than mangled English for Korean text. + print(f"[bridge] {TTS_ENGINE} synth failed; no audio (Piper fallback disabled)", flush=True) return None - print("[bridge] melo synth failed; falling back to Piper", flush=True) + print(f"[bridge] {TTS_ENGINE} synth failed; falling back to Piper", flush=True) return _piper_synthesize(text) diff --git a/bridge/settings_web.py b/bridge/settings_web.py index bdb97af..bb5c9ac 100644 --- a/bridge/settings_web.py +++ b/bridge/settings_web.py @@ -22,8 +22,7 @@ from typing import Any, Dict FIELDS = [ ("ollama_chat_model", "LLM 모델", "model"), ("whisper_model", "STT(Whisper) 모델", "select:tiny,base,small,medium,large,large-v3"), - ("tts_engine", "TTS 엔진", "select:melo,piper"), - ("melo_speed", "TTS 속도 (MeloTTS)", "number:0.5:2.5:0.1"), + ("tts_engine", "TTS 엔진", "select:edge,piper"), ("output_language", "출력 언어 (비우면 사용자 언어)", "text"), ("llm_thinking_enabled", "LLM 사고(thinking) 모드", "bool"), ("agentic_max_turns", "에이전트 최대 반복", "number:1:12:1"), @@ -54,9 +53,7 @@ def _current() -> Dict[str, Any]: cfg = _read_config() out: Dict[str, Any] = {} for k in _KEYS: - if k == "melo_speed": - out[k] = cfg.get("melo_speed", os.environ.get("MELO_SPEED", "1.5")) - elif k == "output_language": + if k == "output_language": out[k] = cfg.get("output_language", os.environ.get("OUTPUT_LANGUAGE", "")) else: out[k] = cfg.get(k, "") @@ -78,12 +75,7 @@ def _coerce(updates: Dict[str, Any]) -> Dict[str, Any]: for k, v in updates.items(): if k not in _KEYS: continue - if k == "melo_speed": - try: - v = float(v) - except (TypeError, ValueError): - continue - elif k == "agentic_max_turns": + if k == "agentic_max_turns": try: v = int(v) except (TypeError, ValueError): @@ -114,15 +106,15 @@ def _save(updates: Dict[str, Any]) -> None: def _apply() -> str: - # Restart melo + bridge AFTER this response is sent. Detached (new session) - # so the bridge being killed mid-restart doesn't drop the restart itself, - # and the HTTP client still receives this response. + # Restart the bridge AFTER this response is sent. Detached (new session) so + # the bridge being killed mid-restart doesn't drop the restart itself, and + # the HTTP client still receives this response. (Edge TTS has no worker.) try: subprocess.Popen( - ["sh", "-c", "sleep 1; supervisorctl restart melo-worker bridge"], + ["sh", "-c", "sleep 1; supervisorctl restart bridge"], start_new_session=True, ) - return "1초 후 브리지/TTS 워커가 재시작되어 반영됩니다." + return "1초 후 브리지가 재시작되어 반영됩니다." except Exception as e: # pragma: no cover return str(e) diff --git a/docker-compose.yml b/docker-compose.yml index 3875900..fb7a549 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,16 +68,13 @@ services: WHISPER_COMPUTE_TYPE: ${WHISPER_COMPUTE_TYPE:-float16} # TTS engine. Rendered into /app/config/jarvis.json via envsubst (the # bridge reads that JSON BEFORE the env, so it must carry the real engine, - # not the template's old hardcoded "piper" — otherwise Korean text is read - # by the English Piper voice). Default melo; .env can override. - TTS_ENGINE: ${TTS_ENGINE:-melo} - # MeloTTS on the GPU (cu128 torch baked by docker/setup-melo.sh). CPU synth - # serialised under load and pushed TTS to 7-8s; GPU does ~0.3s/sentence. - MELO_DEVICE: ${MELO_DEVICE:-cuda} - # Speaking rate for MeloTTS. Set here (with a default) so supervisord's - # %(ENV_MELO_SPEED)s passthrough always resolves and an .env override - # actually reaches the melo-worker. Lower it (e.g. 1.1) for a calmer pace. - MELO_SPEED: ${MELO_SPEED:-1.5} + # not a hardcoded one — otherwise Korean text is read by the English Piper + # voice). Default edge; .env can override (e.g. piper for offline). + TTS_ENGINE: ${TTS_ENGINE:-edge} + # Edge TTS voice + rate (the chosen natural Korean voice). NOTE: edge is an + # ONLINE engine — reply text is sent to Microsoft and needs internet. + EDGE_TTS_VOICE: ${EDGE_TTS_VOICE:-ko-KR-HyunsuMultilingualNeural} + EDGE_TTS_RATE: ${EDGE_TTS_RATE:-+45%} # Optional single-language lock for replies (empty = user's own language). OUTPUT_LANGUAGE: ${OUTPUT_LANGUAGE:-ko} # Drop the pre-loop planner LLM call to cut voice-reply latency on small diff --git a/docker/supervisord.conf b/docker/supervisord.conf index c512e1f..1d6c957 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -49,31 +49,8 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:melo-worker] -; Warm MeloTTS Korean voice (speed 1.5) in its own py3.11 venv. The bridge's -; synthesize() POSTs here; if this is down the bridge falls back to Piper. -command=/app/docker/run-if-role.sh full,bot /opt/melo/bin/python /app/bridge/melo_worker.py -directory=/app -; HF_HOME points at the dedicated, image-baked melo cache (warmed in -; setup-melo.sh). The brain's whisper_cache volume is mounted over -; /root/.cache/huggingface, so without this the pre-cached BERT + KR checkpoint -; would be shadowed and re-downloaded (and would fail if the host is offline). -; HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE force pure-cache reads: the pinned old -; transformers/huggingface_hub otherwise retry the network on every load and -; error out instead of falling back to the (complete) baked cache. -; MELO_DEVICE and MELO_SPEED inherit from the container env (compose sets both -; with defaults: cuda / 1.5) so the worker runs MeloTTS on the GPU at the -; configured rate. supervisord interpolates %(ENV_x)s from its own environment, -; which is the container's — so MELO_SPEED must always be set in the env -; (compose guarantees it) or this expansion fails at startup. Hardcoding 1.5 -; here previously shadowed the .env value, so lowering MELO_SPEED had no effect. -environment=MELO_LANGUAGE="KR",MELO_SPEED="%(ENV_MELO_SPEED)s",MELO_DEVICE="%(ENV_MELO_DEVICE)s",MELO_WORKER_HOST="127.0.0.1",MELO_WORKER_PORT="8770",HF_HOME="/opt/melo-cache",HF_HUB_OFFLINE="1",TRANSFORMERS_OFFLINE="1" -priority=280 -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 +# (No TTS worker program: the default Edge TTS engine synthesises in-process in +# the bridge via the `edge-tts` package — no warm model/worker is needed.) [program:bridge] command=/app/docker/run-if-role.sh full,bot /opt/venv/bin/python -m bridge.server