Add full STT LLM TTS test mode

README.md

@@ -42,6 +42,12 @@ STT + LLM 통합 테스트:
 bun run test:sttllm
 ```
 
+STT + LLM + TTS 전체 연결 테스트:
+
+```bat
+bun run test:all
+```
+
 LLM 단독 테스트:
 
 ```bat
@@ -117,8 +123,9 @@ bun run test:tts -- "안녕하세요. 로컬 티티에스 테스트입니다."
 ## 메모
 
 - 이 버전은 `STT`, `STT+LLM`, `LLM` 테스트를 따로 제공합니다.
-- `test:sttllm`에서 TTS가 켜져 있으면 답변을 스피커로 읽어줍니다.
+- `test:sttllm`은 STT와 LLM만 연결합니다.
-- `test:sttllm`에서는 자기 음성을 다시 전사하지 않도록 TTS 재생 중에는 캡처를 잠시 멈춥니다.
+- `test:all`은 STT, LLM, TTS를 모두 연결합니다.
+- `test:all`에서는 자기 음성을 다시 전사하지 않도록 TTS 재생 중에는 캡처를 잠시 멈춥니다.
 - LLM 프롬프트는 `prompts/*.md` 에 분리되어 있습니다.
 - 최소 지연을 위해 파일 저장은 하지 않습니다.
 - VAD는 현재 모델 기반이 아니라 진폭 기반 단순 분리입니다.
@@ -143,7 +150,14 @@ bun run test:tts -- "안녕하세요. 로컬 티티에스 테스트입니다."
 1. `bun run setup:llm`
 2. `bun run setup:tts`
 3. `bun run test:sttllm`
-4. 유튜브, 디스코드 통화, 동영상 같은 소리를 재생해서 전사와 답변, 음성 출력 확인
+4. 유튜브, 디스코드 통화, 동영상 같은 소리를 재생해서 전사와 답변 확인
+
+## Windows 전체 연결 테스트 순서
+
+1. `bun run setup:llm`
+2. `bun run setup:tts`
+3. `bun run test:all`
+4. 유튜브, 디스코드 통화, 동영상 같은 소리를 재생해서 전사, 답변, 음성 출력 확인
 
 ## Windows LLM 테스트 순서
 

docker/melotts/Dockerfile

@@ -17,5 +17,6 @@ RUN python -m unidic download
 RUN python /opt/MeloTTS/melo/init_downloads.py
 
 COPY melo_tts_cli.py /opt/realtime-voice-bot/melo_tts_cli.py
+COPY melo_tts_worker.py /opt/realtime-voice-bot/melo_tts_worker.py
 
 ENTRYPOINT ["python", "/opt/realtime-voice-bot/melo_tts_cli.py"]

docker/melotts/melo_tts_worker.py

@@ -0,0 +1,110 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+from melo.api import TTS
+
+
+LANGUAGE = os.getenv("TTS_LANGUAGE", "KR")
+SPEAKER = os.getenv("TTS_SPEAKER", "KR")
+DEVICE = os.getenv("TTS_DEVICE", "cpu")
+SPEED = float(os.getenv("TTS_SPEED", "1.18"))
+SDP_RATIO = float(os.getenv("TTS_SDP_RATIO", "0.22"))
+NOISE_SCALE = float(os.getenv("TTS_NOISE_SCALE", "0.55"))
+NOISE_SCALE_W = float(os.getenv("TTS_NOISE_SCALE_W", "0.75"))
+
+_MODEL = None
+_SPEAKER_ID = None
+
+
+def load_model():
+    global _MODEL
+    global _SPEAKER_ID
+
+    if _MODEL is not None and _SPEAKER_ID is not None:
+        return _MODEL, _SPEAKER_ID
+
+    model = TTS(language=LANGUAGE, device=DEVICE)
+    speaker_ids = model.hps.data.spk2id
+
+    if SPEAKER not in speaker_ids:
+        supported = ", ".join(sorted(speaker_ids.keys()))
+        raise RuntimeError(f"지원하지 않는 speaker 입니다: {SPEAKER}. 사용 가능: {supported}")
+
+    _MODEL = model
+    _SPEAKER_ID = speaker_ids[SPEAKER]
+    return _MODEL, _SPEAKER_ID
+
+
+def handle_ping():
+    model, speaker_id = load_model()
+    return {
+        "language": LANGUAGE,
+        "speaker": SPEAKER,
+        "speaker_id": speaker_id,
+        "device": DEVICE,
+        "speed": SPEED,
+        "sdp_ratio": SDP_RATIO,
+        "noise_scale": NOISE_SCALE,
+        "noise_scale_w": NOISE_SCALE_W,
+        "speaker_count": len(model.hps.data.spk2id),
+    }
+
+
+def handle_synthesize(params):
+    text = str(params["text"]).strip()
+    output_path = Path(str(params["output_path"]))
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    model, speaker_id = load_model()
+    model.tts_to_file(
+        text,
+        speaker_id,
+        str(output_path),
+        speed=SPEED,
+        sdp_ratio=SDP_RATIO,
+        noise_scale=NOISE_SCALE,
+        noise_scale_w=NOISE_SCALE_W,
+    )
+
+    return {
+        "output_path": str(output_path),
+        "text_length": len(text),
+    }
+
+
+def main():
+    for raw_line in sys.stdin:
+        line = raw_line.strip()
+        if not line:
+            continue
+
+        try:
+            payload = json.loads(line)
+            request_id = str(payload["id"])
+            method = payload["method"]
+            params = payload.get("params", {})
+
+            if method == "ping":
+                result = handle_ping()
+            elif method == "synthesize":
+                result = handle_synthesize(params)
+            else:
+                raise RuntimeError(f"알 수 없는 method 입니다: {method}")
+
+            sys.stdout.write(json.dumps({"id": request_id, "result": result}, ensure_ascii=False) + "\n")
+            sys.stdout.flush()
+        except Exception as error:
+            request_id = "unknown"
+            try:
+                request_id = str(payload.get("id", "unknown"))
+            except Exception:
+                pass
+
+            sys.stdout.write(json.dumps({"id": request_id, "error": str(error)}, ensure_ascii=False) + "\n")
+            sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    main()

package.json

@@ -11,6 +11,7 @@
     "setup:python": "bun run setup:stt",
     "test:stt": "bun src/index.ts test-stt",
     "test:sttllm": "bun src/index.ts test-sttllm",
+    "test:all": "bun src/index.ts test-all",
     "test:llm": "bun src/index.ts test-llm",
     "test:tts": "bun src/index.ts test-tts",
     "devices": "bun src/index.ts devices",

src/index.ts

@@ -11,12 +11,12 @@ import { OllamaLlmService } from "./services/ollama-llm.js";
 
 const mode = process.argv[2] ?? "test-stt";
 
-async function runSttTest(enableLlm: boolean): Promise<void> {
+async function runSttTest(options: { enableLlm: boolean; enableTts: boolean }): Promise<void> {
   const config = loadConfig();
   const logger = new Logger(config.DEBUG ? config.LOG_LEVEL : "error");
   const stt = new FasterWhisperSttService(config, logger);
-  const llm = enableLlm ? new OllamaLlmService(config, logger) : null;
+  const llm = options.enableLlm ? new OllamaLlmService(config, logger) : null;
-  let tts = enableLlm && config.TTS_ENABLED ? new MeloTtsService(config, logger) : null;
+  let tts = options.enableTts && config.TTS_ENABLED ? new MeloTtsService(config, logger) : null;
   let capture = null as ReturnType<typeof spawnLoopbackCapture> | null;
   let shuttingDown: Promise<void> | null = null;
   let suppressCapture = false;
@@ -47,6 +47,11 @@ async function runSttTest(enableLlm: boolean): Promise<void> {
       await stt.destroy().catch((destroyError) => {
         logger.warn("STT destroy failed", destroyError);
       });
+      if (tts) {
+        await tts.destroy().catch((destroyError) => {
+          logger.warn("TTS destroy failed", destroyError);
+        });
+      }
     })();
 
     await shuttingDown;
@@ -70,6 +75,9 @@ async function runSttTest(enableLlm: boolean): Promise<void> {
       capture.kill("SIGKILL");
     }
     void stt.destroy();
+    if (tts) {
+      void tts.destroy();
+    }
   });
 
   console.log("STT 준비중...");
@@ -282,14 +290,23 @@ async function runSttTest(enableLlm: boolean): Promise<void> {
   });
 
   if (config.DEBUG) {
-    console.log(enableLlm ? "실시간 출력장치 STT+LLM 테스트를 시작합니다. Ctrl+C 로 종료합니다." : "실시간 출력장치 STT 테스트를 시작합니다. Ctrl+C 로 종료합니다.");
+    if (options.enableLlm && options.enableTts) {
+      console.log("실시간 출력장치 STT+LLM+TTS 테스트를 시작합니다. Ctrl+C 로 종료합니다.");
+    } else if (options.enableLlm) {
+      console.log("실시간 출력장치 STT+LLM 테스트를 시작합니다. Ctrl+C 로 종료합니다.");
+    } else {
+      console.log("실시간 출력장치 STT 테스트를 시작합니다. Ctrl+C 로 종료합니다.");
+    }
     console.log(`source: ${config.AUDIO_SOURCE ?? "unset"}`);
     console.log(`model: ${config.WHISPER_MODEL}`);
     console.log(`language: ${config.WHISPER_LANGUAGE}`);
     console.log(`beam: ${config.WHISPER_BEAM_SIZE}`);
-    if (enableLlm) {
+    if (options.enableLlm) {
       console.log(`llm: ${config.OLLAMA_MODEL}`);
     }
+    if (options.enableTts) {
+      console.log(`tts: ${config.TTS_IMAGE}`);
+    }
   }
 
   setInterval(() => {
@@ -385,10 +402,13 @@ async function main(): Promise<void> {
       await printAudioDevices();
       return;
     case "test-stt":
-      await runSttTest(false);
+      await runSttTest({ enableLlm: false, enableTts: false });
       return;
     case "test-sttllm":
-      await runSttTest(true);
+      await runSttTest({ enableLlm: true, enableTts: false });
+      return;
+    case "test-all":
+      await runSttTest({ enableLlm: true, enableTts: true });
       return;
     case "test-llm":
       await runLlmCli();
@@ -397,7 +417,7 @@ async function main(): Promise<void> {
       await runTtsTest();
       return;
     default:
-      throw new Error(`알 수 없는 실행 모드입니다: ${mode}. 사용 가능: test-stt, test-sttllm, test-llm, test-tts, devices`);
+      throw new Error(`알 수 없는 실행 모드입니다: ${mode}. 사용 가능: test-stt, test-sttllm, test-all, test-llm, test-tts, devices`);
   }
 }
 

src/services/melo-tts.ts

@@ -1,7 +1,9 @@
-import { spawn } from "node:child_process";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { mkdir, rm } from "node:fs/promises";
+import { once } from "node:events";
 import path from "node:path";
+import { createInterface } from "node:readline";
 
 import type { AppConfig } from "../config.js";
 import { resolveDockerCommand } from "../docker-runtime.js";
@@ -41,19 +43,80 @@ async function run(command: string, args: string[], stdio: "ignore" | "inherit"
   });
 }
 
+interface RpcSuccess<T> {
+  id: string;
+  result: T;
+}
+
+interface RpcFailure {
+  id: string;
+  error: string;
+}
+
+type RpcResponse<T> = RpcSuccess<T> | RpcFailure;
+
+function isFailure<T>(value: RpcResponse<T>): value is RpcFailure {
+  return "error" in value;
+}
+
+interface TtsPingResult {
+  language: string;
+  speaker: string;
+  speaker_id: number;
+  device: string;
+  speed: number;
+  sdp_ratio: number;
+  noise_scale: number;
+  noise_scale_w: number;
+  speaker_count: number;
+}
+
 export class MeloTtsService {
+  private processRef: ChildProcessWithoutNullStreams | null = null;
+  private shuttingDown = false;
+  private warmedUp = false;
+  private readonly pending = new Map<
+    string,
+    {
+      resolve: (value: unknown) => void;
+      reject: (reason?: unknown) => void;
+    }
+  >();
+  private nextId = 1;
+
   constructor(
     private readonly config: AppConfig,
     private readonly logger: Logger,
   ) {}
 
   async warmup(): Promise<void> {
+    if (this.warmedUp) {
+      return;
+    }
+
     await mkdir(path.resolve(process.cwd(), this.config.TTS_CACHE_DIR), { recursive: true });
     await mkdir(path.resolve(process.cwd(), this.config.TTS_OUTPUT_DIR), { recursive: true });
     const docker = await resolveDockerCommand(this.config);
 
     await run(docker, ["--version"]);
     await run(docker, ["image", "inspect", this.config.TTS_IMAGE]);
+
+    await this.start();
+    const result = await this.request<TtsPingResult>("ping", {});
+    this.logger.info("TTS worker ready", result);
+
+    const warmupFileName = `warmup-${randomUUID()}.wav`;
+    const warmupHostPath = path.resolve(process.cwd(), this.config.TTS_OUTPUT_DIR, warmupFileName);
+    try {
+      await this.request("synthesize", {
+        text: "안녕하세요. 로컬 티티에스 준비 테스트입니다.",
+        output_path: `/work/output/${warmupFileName}`,
+      });
+    } finally {
+      await rm(warmupHostPath, { force: true }).catch(() => undefined);
+    }
+
+    this.warmedUp = true;
   }
 
   async speak(text: string): Promise<void> {
@@ -66,62 +129,16 @@ export class MeloTtsService {
     const targetPath = path.resolve(process.cwd(), this.config.TTS_OUTPUT_DIR, fileName);
 
     try {
-      await this.synthesizeToFile(trimmed, targetPath);
+      await this.synthesizeToFile(trimmed, targetPath, fileName);
       await playWavFile(targetPath, this.config.TTS_PLAYBACK_RATE);
     } finally {
       await rm(targetPath, { force: true }).catch(() => undefined);
     }
   }
 
-  async synthesizeToFile(text: string, targetPath: string): Promise<void> {
+  async synthesizeToFile(text: string, targetPath: string, fileName?: string): Promise<void> {
     await this.warmup();
-
+    const resolvedFileName = fileName ?? path.basename(targetPath);
-    const outputDir = path.dirname(targetPath);
-    const cacheDir = path.resolve(process.cwd(), this.config.TTS_CACHE_DIR);
-    const fileName = path.basename(targetPath);
-
-    await mkdir(outputDir, { recursive: true });
-
-    const args = [
-      "run",
-      "--rm",
-      "-v",
-      `${outputDir}:/work/output`,
-      "-v",
-      `${cacheDir}:/cache`,
-      "-e",
-      "HF_HOME=/cache/huggingface",
-      "-e",
-      "HF_HUB_CACHE=/cache/huggingface/hub",
-      "-e",
-      "TRANSFORMERS_CACHE=/cache/transformers",
-    ];
-
-    if (this.config.TTS_DEVICE !== "cpu") {
-      args.push("--gpus", "all");
-    }
-
-    args.push(
-      this.config.TTS_IMAGE,
-      "--text",
-      text,
-      "--output",
-      `/work/output/${fileName}`,
-      "--language",
-      this.config.TTS_LANGUAGE,
-      "--speaker",
-      this.config.TTS_SPEAKER,
-      "--speed",
-      String(this.config.TTS_SPEED),
-      "--sdp-ratio",
-      String(this.config.TTS_SDP_RATIO),
-      "--noise-scale",
-      String(this.config.TTS_NOISE_SCALE),
-      "--noise-scale-w",
-      String(this.config.TTS_NOISE_SCALE_W),
-      "--device",
-      this.config.TTS_DEVICE,
-    );
 
     this.logger.info("Starting MeloTTS synthesis", {
       image: this.config.TTS_IMAGE,
@@ -135,8 +152,45 @@ export class MeloTtsService {
       device: this.config.TTS_DEVICE,
     });
 
-    const docker = await resolveDockerCommand(this.config);
+    await this.request("synthesize", {
-    await run(docker, args, "inherit");
+      text,
+      output_path: `/work/output/${resolvedFileName}`,
+    });
+  }
+
+  async destroy(): Promise<void> {
+    if (!this.processRef) {
+      return;
+    }
+
+    const child = this.processRef;
+    this.shuttingDown = true;
+
+    try {
+      child.stdin.end();
+    } catch {
+      // ignore
+    }
+
+    if (!child.killed && child.exitCode === null) {
+      child.kill("SIGTERM");
+    }
+
+    const timedWait = Promise.race([
+      once(child, "exit"),
+      new Promise<null>((resolve) => setTimeout(() => resolve(null), 1500)),
+    ]);
+
+    await timedWait;
+
+    if (child.exitCode === null && !child.killed) {
+      child.kill("SIGKILL");
+      await once(child, "exit").catch(() => null);
+    }
+
+    this.processRef = null;
+    this.shuttingDown = false;
+    this.warmedUp = false;
   }
 
   private normalizeText(input: string): string {
@@ -155,4 +209,156 @@ export class MeloTtsService {
 
     return `${collapsed}.`;
   }
+
+  private async start(): Promise<void> {
+    if (this.processRef) {
+      return;
+    }
+    if (this.shuttingDown) {
+      throw new Error("tts worker is shutting down");
+    }
+
+    const docker = await resolveDockerCommand(this.config);
+    const outputDir = path.resolve(process.cwd(), this.config.TTS_OUTPUT_DIR);
+    const cacheDir = path.resolve(process.cwd(), this.config.TTS_CACHE_DIR);
+
+    await mkdir(outputDir, { recursive: true });
+    await mkdir(cacheDir, { recursive: true });
+
+    const args = [
+      "run",
+      "--rm",
+      "-i",
+      "-v",
+      `${outputDir}:/work/output`,
+      "-v",
+      `${cacheDir}:/cache`,
+      "-e",
+      "HF_HOME=/cache/huggingface",
+      "-e",
+      "HF_HUB_CACHE=/cache/huggingface/hub",
+      "-e",
+      "TRANSFORMERS_CACHE=/cache/transformers",
+      "-e",
+      `TTS_LANGUAGE=${this.config.TTS_LANGUAGE}`,
+      "-e",
+      `TTS_SPEAKER=${this.config.TTS_SPEAKER}`,
+      "-e",
+      `TTS_DEVICE=${this.config.TTS_DEVICE}`,
+      "-e",
+      `TTS_SPEED=${this.config.TTS_SPEED}`,
+      "-e",
+      `TTS_SDP_RATIO=${this.config.TTS_SDP_RATIO}`,
+      "-e",
+      `TTS_NOISE_SCALE=${this.config.TTS_NOISE_SCALE}`,
+      "-e",
+      `TTS_NOISE_SCALE_W=${this.config.TTS_NOISE_SCALE_W}`,
+      "--entrypoint",
+      "python",
+    ];
+
+    if (this.config.TTS_DEVICE !== "cpu") {
+      args.push("--gpus", "all");
+    }
+
+    args.push(
+      this.config.TTS_IMAGE,
+      "/opt/realtime-voice-bot/melo_tts_worker.py",
+    );
+
+    const env = { ...process.env };
+    if (path.isAbsolute(docker)) {
+      const dockerBinDir = path.dirname(docker);
+      const currentPath = env.PATH ?? env.Path ?? "";
+      env.PATH = `${dockerBinDir}${path.delimiter}${currentPath}`;
+    }
+
+    this.processRef = spawn(docker, args, {
+      stdio: ["pipe", "pipe", "pipe"],
+      windowsHide: true,
+      shell: process.platform === "win32" && !path.isAbsolute(docker),
+      env,
+    });
+
+    const rl = createInterface({
+      input: this.processRef.stdout,
+      crlfDelay: Infinity,
+    });
+
+    rl.on("line", (line) => {
+      this.handleStdoutLine(line);
+    });
+
+    this.processRef.stderr.on("data", (chunk: Buffer) => {
+      const text = chunk.toString().trim();
+      if (text.length > 0) {
+        this.logger.warn(`[melotts] ${text}`);
+      }
+    });
+
+    this.processRef.stdin.on("error", (error) => {
+      this.logger.debug("melotts stdin error", error);
+    });
+
+    this.processRef.on("exit", (code, signal) => {
+      const error = new Error(`melotts worker exited code=${code ?? "null"} signal=${signal ?? "null"}`);
+      for (const entry of this.pending.values()) {
+        entry.reject(error);
+      }
+      this.pending.clear();
+      this.processRef = null;
+    });
+  }
+
+  private async request<T>(method: string, params: Record<string, unknown>): Promise<T> {
+    await this.start();
+
+    if (!this.processRef) {
+      throw new Error("melotts worker is not running");
+    }
+
+    const id = String(this.nextId++);
+    const payload = JSON.stringify({
+      id,
+      method,
+      params,
+    });
+
+    const promise = new Promise<T>((resolve, reject) => {
+      this.pending.set(id, {
+        resolve: (value) => resolve(value as T),
+        reject,
+      });
+    });
+
+    this.processRef.stdin.write(`${payload}\n`);
+    return await promise;
+  }
+
+  private handleStdoutLine(line: string): void {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      return;
+    }
+
+    let message: RpcResponse<unknown>;
+    try {
+      message = JSON.parse(trimmed) as RpcResponse<unknown>;
+    } catch (error) {
+      this.logger.warn("melotts stdout parse failed", error);
+      return;
+    }
+
+    const pending = this.pending.get(message.id);
+    if (!pending) {
+      return;
+    }
+
+    this.pending.delete(message.id);
+    if (isFailure(message)) {
+      pending.reject(new Error(message.error));
+      return;
+    }
+    pending.resolve(message.result);
+  }
 }

src/services/ollama-llm.ts

@@ -378,6 +378,7 @@ export class OllamaLlmService {
         "bun run devices",
         "bun run test:stt",
         "bun run test:sttllm",
+        "bun run test:all",
         "bun run test:llm",
         "bun run test:tts -- \"안녕하세요\"",
       ],

src/setup-tts.ts

@@ -1,5 +1,5 @@
 import process from "node:process";
-import { mkdir, rm } from "node:fs/promises";
+import { mkdir } from "node:fs/promises";
 import path from "node:path";
 import { spawn } from "node:child_process";
 
@@ -57,13 +57,12 @@ export async function setupTts(): Promise<void> {
   await run(docker, ["build", "-t", config.TTS_IMAGE, dockerContext]);
 
   const tts = new MeloTtsService(config, logger);
-  const warmupPath = path.join(outputDir, "warmup.wav");
 
   console.log("MeloTTS 모델 워밍업...");
   try {
-    await tts.synthesizeToFile("안녕하세요. 로컬 티티에스 준비 테스트입니다.", warmupPath);
+    await tts.warmup();
   } finally {
-    await rm(warmupPath, { force: true }).catch(() => undefined);
+    await tts.destroy().catch(() => undefined);
   }
 
   console.log("로컬 TTS 환경 준비 완료");