diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d37752 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +__pycache__/ +*.pyc +.venv/ +venv/ +env/ +tmp/ +*.wav +!static/** +.DS_Store +.hf-cache/ diff --git a/README.md b/README.md index 66c2970..dfddaa7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,79 @@ # voice_copy +참조 음성 몇 초만 있으면 아무 텍스트나 **그 목소리로** 합성해 주는 zero-shot 음성 클로닝 웹앱입니다. +[jamiepine/voicebox](https://github.com/jamiepine/voicebox)의 음성 클로닝 기능(Qwen3-TTS Base 엔진)만 +떼어내 간단한 UI로 감쌌습니다. + +- 엔진: **Qwen3-TTS-12Hz Base** (`qwen-tts` 패키지, zero-shot voice clone) +- 지원 언어: 한국어 · English · 中文 · 日本語 · 독일어 · 프랑스어 · 러시아어 · 포르투갈어 · 스페인어 · 이탈리아어 +- UI: 파일 업로드 또는 마이크 녹음 → 전사 입력 → 합성할 텍스트 입력 → 생성/재생/다운로드 + +## 필요 사항 + +- Python 3.9+ (3.10~3.12 권장) +- NVIDIA GPU 권장 (CPU도 되지만 매우 느림). VRAM 6GB+ 권장. +- (선택) `ffmpeg` — mp3/m4a/webm 업로드를 디코딩할 때 필요. wav/flac와 브라우저 녹음(WAV)은 ffmpeg 없이 동작합니다. + +## 설치 + +```bash +# 1) 자신의 환경에 맞는 torch를 먼저 설치 +pip install torch --index-url https://download.pytorch.org/whl/cu128 # NVIDIA(CUDA 12.8) +# CPU만 있으면: pip install torch + +# 2) 나머지 의존성 +pip install -r requirements.txt +``` + +## 실행 + +```bash +python app.py +# 브라우저에서 http://localhost:8000 접속 +``` + +첫 생성 요청 때 Qwen3-TTS 가중치(수 GB)가 HuggingFace에서 자동으로 다운로드됩니다. 이후에는 캐시됩니다. + +### 환경 변수 + +| 변수 | 기본값 | 설명 | +|------|--------|------| +| `VOICE_COPY_MODEL_SIZE` | `1.7B` | 모델 크기. `1.7B`(품질↑) 또는 `0.6B`(가벼움) | +| `VOICE_COPY_DEVICE` | 자동 | `cuda` / `cpu` / `mps` 강제 지정 | +| `VOICE_COPY_HOST` | `0.0.0.0` | 바인드 호스트 | +| `VOICE_COPY_PORT` | `8000` | 포트 | + +예: `VOICE_COPY_MODEL_SIZE=0.6B python app.py` + +## 사용법 + +1. **참조 음성** — 클로닝할 목소리를 5~15초 정도 업로드하거나 마이크로 녹음합니다. +2. **전사** — 그 참조 음성에서 말한 내용을 그대로 텍스트로 적습니다. (클로닝 품질에 중요) +3. **합성할 텍스트** — 그 목소리로 말하게 할 문장을 입력하고 "목소리 생성"을 누릅니다. + +## API + +- `GET /api/health` — 디바이스/모델/언어 상태 +- `POST /api/clone` — multipart form: `ref_audio`(파일), `ref_text`, `text`, `language`, `instruct`(선택), `seed`(선택) → `audio/wav` 반환 + +```bash +curl -X POST http://localhost:8000/api/clone \ + -F ref_audio=@sample.wav -F ref_text="안녕하세요" \ + -F text="복제된 목소리로 말합니다." -F language=ko \ + --output cloned.wav +``` + +## 구조 + +``` +app.py FastAPI 서버 (UI 제공 + /api/clone) +engine.py Qwen3-TTS 클로닝 래퍼 (voicebox pytorch_backend에서 추출/단순화) +static/index.html 단일 페이지 UI +requirements.txt +``` + +## 라이선스 / 출처 + +음성 클로닝 로직은 [jamiepine/voicebox](https://github.com/jamiepine/voicebox)에서 파생했습니다. +모델 가중치 및 사용은 각 모델(Qwen3-TTS)과 voicebox의 라이선스를 따르며, 목소리 클로닝은 +반드시 본인 또는 동의를 받은 사람의 음성에만 사용하세요. diff --git a/app.py b/app.py new file mode 100644 index 0000000..2740254 --- /dev/null +++ b/app.py @@ -0,0 +1,156 @@ +""" +voice_copy — simple web UI for zero-shot voice cloning. + +Run: + python app.py # then open http://localhost:8000 + # or: uvicorn app:app --host 0.0.0.0 --port 8000 + +Env: + VOICE_COPY_MODEL_SIZE "1.7B" (default) or "0.6B" + VOICE_COPY_DEVICE force "cuda" / "cpu" / "mps" (default: auto) + VOICE_COPY_HOST bind host (default 0.0.0.0) + VOICE_COPY_PORT bind port (default 8000) +""" + +from __future__ import annotations + +import io +import logging +import os +import tempfile +from pathlib import Path + +import numpy as np +import soundfile as sf +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import HTMLResponse, Response + +from engine import HF_MODEL_MAP, LANGUAGE_CODE_TO_NAME, VoiceCloner + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("voice_copy.app") + +BASE = Path(__file__).resolve().parent +STATIC = BASE / "static" +TMP = BASE / "tmp" +TMP.mkdir(exist_ok=True) + +MODEL_SIZE = os.environ.get("VOICE_COPY_MODEL_SIZE", "1.7B") +DEVICE = os.environ.get("VOICE_COPY_DEVICE") or None + +cloner = VoiceCloner(model_size=MODEL_SIZE, device=DEVICE) +app = FastAPI(title="voice_copy", description="Zero-shot voice cloning UI") + + +def _decode_to_wav(raw_bytes: bytes, filename: str) -> str: + """Decode uploaded audio (wav/flac/mp3/m4a/ogg/webm) into a mono wav on disk. + + Tries soundfile first (wav/flac/ogg — no ffmpeg needed); falls back to + librosa/audioread (needs ffmpeg for mp3/m4a/webm from the mic recorder). + """ + suffix = Path(filename or "ref").suffix or ".bin" + src = tempfile.NamedTemporaryFile(dir=TMP, suffix=suffix, delete=False) + src.write(raw_bytes) + src.close() + + try: + try: + audio, sr = sf.read(src.name, dtype="float32", always_2d=False) + if audio.ndim > 1: # stereo -> mono + audio = audio.mean(axis=1) + except Exception: + import librosa + + audio, sr = librosa.load(src.name, sr=None, mono=True) + finally: + try: + os.unlink(src.name) + except OSError: + pass + + wav = tempfile.NamedTemporaryFile(dir=TMP, suffix=".wav", delete=False) + wav.close() + sf.write(wav.name, np.asarray(audio, dtype=np.float32), int(sr), format="WAV") + return wav.name + + +@app.get("/", response_class=HTMLResponse) +def index() -> str: + return (STATIC / "index.html").read_text(encoding="utf-8") + + +@app.get("/api/health") +def health() -> dict: + return { + "device": cloner.device, + "model_size": cloner.model_size, + "model_repo": HF_MODEL_MAP[cloner.model_size], + "model_loaded": cloner.loaded, + "languages": list(LANGUAGE_CODE_TO_NAME.keys()), + } + + +@app.post("/api/clone") +async def clone( + ref_audio: UploadFile = File(...), + ref_text: str = Form(...), + text: str = Form(...), + language: str = Form("auto"), + instruct: str = Form(""), + seed: str = Form(""), +) -> Response: + if not text.strip(): + raise HTTPException(400, "합성할 텍스트(text)를 입력하세요.") + if not ref_text.strip(): + raise HTTPException(400, "참조 오디오의 전사(ref_text)를 입력하세요.") + + raw = await ref_audio.read() + if not raw: + raise HTTPException(400, "참조 오디오 파일이 비어 있습니다.") + + seed_int = None + if seed.strip(): + try: + seed_int = int(seed) + except ValueError: + raise HTTPException(400, "seed는 정수여야 합니다.") + + try: + wav_path = await run_in_threadpool(_decode_to_wav, raw, ref_audio.filename or "ref.wav") + except Exception as exc: # noqa: BLE001 + logger.exception("audio decode failed") + raise HTTPException( + 400, + f"오디오 디코딩 실패: {exc}. wav/flac는 그대로 되고, mp3/m4a/webm은 ffmpeg 설치가 필요합니다.", + ) + + try: + audio, sr = await run_in_threadpool( + cloner.clone, wav_path, ref_text, text, language, instruct or None, seed_int + ) + except Exception as exc: # noqa: BLE001 + logger.exception("clone failed") + raise HTTPException(500, f"생성 실패: {exc}") + finally: + try: + os.unlink(wav_path) + except OSError: + pass + + buf = io.BytesIO() + sf.write(buf, audio, sr, format="WAV") + buf.seek(0) + return Response( + content=buf.read(), + media_type="audio/wav", + headers={"Content-Disposition": 'inline; filename="cloned.wav"'}, + ) + + +if __name__ == "__main__": + import uvicorn + + host = os.environ.get("VOICE_COPY_HOST", "0.0.0.0") + port = int(os.environ.get("VOICE_COPY_PORT", "8000")) + uvicorn.run(app, host=host, port=port) diff --git a/engine.py b/engine.py new file mode 100644 index 0000000..d5d7b0e --- /dev/null +++ b/engine.py @@ -0,0 +1,151 @@ +""" +Zero-shot voice cloning engine. + +Extracted and simplified from jamiepine/voicebox's Qwen3-TTS "Base" backend +(backend/backends/pytorch_backend.py). Only the voice-cloning path is kept: +give a short reference clip + its transcript, then synthesize any text in +that cloned voice. + +Model: Qwen/Qwen3-TTS-12Hz-{1.7B,0.6B}-Base via the `qwen-tts` package. +Weights download automatically from HuggingFace on first use. +""" + +from __future__ import annotations + +import logging +import threading +from typing import Optional, Tuple + +import numpy as np + +logger = logging.getLogger("voice_copy.engine") + +# Qwen3-TTS speaks these languages. "auto" lets the model detect it. +LANGUAGE_CODE_TO_NAME = { + "auto": "auto", + "ko": "korean", + "en": "english", + "zh": "chinese", + "ja": "japanese", + "de": "german", + "fr": "french", + "ru": "russian", + "pt": "portuguese", + "es": "spanish", + "it": "italian", +} + +HF_MODEL_MAP = { + "1.7B": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "0.6B": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", +} + + +def pick_device() -> str: + """Best available torch device: CUDA GPU if present, else CPU.""" + try: + import torch + except Exception: + return "cpu" + if torch.cuda.is_available(): + return "cuda" + # Apple Silicon + if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available(): + return "mps" + return "cpu" + + +class VoiceCloner: + """Lazily-loaded Qwen3-TTS voice-clone wrapper. + + The heavy model is only loaded on the first `clone()` call, so the web + server can start instantly and the (multi-GB) download happens on demand. + """ + + def __init__(self, model_size: str = "1.7B", device: Optional[str] = None): + if model_size not in HF_MODEL_MAP: + raise ValueError(f"Unknown model size {model_size!r}; choose one of {list(HF_MODEL_MAP)}") + self.model = None + self.model_size = model_size + self.device = device or pick_device() + self._lock = threading.Lock() + + @property + def loaded(self) -> bool: + return self.model is not None + + def load(self) -> None: + """Load the model (thread-safe, idempotent).""" + if self.model is not None: + return + with self._lock: + if self.model is not None: + return + import torch + from qwen_tts import Qwen3TTSModel + + model_path = HF_MODEL_MAP[self.model_size] + logger.info("Loading %s on %s (first run downloads weights)...", model_path, self.device) + + if self.device == "cpu": + self.model = Qwen3TTSModel.from_pretrained( + model_path, + torch_dtype=torch.float32, + low_cpu_mem_usage=False, + ) + else: + self.model = Qwen3TTSModel.from_pretrained( + model_path, + device_map=self.device, + torch_dtype=torch.bfloat16, + ) + logger.info("Model loaded.") + + def clone( + self, + ref_audio_path: str, + ref_text: str, + text: str, + language: str = "auto", + instruct: Optional[str] = None, + seed: Optional[int] = None, + ) -> Tuple[np.ndarray, int]: + """Synthesize `text` in the voice of `ref_audio_path`. + + Args: + ref_audio_path: Path to a short reference clip (a few seconds is enough). + ref_text: Exact transcript of the reference clip. + text: The text to speak in the cloned voice. + language: Language code from LANGUAGE_CODE_TO_NAME ("auto" to detect). + instruct: Optional natural-language style hint (e.g. "speak cheerfully"). + seed: Optional random seed for reproducible output. + + Returns: + (audio_float32_mono, sample_rate) + """ + self.load() + import torch + + if seed is not None: + torch.manual_seed(seed) + if self.device == "cuda": + torch.cuda.manual_seed_all(seed) + + # Build the speaker prompt from the reference clip + its transcript. + voice_prompt = self.model.create_voice_clone_prompt( + ref_audio=str(ref_audio_path), + ref_text=ref_text, + x_vector_only_mode=False, + ) + + wavs, sample_rate = self.model.generate_voice_clone( + text=text, + voice_clone_prompt=voice_prompt, + language=LANGUAGE_CODE_TO_NAME.get(language, "auto"), + instruct=instruct or None, + ) + + audio = wavs[0] + if isinstance(audio, torch.Tensor): + audio = audio.squeeze().cpu().numpy() + return np.asarray(audio, dtype=np.float32), int(sample_rate) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ea3bca9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +# voice_copy — zero-shot voice cloning UI (Qwen3-TTS) +# +# NOTE: install a torch build that matches your machine FIRST, e.g. +# CPU: pip install torch +# NVIDIA (CUDA): pip install torch --index-url https://download.pytorch.org/whl/cu128 +# then: pip install -r requirements.txt + +qwen-tts>=0.1.1 # Qwen3-TTS model + generate_voice_clone (pulls transformers/accelerate/torchaudio/soundfile/librosa) +fastapi>=0.109.0 +uvicorn[standard]>=0.27.0 +python-multipart>=0.0.6 # multipart form uploads +soundfile>=0.12.0 +librosa>=0.10.0 +numpy>=1.24 +torch>=2.2.0 # keep the torch you installed above; listed so a bare install still works diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..4e892c6 --- /dev/null +++ b/static/index.html @@ -0,0 +1,208 @@ + + + + + +voice_copy · 음성 클로닝 + + + +
+

voice_copy …

+

참조 음성 몇 초 + 그 전사를 주면, 아무 텍스트나 그 목소리로 합성합니다. (Qwen3-TTS zero-shot 클로닝)

+ +
+ + +
+ + + 마이크로 직접 녹음할 수도 있어요 +
+ +
+ +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+ + +
+ + + +