/** * HTTP client for the Python brain bridge (bridge/server.py). * All AI work (STT, reply engine, TTS) lives behind these calls. */ import { config } from "./config.ts"; export type BroadcastAction = "start" | "stop"; export interface ConverseResult { transcript: string; language?: string | null; reply: string; error?: string | null; /** Broadcast directive the brain requested this turn (setBroadcast tool). */ broadcast_action?: BroadcastAction | null; /** base64-encoded 16-bit PCM WAV of the spoken reply, or null if TTS off */ audio_b64?: string | null; } export interface TextResult { reply: string; error?: string | null; broadcast_action?: BroadcastAction | null; audio_b64?: string | null; } /** Full voice turn: WAV in -> {transcript, reply, reply audio}. ``broadcasting`` * is the current live screen-share state so the brain routes search (Chrome * while live, Gemini when off). */ export async function converse(wav: Buffer, broadcasting?: boolean): Promise { const qs = broadcasting === undefined ? "" : `?broadcasting=${broadcasting ? "1" : "0"}`; const res = await fetch(`${config.bridgeUrl}/converse${qs}`, { method: "POST", headers: { "content-type": "audio/wav" }, body: wav, }); if (!res.ok) throw new Error(`bridge /converse ${res.status}: ${await res.text()}`); return (await res.json()) as ConverseResult; } /** Metadata for a streamed turn: everything except the audio clips. */ export interface ConverseMeta { transcript: string; language?: string | null; reply: string; error?: string | null; broadcast_action?: BroadcastAction | null; /** Why this turn produced (or didn't produce) a transcript/reply. */ note?: string; /** Per-stage timing (seconds) for diagnosing latency. */ stt_sec?: number; think_sec?: number; /** Wall-clock LLM window (epoch ms) so the transcript channel can show when * the reply engine started/finished. */ llm_start_ms?: number; llm_end_ms?: number; } /** Final event of a streamed turn, carrying TTS timing (synthesis runs after * the meta line, so it can't be reported there). */ export interface ConverseEnd { tts_sec?: number; /** Wall-clock TTS window (epoch ms). */ tts_start_ms?: number; tts_end_ms?: number; } export interface ConverseStreamHandlers { /** Fired once, before any audio, with the transcript/reply/broadcast directive. */ onMeta?: (meta: ConverseMeta) => void | Promise; /** Fired per sentence as its audio finishes synthesising (in order). */ onAudio?: (wav: Buffer) => void | Promise; /** Fired once after the last clip, with TTS timing. */ onEnd?: (end: ConverseEnd) => void | Promise; } /** Parse a byte stream of newline-delimited JSON into objects, one per line. */ export async function* ndjson( stream: AsyncIterable, ): AsyncGenerator { const decoder = new TextDecoder(); let buf = ""; for await (const chunk of stream) { buf += decoder.decode(chunk, { stream: true }); let nl: number; while ((nl = buf.indexOf("\n")) >= 0) { const line = buf.slice(0, nl).trim(); buf = buf.slice(nl + 1); if (line) yield JSON.parse(line); } } const last = buf.trim(); if (last) yield JSON.parse(last); } /** Streaming voice turn: the bridge emits the transcript/reply first, then one * audio clip per sentence as it is synthesised. Handlers run in arrival order, * so playing each clip on arrival starts the first sentence while the rest are * still being spoken. Mirrors {@link converse} but pipelines TTS. */ export async function converseStream( wav: Buffer, broadcasting: boolean | undefined, handlers: ConverseStreamHandlers, ): Promise { const qs = broadcasting === undefined ? "" : `?broadcasting=${broadcasting ? "1" : "0"}`; const res = await fetch(`${config.bridgeUrl}/converse_stream${qs}`, { method: "POST", headers: { "content-type": "audio/wav" }, body: wav, }); if (!res.ok || !res.body) { throw new Error(`bridge /converse_stream ${res.status}: ${await res.text().catch(() => "")}`); } for await (const ev of ndjson(res.body as AsyncIterable)) { if (ev.type === "meta") { await handlers.onMeta?.(ev as ConverseMeta); } else if (ev.type === "audio" && ev.audio_b64) { const clip = decodeWav(ev.audio_b64); if (clip) await handlers.onAudio?.(clip); } else if (ev.type === "end") { await handlers.onEnd?.(ev as ConverseEnd); } } } /** Text-only turn (used by /자비스 ask). */ export async function ask(text: string): Promise { const res = await fetch(`${config.bridgeUrl}/text`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text }), }); if (!res.ok) throw new Error(`bridge /text ${res.status}: ${await res.text()}`); return (await res.json()) as TextResult; } export async function health(): Promise { const res = await fetch(`${config.bridgeUrl}/health`); return res.json(); } export function decodeWav(audio_b64?: string | null): Buffer | null { if (!audio_b64) return null; return Buffer.from(audio_b64, "base64"); }