javis_bot/bot/src/bridge.ts

/**
 * HTTP client for the Python brain bridge (bridge/server.py).
 * All AI work (STT, reply engine, TTS) lives behind these calls.
 */
import { config } from "./config.ts";

export type BroadcastAction = "start" | "stop";

export interface ConverseResult {
  transcript: string;
  language?: string | null;
  reply: string;
  error?: string | null;
  /** Broadcast directive the brain requested this turn (setBroadcast tool). */
  broadcast_action?: BroadcastAction | null;
  /** base64-encoded 16-bit PCM WAV of the spoken reply, or null if TTS off */
  audio_b64?: string | null;
}

export interface TextResult {
  reply: string;
  error?: string | null;
  broadcast_action?: BroadcastAction | null;
  audio_b64?: string | null;
}

/** Full voice turn: WAV in -> {transcript, reply, reply audio}. ``broadcasting``
 *  is the current live screen-share state so the brain routes search (Chrome
 *  while live, Gemini when off). */
export async function converse(wav: Buffer, broadcasting?: boolean): Promise<ConverseResult> {
  const qs = broadcasting === undefined ? "" : `?broadcasting=${broadcasting ? "1" : "0"}`;
  const res = await fetch(`${config.bridgeUrl}/converse${qs}`, {
    method: "POST",
    headers: { "content-type": "audio/wav" },
    body: wav,
  });
  if (!res.ok) throw new Error(`bridge /converse ${res.status}: ${await res.text()}`);
  return (await res.json()) as ConverseResult;
}

/** Metadata for a streamed turn: everything except the audio clips. */
export interface ConverseMeta {
  transcript: string;
  language?: string | null;
  reply: string;
  error?: string | null;
  broadcast_action?: BroadcastAction | null;
  /** Why this turn produced (or didn't produce) a transcript/reply. */
  note?: string;
  /** Per-stage timing (seconds) for diagnosing latency. */
  stt_sec?: number;
  think_sec?: number;
  /** Wall-clock LLM window (epoch ms) so the transcript channel can show when
   *  the reply engine started/finished. */
  llm_start_ms?: number;
  llm_end_ms?: number;
}

/** Final event of a streamed turn, carrying TTS timing (synthesis runs after
 *  the meta line, so it can't be reported there). */
export interface ConverseEnd {
  tts_sec?: number;
  /** Wall-clock TTS window (epoch ms). */
  tts_start_ms?: number;
  tts_end_ms?: number;
}

export interface ConverseStreamHandlers {
  /** Fired once, before any audio, with the transcript/reply/broadcast directive. */
  onMeta?: (meta: ConverseMeta) => void | Promise<void>;
  /** Fired per sentence as its audio finishes synthesising (in order). */
  onAudio?: (wav: Buffer) => void | Promise<void>;
  /** Fired once after the last clip, with TTS timing. */
  onEnd?: (end: ConverseEnd) => void | Promise<void>;
}

/** Parse a byte stream of newline-delimited JSON into objects, one per line. */
export async function* ndjson(
  stream: AsyncIterable<Uint8Array>,
): AsyncGenerator<any> {
  const decoder = new TextDecoder();
  let buf = "";
  for await (const chunk of stream) {
    buf += decoder.decode(chunk, { stream: true });
    let nl: number;
    while ((nl = buf.indexOf("\n")) >= 0) {
      const line = buf.slice(0, nl).trim();
      buf = buf.slice(nl + 1);
      if (line) yield JSON.parse(line);
    }
  }
  const last = buf.trim();
  if (last) yield JSON.parse(last);
}

/** Streaming voice turn: the bridge emits the transcript/reply first, then one
 *  audio clip per sentence as it is synthesised. Handlers run in arrival order,
 *  so playing each clip on arrival starts the first sentence while the rest are
 *  still being spoken. Mirrors {@link converse} but pipelines TTS. */
export async function converseStream(
  wav: Buffer,
  broadcasting: boolean | undefined,
  handlers: ConverseStreamHandlers,
): Promise<void> {
  const qs = broadcasting === undefined ? "" : `?broadcasting=${broadcasting ? "1" : "0"}`;
  const res = await fetch(`${config.bridgeUrl}/converse_stream${qs}`, {
    method: "POST",
    headers: { "content-type": "audio/wav" },
    body: wav,
  });
  if (!res.ok || !res.body) {
    throw new Error(`bridge /converse_stream ${res.status}: ${await res.text().catch(() => "")}`);
  }
  for await (const ev of ndjson(res.body as AsyncIterable<Uint8Array>)) {
    if (ev.type === "meta") {
      await handlers.onMeta?.(ev as ConverseMeta);
    } else if (ev.type === "audio" && ev.audio_b64) {
      const clip = decodeWav(ev.audio_b64);
      if (clip) await handlers.onAudio?.(clip);
    } else if (ev.type === "end") {
      await handlers.onEnd?.(ev as ConverseEnd);
    }
  }
}

/** Text-only turn (used by /자비스 ask). */
export async function ask(text: string): Promise<TextResult> {
  const res = await fetch(`${config.bridgeUrl}/text`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ text }),
  });
  if (!res.ok) throw new Error(`bridge /text ${res.status}: ${await res.text()}`);
  return (await res.json()) as TextResult;
}

export async function health(): Promise<any> {
  const res = await fetch(`${config.bridgeUrl}/health`);
  return res.json();
}

export function decodeWav(audio_b64?: string | null): Buffer | null {
  if (!audio_b64) return null;
  return Buffer.from(audio_b64, "base64");
}