Add Discord-native hybrid front-end for Jarvis (bot + bridge)
Some checks failed
Release / semantic-release (push) Successful in 59s
tests / Unit tests (Linux, Python 3.11) (push) Successful in 13m45s
Release / build-linux (push) Failing after 7m47s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled

Transform isair/jarvis into a Discord-controlled voice assistant running on
the Ubuntu VNC desktop, keeping the mature ~39k-line Python brain intact.

- bot/ (Node + bun, discord.js): /자비스 slash commands (ephemeral),
  voice channel join + voice receive/playback, pluggable VNC screen broadcast
  (selfbot live / noVNC / screenshot)
- bridge/ (Python, Flask): wraps jarvis STT + run_reply_engine + Piper TTS
  behind a thin localhost HTTP API
- .env.example, scripts/ (start_bridge/start_bot/dev), README rewrite,
  docs/language-comparison.md and docs/vnc-xfce-setup.md

Language decision: hybrid (Python brain + Node/bun Discord layer) because
Discord blocks bot video; native screen broadcast only works via a Node
selfbot library.
This commit is contained in:
javis-bot
2026-06-09 14:51:05 +09:00
parent a5bf8d1826
commit c4abf63f38
308 changed files with 94135 additions and 1 deletions

51
bot/src/stream/index.ts Normal file
View File

@@ -0,0 +1,51 @@
/**
* Pluggable VNC screen-broadcast backends.
*
* Per the chosen design (option 1): the streaming method is swappable via
* STREAM_BACKEND in .env. The default is the real live "Go Live" stream via a
* selfbot account (only way to get a native Discord video broadcast), with safe
* fallbacks (noVNC link / periodic screenshots) available without code changes.
*/
import type { AppConfig } from "../config.ts";
export interface StreamContext {
guildId: string;
voiceChannelId: string;
/** Post an image to the invoking text channel (used by the screenshot backend). */
postImage?: (png: Buffer, name: string) => Promise<void>;
}
export interface ScreenStreamer {
readonly kind: AppConfig["streamBackend"];
/** Start broadcasting. Returns a short user-facing status/link message. */
start(ctx: StreamContext): Promise<string>;
stop(): Promise<void>;
isActive(): boolean;
}
export async function createStreamer(config: AppConfig): Promise<ScreenStreamer> {
switch (config.streamBackend) {
case "selfbot": {
const { SelfbotStreamer } = await import("./selfbot.ts");
return new SelfbotStreamer(config);
}
case "novnc": {
const { NoVncStreamer } = await import("./novnc.ts");
return new NoVncStreamer(config);
}
case "screenshot": {
const { ScreenshotStreamer } = await import("./screenshot.ts");
return new ScreenshotStreamer(config);
}
case "none":
default:
return {
kind: "none",
async start() {
return "화면 송출이 비활성화되어 있습니다 (STREAM_BACKEND=none).";
},
async stop() {},
isActive: () => false,
};
}
}

34
bot/src/stream/novnc.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* noVNC link backend (safe, real-time, no ban risk).
*
* Does not broadcast natively into Discord. Instead it shares a noVNC web URL
* that anyone can open in a browser to watch (and optionally control) the VNC
* desktop live. Set NOVNC_URL in .env (e.g. http://192.168.10.9:6080/vnc.html).
*
* Stand up noVNC once on the host with websockify, e.g.:
* websockify --web=/usr/share/novnc 6080 localhost:5901
*/
import type { AppConfig } from "../config.ts";
import type { ScreenStreamer, StreamContext } from "./index.ts";
export class NoVncStreamer implements ScreenStreamer {
readonly kind = "novnc" as const;
private active = false;
constructor(private config: AppConfig) {}
isActive() {
return this.active;
}
async start(_ctx: StreamContext): Promise<string> {
if (!this.config.novncUrl) {
return "NOVNC_URL이 설정되지 않았습니다 (.env). 예: http://192.168.10.9:6080/vnc.html";
}
this.active = true;
return `🖥️ VNC 화면 실시간 보기 (브라우저): ${this.config.novncUrl}`;
}
async stop(): Promise<void> {
this.active = false;
}
}

View File

@@ -0,0 +1,62 @@
/**
* Screenshot backend (safe, no ban risk, not real-time).
*
* Periodically grabs a frame from the VNC X display with ffmpeg's x11grab and
* posts it to the invoking text channel. Low FPS, but works with a normal bot
* account and never touches Discord's selfbot surface.
*/
import { spawn } from "node:child_process";
import type { AppConfig } from "../config.ts";
import type { ScreenStreamer, StreamContext } from "./index.ts";
function grabFrame(display: string, size: string): Promise<Buffer> {
return new Promise((resolve, reject) => {
const ff = spawn("ffmpeg", [
"-loglevel", "error",
"-f", "x11grab",
"-video_size", size,
"-i", display,
"-frames:v", "1",
"-f", "image2pipe",
"-vcodec", "png",
"pipe:1",
]);
const chunks: Buffer[] = [];
ff.stdout.on("data", (c) => chunks.push(c));
ff.on("error", reject);
ff.on("close", (code) =>
code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg exited ${code}`)),
);
});
}
export class ScreenshotStreamer implements ScreenStreamer {
readonly kind = "screenshot" as const;
private timer: ReturnType<typeof setInterval> | null = null;
constructor(private config: AppConfig) {}
isActive() {
return this.timer !== null;
}
async start(ctx: StreamContext): Promise<string> {
if (!ctx.postImage) return "스크린샷을 올릴 텍스트 채널 컨텍스트가 없습니다.";
if (this.timer) return "이미 스크린샷 송출 중입니다.";
const tick = async () => {
try {
const png = await grabFrame(this.config.vncDisplay, this.config.vncResolution);
await ctx.postImage!(png, "vnc.png");
} catch (e) {
console.error("[screenshot] grab failed:", e);
}
};
this.timer = setInterval(tick, this.config.screenshotIntervalSec * 1000);
void tick();
return `📸 ${this.config.screenshotIntervalSec}초마다 VNC 스크린샷을 이 채널에 올립니다.`;
}
async stop(): Promise<void> {
if (this.timer) clearInterval(this.timer);
this.timer = null;
}
}

116
bot/src/stream/selfbot.ts Normal file
View File

@@ -0,0 +1,116 @@
/**
* Selfbot live-stream backend (default).
*
* Streams the VNC X display (:1) into the voice channel as a real Discord
* "Go Live" broadcast. Discord blocks video from *bot* accounts, so this path
* requires a USER account token (a "selfbot"), which violates Discord ToS and
* can get the account banned. Use a throwaway/burner account, never your main.
*
* Dependencies are optional (native): install with
* bun add discord.js-selfbot-v13 @dank074/discord-video-stream
* They are dynamically imported so the core bot installs/runs without them.
*
* Library API targets @dank074/discord-video-stream v6 (Streamer / prepareStream
* / playStream). If a different major is installed, the import guard below will
* point you at the docs rather than crash cryptically.
*/
import type { AppConfig } from "../config.ts";
import type { ScreenStreamer, StreamContext } from "./index.ts";
export class SelfbotStreamer implements ScreenStreamer {
readonly kind = "selfbot" as const;
private config: AppConfig;
private streamer: any = null;
private controller: AbortController | null = null;
private active = false;
constructor(config: AppConfig) {
this.config = config;
}
isActive() {
return this.active;
}
private async loadLib() {
let selfbot: any, videoStream: any;
try {
selfbot = await import("discord.js-selfbot-v13");
// Optional native dep; resolved at runtime only. Version/name can vary by
// upstream release, so we don't hard-bind its types at compile time.
// @ts-ignore - optional dependency, may be absent until `bun add`ed
videoStream = await import("@dank074/discord-video-stream");
} catch (e) {
throw new Error(
"셀프봇 송출 의존성이 없습니다. 설치: bun add discord.js-selfbot-v13 @dank074/discord-video-stream\n" +
`원본 오류: ${(e as Error).message}`,
);
}
if (!videoStream.Streamer || !videoStream.prepareStream || !videoStream.playStream) {
throw new Error(
"@dank074/discord-video-stream v6 API(Streamer/prepareStream/playStream)를 찾지 못했습니다. " +
"package.json 버전을 ^4.2.1(=v6 npm 태그)로 맞추거나 docs를 확인하세요.",
);
}
return { selfbot, videoStream };
}
async start(ctx: StreamContext): Promise<string> {
if (this.active) return "이미 송출 중입니다.";
if (!this.config.selfbotToken) {
return "DISCORD_SELFBOT_TOKEN이 설정되지 않았습니다 (.env). 버너 계정 토큰을 넣어주세요.";
}
const { selfbot, videoStream } = await this.loadLib();
const { Streamer, prepareStream, playStream, Utils } = videoStream;
this.streamer = new Streamer(new selfbot.Client());
await this.streamer.client.login(this.config.selfbotToken);
await this.streamer.joinVoice(ctx.guildId, ctx.voiceChannelId);
// Grab the VNC X display with ffmpeg's x11grab and let the library
// encode/transport it. NVENC (RTX 5050) is used if available.
const input = `x11grab:${this.config.vncDisplay}`;
const { command, output } = prepareStream(
input,
{
width: parseInt(this.config.vncResolution.split("x")[0] ?? "1920", 10),
height: parseInt(this.config.vncResolution.split("x")[1] ?? "1080", 10),
frameRate: this.config.vncFramerate,
bitrateVideo: this.config.vncBitrateKbps,
videoCodec: Utils?.normalizeVideoCodec ? Utils.normalizeVideoCodec("H264") : "H264",
// x11grab needs to be set as the input format for ffmpeg
customHeaders: undefined,
inputFormat: "x11grab",
inputSize: this.config.vncResolution,
},
(this.controller = new AbortController()).signal,
);
command.on("error", (err: Error) => {
if (!this.controller?.signal.aborted) console.error("[selfbot] ffmpeg error:", err);
});
this.active = true;
// Fire-and-forget; resolves when the stream ends.
playStream(output, this.streamer, { type: "go-live" })
.catch((err: Error) => console.error("[selfbot] playStream:", err))
.finally(() => {
this.active = false;
});
return "🔴 셀프봇으로 VNC 화면을 음성채널에 실시간 송출 중입니다 (Go Live).";
}
async stop(): Promise<void> {
this.controller?.abort();
this.controller = null;
try {
this.streamer?.leaveVoice?.();
this.streamer?.client?.destroy?.();
} catch {
/* ignore */
}
this.streamer = null;
this.active = false;
}
}