Add Discord-native hybrid front-end for Jarvis (bot + bridge)
Some checks failed
Release / semantic-release (push) Successful in 59s
tests / Unit tests (Linux, Python 3.11) (push) Successful in 13m45s
Release / build-linux (push) Failing after 7m47s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
Some checks failed
Release / semantic-release (push) Successful in 59s
tests / Unit tests (Linux, Python 3.11) (push) Successful in 13m45s
Release / build-linux (push) Failing after 7m47s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
Transform isair/jarvis into a Discord-controlled voice assistant running on the Ubuntu VNC desktop, keeping the mature ~39k-line Python brain intact. - bot/ (Node + bun, discord.js): /자비스 slash commands (ephemeral), voice channel join + voice receive/playback, pluggable VNC screen broadcast (selfbot live / noVNC / screenshot) - bridge/ (Python, Flask): wraps jarvis STT + run_reply_engine + Piper TTS behind a thin localhost HTTP API - .env.example, scripts/ (start_bridge/start_bot/dev), README rewrite, docs/language-comparison.md and docs/vnc-xfce-setup.md Language decision: hybrid (Python brain + Node/bun Discord layer) because Discord blocks bot video; native screen broadcast only works via a Node selfbot library.
This commit is contained in:
51
bot/src/stream/index.ts
Normal file
51
bot/src/stream/index.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Pluggable VNC screen-broadcast backends.
|
||||
*
|
||||
* Per the chosen design (option 1): the streaming method is swappable via
|
||||
* STREAM_BACKEND in .env. The default is the real live "Go Live" stream via a
|
||||
* selfbot account (only way to get a native Discord video broadcast), with safe
|
||||
* fallbacks (noVNC link / periodic screenshots) available without code changes.
|
||||
*/
|
||||
import type { AppConfig } from "../config.ts";
|
||||
|
||||
export interface StreamContext {
|
||||
guildId: string;
|
||||
voiceChannelId: string;
|
||||
/** Post an image to the invoking text channel (used by the screenshot backend). */
|
||||
postImage?: (png: Buffer, name: string) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface ScreenStreamer {
|
||||
readonly kind: AppConfig["streamBackend"];
|
||||
/** Start broadcasting. Returns a short user-facing status/link message. */
|
||||
start(ctx: StreamContext): Promise<string>;
|
||||
stop(): Promise<void>;
|
||||
isActive(): boolean;
|
||||
}
|
||||
|
||||
export async function createStreamer(config: AppConfig): Promise<ScreenStreamer> {
|
||||
switch (config.streamBackend) {
|
||||
case "selfbot": {
|
||||
const { SelfbotStreamer } = await import("./selfbot.ts");
|
||||
return new SelfbotStreamer(config);
|
||||
}
|
||||
case "novnc": {
|
||||
const { NoVncStreamer } = await import("./novnc.ts");
|
||||
return new NoVncStreamer(config);
|
||||
}
|
||||
case "screenshot": {
|
||||
const { ScreenshotStreamer } = await import("./screenshot.ts");
|
||||
return new ScreenshotStreamer(config);
|
||||
}
|
||||
case "none":
|
||||
default:
|
||||
return {
|
||||
kind: "none",
|
||||
async start() {
|
||||
return "화면 송출이 비활성화되어 있습니다 (STREAM_BACKEND=none).";
|
||||
},
|
||||
async stop() {},
|
||||
isActive: () => false,
|
||||
};
|
||||
}
|
||||
}
|
||||
34
bot/src/stream/novnc.ts
Normal file
34
bot/src/stream/novnc.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* noVNC link backend (safe, real-time, no ban risk).
|
||||
*
|
||||
* Does not broadcast natively into Discord. Instead it shares a noVNC web URL
|
||||
* that anyone can open in a browser to watch (and optionally control) the VNC
|
||||
* desktop live. Set NOVNC_URL in .env (e.g. http://192.168.10.9:6080/vnc.html).
|
||||
*
|
||||
* Stand up noVNC once on the host with websockify, e.g.:
|
||||
* websockify --web=/usr/share/novnc 6080 localhost:5901
|
||||
*/
|
||||
import type { AppConfig } from "../config.ts";
|
||||
import type { ScreenStreamer, StreamContext } from "./index.ts";
|
||||
|
||||
export class NoVncStreamer implements ScreenStreamer {
|
||||
readonly kind = "novnc" as const;
|
||||
private active = false;
|
||||
constructor(private config: AppConfig) {}
|
||||
|
||||
isActive() {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
async start(_ctx: StreamContext): Promise<string> {
|
||||
if (!this.config.novncUrl) {
|
||||
return "NOVNC_URL이 설정되지 않았습니다 (.env). 예: http://192.168.10.9:6080/vnc.html";
|
||||
}
|
||||
this.active = true;
|
||||
return `🖥️ VNC 화면 실시간 보기 (브라우저): ${this.config.novncUrl}`;
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
62
bot/src/stream/screenshot.ts
Normal file
62
bot/src/stream/screenshot.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Screenshot backend (safe, no ban risk, not real-time).
|
||||
*
|
||||
* Periodically grabs a frame from the VNC X display with ffmpeg's x11grab and
|
||||
* posts it to the invoking text channel. Low FPS, but works with a normal bot
|
||||
* account and never touches Discord's selfbot surface.
|
||||
*/
|
||||
import { spawn } from "node:child_process";
|
||||
import type { AppConfig } from "../config.ts";
|
||||
import type { ScreenStreamer, StreamContext } from "./index.ts";
|
||||
|
||||
function grabFrame(display: string, size: string): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ff = spawn("ffmpeg", [
|
||||
"-loglevel", "error",
|
||||
"-f", "x11grab",
|
||||
"-video_size", size,
|
||||
"-i", display,
|
||||
"-frames:v", "1",
|
||||
"-f", "image2pipe",
|
||||
"-vcodec", "png",
|
||||
"pipe:1",
|
||||
]);
|
||||
const chunks: Buffer[] = [];
|
||||
ff.stdout.on("data", (c) => chunks.push(c));
|
||||
ff.on("error", reject);
|
||||
ff.on("close", (code) =>
|
||||
code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg exited ${code}`)),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export class ScreenshotStreamer implements ScreenStreamer {
|
||||
readonly kind = "screenshot" as const;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
constructor(private config: AppConfig) {}
|
||||
|
||||
isActive() {
|
||||
return this.timer !== null;
|
||||
}
|
||||
|
||||
async start(ctx: StreamContext): Promise<string> {
|
||||
if (!ctx.postImage) return "스크린샷을 올릴 텍스트 채널 컨텍스트가 없습니다.";
|
||||
if (this.timer) return "이미 스크린샷 송출 중입니다.";
|
||||
const tick = async () => {
|
||||
try {
|
||||
const png = await grabFrame(this.config.vncDisplay, this.config.vncResolution);
|
||||
await ctx.postImage!(png, "vnc.png");
|
||||
} catch (e) {
|
||||
console.error("[screenshot] grab failed:", e);
|
||||
}
|
||||
};
|
||||
this.timer = setInterval(tick, this.config.screenshotIntervalSec * 1000);
|
||||
void tick();
|
||||
return `📸 ${this.config.screenshotIntervalSec}초마다 VNC 스크린샷을 이 채널에 올립니다.`;
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
116
bot/src/stream/selfbot.ts
Normal file
116
bot/src/stream/selfbot.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Selfbot live-stream backend (default).
|
||||
*
|
||||
* Streams the VNC X display (:1) into the voice channel as a real Discord
|
||||
* "Go Live" broadcast. Discord blocks video from *bot* accounts, so this path
|
||||
* requires a USER account token (a "selfbot"), which violates Discord ToS and
|
||||
* can get the account banned. Use a throwaway/burner account, never your main.
|
||||
*
|
||||
* Dependencies are optional (native): install with
|
||||
* bun add discord.js-selfbot-v13 @dank074/discord-video-stream
|
||||
* They are dynamically imported so the core bot installs/runs without them.
|
||||
*
|
||||
* Library API targets @dank074/discord-video-stream v6 (Streamer / prepareStream
|
||||
* / playStream). If a different major is installed, the import guard below will
|
||||
* point you at the docs rather than crash cryptically.
|
||||
*/
|
||||
import type { AppConfig } from "../config.ts";
|
||||
import type { ScreenStreamer, StreamContext } from "./index.ts";
|
||||
|
||||
export class SelfbotStreamer implements ScreenStreamer {
|
||||
readonly kind = "selfbot" as const;
|
||||
private config: AppConfig;
|
||||
private streamer: any = null;
|
||||
private controller: AbortController | null = null;
|
||||
private active = false;
|
||||
|
||||
constructor(config: AppConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
isActive() {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
private async loadLib() {
|
||||
let selfbot: any, videoStream: any;
|
||||
try {
|
||||
selfbot = await import("discord.js-selfbot-v13");
|
||||
// Optional native dep; resolved at runtime only. Version/name can vary by
|
||||
// upstream release, so we don't hard-bind its types at compile time.
|
||||
// @ts-ignore - optional dependency, may be absent until `bun add`ed
|
||||
videoStream = await import("@dank074/discord-video-stream");
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
"셀프봇 송출 의존성이 없습니다. 설치: bun add discord.js-selfbot-v13 @dank074/discord-video-stream\n" +
|
||||
`원본 오류: ${(e as Error).message}`,
|
||||
);
|
||||
}
|
||||
if (!videoStream.Streamer || !videoStream.prepareStream || !videoStream.playStream) {
|
||||
throw new Error(
|
||||
"@dank074/discord-video-stream v6 API(Streamer/prepareStream/playStream)를 찾지 못했습니다. " +
|
||||
"package.json 버전을 ^4.2.1(=v6 npm 태그)로 맞추거나 docs를 확인하세요.",
|
||||
);
|
||||
}
|
||||
return { selfbot, videoStream };
|
||||
}
|
||||
|
||||
async start(ctx: StreamContext): Promise<string> {
|
||||
if (this.active) return "이미 송출 중입니다.";
|
||||
if (!this.config.selfbotToken) {
|
||||
return "DISCORD_SELFBOT_TOKEN이 설정되지 않았습니다 (.env). 버너 계정 토큰을 넣어주세요.";
|
||||
}
|
||||
const { selfbot, videoStream } = await this.loadLib();
|
||||
const { Streamer, prepareStream, playStream, Utils } = videoStream;
|
||||
|
||||
this.streamer = new Streamer(new selfbot.Client());
|
||||
await this.streamer.client.login(this.config.selfbotToken);
|
||||
await this.streamer.joinVoice(ctx.guildId, ctx.voiceChannelId);
|
||||
|
||||
// Grab the VNC X display with ffmpeg's x11grab and let the library
|
||||
// encode/transport it. NVENC (RTX 5050) is used if available.
|
||||
const input = `x11grab:${this.config.vncDisplay}`;
|
||||
const { command, output } = prepareStream(
|
||||
input,
|
||||
{
|
||||
width: parseInt(this.config.vncResolution.split("x")[0] ?? "1920", 10),
|
||||
height: parseInt(this.config.vncResolution.split("x")[1] ?? "1080", 10),
|
||||
frameRate: this.config.vncFramerate,
|
||||
bitrateVideo: this.config.vncBitrateKbps,
|
||||
videoCodec: Utils?.normalizeVideoCodec ? Utils.normalizeVideoCodec("H264") : "H264",
|
||||
// x11grab needs to be set as the input format for ffmpeg
|
||||
customHeaders: undefined,
|
||||
inputFormat: "x11grab",
|
||||
inputSize: this.config.vncResolution,
|
||||
},
|
||||
(this.controller = new AbortController()).signal,
|
||||
);
|
||||
|
||||
command.on("error", (err: Error) => {
|
||||
if (!this.controller?.signal.aborted) console.error("[selfbot] ffmpeg error:", err);
|
||||
});
|
||||
|
||||
this.active = true;
|
||||
// Fire-and-forget; resolves when the stream ends.
|
||||
playStream(output, this.streamer, { type: "go-live" })
|
||||
.catch((err: Error) => console.error("[selfbot] playStream:", err))
|
||||
.finally(() => {
|
||||
this.active = false;
|
||||
});
|
||||
|
||||
return "🔴 셀프봇으로 VNC 화면을 음성채널에 실시간 송출 중입니다 (Go Live).";
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.controller?.abort();
|
||||
this.controller = null;
|
||||
try {
|
||||
this.streamer?.leaveVoice?.();
|
||||
this.streamer?.client?.destroy?.();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
this.streamer = null;
|
||||
this.active = false;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user