Compare commits
259 Commits
v1.4.0
...
app-v0.1.3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e428aad58a | ||
|
|
a665dd8d21 | ||
|
|
a5afdbb221 | ||
|
|
9199ccbd0e | ||
|
|
2a95f58a0e | ||
|
|
81d1e26f61 | ||
|
|
88ee5ec2cb | ||
|
|
2e90c47a95 | ||
|
|
c7655c58f4 | ||
|
|
b4f8f71dd2 | ||
|
|
9ba26c5bac | ||
|
|
1bef5db912 | ||
|
|
17927b6450 | ||
|
|
ecde1e6bd0 | ||
|
|
282f114e60 | ||
|
|
e99831e540 | ||
|
|
68d0333698 | ||
|
|
af1276cf6b | ||
|
|
59c3f33c2c | ||
|
|
59a9277613 | ||
|
|
d31d75f18f | ||
|
|
64dceb816e | ||
|
|
2238ddbe42 | ||
|
|
680d4fbdc1 | ||
|
|
c51e5b2f68 | ||
|
|
1c06b16bd4 | ||
|
|
44387685af | ||
|
|
61b68130c6 | ||
|
|
f1600f09c2 | ||
|
|
ce03424ba4 | ||
|
|
d02092662b | ||
|
|
4adc6a8d72 | ||
|
|
4993942040 | ||
|
|
ff78c0ac41 | ||
|
|
8c61d7a821 | ||
|
|
07615ee7a1 | ||
|
|
4d0ae91e8a | ||
|
|
7da5bca783 | ||
|
|
a6932d9b16 | ||
|
|
6a9b66bdcd | ||
|
|
cda8db7fce | ||
|
|
65294f3f97 | ||
|
|
420c9c73f3 | ||
|
|
9a91bc9ab6 | ||
|
|
ec2672cd4a | ||
|
|
fdcaae4820 | ||
|
|
72518ca50e | ||
|
|
80ce9281a3 | ||
|
|
cbe898d3d0 | ||
|
|
1cdfbf5151 | ||
|
|
b2b6f92943 | ||
|
|
1f2a8ec05e | ||
|
|
04b516dbc1 | ||
|
|
b386c905b2 | ||
|
|
b29150a767 | ||
|
|
6fc3ad3d57 | ||
|
|
fd7ccab55b | ||
|
|
2b832c1c3b | ||
|
|
3d8801b691 | ||
|
|
1a12755649 | ||
|
|
fbdf274b9d | ||
|
|
2a6b350079 | ||
|
|
53d522db69 | ||
|
|
6c1bc31236 | ||
|
|
27fcea4e3d | ||
|
|
46f71e7f03 | ||
|
|
a20ebe4eae | ||
|
|
2a140d8490 | ||
|
|
27eb1315b9 | ||
|
|
8a37144cd8 | ||
|
|
dbe5c617bc | ||
|
|
d17a3bbb61 | ||
|
|
f991981776 | ||
|
|
a85ad3e00f | ||
|
|
304a6f2284 | ||
|
|
a177d38ab2 | ||
|
|
53bccb620d | ||
|
|
bd7b69d4f4 | ||
|
|
cb2dda25b6 | ||
|
|
f71c72d3a1 | ||
|
|
61d9c06a6b | ||
|
|
7e51ebd0ba | ||
|
|
324fd2ea0a | ||
|
|
a5bd25b86f | ||
|
|
e4b6327cf9 | ||
|
|
04b603facd | ||
|
|
e8db417752 | ||
|
|
f952b72ed9 | ||
|
|
5aa014543c | ||
|
|
ea2d3ba958 | ||
|
|
3d75d74f92 | ||
|
|
c20c83b2ba | ||
|
|
511791f39b | ||
|
|
b4ea9a6af4 | ||
|
|
1ae2521d58 | ||
|
|
5ce832f51a | ||
|
|
ebf264d54b | ||
|
|
3d58cf8ab1 | ||
|
|
5316f4e9b0 | ||
|
|
4a81b88d41 | ||
|
|
aa6d950af6 | ||
|
|
7e096b449e | ||
|
|
b6bd1be6fb | ||
|
|
1400a3ff97 | ||
|
|
af6f2c855d | ||
|
|
7e83b09dd5 | ||
|
|
c9101e0d6a | ||
|
|
9b53af169d | ||
|
|
cfab18a47e | ||
|
|
1bd7defbb4 | ||
|
|
56212c06dc | ||
|
|
077db5031c | ||
|
|
55d19364a0 | ||
|
|
b20b5795b6 | ||
|
|
24d0798bae | ||
|
|
69f1bf4fa9 | ||
|
|
d0cb540fa0 | ||
|
|
9185265010 | ||
|
|
b94e9dc112 | ||
|
|
d531eb0ffc | ||
|
|
71f9b0cd54 | ||
|
|
fba258821b | ||
|
|
648e8304b0 | ||
|
|
8b3905f1c2 | ||
|
|
3ef4281180 | ||
|
|
d4bd4acfe9 | ||
|
|
0d9c6ec984 | ||
|
|
91d0109139 | ||
|
|
76144fb19b | ||
|
|
ac8c30022d | ||
|
|
81ba5e3521 | ||
|
|
41026a1eff | ||
|
|
5c510bb602 | ||
|
|
1a6d6447f6 | ||
|
|
a4436698c8 | ||
|
|
b68eef3ec1 | ||
|
|
20300c017e | ||
|
|
e8ddde5be6 | ||
|
|
371e87f553 | ||
|
|
0973ad9815 | ||
|
|
eb0fa2f159 | ||
|
|
390f6f0b86 | ||
|
|
07c701be54 | ||
|
|
6d435caef0 | ||
|
|
e6ee152774 | ||
|
|
b88e20134f | ||
|
|
46e4e75d05 | ||
|
|
7623d6d002 | ||
|
|
cefe9b9253 | ||
|
|
1bebc1c531 | ||
|
|
5507ec9b71 | ||
|
|
bd2eb49e3d | ||
|
|
9ff307d7fe | ||
|
|
9e75df266b | ||
|
|
c328bda699 | ||
|
|
19f984ae73 | ||
|
|
2f69557e1b | ||
|
|
a70ccda37e | ||
|
|
de35573e5f | ||
|
|
f892a1dce9 | ||
|
|
e885630e91 | ||
|
|
46eebcd256 | ||
|
|
6fc3983a40 | ||
|
|
b88b28140f | ||
|
|
3ec430fd25 | ||
|
|
4bb3895ca0 | ||
|
|
69a884b182 | ||
|
|
a45e198df6 | ||
|
|
1745fcd04c | ||
|
|
4621e6c1fd | ||
|
|
0182823592 | ||
|
|
fe4e296361 | ||
|
|
3dbf1e8c56 | ||
|
|
b3dcd12ec9 | ||
|
|
864b2e7fdb | ||
|
|
7a288129de | ||
|
|
54a465948a | ||
|
|
6b23a8adc7 | ||
|
|
beaacb39cd | ||
|
|
5efa760986 | ||
|
|
f25c26cf53 | ||
|
|
8479112ddd | ||
|
|
5838f9edd8 | ||
|
|
1309525aa8 | ||
|
|
590cd07268 | ||
|
|
a7685e9078 | ||
|
|
b4926d5e19 | ||
|
|
8f2470af2a | ||
|
|
cec52165b2 | ||
|
|
dff18c5c7e | ||
|
|
726fca3377 | ||
|
|
f66f57f02f | ||
|
|
357f36afcc | ||
|
|
6d0b3d5bec | ||
|
|
37e27a2648 | ||
|
|
a6908c386e | ||
|
|
457b8636fa | ||
|
|
a4f77c3261 | ||
|
|
aa34f707b5 | ||
|
|
ba662ae199 | ||
|
|
d0938c10f9 | ||
|
|
0e50c558d1 | ||
|
|
3f0ed4343b | ||
|
|
b3c26417d1 | ||
|
|
2a85b6afa5 | ||
|
|
a2e4f064d2 | ||
|
|
7da86262de | ||
|
|
290a4c9dcf | ||
|
|
ca5bb520b9 | ||
|
|
3794bb7149 | ||
|
|
d0ffa7f55e | ||
|
|
83304255f8 | ||
|
|
165142be9a | ||
|
|
8dfab3ab6e | ||
|
|
8124451402 | ||
|
|
13de978cf9 | ||
|
|
c63a859ced | ||
|
|
66009d910d | ||
|
|
bf4138c9ee | ||
|
|
d2b21a1afa | ||
|
|
5da88bdcbb | ||
|
|
8cb7e57cda | ||
|
|
85a562c171 | ||
|
|
903e0eb9de | ||
|
|
73fa93f346 | ||
|
|
01037944a0 | ||
|
|
bb0defd384 | ||
|
|
cbe1785e08 | ||
|
|
26fd6e4a1f | ||
|
|
810c62ab29 | ||
|
|
e0b172e796 | ||
|
|
af0c773998 | ||
|
|
dc2e709d31 | ||
|
|
ff9b868e62 | ||
|
|
5762a22a67 | ||
|
|
653d6ecc03 | ||
|
|
4f90701112 | ||
|
|
4a15384bc6 | ||
|
|
bb60c4db3b | ||
|
|
8562548571 | ||
|
|
db51bb92e6 | ||
|
|
ffc16665e5 | ||
|
|
5629da7e9f | ||
|
|
83999a5b0b | ||
|
|
d5fd218c86 | ||
|
|
23b1fe692b | ||
|
|
7bb9718c34 | ||
|
|
7da2fcb5e5 | ||
|
|
680f5a656a | ||
|
|
b52ffd2b18 | ||
|
|
140fc56f18 | ||
|
|
5ee47827f3 | ||
|
|
c189ce2e65 | ||
|
|
086dd5cde7 | ||
|
|
f64d76e737 | ||
|
|
11c3621093 | ||
|
|
7ad5d99380 | ||
|
|
39a0944105 | ||
|
|
b9f637faa4 |
84
.env.example
84
.env.example
@@ -29,29 +29,45 @@ BRIDGE_PORT=8765
|
||||
JARVIS_BRAIN_ENABLED=1
|
||||
JARVIS_TTS_ENABLED=1
|
||||
# faster-whisper device/compute. GPU by default (RTX 5050 / sm_120, verified).
|
||||
# Falls back to CPU automatically if no GPU is passed to the container.
|
||||
# On a NO-GPU host (e.g. an app host that offloads the LLM to a separate GPU box)
|
||||
# set WHISPER_DEVICE=cpu AND WHISPER_COMPUTE_TYPE=int8 — float16 is GPU-only, so
|
||||
# CPU needs int8 (STT is slower on CPU).
|
||||
WHISPER_DEVICE=cuda
|
||||
WHISPER_COMPUTE_TYPE=float16
|
||||
# Optional explicit Piper voice model (.onnx). If empty, the jarvis default is used.
|
||||
TTS_PIPER_MODEL_PATH=
|
||||
# TTS engine: "melo" (default) uses the MeloTTS Korean voice served by the warm
|
||||
# melo-worker (Korean speaker, speed 1.5). Set to "piper" to use Piper directly.
|
||||
# TTS engine:
|
||||
# melo (default) GPU MeloTTS — offline Korean voice, privacy-first (text never
|
||||
# leaves the host), synthesises on the GPU via the melo-worker. Needs a
|
||||
# GPU (MELO_DEVICE=cuda); falls back to CPU (slow) with MELO_DEVICE=cpu.
|
||||
# edge Microsoft Edge TTS — natural Korean neural voice, but ONLINE: reply
|
||||
# text is sent to Microsoft's servers and needs internet.
|
||||
# piper offline English voice (fallback).
|
||||
TTS_ENGINE=melo
|
||||
# Melo-only by default: if MeloTTS synthesis fails the bridge returns no audio
|
||||
# rather than speaking Korean through the English Piper voice (which mangles it).
|
||||
# Set to 1 only if you explicitly want the Piper fallback.
|
||||
MELO_FALLBACK_PIPER=0
|
||||
# Where the bridge reaches the in-container MeloTTS worker, and how long it
|
||||
# waits for a synthesis. Speaking rate is set on the worker via MELO_SPEED.
|
||||
MELO_WORKER_URL=http://127.0.0.1:8770
|
||||
MELO_TIMEOUT=30
|
||||
# GPU MeloTTS knobs (used when TTS_ENGINE=melo).
|
||||
MELO_DEVICE=cuda
|
||||
MELO_SPEED=1.5
|
||||
# Edge voice + speaking rate (used when TTS_ENGINE=edge). Rate is a percentage
|
||||
# (+45% ≈ 1.45×). Korean voices: ko-KR-HyunsuMultilingualNeural (M),
|
||||
# ko-KR-InJoonNeural (M), ko-KR-SunHiNeural (F).
|
||||
EDGE_TTS_VOICE=ko-KR-HyunsuMultilingualNeural
|
||||
EDGE_TTS_RATE=+45%
|
||||
# Neural-only by default: if synthesis fails the bridge returns no audio rather
|
||||
# than speaking Korean through the English Piper voice. Set to 1 to allow the
|
||||
# Piper fallback.
|
||||
MELO_FALLBACK_PIPER=0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Jarvis brain (Ollama-backed). In Docker these populate the rendered
|
||||
# config (docker/jarvis-config.template.json). See src/jarvis/config.py.
|
||||
# ---------------------------------------------------------------------------
|
||||
# In docker-compose this is overridden to http://ollama:11434 automatically.
|
||||
# In docker-compose this defaults to the in-stack `ollama` service
|
||||
# (http://ollama:11434). To use an EXTERNAL Ollama on another LAN machine (e.g. a
|
||||
# GPU host), set it here to that host's IP — the compose now respects this value:
|
||||
# OLLAMA_BASE_URL=http://192.168.10.9:11434 # Ollama runs on the .9 host
|
||||
# In that split setup: do NOT start the in-stack ollama/ollama-init (run
|
||||
# `docker compose up -d javis`), pull the models on the LLM host, and make that
|
||||
# host's Ollama listen on the LAN (OLLAMA_HOST=0.0.0.0:11434).
|
||||
OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
# qwen2.5:3b — small non-reasoning instruct model. ~2.4GB, runs 100% on the GPU
|
||||
# (the 8B offloads ~8% to CPU), warm voice turns ~2-4s vs ~5-7s on 8B. Clean
|
||||
@@ -59,11 +75,15 @@ OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
# free-form chit-chat. Swap back to qwen3:8b for the strongest tool-calling.
|
||||
OLLAMA_CHAT_MODEL=qwen2.5:3b
|
||||
# Model for the auxiliary small-model calls: intent judge, tool router, weather
|
||||
# place extraction, query decomposition. BLANK (default) reuses OLLAMA_CHAT_MODEL
|
||||
# so the stack runs on one already-warm model. The code's built-in default
|
||||
# (gemma4:e2b) is NOT pulled by this stack, so leaving this unset previously made
|
||||
# every router/extractor call silently fail. Only set this if you also pull the
|
||||
# model into Ollama.
|
||||
# place extraction, query decomposition. These are classification/JSON calls,
|
||||
# NOT the spoken answer, so a small fast model here cuts 2-3 big-model round
|
||||
# trips per command without touching answer quality. BLANK uses the stack
|
||||
# default qwen2.5:3b, which ollama-init pulls automatically. Set it equal to
|
||||
# OLLAMA_CHAT_MODEL to run everything on one resident model instead (saves VRAM
|
||||
# at the cost of slower routing when the chat model is large).
|
||||
# NEVER set this LARGER than OLLAMA_CHAT_MODEL: the auxiliary calls would then
|
||||
# run on the bigger, slower model and add latency to every command (the exact
|
||||
# opposite of the split's purpose). Keep it <= the chat model, blank, or equal.
|
||||
OLLAMA_INTENT_MODEL=
|
||||
OLLAMA_EMBED_MODEL=nomic-embed-text
|
||||
WHISPER_MODEL=small
|
||||
@@ -77,7 +97,7 @@ OUTPUT_LANGUAGE=
|
||||
# Operator instruction folder: every *.md in this dir is appended to the main
|
||||
# reply LLM's system prompt (filename order), re-read each turn so edits apply
|
||||
# without a rebuild/restart. ./agents is bind-mounted here read-only; only
|
||||
# change this to relocate the folder inside the container. See README "운영자 지시문".
|
||||
# change this to relocate the folder inside the container. See docs/v1/README.md "운영자 지시문".
|
||||
AGENTS_DIR=/app/agents
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -102,7 +122,7 @@ CHROME_START_URL=about:blank
|
||||
# Screen-share + browser mode.
|
||||
# true = the bot may go Live (screen-share the VNC desktop) and drive the
|
||||
# on-screen browser for real-time info (search / play / read screen).
|
||||
# false = no screen share; voice only, real-time info via the Gemini API.
|
||||
# false = no screen share; voice only, real-time info via the normal web search.
|
||||
STREAM_BROWSER=true
|
||||
# Optional: profile dir for browser-based Google search in plain text turns
|
||||
# (no active broadcast). When set, the search helper opens Chrome against this
|
||||
@@ -113,27 +133,6 @@ STREAM_BROWSER=true
|
||||
# Google does not challenge it). Use a DEDICATED dir, not your everyday Chrome
|
||||
# profile, to avoid the "profile in use" lock while Chrome is open.
|
||||
CHROME_USER_DATA_DIR=
|
||||
# Gemini auth for real-time info when STREAM_BROWSER=false.
|
||||
# oauth = use the Gemini CLI with a Google-account login (no API key).
|
||||
# Install once: npm i -g @google/gemini-cli ; then run `gemini` and
|
||||
# "Sign in with Google". Uses the CLI's built-in web-search grounding.
|
||||
# NOTE (2026-06): Google is blocking personal Google accounts on this
|
||||
# path ("This client is no longer supported for Gemini Code Assist for
|
||||
# individuals"). Workspace/org accounts may still work; personal
|
||||
# accounts should use apikey below instead.
|
||||
# apikey = REST path; needs GEMINI_API_KEY below
|
||||
# (get one at https://aistudio.google.com/app/apikey). Recommended for
|
||||
# personal Google accounts now that individual OAuth login is blocked.
|
||||
# Either way, real-time search fail-opens to DDG/Brave/Wikipedia if Gemini is
|
||||
# unavailable, so this is optional, not required.
|
||||
GEMINI_AUTH=oauth
|
||||
GEMINI_API_KEY=
|
||||
GEMINI_MODEL=gemini-2.0-flash
|
||||
# OAuth login source for Docker. The container mounts this into ~/.gemini.
|
||||
# Default (blank) = ./docker/gemini-oauth (project-local, cross-platform). Seed
|
||||
# it once: cp -r ~/.gemini/. docker/gemini-oauth/ (copy the whole login state).
|
||||
# Or point at an existing host login instead, e.g. GEMINI_OAUTH_DIR=~/.gemini
|
||||
GEMINI_OAUTH_DIR=
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VNC screen broadcast
|
||||
@@ -193,6 +192,10 @@ VOICE_SILENCE_MS=800
|
||||
# full (default) everything in one container
|
||||
# browser ONLY the desktop + Chrome + control-server (driven over the LAN)
|
||||
# bot ONLY the bot + bridge + TTS (drives a REMOTE browser)
|
||||
# brain ONLY the bridge (STT + reply + TTS) on a GPU host — no bot/desktop.
|
||||
# The app host reaches it via BRIDGE_URL. See docs/DEPLOY.md "GPU split".
|
||||
# app the bot + desktop/Chrome, but NOT the local bridge: offloads
|
||||
# STT/LLM/TTS to a remote brain host (set BRIDGE_URL=http://<brain>:8765).
|
||||
JARVIS_ROLE=full
|
||||
|
||||
# --- GPU per OS: pick the matching compose override via COMPOSE_FILE ---
|
||||
@@ -226,6 +229,7 @@ COMPOSE_FILE=docker-compose.yml:docker-compose.gpu-linux.yml
|
||||
# OLLAMA_CHAT_MODEL=qwen2.5:7b # quality (needs ~5GB VRAM + whisper small)
|
||||
# OLLAMA_CHAT_MODEL=qwen2.5:3b # speed (fits easily, faster on 8GB GPUs)
|
||||
# WHISPER_MODEL=small # small frees VRAM for a bigger LLM; medium=more accurate
|
||||
# STT_BEAM_SIZE=5 # beam search (5) > greedy (1) for accuracy; lower for speed
|
||||
# MELO_DEVICE=cuda # cpu if no GPU on the bot host
|
||||
|
||||
# --- Settings web UI (http://localhost:8765/settings on the bot host) ---
|
||||
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -29,7 +29,6 @@ src/jarvis/_version.py
|
||||
.env.bak*
|
||||
*.bak
|
||||
|
||||
# Gemini CLI OAuth login (account tokens) seeded for GEMINI_AUTH=oauth in Docker.
|
||||
# Keep the dir (.gitkeep) but never commit the login files.
|
||||
docker/gemini-oauth/*
|
||||
!docker/gemini-oauth/.gitkeep
|
||||
# Gemini CLI OAuth login (account tokens) from the removed Gemini search backend. An
|
||||
# older checkout may still hold docker/gemini-oauth/oauth_creds.json; never commit it.
|
||||
docker/gemini-oauth/
|
||||
|
||||
16
CLAUDE.md
16
CLAUDE.md
@@ -1,6 +1,6 @@
|
||||
Data privacy comes first, always.
|
||||
|
||||
All user-facing command line output should make use of emojis. Especially an initial emoji to start off the lines that depict what the line is about. Output should make use of indentation spacing to establish a visual hierarchy and aim to make output as easy to sift through as possible. Exception: Windows .bat scripts cannot use emojis (cmd.exe doesn't render Unicode properly).
|
||||
This assistant is used through a Discord bot with voice (TTS) replies, not a CLI. Do not add emojis to user-facing assistant output. Keep output plain and readable. (Runtime assistant behaviour lives in `agents/*.md`, which is injected into the reply LLM's prompt.)
|
||||
|
||||
Any important point in our logical flows should have debug logs using the `debug_log` method from `src/jarvis/debug.py`. Avoid excessive logging to keep the logs easily readable and actionable.
|
||||
|
||||
@@ -21,13 +21,25 @@ Any code change must either adhere to our spec files perfectly or you should ask
|
||||
| `src/jarvis/tools/builtin/tool_search.spec.md` | toolSearchTool escape hatch for mid-loop tool routing | Re-runs the same router; never removes stop/self; capped per reply |
|
||||
| `src/jarvis/tools/external/mcp_runtime.spec.md` | Persistent MCP runtime: per-server long-lived stdio session, queue-based dispatch, retry on transient session loss | One worker per server keyed by config; calls to the same server serialise; `MCPServerSessionError` for session-level failures; opt-in `idle_timeout_sec` for stateless servers |
|
||||
| `src/jarvis/reply/prompts/prompts.spec.md` | System/user prompt templates | — |
|
||||
| `src/jarvis/tools/builtin/web_search.spec.md` | webSearch tool: STREAM_BROWSER routing (browser/Gemini), cascade fetch, SSRF guard, prompt-injection fence, links-only envelope | Untrusted web content is fenced as data, not instructions; rank preference over speed; honest failure over confabulation |
|
||||
| `src/jarvis/tools/builtin/web_search.spec.md` | webSearch tool: live-broadcast Chrome, then Exa (keyless), Brave (opt-in), title-guarded Wikipedia; foreign-script drop, SSRF guard, prompt-injection fence, honest envelopes | Untrusted web content is fenced as data, not instructions; rank preference over speed; honest failure over confabulation |
|
||||
| `src/jarvis/tools/builtin/chat_channels.spec.md` | chatChannels tool (v2): list, read, search, send, edit, react, delete, pin registered Discord chat channels through the voice worker and app | Offered only with a chat backend; per-channel switches; chat is fenced data; delete needs the user's yes in a later turn |
|
||||
| `src/jarvis/tools/builtin/schedule.spec.md` | schedule tool (v2): reminders, timers and scheduled tasks of the asker; the app keeps, fires and delivers them | Offered only with a schedule backend; only the asker's own; tasks get read-only tools; never claimed without the tool |
|
||||
| `src/jarvis/tools/builtin/personal_notes.spec.md` | personalNotes tool (v2): each person's standing instructions for how they are answered; the app keeps them and sends the asker's with every reply | Offered only with a notes backend; only the asker's own, only in their answers; fenced, never permits an action |
|
||||
| `src/jarvis/tools/builtin/collected_news.spec.md` | collectedNews tool (v2): read the background collection's results (targets, newest items) | Offered only with the app's collection file and the feature on; web content fenced as data; read-only, no LLM |
|
||||
| `src/jarvis/tools/builtin/screen_view.spec.md` | screenView tool (v2): see the whole screen, one monitor or one window through the app; pictures go to the reply model for that reply only | Offered only when the app makes screen viewing available; the app decides who may see; screen text is data; pictures never kept or logged |
|
||||
| `src/jarvis/tools/builtin/pc_control.spec.md` | pcControl tool (v2 M7): list windows, read a window, click, type, keys, run and close programs through the app's agent-desktop | Offered only when the app makes PC control available; ask-level actions need the asker's own next message; payments, passwords, deleting files and admin never; screen text is data |
|
||||
| `src/jarvis/tools/builtin/bot_browser.spec.md` | botBrowser tool (v2): open, read, click, type, back in the bot's own Chrome over CDP | Offered only when the app started the bot browser; only http(s); page content is fenced data; no LLM call |
|
||||
| `src/jarvis/tools/builtin/browse_and_play.spec.md` | browseAndPlay tool: play YouTube on the shared screen (screen-share mode only) | Node layer owns Chrome/CDP; mode-gated; fail-open, no LLM call |
|
||||
| `src/jarvis/tools/builtin/nutrition/log_meal.spec.md` | logMeal tool: single-property schema for planner fast-path, internal nutrition extraction, untrusted-data fence, follow-ups | Public schema is a single optional `meal` string; nutrition fields are internal; user text is fenced as data |
|
||||
| `src/jarvis/utils/location.spec.md` | GeoIP location detection | Privacy-first; local GeoLite2 DB only |
|
||||
| `src/jarvis/memory/graph.spec.md` | Node graph memory (v2), self-organising tree, UI explorer | Dynamic structure; access-aware; auto-split/merge (future) |
|
||||
| `src/jarvis/memory/history.spec.md` | v2 conversation history store: voice transcripts, answers and registered chat channel messages; per-asker scope (own, chat channels, others only while co-present); language-agnostic prefix search; size-budgeted context for the reply engine; answer traces (how each answer was made) for the 대화 screen | Only what the asker may see; traces never reach a prompt; logs carry no record text; whole history searched, little sent |
|
||||
| `src/jarvis/memory/summariser.spec.md` | Diary summariser prompt contract, hygiene rules (deflection, attribution, topic separation), post-process scrub, and bulk-sweep clean button | Two-layer defence: prompt + deterministic scrub; corrupted summaries poison every downstream consumer |
|
||||
| `src/jarvis/memory/recall_gate.spec.md` | Deterministic skip-enrichment heuristic when the hot window covers a follow-up | Fail-open; language-agnostic via `\w{3,}` + `re.UNICODE`; planner intent always wins |
|
||||
| `src/voice_worker/voice_worker.spec.md` | v2 voice worker: multi-speaker streaming STT (shared Whisper, batched) and sentence TTS over a local WebSocket | Loopback only with an app token; no transcript text in logs; finals before partials; language-agnostic |
|
||||
| `app/src/conversation.spec.md` | v2 call detection and response policies in the Rust app: wake words with near matches, call modes, designated people, voice stop, the four response policies with settling, interruption and missed requests | Pure types tested with fake times; language-agnostic matching on normalised letters; nothing dropped silently |
|
||||
| `app/src/admin.spec.md` | v2 M5 admin backend for the management app: data folder and its location file, settings editing for the screens, stored secrets (DPAPI on Windows), log hub (tabs, daily files, error report, zip export), service control and dashboard data, runtime install, the LLM (Strata) the services start, the Tauri window and the Windows build | One folder; edits keep the user's file (only non-defaults, unknown keys kept, all or nothing); secrets never in a log, copy or export |
|
||||
| `app/src/discord_core.spec.md` | v2 Discord voice core in the Rust app: join/leave, per-speaker receive (own reorder window and Opus decoding after songbird decrypts) to the voice worker, sentence playback and stop, slash command, registered chat channels (read/send), voice worker process | Hears only the channel it is in and reads only registered channels; join notice on every join; chat is data and sent text never pings; no transcript or chat text in logs |
|
||||
|
||||
The LLM contexts graph at `docs/llm_contexts.md` maps every LLM call in the app (model, gating, inputs, outputs, limits, flow). Keep it up-to-date at all times: any change that adds, removes, or alters an LLM context (model resolution, timeout, cap, prompt source, gating flag, data-flow edge) must update `docs/llm_contexts.md` in the same PR.
|
||||
|
||||
|
||||
548
DESIGN.md
Normal file
548
DESIGN.md
Normal file
@@ -0,0 +1,548 @@
|
||||
---
|
||||
version: alpha
|
||||
name: Linear-design-analysis
|
||||
description: "A near-black product-focused marketing canvas built around #010102 (the deepest dark surface of any tool in this collection), light gray text (#f7f8f8), and the signature Linear lavender-blue (#5e6ad2) used as the single chromatic accent. The system reads as software-craft documentation: dense, technical, and quietly luxurious. Display type is set in the Linear custom sans (SF Pro Display fallback) at 500–700 with measured negative tracking. Cards live as charcoal panels (#0f1011) with hairline borders. The accent lavender appears on the brand mark, focus rings, and a few intentional CTAs — never decoratively. Page rhythm leans on product UI screenshots framed in dark panels rather than atmospheric color."
|
||||
|
||||
colors:
|
||||
primary: "#5e6ad2"
|
||||
on-primary: "#ffffff"
|
||||
primary-hover: "#828fff"
|
||||
primary-focus: "#5e69d1"
|
||||
ink: "#f7f8f8"
|
||||
ink-muted: "#d0d6e0"
|
||||
ink-subtle: "#8a8f98"
|
||||
ink-tertiary: "#62666d"
|
||||
canvas: "#010102"
|
||||
surface-1: "#0f1011"
|
||||
surface-2: "#141516"
|
||||
surface-3: "#18191a"
|
||||
surface-4: "#191a1b"
|
||||
hairline: "#23252a"
|
||||
hairline-strong: "#34343a"
|
||||
hairline-tertiary: "#3e3e44"
|
||||
inverse-canvas: "#ffffff"
|
||||
inverse-surface-1: "#f5f6f6"
|
||||
inverse-surface-2: "#f6f7f7"
|
||||
inverse-ink: "#000000"
|
||||
brand-secure: "#7a7fad"
|
||||
semantic-success: "#27a644"
|
||||
semantic-overlay: "#000000"
|
||||
|
||||
typography:
|
||||
display-xl:
|
||||
fontFamily: Linear Display
|
||||
fontSize: 80px
|
||||
fontWeight: 600
|
||||
lineHeight: 1.05
|
||||
letterSpacing: -3.0px
|
||||
display-lg:
|
||||
fontFamily: Linear Display
|
||||
fontSize: 56px
|
||||
fontWeight: 600
|
||||
lineHeight: 1.10
|
||||
letterSpacing: -1.8px
|
||||
display-md:
|
||||
fontFamily: Linear Display
|
||||
fontSize: 40px
|
||||
fontWeight: 600
|
||||
lineHeight: 1.15
|
||||
letterSpacing: -1.0px
|
||||
headline:
|
||||
fontFamily: Linear Display
|
||||
fontSize: 28px
|
||||
fontWeight: 600
|
||||
lineHeight: 1.20
|
||||
letterSpacing: -0.6px
|
||||
card-title:
|
||||
fontFamily: Linear Display
|
||||
fontSize: 22px
|
||||
fontWeight: 500
|
||||
lineHeight: 1.25
|
||||
letterSpacing: -0.4px
|
||||
subhead:
|
||||
fontFamily: Linear Display
|
||||
fontSize: 20px
|
||||
fontWeight: 400
|
||||
lineHeight: 1.40
|
||||
letterSpacing: -0.2px
|
||||
body-lg:
|
||||
fontFamily: Linear Text
|
||||
fontSize: 18px
|
||||
fontWeight: 400
|
||||
lineHeight: 1.50
|
||||
letterSpacing: -0.1px
|
||||
body:
|
||||
fontFamily: Linear Text
|
||||
fontSize: 16px
|
||||
fontWeight: 400
|
||||
lineHeight: 1.50
|
||||
letterSpacing: -0.05px
|
||||
body-sm:
|
||||
fontFamily: Linear Text
|
||||
fontSize: 14px
|
||||
fontWeight: 400
|
||||
lineHeight: 1.50
|
||||
letterSpacing: 0
|
||||
caption:
|
||||
fontFamily: Linear Text
|
||||
fontSize: 12px
|
||||
fontWeight: 400
|
||||
lineHeight: 1.40
|
||||
letterSpacing: 0
|
||||
button:
|
||||
fontFamily: Linear Text
|
||||
fontSize: 14px
|
||||
fontWeight: 500
|
||||
lineHeight: 1.20
|
||||
letterSpacing: 0
|
||||
eyebrow:
|
||||
fontFamily: Linear Text
|
||||
fontSize: 13px
|
||||
fontWeight: 500
|
||||
lineHeight: 1.30
|
||||
letterSpacing: 0.4px
|
||||
mono:
|
||||
fontFamily: Linear Mono
|
||||
fontSize: 13px
|
||||
fontWeight: 400
|
||||
lineHeight: 1.50
|
||||
letterSpacing: 0
|
||||
|
||||
rounded:
|
||||
xs: 4px
|
||||
sm: 6px
|
||||
md: 8px
|
||||
lg: 12px
|
||||
xl: 16px
|
||||
xxl: 24px
|
||||
pill: 9999px
|
||||
full: 9999px
|
||||
|
||||
spacing:
|
||||
xxs: 4px
|
||||
xs: 8px
|
||||
sm: 12px
|
||||
md: 16px
|
||||
lg: 24px
|
||||
xl: 32px
|
||||
xxl: 48px
|
||||
section: 96px
|
||||
|
||||
components:
|
||||
button-primary:
|
||||
backgroundColor: "{colors.primary}"
|
||||
textColor: "{colors.on-primary}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: 8px 14px
|
||||
button-primary-pressed:
|
||||
backgroundColor: "{colors.primary-focus}"
|
||||
textColor: "{colors.on-primary}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.md}"
|
||||
button-primary-hover:
|
||||
backgroundColor: "{colors.primary-hover}"
|
||||
textColor: "{colors.on-primary}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.md}"
|
||||
button-secondary:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: 8px 14px
|
||||
button-tertiary:
|
||||
backgroundColor: "{colors.canvas}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: 8px 14px
|
||||
button-inverse:
|
||||
backgroundColor: "{colors.inverse-canvas}"
|
||||
textColor: "{colors.inverse-ink}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: 8px 14px
|
||||
pricing-card:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: 24px
|
||||
pricing-card-featured:
|
||||
backgroundColor: "{colors.surface-2}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: 24px
|
||||
feature-card:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: 24px
|
||||
product-screenshot-card:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.xl}"
|
||||
padding: 24px
|
||||
testimonial-card:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body-lg}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: 32px
|
||||
customer-logo-tile:
|
||||
backgroundColor: "{colors.canvas}"
|
||||
textColor: "{colors.ink-subtle}"
|
||||
typography: "{typography.caption}"
|
||||
rounded: "{rounded.xs}"
|
||||
padding: 16px
|
||||
text-input:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: 8px 12px
|
||||
text-input-focused:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.md}"
|
||||
padding: 8px 12px
|
||||
pricing-tab-default:
|
||||
backgroundColor: "{colors.canvas}"
|
||||
textColor: "{colors.ink-subtle}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.pill}"
|
||||
padding: 6px 14px
|
||||
pricing-tab-selected:
|
||||
backgroundColor: "{colors.surface-2}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.button}"
|
||||
rounded: "{rounded.pill}"
|
||||
padding: 6px 14px
|
||||
cta-banner:
|
||||
backgroundColor: "{colors.surface-1}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.headline}"
|
||||
rounded: "{rounded.lg}"
|
||||
padding: 48px
|
||||
changelog-row:
|
||||
backgroundColor: "{colors.canvas}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body}"
|
||||
rounded: "{rounded.xs}"
|
||||
padding: 24px 0
|
||||
status-badge:
|
||||
backgroundColor: "{colors.surface-2}"
|
||||
textColor: "{colors.ink-muted}"
|
||||
typography: "{typography.caption}"
|
||||
rounded: "{rounded.pill}"
|
||||
padding: 2px 8px
|
||||
top-nav:
|
||||
backgroundColor: "{colors.canvas}"
|
||||
textColor: "{colors.ink}"
|
||||
typography: "{typography.body-sm}"
|
||||
rounded: "{rounded.xs}"
|
||||
height: 56px
|
||||
footer:
|
||||
backgroundColor: "{colors.canvas}"
|
||||
textColor: "{colors.ink-subtle}"
|
||||
typography: "{typography.caption}"
|
||||
rounded: "{rounded.xs}"
|
||||
padding: 64px 32px
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Linear's marketing canvas is the deepest dark surface in this collection — `{colors.canvas}` is #010102, essentially pure black with a faint blue tint. On top sits a four-step surface ladder (`{colors.surface-1}` through `{colors.surface-4}`) for cards, panels, and lifted tiles, with hairline borders running from `{colors.hairline}` (#23252a) up through `{colors.hairline-strong}` and `{colors.hairline-tertiary}`. Light gray text (`{colors.ink}` #f7f8f8) carries the body and headlines.
|
||||
|
||||
The single chromatic accent is **Linear lavender-blue** `{colors.primary}` (#5e6ad2) — used on the brand mark, focus rings, and the primary CTA button. A lighter hover state (`{colors.primary-hover}` #828fff) and a focus-tinted variant (`{colors.primary-focus}` #5e69d1) extend the same hue. Linear avoids saturated greens, oranges, reds, etc. on the marketing canvas — the only semantic color is `{colors.semantic-success}` (#27a644) for status pills and the rare success indicator.
|
||||
|
||||
Display type runs Linear's custom sans (with `SF Pro Display` fallback) at weight 500–700 with negative letter-spacing scaling from -3.0px at 80px down to 0 at body. The body family is Linear's text cut, and a Linear Mono is reserved for code snippets in product screenshots.
|
||||
|
||||
The page rhythm is **dense product screenshots** — Linear's marketing leads with high-fidelity captures of the product UI (issue list, project view, dashboard) framed in `{colors.surface-1}` panels with `{rounded.xl}` 16px corners. The chrome is intentionally minimal so the app screenshots can do the heavy lifting.
|
||||
|
||||
**Key Characteristics:**
|
||||
- **Dark-canvas marketing system** — `{colors.canvas}` (#010102) is the deepest dark in this collection.
|
||||
- **Lavender-blue brand accent** (`{colors.primary}` #5e6ad2) — used scarcely on brand mark, focus, and the primary CTA.
|
||||
- Four-step surface ladder (canvas → surface-1 → surface-2 → surface-3 → surface-4) carries hierarchy without shadow.
|
||||
- Display tracking pulls aggressively negative (-3.0px at 80px); body holds at -0.05px.
|
||||
- Cards use `{rounded.lg}` 12px corners with 1px hairline borders — never pill, rarely 16px.
|
||||
- **Product UI screenshots** dominate the page. The marketing chrome is a dark frame for the app.
|
||||
- No second chromatic color. No atmospheric gradients. No spotlight cards.
|
||||
|
||||
## Colors
|
||||
|
||||
> Source pages: linear.app (home), /intake, /pricing, /contact/sales, /build.
|
||||
|
||||
### Brand & Accent
|
||||
- **Lavender-Blue** ({colors.primary}): The signature Linear accent — primary CTA, brand mark, link emphasis.
|
||||
- **Lavender Hover** ({colors.primary-hover}): Lighter lavender (#828fff) — hovered state of the primary CTA.
|
||||
- **Lavender Focus** ({colors.primary-focus}): Focus-ring tint (#5e69d1) — focused inputs, focused buttons.
|
||||
- **Brand Secure** ({colors.brand-secure}): Muted lavender-gray (#7a7fad) — used in "Linear Security" surfaces.
|
||||
|
||||
### Surface
|
||||
- **Canvas** ({colors.canvas}): Default page background — #010102, near-pure black with a faint blue tint.
|
||||
- **Surface 1** ({colors.surface-1}): One step above canvas — feature cards, pricing cards, product screenshot panels.
|
||||
- **Surface 2** ({colors.surface-2}): Two steps above — featured pricing card, hovered cards.
|
||||
- **Surface 3** ({colors.surface-3}): Three steps above — line-tertiary backgrounds, sub-nav.
|
||||
- **Surface 4** ({colors.surface-4}): Four steps above — bg-level-3, deepest lifted surface.
|
||||
- **Hairline** ({colors.hairline}): 1px borders on cards and dividers.
|
||||
- **Hairline Strong** ({colors.hairline-strong}): Stronger 1px borders — input focus rings.
|
||||
- **Hairline Tertiary** ({colors.hairline-tertiary}): Tertiary borders for nested surfaces.
|
||||
- **Inverse Canvas** ({colors.inverse-canvas}): Pure white — surface of the inverse pill CTA on a small set of section openers.
|
||||
- **Inverse Surface 1** ({colors.inverse-surface-1}): One step above inverse canvas.
|
||||
- **Inverse Surface 2** ({colors.inverse-surface-2}): Two steps above inverse canvas.
|
||||
|
||||
### Text
|
||||
- **Ink** ({colors.ink}): All headlines and emphasized body type — light gray #f7f8f8.
|
||||
- **Ink Muted** ({colors.ink-muted}): Secondary type at #d0d6e0 — meta info on hero panels.
|
||||
- **Ink Subtle** ({colors.ink-subtle}): Tertiary type at #8a8f98 — deselected pricing tabs, footer columns.
|
||||
- **Ink Tertiary** ({colors.ink-tertiary}): Quaternary at #62666d — disabled, footnotes.
|
||||
|
||||
### Semantic
|
||||
- **Success Green** ({colors.semantic-success}): Status pills, success indicators. The only semantic color on marketing.
|
||||
- **Overlay** ({colors.semantic-overlay}): Pure black overlay scrim for modals.
|
||||
|
||||
## Typography
|
||||
|
||||
### Font Family
|
||||
|
||||
- **Linear Display** — Linear's custom display sans; fallback `SF Pro Display, -apple-system, system-ui, Segoe UI, Roboto`. Carries display-xl through subhead.
|
||||
- **Linear Text** — Linear's custom text sans (a slightly different cut tuned for body sizes); same fallback stack. Carries body sizes, button labels, captions.
|
||||
- **Linear Mono** — Linear's custom mono; fallback `ui-monospace, SF Mono, Menlo`. Used for code snippets in product screenshots and for status / ID tokens.
|
||||
|
||||
The marketing surface treats Display and Text as one continuous voice; the family change is silent.
|
||||
|
||||
### Hierarchy
|
||||
|
||||
| Token | Size | Weight | Line Height | Letter Spacing | Use |
|
||||
|---|---|---|---|---|---|
|
||||
| `{typography.display-xl}` | 80px | 600 | 1.05 | -3.0px | Largest hero headline |
|
||||
| `{typography.display-lg}` | 56px | 600 | 1.10 | -1.8px | Section opener headlines |
|
||||
| `{typography.display-md}` | 40px | 600 | 1.15 | -1.0px | Sub-section headlines |
|
||||
| `{typography.headline}` | 28px | 600 | 1.20 | -0.6px | Pricing tier titles, CTA banner heading |
|
||||
| `{typography.card-title}` | 22px | 500 | 1.25 | -0.4px | Feature card title |
|
||||
| `{typography.subhead}` | 20px | 400 | 1.40 | -0.2px | Lead body, intro paragraphs |
|
||||
| `{typography.body-lg}` | 18px | 400 | 1.50 | -0.1px | Hero subhead, lead paragraphs |
|
||||
| `{typography.body}` | 16px | 400 | 1.50 | -0.05px | Default body |
|
||||
| `{typography.body-sm}` | 14px | 400 | 1.50 | 0 | Card body, footer columns |
|
||||
| `{typography.caption}` | 12px | 400 | 1.40 | 0 | Captions, meta, status |
|
||||
| `{typography.button}` | 14px | 500 | 1.20 | 0 | All button labels |
|
||||
| `{typography.eyebrow}` | 13px | 500 | 1.30 | 0.4px | Section eyebrow (slight positive tracking) |
|
||||
| `{typography.mono}` | 13px | 400 | 1.50 | 0 | Linear Mono for code in product screenshots |
|
||||
|
||||
### Principles
|
||||
|
||||
- **Aggressive negative tracking on display** (-3.0px at 80px ≈ 4% of size).
|
||||
- **Single voice from display to body.** Display-xl at 600 → body at 400 — same family, narrower weights.
|
||||
- **Eyebrow uses positive tracking** (+0.4px) — contrast against the negative-tracked display marks the eyebrow as taxonomy.
|
||||
- **Mono only in code contexts.** Linear Mono lives inside product screenshots — not on marketing chrome.
|
||||
|
||||
### Note on Font Substitutes
|
||||
|
||||
Linear's custom typeface isn't publicly distributed; the documented fallback `SF Pro Display, -apple-system, system-ui` is the recommended substitute on macOS. For cross-platform implementation, **Inter** at weight 500 / 600 / 700 is the closest free substitute. **Geist Sans** is also viable. For mono, **JetBrains Mono** or **Geist Mono** at weight 400 closely approximates Linear Mono.
|
||||
|
||||
## Layout
|
||||
|
||||
### Spacing System
|
||||
|
||||
- **Base unit**: 4px.
|
||||
- **Tokens (front matter)**: `{spacing.xxs}` 4px · `{spacing.xs}` 8px · `{spacing.sm}` 12px · `{spacing.md}` 16px · `{spacing.lg}` 24px · `{spacing.xl}` 32px · `{spacing.xxl}` 48px · `{spacing.section}` 96px.
|
||||
- Card interior padding: `{spacing.lg}` 24px on feature/pricing cards; `{spacing.xl}` 32px on testimonial cards; `{spacing.xxl}` 48px on CTA banners.
|
||||
- Pill button padding: 8px vertical · 14px horizontal — Linear's compact button spec.
|
||||
- Form input padding: 8px vertical · 12px horizontal.
|
||||
|
||||
### Grid & Container
|
||||
|
||||
- Max content width sits around 1280px.
|
||||
- Card grids are 3-up at desktop, 2-up at tablet, 1-up at mobile.
|
||||
- Pricing tier grid is 3-up; comparison strip below shows checkmarks per tier.
|
||||
- Product screenshot panels span full content width — they're the protagonist.
|
||||
|
||||
### Whitespace Philosophy
|
||||
|
||||
The dark canvas IS the whitespace. Sections separate by lift onto surface-1 panels, not by gaps in white. Within a panel, generous `{spacing.lg}` 24px gaps between content blocks; `{spacing.section}` 96px between sections.
|
||||
|
||||
## Elevation & Depth
|
||||
|
||||
| Level | Treatment | Use |
|
||||
|---|---|---|
|
||||
| 0 (flat) | No shadow, no border | Default for body type, hero text, footer |
|
||||
| 1 (charcoal lift) | `{colors.surface-1}` background on canvas, 1px `{colors.hairline}` | Default cards, product panels |
|
||||
| 2 (surface-2 lift) | `{colors.surface-2}` background, 1px `{colors.hairline-strong}` | Featured pricing card, hovered cards |
|
||||
| 3 (surface-3 lift) | `{colors.surface-3}` background | Sub-nav, dropdown menus |
|
||||
| 4 (focus ring) | 2px `{colors.primary-focus}` outline at 50% opacity | Focused input, focused button |
|
||||
|
||||
Linear's depth is carried by surface ladder + hairline borders. The brand resists drop shadows on dark almost entirely.
|
||||
|
||||
### Decorative Depth
|
||||
|
||||
- **Product UI screenshots** dominate as decorative depth.
|
||||
- **No atmospheric gradients, no spotlight cards.**
|
||||
- **Subtle white edge highlight** on the top edge of lifted panels — gives the dark surface a faint "pixel rendered" feel.
|
||||
|
||||
## Shapes
|
||||
|
||||
### Border Radius Scale
|
||||
|
||||
| Token | Value | Use |
|
||||
|---|---|---|
|
||||
| `{rounded.xs}` | 4px | Small chips, status badges |
|
||||
| `{rounded.sm}` | 6px | Inline tags |
|
||||
| `{rounded.md}` | 8px | All buttons, form inputs |
|
||||
| `{rounded.lg}` | 12px | Pricing cards, feature cards, testimonial cards |
|
||||
| `{rounded.xl}` | 16px | Product screenshot panels |
|
||||
| `{rounded.xxl}` | 24px | Oversized CTA banners (rare) |
|
||||
| `{rounded.pill}` | 9999px | Pricing tab toggles, status pills |
|
||||
| `{rounded.full}` | 9999px | Avatar circles |
|
||||
|
||||
### Photography & Illustration Geometry
|
||||
|
||||
- Product UI screenshots dominate; they sit in `{rounded.xl}` 16px tiles with `{spacing.lg}` 24px outer padding.
|
||||
- Customer logo tiles render at small sizes (~24px logo height) on `{colors.canvas}` with no border.
|
||||
- Avatar circles in testimonial cards use `{rounded.full}` at 32–40px sizes.
|
||||
|
||||
## Components
|
||||
|
||||
### Buttons
|
||||
|
||||
**`button-primary`** — Lavender CTA. The default primary CTA across all pages.
|
||||
- Background `{colors.primary}`, text `{colors.on-primary}`, type `{typography.button}`, padding 8px 14px, rounded `{rounded.md}`.
|
||||
- Pressed state lives in `button-primary-pressed` (background shifts to `{colors.primary-focus}`).
|
||||
- Hover state lives in `button-primary-hover` (background shifts to `{colors.primary-hover}` lighter lavender).
|
||||
|
||||
**`button-secondary`** — Charcoal button. Used for secondary CTAs ("Sign in", "Read changelog").
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.button}`, padding 8px 14px, rounded `{rounded.md}`. 1px `{colors.hairline}` border.
|
||||
|
||||
**`button-tertiary`** — Plain text button.
|
||||
- Background `{colors.canvas}`, text `{colors.ink}`, type `{typography.button}`, rounded `{rounded.md}`, padding 8px 14px.
|
||||
|
||||
**`button-inverse`** — White-on-dark inverse CTA.
|
||||
- Background `{colors.inverse-canvas}`, text `{colors.inverse-ink}`, type `{typography.button}`, rounded `{rounded.md}`, padding 8px 14px.
|
||||
|
||||
### Pricing Tabs
|
||||
|
||||
**`pricing-tab-default`** + **`pricing-tab-selected`** — Pill-toggle on `/pricing`.
|
||||
- Default: `{colors.canvas}` background, `{colors.ink-subtle}` text, rounded `{rounded.pill}`, padding 6px 14px.
|
||||
- Selected: `{colors.surface-2}` background, `{colors.ink}` text — selected = surface lift.
|
||||
|
||||
### Cards & Containers
|
||||
|
||||
**`pricing-card`** — Each tier on `/pricing`.
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.body}`, rounded `{rounded.lg}`, padding 24px. 1px `{colors.hairline}` border.
|
||||
|
||||
**`pricing-card-featured`** — Recommended tier — surface lift to surface-2.
|
||||
- Background `{colors.surface-2}`, otherwise identical structure.
|
||||
|
||||
**`feature-card`** — Generic feature highlight tile.
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.body}`, rounded `{rounded.lg}`, padding 24px.
|
||||
|
||||
**`product-screenshot-card`** — The dominant card type — frames a high-fidelity Linear app UI screenshot.
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.body}`, rounded `{rounded.xl}`, padding 24px.
|
||||
|
||||
**`testimonial-card`** — Customer quote with avatar + name + role.
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.body-lg}`, rounded `{rounded.lg}`, padding 32px.
|
||||
|
||||
**`customer-logo-tile`** — Small tile in the customer marquee.
|
||||
- Background `{colors.canvas}`, text `{colors.ink-subtle}`, type `{typography.caption}`, rounded `{rounded.xs}`, padding 16px.
|
||||
|
||||
**`cta-banner`** — Closing CTA panel near page bottom.
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.headline}`, rounded `{rounded.lg}`, padding 48px.
|
||||
|
||||
### Inputs & Forms
|
||||
|
||||
**`text-input`** + **`text-input-focused`** — Form fields on `/contact/sales` and signup overlays.
|
||||
- Background `{colors.surface-1}`, text `{colors.ink}`, type `{typography.body}`, rounded `{rounded.md}`, padding 8px 12px.
|
||||
- Focused state retains the same surface; the focus ring is a 2px `{colors.primary-focus}` outline at 50% opacity.
|
||||
|
||||
### Status & Build Page
|
||||
|
||||
**`changelog-row`** — Each row in `/build` (changelog page) listing version, date, and changes.
|
||||
- Background `{colors.canvas}`, text `{colors.ink}`, type `{typography.body}`, rounded `{rounded.xs}`, padding 24px 0. 1px `{colors.hairline}` bottom rule.
|
||||
|
||||
**`status-badge`** — Small status pill.
|
||||
- Background `{colors.surface-2}`, text `{colors.ink-muted}`, type `{typography.caption}`, rounded `{rounded.pill}`, padding 2px 8px.
|
||||
|
||||
### Navigation
|
||||
|
||||
**`top-nav`** — Sticky dark bar with the Linear wordmark left, primary nav links centered, and a `button-secondary` ("Sign in") + `button-primary` ("Get started") pair right.
|
||||
- Background `{colors.canvas}`, text `{colors.ink}`, type `{typography.body-sm}`, height 56px.
|
||||
|
||||
### Footer
|
||||
|
||||
**`footer`** — Dense link grid on `{colors.canvas}` with the Linear wordmark left.
|
||||
- Background `{colors.canvas}`, text `{colors.ink-subtle}`, type `{typography.caption}`, padding 64px 32px.
|
||||
|
||||
## Do's and Don'ts
|
||||
|
||||
### Do
|
||||
|
||||
- Reserve `{colors.canvas}` (#010102) as the system's anchor surface — the faint blue tint is intentional.
|
||||
- Use `{colors.primary}` lavender ONLY for: brand mark, primary CTA, focus ring, link emphasis.
|
||||
- Use the four-step surface ladder for hierarchy. Avoid skipping levels.
|
||||
- Pair display weight 600 with body weight 400 — Linear resists 700+ display weights.
|
||||
- Apply negative letter-spacing aggressively on display.
|
||||
- Use product UI screenshots as the protagonist of every section.
|
||||
- Compose CTAs as `{rounded.md}` 8px corners.
|
||||
|
||||
### Don't
|
||||
|
||||
- Don't ship a light-mode marketing page.
|
||||
- Don't use lavender as a section background or card fill.
|
||||
- Don't introduce a second chromatic accent (orange, pink, green for marketing).
|
||||
- Don't add atmospheric gradients or spotlight cards.
|
||||
- Don't pill-round CTAs.
|
||||
- Don't use `#000000` true black as the canvas.
|
||||
- Don't combine multiple bright accents in product screenshot mockups.
|
||||
|
||||
## Responsive Behavior
|
||||
|
||||
### Breakpoints
|
||||
|
||||
| Name | Width | Key Changes |
|
||||
|---|---|---|
|
||||
| Desktop-XL | 1440px | Default desktop layout |
|
||||
| Desktop | 1280px | Card grid 3-up maintained |
|
||||
| Tablet | 1024px | Card grid 3-up → 2-up |
|
||||
| Mobile-Lg | 768px | Pricing comparison becomes accordion; nav hamburger |
|
||||
| Mobile | 480px | Single-column; display-xl scales 80px → ~36px |
|
||||
|
||||
### Touch Targets
|
||||
|
||||
- CTAs hold ≥40px tap height across viewports.
|
||||
- Pricing tab pills hold ≥36px tap height; touch viewports grow to ≥44px.
|
||||
- Form inputs hold ≥44px tap target on touch.
|
||||
|
||||
### Collapsing Strategy
|
||||
|
||||
- **Top nav**: links collapse to hamburger below 768px.
|
||||
- **Card grids**: 3-up → 2-up at 1024px → 1-up below 768px.
|
||||
- **Pricing comparison**: per-tier accordion below 768px.
|
||||
- **Display type**: `{typography.display-xl}` 80px scales toward `{typography.display-md}` 40px on mobile.
|
||||
|
||||
### Image Behavior
|
||||
|
||||
- Product UI screenshots maintain aspect ratio and never crop.
|
||||
- Customer logos in the marquee may collapse from 6-up to 3-up below 768px.
|
||||
|
||||
## Iteration Guide
|
||||
|
||||
1. Focus on ONE component at a time and reference it by its `components:` token name.
|
||||
2. When introducing a section, decide first which surface lift it lives on.
|
||||
3. Default body to `{typography.body}` at weight 400.
|
||||
4. Run `npx @google/design.md lint DESIGN.md` after edits.
|
||||
5. Add new variants as separate component entries.
|
||||
6. Treat lavender as scarce: brand mark, primary CTA, focus, link emphasis.
|
||||
7. Lead every section with a product UI screenshot.
|
||||
|
||||
## Known Gaps
|
||||
|
||||
- The four-step surface ladder values are extracted directly from Linear's `--color-bg-level-3`, `--color-line-tint`, etc. CSS variables; they are Linear's canonical surface spec.
|
||||
- Form-field error and validation styling is not visible on the inspected pages.
|
||||
- Light mode is not documented because the marketing site does not ship a light theme.
|
||||
- Linear's actual product UI uses a richer color-tag palette (red, orange, yellow, green, blue, purple) for issue priorities and project labels — those colors live in the in-product surfaces shown in mockups.
|
||||
- The custom display, text, and mono families are proprietary; an open-source substitute is acceptable.
|
||||
34
Dockerfile
34
Dockerfile
@@ -39,16 +39,11 @@ RUN wget -q -O /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-
|
||||
# --- bun (Discord bot runtime/package manager) ---
|
||||
RUN curl -fsSL https://bun.sh/install | bash
|
||||
|
||||
# --- Node.js + Gemini CLI (the GEMINI_AUTH=oauth real-time search path shells
|
||||
# out to `gemini`; it needs a real Node runtime, not bun). The account
|
||||
# login itself is provided at runtime via the mounted ~/.gemini (see
|
||||
# docker-compose.yml); here we only install the binary. ---
|
||||
ARG GEMINI_CLI_VERSION=0.46.0
|
||||
# --- Node.js (the on-screen browser search helper, browse-search.mjs, needs a
|
||||
# real Node runtime, not bun) ---
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||
&& apt-get install -y --no-install-recommends nodejs \
|
||||
&& npm install -g "@google/gemini-cli@${GEMINI_CLI_VERSION}" \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& gemini --version
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# --- Python brain/bridge deps (slim set) ---
|
||||
COPY bridge/requirements-bridge.txt /app/bridge/requirements-bridge.txt
|
||||
@@ -65,20 +60,23 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \
|
||||
> /etc/ld.so.conf.d/nvidia-cu12.conf 2>/dev/null \
|
||||
&& /sbin/ldconfig || true
|
||||
|
||||
# --- MeloTTS Korean voice (separate /opt/melo py3.11 venv; see setup-melo.sh).
|
||||
# Heavy layer (torch CPU + transformers + MeCab); placed before the app
|
||||
# COPY so it stays cached across source-only changes. ---
|
||||
# --- Korean voice: GPU MeloTTS (offline, privacy-first) in a separate /opt/melo
|
||||
# py3.11 venv; see docker/setup-melo.sh. Heavy layer (torch cu128 GPU +
|
||||
# transformers + MeCab + baked KR model), placed before the app COPY so it
|
||||
# stays cached across source-only changes. Runtime device is MELO_DEVICE=cuda
|
||||
# (compose), so TTS synthesises on the GPU next to Ollama + Whisper. Started
|
||||
# only in full/bot/brain roles (see docker/supervisord.conf). The offline
|
||||
# Edge TTS (`edge-tts`, in requirements-bridge.txt) remains available as an
|
||||
# opt-in fallback via TTS_ENGINE=edge, but it is ONLINE (text leaves the
|
||||
# host), so MeloTTS is the default. ---
|
||||
COPY docker/setup-melo.sh /app/docker/setup-melo.sh
|
||||
# Strip CR before running: a Windows checkout (autocrlf) yields CRLF, which makes
|
||||
# bash read line 18 as `set -euxo pipefail\r` and abort with
|
||||
# "set: pipefail: invalid option name". .gitattributes pins *.sh to LF, but this
|
||||
# keeps the build working even on a not-yet-renormalised working tree.
|
||||
# Strip CR first: a Windows (autocrlf) checkout yields CRLF, which makes bash
|
||||
# read `set -euxo pipefail\r` and abort. Idempotent on an LF tree.
|
||||
RUN sed -i 's/\r$//' /app/docker/setup-melo.sh && bash /app/docker/setup-melo.sh
|
||||
|
||||
# --- Human input + window management for the on-screen Chrome control tool.
|
||||
# Placed AFTER the heavy melo layer so it doesn't bust that cache. xdotool
|
||||
# injects real X pointer/keyboard events (visible cursor, char-by-char
|
||||
# typing) into the broadcast; wmctrl lists/moves windows. ---
|
||||
# xdotool injects real X pointer/keyboard events (visible cursor,
|
||||
# char-by-char typing) into the broadcast; wmctrl lists/moves windows. ---
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
xdotool wmctrl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
351
README.md
351
README.md
@@ -1,280 +1,93 @@
|
||||
# Javis Bot
|
||||
|
||||
Ubuntu 데스크톱(VNC) 위에서 도는 **디스코드 네이티브 음성 비서**입니다.
|
||||
[isair/jarvis](https://github.com/isair/jarvis)의 성숙한 AI "두뇌"(메모리·툴·답변엔진·STT/TTS)를 그대로 쓰면서,
|
||||
입출력 인터페이스를 로컬 마이크/스피커에서 **디스코드 음성 + 화면 방송**으로 바꾼 하이브리드 구성입니다.
|
||||
|
||||
- 🎙️ 디스코드 음성 채널에서 말로 대화 (음성 입력 → 두뇌 → 음성 출력)
|
||||
- 🖥️ VNC 화면을 디스코드로 송출해서 같이 보기 (셀프봇 실시간 / noVNC / 스크린샷 선택)
|
||||
- ⌨️ `/자비스` 슬래시 명령으로 호출 — 호출한 사람이 음성 채널에 있으면 그 채널로 접속
|
||||
- 🔒 모든 슬래시 명령 응답은 **호출한 사람만 보이는 ephemeral** 메시지
|
||||
- 🧠 크롬/웹 제어, 메모리, MCP 툴 등 jarvis의 기능 유지
|
||||
|
||||
> 언어 선택 근거(Python 유지 vs 재작성)는 [docs/language-comparison.md](docs/language-comparison.md) 참고.
|
||||
> VNC + XFCE 호스트 셋업은 [docs/vnc-xfce-setup.md](docs/vnc-xfce-setup.md) 참고.
|
||||
> 원본 jarvis README는 [docs/UPSTREAM-README.md](docs/UPSTREAM-README.md)에 보존했습니다.
|
||||
|
||||
---
|
||||
|
||||
## 아키텍처 (하이브리드)
|
||||
|
||||
```
|
||||
Discord ──voice / video / slash──▶ bot/ (Node + bun, discord.js)
|
||||
│ HTTP(localhost)
|
||||
▼
|
||||
bridge/ (Python, Flask)
|
||||
│ in-process import
|
||||
▼
|
||||
src/jarvis (기존 두뇌: STT·답변엔진·메모리·툴·TTS)
|
||||
```
|
||||
|
||||
- **bot/** — 디스코드 관련 전부. 슬래시 명령, 음성 송수신, VNC 화면 송출. AI 로직 없음.
|
||||
- **bridge/** — 얇은 HTTP 서비스. 음성(WAV) → 텍스트(STT) → 두뇌(답변) → 음성(TTS).
|
||||
- **src/jarvis** — 원본 jarvis 두뇌. 거의 손대지 않음. (PyQt 데스크톱 GUI/단축키 받아쓰기는 이 배포에선 사용하지 않음.)
|
||||
|
||||
왜 이렇게? 디스코드 봇은 정책상 영상(Go Live)을 송출할 수 없고, 봇 영상 송출이 되는 라이브러리는 Node 전용 + 셀프봇만 가능합니다. 반면 jarvis 두뇌는 검증된 Python 39k줄입니다. 그래서 영상이 가능한 Node로 인터페이스만 새로 짜고 두뇌는 Python 그대로 두는 하이브리드가 비용/위험 대비 최선입니다.
|
||||
|
||||
---
|
||||
|
||||
## 요구 사항
|
||||
|
||||
Docker로 돌리면(권장) 호스트에는 Docker + (GPU 쓸 경우) NVIDIA 드라이버만 있으면 되고, Python/bun/Ollama/ffmpeg/Whisper/Piper는 전부 컨테이너 안에 포함됩니다.
|
||||
|
||||
OS별 호스트 준비물:
|
||||
|
||||
| | Linux (Ubuntu 등) | Windows 11 |
|
||||
|---|---|---|
|
||||
| 컨테이너 런타임 | Docker Engine (CDI 지원, Docker 25+) | Docker Desktop + WSL2 백엔드 |
|
||||
| GPU 가속(선택) | `nvidia-container-toolkit` + `nvidia-ctk cdi generate` | NVIDIA 드라이버 + Docker Desktop GPU(WSL2) 활성화 |
|
||||
| GPU 넣는 compose | `docker-compose.gpu-linux.yml` | `docker-compose.gpu-windows.yml` |
|
||||
|
||||
- 디스코드 **봇** 토큰 1개 (음성/슬래시) — 또는 (셀프봇 송출 사용 시) 디스코드 **버너 유저** 토큰 1개
|
||||
- (도커 없이 수동 실행 시에만) Python 3.11+, [bun](https://bun.sh), Ollama, `ffmpeg`를 호스트에 직접 설치 — 아래 "수동" 절 참고
|
||||
|
||||
> VNC 데스크톱 호스트를 직접 구성하는 경우(도커 미사용)는 `docs/vnc-xfce-setup.md` 참고. 도커 실행에서는 VNC+XFCE가 컨테이너 안에 이미 들어 있습니다.
|
||||
|
||||
---
|
||||
|
||||
## 실행 — Docker (권장)
|
||||
|
||||
환경 설정 없이 통째로 컨테이너에서 돌립니다. VNC 데스크톱 + 크롬 + Python 브릿지 + Node 봇이 한 컨테이너(`javis`)에, LLM 백엔드(Ollama)가 별도 컨테이너에 뜹니다. **올리기만 하면 Ollama 모델까지 자동으로** 받아집니다.
|
||||
|
||||
베이스 `docker-compose.yml`에는 GPU 설정이 없습니다(이식성 유지). GPU는 OS에 맞는 override 파일을 같이 얹어서 켭니다. **돌리는 OS에 따라 명령이 다릅니다:**
|
||||
|
||||
```bash
|
||||
# ── Linux (Ubuntu 등, nvidia-container-toolkit + CDI) ──
|
||||
docker compose -f docker-compose.yml -f docker-compose.gpu-linux.yml up -d --build
|
||||
|
||||
# ── Windows 11 (Docker Desktop + WSL2 + NVIDIA) ──
|
||||
docker compose -f docker-compose.yml -f docker-compose.gpu-windows.yml up -d --build
|
||||
|
||||
# ── GPU 없이 (CPU 전용 호스트) ──
|
||||
# .env 에 WHISPER_DEVICE=cpu, MELO_DEVICE=cpu 를 넣고 베이스만 사용
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
매번 `-f`를 치기 싫으면 `.env`에 한 줄 넣어두면 그냥 `docker compose up -d`로 됩니다(override가 자동 적용):
|
||||
|
||||
```bash
|
||||
# Linux / macOS (구분자 = 콜론 ":")
|
||||
COMPOSE_FILE=docker-compose.yml:docker-compose.gpu-linux.yml
|
||||
# Windows 11 (구분자 = 세미콜론 ";" — 콜론은 드라이브 문자 C: 와 충돌)
|
||||
COMPOSE_FILE=docker-compose.yml;docker-compose.gpu-windows.yml
|
||||
```
|
||||
|
||||
> ⚠️ `COMPOSE_FILE`의 파일 구분자는 OS마다 다릅니다: Linux/macOS는 `:`, Windows는 `;`. Windows에서 `:`를 쓰면 Docker가 전체를 파일 하나 이름으로 읽어 `... The system cannot find the file specified` 에러가 납니다. 헷갈리면 `COMPOSE_FILE`을 비워두고 실행 시 직접 지정하세요: `docker compose -f docker-compose.yml -f docker-compose.gpu-windows.yml up -d --build`.
|
||||
|
||||
> Linux와 Windows는 GPU를 컨테이너에 넣는 방식이 달라서 override 파일이 갈립니다. Linux는 CDI(`devices: nvidia.com/gpu=all`), Windows(Docker Desktop)는 Compose의 `deploy.resources.reservations.devices`(`driver: nvidia`)를 씁니다. 호스트 사전 준비는 아래 "GPU 가속" 절 참고.
|
||||
|
||||
`docker compose up` 한 번이면 자동으로:
|
||||
- Ollama 서버가 뜨고, `ollama-init`이 채팅/임베딩 모델을 **자동 pull**
|
||||
- VNC+XFCE 데스크톱 + 크롬 + Python 브릿지가 기동
|
||||
- Whisper STT 모델 / Piper TTS 음성 자동 다운로드(볼륨에 캐시)
|
||||
|
||||
화면 보기: VNC 뷰어 → `localhost:5901` (비밀번호 = `.env`의 `VNC_PASSWORD`, 기본 `javis123`) 또는 브라우저 → `http://localhost:6080/vnc.html`.
|
||||
로그: `docker compose logs -f javis`.
|
||||
|
||||
### 디스코드 토큰은 마지막에
|
||||
|
||||
토큰 없이도 위의 모든 게 정상 동작합니다(봇만 대기). 준비되면 `.env`를 만들어 토큰을 채웁니다.
|
||||
|
||||
기본 모드는 **유저봇(selfbot)** 입니다. 음성 참여와 화면 송출(Go Live)을 한 유저 계정으로 처리하며, Discord가 일반 봇 계정에는 Go Live를 허용하지 않기 때문에 이 방식이 기본입니다.
|
||||
|
||||
```bash
|
||||
cp .env.example .env # DISCORD_SELFBOT_TOKEN(버너 계정) + DISCORD_VOICE_CHANNEL_ID 채우기
|
||||
docker compose up -d # 유저봇이 로그인해 지정 음성채널에 자동 참여
|
||||
```
|
||||
|
||||
유저봇은 슬래시 명령을 쓸 수 없으므로 텍스트로 제어합니다: 음성 채널에서 `!자비스 join` / `!자비스 leave`. `DISCORD_VOICE_CHANNEL_ID`를 채워두면 시작 시 자동 참여합니다.
|
||||
|
||||
> ⚠️ 유저봇은 Discord ToS 위반이며 계정 정지 위험이 있습니다. 반드시 일회용 **버너 계정** 토큰만 사용하세요. 자세한 주의사항은 아래 "셀프봇 주의" 절을 참고하세요.
|
||||
|
||||
일반 봇(슬래시 명령 `/자비스`)으로 돌리려면 `DISCORD_BOT_TOKEN` / `DISCORD_APP_ID` / `DISCORD_GUILD_ID`를 채우세요. 다만 일반 봇은 화면 송출(Go Live)을 할 수 없습니다. `DISCORD_BOT_TOKEN`이 비어 있고 `DISCORD_SELFBOT_TOKEN`이 있으면 자동으로 유저봇 모드로 동작합니다. (`OLLAMA_CHAT_MODEL` 등 모델을 바꾸려면 `.env`에서 지정 후 `docker compose up -d`.)
|
||||
|
||||
### GPU 가속 (OS별)
|
||||
|
||||
LLM(Ollama), Whisper STT, MeloTTS가 GPU에서 돕니다(env 기본 `WHISPER_DEVICE=cuda`, `MELO_DEVICE=cuda`). NVIDIA Blackwell(sm_120, 예: RTX 5050/5070Ti)에서 검증: 컨테이너 내 torch cu128 CUDA 동작, Ollama GPU 오프로드, faster-whisper float16, MeloTTS GPU 합성 모두 확인.
|
||||
|
||||
GPU는 위 "실행 — Docker"의 OS별 override 파일로 켜집니다. 호스트 사전 준비는 OS마다 다릅니다:
|
||||
|
||||
**Linux (Ubuntu 등) — CDI 방식, 1회:**
|
||||
|
||||
```bash
|
||||
# nvidia-container-toolkit 설치 후 CDI 스펙 생성 (Docker 25+ CDI, 데몬 재시작 불필요)
|
||||
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
|
||||
docker run --rm --device nvidia.com/gpu=all ubuntu nvidia-smi -L # GPU 보이면 OK
|
||||
```
|
||||
|
||||
`docker-compose.gpu-linux.yml`이 두 컨테이너에 `devices: ["nvidia.com/gpu=all"]`(CDI)로 GPU를 넣습니다.
|
||||
|
||||
**Windows 11 — Docker Desktop + WSL2:**
|
||||
|
||||
- 최신 NVIDIA 게임/스튜디오 드라이버 설치(별도 CUDA 툴킷 불필요).
|
||||
- Docker Desktop → Settings → Resources → WSL Integration 활성화(WSL2 백엔드). 최신 Docker Desktop은 WSL2에서 GPU를 자동 노출합니다.
|
||||
- 확인: PowerShell에서 `docker run --rm --gpus all nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi`.
|
||||
- `docker-compose.gpu-windows.yml`이 `deploy.resources.reservations.devices`(`driver: nvidia`, `count: all`)로 GPU를 넣습니다.
|
||||
|
||||
**공통:**
|
||||
|
||||
- 모델: 베이스 compose 기본은 `qwen2.5:3b`(8GB VRAM에서 도구호출 안정적). 더 무겁게(`qwen2.5:7b`, `qwen3:8b` 등) 쓰려면 `.env`의 `OLLAMA_CHAT_MODEL` 변경.
|
||||
- **GPU가 없거나 인식 실패 시 자동으로 CPU 폴백**(Whisper)하므로 안전합니다. 명시적으로 CPU만 쓰려면 override 파일 없이 베이스만 올리고 `.env`에 `WHISPER_DEVICE=cpu`, `MELO_DEVICE=cpu`를 두세요.
|
||||
|
||||
- 데이터(메모리 DB), Whisper 캐시, Piper 음성은 named volume에 영속됩니다.
|
||||
- 셀프봇 영상 송출 의존성은 이미지에 기본 포함하지 않습니다. 쓰려면 컨테이너에서 `cd /app/bot && bun add discord.js-selfbot-v13 @dank074/discord-video-stream` 후 재시작(또는 Dockerfile에 추가).
|
||||
|
||||
---
|
||||
|
||||
## 실행 — 수동(도커 없이)
|
||||
|
||||
도커 없이 호스트에서 직접 돌릴 때는 OS별로 venv 활성화·ffmpeg 설치·실행 스크립트가 다릅니다.
|
||||
|
||||
**Linux / macOS:**
|
||||
|
||||
```bash
|
||||
# 1) 환경 변수
|
||||
cp .env.example .env # DISCORD_BOT_TOKEN / DISCORD_APP_ID / DISCORD_GUILD_ID 등 채우기
|
||||
|
||||
# 2) Python 두뇌 + 브릿지 의존성
|
||||
python3 -m venv .venv && . .venv/bin/activate
|
||||
pip install -r requirements.txt # jarvis 두뇌
|
||||
pip install flask # 브릿지(없으면)
|
||||
|
||||
# 3) 디스코드 봇 의존성 (bun)
|
||||
cd bot && bun install && cd ..
|
||||
|
||||
# 4) 한 번에 실행 (브릿지 + 봇)
|
||||
./scripts/dev.sh
|
||||
# 또는 따로: ./scripts/start_bridge.sh / ./scripts/start_bot.sh
|
||||
```
|
||||
|
||||
- `ffmpeg`: Ubuntu `sudo apt install ffmpeg`, macOS `brew install ffmpeg`.
|
||||
|
||||
**Windows 11 (PowerShell):**
|
||||
|
||||
```powershell
|
||||
# 1) 환경 변수
|
||||
copy .env.example .env # 같은 키들 채우기
|
||||
|
||||
# 2) Python 두뇌 + 브릿지 의존성 (venv 활성화 경로가 다름)
|
||||
py -3 -m venv .venv; .\.venv\Scripts\Activate.ps1
|
||||
pip install -r requirements.txt
|
||||
pip install flask
|
||||
|
||||
# 3) 디스코드 봇 의존성 (bun — Windows 네이티브 또는 WSL2)
|
||||
cd bot; bun install; cd ..
|
||||
|
||||
# 4) 실행: .sh 스크립트는 bash 전용이라 Windows에서는 두 프로세스를 따로 띄웁니다
|
||||
# (PowerShell 창 2개, 또는 WSL2에서 위 Linux 절차 그대로 사용 권장)
|
||||
python -m bridge.server # 창 1: 브릿지
|
||||
cd bot; bun run register; bun run start # 창 2: (일반 봇이면) 슬래시 등록 후 봇 기동
|
||||
```
|
||||
|
||||
- `ffmpeg`: `winget install Gyan.FFmpeg` 또는 `choco install ffmpeg` 후 PATH 확인.
|
||||
- `scripts/*.sh`(dev/start_bridge/start_bot)는 bash 스크립트라 순수 Windows에선 동작하지 않습니다. 가장 간단한 길은 **WSL2 안에서 위 Linux 절차를 그대로** 쓰는 것입니다(도커도 WSL2 백엔드와 동일).
|
||||
|
||||
봇이 뜨면 디스코드에서 `/자비스 join` 으로 음성 채널에 부르세요.
|
||||
|
||||
---
|
||||
|
||||
## 슬래시 명령 (`/자비스`)
|
||||
|
||||
| 명령 | 동작 |
|
||||
# Javis Bot v2
|
||||
|
||||
게임PC 한 대에서 **완전 로컬**로 도는 디스코드 음성 비서입니다. 설치 파일 하나로 깔리는 **Windows 관리 앱(exe)** 에서 모든 설정과 관리를 합니다.
|
||||
|
||||
> **상태: 설계 단계.** 이 문서와 [docs/v2/](docs/v2/)는 v2 계획입니다. 지금 저장소의 코드는 v1(Docker + VNC 구성)이고, v1 사용법은 [docs/v1/README.md](docs/v1/README.md)에 있습니다.
|
||||
|
||||
## 개인정보 우선
|
||||
|
||||
- 음성 인식(STT), 답변(LLM), 음성 합성(TTS)을 모두 게임PC 안에서 처리합니다. 클라우드 AI API는 쓰지 않습니다. 밖으로 나가는 요청은 웹 검색과 날씨 조회뿐입니다.
|
||||
- 음성 원본과 화면 캡처는 기본적으로 저장하지 않습니다.
|
||||
- 대화 기록은 사람별로 보고 지울 수 있습니다.
|
||||
- 봇 토큰 같은 비밀값은 Windows DPAPI로 암호화해서 저장합니다.
|
||||
|
||||
자세한 내용: [docs/v2/privacy.md](docs/v2/privacy.md)
|
||||
|
||||
## 주요 기능
|
||||
|
||||
- **빠른 음성 대화**: 말하는 도중에 전사하고 답은 문장 단위로 바로 읽어서, 말이 끝난 뒤 첫 소리까지 약 1~1.5초를 목표로 합니다.
|
||||
- **최대 10명 동시 대화**: 화자별로 따로 받아 전사합니다. 여러 명이 동시에 부를 때 어떻게 답할지(처음 대화만 / 들어온 순서대로 / 마지막 대화만 / 제일 중요한 대화만)는 GUI에서 고릅니다.
|
||||
- **호출어와 지정한 사람**:
|
||||
- 호출어는 GUI에서 여러 개 등록합니다.
|
||||
- "지정한 사람만 응답"과 "지정한 사람은 호출어 없이 대답"을 각각 켜고 끕니다.
|
||||
- **AI 채팅 채널**: 등록한 디스코드 채팅 채널을 자비스가 읽고, 메시지를 보내고, 자기 메시지를 고칩니다. 채널 ID, 이름, 설명은 GUI에서 편집하고, 고친 이름·설명은 실제 디스코드 채널 이름·주제에도 반영합니다(끌 수 있음).
|
||||
- **예약 알림과 정기 작업**: "10분 뒤에 알려 줘", "내일 7시에 회의 알려 줘", "평일 아침 9시에 날씨 알려 줘"처럼 말이나 채팅으로 맡깁니다. 맡긴 사람이 자비스와 같은 음성 채널에 있으면 음성으로, 아니면 채팅 채널에 글로 전합니다. 정기 작업은 날씨, 웹 검색처럼 찾아보는 기능만 쓰고, 관리 앱 "예약" 화면에서 목록을 보고 지울 수 있습니다.
|
||||
- **수집 소식 알림**: 백그라운드 수집 대상마다 새 글을 바로, 또는 하루 한 번 정한 채팅 채널에 제목과 링크로 알립니다(기본은 알리지 않음). "수집한 소식 정리해 줘"처럼 물으면 모은 글을 읽고 답하며, 정기 작업으로 맡길 수도 있습니다.
|
||||
- **사람별 지침**: "앞으로 나한테는 반말로 해", "날씨는 섭씨로만"처럼 말하면 그 사람의 지침으로 저장하고, 그 사람에게 답할 때만 따릅니다. 관리 앱 "사람별 지침" 화면에서 보고 고칠 수 있습니다.
|
||||
- **대화 기억**: 본인이 말했던 기록, 등록한 채팅 채널, 같은 음성 채널에 함께 있던 다른 사람 말에서 이번 질문과 관련된 내용을 찾아 답할 때 참고합니다. 다른 사람 말 참고는 GUI에서 끌 수 있습니다.
|
||||
- **화면 보기**:
|
||||
- 전체 화면, 모니터별, 프로그램(창)별 화면을 자비스가 직접 봅니다. 관리 앱 "PC 제어" 화면에 등록한 사람만 시킬 수 있습니다.
|
||||
- 비밀번호 관리자처럼 보면 안 되는 프로그램은 "화면 보기에서 뺄 프로그램"에 넣으면 그 창을 고르지 않고, 전체 화면에서도 검게 가립니다. 캡처한 그림은 그 답에만 쓰고 저장하지 않습니다.
|
||||
- "2번 화면에 뭐 떠 있어?" 같은 질문도 1.5~2.5초 안에 답하는 것을 목표로 합니다.
|
||||
- 다른 사람의 디스코드 화면공유는 자비스가 말로 공유 창 분리(팝아웃)를 부탁하고, 분리된 창만 캡처해서 봅니다.
|
||||
- **게임 모드**:
|
||||
- 전체화면 게임과 테두리 없는 창 모드 게임을 감지합니다.
|
||||
- 게임 중에는 LLM이 쓰던 VRAM을 게임에 내주도록 앱이 자동으로 조절합니다. 모델은 하나만 쓰고, PC 성능이 부족하면 그 PC에 맞게 설정을 낮춥니다.
|
||||
- 원하면 게임 중에는 봇을 일시정지할 수 있습니다.
|
||||
- **안티치트 보호**: 뱅가드 같은 안티치트가 걸린 게임이 실행 중이면 키보드·마우스 자동 조작을 막고 그 이유를 알려 줍니다.
|
||||
- **기존 기능 유지**: 메모리, 날씨, 웹 검색, 식단 기록, MCP, 브라우저 제어를 그대로 가져갑니다.
|
||||
- **기상청 날씨**: 관리 앱 "기능" 화면에 기상청 API허브 인증키를 넣으면 한국 안의 현재 날씨와 10일 뒤까지의 예보("내일 날씨", "주말 날씨")를 기상청에서 받습니다. "부산 연제구"처럼 구 단위로 물으면 그 구의 날씨를 답하고(장소는 OpenStreetMap에서 찾습니다), "자세히" 물으면 습도, 바람, 강수량까지 답합니다(초단기실황조회, 초단기예보조회, 단기예보조회, 중기기온조회, 중기육상예보조회 활용신청 필요). 키가 없거나 한국 밖이면 open-meteo에서 받고, 현재 날씨는 그다음 met.no에서 받습니다.
|
||||
- **관리 앱**:
|
||||
- 첫 실행 마법사, 대시보드, 모델 다운로드, 기능별 켜기/끄기를 제공합니다.
|
||||
- 로그 화면에서 로그를 보고, 복사하고, 내려받을 수 있습니다. 비밀값은 자동으로 가립니다.
|
||||
- 로그 화면의 "대화"에서 날짜별 대화 내용(사람 말, 자비스 생각과 도구 호출, 답)을 txt로, "목소리"에서 녹음한 목소리를 zip으로 저장합니다. 목소리 녹음은 기본으로 꺼져 있고, 켜면 이 PC에만 7일 동안 남습니다.
|
||||
|
||||
## 요구 사항 (예정)
|
||||
|
||||
| 항목 | 기준 |
|
||||
|---|---|
|
||||
| `/자비스 join` | 호출자가 있는 음성 채널에 접속해 듣기 시작 |
|
||||
| `/자비스 leave` | 음성 채널에서 나감 |
|
||||
| `/자비스 ask 질문:<내용>` | 텍스트로 질문하고 답을 받음 |
|
||||
| `/자비스 stream` | VNC 화면을 디스코드에 송출 시작 |
|
||||
| `/자비스 stop` | 송출 중단 |
|
||||
| `/자비스 status` | 브릿지 두뇌/세션/송출 상태 확인 |
|
||||
| OS | Windows 11 |
|
||||
| GPU | NVIDIA RTX 계열, VRAM 16GB 권장 (기준 PC: RTX 5070 Ti 16GB) |
|
||||
| 드라이버 | NVIDIA 580 이상 (CUDA 13). 낮으면 CUDA 12 빌드를 씁니다 |
|
||||
| RAM | 32GB 이상 (기준 PC: 128GB) |
|
||||
| 디스크 | 최소 120GB, 권장 200GB 이상 여유 (NVMe 권장) |
|
||||
| 디스코드 | 봇 토큰 1개. 채팅 채널을 쓰려면 개발자 포털에서 Message Content Intent를 켭니다 |
|
||||
|
||||
모든 응답은 **호출한 사람에게만** 보입니다(ephemeral).
|
||||
Python, CUDA 라이브러리, 모델, 브라우저는 첫 실행 때 사용자가 고른 데이터 폴더에 자동으로 받습니다. 따로 설치할 것은 NVIDIA 드라이버뿐입니다.
|
||||
|
||||
---
|
||||
## 설치 (예정)
|
||||
|
||||
## VNC 화면 송출 백엔드 (`STREAM_BACKEND`)
|
||||
1. 설치 파일(exe)을 실행합니다. 서명되지 않은 파일이라 SmartScreen 경고가 뜨면 "추가 정보 → 실행"을 누릅니다.
|
||||
2. 첫 실행 마법사를 따라갑니다.
|
||||
1. 데이터 폴더를 고릅니다.
|
||||
2. PC 점검을 합니다.
|
||||
3. 받을 모델 크기와 예상 시간을 확인합니다.
|
||||
4. 자동 설치와 자가 점검(이 PC에 맞는 설정 정하기)을 기다립니다.
|
||||
5. 디스코드 봇을 연결합니다.
|
||||
6. 테스트합니다.
|
||||
3. 대시보드에서 봇을 시작합니다.
|
||||
|
||||
`.env`에서 교체 가능합니다. 코드 변경 없이 위험/방식만 바꿉니다.
|
||||
## 알려진 제한 (예정)
|
||||
|
||||
| 값 | 방식 | 실시간 | 디스코드 native | 밴 위험 |
|
||||
|---|---|---|---|---|
|
||||
| `selfbot` (기본) | 버너 유저 계정으로 Go Live 실시간 송출 | ✅ | ✅ | ⚠️ ToS 위반·정지 위험 |
|
||||
| `novnc` | noVNC 브라우저 링크 공유 | ✅ | ❌ | 없음 |
|
||||
| `screenshot` | N초마다 채널에 스크린샷 업로드 | ❌ | ❌ | 없음 |
|
||||
| `none` | 비활성화 | — | — | — |
|
||||
- 저작권 보호 영상, 일부 게임의 독점 전체화면, 최소화된 창은 캡처가 검게 나올 수 있습니다. 게임은 테두리 없는 창 모드면 대부분 보입니다.
|
||||
- 화면 보기는 이미지 입력을 지원하는 LLM에서만 됩니다.
|
||||
- 디스코드 봇은 다른 사람의 메시지를 수정할 수 없습니다. 삭제와 고정만 됩니다.
|
||||
- 공식 봇 계정은 화면공유 영상을 직접 받을 수 없습니다. 그래서 게임PC의 디스코드 앱 창을 캡처하는 방식을 씁니다.
|
||||
|
||||
### 셀프봇(selfbot) 주의
|
||||
## 문서
|
||||
|
||||
- 디스코드 봇은 영상 송출이 불가능해, 실시간 화면 방송은 **유저 계정 토큰(셀프봇)** 으로만 됩니다.
|
||||
- 이는 Discord ToS 위반이며 계정이 영구 정지될 수 있습니다.
|
||||
- 반드시 **버너(일회용) 계정**을 만들어 그 토큰을 `DISCORD_SELFBOT_TOKEN`에 넣고, 본계정은 절대 쓰지 마세요.
|
||||
- 영상 송출만 조용히 하는 패턴은 상대적으로 위험이 낮지만 0은 아닙니다.
|
||||
- 의존성(네이티브)은 선택 설치입니다:
|
||||
```bash
|
||||
cd bot && bun add discord.js-selfbot-v13 @dank074/discord-video-stream
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 환경 변수
|
||||
|
||||
전체 목록과 설명은 [`.env.example`](.env.example)에 있습니다. 핵심:
|
||||
|
||||
- `DISCORD_BOT_TOKEN`, `DISCORD_APP_ID`, `DISCORD_GUILD_ID` — 봇/길드
|
||||
- `BRIDGE_URL` — 봇이 호출할 브릿지 주소 (기본 `http://127.0.0.1:8765`)
|
||||
- `STREAM_BACKEND`, `DISCORD_SELFBOT_TOKEN`, `NOVNC_URL` — 화면 송출
|
||||
- `VNC_DISPLAY=:1`, `VNC_RESOLUTION`, `VNC_FRAMERATE`, `VNC_BITRATE_KBPS` — 캡처
|
||||
- `WHISPER_DEVICE/COMPUTE_TYPE`, `MELO_DEVICE` — GPU 호스트면 `cuda`/`float16`, CPU 전용이면 `cpu`(GPU 자체는 OS별 override compose 파일로 켬)
|
||||
- `OLLAMA_CHAT_MODEL` — 두뇌 LLM (기본 `qwen2.5:3b`)
|
||||
- `COMPOSE_FILE` — OS별 GPU override를 매번 `-f`로 안 치고 자동 적용 (위 "실행 — Docker" 참고)
|
||||
- `output_language` — 출력 언어 고정(비우면 사용자 언어). 설정 웹 UI(`/settings`)에서 바꾸면 env 기본값보다 우선하며 컨테이너 재생성 후에도 유지됩니다.
|
||||
- `AGENTS_DIR` — 운영자 지시문 폴더(기본 `/app/agents`, `./agents`가 read-only로 마운트됨). 아래 "운영자 지시문" 참고.
|
||||
|
||||
---
|
||||
|
||||
## 운영자 지시문 (`agents/*.md`)
|
||||
|
||||
`agents/` 폴더에 마크다운 파일을 넣으면 그 내용이 어시스턴트의 메인 답변 시스템 프롬프트 뒤에 그대로 추가됩니다. 페르소나(집사 성격)는 그대로 두고 규칙·말투·금칙어 등을 덧붙일 때 쓰세요.
|
||||
|
||||
- `agents/` 안의 모든 `*.md`를 **파일명 순서**로 이어 붙입니다. 순서를 정하려면 `00-tone.md`, `10-rules.md`처럼 숫자 접두사를 쓰세요.
|
||||
- **매 답변마다 다시 읽습니다.** 파일을 저장하면 다음 발화부터 바로 반영되며, 재빌드/재시작이 필요 없습니다(폴더가 read-only로 마운트됨).
|
||||
- 폴더가 없거나 비어 있으면 아무 일도 일어나지 않습니다(fail-open).
|
||||
- `agents/example.md.sample`을 `rules.md` 등 `*.md`로 복사해서 시작하세요. `.sample` 파일은 로드되지 않습니다.
|
||||
|
||||
---
|
||||
|
||||
## 현재 상태 / 남은 작업
|
||||
|
||||
이 레포는 동작하는 **스캐폴드**입니다. 구조·명령·송출 백엔드·브릿지 연동은 완성되어 있고, 실제 토큰/모델/VNC 디스플레이를 붙여 런타임 검증이 필요한 부분이 남아 있습니다.
|
||||
|
||||
- [ ] 실제 디스코드 봇/버너 토큰으로 음성 송수신 end-to-end 검증
|
||||
- [ ] faster-whisper(CUDA) + Piper 모델로 STT/TTS 실측
|
||||
- [ ] 셀프봇 영상 송출 라이브러리 버전별 API 실연결(현재 v6 API 기준 작성)
|
||||
- [ ] Ollama 모델 다운로드 및 두뇌 응답 품질 점검
|
||||
|
||||
---
|
||||
| 문서 | 내용 |
|
||||
|---|---|
|
||||
| [docs/v2/architecture.md](docs/v2/architecture.md) | 목표, v1과 비교, 하드웨어 자원 배분, 전체 구조, 기술 선택 |
|
||||
| [docs/v2/voice-and-llm.md](docs/v2/voice-and-llm.md) | 실시간 음성 파이프라인(STT, TTS, 지연 목표)과 LLM(Strata, 모델) |
|
||||
| [docs/v2/conversation.md](docs/v2/conversation.md) | 10명 동시 대화, 호출어와 지정한 사람, 응답 정책, AI 채팅 채널 |
|
||||
| [docs/v2/memory.md](docs/v2/memory.md) | 이전 대화 참고(대화 기억) |
|
||||
| [docs/v2/desktop-app.md](docs/v2/desktop-app.md) | 관리 앱: 첫 실행 마법사, 데이터 폴더, 설정 화면, 로그 화면 |
|
||||
| [docs/v2/pc-features.md](docs/v2/pc-features.md) | 게임 모드, 화면 보기, 키보드·마우스와 안티치트, 봇용 크롬창, 화면공유 보기, 정보 수집 |
|
||||
| [docs/v2/privacy.md](docs/v2/privacy.md) | 개인정보·법적 안전장치 |
|
||||
| [docs/v2/roadmap.md](docs/v2/roadmap.md) | 구현 단계(M0~M8)와 위험 요소 |
|
||||
| [docs/v2/decisions.md](docs/v2/decisions.md) | 확정된 결정 |
|
||||
|
||||
## 크레딧
|
||||
|
||||
- 두뇌: [isair/jarvis](https://github.com/isair/jarvis) (라이선스는 [LICENSE](LICENSE) 참고)
|
||||
- 디스코드 음성: [discord.js](https://discord.js.org) / [@discordjs/voice](https://github.com/discordjs/voice)
|
||||
- 영상 송출: [@dank074/discord-video-stream](https://github.com/Discord-RE/Discord-video-stream)
|
||||
두뇌(메모리, 도구, 답변 엔진)는 [isair/jarvis](https://github.com/isair/jarvis)를 기반으로 합니다. 원본 README는 [docs/UPSTREAM-README.md](docs/UPSTREAM-README.md)에 있습니다.
|
||||
|
||||
13
agents/llm.md
Normal file
13
agents/llm.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# 자비스 운영자 지시
|
||||
|
||||
- 너의 이름은 자비스다.
|
||||
- 모든 답변은 음성(TTS)으로 읽혀 나간다. 그러니 무조건 한 문장으로만 답한다. 두 문장 이상 쓰지 않는다. 목록, 마크다운, 이모지, 그리고 소리 내어 읽기 어려운 특수문자는 쓰지 않는다.
|
||||
- 정해진 문구에만 반응하지 말고, 실제 사람처럼 말의 뉘앙스와 맥락으로 의도를 알아듣고 처리한다.
|
||||
|
||||
화면 속 크롬(방송 화면)에서 유튜브를 다룰 때 (화면에 보여야 하므로 반드시 on-screen 브라우저 제어 도구로 수행한다):
|
||||
|
||||
- "유튜브 켜줘" → 방송 크롬에서 유튜브를 연다.
|
||||
- "유튜브에서 OO 검색해줘" → 유튜브로 가서 검색창에 OO를 사람이 직접 타이핑하듯 입력하고 검색한다.
|
||||
- "위에서 N번째 영상 재생해줘" 또는 "왼쪽에서 N번째 영상 재생해줘" → 검색 결과 목록에서 그 위치의 영상을 재생한다.
|
||||
- "일시정지해줘" → 현재 영상을 일시정지한다. "다시 재생해줘" → 이어서 재생한다.
|
||||
- "영상 종료" 또는 "그만 보여줘" → 뒤로 가서 직전 화면으로 돌아간다.
|
||||
1
app/.gitignore
vendored
Normal file
1
app/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
target/
|
||||
4778
app/Cargo.lock
generated
Normal file
4778
app/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
57
app/Cargo.toml
Normal file
57
app/Cargo.toml
Normal file
@@ -0,0 +1,57 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.33"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
license = "MIT"
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "jarvis_app"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "jarvis-app"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
serenity = { version = "0.12.5", default-features = false, features = ["client", "gateway", "model", "cache", "http", "voice", "rustls_backend"] }
|
||||
songbird = { version = "0.6.1", features = ["receive", "serenity", "gateway", "driver", "rustls"] }
|
||||
symphonia = { version = "0.5", default-features = false, features = ["wav", "pcm"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-tungstenite = "0.26"
|
||||
futures-util = "0.3"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
rand = "0.9"
|
||||
base64 = "0.22"
|
||||
async-trait = "0.1"
|
||||
opus2 = "0.4"
|
||||
fs4 = "1.1"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream"] }
|
||||
sha2 = "0.10"
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
|
||||
zip = { version = "9", default-features = false, features = ["deflate"] }
|
||||
unicode-normalization = "0.1"
|
||||
jpeg-encoder = "0.6"
|
||||
rusqlite = { version = "0.37", features = ["bundled"] }
|
||||
|
||||
[dev-dependencies]
|
||||
jpeg-decoder = "0.3"
|
||||
|
||||
[[test]]
|
||||
name = "vdisplay_elevated"
|
||||
harness = false
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 4
|
||||
strip = "symbols"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp", "Win32_Graphics_Dwm"] }
|
||||
windows = { version = "0.62", features = ["Foundation", "Graphics", "Graphics_Capture", "Graphics_DirectX", "Graphics_DirectX_Direct3D11", "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_System_WinRT", "Win32_System_WinRT_Direct3D11", "Win32_System_WinRT_Graphics_Capture"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
2
app/shell/.gitignore
vendored
Normal file
2
app/shell/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
target/
|
||||
gen/
|
||||
8038
app/shell/Cargo.lock
generated
Normal file
8038
app/shell/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
27
app/shell/Cargo.toml
Normal file
27
app/shell/Cargo.toml
Normal file
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.33"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
license = "MIT"
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "jarvis"
|
||||
path = "src/main.rs"
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2", features = [] }
|
||||
|
||||
[dependencies]
|
||||
jarvis-app = { path = ".." }
|
||||
tauri = { version = "2", features = ["tray-icon", "image-png"] }
|
||||
tauri-plugin-single-instance = "2"
|
||||
serde_json = "1"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 4
|
||||
strip = "symbols"
|
||||
3
app/shell/build.rs
Normal file
3
app/shell/build.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
tauri_build::build()
|
||||
}
|
||||
7
app/shell/capabilities/default.json
Normal file
7
app/shell/capabilities/default.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"$schema": "../gen/schemas/desktop-schema.json",
|
||||
"identifier": "default",
|
||||
"description": "The management window: core only (the app's own `admin` command needs no permission).",
|
||||
"windows": ["main"],
|
||||
"permissions": ["core:default"]
|
||||
}
|
||||
BIN
app/shell/icons/icon.ico
Normal file
BIN
app/shell/icons/icon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 55 KiB |
BIN
app/shell/icons/icon.png
Normal file
BIN
app/shell/icons/icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
BIN
app/shell/icons/tray.png
Normal file
BIN
app/shell/icons/tray.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
347
app/shell/installer.nsi
Normal file
347
app/shell/installer.nsi
Normal file
@@ -0,0 +1,347 @@
|
||||
; The Jarvis installer (admin.spec.md, "Installer"). Built on Linux by installer.sh with makensis:
|
||||
; makensis -DVERSION=0.1.0 -DSRC=<Jarvis folder> -DOUT=<setup.exe> -DICON=<icon.ico> -DICON_FILE=<jarvis-<hash>.ico> -DWEBVIEW2=<bootstrapper>
|
||||
; -DUNINSTALL_LIST=<the Delete/RMDir lines of the installed files> installer.nsi
|
||||
; Program Files, one administrator prompt; the data folder is chosen by the app's first-run wizard
|
||||
; and never touched here. /S installs silently (the app's updates: /S /RUN starts the app afterwards).
|
||||
|
||||
Unicode true
|
||||
SetCompressor /SOLID lzma
|
||||
!include "MUI2.nsh"
|
||||
!include "x64.nsh"
|
||||
!include "LogicLib.nsh"
|
||||
!include "FileFunc.nsh"
|
||||
|
||||
!define UNINST_KEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\JarvisBot"
|
||||
!ifndef WEBVIEW2_KEY ; a test build points it elsewhere to act as a PC without the runtime
|
||||
!define WEBVIEW2_KEY "SOFTWARE\WOW6432Node\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}"
|
||||
!endif
|
||||
|
||||
Name "Jarvis"
|
||||
OutFile "${OUT}"
|
||||
InstallDir "$PROGRAMFILES64\Jarvis"
|
||||
RequestExecutionLevel admin
|
||||
BrandingText "Jarvis ${VERSION}"
|
||||
VIProductVersion "${VERSION}.0"
|
||||
VIAddVersionKey /LANG=0 "ProductName" "Jarvis"
|
||||
VIAddVersionKey /LANG=0 "FileVersion" "${VERSION}"
|
||||
VIAddVersionKey /LANG=0 "FileDescription" "Jarvis 설치"
|
||||
VIAddVersionKey /LANG=0 "LegalCopyright" "MIT"
|
||||
|
||||
!define MUI_ICON "${ICON}"
|
||||
!define MUI_UNICON "${ICON}"
|
||||
!define MUI_ABORTWARNING
|
||||
!define MUI_FINISHPAGE_RUN
|
||||
!define MUI_FINISHPAGE_RUN_TEXT "Jarvis 실행"
|
||||
!define MUI_FINISHPAGE_RUN_FUNCTION RunAsUser
|
||||
!insertmacro MUI_PAGE_DIRECTORY
|
||||
!insertmacro MUI_PAGE_COMPONENTS
|
||||
!insertmacro MUI_PAGE_INSTFILES
|
||||
!insertmacro MUI_PAGE_FINISH
|
||||
!insertmacro MUI_UNPAGE_CONFIRM
|
||||
!insertmacro MUI_UNPAGE_INSTFILES
|
||||
!insertmacro MUI_LANGUAGE "Korean"
|
||||
|
||||
; Whether jarvis.exe runs (any user's), from tasklist's answer: OUT = 1 or 0.
|
||||
!macro JarvisRuns PREFIX OUT
|
||||
nsExec::ExecToStack 'tasklist /FI "IMAGENAME eq jarvis.exe" /NH'
|
||||
Pop $0
|
||||
Call ${PREFIX}StrHasJarvis
|
||||
Pop ${OUT}
|
||||
!macroend
|
||||
|
||||
!macro StrHasJarvisFn PREFIX
|
||||
Function ${PREFIX}StrHasJarvis
|
||||
Exch $R0
|
||||
Push $R1
|
||||
Push $R2
|
||||
Push $R3
|
||||
StrCpy $R1 0
|
||||
StrCpy $R2 0
|
||||
loop:
|
||||
StrCpy $R3 $R0 10 $R1
|
||||
StrCmp $R3 "" done
|
||||
StrCmp $R3 "jarvis.exe" found
|
||||
IntOp $R1 $R1 + 1
|
||||
Goto loop
|
||||
found:
|
||||
StrCpy $R2 1
|
||||
done:
|
||||
StrCpy $R0 $R2
|
||||
Pop $R3
|
||||
Pop $R2
|
||||
Pop $R1
|
||||
Exch $R0
|
||||
FunctionEnd
|
||||
!macroend
|
||||
!insertmacro StrHasJarvisFn ""
|
||||
!insertmacro StrHasJarvisFn "un."
|
||||
|
||||
; Wait until Jarvis has closed: silently up to 60 s (an update closes it itself), else ask.
|
||||
!macro WaitClosed PREFIX
|
||||
StrCpy $5 0
|
||||
${Do}
|
||||
!insertmacro JarvisRuns "${PREFIX}" $4
|
||||
${If} $4 == 0
|
||||
${Break}
|
||||
${EndIf}
|
||||
${If} ${Silent}
|
||||
${If} $5 >= 60
|
||||
Abort
|
||||
${EndIf}
|
||||
Sleep 1000
|
||||
IntOp $5 $5 + 1
|
||||
${Else}
|
||||
MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Jarvis가 실행 중입니다. Jarvis 창을 닫은 뒤 다시 시도를 눌러 주세요." IDRETRY +2
|
||||
Abort
|
||||
${EndIf}
|
||||
${Loop}
|
||||
!macroend
|
||||
|
||||
; The driver's result file (why it failed) as text: the app writes it in UTF-8, which FileRead
|
||||
; would take for the system's ANSI code page. OUT is empty when there is no file.
|
||||
!macro ReadUtf8 FILE OUT
|
||||
StrCpy ${OUT} ""
|
||||
System::Call 'kernel32::CreateFileW(w "${FILE}", i 0x80000000, i 1, p 0, i 3, i 0, p 0) p .r7'
|
||||
${If} $7 P<> -1
|
||||
System::Alloc 1024 ; zeroed, and at most 1023 bytes read: always ends with a NUL
|
||||
Pop $6
|
||||
System::Call 'kernel32::ReadFile(p r7, p r6, i 1023, *i .r5, p 0)'
|
||||
System::Call 'kernel32::CloseHandle(p r7)'
|
||||
System::Call 'kernel32::MultiByteToWideChar(i 65001, i 0, p r6, i -1, w .s, i ${NSIS_MAX_STRLEN})'
|
||||
Pop ${OUT}
|
||||
System::Free $6
|
||||
${EndIf}
|
||||
!macroend
|
||||
|
||||
; The installer's lock: Jarvis does not start while it is held (app/src/setup_lock.rs); taken
|
||||
; before waiting for a running Jarvis, closed once every change is done (before the app is started
|
||||
; again). Another installer or uninstaller holding it (or a lock that cannot be made) means this
|
||||
; one stops at once: silently with exit code 5.
|
||||
!macro TakeLock
|
||||
System::Call 'kernel32::CreateMutexW(p 0, i 1, w "Global\JarvisBotSetup") p .r9 ?e'
|
||||
Pop $8
|
||||
${If} $9 == 0
|
||||
${OrIf} $8 == 183 ; ERROR_ALREADY_EXISTS
|
||||
${If} $9 != 0
|
||||
System::Call 'kernel32::CloseHandle(p r9)'
|
||||
${EndIf}
|
||||
${IfNot} ${Silent}
|
||||
MessageBox MB_ICONSTOP "다른 Jarvis 설치나 제거가 진행 중입니다. 끝난 뒤 다시 실행해 주세요."
|
||||
${EndIf}
|
||||
SetErrorLevel 5
|
||||
Abort
|
||||
${EndIf}
|
||||
!macroend
|
||||
|
||||
; Closing the only handle removes the lock (the sections run on another thread than .onInit, so
|
||||
; ReleaseMutex would not be allowed there; the handle count is what counts).
|
||||
!macro DropLock
|
||||
${If} $9 != 0
|
||||
System::Call 'kernel32::CloseHandle(p r9)'
|
||||
StrCpy $9 0
|
||||
${EndIf}
|
||||
!macroend
|
||||
|
||||
Function .onInit
|
||||
${IfNot} ${RunningX64}
|
||||
MessageBox MB_ICONSTOP "Jarvis는 64비트 Windows 10/11에서만 실행됩니다."
|
||||
Abort
|
||||
${EndIf}
|
||||
SetRegView 64
|
||||
; An update goes where Jarvis is installed (the key is in the 64-bit view, which InstallDirRegKey
|
||||
; would not read); a folder given with /D wins.
|
||||
ReadRegStr $0 HKLM "Software\JarvisBot" "InstallDir"
|
||||
${If} $0 != ""
|
||||
${AndIf} $INSTDIR == "$PROGRAMFILES64\Jarvis"
|
||||
StrCpy $INSTDIR $0
|
||||
${EndIf}
|
||||
; Shortcuts for every user of the PC, like the Program Files install itself.
|
||||
SetShellVarContext all
|
||||
!insertmacro TakeLock
|
||||
!insertmacro WaitClosed ""
|
||||
FunctionEnd
|
||||
|
||||
Function un.onInit
|
||||
SetRegView 64
|
||||
SetShellVarContext all
|
||||
!insertmacro TakeLock
|
||||
!insertmacro WaitClosed "un."
|
||||
FunctionEnd
|
||||
|
||||
; The window needs the WebView2 runtime (Windows 11 has it; an older Windows 10 may not). Checked
|
||||
; before any file is copied: without it nothing is installed (and /RUN starts nothing).
|
||||
Function WebView2Present
|
||||
ReadRegStr $0 HKLM "${WEBVIEW2_KEY}" "pv"
|
||||
ReadRegStr $1 HKCU "${WEBVIEW2_KEY}" "pv"
|
||||
StrCpy $2 0
|
||||
${If} $0 != ""
|
||||
${AndIf} $0 != "0.0.0.0"
|
||||
StrCpy $2 1
|
||||
${EndIf}
|
||||
${If} $1 != ""
|
||||
${AndIf} $1 != "0.0.0.0"
|
||||
StrCpy $2 1
|
||||
${EndIf}
|
||||
Push $2
|
||||
FunctionEnd
|
||||
|
||||
Function EnsureWebView2
|
||||
Call WebView2Present
|
||||
Pop $2
|
||||
${If} $2 == 1
|
||||
Return
|
||||
${EndIf}
|
||||
SetOutPath "$PLUGINSDIR"
|
||||
File "/oname=webview2.exe" "${WEBVIEW2}"
|
||||
${Do}
|
||||
DetailPrint "WebView2 런타임 설치"
|
||||
ExecWait '"$PLUGINSDIR\webview2.exe" /silent /install' $0
|
||||
Call WebView2Present
|
||||
Pop $2
|
||||
${If} $2 == 1
|
||||
Return
|
||||
${EndIf}
|
||||
DetailPrint "WebView2 런타임을 설치하지 못했습니다 (결과 $0)"
|
||||
${If} ${Silent}
|
||||
SetErrorLevel 3
|
||||
Abort "WebView2 런타임을 설치하지 못했습니다."
|
||||
${EndIf}
|
||||
MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Jarvis 창에 필요한 WebView2 런타임을 설치하지 못했습니다. 인터넷 연결을 확인하고 다시 시도해 주세요." IDRETRY +3
|
||||
SetErrorLevel 3
|
||||
Abort "WebView2 런타임이 없어 설치하지 않았습니다."
|
||||
${Loop}
|
||||
FunctionEnd
|
||||
|
||||
; Started from this elevated installer the app would run as administrator; Explorer starts it as
|
||||
; the signed-in user.
|
||||
Function RunAsUser
|
||||
Exec '"$WINDIR\explorer.exe" "$INSTDIR\jarvis.exe"'
|
||||
FunctionEnd
|
||||
|
||||
Section "Jarvis (필수)" SecMain
|
||||
SectionIn RO
|
||||
; Again right before changing anything: a Jarvis started since the first look closes first.
|
||||
!insertmacro WaitClosed ""
|
||||
Call EnsureWebView2
|
||||
SetOutPath "$INSTDIR"
|
||||
File "${SRC}/jarvis.exe"
|
||||
File "${SRC}/WebView2Loader.dll"
|
||||
; The shortcuts' icon, named by its content: Windows keeps an icon it has shown by the file's path,
|
||||
; so a changed icon under the old path would stay the old one on the desktop and in search.
|
||||
Delete "$INSTDIR\jarvis-*.ico"
|
||||
File "/oname=${ICON_FILE}" "${ICON}"
|
||||
File /r "${SRC}/python-src"
|
||||
File /r "${SRC}/display-driver"
|
||||
WriteUninstaller "$INSTDIR\uninstall.exe"
|
||||
WriteRegStr HKLM "Software\JarvisBot" "InstallDir" "$INSTDIR"
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "DisplayName" "Jarvis"
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "DisplayVersion" "${VERSION}"
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "Publisher" "Jarvis"
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "DisplayIcon" "$INSTDIR\jarvis.exe"
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "InstallLocation" "$INSTDIR"
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "UninstallString" '"$INSTDIR\uninstall.exe"'
|
||||
WriteRegStr HKLM "${UNINST_KEY}" "QuietUninstallString" '"$INSTDIR\uninstall.exe" /S'
|
||||
WriteRegDWORD HKLM "${UNINST_KEY}" "NoModify" 1
|
||||
WriteRegDWORD HKLM "${UNINST_KEY}" "NoRepair" 1
|
||||
WriteRegDWORD HKLM "${UNINST_KEY}" "EstimatedSize" ${SIZE_KB}
|
||||
SectionEnd
|
||||
|
||||
Section "시작 메뉴 바로 가기" SecStart
|
||||
CreateShortCut "$SMPROGRAMS\Jarvis.lnk" "$INSTDIR\jarvis.exe" "" "$INSTDIR\${ICON_FILE}" 0
|
||||
SectionEnd
|
||||
|
||||
Section /o "바탕 화면 바로 가기" SecDesktop
|
||||
CreateShortCut "$DESKTOP\Jarvis.lnk" "$INSTDIR\jarvis.exe" "" "$INSTDIR\${ICON_FILE}" 0
|
||||
SectionEnd
|
||||
|
||||
; Every shortcut to jarvis.exe on the desktops and in the start menus, their folders included (the
|
||||
; person's own too, and the installer's from an earlier version, which an update does not make
|
||||
; again) shows this version's icon file; then Explorer redraws its icons. A shortcut to another
|
||||
; Jarvis install's jarvis.exe (a folder with uninstall.exe and python-src, left behind when an
|
||||
; earlier update installed into Program Files instead) starts this one instead, so it opens the
|
||||
; Jarvis that runs, not an old copy. The paths go in through
|
||||
; the environment, so no install folder name can break the command. The 64-bit PowerShell
|
||||
; (SysNative from this 32-bit installer): a 32-bit one reads a link to Program Files as one to
|
||||
; Program Files (x86).
|
||||
Section "-Icons"
|
||||
System::Call 'kernel32::SetEnvironmentVariable(t "JARVIS_EXE", t "$INSTDIR\jarvis.exe")'
|
||||
System::Call 'kernel32::SetEnvironmentVariable(t "JARVIS_ICON", t "$INSTDIR\${ICON_FILE},0")'
|
||||
System::Call 'kernel32::SetEnvironmentVariable(t "JARVIS_DIR", t "$INSTDIR")'
|
||||
nsExec::Exec `"$WINDIR\SysNative\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -NonInteractive -Command "$$w = New-Object -ComObject WScript.Shell; foreach ($$d in 'Desktop', 'CommonDesktopDirectory', 'Programs', 'CommonPrograms') { Get-ChildItem -LiteralPath ([Environment]::GetFolderPath($$d)) -Filter *.lnk -Recurse:($$d -like '*Programs') -EA 0 | ForEach-Object { $$l = $$w.CreateShortcut($$_.FullName); $$t = $$l.TargetPath; if ($$t -and (Split-Path -Leaf $$t) -ieq 'jarvis.exe' -and (Test-Path -LiteralPath (Join-Path (Split-Path $$t) 'uninstall.exe')) -and (Test-Path -LiteralPath (Join-Path (Split-Path $$t) 'python-src'))) { if ($$t -ine $$env:JARVIS_EXE) { $$l.TargetPath = $$env:JARVIS_EXE; $$l.WorkingDirectory = $$env:JARVIS_DIR }; $$l.IconLocation = $$env:JARVIS_ICON; $$l.Save() } } }"`
|
||||
Pop $0
|
||||
${If} $0 != 0
|
||||
DetailPrint "바로 가기 아이콘을 바꾸지 못했습니다 ($0)"
|
||||
${EndIf}
|
||||
System::Call 'shell32::SHChangeNotify(i 0x08000000, i 0, p 0, p 0)' ; SHCNE_ASSOCCHANGED
|
||||
SectionEnd
|
||||
|
||||
Section "가상 모니터 드라이버" SecDriver
|
||||
; The app's own elevated half (admin.spec.md, "Virtual monitor"); Windows asks once whether to
|
||||
; trust the SignPath Foundation. It can also be installed later from the features screen. Not in
|
||||
; a silent install (the app's updates): an update never asks Windows' question and never brings
|
||||
; back a driver the person removed.
|
||||
${If} ${Silent}
|
||||
Return
|
||||
${EndIf}
|
||||
ExecWait '"$INSTDIR\jarvis.exe" --display-driver install --result "$PLUGINSDIR\vdd.txt"' $0
|
||||
${If} $0 != 0
|
||||
!insertmacro ReadUtf8 "$PLUGINSDIR\vdd.txt" $2
|
||||
DetailPrint "가상 모니터 드라이버를 설치하지 못했습니다: $2"
|
||||
${IfNot} ${Silent}
|
||||
MessageBox MB_ICONEXCLAMATION "가상 모니터 드라이버를 설치하지 못했습니다. 나중에 기능 화면에서 다시 설치할 수 있습니다.$\r$\n$2"
|
||||
${EndIf}
|
||||
${EndIf}
|
||||
SectionEnd
|
||||
|
||||
; Last: every change is done, so the lock goes before the app starts again (silent updates with
|
||||
; /RUN here, the finish page's "Jarvis 실행" after this).
|
||||
Section "-Done"
|
||||
!insertmacro DropLock
|
||||
${GetParameters} $0
|
||||
ClearErrors
|
||||
${GetOptions} $0 "/RUN" $1
|
||||
${IfNot} ${Errors}
|
||||
Call RunAsUser
|
||||
${EndIf}
|
||||
SectionEnd
|
||||
|
||||
LangString DESC_Main ${LANG_KOREAN} "관리 앱과 음성 워커·두뇌 소스입니다. 모델과 런타임은 첫 실행 때 고른 데이터 폴더에 받습니다."
|
||||
LangString DESC_Driver ${LANG_KOREAN} "봇용 크롬창을 실제 모니터에 보이지 않는 가상 모니터에 두어 디스코드 화면 공유에서 고를 수 있게 합니다. Windows가 장치 소프트웨어 설치를 한 번 묻습니다."
|
||||
!insertmacro MUI_FUNCTION_DESCRIPTION_BEGIN
|
||||
!insertmacro MUI_DESCRIPTION_TEXT ${SecMain} $(DESC_Main)
|
||||
!insertmacro MUI_DESCRIPTION_TEXT ${SecDriver} $(DESC_Driver)
|
||||
!insertmacro MUI_FUNCTION_DESCRIPTION_END
|
||||
|
||||
Section "Uninstall"
|
||||
!insertmacro WaitClosed "un."
|
||||
; The virtual monitor's driver goes with the app when this app installed it (code 0). Code 2:
|
||||
; nothing of this app's to remove (none, or another program's copy). Anything else: it stays,
|
||||
; so the uninstall stops here and keeps the app, which can remove it or try again.
|
||||
ExecWait '"$INSTDIR\jarvis.exe" --display-driver uninstall --result "$TEMP\jarvis-vdd-uninstall.txt"' $0
|
||||
${If} $0 != 0
|
||||
${AndIf} $0 != 2
|
||||
!insertmacro ReadUtf8 "$TEMP\jarvis-vdd-uninstall.txt" $2
|
||||
Delete "$TEMP\jarvis-vdd-uninstall.txt"
|
||||
DetailPrint "가상 모니터 드라이버를 지우지 못했습니다 ($0): $2"
|
||||
${IfNot} ${Silent}
|
||||
MessageBox MB_ICONSTOP "가상 모니터 드라이버를 지우지 못해 제거를 멈췄습니다. Jarvis의 기능 화면에서 드라이버를 제거한 뒤 다시 시도해 주세요.$\r$\n$2"
|
||||
${EndIf}
|
||||
SetErrorLevel 4
|
||||
Abort "가상 모니터 드라이버를 지우지 못했습니다."
|
||||
${EndIf}
|
||||
Delete "$TEMP\jarvis-vdd-uninstall.txt"
|
||||
; Only the files the installer put there (the list is made from the package at build time): a
|
||||
; folder the person chose may hold other things; folders go only when empty.
|
||||
Delete "$INSTDIR\jarvis.exe"
|
||||
Delete "$INSTDIR\WebView2Loader.dll"
|
||||
Delete "$INSTDIR\jarvis-*.ico"
|
||||
!include "${UNINSTALL_LIST}"
|
||||
Delete "$INSTDIR\uninstall.exe"
|
||||
RMDir "$INSTDIR"
|
||||
Delete "$SMPROGRAMS\Jarvis.lnk"
|
||||
Delete "$DESKTOP\Jarvis.lnk"
|
||||
DeleteRegKey HKLM "${UNINST_KEY}"
|
||||
DeleteRegKey HKLM "Software\JarvisBot"
|
||||
DetailPrint "데이터 폴더(모델, 대화 기록, 설정)는 지우지 않았습니다."
|
||||
!insertmacro DropLock
|
||||
SectionEnd
|
||||
28
app/shell/installer.sh
Executable file
28
app/shell/installer.sh
Executable file
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# The Jarvis installer (admin.spec.md, "Installer"): package.sh's Jarvis/ folder into one NSIS setup.
|
||||
# app/shell/installer.sh <output folder> -> <output folder>/Jarvis-Setup-<version>.exe (+ .sha256)
|
||||
# Needs makensis 3 (NSISDIR set when it is not installed system-wide) and curl.
|
||||
set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
out=$(mkdir -p "${1:?output folder}" && cd "$1" && pwd)
|
||||
version=$(sed -n 's/^version = "\(.*\)"/\1/p' "$here/Cargo.toml" | head -1)
|
||||
target=${CARGO_TARGET_DIR:-$here/target}
|
||||
sh "$here/package.sh" "$out" >/dev/null
|
||||
# The WebView2 runtime's bootstrapper (Microsoft, about 2 MB), run only on a PC without the runtime.
|
||||
wv2="$target/MicrosoftEdgeWebview2Setup.exe"
|
||||
[ -s "$wv2" ] || curl -fsSL -o "$wv2" "https://go.microsoft.com/fwlink/p/?LinkId=2124703"
|
||||
# What the uninstaller removes: exactly the files installed under python-src and display-driver,
|
||||
# then their folders, deepest first, each only when empty.
|
||||
list="$target/uninstall-list.nsh"
|
||||
(cd "$out/Jarvis" && find python-src display-driver -type f | sort | sed 's|/|\\|g; s|^| Delete "$INSTDIR\\|; s|$|"|'
|
||||
cd "$out/Jarvis" && find python-src display-driver -type d | sort -r | sed 's|/|\\|g; s|^| RMDir "$INSTDIR\\|; s|$|"|') > "$list"
|
||||
setup="$out/Jarvis-Setup-$version.exe"
|
||||
# The uninstall entry's size: what this setup installs, counted here. Counting the install folder on
|
||||
# the user's PC would also walk a data folder kept inside it (models, runtime: many thousand files).
|
||||
size_kb=$(cd "$out/Jarvis" && du -sk --apparent-size -c jarvis.exe WebView2Loader.dll python-src display-driver | tail -1 | cut -f1)
|
||||
# The shortcuts' icon file is named by its content, so a new icon gets a path Windows has not seen.
|
||||
icon_file="jarvis-$(sha256sum "$here/icons/icon.ico" | cut -c1-12).ico"
|
||||
makensis -V2 -DVERSION="$version" -DSRC="$out/Jarvis" -DOUT="$setup" -DICON="$here/icons/icon.ico" -DICON_FILE="$icon_file" \
|
||||
-DWEBVIEW2="$wv2" -DUNINSTALL_LIST="$list" -DSIZE_KB="$size_kb" ${NSIS_EXTRA:-} "$here/installer.nsi"
|
||||
(cd "$out" && sha256sum "Jarvis-Setup-$version.exe" > "Jarvis-Setup-$version.exe.sha256")
|
||||
echo "$setup"
|
||||
33
app/shell/package.sh
Executable file
33
app/shell/package.sh
Executable file
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
# Build the Windows management app from Linux and lay it out ready to zip (admin.spec.md, "Windows build"):
|
||||
# Jarvis/jarvis.exe, Jarvis/WebView2Loader.dll, Jarvis/python-src/{voice_worker,jarvis},
|
||||
# Jarvis/display-driver/ (Virtual Display Driver 25.7.23, MIT, SignPath-signed; "Virtual monitor")
|
||||
# Needs the x86_64-pc-windows-gnu Rust target, mingw-w64, cmake (for Opus), curl and unzip.
|
||||
# app/shell/package.sh <output folder>
|
||||
set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
out=${1:?output folder}
|
||||
target=${CARGO_TARGET_DIR:-$here/target}
|
||||
cargo build --release --locked --target x86_64-pc-windows-gnu --manifest-path "$here/Cargo.toml"
|
||||
rel="$target/x86_64-pc-windows-gnu/release"
|
||||
rm -rf "$out/Jarvis"
|
||||
mkdir -p "$out/Jarvis/python-src"
|
||||
cp "$rel/jarvis.exe" "$rel/WebView2Loader.dll" "$out/Jarvis/"
|
||||
# The worker and brain sources only (the worker venv gets its packages from the requirement files).
|
||||
for pkg in voice_worker jarvis; do
|
||||
(cd "$here/../../src" && find "$pkg" -name __pycache__ -prune -o -name '*.spec.md' -prune -o -type f -print) |
|
||||
while read -r f; do mkdir -p "$out/Jarvis/python-src/$(dirname "$f")"; cp "$here/../../src/$f" "$out/Jarvis/python-src/$f"; done
|
||||
done
|
||||
# The virtual monitor's driver, pinned by hash; its sample settings file is not used (the app
|
||||
# writes its own).
|
||||
vdd_url=https://github.com/VirtualDrivers/Virtual-Display-Driver/releases/download/25.7.23/VirtualDisplayDriver-x86.Driver.Only.zip
|
||||
vdd_sha=e24210692b442b39af763536330ce78b423f19342b7a7792c26de3944e418b3a
|
||||
vdd_zip="$target/vdd-25.7.23.zip"
|
||||
[ -f "$vdd_zip" ] || curl -fsSL -o "$vdd_zip" "$vdd_url"
|
||||
echo "$vdd_sha $vdd_zip" | sha256sum -c - >/dev/null || { rm -f "$vdd_zip"; echo "display driver hash mismatch" >&2; exit 1; }
|
||||
tmp=$(mktemp -d "$target/vdd.XXXXXX")
|
||||
unzip -q "$vdd_zip" -d "$tmp"
|
||||
mkdir -p "$out/Jarvis/display-driver"
|
||||
cp "$tmp/VirtualDisplayDriver/MttVDD.inf" "$tmp/VirtualDisplayDriver/mttvdd.cat" "$tmp/VirtualDisplayDriver/MttVDD.dll" "$out/Jarvis/display-driver/"
|
||||
rm -rf "$tmp"
|
||||
echo "$out/Jarvis"
|
||||
175
app/shell/src/main.rs
Normal file
175
app/shell/src/main.rs
Normal file
@@ -0,0 +1,175 @@
|
||||
//! Jarvis management app: one Tauri 2 window over the app core (`app/src/admin.spec.md`, "Window
|
||||
//! and Windows build"). The screens (`app/ui`) call `invoke("admin", {command, args})`, passed to the
|
||||
//! same `AdminApi` the development server uses. Quitting first stops what the app started; if that
|
||||
//! cannot be confirmed the window stays open and says so, and quitting again quits. The close button
|
||||
//! quits, or hides the window to the tray when `app.closeButton` is "tray"; the tray icon opens it
|
||||
//! again or quits. Starting Jarvis again while it runs shows the running one's window.
|
||||
//!
|
||||
//! jarvis [--location FILE]
|
||||
//! jarvis --display-driver install|uninstall --result FILE (elevated, started by the app)
|
||||
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use jarvis_app::admin_api::AdminApi;
|
||||
use serde_json::Value;
|
||||
use tauri::menu::{Menu, MenuItem};
|
||||
use tauri::tray::{MouseButton, MouseButtonState, TrayIconBuilder, TrayIconEvent};
|
||||
use tauri::{AppHandle, Emitter, Manager, RunEvent, WebviewUrl, WebviewWindowBuilder, WindowEvent};
|
||||
|
||||
/// Where closing stands: open, closing (stopping what runs), or the stop could not be confirmed.
|
||||
const OPEN: u8 = 0;
|
||||
const CLOSING: u8 = 1;
|
||||
const CLOSE_FAILED: u8 = 2;
|
||||
|
||||
struct Api {
|
||||
api: Arc<AdminApi>,
|
||||
phase: AtomicU8,
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
async fn admin(app: tauri::AppHandle, state: tauri::State<'_, Api>, command: String, args: Option<Value>) -> Result<Value, String> {
|
||||
let out = state.api.call(&command, args.unwrap_or(Value::Null)).await?;
|
||||
// An update's installer was started (what ran is stopped): quit so it can replace the app.
|
||||
if out["exit"] == Value::Bool(true) {
|
||||
state.phase.store(CLOSING, Ordering::SeqCst);
|
||||
app.exit(0);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Bring the window back: from the tray, or when Jarvis is started again.
|
||||
fn show_window(app: &AppHandle) {
|
||||
if let Some(w) = app.get_webview_window("main") {
|
||||
let _ = w.unminimize();
|
||||
let _ = w.show();
|
||||
let _ = w.set_focus();
|
||||
}
|
||||
}
|
||||
|
||||
/// Quit (the close button set to quit, or the tray's "완전 종료"): stop what the app started, then
|
||||
/// exit; the window shows while that runs. If the stop fails the window stays with the reason, and
|
||||
/// quitting again exits anyway.
|
||||
fn quit(app: &AppHandle) {
|
||||
let state = app.state::<Api>();
|
||||
match state.phase.load(Ordering::SeqCst) {
|
||||
CLOSE_FAILED => {
|
||||
jarvis_app::warn_log!("app", "quitting again: quitting although the stop was not confirmed");
|
||||
app.exit(0);
|
||||
}
|
||||
CLOSING => {}
|
||||
_ => {
|
||||
state.phase.store(CLOSING, Ordering::SeqCst);
|
||||
show_window(app);
|
||||
let _ = app.emit("closing", ());
|
||||
let (handle, api) = (app.clone(), state.api.clone());
|
||||
tauri::async_runtime::spawn(async move {
|
||||
jarvis_app::info_log!("app", "quitting: stopping what the app started");
|
||||
match api.shutdown().await {
|
||||
Ok(()) => handle.exit(0),
|
||||
Err(e) => {
|
||||
handle.state::<Api>().phase.store(CLOSE_FAILED, Ordering::SeqCst);
|
||||
let _ = handle.emit("close-failed", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn arg(name: &str) -> Option<String> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
args.iter().position(|a| a == name).and_then(|i| args.get(i + 1).cloned())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// The elevated half of installing or removing the virtual monitor's driver: no window.
|
||||
if let Some(code) = jarvis_app::vdisplay::elevated_main(&std::env::args().collect::<Vec<_>>()) {
|
||||
std::process::exit(code);
|
||||
}
|
||||
// Not while an installer installs, updates or removes Jarvis (admin.spec.md, "Installer").
|
||||
if jarvis_app::setup_lock::held() {
|
||||
jarvis_app::setup_lock::tell();
|
||||
std::process::exit(3);
|
||||
}
|
||||
let location = arg("--location").map(PathBuf::from).or_else(jarvis_app::datadir::default_location_file)
|
||||
.expect("no location file path: pass --location");
|
||||
// Log files go to the data folder once there is one; until then memory only.
|
||||
let logs = jarvis_app::datadir::DataDir::locate(&location).map(|d| d.logs());
|
||||
let hub = jarvis_app::logs::LogHub::new(logs.as_deref());
|
||||
jarvis_app::logs::install(hub.clone());
|
||||
let api = Arc::new(AdminApi::new(&location, hub));
|
||||
// The window's browser data: in the data folder, or a scratch folder in the system temp folder
|
||||
// before there is one. WebView2 holds that folder until this process has ended, so scratch
|
||||
// folders of earlier runs are removed here, at the next start.
|
||||
jarvis_app::datadir::remove_scratch_webview(&std::env::temp_dir(), None);
|
||||
let (webview_data, _) = jarvis_app::datadir::webview_data_dir(&location);
|
||||
|
||||
let app = tauri::Builder::default()
|
||||
// First, so a second start only shows this one's window and exits.
|
||||
.plugin(tauri_plugin_single_instance::init(|app, _, _| show_window(app)))
|
||||
.manage(Api { api, phase: AtomicU8::new(OPEN) })
|
||||
.invoke_handler(tauri::generate_handler![admin])
|
||||
.setup({
|
||||
move |app| {
|
||||
WebviewWindowBuilder::new(app, "main", WebviewUrl::App("index.html".into()))
|
||||
.title("Jarvis 관리")
|
||||
.inner_size(1200.0, 820.0)
|
||||
.min_inner_size(900.0, 600.0)
|
||||
.data_directory(webview_data)
|
||||
.build()?;
|
||||
let open = MenuItem::with_id(app, "open", "열기", true, None::<&str>)?;
|
||||
let quit_item = MenuItem::with_id(app, "quit", "완전 종료", true, None::<&str>)?;
|
||||
TrayIconBuilder::with_id("main")
|
||||
.tooltip("Jarvis")
|
||||
.menu(&Menu::with_items(app, &[&open, &quit_item])?)
|
||||
.show_menu_on_left_click(false)
|
||||
.on_menu_event(|app, e| match e.id.as_ref() {
|
||||
"open" => show_window(app),
|
||||
"quit" => quit(app),
|
||||
_ => {}
|
||||
})
|
||||
.on_tray_icon_event(|tray, e| {
|
||||
if let TrayIconEvent::Click { button: MouseButton::Left, button_state: MouseButtonState::Up, .. } = e {
|
||||
show_window(tray.app_handle());
|
||||
}
|
||||
})
|
||||
.icon(tauri::image::Image::from_bytes(include_bytes!("../icons/tray.png"))?)
|
||||
.build(app)?;
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.on_window_event(|window, event| {
|
||||
let WindowEvent::CloseRequested { api: close, .. } = event else { return };
|
||||
let state = window.state::<Api>();
|
||||
match state.phase.load(Ordering::SeqCst) {
|
||||
// After a failed stop the window closes, and with it the app.
|
||||
CLOSE_FAILED => jarvis_app::warn_log!("app", "window closed again: quitting although the stop was not confirmed"),
|
||||
CLOSING => close.prevent_close(),
|
||||
_ if state.api.close_to_tray() => {
|
||||
close.prevent_close();
|
||||
let _ = window.hide();
|
||||
jarvis_app::info_log!("app", "window hidden to the tray");
|
||||
}
|
||||
_ => {
|
||||
close.prevent_close();
|
||||
quit(window.app_handle());
|
||||
}
|
||||
}
|
||||
})
|
||||
.build(tauri::generate_context!())
|
||||
.expect("the window did not start");
|
||||
app.run(move |handle, event| {
|
||||
match event {
|
||||
// Quitting another way (the system shutting down) still stops what runs.
|
||||
RunEvent::ExitRequested { .. } if handle.state::<Api>().phase.load(Ordering::SeqCst) == OPEN => {
|
||||
let api = handle.state::<Api>().api.clone();
|
||||
let _ = tauri::async_runtime::block_on(api.shutdown());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
});
|
||||
}
|
||||
19
app/shell/tauri.conf.json
Normal file
19
app/shell/tauri.conf.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "Jarvis",
|
||||
"identifier": "kr.jarvis.manager",
|
||||
"build": {
|
||||
"frontendDist": "../ui"
|
||||
},
|
||||
"app": {
|
||||
"withGlobalTauri": true,
|
||||
"windows": [],
|
||||
"security": {
|
||||
"csp": "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src ipc: http://ipc.localhost"
|
||||
}
|
||||
},
|
||||
"bundle": {
|
||||
"active": false,
|
||||
"icon": ["icons/icon.ico", "icons/icon.png"]
|
||||
}
|
||||
}
|
||||
205
app/src/activity.rs
Normal file
205
app/src/activity.rs
Normal file
@@ -0,0 +1,205 @@
|
||||
//! What Jarvis did outside its answers (`app/src/admin.spec.md`, "Activity"; `docs/v2/proactive.md`
|
||||
//! 제안 5): chat channel actions and schedule deliveries, by kind, target, who asked and outcome,
|
||||
//! never their text. Day files `activity/<YYYY-MM-DD>.jsonl` in the data folder, kept `KEEP_DAYS`
|
||||
//! (old ones deleted at the start and on the first record of each new day).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use chrono::{Local, NaiveDate};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const KEEP_DAYS: i64 = 30;
|
||||
|
||||
static FILE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Entry {
|
||||
/// Local time, RFC 3339.
|
||||
pub at: String,
|
||||
/// What was done: `chat.send`, `chat.edit`, `chat.react`, `chat.delete`, `chat.pin`,
|
||||
/// `chat.unpin`, `schedule.remind`, `schedule.task`.
|
||||
pub kind: String,
|
||||
/// What it was done to: a channel or message id, or the schedule's id and where it went.
|
||||
pub target: String,
|
||||
/// Who asked (Discord user id and name), when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub by: Option<String>,
|
||||
pub ok: bool,
|
||||
/// Why it failed (never content).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
pub struct Activity {
|
||||
dir: PathBuf,
|
||||
/// The day old files were last deleted; a record on a later day deletes again.
|
||||
pruned: Mutex<NaiveDate>,
|
||||
}
|
||||
|
||||
/// Recorded instead of an id that is not one (the model may put any text there).
|
||||
pub const INVALID_ID: &str = "(잘못된 ID)";
|
||||
/// Recorded instead of a chat action's error (it may quote what the model wrote).
|
||||
pub const CHAT_FAILED: &str = "채팅 동작 실패";
|
||||
const CHAT_ACTIONS: &[&str] = &["send", "edit", "react", "delete", "pin", "unpin"];
|
||||
|
||||
/// `s` when it is a Discord id (digits only), else `INVALID_ID`.
|
||||
pub fn id_or_mark(s: &str) -> String {
|
||||
if (1..=20).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_digit()) { s.to_owned() } else { INVALID_ID.into() }
|
||||
}
|
||||
|
||||
/// The kind of a chat action: `chat.<action>` for the known ones, else `chat.other`.
|
||||
pub fn chat_kind(action: &str) -> String {
|
||||
format!("chat.{}", if CHAT_ACTIONS.contains(&action) { action } else { "other" })
|
||||
}
|
||||
|
||||
/// The target of a chat action, from ids only.
|
||||
pub fn chat_target(channel: &str, message: Option<&str>) -> String {
|
||||
match message {
|
||||
Some(m) => format!("채널 {} 메시지 {}", id_or_mark(channel), id_or_mark(m)),
|
||||
None => format!("채널 {}", id_or_mark(channel)),
|
||||
}
|
||||
}
|
||||
|
||||
fn day_name(d: NaiveDate) -> String {
|
||||
format!("{d}.jsonl")
|
||||
}
|
||||
|
||||
impl Activity {
|
||||
/// The log in `data`; day files older than `KEEP_DAYS` are deleted.
|
||||
pub fn new(data: &Path) -> Self {
|
||||
let today = Local::now().date_naive();
|
||||
let a = Self { dir: data.join("activity"), pruned: Mutex::new(today) };
|
||||
a.prune(today);
|
||||
a
|
||||
}
|
||||
|
||||
fn days(&self) -> Vec<(NaiveDate, PathBuf)> {
|
||||
let mut out: Vec<(NaiveDate, PathBuf)> = std::fs::read_dir(&self.dir).into_iter().flatten().flatten()
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name().to_str()?.to_owned();
|
||||
let d = NaiveDate::parse_from_str(name.strip_suffix(".jsonl")?, "%Y-%m-%d").ok()?;
|
||||
Some((d, e.path()))
|
||||
})
|
||||
.collect();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
fn prune(&self, today: NaiveDate) {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for (d, f) in self.days() {
|
||||
if (today - d).num_days() > KEEP_DAYS {
|
||||
let _ = std::fs::remove_file(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append one entry (a failure to write is logged, never raised: the action itself happened).
|
||||
pub fn record(&self, kind: &str, target: &str, by: Option<String>, ok: bool, note: Option<String>) {
|
||||
let now = Local::now();
|
||||
let today = now.date_naive();
|
||||
let rolled = {
|
||||
let mut p = self.pruned.lock().unwrap_or_else(|e| e.into_inner());
|
||||
std::mem::replace(&mut *p, today) != today
|
||||
};
|
||||
if rolled {
|
||||
self.prune(today);
|
||||
}
|
||||
let e = Entry { at: now.to_rfc3339_opts(chrono::SecondsFormat::Secs, false), kind: kind.into(),
|
||||
target: target.into(), by, ok, note };
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let file = self.dir.join(day_name(now.date_naive()));
|
||||
let line = serde_json::to_string(&e).unwrap() + "\n";
|
||||
let wrote = std::fs::create_dir_all(&self.dir).and_then(|_| {
|
||||
use std::io::Write;
|
||||
std::fs::OpenOptions::new().create(true).append(true).open(&file)?.write_all(line.as_bytes())
|
||||
});
|
||||
match wrote {
|
||||
Ok(()) => {
|
||||
crate::debug_log!("conversation", "activity {kind} ok={ok}");
|
||||
}
|
||||
Err(err) => {
|
||||
crate::warn_log!("conversation", "Warning: could not record activity {kind}: {err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The newest `limit` entries, newest first.
|
||||
pub fn recent(&self, limit: usize) -> Vec<Entry> {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut out = vec![];
|
||||
for (_, f) in self.days().into_iter().rev() {
|
||||
let text = std::fs::read_to_string(&f).unwrap_or_default();
|
||||
for line in text.lines().rev() {
|
||||
if let Ok(e) = serde_json::from_str::<Entry>(line) {
|
||||
out.push(e);
|
||||
if out.len() == limit {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn entries_come_back_newest_first_up_to_the_limit() {
|
||||
let data = crate::datadir::temp_dir("activity");
|
||||
let a = Activity::new(&data);
|
||||
a.record("chat.send", "123", Some("1 가".into()), true, None);
|
||||
a.record("schedule.remind", "7 → 음성", None, false, Some("음성 채널에 없음".into()));
|
||||
let got = a.recent(10);
|
||||
assert_eq!(got.iter().map(|e| e.kind.as_str()).collect::<Vec<_>>(), ["schedule.remind", "chat.send"]);
|
||||
assert_eq!(got[1].by.as_deref(), Some("1 가"));
|
||||
assert!(!got[0].ok && got[0].note.is_some());
|
||||
assert_eq!(a.recent(1).len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chat_action_is_recorded_by_ids_and_known_words_only() {
|
||||
// What the model put in the fields never reaches the log, valid or not.
|
||||
assert_eq!(chat_kind("edit"), "chat.edit");
|
||||
assert_eq!(chat_kind("비밀 회의"), "chat.other");
|
||||
assert_eq!(chat_target("223456789012345678", Some("323456789012345678")),
|
||||
"채널 223456789012345678 메시지 323456789012345678");
|
||||
let bad = chat_target("일정 채널", Some("비밀 회의"));
|
||||
assert!(!bad.contains("비밀") && !bad.contains("일정"), "{bad}");
|
||||
assert_eq!(chat_target("223456789012345678", None), "채널 223456789012345678");
|
||||
assert_eq!(id_or_mark("12a"), INVALID_ID);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_run_still_deletes_old_day_files_when_the_day_changes() {
|
||||
let data = crate::datadir::temp_dir("activity-rollover");
|
||||
let a = Activity::new(&data);
|
||||
let old = Local::now().date_naive() - chrono::Duration::days(KEEP_DAYS + 1);
|
||||
std::fs::create_dir_all(data.join("activity")).unwrap();
|
||||
std::fs::write(data.join("activity").join(day_name(old)), "{}\n").unwrap();
|
||||
// The app has been running since yesterday.
|
||||
*a.pruned.lock().unwrap() = Local::now().date_naive() - chrono::Duration::days(1);
|
||||
a.record("chat.send", "채널 1", None, true, None);
|
||||
assert!(!data.join("activity").join(day_name(old)).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn day_files_past_the_keep_days_are_deleted() {
|
||||
let data = crate::datadir::temp_dir("activity-prune");
|
||||
let dir = data.join("activity");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let today = Local::now().date_naive();
|
||||
let old = today - chrono::Duration::days(KEEP_DAYS + 1);
|
||||
let kept = today - chrono::Duration::days(KEEP_DAYS);
|
||||
let line = r#"{"at":"x","kind":"chat.send","target":"1","ok":true}"#;
|
||||
std::fs::write(dir.join(day_name(old)), line).unwrap();
|
||||
std::fs::write(dir.join(day_name(kept)), line).unwrap();
|
||||
let a = Activity::new(&data);
|
||||
assert!(!dir.join(day_name(old)).exists());
|
||||
assert_eq!(a.recent(10).len(), 1);
|
||||
}
|
||||
}
|
||||
1086
app/src/admin.spec.md
Normal file
1086
app/src/admin.spec.md
Normal file
File diff suppressed because it is too large
Load Diff
1051
app/src/admin_api.rs
Normal file
1051
app/src/admin_api.rs
Normal file
File diff suppressed because it is too large
Load Diff
105
app/src/bin/jarvis-admin-dev.rs
Normal file
105
app/src/bin/jarvis-admin-dev.rs
Normal file
@@ -0,0 +1,105 @@
|
||||
//! Development server for the management app's screens (`app/src/admin.spec.md`, "Screens"):
|
||||
//! serves `app/ui` and passes `POST /api/<command>` to the same `AdminApi` the Tauri shell uses,
|
||||
//! so the screens can be opened and tested in a browser. Loopback only, and every API call needs
|
||||
//! the random token printed at start (in the page URL's `#token=`).
|
||||
//!
|
||||
//! jarvis-admin-dev [--port N] [--ui DIR] [--location FILE]
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use jarvis_app::admin_api::AdminApi;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
fn arg(name: &str) -> Option<String> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
args.iter().position(|a| a == name).and_then(|i| args.get(i + 1).cloned())
|
||||
}
|
||||
|
||||
fn content_type(path: &Path) -> &'static str {
|
||||
match path.extension().and_then(|e| e.to_str()) {
|
||||
Some("html") => "text/html; charset=utf-8",
|
||||
Some("js") => "text/javascript; charset=utf-8",
|
||||
Some("css") => "text/css; charset=utf-8",
|
||||
Some("svg") => "image/svg+xml",
|
||||
Some("png") => "image/png",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
async fn respond(s: &mut TcpStream, status: &str, ctype: &str, body: &[u8]) {
|
||||
let head = format!("HTTP/1.1 {status}\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n", body.len());
|
||||
let _ = s.write_all(head.as_bytes()).await;
|
||||
let _ = s.write_all(body).await;
|
||||
}
|
||||
|
||||
async fn handle(mut s: TcpStream, api: Arc<AdminApi>, ui: Arc<PathBuf>, token: Arc<String>) {
|
||||
let mut buf = Vec::new();
|
||||
let mut chunk = [0u8; 8192];
|
||||
let head_end = loop {
|
||||
let Ok(n) = s.read(&mut chunk).await else { return };
|
||||
if n == 0 { return; }
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") { break i + 4; }
|
||||
if buf.len() > 64 * 1024 { return; }
|
||||
};
|
||||
let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
|
||||
let mut lines = head.lines();
|
||||
let mut first = lines.next().unwrap_or("").split_whitespace();
|
||||
let (method, path) = (first.next().unwrap_or(""), first.next().unwrap_or("/").to_owned());
|
||||
let header = |name: &str| head.lines().find_map(|l| l.split_once(':')
|
||||
.filter(|(k, _)| k.trim().eq_ignore_ascii_case(name)).map(|(_, v)| v.trim().to_owned()));
|
||||
let len: usize = header("content-length").and_then(|v| v.parse().ok()).unwrap_or(0).min(4 * 1024 * 1024);
|
||||
while buf.len() < head_end + len {
|
||||
let Ok(n) = s.read(&mut chunk).await else { return };
|
||||
if n == 0 { break; }
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
}
|
||||
if method == "POST" && path.starts_with("/api/") {
|
||||
if header("x-admin-token").as_deref() != Some(token.as_str()) {
|
||||
return respond(&mut s, "403 Forbidden", "text/plain", b"bad token").await;
|
||||
}
|
||||
let args: Value = serde_json::from_slice(&buf[head_end..(head_end + len).min(buf.len())]).unwrap_or(json!({}));
|
||||
let out = match api.call(&path[5..], args).await {
|
||||
Ok(v) => json!({"ok": true, "value": v}),
|
||||
Err(e) => json!({"ok": false, "error": e}),
|
||||
};
|
||||
return respond(&mut s, "200 OK", "application/json", out.to_string().as_bytes()).await;
|
||||
}
|
||||
if method != "GET" {
|
||||
return respond(&mut s, "405 Method Not Allowed", "text/plain", b"").await;
|
||||
}
|
||||
let rel = path.split(['?', '#']).next().unwrap_or("/").trim_start_matches('/');
|
||||
let rel = if rel.is_empty() { "index.html" } else { rel };
|
||||
if rel.split('/').any(|p| p == ".." || p.is_empty()) {
|
||||
return respond(&mut s, "404 Not Found", "text/plain", b"").await;
|
||||
}
|
||||
let file = ui.join(rel);
|
||||
match tokio::fs::read(&file).await {
|
||||
Ok(b) => respond(&mut s, "200 OK", content_type(&file), &b).await,
|
||||
Err(_) => respond(&mut s, "404 Not Found", "text/plain", b"").await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let ui = arg("--ui").map(PathBuf::from).unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("ui"));
|
||||
let location = arg("--location").map(PathBuf::from).or_else(jarvis_app::datadir::default_location_file)
|
||||
.expect("no location file path: pass --location");
|
||||
let port: u16 = arg("--port").and_then(|p| p.parse().ok()).unwrap_or(0);
|
||||
// Log files go to the data folder once there is one (from the next start after the wizard).
|
||||
let logs = jarvis_app::datadir::DataDir::locate(&location).map(|d| d.logs());
|
||||
let hub = jarvis_app::logs::LogHub::new(logs.as_deref());
|
||||
jarvis_app::logs::install(hub.clone());
|
||||
let api = Arc::new(AdminApi::new(&location, hub));
|
||||
let token: String = (0..16).map(|_| format!("{:02x}", rand::random::<u8>())).collect();
|
||||
let listener = TcpListener::bind(("127.0.0.1", port)).await.expect("bind");
|
||||
println!("http://127.0.0.1:{}/#token={token}", listener.local_addr().unwrap().port());
|
||||
let (ui, token) = (Arc::new(ui), Arc::new(token));
|
||||
loop {
|
||||
let Ok((s, _)) = listener.accept().await else { continue };
|
||||
tokio::spawn(handle(s, api.clone(), ui.clone(), token.clone()));
|
||||
}
|
||||
}
|
||||
646
app/src/browser.rs
Normal file
646
app/src/browser.rs
Normal file
@@ -0,0 +1,646 @@
|
||||
//! The bot browser (`app/src/admin.spec.md`, "Bot browser"): the bot's own Chrome in the data
|
||||
//! folder, driven over CDP, and the egress proxy every Chrome connection goes through so it can
|
||||
//! only reach public addresses.
|
||||
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const HEAD_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_HEAD: usize = 64 * 1024;
|
||||
const READY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const CLOSE_WAIT: Duration = Duration::from_secs(5);
|
||||
/// How often the windows the bot opens on the virtual monitor are looked for.
|
||||
const WATCH_EVERY: Duration = Duration::from_millis(200);
|
||||
|
||||
/// Whether an address is on the public internet: not this PC, the local network or a special range.
|
||||
pub fn is_public(ip: &IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
let o = v4.octets();
|
||||
!(v4.is_private() || v4.is_loopback() || v4.is_link_local() || v4.is_broadcast()
|
||||
|| v4.is_documentation() || v4.is_unspecified() || v4.is_multicast()
|
||||
|| o[0] == 0 // "this network"
|
||||
|| (o[0] == 100 && (64..128).contains(&o[1])) // CGNAT 100.64/10
|
||||
|| (o[0] == 192 && o[1] == 0 && o[2] == 0) // IETF 192.0.0/24
|
||||
|| (o[0] == 198 && (18..20).contains(&o[1])) // benchmarking 198.18/15
|
||||
|| o[0] >= 240) // reserved
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
if let Some(v4) = v6.to_ipv4_mapped() {
|
||||
return is_public(&IpAddr::V4(v4));
|
||||
}
|
||||
// Only global unicast 2000::/3 is routed on the internet (this also leaves out site
|
||||
// local fec0::/10, unique and link local, IPv4-compatible ::a.b.c.d and NAT64).
|
||||
let s = v6.segments();
|
||||
(s[0] & 0xe000) == 0x2000
|
||||
&& !(s[0] == 0x2001 && s[1] < 0x0200) // IETF 2001::/23 (Teredo, benchmarking, ...)
|
||||
&& !(s[0] == 0x2001 && s[1] == 0x0db8) // documentation
|
||||
&& s[0] != 0x2002 // 6to4 (embeds any IPv4)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- egress proxy ---------------------------------------------------------------------------------
|
||||
|
||||
/// A loopback HTTP proxy that connects only to addresses `allow` accepts (`is_public` for Chrome).
|
||||
pub struct EgressProxy {
|
||||
pub port: u16,
|
||||
shutdown: tokio::sync::watch::Sender<bool>,
|
||||
task: Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl EgressProxy {
|
||||
pub async fn start(allow: fn(&IpAddr) -> bool) -> std::io::Result<Self> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let port = listener.local_addr()?.port();
|
||||
let (shutdown, rx) = tokio::sync::watch::channel(false);
|
||||
let mut stopping = rx.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
// Every connection is a task of this set, so a stop ends the open tunnels too.
|
||||
let mut conns = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
r = listener.accept() => if let Ok((client, _)) = r {
|
||||
let mut stop = rx.clone();
|
||||
conns.spawn(async move {
|
||||
tokio::select! {
|
||||
_ = serve(client, allow) => {}
|
||||
_ = stop.wait_for(|s| *s) => {}
|
||||
}
|
||||
});
|
||||
},
|
||||
_ = stopping.wait_for(|s| *s) => break,
|
||||
Some(_) = conns.join_next(), if !conns.is_empty() => {}
|
||||
}
|
||||
}
|
||||
drop(listener);
|
||||
conns.shutdown().await;
|
||||
});
|
||||
Ok(Self { port, shutdown, task: Mutex::new(Some(task)) })
|
||||
}
|
||||
|
||||
/// Stop taking connections, close every open one and wait until they are gone.
|
||||
pub async fn stop(&self) {
|
||||
let _ = self.shutdown.send(true);
|
||||
let task = self.task.lock().unwrap().take();
|
||||
if let Some(t) = task {
|
||||
let _ = t.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EgressProxy {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.shutdown.send(true);
|
||||
}
|
||||
}
|
||||
|
||||
/// The request head (up to the blank line) and whatever came after it.
|
||||
async fn read_head(s: &mut TcpStream) -> std::io::Result<Option<(String, Vec<u8>)>> {
|
||||
let mut buf = Vec::new();
|
||||
let mut chunk = [0u8; 8192];
|
||||
loop {
|
||||
if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
|
||||
let rest = buf.split_off(end + 4);
|
||||
return Ok(String::from_utf8(buf).ok().map(|h| (h, rest)));
|
||||
}
|
||||
if buf.len() > MAX_HEAD {
|
||||
return Ok(None);
|
||||
}
|
||||
let n = s.read(&mut chunk).await?;
|
||||
if n == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
buf.extend_from_slice(&chunk[..n]);
|
||||
}
|
||||
}
|
||||
|
||||
/// `host:port` with an optional `[..]` around an IPv6 address.
|
||||
fn split_host_port(s: &str) -> Option<(String, u16)> {
|
||||
let (host, port) = s.rsplit_once(':')?;
|
||||
let host = host.strip_prefix('[').and_then(|h| h.strip_suffix(']')).unwrap_or(host);
|
||||
if host.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some((host.to_owned(), port.parse().ok()?))
|
||||
}
|
||||
|
||||
/// Resolve once, require every address to be allowed, and connect only to those.
|
||||
async fn connect_allowed(host: &str, port: u16, allow: fn(&IpAddr) -> bool) -> Result<TcpStream, &'static str> {
|
||||
let addrs: Vec<SocketAddr> = match host.parse::<IpAddr>() {
|
||||
Ok(ip) => vec![SocketAddr::new(ip, port)],
|
||||
Err(_) => match tokio::time::timeout(RESOLVE_TIMEOUT, tokio::net::lookup_host((host, port))).await {
|
||||
Ok(Ok(a)) => a.collect(),
|
||||
_ => return Err("name not resolved"),
|
||||
},
|
||||
};
|
||||
if addrs.is_empty() || !addrs.iter().all(|a| allow(&a.ip())) {
|
||||
return Err("not a public address");
|
||||
}
|
||||
for a in &addrs {
|
||||
if let Ok(Ok(s)) = tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(a)).await {
|
||||
return Ok(s);
|
||||
}
|
||||
}
|
||||
Err("could not connect")
|
||||
}
|
||||
|
||||
async fn reply(client: &mut TcpStream, status: &str) -> std::io::Result<()> {
|
||||
client.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await
|
||||
}
|
||||
|
||||
async fn serve(mut client: TcpStream, allow: fn(&IpAddr) -> bool) -> std::io::Result<()> {
|
||||
let Ok(Ok(Some((head, rest)))) = tokio::time::timeout(HEAD_TIMEOUT, read_head(&mut client)).await else {
|
||||
return reply(&mut client, "400 Bad Request").await;
|
||||
};
|
||||
let mut lines = head.split("\r\n");
|
||||
let mut first = lines.next().unwrap_or("").split(' ');
|
||||
let (method, target, version) = (first.next().unwrap_or(""), first.next().unwrap_or(""), first.next().unwrap_or(""));
|
||||
if method.eq_ignore_ascii_case("CONNECT") {
|
||||
let Some((host, port)) = split_host_port(target) else { return reply(&mut client, "400 Bad Request").await };
|
||||
let mut up = match connect_allowed(&host, port, allow).await {
|
||||
Ok(s) => s,
|
||||
Err(why) => return refused(&mut client, why).await,
|
||||
};
|
||||
client.write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n").await?;
|
||||
up.write_all(&rest).await?;
|
||||
tokio::io::copy_bidirectional(&mut client, &mut up).await?;
|
||||
return Ok(());
|
||||
}
|
||||
let Ok(url) = reqwest::Url::parse(target) else { return reply(&mut client, "400 Bad Request").await };
|
||||
let (Some(host), Some(port)) = (url.host_str(), url.port_or_known_default()) else {
|
||||
return reply(&mut client, "400 Bad Request").await;
|
||||
};
|
||||
if url.scheme() != "http" || !version.starts_with("HTTP/") {
|
||||
return reply(&mut client, "400 Bad Request").await;
|
||||
}
|
||||
let host = host.trim_start_matches('[').trim_end_matches(']').to_owned();
|
||||
let mut up = match connect_allowed(&host, port, allow).await {
|
||||
Ok(s) => s,
|
||||
Err(why) => return refused(&mut client, why).await,
|
||||
};
|
||||
// Origin form, no proxy headers, one request per connection (the next may be for another host).
|
||||
let path = match url.query() {
|
||||
Some(q) => format!("{}?{q}", url.path()),
|
||||
None => url.path().to_owned(),
|
||||
};
|
||||
let mut out = format!("{method} {path} {version}\r\n");
|
||||
for l in lines.filter(|l| !l.is_empty()) {
|
||||
let name = l.split(':').next().unwrap_or("").trim().to_ascii_lowercase();
|
||||
if !matches!(name.as_str(), "proxy-connection" | "proxy-authorization" | "connection" | "keep-alive") {
|
||||
out.push_str(l);
|
||||
out.push_str("\r\n");
|
||||
}
|
||||
}
|
||||
out.push_str("Connection: close\r\n\r\n");
|
||||
up.write_all(out.as_bytes()).await?;
|
||||
up.write_all(&rest).await?;
|
||||
tokio::io::copy_bidirectional(&mut client, &mut up).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refused(client: &mut TcpStream, why: &str) -> std::io::Result<()> {
|
||||
// Never the host or the address: they are what the bot was asked to open.
|
||||
crate::debug_log!("browser", "proxy refused a connection: {why}");
|
||||
reply(client, "403 Forbidden").await
|
||||
}
|
||||
|
||||
// ---- Chrome ---------------------------------------------------------------------------------------
|
||||
|
||||
/// The arguments the bot's Chrome runs with (its profile, CDP on a free port, every connection
|
||||
/// through the proxy on `proxy_port`): headless, or with `screen` a real window the app opens later
|
||||
/// over CDP (admin.spec.md, "Virtual monitor").
|
||||
pub fn chrome_args(profile: &Path, proxy_port: u16, no_sandbox: bool, screen: bool) -> Vec<String> {
|
||||
let mut a = vec![
|
||||
format!("--user-data-dir={}", profile.display()),
|
||||
"--remote-debugging-port=0".into(),
|
||||
format!("--proxy-server=http://127.0.0.1:{proxy_port}"),
|
||||
"--proxy-bypass-list=<-loopback>".into(),
|
||||
"--disable-quic".into(),
|
||||
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp".into(),
|
||||
"--no-first-run".into(),
|
||||
"--no-default-browser-check".into(),
|
||||
"--disable-sync".into(),
|
||||
"--disable-background-networking".into(),
|
||||
"--disable-component-update".into(),
|
||||
"--disable-default-apps".into(),
|
||||
"--disable-features=Translate,MediaRouter,OptimizationHints".into(),
|
||||
"--window-size=1280,800".into(),
|
||||
// Video and sound start when a page starts them: the tool clicks with page scripts, which
|
||||
// Chrome does not count as a user's gesture, so a video page opened for "play X" stayed paused.
|
||||
"--autoplay-policy=no-user-gesture-required".into(),
|
||||
];
|
||||
if no_sandbox {
|
||||
a.push("--no-sandbox".into());
|
||||
}
|
||||
if cfg!(windows) {
|
||||
// Started by an app running as administrator, Chrome would start again as the user and end
|
||||
// the process the app holds, which could then neither see nor stop it.
|
||||
a.push("--do-not-de-elevate".into());
|
||||
}
|
||||
if screen {
|
||||
// A window opened at start would take the focus from the game the user is playing.
|
||||
a.extend(["--no-startup-window", "--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding",
|
||||
"--disable-background-timer-throttling"].map(String::from));
|
||||
} else {
|
||||
a.insert(0, "--headless=new".into());
|
||||
a.push("about:blank".into());
|
||||
}
|
||||
a
|
||||
}
|
||||
|
||||
pub struct BotBrowser {
|
||||
child: Mutex<Option<tokio::process::Child>>,
|
||||
proxy: EgressProxy,
|
||||
pub cdp_port: u16,
|
||||
/// While set (with a screen, on Windows), a thread settles the windows the bot opens.
|
||||
watching: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl Drop for BotBrowser {
|
||||
fn drop(&mut self) {
|
||||
self.watching.store(false, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl BotBrowser {
|
||||
/// Start Chrome `exe` with profile folder `profile` and wait until CDP answers; with `screen`,
|
||||
/// its one window sits there (the virtual monitor), opened without taking the focus and kept out
|
||||
/// of Alt+Tab and the taskbar.
|
||||
pub async fn start(exe: &Path, profile: &Path, screen: Option<crate::vdisplay::Rect>) -> Result<Self, String> {
|
||||
let proxy = EgressProxy::start(is_public).await.map_err(|e| format!("proxy: {e}"))?;
|
||||
std::fs::create_dir_all(profile).map_err(|e| e.to_string())?;
|
||||
turn_off_translation(profile);
|
||||
let port_file = profile.join("DevToolsActivePort");
|
||||
let _ = std::fs::remove_file(&port_file);
|
||||
let no_sandbox = cfg!(not(windows)) && std::env::var("JARVIS_CHROME_NO_SANDBOX").is_ok_and(|v| v == "1");
|
||||
let mut cmd = tokio::process::Command::new(exe);
|
||||
cmd.args(chrome_args(profile, proxy.port, no_sandbox, screen.is_some())).stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).kill_on_drop(true);
|
||||
#[cfg(unix)]
|
||||
cmd.process_group(0);
|
||||
#[cfg(windows)]
|
||||
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
|
||||
let mut child = cmd.spawn().map_err(|e| format!("Chrome을 실행하지 못했습니다: {e}"))?;
|
||||
let until = tokio::time::Instant::now() + READY_TIMEOUT;
|
||||
let cdp_port = loop {
|
||||
if let Some(p) = std::fs::read_to_string(&port_file).ok()
|
||||
.and_then(|t| t.lines().next().and_then(|l| l.trim().parse::<u16>().ok())) {
|
||||
if version(p).await.is_some() {
|
||||
break p;
|
||||
}
|
||||
}
|
||||
if let Ok(Some(status)) = child.try_wait() {
|
||||
return Err(format!("Chrome이 바로 끝났습니다 ({status})"));
|
||||
}
|
||||
if tokio::time::Instant::now() > until {
|
||||
kill_tree(&mut child).await;
|
||||
return Err("Chrome이 30초 안에 준비되지 않았습니다".into());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
};
|
||||
let b = Self { child: Mutex::new(Some(child)), proxy, cdp_port, watching: Arc::new(AtomicBool::new(false)) };
|
||||
if let Some(r) = screen {
|
||||
if let Err(e) = b.open_window(r).await {
|
||||
b.stop().await;
|
||||
return Err(e);
|
||||
}
|
||||
if cfg!(windows) {
|
||||
// The bot opens a window per person (bot_browser.spec.md), each settled like the first.
|
||||
b.watching.store(true, Ordering::Relaxed);
|
||||
let (on, pid) = (b.watching.clone(), b.pid());
|
||||
std::thread::spawn(move || {
|
||||
while on.load(Ordering::Relaxed) {
|
||||
if crate::vdisplay::settle_windows(pid) > 0 {
|
||||
crate::debug_log!("browser", "a new bot browser window settled on the virtual monitor");
|
||||
}
|
||||
std::thread::sleep(WATCH_EVERY);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
crate::info_log!("browser", "bot browser ready (CDP on 127.0.0.1:{cdp_port}{})",
|
||||
if screen.is_some() { ", on the virtual monitor" } else { "" });
|
||||
Ok(b)
|
||||
}
|
||||
|
||||
/// The window on `r`: a background page in a new window (CDP), then out of the window switcher
|
||||
/// and fitted to the monitor's work area.
|
||||
/// It holds the first page, which the bot never closes, so the window stays; the bot's own
|
||||
/// windows open where it is.
|
||||
async fn open_window(&self, r: crate::vdisplay::Rect) -> Result<(), String> {
|
||||
let ws = version(self.cdp_port).await.and_then(|v| v["webSocketDebuggerUrl"].as_str().map(str::to_owned))
|
||||
.ok_or("Chrome의 CDP 주소를 읽지 못했습니다")?;
|
||||
cdp_call(&ws, serde_json::json!({"id": 1, "method": "Target.createTarget", "params": {"url": "about:blank",
|
||||
"newWindow": true, "background": true, "left": r.x, "top": r.y, "width": r.w, "height": r.h}})).await
|
||||
.map_err(|e| format!("Chrome 창을 열지 못했습니다: {e}"))?;
|
||||
let pid = self.pid();
|
||||
for _ in 0..25 {
|
||||
if cfg!(not(windows)) || crate::vdisplay::settle_windows(pid) > 0 {
|
||||
return Ok(());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
Err("Chrome 창을 찾지 못했습니다".into())
|
||||
}
|
||||
|
||||
/// Chrome's main process (0 once it ended), the one owning its windows.
|
||||
pub fn pid(&self) -> u32 {
|
||||
self.child.lock().unwrap().as_ref().and_then(|c| c.id()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn cdp_url(&self) -> String {
|
||||
format!("http://127.0.0.1:{}", self.cdp_port)
|
||||
}
|
||||
|
||||
/// Whether Chrome still runs.
|
||||
pub fn running(&self) -> bool {
|
||||
self.child.lock().unwrap().as_mut().is_some_and(|c| matches!(c.try_wait(), Ok(None)))
|
||||
}
|
||||
|
||||
/// `Browser.close`, then up to 5 s for Chrome to end, else it is killed; the proxy ends too.
|
||||
pub async fn stop(&self) {
|
||||
self.watching.store(false, Ordering::Relaxed);
|
||||
let child = self.child.lock().unwrap().take();
|
||||
if let Some(mut child) = child {
|
||||
if let Some(ws) = version(self.cdp_port).await.and_then(|v| v["webSocketDebuggerUrl"].as_str().map(str::to_owned)) {
|
||||
let _ = close_over_cdp(&ws).await;
|
||||
}
|
||||
if !matches!(tokio::time::timeout(CLOSE_WAIT, child.wait()).await, Ok(Ok(_))) {
|
||||
crate::warn_log!("browser", "Chrome did not close in {} s; killing it", CLOSE_WAIT.as_secs());
|
||||
kill_tree(&mut child).await;
|
||||
}
|
||||
crate::info_log!("browser", "bot browser stopped");
|
||||
}
|
||||
self.proxy.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// CDP's `/json/version`, if it answers.
|
||||
async fn version(port: u16) -> Option<serde_json::Value> {
|
||||
let client = reqwest::Client::builder().no_proxy().timeout(Duration::from_secs(2)).build().ok()?;
|
||||
let text = client.get(format!("http://127.0.0.1:{port}/json/version")).send().await.ok()?.text().await.ok()?;
|
||||
serde_json::from_str(&text).ok()
|
||||
}
|
||||
|
||||
async fn close_over_cdp(ws: &str) -> Result<(), String> {
|
||||
use futures_util::SinkExt;
|
||||
let (mut s, _) = tokio::time::timeout(Duration::from_secs(2), tokio_tungstenite::connect_async(ws)).await
|
||||
.map_err(|_| "timeout".to_string())?.map_err(|e| e.to_string())?;
|
||||
s.send(tokio_tungstenite::tungstenite::Message::text(r#"{"id":1,"method":"Browser.close"}"#)).await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// One CDP call on the browser connection `ws`; its result, or the error Chrome gave (5 s at most).
|
||||
async fn cdp_call(ws: &str, msg: serde_json::Value) -> Result<serde_json::Value, String> {
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
let id = msg["id"].clone();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
let (mut s, _) = tokio_tungstenite::connect_async(ws).await.map_err(|e| e.to_string())?;
|
||||
s.send(tokio_tungstenite::tungstenite::Message::text(msg.to_string())).await.map_err(|e| e.to_string())?;
|
||||
while let Some(m) = s.next().await {
|
||||
let m = m.map_err(|e| e.to_string())?;
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(m.to_text().unwrap_or("")) else { continue };
|
||||
if v["id"] == id {
|
||||
return match v.get("error") {
|
||||
Some(e) => Err(e["message"].as_str().unwrap_or("error").to_owned()),
|
||||
None => Ok(v["result"].clone()),
|
||||
};
|
||||
}
|
||||
}
|
||||
Err("connection closed".to_owned())
|
||||
}).await.map_err(|_| "timeout".to_owned())?
|
||||
}
|
||||
|
||||
async fn kill_tree(child: &mut tokio::process::Child) {
|
||||
if let Some(pid) = child.id() {
|
||||
#[cfg(unix)]
|
||||
// SAFETY: a plain signal to Chrome's own process group (made with process_group(0)).
|
||||
unsafe {
|
||||
libc::kill(-(pid as i32), libc::SIGKILL);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let mut tk = tokio::process::Command::new("taskkill");
|
||||
tk.args(["/T", "/F", "/PID", &pid.to_string()]).stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).creation_flags(0x0800_0000);
|
||||
let _ = tk.status().await;
|
||||
}
|
||||
}
|
||||
let _ = child.kill().await;
|
||||
}
|
||||
|
||||
/// Chrome no longer reads `--disable-features=Translate`: without this preference it offers to
|
||||
/// translate every page in another language, a bubble that covers the page on the shared screen.
|
||||
/// Written before Chrome starts; the profile's other preferences stay.
|
||||
fn turn_off_translation(profile: &Path) {
|
||||
let file = profile.join("Default").join("Preferences");
|
||||
let mut prefs = std::fs::read_to_string(&file).ok()
|
||||
.and_then(|t| serde_json::from_str::<serde_json::Value>(&t).ok())
|
||||
.filter(serde_json::Value::is_object).unwrap_or_else(|| serde_json::json!({}));
|
||||
if !prefs["translate"].is_object() {
|
||||
prefs["translate"] = serde_json::json!({});
|
||||
}
|
||||
prefs["translate"]["enabled"] = false.into();
|
||||
let written = std::fs::create_dir_all(file.parent().unwrap_or(profile))
|
||||
.and_then(|_| std::fs::write(&file, prefs.to_string()));
|
||||
if let Err(e) = written {
|
||||
crate::warn_log!("browser", "could not turn off translation in the profile: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the bot browser's profile lives in the data folder.
|
||||
pub fn profile_dir(data: &Path) -> PathBuf {
|
||||
data.join("browser").join("profile")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
fn only_public_addresses_are_public() {
|
||||
let public = ["8.8.8.8", "1.1.1.1", "142.250.1.1", "2606:4700:4700::1111", "2a00:1450:4001::1",
|
||||
"2001:4860:4860::8888", "2400:cb00::1"];
|
||||
let not = ["127.0.0.1", "10.0.0.1", "172.16.5.4", "192.168.10.9", "169.254.169.254", "100.64.0.1",
|
||||
"0.0.0.0", "0.1.2.3", "192.0.0.8", "192.0.2.1", "198.18.0.1", "203.0.113.5", "224.0.0.1", "240.0.0.1",
|
||||
"255.255.255.255", "::1", "::", "fc00::1", "fd12::1", "fe80::1", "ff02::1", "2001:db8::1",
|
||||
"::ffff:127.0.0.1", "::ffff:192.168.0.1", "64:ff9b::a00:1", "2002:c0a8:1::1", "2001:0:4136:e378::1",
|
||||
"fec0::1", "fef0::1", "::1.2.3.4", "100::1", "2001:2::1", "2001:10::1", "4000::1", "e000::1"];
|
||||
for a in public {
|
||||
assert!(is_public(&a.parse().unwrap()), "{a} should be public");
|
||||
}
|
||||
for a in not {
|
||||
assert!(!is_public(&a.parse().unwrap()), "{a} should not be public");
|
||||
}
|
||||
}
|
||||
|
||||
/// A local server recording each connection's first bytes and answering `answer`.
|
||||
async fn server(answer: &'static [u8]) -> (u16, Arc<Mutex<Vec<String>>>) {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let s2 = seen.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut c, _)) = l.accept().await {
|
||||
let seen = s2.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let n = c.read(&mut buf).await.unwrap_or(0);
|
||||
seen.lock().unwrap().push(String::from_utf8_lossy(&buf[..n]).into_owned());
|
||||
let _ = c.write_all(answer).await;
|
||||
});
|
||||
}
|
||||
});
|
||||
(port, seen)
|
||||
}
|
||||
|
||||
async fn ask(proxy: u16, req: String) -> String {
|
||||
let mut c = TcpStream::connect(("127.0.0.1", proxy)).await.unwrap();
|
||||
c.write_all(req.as_bytes()).await.unwrap();
|
||||
let mut out = Vec::new();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), c.read_to_end(&mut out)).await;
|
||||
String::from_utf8_lossy(&out).into_owned()
|
||||
}
|
||||
|
||||
fn loopback(ip: &IpAddr) -> bool {
|
||||
ip.is_loopback()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_proxy_refuses_this_pc_and_the_local_network() {
|
||||
let (port, seen) = server(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nhi").await;
|
||||
let p = EgressProxy::start(is_public).await.unwrap();
|
||||
for req in [format!("CONNECT 127.0.0.1:{port} HTTP/1.1\r\nHost: x\r\n\r\n"),
|
||||
format!("CONNECT localhost:{port} HTTP/1.1\r\n\r\n"),
|
||||
format!("CONNECT [::1]:{port} HTTP/1.1\r\n\r\n"),
|
||||
format!("GET http://127.0.0.1:{port}/a HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"),
|
||||
format!("GET http://localhost:{port}/a HTTP/1.1\r\n\r\n"),
|
||||
"CONNECT 192.168.10.1:443 HTTP/1.1\r\n\r\n".to_string(),
|
||||
"GET http://10.0.0.1/ HTTP/1.1\r\n\r\n".to_string()] {
|
||||
let r = ask(p.port, req.clone()).await;
|
||||
assert!(r.starts_with("HTTP/1.1 403"), "{req:?} -> {r:?}");
|
||||
}
|
||||
assert!(seen.lock().unwrap().is_empty());
|
||||
for bad in ["nonsense\r\n\r\n", "GET /relative HTTP/1.1\r\n\r\n", "GET ftp://a.example/ HTTP/1.1\r\n\r\n",
|
||||
"CONNECT nohost HTTP/1.1\r\n\r\n"] {
|
||||
assert!(ask(p.port, bad.into()).await.starts_with("HTTP/1.1 400"), "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_proxy_tunnels_and_forwards_to_allowed_addresses() {
|
||||
let (port, seen) = server(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nhi").await;
|
||||
let p = EgressProxy::start(loopback).await.unwrap();
|
||||
let r = ask(p.port, format!("CONNECT 127.0.0.1:{port} HTTP/1.1\r\n\r\nping")).await;
|
||||
assert!(r.starts_with("HTTP/1.1 200 Connection Established\r\n\r\nHTTP/1.1 200 OK"), "{r:?}");
|
||||
let r = ask(p.port, format!("GET http://127.0.0.1:{port}/a/b?c=1 HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n\
|
||||
Proxy-Connection: keep-alive\r\nConnection: keep-alive\r\nAccept: */*\r\n\r\n")).await;
|
||||
assert!(r.ends_with("hi"), "{r:?}");
|
||||
let seen = seen.lock().unwrap().clone();
|
||||
assert_eq!(seen[0], "ping");
|
||||
assert_eq!(seen[1], format!("GET /a/b?c=1 HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAccept: */*\r\nConnection: close\r\n\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_profile_never_offers_to_translate_and_keeps_its_other_preferences() {
|
||||
let dir = std::env::temp_dir().join(format!("jarvis-translate-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let prefs = |d: &Path| -> serde_json::Value {
|
||||
serde_json::from_str(&std::fs::read_to_string(d.join("Default").join("Preferences")).unwrap()).unwrap()
|
||||
};
|
||||
turn_off_translation(&dir);
|
||||
assert_eq!(prefs(&dir)["translate"]["enabled"], false, "a new profile");
|
||||
|
||||
std::fs::write(dir.join("Default").join("Preferences"),
|
||||
r#"{"homepage":"x","translate":{"enabled":true,"blocked_languages":["ko"]}}"#).unwrap();
|
||||
turn_off_translation(&dir);
|
||||
let p = prefs(&dir);
|
||||
assert_eq!(p["translate"]["enabled"], false, "a profile Chrome already used");
|
||||
assert_eq!(p["homepage"], "x");
|
||||
assert_eq!(p["translate"]["blocked_languages"][0], "ko");
|
||||
|
||||
std::fs::write(dir.join("Default").join("Preferences"), "not json").unwrap();
|
||||
turn_off_translation(&dir);
|
||||
assert_eq!(prefs(&dir)["translate"]["enabled"], false, "a damaged file");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chrome_goes_through_the_proxy_with_its_own_profile() {
|
||||
let a = chrome_args(Path::new("/d/browser/profile"), 4321, false, false);
|
||||
for want in ["--headless=new", "--user-data-dir=/d/browser/profile", "--remote-debugging-port=0",
|
||||
"--proxy-server=http://127.0.0.1:4321", "--proxy-bypass-list=<-loopback>", "--disable-quic",
|
||||
"--autoplay-policy=no-user-gesture-required"] {
|
||||
assert!(a.iter().any(|x| x == want), "{want}");
|
||||
}
|
||||
assert!(!a.iter().any(|x| x == "--no-sandbox"));
|
||||
assert!(chrome_args(Path::new("/p"), 1, true, false).iter().any(|x| x == "--no-sandbox"));
|
||||
assert_eq!(a.iter().any(|x| x == "--do-not-de-elevate"), cfg!(windows), "the app keeps hold of Chrome on Windows");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_a_screen_chrome_has_a_window_it_opens_later_and_keeps_drawing_when_covered() {
|
||||
let a = chrome_args(Path::new("/p"), 4321, false, true);
|
||||
assert!(!a.iter().any(|x| x.starts_with("--headless")));
|
||||
// No window at start: the app opens it over CDP in the background, so it never takes focus.
|
||||
assert!(a.iter().any(|x| x == "--no-startup-window"));
|
||||
assert!(!a.iter().any(|x| x == "about:blank"));
|
||||
for want in ["--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding",
|
||||
"--proxy-server=http://127.0.0.1:4321", "--disable-quic", "--autoplay-policy=no-user-gesture-required"] {
|
||||
assert!(a.iter().any(|x| x == want), "{want}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod stop_tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopping_the_proxy_closes_the_tunnels_it_holds() {
|
||||
// An echo server the tunnel reaches.
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
let (mut c, _) = l.accept().await.unwrap();
|
||||
let mut b = [0u8; 64];
|
||||
loop {
|
||||
match c.read(&mut b).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => { let _ = c.write_all(&b[..n]).await; }
|
||||
}
|
||||
}
|
||||
let _ = closed_tx.send(());
|
||||
});
|
||||
let p = EgressProxy::start(|ip| ip.is_loopback()).await.unwrap();
|
||||
let mut c = TcpStream::connect(("127.0.0.1", p.port)).await.unwrap();
|
||||
c.write_all(format!("CONNECT 127.0.0.1:{port} HTTP/1.1\r\n\r\n").as_bytes()).await.unwrap();
|
||||
let mut b = vec![0u8; 39];
|
||||
c.read_exact(&mut b).await.unwrap();
|
||||
assert!(b.starts_with(b"HTTP/1.1 200 Connection Established"));
|
||||
c.write_all(b"ping").await.unwrap();
|
||||
let mut e = [0u8; 4];
|
||||
c.read_exact(&mut e).await.unwrap();
|
||||
assert_eq!(&e, b"ping");
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), p.stop()).await.expect("stop waits only for the cleanup");
|
||||
// The open tunnel is gone on both sides, and no new connection is taken.
|
||||
let _ = c.write_all(b"after").await;
|
||||
let mut rest = Vec::new();
|
||||
let n = tokio::time::timeout(Duration::from_secs(2), c.read_to_end(&mut rest)).await.expect("closed").unwrap_or(0);
|
||||
assert_eq!(n, 0, "nothing passes after the stop");
|
||||
tokio::time::timeout(Duration::from_secs(2), closed_rx).await.expect("the upstream side closed").unwrap();
|
||||
assert!(TcpStream::connect(("127.0.0.1", p.port)).await.is_err());
|
||||
}
|
||||
}
|
||||
267
app/src/calls.rs
Normal file
267
app/src/calls.rs
Normal file
@@ -0,0 +1,267 @@
|
||||
//! Call detection: does Jarvis answer this utterance? (`app/src/conversation.spec.md`).
|
||||
//!
|
||||
//! A pure function of the settings, the speaker and the text. Wake words and stop words are settings;
|
||||
//! matching works on Unicode-normalised letters, so it behaves the same in any script.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CallMode {
|
||||
/// Answer only when called by a wake word (default).
|
||||
WakeWord,
|
||||
/// Also answer what the brain judges to be said to Jarvis.
|
||||
Addressed,
|
||||
/// Answer every utterance.
|
||||
Everything,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CallSettings {
|
||||
pub wake_words: Vec<String>,
|
||||
/// Share of a wake word's normalised letters a heard word may differ by (0-0.5).
|
||||
pub similarity: f64,
|
||||
pub mode: CallMode,
|
||||
pub designated: HashSet<u64>,
|
||||
pub designated_only: bool,
|
||||
pub designated_no_wake_word: bool,
|
||||
pub stop_words: Vec<String>,
|
||||
/// After a wake word said alone, how long the speaker's next utterance may take to start and
|
||||
/// still be joined to it (the pause after "Jarvis," often ends an utterance).
|
||||
pub follow_up: Duration,
|
||||
/// Play the earcon when Jarvis takes a voice call (`call.earcon`).
|
||||
pub earcon: bool,
|
||||
}
|
||||
|
||||
impl Default for CallSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
wake_words: vec!["자비스".into()],
|
||||
similarity: 0.2,
|
||||
mode: CallMode::WakeWord,
|
||||
designated: HashSet::new(),
|
||||
designated_only: false,
|
||||
designated_no_wake_word: false,
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
earcon: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Call {
|
||||
Ignore,
|
||||
/// Answer it. `direct`: called by a wake word.
|
||||
Request { direct: bool },
|
||||
/// Ask the brain whether it was said to Jarvis (`call.mode` `addressed`).
|
||||
Ask,
|
||||
/// Stop the current answer and drop the waiting requests.
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Most letters after a stop word that still count as the stop ("멈춰 줘", "그만해").
|
||||
const STOP_TAIL: usize = 3;
|
||||
|
||||
/// NFKD, lower case, letters and digits only.
|
||||
pub fn normalise(text: &str) -> Vec<char> {
|
||||
text.nfkd().flat_map(char::to_lowercase).filter(|c| c.is_alphanumeric()).collect()
|
||||
}
|
||||
|
||||
/// The stretch `[start, end)` of `text` closest to `word`, if within `max_edits` edits
|
||||
/// (approximate substring matching: the match may start and end anywhere in `text`).
|
||||
fn find(text: &[char], word: &[char], max_edits: usize) -> Option<(usize, usize)> {
|
||||
if word.is_empty() || text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (n, m) = (text.len(), word.len());
|
||||
// d[i][j]: fewest edits turning word[..i] into a stretch of text ending at j; start[i][j] where it began.
|
||||
let mut d = vec![vec![0usize; n + 1]; m + 1];
|
||||
let mut start = vec![vec![0usize; n + 1]; m + 1];
|
||||
for j in 0..=n {
|
||||
start[0][j] = j;
|
||||
}
|
||||
for i in 1..=m {
|
||||
d[i][0] = i;
|
||||
start[i][0] = 0;
|
||||
for j in 1..=n {
|
||||
let sub = d[i - 1][j - 1] + usize::from(word[i - 1] != text[j - 1]);
|
||||
let del = d[i - 1][j] + 1; // word letter missing from text
|
||||
let ins = d[i][j - 1] + 1; // extra letter in text
|
||||
let (best, from) = if sub <= del && sub <= ins {
|
||||
(sub, start[i - 1][j - 1])
|
||||
} else if del <= ins {
|
||||
(del, start[i - 1][j])
|
||||
} else {
|
||||
(ins, start[i][j - 1])
|
||||
};
|
||||
d[i][j] = best;
|
||||
start[i][j] = from;
|
||||
}
|
||||
}
|
||||
(1..=n)
|
||||
.filter(|&j| d[m][j] <= max_edits)
|
||||
.min_by_key(|&j| (d[m][j], j))
|
||||
.map(|j| (start[m][j], j))
|
||||
}
|
||||
|
||||
impl CallSettings {
|
||||
fn allowed_edits(&self, word: &[char]) -> usize {
|
||||
(self.similarity.clamp(0.0, 0.5) * word.len() as f64 + 1e-9).floor() as usize
|
||||
}
|
||||
|
||||
/// The text with the first heard wake word removed, if one is heard.
|
||||
fn without_wake_word(&self, text: &[char]) -> Option<Vec<char>> {
|
||||
// The earliest heard wake word is the one used to address Jarvis.
|
||||
let (s, e) = self
|
||||
.wake_words
|
||||
.iter()
|
||||
.map(|w| normalise(w))
|
||||
.filter_map(|word| find(text, &word, self.allowed_edits(&word)))
|
||||
.min()?;
|
||||
Some(text[..s].iter().chain(&text[e..]).copied().collect())
|
||||
}
|
||||
|
||||
fn is_stop(&self, rest: &[char]) -> bool {
|
||||
self.stop_words.iter().map(|w| normalise(w)).any(|w| {
|
||||
!w.is_empty() && rest.starts_with(&w) && rest.len() - w.len() <= STOP_TAIL
|
||||
})
|
||||
}
|
||||
|
||||
/// A wake word is heard and nothing else (no letters or digits around it).
|
||||
pub fn wake_word_only(&self, text: &str) -> bool {
|
||||
self.without_wake_word(&normalise(text)).is_some_and(|rest| rest.is_empty())
|
||||
}
|
||||
|
||||
/// Decide what to do with one finished utterance.
|
||||
pub fn judge(&self, speaker: u64, text: &str) -> Call {
|
||||
let norm = normalise(text);
|
||||
if norm.is_empty() {
|
||||
return Call::Ignore;
|
||||
}
|
||||
let designated = self.designated.contains(&speaker);
|
||||
if self.designated_only && !designated {
|
||||
return Call::Ignore;
|
||||
}
|
||||
if let Some(rest) = self.without_wake_word(&norm) {
|
||||
if self.is_stop(&rest) {
|
||||
return Call::Stop;
|
||||
}
|
||||
return Call::Request { direct: true };
|
||||
}
|
||||
if self.designated_no_wake_word && designated {
|
||||
return Call::Request { direct: false };
|
||||
}
|
||||
match self.mode {
|
||||
CallMode::Everything => Call::Request { direct: false },
|
||||
CallMode::Addressed => Call::Ask,
|
||||
CallMode::WakeWord => Call::Ignore,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const A: u64 = 1;
|
||||
const B: u64 = 2;
|
||||
|
||||
fn s() -> CallSettings {
|
||||
CallSettings::default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wake_word_with_nothing_else_is_told_apart() {
|
||||
let c = s();
|
||||
assert!(c.wake_word_only("자비스,"));
|
||||
assert!(c.wake_word_only("자비즈?"));
|
||||
assert!(!c.wake_word_only("자비스, 지금 몇 시야?"));
|
||||
assert!(!c.wake_word_only("서비스"));
|
||||
assert!(!c.wake_word_only(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_and_near_wake_words_call_but_a_different_word_does_not() {
|
||||
let c = s();
|
||||
assert_eq!(c.judge(A, "자비스, 오늘 날씨 어때?"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "자비즈 날씨 알려 줘"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "오늘 날씨 어때 자비스"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "서비스 센터 몇 시에 열어?"), Call::Ignore);
|
||||
assert_eq!(c.judge(A, "오늘 날씨 어때?"), Call::Ignore);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latin_wake_words_ignore_case_spacing_and_accents() {
|
||||
let c = CallSettings { wake_words: vec!["Jarvis".into()], ..s() };
|
||||
assert_eq!(c.judge(A, "hey JARVIS, what's up"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "Jervis, lights on"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "Jär vis?"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "the service is down"), Call::Ignore);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn similarity_zero_needs_the_exact_word() {
|
||||
let c = CallSettings { similarity: 0.0, ..s() };
|
||||
assert_eq!(c.judge(A, "자비즈 날씨"), Call::Ignore);
|
||||
assert_eq!(c.judge(A, "자비스 날씨"), Call::Request { direct: true });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_needs_the_wake_word_and_little_else() {
|
||||
let c = s();
|
||||
assert_eq!(c.judge(A, "자비스, 멈춰!"), Call::Stop);
|
||||
assert_eq!(c.judge(A, "자비스 그만해"), Call::Stop);
|
||||
assert_eq!(c.judge(A, "자비스 멈춰 줘"), Call::Stop);
|
||||
assert_eq!(c.judge(A, "멈춰"), Call::Ignore);
|
||||
assert_eq!(c.judge(A, "자비스 그만 얘기하고 날씨 알려 줘"), Call::Request { direct: true });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_punctuation_only_text_is_ignored() {
|
||||
let c = CallSettings { mode: CallMode::Everything, ..s() };
|
||||
assert_eq!(c.judge(A, ""), Call::Ignore);
|
||||
assert_eq!(c.judge(A, " ... ?! "), Call::Ignore);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn modes_decide_utterances_without_a_wake_word() {
|
||||
let every = CallSettings { mode: CallMode::Everything, ..s() };
|
||||
assert_eq!(every.judge(A, "날씨 어때"), Call::Request { direct: false });
|
||||
let addressed = CallSettings { mode: CallMode::Addressed, ..s() };
|
||||
assert_eq!(addressed.judge(A, "날씨 어때"), Call::Ask);
|
||||
assert_eq!(addressed.judge(A, "자비스 날씨 어때"), Call::Request { direct: true });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn designated_switches_combine_as_the_design_table_says() {
|
||||
let base = CallSettings { designated: [A].into(), ..s() };
|
||||
// only on, no-wake off: designated people answered only when called.
|
||||
let c = CallSettings { designated_only: true, ..base.clone() };
|
||||
assert_eq!(c.judge(A, "자비스 날씨"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "날씨"), Call::Ignore);
|
||||
assert_eq!(c.judge(B, "자비스 날씨"), Call::Ignore);
|
||||
assert_eq!(c.judge(B, "자비스 멈춰"), Call::Ignore);
|
||||
// only on, no-wake on: designated people only, without a wake word.
|
||||
let c = CallSettings { designated_only: true, designated_no_wake_word: true, ..base.clone() };
|
||||
assert_eq!(c.judge(A, "날씨"), Call::Request { direct: false });
|
||||
assert_eq!(c.judge(B, "자비스 날씨"), Call::Ignore);
|
||||
// only off, no-wake on: anyone by wake word, designated people without.
|
||||
let c = CallSettings { designated_no_wake_word: true, ..base.clone() };
|
||||
assert_eq!(c.judge(A, "날씨"), Call::Request { direct: false });
|
||||
assert_eq!(c.judge(B, "날씨"), Call::Ignore);
|
||||
assert_eq!(c.judge(B, "자비스 날씨"), Call::Request { direct: true });
|
||||
// both off (default): anyone, by wake word only.
|
||||
assert_eq!(base.judge(A, "날씨"), Call::Ignore);
|
||||
assert_eq!(base.judge(B, "자비스 날씨"), Call::Request { direct: true });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn several_wake_words_all_count() {
|
||||
let c = CallSettings { wake_words: vec!["자비스".into(), "javis".into()], ..s() };
|
||||
assert_eq!(c.judge(A, "javis 날씨"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "javis stop"), Call::Stop);
|
||||
}
|
||||
}
|
||||
431
app/src/channel_sync.rs
Normal file
431
app/src/channel_sync.rs
Normal file
@@ -0,0 +1,431 @@
|
||||
//! Chat channel names and topics kept in step with Discord (`app/src/admin.spec.md`, "Chat channel
|
||||
//! names and topics"; `docs/v2/conversation.md`, "채널 이름·주제 반영"). When the chat channel
|
||||
//! settings are saved, each channel with `syncDiscord` whose name or description changed gets them as
|
||||
//! its Discord name and topic; Discord's own form of the name (spaces become `-`) is written back.
|
||||
//! Discord allows only a few such edits per channel in ten minutes: a refused edit is "pending" and
|
||||
//! tried again by itself after the wait Discord names. Without the permission (or any other refusal)
|
||||
//! the edit is "failed" with the reason, and the app keeps what the user wrote.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::settings::ChatChannelSettings;
|
||||
|
||||
/// Discord's limit for a channel topic.
|
||||
pub const TOPIC_MAX: usize = 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct ChannelInfo {
|
||||
pub name: String,
|
||||
pub topic: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ApiError {
|
||||
/// Too many edits: try again after this long.
|
||||
RateLimited(Duration),
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
/// Discord's channel endpoints (a fake in tests).
|
||||
#[async_trait]
|
||||
pub trait ChannelApi: Send + Sync {
|
||||
async fn info(&self, id: u64) -> Result<ChannelInfo, ApiError>;
|
||||
/// Set what is given; returns the channel as Discord stored it.
|
||||
async fn edit(&self, id: u64, name: Option<&str>, topic: Option<&str>) -> Result<ChannelInfo, ApiError>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(tag = "state", rename_all = "camelCase")]
|
||||
pub enum SyncState {
|
||||
Applied { name: String },
|
||||
#[serde(rename_all = "camelCase")]
|
||||
Pending { retry_in_sec: u64 },
|
||||
Failed { reason: String },
|
||||
}
|
||||
|
||||
/// Refuse (before anything is saved) a description Discord would not take as a topic.
|
||||
pub fn check_topics(channels: &[ChatChannelSettings]) -> Result<(), String> {
|
||||
match channels.iter().find(|c| c.sync_discord && c.description.chars().count() > TOPIC_MAX) {
|
||||
Some(c) => Err(format!("채널 {}의 설명이 {}자를 넘어 디스코드 주제로 쓸 수 없습니다 ({}자)", c.id, TOPIC_MAX,
|
||||
c.description.chars().count())),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The channels to bring to Discord after a save: `syncDiscord` on and new, or with another name or
|
||||
/// description than before.
|
||||
pub fn changed(old: &[ChatChannelSettings], new: &[ChatChannelSettings]) -> Vec<ChatChannelSettings> {
|
||||
new.iter().filter(|c| c.sync_discord)
|
||||
.filter(|c| old.iter().find(|o| o.id == c.id)
|
||||
.is_none_or(|o| o.name != c.name || o.description != c.description || !o.sync_discord))
|
||||
.cloned().collect()
|
||||
}
|
||||
|
||||
pub struct Syncer {
|
||||
api: Arc<dyn ChannelApi>,
|
||||
settings_file: PathBuf,
|
||||
status: Mutex<HashMap<u64, SyncState>>,
|
||||
/// Channels with a retry already waiting (another save while waiting adds none), and whether
|
||||
/// that channel was new (so the retry still keeps empty fields out).
|
||||
retrying: Mutex<HashMap<u64, bool>>,
|
||||
/// One channel edit at a time: a later save waits for an earlier one, so a slow answer to the
|
||||
/// earlier never lands after the later.
|
||||
edit_lock: tokio::sync::Mutex<()>,
|
||||
}
|
||||
|
||||
impl Syncer {
|
||||
pub fn new(api: Arc<dyn ChannelApi>, settings_file: PathBuf) -> Arc<Self> {
|
||||
Arc::new(Self { api, settings_file, status: Mutex::default(), retrying: Mutex::default(),
|
||||
edit_lock: tokio::sync::Mutex::new(()) })
|
||||
}
|
||||
|
||||
/// A channel's name and topic on Discord now (the screens fill a newly registered channel with them).
|
||||
pub async fn info(&self, id: u64) -> Result<ChannelInfo, String> {
|
||||
self.api.info(id).await.map_err(|e| match e {
|
||||
ApiError::RateLimited(w) => format!("디스코드가 {}초 뒤에 다시 물어 달라고 합니다", w.as_secs()),
|
||||
ApiError::Failed(r) => r,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn status(&self) -> HashMap<u64, SyncState> {
|
||||
self.status.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Bring these channels to Discord now; `old` tells which ones are new (a new one never clears a
|
||||
/// Discord name or topic with an empty field).
|
||||
pub async fn sync(self: &Arc<Self>, channels: Vec<ChatChannelSettings>, old: &[ChatChannelSettings])
|
||||
-> Vec<(u64, SyncState)> {
|
||||
let mut out = vec![];
|
||||
for c in channels {
|
||||
let is_new = !old.iter().any(|o| o.id == c.id);
|
||||
let state = self.sync_one(&c, is_new).await;
|
||||
self.status.lock().unwrap().insert(c.id, state.clone());
|
||||
out.push((c.id, state));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
async fn sync_one(self: &Arc<Self>, c: &ChatChannelSettings, is_new: bool) -> SyncState {
|
||||
let _one_at_a_time = self.edit_lock.lock().await;
|
||||
let current = match self.api.info(c.id).await {
|
||||
Ok(i) => i,
|
||||
Err(e) => return self.refused(c.id, e, is_new),
|
||||
};
|
||||
let keep = |v: &str| !(is_new && v.trim().is_empty());
|
||||
let name = (keep(&c.name) && c.name != current.name).then_some(c.name.as_str());
|
||||
let topic = (keep(&c.description) && c.description != current.topic).then_some(c.description.as_str());
|
||||
if name.is_none() && topic.is_none() {
|
||||
return SyncState::Applied { name: current.name };
|
||||
}
|
||||
match self.api.edit(c.id, name, topic).await {
|
||||
Ok(stored) => {
|
||||
if keep(&c.name) && stored.name != c.name {
|
||||
self.write_name(c.id, &c.name, &stored.name);
|
||||
}
|
||||
crate::info_log!("chat", "channel {} name and topic updated on Discord", c.id);
|
||||
SyncState::Applied { name: stored.name }
|
||||
}
|
||||
Err(e) => self.refused(c.id, e, is_new),
|
||||
}
|
||||
}
|
||||
|
||||
fn refused(self: &Arc<Self>, id: u64, e: ApiError, is_new: bool) -> SyncState {
|
||||
match e {
|
||||
ApiError::RateLimited(wait) => {
|
||||
crate::info_log!("chat", "channel {id}: Discord asks to wait {}s before the next edit", wait.as_secs());
|
||||
let first = {
|
||||
let mut r = self.retrying.lock().unwrap();
|
||||
let first = !r.contains_key(&id);
|
||||
*r.entry(id).or_insert(is_new) |= is_new;
|
||||
first
|
||||
};
|
||||
if first {
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(wait).await;
|
||||
let is_new = me.retrying.lock().unwrap().remove(&id).unwrap_or(false);
|
||||
me.retry(id, is_new).await;
|
||||
});
|
||||
}
|
||||
SyncState::Pending { retry_in_sec: wait.as_secs().max(1) }
|
||||
}
|
||||
ApiError::Failed(reason) => {
|
||||
crate::warn_log!("chat", "channel {id}: name and topic not updated on Discord: {reason}");
|
||||
SyncState::Failed { reason }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try a pending channel again with what the settings hold now.
|
||||
async fn retry(self: &Arc<Self>, id: u64, is_new: bool) {
|
||||
let Ok(settings) = crate::settings::load_settings(&self.settings_file) else { return };
|
||||
match settings.chat_channels.into_iter().find(|c| c.id == id && c.sync_discord) {
|
||||
Some(c) => {
|
||||
let state = self.sync_one(&c, is_new).await;
|
||||
self.status.lock().unwrap().insert(id, state);
|
||||
}
|
||||
None => {
|
||||
self.status.lock().unwrap().remove(&id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Show Discord's form of the name (spaces become `-`) in the app too, only while the settings
|
||||
/// still hold the name that was sent (a later save keeps its own).
|
||||
fn write_name(&self, id: u64, sent: &str, name: &str) {
|
||||
let Ok(view) = crate::settings_edit::view(&self.settings_file) else { return };
|
||||
let Some(Value::Array(mut list)) = view.get("chatChannels").cloned() else { return };
|
||||
let mut changed = false;
|
||||
for c in &mut list {
|
||||
if c["id"].as_str() == Some(id.to_string().as_str()) && c["name"].as_str() == Some(sent) {
|
||||
c["name"] = json!(name);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return;
|
||||
}
|
||||
let mut changes = serde_json::Map::new();
|
||||
changes.insert("chatChannels".into(), Value::Array(list));
|
||||
if let Err(e) = crate::settings_edit::apply(&self.settings_file, &changes) {
|
||||
crate::warn_log!("chat", "channel {id}: could not write Discord's name back: {}", e.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Discord's REST API with the stored bot token. Serenity's client waits out rate limits by itself
|
||||
/// (up to ten minutes for channel edits), so these two calls go out directly and report the wait.
|
||||
pub struct DiscordRest {
|
||||
pub secrets_file: PathBuf,
|
||||
pub base: String,
|
||||
}
|
||||
|
||||
impl DiscordRest {
|
||||
pub fn new(secrets_file: PathBuf) -> Self {
|
||||
Self { secrets_file, base: "https://discord.com/api/v10".into() }
|
||||
}
|
||||
|
||||
async fn call(&self, req: impl FnOnce(&reqwest::Client, String) -> reqwest::RequestBuilder) -> Result<ChannelInfo, ApiError> {
|
||||
let token = crate::secrets::Secrets::new(&self.secrets_file).get(crate::secrets::DISCORD_TOKEN)
|
||||
.ok().flatten().ok_or_else(|| ApiError::Failed("봇 토큰이 저장되어 있지 않습니다".into()))?;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build()
|
||||
.map_err(|e| ApiError::Failed(e.to_string()))?;
|
||||
let r = req(&client, self.base.clone()).header("Authorization", format!("Bot {token}"))
|
||||
.header("User-Agent", "DiscordBot (https://github.com/jarvis, 0.1)").send().await
|
||||
.map_err(|e| ApiError::Failed(format!("디스코드에 연결하지 못했습니다: {e}")))?;
|
||||
let status = r.status().as_u16();
|
||||
let body: Value = serde_json::from_str(&r.text().await.unwrap_or_default()).unwrap_or_default();
|
||||
match status {
|
||||
200..=299 => Ok(ChannelInfo { name: body["name"].as_str().unwrap_or_default().to_owned(),
|
||||
topic: body["topic"].as_str().unwrap_or_default().to_owned() }),
|
||||
429 => Err(ApiError::RateLimited(Duration::from_secs_f64(body["retry_after"].as_f64().unwrap_or(60.0).ceil()))),
|
||||
403 => Err(ApiError::Failed("봇에 이 채널의 채널 관리 권한이 없습니다".into())),
|
||||
404 => Err(ApiError::Failed("디스코드에서 이 채널을 찾지 못했습니다".into())),
|
||||
_ => Err(ApiError::Failed(format!("디스코드가 거절했습니다 ({status}): {}",
|
||||
body["message"].as_str().unwrap_or_default()))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChannelApi for DiscordRest {
|
||||
async fn info(&self, id: u64) -> Result<ChannelInfo, ApiError> {
|
||||
self.call(|c, base| c.get(format!("{base}/channels/{id}"))).await
|
||||
}
|
||||
|
||||
async fn edit(&self, id: u64, name: Option<&str>, topic: Option<&str>) -> Result<ChannelInfo, ApiError> {
|
||||
let mut body = serde_json::Map::new();
|
||||
if let Some(n) = name {
|
||||
body.insert("name".into(), json!(n));
|
||||
}
|
||||
if let Some(t) = topic {
|
||||
body.insert("topic".into(), json!(t));
|
||||
}
|
||||
let body = Value::Object(body).to_string();
|
||||
self.call(move |c, base| c.patch(format!("{base}/channels/{id}"))
|
||||
.header("Content-Type", "application/json").body(body)).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
|
||||
/// Discord as the tests need it: names lose their spaces, a channel may refuse, and the first
|
||||
/// `limited` edits are rate limited.
|
||||
#[derive(Default)]
|
||||
struct FakeDiscord {
|
||||
channels: Mutex<HashMap<u64, ChannelInfo>>,
|
||||
forbidden: Vec<u64>,
|
||||
limited: Mutex<u32>,
|
||||
edits: Mutex<Vec<(u64, Option<String>, Option<String>)>>,
|
||||
/// How long each next edit takes (the first save's slow answer).
|
||||
delays: Mutex<Vec<Duration>>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl ChannelApi for FakeDiscord {
|
||||
async fn info(&self, id: u64) -> Result<ChannelInfo, ApiError> {
|
||||
self.channels.lock().unwrap().get(&id).cloned().ok_or_else(|| ApiError::Failed("없음".into()))
|
||||
}
|
||||
async fn edit(&self, id: u64, name: Option<&str>, topic: Option<&str>) -> Result<ChannelInfo, ApiError> {
|
||||
if self.forbidden.contains(&id) {
|
||||
return Err(ApiError::Failed("봇에 이 채널의 채널 관리 권한이 없습니다".into()));
|
||||
}
|
||||
let delay = { let mut d = self.delays.lock().unwrap(); if d.is_empty() { None } else { Some(d.remove(0)) } };
|
||||
if let Some(d) = delay {
|
||||
tokio::time::sleep(d).await;
|
||||
}
|
||||
{
|
||||
let mut l = self.limited.lock().unwrap();
|
||||
if *l > 0 {
|
||||
*l -= 1;
|
||||
return Err(ApiError::RateLimited(Duration::from_millis(200)));
|
||||
}
|
||||
}
|
||||
self.edits.lock().unwrap().push((id, name.map(String::from), topic.map(String::from)));
|
||||
let mut chans = self.channels.lock().unwrap();
|
||||
let c = chans.get_mut(&id).unwrap();
|
||||
if let Some(n) = name {
|
||||
c.name = n.replace(' ', "-");
|
||||
}
|
||||
if let Some(t) = topic {
|
||||
c.topic = t.into();
|
||||
}
|
||||
Ok(c.clone())
|
||||
}
|
||||
}
|
||||
|
||||
const A: u64 = 100000000000000001;
|
||||
const B: u64 = 100000000000000002;
|
||||
|
||||
fn ch(id: u64, name: &str, description: &str) -> ChatChannelSettings {
|
||||
ChatChannelSettings { id, name: name.into(), description: description.into(), read: true, send: true, edit: true,
|
||||
react: true, delete: false, pin: false, answer_all: false, sync_discord: true }
|
||||
}
|
||||
|
||||
fn fake(forbidden: Vec<u64>, limited: u32) -> Arc<FakeDiscord> {
|
||||
let f = FakeDiscord { forbidden, limited: Mutex::new(limited), ..Default::default() };
|
||||
f.channels.lock().unwrap().insert(A, ChannelInfo { name: "일정".into(), topic: "".into() });
|
||||
f.channels.lock().unwrap().insert(B, ChannelInfo { name: "잡담".into(), topic: "아무 말".into() });
|
||||
Arc::new(f)
|
||||
}
|
||||
|
||||
fn settings_with(chans: &[ChatChannelSettings]) -> PathBuf {
|
||||
let f = temp_dir("sync").join("settings.json");
|
||||
std::fs::create_dir_all(f.parent().unwrap()).unwrap();
|
||||
let list: Vec<Value> = chans.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name,
|
||||
"description": c.description})).collect();
|
||||
std::fs::write(&f, json!({"chatChannels": list}).to_string()).unwrap();
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_new_or_edited_channels_that_keep_discord_in_step_are_sent() {
|
||||
let old = vec![ch(A, "일정", ""), ch(B, "잡담", "아무 말")];
|
||||
let mut off = ch(B, "잡담2", "아무 말");
|
||||
off.sync_discord = false;
|
||||
assert!(changed(&old, &old).is_empty());
|
||||
assert_eq!(changed(&old, &[ch(A, "일정 표", ""), off]).iter().map(|c| c.id).collect::<Vec<_>>(), [A]);
|
||||
assert_eq!(changed(&[], &[ch(B, "잡담", "아무 말")]).len(), 1, "a newly registered channel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_topic_over_discord_s_limit_is_refused_before_saving() {
|
||||
let long = "가".repeat(TOPIC_MAX + 1);
|
||||
assert!(check_topics(&[ch(A, "일정", &long)]).unwrap_err().contains("1024"));
|
||||
let mut off = ch(A, "일정", &long);
|
||||
off.sync_discord = false;
|
||||
assert!(check_topics(&[off]).is_ok(), "not sent to Discord: any length");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_edit_reaches_discord_and_discord_s_form_of_the_name_comes_back() {
|
||||
let d = fake(vec![], 0);
|
||||
let file = settings_with(&[ch(A, "일정 표", "할 일")]);
|
||||
let s = Syncer::new(d.clone(), file.clone());
|
||||
let out = s.sync(vec![ch(A, "일정 표", "할 일")], &[ch(A, "일정", "")]).await;
|
||||
assert_eq!(out, [(A, SyncState::Applied { name: "일정-표".into() })]);
|
||||
assert_eq!(d.channels.lock().unwrap()[&A], ChannelInfo { name: "일정-표".into(), topic: "할 일".into() });
|
||||
let saved = crate::settings::load_settings(&file).unwrap();
|
||||
assert_eq!(saved.chat_channels[0].name, "일정-표", "the app shows Discord's name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn without_permission_it_fails_with_the_reason_and_the_app_keeps_the_text() {
|
||||
let d = fake(vec![A], 0);
|
||||
let file = settings_with(&[ch(A, "일정 표", "할 일")]);
|
||||
let s = Syncer::new(d.clone(), file.clone());
|
||||
let out = s.sync(vec![ch(A, "일정 표", "할 일")], &[]).await;
|
||||
assert!(matches!(&out[0].1, SyncState::Failed { reason } if reason.contains("권한")), "{out:?}");
|
||||
assert_eq!(crate::settings::load_settings(&file).unwrap().chat_channels[0].name, "일정 표");
|
||||
assert_eq!(d.channels.lock().unwrap()[&A].name, "일정");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_rate_limited_edit_is_pending_and_goes_through_by_itself_with_the_latest_text() {
|
||||
let d = fake(vec![], 1);
|
||||
let file = settings_with(&[ch(A, "일정 표", "할 일")]);
|
||||
let s = Syncer::new(d.clone(), file.clone());
|
||||
let out = s.sync(vec![ch(A, "일정 표", "할 일")], &[ch(A, "일정", "")]).await;
|
||||
assert!(matches!(out[0].1, SyncState::Pending { .. }), "{out:?}");
|
||||
// Edited again while waiting: the retry sends what the settings hold then.
|
||||
std::fs::write(&file, json!({"chatChannels": [{"id": A.to_string(), "name": "일정 표 2", "description": "할 일"}]})
|
||||
.to_string()).unwrap();
|
||||
// A second save while waiting is pending too, but adds no second retry.
|
||||
*d.limited.lock().unwrap() = 1;
|
||||
let again = s.sync(vec![ch(A, "일정 표 2", "할 일")], &[ch(A, "일정 표", "할 일")]).await;
|
||||
assert!(matches!(again[0].1, SyncState::Pending { .. }), "{again:?}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
assert_eq!(s.status()[&A], SyncState::Applied { name: "일정-표-2".into() });
|
||||
assert_eq!(d.channels.lock().unwrap()[&A].name, "일정-표-2");
|
||||
assert_eq!(d.edits.lock().unwrap().len(), 1, "one retry, one edit");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_new_channel_with_empty_fields_never_clears_discord_and_unchanged_ones_are_not_edited() {
|
||||
let d = fake(vec![], 0);
|
||||
let s = Syncer::new(d.clone(), settings_with(&[]));
|
||||
let out = s.sync(vec![ch(B, "", "")], &[]).await;
|
||||
assert_eq!(out, [(B, SyncState::Applied { name: "잡담".into() })]);
|
||||
assert_eq!(d.channels.lock().unwrap()[&B].topic, "아무 말");
|
||||
s.sync(vec![ch(B, "잡담", "아무 말")], &[]).await;
|
||||
assert!(d.edits.lock().unwrap().is_empty(), "nothing differs from Discord: no edit");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_retried_new_channel_still_never_clears_discord_with_an_empty_field() {
|
||||
let d = fake(vec![], 1);
|
||||
let file = settings_with(&[ch(B, "잡담 2", "")]);
|
||||
let s = Syncer::new(d.clone(), file);
|
||||
let out = s.sync(vec![ch(B, "잡담 2", "")], &[]).await;
|
||||
assert!(matches!(out[0].1, SyncState::Pending { .. }), "{out:?}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
assert_eq!(s.status()[&B], SyncState::Applied { name: "잡담-2".into() });
|
||||
assert_eq!(d.channels.lock().unwrap()[&B].topic, "아무 말", "the retry kept the empty description out");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_slow_first_save_never_overwrites_a_later_one() {
|
||||
let d = fake(vec![], 0);
|
||||
*d.delays.lock().unwrap() = vec![Duration::from_millis(300)];
|
||||
let file = settings_with(&[ch(A, "일정 표", "")]);
|
||||
let s = Syncer::new(d.clone(), file.clone());
|
||||
let s1 = s.clone();
|
||||
let first = tokio::spawn(async move { s1.sync(vec![ch(A, "일정 표", "")], &[ch(A, "일정", "")]).await });
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
// Saved again while Discord still answers the first edit.
|
||||
std::fs::write(&file, json!({"chatChannels": [{"id": A.to_string(), "name": "회의", "description": ""}]}).to_string()).unwrap();
|
||||
s.sync(vec![ch(A, "회의", "")], &[ch(A, "일정 표", "")]).await;
|
||||
first.await.unwrap();
|
||||
assert_eq!(d.channels.lock().unwrap()[&A].name, "회의", "Discord ends with the later save");
|
||||
assert_eq!(crate::settings::load_settings(&file).unwrap().chat_channels[0].name, "회의", "and so does the app");
|
||||
}
|
||||
}
|
||||
215
app/src/chat.rs
Normal file
215
app/src/chat.rs
Normal file
@@ -0,0 +1,215 @@
|
||||
//! Registered chat channels: which ones the bot may read or write, and message splitting.
|
||||
|
||||
use crate::settings::ChatChannelSettings;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Discord's message length limit, in UTF-16 code units as Discord counts.
|
||||
pub const MESSAGE_LIMIT: usize = 2000;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ChatMessage {
|
||||
pub channel_id: u64,
|
||||
pub message_id: u64,
|
||||
pub author_id: u64,
|
||||
pub author_name: String,
|
||||
pub author_is_bot: bool,
|
||||
/// Written by this bot. Live events never carry these; history reads do.
|
||||
pub from_self: bool,
|
||||
/// What the person wrote. Data, never instructions (privacy.md).
|
||||
pub content: String,
|
||||
/// RFC 3339.
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ChatAccessError(pub String);
|
||||
|
||||
impl std::fmt::Display for ChatAccessError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for ChatAccessError {}
|
||||
|
||||
pub struct ChatChannels {
|
||||
by_id: HashMap<u64, ChatChannelSettings>,
|
||||
}
|
||||
|
||||
impl ChatChannels {
|
||||
pub fn new(channels: &[ChatChannelSettings]) -> Self {
|
||||
Self { by_id: channels.iter().map(|c| (c.id, c.clone())).collect() }
|
||||
}
|
||||
|
||||
/// Some channel is readable, so the bot needs the message intents.
|
||||
pub fn any_readable(&self) -> bool {
|
||||
self.by_id.values().any(|c| c.read)
|
||||
}
|
||||
|
||||
pub fn can_read(&self, id: u64) -> bool {
|
||||
self.by_id.get(&id).is_some_and(|c| c.read)
|
||||
}
|
||||
|
||||
pub fn can_send(&self, id: u64) -> bool {
|
||||
self.by_id.get(&id).is_some_and(|c| c.send)
|
||||
}
|
||||
|
||||
pub fn check_read(&self, id: u64) -> Result<(), ChatAccessError> {
|
||||
match self.by_id.get(&id) {
|
||||
None => Err(ChatAccessError(format!("channel {id} is not a registered chat channel"))),
|
||||
Some(c) if !c.read => Err(ChatAccessError(format!("reading channel {id} is turned off"))),
|
||||
Some(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn check_send(&self, id: u64) -> Result<(), ChatAccessError> {
|
||||
match self.by_id.get(&id) {
|
||||
None => Err(ChatAccessError(format!("channel {id} is not a registered chat channel"))),
|
||||
Some(c) if !c.send => Err(ChatAccessError(format!("sending to channel {id} is turned off"))),
|
||||
Some(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The switch a chat action needs (`send`, `edit`, `react`, `delete`, `pin`, `unpin`).
|
||||
pub fn check_action(&self, id: u64, action: &str) -> Result<(), ChatAccessError> {
|
||||
let Some(c) = self.by_id.get(&id) else {
|
||||
return Err(ChatAccessError(format!("channel {id} is not a registered chat channel")));
|
||||
};
|
||||
let on = match action {
|
||||
"send" => c.send,
|
||||
"edit" => c.edit,
|
||||
"react" => c.react,
|
||||
"delete" => c.delete,
|
||||
"pin" | "unpin" => c.pin,
|
||||
_ => return Err(ChatAccessError(format!("unknown chat action {action}"))),
|
||||
};
|
||||
if on { Ok(()) } else { Err(ChatAccessError(format!("{action} is turned off for channel {id}"))) }
|
||||
}
|
||||
|
||||
pub fn all(&self) -> impl Iterator<Item = &ChatChannelSettings> {
|
||||
self.by_id.values()
|
||||
}
|
||||
}
|
||||
|
||||
/// Split text into messages of at most `limit` UTF-16 units, breaking at the last newline, else the
|
||||
/// last space, else hard (never inside a character). Nothing is lost except the newline or space a
|
||||
/// break happens at.
|
||||
pub fn split_message(text: &str, limit: usize) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut rest = text;
|
||||
loop {
|
||||
// Byte offset just past the longest prefix within the limit, and whether more follows.
|
||||
let mut units = 0;
|
||||
let mut cut = rest.len();
|
||||
let mut window_end = rest.len(); // the prefix plus one more character, to see a break right at the limit
|
||||
for (i, ch) in rest.char_indices() {
|
||||
if units + ch.len_utf16() > limit {
|
||||
cut = i;
|
||||
window_end = i + ch.len_utf8();
|
||||
break;
|
||||
}
|
||||
units += ch.len_utf16();
|
||||
}
|
||||
if cut == rest.len() {
|
||||
if !rest.is_empty() {
|
||||
out.push(rest.to_owned());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
let window = &rest[..window_end];
|
||||
let brk = window.rfind('\n').filter(|&b| b > 0).or_else(|| window.rfind(' ').filter(|&b| b > 0));
|
||||
match brk {
|
||||
Some(b) => {
|
||||
out.push(rest[..b].to_owned());
|
||||
rest = &rest[b + 1..];
|
||||
}
|
||||
None => {
|
||||
out.push(rest[..cut].to_owned());
|
||||
rest = &rest[cut..];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Message Content intent bits in the application flags (GATEWAY_MESSAGE_CONTENT and its _LIMITED form).
|
||||
pub fn has_message_content_intent(application_flags: u64) -> bool {
|
||||
application_flags & ((1 << 18) | (1 << 19)) != 0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn utf16_len(s: &str) -> usize {
|
||||
s.encode_utf16().count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_text_is_left_alone() {
|
||||
assert_eq!(split_message("안녕하세요", MESSAGE_LIMIT), vec!["안녕하세요"]);
|
||||
assert!(split_message("", MESSAGE_LIMIT).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaks_at_the_last_newline_then_space_and_loses_only_the_break() {
|
||||
assert_eq!(split_message("aaaa bbbb\ncccc dddd", 12), vec!["aaaa bbbb", "cccc dddd"]);
|
||||
assert_eq!(split_message("aaaa bbbb cccc", 10), vec!["aaaa bbbb", "cccc"]);
|
||||
// A break exactly at the limit is used.
|
||||
assert_eq!(split_message("aaaa bbbb", 4), vec!["aaaa", "bbbb"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hard_cuts_keep_emoji_whole() {
|
||||
let text = format!("{}😀y", "x".repeat(MESSAGE_LIMIT - 1));
|
||||
let parts = split_message(&text, MESSAGE_LIMIT);
|
||||
assert!(parts.iter().all(|p| utf16_len(p) <= MESSAGE_LIMIT));
|
||||
assert_eq!(parts.concat(), text);
|
||||
assert!(parts[1].starts_with('😀'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_part_fits_for_long_mixed_text() {
|
||||
let text: Vec<String> = (0..900).map(|i| format!("줄 {i} 내용입니다")).collect();
|
||||
let text = text.join("\n");
|
||||
let parts = split_message(&text, MESSAGE_LIMIT);
|
||||
assert!(parts.len() > 1);
|
||||
assert!(parts.iter().all(|p| !p.is_empty() && utf16_len(p) <= MESSAGE_LIMIT));
|
||||
assert_eq!(parts.join("\n"), text);
|
||||
}
|
||||
|
||||
fn ch(id: u64, read: bool, send: bool) -> ChatChannelSettings {
|
||||
ChatChannelSettings { id, name: String::new(), description: String::new(), read, send, edit: true,
|
||||
react: true, delete: false, pin: false, answer_all: false, sync_discord: true }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_action_needs_its_own_switch() {
|
||||
let mut open = ch(1, true, true);
|
||||
open.delete = true;
|
||||
let c = ChatChannels::new(&[open, ch(2, true, false)]);
|
||||
for a in ["send", "edit", "react", "delete"] {
|
||||
assert!(c.check_action(1, a).is_ok(), "{a}");
|
||||
}
|
||||
assert!(c.check_action(1, "pin").is_err() && c.check_action(1, "unpin").is_err());
|
||||
assert!(c.check_action(2, "send").is_err() && c.check_action(2, "delete").is_err());
|
||||
assert!(c.check_action(2, "react").is_ok());
|
||||
assert!(c.check_action(3, "send").unwrap_err().0.contains("not a registered"));
|
||||
assert!(c.check_action(1, "explode").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_channel_follows_its_switches_and_strangers_are_refused() {
|
||||
let c = ChatChannels::new(&[ch(1, true, false), ch(2, false, true)]);
|
||||
assert!(c.can_read(1) && !c.can_send(1) && !c.can_read(2) && c.can_send(2));
|
||||
assert!(c.check_send(1).is_err() && c.check_read(2).is_err());
|
||||
assert!(c.check_read(3).unwrap_err().0.contains("not a registered"));
|
||||
assert!(c.any_readable());
|
||||
assert!(!ChatChannels::new(&[ch(2, false, true)]).any_readable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_content_intent_full_or_limited() {
|
||||
assert!(has_message_content_intent(1 << 18));
|
||||
assert!(has_message_content_intent(1 << 19));
|
||||
assert!(!has_message_content_intent((1 << 15) | (1 << 13)));
|
||||
}
|
||||
}
|
||||
134
app/src/collect.rs
Normal file
134
app/src/collect.rs
Normal file
@@ -0,0 +1,134 @@
|
||||
//! Background collection (`app/src/admin.spec.md`, "Background collection"): the worker's
|
||||
//! collection file, and the collected items for the 백그라운드 수집 screen.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::settings::{valid_collect_name, Settings};
|
||||
|
||||
/// The worker's environment variable with the collection file (`voice_worker/collect.py`).
|
||||
pub const COLLECT_ENV_VAR: &str = "JARVIS_COLLECT_PATH";
|
||||
|
||||
/// Write `config/collect.json` and return it. With the feature off it lists no targets: the worker
|
||||
/// then collects nothing but still deletes day files older than the keep days.
|
||||
pub fn write_config(data: &Path, settings: &Settings) -> std::io::Result<PathBuf> {
|
||||
let file = data.join("config").join("collect.json");
|
||||
let targets: Vec<Value> = settings.collect.targets.iter().filter(|t| t.enabled && settings.features.collect)
|
||||
.map(|t| json!({"name": t.name, "kind": t.kind, "value": t.value, "everyMinutes": t.every_minutes}))
|
||||
.collect();
|
||||
let cfg = json!({"dir": data.join("collect").to_string_lossy(), "keepDays": settings.collect.keep_days,
|
||||
"targets": targets});
|
||||
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
|
||||
text.push('\n');
|
||||
crate::datadir::write_atomic(&file, text.as_bytes())?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// The day files of one target folder, newest first.
|
||||
fn day_files(folder: &Path) -> Vec<PathBuf> {
|
||||
let mut files: Vec<PathBuf> = std::fs::read_dir(folder).into_iter().flatten().flatten().map(|e| e.path())
|
||||
.filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.len() == 16 && n.ends_with(".jsonl")))
|
||||
.collect();
|
||||
files.sort();
|
||||
files.reverse();
|
||||
files
|
||||
}
|
||||
|
||||
/// The target folders that hold results, sorted.
|
||||
pub fn result_targets(data: &Path) -> Vec<String> {
|
||||
let root = data.join("collect");
|
||||
let mut names: Vec<String> = std::fs::read_dir(&root).into_iter().flatten().flatten()
|
||||
.filter(|e| e.path().is_dir())
|
||||
.filter_map(|e| e.file_name().to_str().map(str::to_owned))
|
||||
.filter(|n| valid_collect_name(n) && !day_files(&root.join(n)).is_empty())
|
||||
.collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Collected items newest first, each with its `target`; `target` None reads every target folder.
|
||||
pub fn results(data: &Path, target: Option<&str>, limit: usize) -> Result<Vec<Value>, String> {
|
||||
let root = data.join("collect");
|
||||
let names: Vec<String> = match target {
|
||||
Some(t) if !valid_collect_name(t) => return Err(format!("not a collection target name: {t}")),
|
||||
Some(t) => vec![t.to_owned()],
|
||||
None => result_targets(data),
|
||||
};
|
||||
let mut out: Vec<Value> = Vec::new();
|
||||
for name in names {
|
||||
let mut n = 0;
|
||||
'files: for f in day_files(&root.join(&name)) {
|
||||
let text = std::fs::read_to_string(&f).unwrap_or_default();
|
||||
for line in text.lines().rev() {
|
||||
if let Ok(Value::Object(mut item)) = serde_json::from_str::<Value>(line) {
|
||||
item.insert("target".into(), Value::String(name.clone()));
|
||||
out.push(Value::Object(item));
|
||||
n += 1;
|
||||
if n == limit {
|
||||
break 'files;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// `at` is the local time with its offset in one format, so the text order is the time order.
|
||||
out.sort_by(|a, b| b["at"].as_str().unwrap_or("").cmp(a["at"].as_str().unwrap_or("")));
|
||||
out.truncate(limit);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use crate::settings::parse_settings;
|
||||
|
||||
fn settings(v: Value) -> Settings {
|
||||
parse_settings(&v).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_worker_gets_only_enabled_targets_and_the_keep_days_also_without_targets() {
|
||||
let data = temp_dir("collect-cfg");
|
||||
let s = settings(json!({"collect": {"keepDays": 3, "targets": [
|
||||
{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30},
|
||||
{"name": "b", "kind": "site", "value": "https://b.example", "enabled": false}]}}));
|
||||
let read = |f: &Path| serde_json::from_str::<Value>(&std::fs::read_to_string(f).unwrap()).unwrap();
|
||||
let dir = data.join("collect").to_string_lossy().into_owned();
|
||||
let file = write_config(&data, &s).unwrap();
|
||||
assert_eq!(read(&file), json!({"dir": dir, "keepDays": 3,
|
||||
"targets": [{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30}]}));
|
||||
// Feature off or no targets: nothing to collect, but old results still go after keepDays.
|
||||
let mut off = s.clone();
|
||||
off.features.collect = false;
|
||||
assert_eq!(read(&write_config(&data, &off).unwrap()), json!({"dir": dir, "keepDays": 3, "targets": []}));
|
||||
assert_eq!(read(&write_config(&data, &settings(json!({}))).unwrap()), json!({"dir": dir, "keepDays": 30, "targets": []}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn results_come_newest_first_with_their_target_and_only_from_target_folders() {
|
||||
let data = temp_dir("collect-res");
|
||||
let put = |name: &str, day: &str, lines: &[&str]| {
|
||||
let d = data.join("collect").join(name);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
std::fs::write(d.join(format!("{day}.jsonl")), lines.join("\n") + "\n").unwrap();
|
||||
};
|
||||
put("뉴스", "2026-11-12", &[r#"{"at": "2026-11-12T09:00:00+09:00", "title": "old"}"#]);
|
||||
put("뉴스", "2026-11-13", &[r#"{"at": "2026-11-13T09:00:00+09:00", "title": "n1"}"#, "not json",
|
||||
r#"{"at": "2026-11-13T10:00:00+09:00", "title": "n2"}"#]);
|
||||
put("site", "2026-11-13", &[r#"{"at": "2026-11-13T09:30:00+09:00", "title": "s1"}"#]);
|
||||
std::fs::write(data.join("collect").join("뉴스").join("state.json"), r#"{"at": "9999"}"#).unwrap();
|
||||
let titles = |v: Vec<Value>| v.iter().map(|i| format!("{}:{}", i["target"].as_str().unwrap(),
|
||||
i["title"].as_str().unwrap())).collect::<Vec<_>>();
|
||||
assert_eq!(titles(results(&data, None, 100).unwrap()), ["뉴스:n2", "site:s1", "뉴스:n1", "뉴스:old"]);
|
||||
assert_eq!(titles(results(&data, Some("뉴스"), 2).unwrap()), ["뉴스:n2", "뉴스:n1"]);
|
||||
assert_eq!(titles(results(&data, None, 2).unwrap()), ["뉴스:n2", "site:s1"]);
|
||||
assert!(results(&data, Some("missing"), 10).unwrap().is_empty());
|
||||
// Every target with results is listed, also one whose items are all older than the newest.
|
||||
std::fs::create_dir_all(data.join("collect").join("empty")).unwrap();
|
||||
assert_eq!(result_targets(&data), ["site", "뉴스"]);
|
||||
assert!(results(&data, Some(".."), 10).is_err());
|
||||
assert!(results(&data, Some("../config"), 10).is_err());
|
||||
}
|
||||
}
|
||||
236
app/src/collect_notify.rs
Normal file
236
app/src/collect_notify.rs
Normal file
@@ -0,0 +1,236 @@
|
||||
//! News notifications (`app/src/admin.spec.md`, "News notifications"; `docs/v2/proactive.md` 제안 2):
|
||||
//! new items of the background collection announced in a chat channel, at once or once a day, as
|
||||
//! titles and links only (no LLM, nothing written by Jarvis about them).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{DateTime, Local, Timelike};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::settings::Settings;
|
||||
|
||||
/// Items listed per target in one message; the rest are counted.
|
||||
pub const LISTED: usize = 10;
|
||||
const TITLE_CHARS: usize = 200;
|
||||
pub const TICK: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct State {
|
||||
/// Per target: the `at` of the newest item announced (or seen when its notifications began).
|
||||
#[serde(default)]
|
||||
seen: HashMap<String, String>,
|
||||
/// Per daily target: the day its digest last went out (YYYY-MM-DD).
|
||||
#[serde(default)]
|
||||
digest_day: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Message {
|
||||
pub target: String,
|
||||
pub count: usize,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
fn state_file(data: &Path) -> PathBuf {
|
||||
data.join("config").join("collect-notify.json")
|
||||
}
|
||||
|
||||
pub fn load(data: &Path) -> State {
|
||||
std::fs::read_to_string(state_file(data)).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn save(data: &Path, s: &State) -> std::io::Result<()> {
|
||||
crate::datadir::write_atomic(&state_file(data), serde_json::to_string_pretty(s).unwrap().as_bytes())
|
||||
}
|
||||
|
||||
fn line(item: &Value) -> String {
|
||||
let title: String = item["title"].as_str().unwrap_or("").split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
.chars().take(TITLE_CHARS).collect();
|
||||
let url = item["url"].as_str().unwrap_or("");
|
||||
let title = if title.is_empty() { "(제목 없음)".to_owned() } else { title };
|
||||
// <url> keeps Discord from showing a preview for every link.
|
||||
if url.starts_with("http://") || url.starts_with("https://") { format!("- {title} <{url}>") } else { format!("- {title}") }
|
||||
}
|
||||
|
||||
fn message(target: &str, new: &[Value], daily: bool) -> Message {
|
||||
let head = if daily { format!("수집 소식 하루 요약: {target} 새 글 {}건", new.len()) }
|
||||
else { format!("수집 소식: {target} 새 글 {}건", new.len()) };
|
||||
let mut lines = vec![head];
|
||||
lines.extend(new.iter().take(LISTED).map(line));
|
||||
if new.len() > LISTED {
|
||||
lines.push(format!("외 {}건", new.len() - LISTED));
|
||||
}
|
||||
Message { target: target.to_owned(), count: new.len(), text: lines.join("\n") }
|
||||
}
|
||||
|
||||
/// The messages due at `now`, updating `state`. Nothing without the feature or a channel. A target
|
||||
/// whose notifications just began only marks what it has as seen (its first digest still comes
|
||||
/// that day); `now` targets announce what is new since then; `daily` ones once a day at
|
||||
/// `digest_hour`, what came since their last digest.
|
||||
pub fn due(data: &Path, s: &Settings, now: DateTime<Local>, state: &mut State) -> Vec<Message> {
|
||||
if !s.features.collect || s.collect.notify_channel.is_none() {
|
||||
return vec![];
|
||||
}
|
||||
let today = now.date_naive().to_string();
|
||||
let mut out = vec![];
|
||||
for t in s.collect.targets.iter().filter(|t| t.enabled && t.notify != "off") {
|
||||
let items = crate::collect::results(data, Some(&t.name), 500).unwrap_or_default();
|
||||
let newest = items.first().and_then(|i| i["at"].as_str()).map(str::to_owned);
|
||||
let Some(last) = state.seen.get(&t.name).cloned() else {
|
||||
// Only what is there now is marked seen; that day's digest still goes out at its hour.
|
||||
state.seen.insert(t.name.clone(), newest.unwrap_or_else(|| now.to_rfc3339_opts(chrono::SecondsFormat::Secs, false)));
|
||||
continue;
|
||||
};
|
||||
if t.notify == "daily" && (now.hour() != s.collect.digest_hour || state.digest_day.get(&t.name) == Some(&today)) {
|
||||
continue;
|
||||
}
|
||||
// `at` is the local time with its offset in one format, so text order is time order.
|
||||
let new: Vec<Value> = items.into_iter().filter(|i| i["at"].as_str().is_some_and(|a| a > last.as_str())).collect();
|
||||
if t.notify == "daily" {
|
||||
state.digest_day.insert(t.name.clone(), today.clone());
|
||||
}
|
||||
if new.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(n) = new[0]["at"].as_str() {
|
||||
state.seen.insert(t.name.clone(), n.to_owned());
|
||||
}
|
||||
out.push(message(&t.name, &new, t.notify == "daily"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One round: the messages due now, or None when the updated state could not be saved (they would
|
||||
/// then be sent again every minute; the next round tries again).
|
||||
pub fn prepare(data: &Path, s: &Settings, now: DateTime<Local>) -> Option<Vec<Message>> {
|
||||
let mut state = load(data);
|
||||
let msgs = due(data, s, now, &mut state);
|
||||
match save(data, &state) {
|
||||
Ok(()) => Some(msgs),
|
||||
Err(e) => {
|
||||
crate::warn_log!("conversation", "Warning: could not save the news notification state, nothing sent: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Announce due news every minute until aborted; each post is recorded in the activity log.
|
||||
pub async fn run(data: PathBuf, settings: Settings, d: std::sync::Arc<dyn crate::schedule::Deliver>,
|
||||
activity: std::sync::Arc<crate::activity::Activity>) {
|
||||
loop {
|
||||
let msgs = prepare(&data, &settings, Local::now()).unwrap_or_default();
|
||||
if let Some(channel) = settings.collect.notify_channel {
|
||||
for m in msgs {
|
||||
let posted = d.post(channel, &m.text).await;
|
||||
crate::info_log!("conversation", "news {} items={} posted={}", m.target, m.count, posted.is_ok());
|
||||
activity.record("collect.notify", &format!("수집 {} → 채널 {channel}", m.target), None, posted.is_ok(),
|
||||
posted.err().map(|_| "채팅 채널에 보내지 못함".to_owned()));
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(TICK).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::TimeZone;
|
||||
use serde_json::json;
|
||||
|
||||
fn settings(notify: &str) -> Settings {
|
||||
crate::settings::parse_settings(&json!({
|
||||
"chatChannels": [{"id": "123456789012345678"}],
|
||||
"collect": {"notifyChannel": "123456789012345678", "digestHour": 9, "targets": [
|
||||
{"name": "뉴스", "kind": "rss", "value": "https://feed.example/rss", "notify": notify}]}})).unwrap()
|
||||
}
|
||||
|
||||
fn write(data: &Path, day: &str, items: &[(&str, &str)]) {
|
||||
let dir = data.join("collect").join("뉴스");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let mut text = std::fs::read_to_string(dir.join(format!("{day}.jsonl"))).unwrap_or_default();
|
||||
for (at, title) in items {
|
||||
text.push_str(&json!({"at": at, "title": title, "url": format!("https://e.example/{title}")}).to_string());
|
||||
text.push('\n');
|
||||
}
|
||||
std::fs::write(dir.join(format!("{day}.jsonl")), text).unwrap();
|
||||
}
|
||||
|
||||
fn at(h: u32, m: u32) -> DateTime<Local> {
|
||||
Local.with_ymd_and_hms(2026, 11, 14, h, m, 0).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_once_announces_only_what_is_new_since_notifications_began() {
|
||||
let data = crate::datadir::temp_dir("news-now");
|
||||
let s = settings("now");
|
||||
write(&data, "2026-11-14", &[("2026-11-14T08:00:00+09:00", "old")]);
|
||||
let mut st = State::default();
|
||||
assert!(due(&data, &s, at(8, 30), &mut st).is_empty(), "what was there before is not announced");
|
||||
write(&data, "2026-11-14", &[("2026-11-14T08:40:00+09:00", "new one")]);
|
||||
let m = due(&data, &s, at(8, 41), &mut st);
|
||||
assert_eq!(m.len(), 1);
|
||||
assert_eq!(m[0].count, 1);
|
||||
assert!(m[0].text.contains("new one <https://e.example/new one>") && !m[0].text.contains("old"), "{}", m[0].text);
|
||||
assert!(due(&data, &s, at(8, 42), &mut st).is_empty(), "each item once");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_daily_digest_goes_out_once_at_its_hour_with_everything_since_the_last() {
|
||||
let data = crate::datadir::temp_dir("news-daily");
|
||||
let s = settings("daily");
|
||||
let mut st = State::default();
|
||||
// Turned on at 7: what is there is seen; what comes later goes in that same day's digest.
|
||||
assert!(due(&data, &s, at(7, 0), &mut st).is_empty());
|
||||
let many: Vec<(String, String)> = (0..12).map(|i| (format!("2026-11-14T08:{i:02}:00+09:00"), format!("t{i}"))).collect();
|
||||
write(&data, "2026-11-14", &many.iter().map(|(a, t)| (a.as_str(), t.as_str())).collect::<Vec<_>>());
|
||||
assert!(due(&data, &s, at(8, 59), &mut st).is_empty(), "not before its hour");
|
||||
let m = due(&data, &s, at(9, 0), &mut st);
|
||||
assert_eq!(m.len(), 1);
|
||||
assert_eq!(m[0].count, 12);
|
||||
assert!(m[0].text.contains("외 2건"), "{}", m[0].text);
|
||||
assert!(due(&data, &s, at(9, 30), &mut st).is_empty(), "once a day");
|
||||
// The next day only what came after.
|
||||
write(&data, "2026-11-15", &[("2026-11-15T06:00:00+09:00", "next day")]);
|
||||
let next = Local.with_ymd_and_hms(2026, 11, 15, 9, 0, 0).unwrap();
|
||||
let m = due(&data, &s, next, &mut st);
|
||||
assert_eq!(m.len(), 1);
|
||||
assert_eq!(m[0].count, 1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_round_whose_state_cannot_be_saved_sends_nothing() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let data = crate::datadir::temp_dir("news-readonly");
|
||||
let s = settings("now");
|
||||
write(&data, "2026-11-14", &[("2026-11-14T08:00:00+09:00", "old")]);
|
||||
assert_eq!(prepare(&data, &s, at(8, 0)), Some(vec![]));
|
||||
write(&data, "2026-11-14", &[("2026-11-14T08:30:00+09:00", "new")]);
|
||||
let config = data.join("config");
|
||||
std::fs::set_permissions(&config, std::fs::Permissions::from_mode(0o555)).unwrap();
|
||||
let blocked = prepare(&data, &s, at(8, 31));
|
||||
std::fs::set_permissions(&config, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
assert_eq!(blocked, None, "not sent: it would be sent again every minute");
|
||||
assert_eq!(prepare(&data, &s, at(8, 32)).map(|m| m.len()), Some(1), "sent once the state can be saved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_is_announced_without_a_channel_the_feature_or_with_notify_off() {
|
||||
let data = crate::datadir::temp_dir("news-off");
|
||||
write(&data, "2026-11-14", &[("2026-11-14T08:00:00+09:00", "x")]);
|
||||
let mut off = settings("now");
|
||||
off.collect.notify_channel = None;
|
||||
let mut nofeature = settings("now");
|
||||
nofeature.features.collect = false;
|
||||
for s in [off, nofeature, settings("off")] {
|
||||
let mut st = State::default();
|
||||
due(&data, &s, at(8, 0), &mut st);
|
||||
write(&data, "2026-11-14", &[("2026-11-14T08:30:00+09:00", "y")]);
|
||||
assert!(due(&data, &s, at(8, 31), &mut st).is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
676
app/src/control.rs
Normal file
676
app/src/control.rs
Normal file
@@ -0,0 +1,676 @@
|
||||
//! Starting and stopping the services, and the dashboard snapshot (`app/src/admin.spec.md`,
|
||||
//! "Service control" and "Dashboard data").
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::dashboard::Dashboard;
|
||||
use crate::discord::CoreEvent;
|
||||
use crate::system::Usage;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum State {
|
||||
Stopped,
|
||||
Starting,
|
||||
Running,
|
||||
Stopping,
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// The running services as the controller sees them.
|
||||
#[async_trait]
|
||||
pub trait Running: Send + Sync + 'static {
|
||||
/// Stop everything; an error means something may still run (the LLM's server still answers).
|
||||
async fn stop(&self) -> Result<(), String>;
|
||||
/// Worker, voice channel members and missed requests, for the snapshot.
|
||||
fn status(&self, now: Instant) -> Value;
|
||||
/// Speak at this speed (`voice.tts.speed`) from the next sentence on.
|
||||
fn set_tts_speed(&self, _speed: f64) {}
|
||||
}
|
||||
|
||||
/// What starts the services (the real ones, or a fake in tests).
|
||||
#[async_trait]
|
||||
pub trait Host: Send + Sync + 'static {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String>;
|
||||
/// Stored secrets, masked in error messages.
|
||||
fn secrets(&self) -> Vec<String>;
|
||||
/// Called when a start is accepted (state `starting`, under the controller's lock): a fresh
|
||||
/// cancel, so a stop from now on cancels this start and an earlier stop does not.
|
||||
fn prepare_start(&self) {}
|
||||
/// Stop a start in progress (the LLM's first start downloads the model); `start` then returns
|
||||
/// `Err("cancelled")` if it had not finished.
|
||||
fn cancel_start(&self) {}
|
||||
/// The LLM's state for the dashboard, also while starting.
|
||||
fn llm(&self) -> Value {
|
||||
Value::Null
|
||||
}
|
||||
/// After a failed run (a start or a stop that could not clear everything): stop what may be left.
|
||||
async fn cleanup(&self) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
state: State,
|
||||
error: Option<String>,
|
||||
running: Option<Arc<dyn Running>>,
|
||||
/// The running services' dashboard and the task feeding it their events. Each start gets a new
|
||||
/// one, so an event of an earlier run can never reach a later dashboard.
|
||||
dashboard: Arc<Mutex<Dashboard>>,
|
||||
feeder: Option<tokio::task::JoinHandle<()>>,
|
||||
/// Stop was asked for while starting.
|
||||
stop_asked: bool,
|
||||
/// The latest speech speed saved (equal to the settings file's): a start may have read the
|
||||
/// settings before it was saved, so the services get it again once they run.
|
||||
tts_speed: Option<f64>,
|
||||
/// When the latest start began (Unix seconds): the 대화 screen's current session.
|
||||
session: Option<f64>,
|
||||
}
|
||||
|
||||
pub struct Controller {
|
||||
host: Box<dyn Host>,
|
||||
inner: Mutex<Inner>,
|
||||
usage: tokio::sync::Mutex<Option<(Instant, Usage)>>,
|
||||
}
|
||||
|
||||
impl Controller {
|
||||
pub fn new(host: impl Host) -> Arc<Self> {
|
||||
Arc::new(Self { host: Box::new(host), inner: Mutex::new(Inner { state: State::Stopped, error: None, running: None,
|
||||
dashboard: Arc::default(), feeder: None, stop_asked: false, tts_speed: None, session: None }), usage: tokio::sync::Mutex::new(None) })
|
||||
}
|
||||
|
||||
pub fn state(&self) -> State {
|
||||
self.inner.lock().unwrap().state
|
||||
}
|
||||
|
||||
/// When the latest start began (Unix seconds), none before the first start of this app run.
|
||||
pub fn session_start(&self) -> Option<f64> {
|
||||
self.inner.lock().unwrap().session
|
||||
}
|
||||
|
||||
/// Start from stopped or failed; a failed start leaves nothing running.
|
||||
pub async fn start(&self) -> Result<(), String> {
|
||||
self.begin_start()?;
|
||||
self.run_start().await
|
||||
}
|
||||
|
||||
/// The same in the background: the state is `starting` when this returns, so a stop right after
|
||||
/// it cancels the start and a second start is refused.
|
||||
pub fn start_in_background(self: &Arc<Self>) -> Result<(), String> {
|
||||
self.begin_start()?;
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = me.run_start().await;
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn begin_start(&self) -> Result<(), String> {
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
if !matches!(i.state, State::Stopped | State::Failed) {
|
||||
return Err(format!("cannot start while {:?}", i.state).to_lowercase());
|
||||
}
|
||||
i.state = State::Starting;
|
||||
i.error = None;
|
||||
i.stop_asked = false;
|
||||
i.session = Some(chrono::Utc::now().timestamp_millis() as f64 / 1000.0);
|
||||
self.host.prepare_start();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_start(&self) -> Result<(), String> {
|
||||
crate::info_log!("app", "starting the services");
|
||||
let started = self.host.start().await;
|
||||
// The stop flag is read under the same lock that makes the state Running, so a stop asked
|
||||
// for while starting is never lost.
|
||||
let started = {
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
if std::mem::take(&mut i.stop_asked) {
|
||||
i.state = State::Stopping;
|
||||
started
|
||||
} else {
|
||||
return match started {
|
||||
Ok((running, mut events)) => {
|
||||
let dash: Arc<Mutex<Dashboard>> = Arc::default();
|
||||
let d2 = dash.clone();
|
||||
let feeder = tokio::spawn(async move {
|
||||
while let Some(ev) = events.recv().await {
|
||||
d2.lock().unwrap().on_event(&ev, Instant::now());
|
||||
}
|
||||
});
|
||||
if let Some(speed) = i.tts_speed {
|
||||
running.set_tts_speed(speed);
|
||||
}
|
||||
i.state = State::Running;
|
||||
i.running = Some(running);
|
||||
i.dashboard = dash;
|
||||
i.feeder = Some(feeder);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
let e = crate::logs::redact(&e, &self.host.secrets());
|
||||
crate::error_log!("app", "the services did not start: {e}");
|
||||
i.state = State::Failed;
|
||||
i.error = Some(e.clone());
|
||||
Err(e)
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
// Stopped while starting: whatever came up goes down again. If that fails, or the cancelled
|
||||
// start could not clear the LLM's server, the state says so.
|
||||
let left = match started {
|
||||
Ok((running, _)) => running.stop().await.err(),
|
||||
Err(e) if self.host.llm()["state"] == "failed" => Some(e),
|
||||
Err(_) => None,
|
||||
};
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
if let Some(e) = left {
|
||||
let e = crate::logs::redact(&e, &self.host.secrets());
|
||||
crate::error_log!("app", "start cancelled, but: {e}");
|
||||
i.state = State::Failed;
|
||||
i.error = Some(e);
|
||||
} else {
|
||||
i.state = State::Stopped;
|
||||
crate::info_log!("app", "start cancelled");
|
||||
}
|
||||
Err("cancelled".into())
|
||||
}
|
||||
|
||||
/// Give the running services a new speech speed. False when nothing runs yet; a start in
|
||||
/// progress passes it on once the services run.
|
||||
pub fn set_tts_speed(&self, speed: f64) -> bool {
|
||||
let running = {
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
i.tts_speed = Some(speed);
|
||||
i.running.clone()
|
||||
};
|
||||
running.map(|r| r.set_tts_speed(speed)).is_some()
|
||||
}
|
||||
|
||||
/// Stop from running, or cancel a start; what was said is forgotten.
|
||||
pub async fn stop(&self) -> Result<(), String> {
|
||||
let running = {
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
if i.state == State::Starting {
|
||||
i.stop_asked = true;
|
||||
drop(i);
|
||||
self.host.cancel_start();
|
||||
return Ok(());
|
||||
}
|
||||
if i.state != State::Running {
|
||||
return Err(format!("cannot stop while {:?}", i.state).to_lowercase());
|
||||
}
|
||||
i.state = State::Stopping;
|
||||
i.running.take()
|
||||
};
|
||||
let stopped = match running {
|
||||
Some(r) => r.stop().await,
|
||||
None => Ok(()),
|
||||
};
|
||||
// Stop feeding before forgetting: nothing still queued may reach the screen afterwards.
|
||||
let feeder = self.inner.lock().unwrap().feeder.take();
|
||||
if let Some(f) = feeder {
|
||||
f.abort();
|
||||
let _ = f.await;
|
||||
}
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
i.dashboard = Arc::default();
|
||||
match stopped {
|
||||
Ok(()) => {
|
||||
i.state = State::Stopped;
|
||||
drop(i);
|
||||
crate::info_log!("app", "services stopped");
|
||||
Ok(())
|
||||
}
|
||||
// Something may still run: shown as failed with the reason; a start (Strata's
|
||||
// `--restart`) or another stop attempt can follow.
|
||||
Err(e) => {
|
||||
let e = crate::logs::redact(&e, &self.host.secrets());
|
||||
i.state = State::Failed;
|
||||
i.error = Some(e.clone());
|
||||
drop(i);
|
||||
crate::error_log!("app", "the services did not stop cleanly: {e}");
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The app is closing: stop what runs (a start in progress is cancelled and waited for, up to
|
||||
/// 150 s), then, if the state is failed, clear what may be left. Ok means nothing is left running.
|
||||
pub async fn shutdown(&self) -> Result<(), String> {
|
||||
if matches!(self.state(), State::Running | State::Starting) {
|
||||
let _ = self.stop().await;
|
||||
}
|
||||
let until = Instant::now() + Duration::from_secs(150);
|
||||
while matches!(self.state(), State::Starting | State::Stopping) {
|
||||
if Instant::now() >= until {
|
||||
return Err("서비스가 아직 멈추지 않았습니다".into());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
if self.state() != State::Failed {
|
||||
return Ok(());
|
||||
}
|
||||
match self.host.cleanup().await {
|
||||
Ok(()) => {
|
||||
let mut i = self.inner.lock().unwrap();
|
||||
i.state = State::Stopped;
|
||||
i.error = None;
|
||||
drop(i);
|
||||
crate::info_log!("app", "cleared what the failed run left");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
let e = crate::logs::redact(&e, &self.host.secrets());
|
||||
crate::error_log!("app", "could not clear what the failed run left: {e}");
|
||||
self.inner.lock().unwrap().error = Some(e.clone());
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn restart(&self) -> Result<(), String> {
|
||||
if self.state() == State::Running {
|
||||
self.stop().await?;
|
||||
}
|
||||
self.start().await
|
||||
}
|
||||
|
||||
/// Everything the dashboard shows.
|
||||
pub async fn snapshot(&self) -> Value {
|
||||
let now = Instant::now();
|
||||
let (state, error, running, dashboard) = {
|
||||
let i = self.inner.lock().unwrap();
|
||||
(i.state, i.error.clone(), i.running.clone(), i.dashboard.clone())
|
||||
};
|
||||
let status = running.map(|r| r.status(now)).unwrap_or_else(|| json!({"worker": null, "voiceMembers": [],
|
||||
"missed": []}));
|
||||
let mut view = dashboard.lock().unwrap().view(&[], now);
|
||||
view["missed"] = status["missed"].clone();
|
||||
view["worker"] = status["worker"].clone();
|
||||
view["voiceMembers"] = status["voiceMembers"].clone();
|
||||
view["browser"] = status.get("browser").cloned().unwrap_or_else(|| json!({"state": "off", "note": null}));
|
||||
view["state"] = json!(state);
|
||||
view["error"] = json!(error);
|
||||
view["system"] = json!(self.system().await);
|
||||
view["llm"] = self.host.llm();
|
||||
view["game"] = status.get("game").cloned().unwrap_or(Value::Null);
|
||||
view["performance"] = status.get("performance").cloned().unwrap_or(Value::Null);
|
||||
view
|
||||
}
|
||||
|
||||
/// Memory and GPU use, sampled at most once a second.
|
||||
async fn system(&self) -> Usage {
|
||||
let mut cache = self.usage.lock().await;
|
||||
if let Some((at, u)) = cache.as_ref() {
|
||||
if at.elapsed() < Duration::from_secs(1) {
|
||||
return u.clone();
|
||||
}
|
||||
}
|
||||
let u = crate::system::usage().await;
|
||||
*cache = Some((Instant::now(), u.clone()));
|
||||
u
|
||||
}
|
||||
}
|
||||
|
||||
/// The real services from a data folder.
|
||||
pub struct ServicesHost {
|
||||
data_dir: PathBuf,
|
||||
llm: Arc<crate::strata::Llm>,
|
||||
}
|
||||
|
||||
impl ServicesHost {
|
||||
pub fn new(data_dir: PathBuf) -> Self {
|
||||
let llm = Arc::new(crate::strata::Llm::new(&data_dir));
|
||||
Self { data_dir, llm }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Host for ServicesHost {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String> {
|
||||
let mut s = crate::services::start_services(&self.data_dir, self.llm.clone()).await.map_err(|e| e.to_string())?;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let mut events = std::mem::replace(&mut s.events, mpsc::unbounded_channel().1);
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = events.recv().await {
|
||||
if tx.send(e).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok((Arc::new(s), rx))
|
||||
}
|
||||
|
||||
fn secrets(&self) -> Vec<String> {
|
||||
crate::secrets::Secrets::new(&self.data_dir.join("config").join("secrets.dat")).values()
|
||||
}
|
||||
|
||||
fn prepare_start(&self) {
|
||||
self.llm.arm();
|
||||
}
|
||||
|
||||
fn cancel_start(&self) {
|
||||
self.llm.cancel();
|
||||
}
|
||||
|
||||
fn llm(&self) -> Value {
|
||||
json!(self.llm.status())
|
||||
}
|
||||
|
||||
async fn cleanup(&self) -> Result<(), String> {
|
||||
if self.llm.status().state == crate::strata::LlmState::Failed {
|
||||
self.llm.stop().await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Running for crate::services::Services {
|
||||
async fn stop(&self) -> Result<(), String> {
|
||||
crate::services::Services::stop(self).await
|
||||
}
|
||||
|
||||
fn status(&self, now: Instant) -> Value {
|
||||
let hello = self.worker.hello();
|
||||
let missed: Vec<_> = self.missed.lock().unwrap().iter().cloned().collect();
|
||||
json!({
|
||||
"worker": {"connected": hello.is_some(), "sttReady": hello.as_ref().map(|h| h.stt_ready),
|
||||
"ttsReady": hello.as_ref().map(|h| h.tts_ready), "restarts": self.worker_process.restarts()},
|
||||
"voiceMembers": self.core.voice_members().into_iter()
|
||||
.map(|(id, name)| json!({"id": id.to_string(), "name": name})).collect::<Vec<_>>(),
|
||||
"missed": crate::dashboard::missed_view(&missed, now),
|
||||
"browser": self.browser.view(),
|
||||
"game": self.game.view(),
|
||||
"performance": self.llm.stage_view(),
|
||||
})
|
||||
}
|
||||
|
||||
fn set_tts_speed(&self, speed: f64) {
|
||||
self.worker.set_tts_speed(speed);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Counts {
|
||||
starts: AtomicU32,
|
||||
stops: AtomicU32,
|
||||
stop_fails: std::sync::atomic::AtomicBool,
|
||||
cleanups: AtomicU32,
|
||||
cleanup_fails: std::sync::atomic::AtomicBool,
|
||||
speeds: Mutex<Vec<f64>>,
|
||||
}
|
||||
|
||||
struct FakeRunning(Arc<Counts>);
|
||||
#[async_trait]
|
||||
impl Running for FakeRunning {
|
||||
async fn stop(&self) -> Result<(), String> {
|
||||
self.0.stops.fetch_add(1, Ordering::SeqCst);
|
||||
if self.0.stop_fails.load(Ordering::SeqCst) { Err("Strata를 끄지 못했습니다: still answering".into()) } else { Ok(()) }
|
||||
}
|
||||
fn status(&self, _: Instant) -> Value {
|
||||
json!({"worker": {"connected": true}, "voiceMembers": [{"id": "11", "name": "민수"}], "missed": []})
|
||||
}
|
||||
fn set_tts_speed(&self, speed: f64) {
|
||||
self.0.speeds.lock().unwrap().push(speed);
|
||||
}
|
||||
}
|
||||
|
||||
struct FakeHost {
|
||||
counts: Arc<Counts>,
|
||||
fail_first: AtomicU32,
|
||||
events: Mutex<Option<mpsc::UnboundedSender<CoreEvent>>>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl Host for FakeHost {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String> {
|
||||
self.counts.starts.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
if self.fail_first.load(Ordering::SeqCst) > 0 {
|
||||
self.fail_first.fetch_sub(1, Ordering::SeqCst);
|
||||
return Err("login failed for token Mt0kenSecretValue".into());
|
||||
}
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
*self.events.lock().unwrap() = Some(tx);
|
||||
Ok((Arc::new(FakeRunning(self.counts.clone())), rx))
|
||||
}
|
||||
fn secrets(&self) -> Vec<String> {
|
||||
vec!["Mt0kenSecretValue".into()]
|
||||
}
|
||||
}
|
||||
|
||||
fn controller(fail_first: u32) -> (Arc<Controller>, Arc<Counts>, Arc<FakeHost>) {
|
||||
let counts = Arc::new(Counts::default());
|
||||
let host = Arc::new(FakeHost { counts: counts.clone(), fail_first: AtomicU32::new(fail_first),
|
||||
events: Mutex::new(None) });
|
||||
struct Shared(Arc<FakeHost>);
|
||||
#[async_trait]
|
||||
impl Host for Shared {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String> {
|
||||
self.0.start().await
|
||||
}
|
||||
fn secrets(&self) -> Vec<String> {
|
||||
self.0.secrets()
|
||||
}
|
||||
async fn cleanup(&self) -> Result<(), String> {
|
||||
self.0.counts.cleanups.fetch_add(1, Ordering::SeqCst);
|
||||
if self.0.counts.cleanup_fails.load(Ordering::SeqCst) { Err("Strata를 끄지 못했습니다".into()) } else { Ok(()) }
|
||||
}
|
||||
}
|
||||
(Controller::new(Shared(host.clone())), counts, host)
|
||||
}
|
||||
|
||||
/// A start that takes a while (the LLM's first start) and honours a cancel or ignores it.
|
||||
struct SlowHost {
|
||||
counts: Arc<Counts>,
|
||||
honours_cancel: bool,
|
||||
cancelled: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl Host for SlowHost {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String> {
|
||||
self.counts.starts.fetch_add(1, Ordering::SeqCst);
|
||||
for _ in 0..50 {
|
||||
if self.honours_cancel && self.cancelled.load(Ordering::SeqCst) {
|
||||
return Err("cancelled".into());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Ok((Arc::new(FakeRunning(self.counts.clone())), mpsc::unbounded_channel().1))
|
||||
}
|
||||
fn secrets(&self) -> Vec<String> {
|
||||
vec![]
|
||||
}
|
||||
fn prepare_start(&self) {
|
||||
self.cancelled.store(false, Ordering::SeqCst); // as the LLM does: armed when a start is accepted
|
||||
}
|
||||
fn cancel_start(&self) {
|
||||
self.cancelled.store(true, Ordering::SeqCst);
|
||||
}
|
||||
fn llm(&self) -> Value {
|
||||
json!({"state": "starting", "percent": 12.5})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_new_speech_speed_reaches_the_running_services_without_a_restart() {
|
||||
let counts = Arc::new(Counts::default());
|
||||
let c = Controller::new(FakeHost { counts: counts.clone(), fail_first: AtomicU32::new(0), events: Mutex::new(None) });
|
||||
assert!(!c.set_tts_speed(1.7), "nothing runs: the next start reads the settings");
|
||||
c.start().await.unwrap();
|
||||
assert!(c.set_tts_speed(1.8));
|
||||
assert_eq!(counts.speeds.lock().unwrap().last(), Some(&1.8), "the latest speed is the one they use");
|
||||
assert_eq!(counts.starts.load(Ordering::SeqCst), 1, "no restart");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_speech_speed_saved_while_starting_reaches_the_services_once_they_run() {
|
||||
// The start read the settings before the save, so the new speed must follow it.
|
||||
let counts = Arc::new(Counts::default());
|
||||
let c = Controller::new(FakeHost { counts: counts.clone(), fail_first: AtomicU32::new(0), events: Mutex::new(None) });
|
||||
c.start_in_background().unwrap();
|
||||
assert_eq!(c.state(), State::Starting);
|
||||
assert!(!c.set_tts_speed(1.9), "not running yet");
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while c.state() != State::Running { tokio::time::sleep(Duration::from_millis(5)).await; }
|
||||
}).await.unwrap();
|
||||
assert_eq!(*counts.speeds.lock().unwrap(), [1.9]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_stop_while_starting_cancels_the_start_or_stops_what_came_up() {
|
||||
for honours_cancel in [true, false] {
|
||||
let counts = Arc::new(Counts::default());
|
||||
let c = Controller::new(SlowHost { counts: counts.clone(), honours_cancel, cancelled: Arc::default() });
|
||||
let c2 = c.clone();
|
||||
let start = tokio::spawn(async move { c2.start().await });
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert_eq!(c.snapshot().await["llm"]["percent"], json!(12.5), "progress shows while starting");
|
||||
c.stop().await.unwrap();
|
||||
assert_eq!(start.await.unwrap().unwrap_err(), "cancelled");
|
||||
let s = c.snapshot().await;
|
||||
assert_eq!((s["state"].clone(), s["error"].clone()), (json!("stopped"), Value::Null));
|
||||
// What came up anyway was stopped; a cancelled start left nothing to stop.
|
||||
assert_eq!(counts.stops.load(Ordering::SeqCst), if honours_cancel { 0 } else { 1 });
|
||||
c.start().await.unwrap();
|
||||
assert_eq!(c.state(), State::Running, "it can start again");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_background_start_is_starting_at_once_so_an_immediate_stop_or_second_start_is_handled() {
|
||||
let counts = Arc::new(Counts::default());
|
||||
let c = Controller::new(SlowHost { counts: counts.clone(), honours_cancel: true, cancelled: Arc::default() });
|
||||
c.start_in_background().unwrap();
|
||||
assert_eq!(c.state(), State::Starting, "starting before anything else runs");
|
||||
assert!(c.start_in_background().unwrap_err().contains("starting"), "no second start");
|
||||
c.stop().await.unwrap();
|
||||
for _ in 0..100 {
|
||||
if c.state() == State::Stopped { break; }
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
assert_eq!(c.state(), State::Stopped);
|
||||
assert_eq!(counts.starts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_stop_that_fails_is_reported_and_a_new_start_is_allowed() {
|
||||
let (c, counts, _) = controller(0);
|
||||
c.start().await.unwrap();
|
||||
counts.stop_fails.store(true, Ordering::SeqCst);
|
||||
let e = c.stop().await.unwrap_err();
|
||||
assert!(e.contains("Strata"), "{e}");
|
||||
let s = c.snapshot().await;
|
||||
assert_eq!((s["state"].clone(), s["error"].clone()), (json!("failed"), json!(e)));
|
||||
counts.stop_fails.store(false, Ordering::SeqCst);
|
||||
c.start().await.unwrap();
|
||||
assert_eq!(c.state(), State::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closing_the_app_after_a_failed_stop_clears_what_is_left_or_says_it_could_not() {
|
||||
let (c, counts, _) = controller(0);
|
||||
c.shutdown().await.unwrap(); // nothing ran: nothing to do
|
||||
assert_eq!(counts.cleanups.load(Ordering::SeqCst), 0);
|
||||
c.start().await.unwrap();
|
||||
counts.stop_fails.store(true, Ordering::SeqCst);
|
||||
counts.cleanup_fails.store(true, Ordering::SeqCst);
|
||||
assert!(c.shutdown().await.is_err(), "the stop and the clean-up both failed");
|
||||
assert_eq!(c.state(), State::Failed);
|
||||
counts.cleanup_fails.store(false, Ordering::SeqCst);
|
||||
c.shutdown().await.unwrap();
|
||||
let s = c.snapshot().await;
|
||||
assert_eq!((s["state"].clone(), s["error"].clone()), (json!("stopped"), Value::Null));
|
||||
assert_eq!(counts.cleanups.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn start_and_stop_follow_the_state_rules() {
|
||||
let (c, counts, _) = controller(0);
|
||||
assert!(c.stop().await.is_err(), "nothing to stop");
|
||||
let c2 = c.clone();
|
||||
let first = tokio::spawn(async move { c2.start().await });
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
assert_eq!(c.state(), State::Starting);
|
||||
assert!(c.start().await.unwrap_err().contains("starting"), "one start at a time");
|
||||
first.await.unwrap().unwrap();
|
||||
assert_eq!(c.state(), State::Running);
|
||||
assert!(c.start().await.is_err());
|
||||
c.restart().await.unwrap();
|
||||
assert_eq!((counts.starts.load(Ordering::SeqCst), counts.stops.load(Ordering::SeqCst)), (2, 1));
|
||||
c.stop().await.unwrap();
|
||||
assert_eq!(c.state(), State::Stopped);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_start_is_reported_masked_and_can_be_retried() {
|
||||
let (c, counts, _) = controller(1);
|
||||
let e = c.start().await.unwrap_err();
|
||||
assert_eq!(e, "login failed for token [비밀값]");
|
||||
let snap = c.snapshot().await;
|
||||
assert_eq!((snap["state"].clone(), snap["error"].clone()), (json!("failed"), json!(e)));
|
||||
c.start().await.unwrap();
|
||||
assert_eq!(c.state(), State::Running);
|
||||
assert_eq!(counts.stops.load(Ordering::SeqCst), 0, "a failed start leaves nothing to stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn events_queued_before_a_stop_or_sent_by_an_earlier_run_never_reach_the_screen() {
|
||||
let (c, _, host) = controller(0);
|
||||
c.start().await.unwrap();
|
||||
let old = host.events.lock().unwrap().clone().unwrap();
|
||||
// Queued right before the stop, never handled before it.
|
||||
old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap();
|
||||
old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) },
|
||||
"자비스".into())).unwrap();
|
||||
c.stop().await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
let s = c.snapshot().await;
|
||||
assert_eq!((s["state"].clone(), s["voiceChannel"].clone(), s["captions"].clone()),
|
||||
(json!("stopped"), Value::Null, json!([])));
|
||||
// After a new start, the earlier run's sender still reaches nothing.
|
||||
c.start().await.unwrap();
|
||||
let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78, session: 2 });
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
assert_eq!(c.snapshot().await["voiceChannel"], Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_snapshot_carries_the_events_and_stopping_forgets_what_was_said() {
|
||||
let (c, _, host) = controller(0);
|
||||
c.start().await.unwrap();
|
||||
let tx = host.events.lock().unwrap().clone().unwrap();
|
||||
tx.send(CoreEvent::Ready { user_id: 1, tag: "Jarvis#0001".into() }).unwrap();
|
||||
tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap();
|
||||
tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) },
|
||||
"자비스 안녕".into())).unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let s = c.snapshot().await;
|
||||
assert_eq!(s["state"], json!("running"));
|
||||
assert_eq!(s["discord"], json!("Jarvis#0001"));
|
||||
assert_eq!(s["voiceChannel"], json!("77"));
|
||||
assert_eq!(s["captions"][0]["text"], json!("자비스 안녕"));
|
||||
assert_eq!(s["voiceMembers"][0]["name"], json!("민수"));
|
||||
assert_eq!(s["worker"]["connected"], json!(true));
|
||||
assert!(s["system"]["ramTotalMb"].is_u64() || !cfg!(any(target_os = "linux", windows)));
|
||||
assert_eq!(s["performance"], Value::Null);
|
||||
c.stop().await.unwrap();
|
||||
let s = c.snapshot().await;
|
||||
assert_eq!(s["captions"], json!([]));
|
||||
assert_eq!(s["worker"], Value::Null);
|
||||
}
|
||||
}
|
||||
1019
app/src/conversation.rs
Normal file
1019
app/src/conversation.rs
Normal file
File diff suppressed because it is too large
Load Diff
221
app/src/conversation.spec.md
Normal file
221
app/src/conversation.spec.md
Normal file
@@ -0,0 +1,221 @@
|
||||
# Conversation spec: call detection and response policies (v2 M4)
|
||||
|
||||
The part of the app that decides which finished utterances (and, later, chat messages) Jarvis answers
|
||||
and in what order, and when an answer that is being spoken stops. It sits between the Discord voice
|
||||
core (`discord_core.spec.md`), which reports what was heard and plays speech, and the voice worker's
|
||||
brain (`src/voice_worker/voice_worker.spec.md`, `reply`), which writes the answer. The user-facing
|
||||
design is `docs/v2/conversation.md`; this spec is the contract the code follows.
|
||||
|
||||
## Principles
|
||||
|
||||
- **Plain types, tested without Discord.** Call detection (`app/src/calls.rs`) is a pure function of
|
||||
the settings, the speaker and the text. The response policy (`app/src/policy.rs`) is a state machine
|
||||
driven by events with the time passed in, so every rule is tested with fake times.
|
||||
- **Language-agnostic.** Nothing matches a language's words in code. Wake words and stop words are
|
||||
settings; matching works on Unicode-normalised letters, so it behaves the same in any script.
|
||||
- **No text in logs.** Logs carry speaker IDs, decisions, reasons, counts and timings, never what was
|
||||
said.
|
||||
- **Nothing is dropped silently.** Every request the policy does not answer, and every answer it stops,
|
||||
is reported as missed with a reason (the dashboard's "놓친 요청" list, M5).
|
||||
|
||||
## Call detection
|
||||
|
||||
Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`):
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `voice.wakeWords` | `["자비스"]` | Wake words for voice (also the STT hint). |
|
||||
| `call.chatWakeWords` | `null` | Wake words for chat; `null` uses `voice.wakeWords`. |
|
||||
| `call.similarity` | `0.2` | How far a heard word may be from a wake word, as a share of the wake word's normalised letters (0-0.5). `0` needs an exact match. |
|
||||
| `call.mode` | `"wakeWord"` | `wakeWord`: answer only when called by a wake word. `addressed`: also answer what is judged to be said to Jarvis. `everything`: answer every utterance. |
|
||||
| `call.designated` | `[]` | Discord user IDs of the designated people. |
|
||||
| `call.designatedOnly` | `false` | Answer only designated people. |
|
||||
| `call.designatedNoWakeWord` | `false` | Answer designated people without a wake word. |
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
away from "자비스" (of six) while "서비스" is two away.
|
||||
|
||||
A wake word is heard when some stretch of the normalised text is at most
|
||||
`floor(similarity × letters of the wake word)` edits (insert, delete, substitute) from it. With the
|
||||
default 0.2, "자비스" (six letters) allows one edit. A wake word anywhere in the utterance counts.
|
||||
|
||||
Decision for one utterance (`judge(speaker, text)`), in order:
|
||||
|
||||
1. Empty text (after normalisation): ignore.
|
||||
2. `designatedOnly` and the speaker is not designated: ignore.
|
||||
3. A wake word is heard and the rest of the text, with that stretch removed, starts with a stop word
|
||||
and has at most 3 more letters ("멈춰", "멈춰 줘", "그만해"): **stop**.
|
||||
4. A wake word is heard: **request**, `direct` (called by name).
|
||||
5. `designatedNoWakeWord` and the speaker is designated: **request**, not direct.
|
||||
6. `call.mode` `everything`: **request**, not direct. `addressed`: when the speaker is the only person in
|
||||
Jarvis's voice channel (the people it hears, not Jarvis or ignored bots; the speaker always counts,
|
||||
also while the channel list lags), **request**, not direct: alone with Jarvis, whatever they say is
|
||||
said to it. Otherwise **ask**: the brain judges whether it was said to Jarvis, told how many people
|
||||
are in the channel and how many of them may call Jarvis (all, or with `designatedOnly` the
|
||||
designated ones present), since with one other person most of what the speaker says is for Jarvis
|
||||
and with more most talk is between people (`voice_worker.spec.md`, `addressed`); a yes becomes a
|
||||
request, not direct. `wakeWord`: ignore.
|
||||
|
||||
The text sent to the brain is the utterance as heard; the brain treats the wake word as address.
|
||||
|
||||
**A wake word said alone.** People often pause after the name ("자비스, ... 지금 몇 시야?"), and a
|
||||
pause longer than the worker's `end_silence_ms` ends the utterance there. A voice utterance that is a
|
||||
wake word and nothing else (no letters or digits left once the wake word is removed) is therefore
|
||||
not a request yet: the conversation keeps it for that speaker and joins it to their next final, with
|
||||
a space between, and judges the joined text as one utterance (so "자비스" then "멈춰" is a stop). If
|
||||
the speaker's next utterance already started when the wake word's final came, or starts within
|
||||
`call.followUpMs` of it, the wake word waits for that utterance's final however long it takes;
|
||||
otherwise, after `call.followUpMs`, the wake word alone is the request. Other speakers' utterances
|
||||
never join it, and a stop (voice, slash command or leaving) drops it. So does a lost worker
|
||||
connection (`WorkerLost`): the final it waited for will not come, and the restarted worker's
|
||||
utterances are not continuations of it. Chat messages are not joined:
|
||||
a message is complete as sent.
|
||||
|
||||
**Earcon.** A short two-tone "띠링" (about 0.3 s, made by the app, `app/src/earcon.rs`) tells the
|
||||
speaker Jarvis heard the call, long before the answer is written: it is the first sound after a call.
|
||||
It plays right before each voice answer starts (the policy's `Answer`), and at once when a wake word
|
||||
said alone is kept waiting while no answer is running (the question that joins it then gets no second
|
||||
earcon). It is queued like speech, so it waits behind speech already queued and a stop drops it. A
|
||||
call the policy does not take (busy, replaced, stopped) gets none, nor does a wake word alone while an
|
||||
answer runs. Chat answers get none. `call.earcon: false` turns it off.
|
||||
|
||||
## Response policy
|
||||
|
||||
Settings (section `response`):
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `response.policy` | `"latest"` | `first` (처음 대화만), `inOrder` (들어온 순서대로), `latest` (마지막 대화만), `important` (제일 중요한 대화만). |
|
||||
| `response.queueMax` | `5` | `inOrder`: most waiting requests; past it the oldest is dropped. 1-20. |
|
||||
| `response.collectMs` | `700` | `important`: how long candidates are collected before choosing. 0-3000. |
|
||||
| `response.settleMs` | `800` | Longest wait for an earlier-ended utterance that is still being transcribed. 0-3000. |
|
||||
| `response.priorities` | `{}` | `important`: user ID → priority (higher first, default 0). |
|
||||
| `response.interrupt` | `true` | `latest` and `important`: a new request may stop the running answer (below). `false`: the running answer is always spoken to its end and the new request waits for it (`latest`: the newest waiting request is answered next and older waiting ones are missed, `replaced`, also a request whose final arrives after a newer one's; `important`: the best candidate collected meanwhile is answered next, no ranking against the running answer). A voice stop still stops it. `first` and `inOrder` never interrupt. |
|
||||
|
||||
Terms:
|
||||
|
||||
- A **request** has an id, the speaker, the text, whether it was `direct`, and `ended_at`: when the
|
||||
utterance ended (the final's arrival minus its `wait_ms`).
|
||||
- An **answer** runs from the moment the policy starts it until it is done (its speech ended, failed or
|
||||
was stopped). There is at most one answer at a time: Jarvis has one voice.
|
||||
- An utterance is **open** from its `speech_start` until its final. An open utterance whose speaker
|
||||
has stopped speaking (`speaking_end` sent) is **ending**: its final may still turn out to be a call.
|
||||
|
||||
Rules shared by every policy:
|
||||
|
||||
- **Ordering by end time.** Requests are compared by `ended_at`, not by when their final arrived, so a
|
||||
long utterance is not pushed back by its transcription time.
|
||||
- **Settling.** When a request could start now but some other utterance is ending and stopped earlier
|
||||
than the request's `ended_at` (in `important`: any ending utterance, since it competes on score), the
|
||||
policy waits for that final, at most `settleMs` from the first waiting request's arrival. A final that
|
||||
is not a call ends the wait for it. `latest` does not settle: an earlier utterance would lose anyway.
|
||||
- **Stop.** A stop decision stops the current answer and drops every waiting request, in every policy.
|
||||
The stopped answer and the dropped requests are missed (`stopped`).
|
||||
- An answer that is already done is never started again.
|
||||
|
||||
Per policy (with `response.interrupt` on; off, no answer is stopped for a new request):
|
||||
|
||||
| Policy | No answer running | A new request while an answer runs |
|
||||
|---|---|---|
|
||||
| `first` | Answer the request that ended first (after settling). | Missed (`busy`). |
|
||||
| `inOrder` | Answer the earliest-ended waiting request. | Queued by `ended_at`. Past `queueMax`, the earliest-ended waiting one is missed (`queue_full`). |
|
||||
| `latest` | Answer the latest-ended waiting request; the others are missed (`replaced`). | The current answer stops (missed, `interrupted`) and the new request is answered. A request that ended before the current one is missed (`replaced`). |
|
||||
| `important` | Collect for `collectMs` from the first request, then answer the best; the other candidates are missed (`lower_score`). | Kept as a candidate. If it beats the current answer, the current answer stops (missed, `interrupted`) and the best candidate is answered at once; otherwise, when the current answer ends, the best of the candidates collected meanwhile is answered and the rest are missed (`lower_score`). |
|
||||
|
||||
`important` ranks by (1) priority, (2) `direct`, then (3) the brain's importance ranking, asked for only
|
||||
when two or more candidates tie on (1) and (2). While the ranking is pending nothing new starts; if it
|
||||
does not come within 1.5 s (a ranking that arrives later counts as late), or fails, the earliest-ended
|
||||
of the tied candidates wins. Candidates are kept in `ended_at` order, whenever they arrived.
|
||||
|
||||
Each ranking carries an id (`Rank { id, candidates }`, answered with `ranked(id, order)`). Only the
|
||||
pending ranking's answer is used: an answer to a ranking that was replaced, or dropped (its answer ended,
|
||||
or a stop), is ignored. A candidate that ties the ranked ones while a ranking is pending replaces it with
|
||||
a new ranking over all of them (and the running answer, when the ranking was about interrupting it),
|
||||
keeping the first ranking's deadline, so only candidates that were compared are missed (`lower_score`).
|
||||
A candidate that beats the ranked ones makes the ranking moot.
|
||||
|
||||
A new request that beats the running answer on (1) or (2) interrupts it at once. One that ties it on
|
||||
both is ranked against it (the running answer keeps playing meanwhile): if the ranking puts the new
|
||||
request first, the answer stops (missed, `interrupted`) and the new request is answered; if it puts the
|
||||
running answer first, or does not come in time, the answer goes on and the new request waits. The
|
||||
ranking is dropped when the running answer ends first.
|
||||
|
||||
## Chat calls
|
||||
|
||||
Registered chat channels are answered in chat, by text, beside the voice conversation: a chat message
|
||||
never takes the voice turn and never goes through the response policy.
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `chatChannels[].answer` | `"wakeWord"` | `wakeWord`: answer messages that call a wake word. `everything`: answer every message. |
|
||||
|
||||
- The wake words are `call.chatWakeWords` (or `voice.wakeWords` when it is `null`), with the same
|
||||
similarity and the same designated-people switches as voice; `call.mode` is voice only.
|
||||
- Only messages of a registered channel with reading on reach the loop (the core's rule), and only a
|
||||
channel with sending on is answered: there an answer could not be posted, so none is asked for.
|
||||
- Messages from bots (other bots and Jarvis itself) are never answered, so two bots cannot keep
|
||||
answering each other.
|
||||
- Only a request is answered: a stop in chat ("자비스 멈춰") is not a question and gets no answer.
|
||||
- The brain answers through `reply` with `via: "chat"` and `speak: false`; the answer is posted in the
|
||||
same channel as a Discord reply to the message (so the history records it as an answer to that
|
||||
person), pinging nobody. A failed or empty answer posts nothing. An answer that arrives 5 minutes or
|
||||
more after it was asked is not posted. The worker is told whether each answer was posted
|
||||
(`reply_delivered`).
|
||||
- The worker writes voice and chat answers in separate lanes (`voice_worker.spec.md`), so a voice
|
||||
answer never waits behind another person's chat answer; one person's answers are written one at a
|
||||
time, in the order they asked. The LLM takes one request at a time (`docs/v2/conversation.md`,
|
||||
"LLM 동시 요청 수"; `--parallel 1`, also when a channel answers everything), so the two lanes' LLM
|
||||
calls take turns at Strata.
|
||||
|
||||
## The conversation loop
|
||||
|
||||
`app/src/conversation.rs` runs between the core's events and everything else that reads them (every
|
||||
event is passed on unchanged). It talks to the core through a small `Voice` interface (answer, cancel
|
||||
one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glue is tested with a fake.
|
||||
|
||||
- `Chat` and `ReplyText`: chat calls, above.
|
||||
- `Partial` (getting ready): a partial transcript that call detection already reads as a request
|
||||
(joined to the speaker's waiting wake word alone, as its final would be; not the wake word alone,
|
||||
not a stop) is sent to the brain as `prepare(user, text)`, so the brain judges what the call needs
|
||||
while the speaker is still talking (`voice_worker.spec.md`, `prepare`). Not while an answer runs:
|
||||
the call may not be taken, and the brain is busy answering.
|
||||
- `SpeechStart` opens the speaker's utterance; `SpeakingStopped` marks it ending.
|
||||
- `Final`: a request gets the next id, `ended_at` = arrival minus the final's `wait_ms`, and goes
|
||||
through call detection. Ignore closes the utterance. Stop stops all speech, runs the policy's stop,
|
||||
voids every pending "addressed" question (closing its utterance) and closes the utterance. Request
|
||||
goes to the policy. Ask sends `addressed` to the brain; a yes becomes the request, a no (or a failed
|
||||
judgement) closes the utterance, and no answer within 5 s counts as no, also when the answer arrives
|
||||
after that but before the timer ran.
|
||||
- Policy actions: `Answer` → the earcon (unless it already played for this request), then
|
||||
`answer(user, text)` (the brain answers and the answer is spoken; when it
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
`Left`: the same stop. The core sends `Stopped` before the `SpeakEnd`s of the speech it stopped, so a
|
||||
cancelled answer's end never starts the next waiting request.
|
||||
- A timer wakes the loop at the policy's next deadline or the earliest "addressed" timeout.
|
||||
|
||||
The brain's two judgements are `src/voice_worker/judges.py` (`voice_worker.spec.md`): one short call
|
||||
each to the chat model, the heard text fenced as data. Eval: `evals/test_voice_judges.py`.
|
||||
|
||||
## Tests
|
||||
|
||||
`cargo test` covers call detection (exact and near wake words in Hangul and Latin, a different word
|
||||
two letters away, wake word anywhere, stop with and without a wake word, designated combinations,
|
||||
modes) and every policy rule above with fake times (settling, ordering by end time, busy, queue limit,
|
||||
interruption and `response.interrupt` off, ranking tie, ranking timeout, stop dropping the queue), and the conversation loop with a
|
||||
fake voice side (answering and queueing, a busy drop recorded without text, voice stop, addressed alone
|
||||
with Jarvis answered without asking, the room counts sent with the question, addressed yes,
|
||||
no and timeout, the earcon before each answer, once for a wake word alone, none while busy or when
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
338
app/src/dashboard.rs
Normal file
338
app/src/dashboard.rs
Normal file
@@ -0,0 +1,338 @@
|
||||
//! What the dashboard shows, built from the core's events (`app/src/admin.spec.md`, "Dashboard data").
|
||||
//!
|
||||
//! Pure: `on_event` takes the time, so every rule is tested with made-up events. Captions hold what
|
||||
//! people said for the screen only; nothing here is logged or written.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::time::Instant;
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::conversation::Missed;
|
||||
use crate::discord::CoreEvent;
|
||||
|
||||
pub const CAPTIONS_KEPT: usize = 30;
|
||||
pub const ANSWERS_KEPT: usize = 20;
|
||||
pub const MISSED_SHOWN: usize = 20;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
pub struct Caption {
|
||||
pub speaker: String,
|
||||
pub name: String,
|
||||
pub text: String,
|
||||
#[serde(rename = "final")]
|
||||
pub is_final: bool,
|
||||
#[serde(skip)]
|
||||
utt: u32,
|
||||
}
|
||||
|
||||
/// One voice answer's stages in milliseconds (None: that stage could not be seen, e.g. no answer text
|
||||
/// before the sound).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AnswerTiming {
|
||||
/// Speech end to its final transcript.
|
||||
pub final_ms: Option<u64>,
|
||||
/// The answer asked to its text.
|
||||
pub brain_ms: Option<u64>,
|
||||
/// The text to the first sound of the answer.
|
||||
pub voice_ms: Option<u64>,
|
||||
/// Speech end to the first sound.
|
||||
pub total_ms: Option<u64>,
|
||||
}
|
||||
|
||||
struct Pending {
|
||||
asked: Instant,
|
||||
/// When the request's speech ended and when its final came.
|
||||
heard: (Instant, Instant),
|
||||
reply: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Dashboard {
|
||||
bot_tag: Option<String>,
|
||||
voice_channel: Option<u64>,
|
||||
/// The voice session Jarvis is in (from `Joined`).
|
||||
session: Option<u64>,
|
||||
captions: VecDeque<Caption>,
|
||||
pending: HashMap<String, Pending>,
|
||||
answers: VecDeque<AnswerTiming>,
|
||||
}
|
||||
|
||||
fn ms(from: Instant, to: Instant) -> u64 {
|
||||
to.saturating_duration_since(from).as_millis() as u64
|
||||
}
|
||||
|
||||
impl Dashboard {
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::Ready { tag, .. } => self.bot_tag = Some(tag.clone()),
|
||||
CoreEvent::Joined { channel_id, session, .. } => {
|
||||
self.voice_channel = Some(*channel_id);
|
||||
self.session = Some(*session);
|
||||
}
|
||||
CoreEvent::Left { .. } => {
|
||||
self.voice_channel = None;
|
||||
self.session = None;
|
||||
self.captions.clear();
|
||||
self.pending.clear();
|
||||
}
|
||||
// A transcript belongs to the session its utterance started in: one of an earlier session
|
||||
// (arriving after a leave, or after a move to another channel) is not shown.
|
||||
CoreEvent::Partial(h, text) if self.session.is_some() && h.session == self.session =>
|
||||
self.caption(h.speaker_id, &h.speaker_name, h.utt, text, false),
|
||||
CoreEvent::Final(f) if self.session.is_some() && f.heard.session == self.session => {
|
||||
let h = &f.heard;
|
||||
self.caption(h.speaker_id, &h.speaker_name, h.utt, &f.text, true);
|
||||
}
|
||||
// The request's own times: a second question waiting behind the first does not move them.
|
||||
CoreEvent::AnswerAsked { id, ended_at, final_at, .. } => {
|
||||
self.pending.insert(id.clone(), Pending { asked: now, heard: (*ended_at, *final_at), reply: None });
|
||||
}
|
||||
CoreEvent::ReplyText { id, .. } => {
|
||||
if let Some(p) = self.pending.get_mut(id) {
|
||||
p.reply = Some(now);
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(p) = self.pending.remove(id) {
|
||||
let t = AnswerTiming {
|
||||
final_ms: Some(ms(p.heard.0, p.heard.1)),
|
||||
brain_ms: p.reply.map(|r| ms(p.asked, r)),
|
||||
voice_ms: p.reply.map(|r| ms(r, now)),
|
||||
total_ms: Some(ms(p.heard.0, now)),
|
||||
};
|
||||
if self.answers.len() == ANSWERS_KEPT {
|
||||
self.answers.pop_front();
|
||||
}
|
||||
self.answers.push_back(t);
|
||||
}
|
||||
}
|
||||
// An answer that ends without starting (failed, stopped) has nothing to time.
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
self.pending.remove(&e.id);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// A partial replaces the same utterance's earlier caption; the final replaces the partial
|
||||
/// (an empty final removes it).
|
||||
fn caption(&mut self, speaker: u64, name: &str, utt: u32, text: &str, is_final: bool) {
|
||||
let id = speaker.to_string();
|
||||
let pos = self.captions.iter().position(|c| c.speaker == id && c.utt == utt && !c.is_final);
|
||||
if let Some(i) = pos {
|
||||
self.captions.remove(i);
|
||||
}
|
||||
if text.trim().is_empty() {
|
||||
return;
|
||||
}
|
||||
if self.captions.len() == CAPTIONS_KEPT {
|
||||
self.captions.pop_front();
|
||||
}
|
||||
self.captions.push_back(Caption { speaker: id, name: name.into(), text: text.into(), is_final, utt });
|
||||
}
|
||||
|
||||
/// Forget what was said (the services stopped).
|
||||
pub fn clear(&mut self) {
|
||||
*self = Self::default();
|
||||
}
|
||||
|
||||
pub fn captions(&self) -> Vec<Caption> {
|
||||
self.captions.iter().cloned().collect()
|
||||
}
|
||||
|
||||
pub fn answers(&self) -> Vec<AnswerTiming> {
|
||||
self.answers.iter().copied().collect()
|
||||
}
|
||||
|
||||
/// The median of each stage over the answers kept.
|
||||
pub fn medians(&self) -> AnswerTiming {
|
||||
fn median(mut v: Vec<u64>) -> Option<u64> {
|
||||
v.sort_unstable();
|
||||
(!v.is_empty()).then(|| v[v.len() / 2])
|
||||
}
|
||||
let pick = |f: fn(&AnswerTiming) -> Option<u64>| median(self.answers.iter().filter_map(f).collect());
|
||||
AnswerTiming { final_ms: pick(|a| a.final_ms), brain_ms: pick(|a| a.brain_ms),
|
||||
voice_ms: pick(|a| a.voice_ms), total_ms: pick(|a| a.total_ms) }
|
||||
}
|
||||
|
||||
pub fn view(&self, missed: &[Missed], now: Instant) -> Value {
|
||||
json!({
|
||||
"discord": self.bot_tag,
|
||||
"voiceChannel": self.voice_channel.map(|c| c.to_string()),
|
||||
"captions": self.captions(),
|
||||
"latency": {"answers": self.answers(), "median": self.medians()},
|
||||
"missed": missed_view(missed, now),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Newest first, at most `MISSED_SHOWN`: who, why, how long ago and how long the request was.
|
||||
pub fn missed_view(missed: &[Missed], now: Instant) -> Value {
|
||||
Value::Array(missed.iter().rev().take(MISSED_SHOWN).map(|m| json!({
|
||||
"speaker": m.speaker.to_string(),
|
||||
"reason": m.reason,
|
||||
"secondsAgo": now.saturating_duration_since(m.at).as_secs(),
|
||||
"chars": m.chars,
|
||||
})).collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::discord::{Heard, HeardFinal};
|
||||
use crate::playback::PlaybackEnd;
|
||||
use crate::policy::MissReason;
|
||||
use std::time::Duration;
|
||||
|
||||
const A: u64 = 11;
|
||||
fn heard_in(utt: u32, session: u64) -> Heard {
|
||||
Heard { speaker_id: A, speaker_name: "민수".into(), utt, session: Some(session) }
|
||||
}
|
||||
fn heard(utt: u32) -> Heard {
|
||||
heard_in(utt, 1)
|
||||
}
|
||||
fn fin_in(utt: u32, text: &str, wait_ms: u64, session: u64) -> CoreEvent {
|
||||
CoreEvent::Final(HeardFinal { heard: heard_in(utt, session), text: text.into(), end_reason: "silence".into(),
|
||||
audio_ms: 900, wait_ms, stt_ms: 200 })
|
||||
}
|
||||
fn fin(utt: u32, text: &str, wait_ms: u64) -> CoreEvent {
|
||||
fin_in(utt, text, wait_ms, 1)
|
||||
}
|
||||
fn at(t: Instant, ms: u64) -> Instant {
|
||||
t + Duration::from_millis(ms)
|
||||
}
|
||||
fn joined() -> CoreEvent {
|
||||
CoreEvent::Joined { guild_id: 1, channel_id: 5, session: 1 }
|
||||
}
|
||||
fn left() -> CoreEvent {
|
||||
CoreEvent::Left { guild_id: 1, channel_id: 5, reason: "command".into() }
|
||||
}
|
||||
fn asked(id: &str, t: Instant, ended: u64, final_at: u64) -> CoreEvent {
|
||||
CoreEvent::AnswerAsked { id: id.into(), user: A, ended_at: at(t, ended), final_at: at(t, final_at) }
|
||||
}
|
||||
fn reply(id: &str) -> CoreEvent {
|
||||
CoreEvent::ReplyText { id: id.into(), text: "네".into(), failed: false }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_partial_is_replaced_by_the_next_and_then_by_the_final() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
d.on_event(&joined(), t);
|
||||
d.on_event(&CoreEvent::Partial(heard(1), "자비스 오늘".into()), t);
|
||||
d.on_event(&CoreEvent::Partial(heard(1), "자비스 오늘 날씨".into()), t);
|
||||
assert_eq!(d.captions().iter().map(|c| (c.text.as_str(), c.is_final)).collect::<Vec<_>>(),
|
||||
vec![("자비스 오늘 날씨", false)]);
|
||||
d.on_event(&fin(1, "자비스 오늘 날씨 어때?", 700), t);
|
||||
d.on_event(&CoreEvent::Partial(heard(2), "그리고".into()), t);
|
||||
assert_eq!(d.captions().iter().map(|c| (c.text.as_str(), c.is_final)).collect::<Vec<_>>(),
|
||||
vec![("자비스 오늘 날씨 어때?", true), ("그리고", false)]);
|
||||
// An empty final removes its partial.
|
||||
d.on_event(&fin(2, "", 300), t);
|
||||
assert_eq!(d.captions().len(), 1);
|
||||
assert_eq!(d.captions()[0].name, "민수");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn captions_are_capped_and_cleared_when_jarvis_leaves_and_late_ones_stay_out() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
d.on_event(&joined(), t);
|
||||
for i in 0..(CAPTIONS_KEPT as u32 + 5) {
|
||||
d.on_event(&fin(i, &format!("말 {i}"), 100), t);
|
||||
}
|
||||
let caps = d.captions();
|
||||
assert_eq!(caps.len(), CAPTIONS_KEPT);
|
||||
assert_eq!(caps.last().unwrap().text, format!("말 {}", CAPTIONS_KEPT + 4));
|
||||
d.on_event(&left(), t);
|
||||
assert!(d.captions().is_empty());
|
||||
// A transcript of the session that ended arrives after the leave: it does not come back.
|
||||
d.on_event(&CoreEvent::Partial(heard(90), "늦게 온 말".into()), t);
|
||||
d.on_event(&fin(90, "늦게 온 말", 100), t);
|
||||
assert!(d.captions().is_empty());
|
||||
assert_eq!(d.view(&[], t)["voiceChannel"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_transcript_of_the_channel_jarvis_moved_away_from_stays_out_of_the_new_ones_captions() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
d.on_event(&joined(), t);
|
||||
d.on_event(&CoreEvent::SpeechStart(heard_in(7, 1)), t); // started in channel A, session 1
|
||||
d.on_event(&left(), t);
|
||||
d.on_event(&CoreEvent::Joined { guild_id: 1, channel_id: 6, session: 2 }, t);
|
||||
d.on_event(&CoreEvent::Partial(heard_in(7, 1), "A에서 한 말".into()), t);
|
||||
d.on_event(&fin_in(7, "A에서 한 말", 300, 1), t);
|
||||
assert!(d.captions().is_empty(), "session 1's utterance is not session 2's");
|
||||
// An utterance whose start was not seen in a session (None) is not shown either.
|
||||
d.on_event(&CoreEvent::Partial(Heard { speaker_id: A, speaker_name: "민수".into(), utt: 8, session: None },
|
||||
"어디서 온 말".into()), t);
|
||||
assert!(d.captions().is_empty());
|
||||
// The same person speaking in B is.
|
||||
d.on_event(&fin_in(9, "B에서 한 말", 300, 2), t);
|
||||
assert_eq!(d.captions().iter().map(|c| c.text.as_str()).collect::<Vec<_>>(), vec!["B에서 한 말"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_is_timed_from_its_own_requests_speech_end_to_its_first_sound() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
d.on_event(&joined(), t);
|
||||
d.on_event(&asked("r1", t, 300, 1000), at(t, 1010));
|
||||
d.on_event(&reply("r1"), at(t, 1700));
|
||||
d.on_event(&CoreEvent::SpeakStart("r1".into()), at(t, 1900));
|
||||
assert_eq!(d.answers(), vec![AnswerTiming { final_ms: Some(700), brain_ms: Some(690), voice_ms: Some(200),
|
||||
total_ms: Some(1600) }]);
|
||||
// An answer stopped before it started is not timed; the earcon (another id) is not either.
|
||||
d.on_event(&asked("r3", t, 2000, 2500), at(t, 3000));
|
||||
d.on_event(&CoreEvent::SpeakStart("r2".into()), at(t, 3000));
|
||||
d.on_event(&CoreEvent::SpeakEnd(PlaybackEnd { id: "r3".into(), played: 0, cancelled: true, failed: false }),
|
||||
at(t, 3100));
|
||||
d.on_event(&CoreEvent::SpeakStart("r3".into()), at(t, 3200));
|
||||
assert_eq!(d.answers().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_second_question_waiting_does_not_change_the_first_answers_times() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
d.on_event(&joined(), t);
|
||||
// First question ends at 0 (final at 700); the same person asks again, ending at 1000, while
|
||||
// the first is still answered. The first answer starts sounding at 3700.
|
||||
d.on_event(&fin(1, "첫 질문", 700), at(t, 700));
|
||||
d.on_event(&asked("r1", t, 0, 700), at(t, 710));
|
||||
d.on_event(&fin(2, "둘째 질문", 700), at(t, 1700));
|
||||
d.on_event(&reply("r1"), at(t, 3500));
|
||||
d.on_event(&CoreEvent::SpeakStart("r1".into()), at(t, 3700));
|
||||
assert_eq!(d.answers()[0].total_ms, Some(3700));
|
||||
assert_eq!(d.answers()[0].final_ms, Some(700));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_medians_cover_the_last_answers_kept() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
for (i, brain) in [500u64, 900, 700].into_iter().enumerate() {
|
||||
let base = i as u64 * 10_000;
|
||||
d.on_event(&asked(&format!("r{i}"), t, base + 400, base + 1000), at(t, base + 1000));
|
||||
d.on_event(&reply(&format!("r{i}")), at(t, base + 1000 + brain));
|
||||
d.on_event(&CoreEvent::SpeakStart(format!("r{i}")), at(t, base + 1000 + brain + 100));
|
||||
}
|
||||
let m = d.medians();
|
||||
assert_eq!((m.final_ms, m.brain_ms, m.voice_ms, m.total_ms), (Some(600), Some(700), Some(100), Some(1400)));
|
||||
for i in 0..ANSWERS_KEPT as u64 {
|
||||
d.on_event(&asked(&format!("x{i}"), t, 100_000, 100_000), at(t, 100_000 + i));
|
||||
d.on_event(&CoreEvent::SpeakStart(format!("x{i}")), at(t, 100_000 + i));
|
||||
}
|
||||
assert_eq!(d.answers().len(), ANSWERS_KEPT);
|
||||
assert_eq!(d.medians().brain_ms, None, "no answer text seen for the answers kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missed_requests_show_newest_first_without_text() {
|
||||
let t = Instant::now();
|
||||
let missed: Vec<Missed> = (0..25).map(|i| Missed { speaker: i, reason: MissReason::Busy,
|
||||
at: t, chars: 7 }).collect();
|
||||
let v = missed_view(&missed, at(t, 3000));
|
||||
let a = v.as_array().unwrap();
|
||||
assert_eq!(a.len(), MISSED_SHOWN);
|
||||
assert_eq!(a[0], json!({"speaker": "24", "reason": "busy", "secondsAgo": 3, "chars": 7}));
|
||||
}
|
||||
}
|
||||
222
app/src/datadir.rs
Normal file
222
app/src/datadir.rs
Normal file
@@ -0,0 +1,222 @@
|
||||
//! The data folder and the file that remembers where it is (`app/src/admin.spec.md`).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
/// The wizard warns below this much free space (desktop-app.md: 200 GB recommended).
|
||||
pub const RECOMMENDED_FREE_BYTES: u64 = 200 * 1_000_000_000;
|
||||
|
||||
/// Folders inside the data folder (desktop-app.md, "데이터 폴더 구조").
|
||||
pub const LAYOUT: &[&str] = &[
|
||||
"config", "runtime", "models/llm", "models/stt", "models/tts", "models/vad", "memory", "browser",
|
||||
"collect", "cache", "logs", "models/hf", "models/nltk",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DataDir(PathBuf);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DataDirError(pub String);
|
||||
|
||||
impl std::fmt::Display for DataDirError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for DataDirError {}
|
||||
|
||||
fn io(what: &'static str, p: &Path) -> impl FnOnce(std::io::Error) -> DataDirError {
|
||||
let p = p.display().to_string();
|
||||
move |e| DataDirError(format!("{what} {p}: {e}"))
|
||||
}
|
||||
|
||||
impl DataDir {
|
||||
/// Make the layout inside `path` (new or existing); nothing is ever deleted.
|
||||
pub fn create(path: &Path) -> Result<Self, DataDirError> {
|
||||
if !path.is_absolute() {
|
||||
return Err(DataDirError(format!("the data folder must be an absolute path: {}", path.display())));
|
||||
}
|
||||
if path.is_file() {
|
||||
return Err(DataDirError(format!("the data folder is a file: {}", path.display())));
|
||||
}
|
||||
for sub in LAYOUT {
|
||||
let d = path.join(sub);
|
||||
std::fs::create_dir_all(&d).map_err(io("could not create", &d))?;
|
||||
}
|
||||
Ok(Self(path.to_path_buf()))
|
||||
}
|
||||
|
||||
/// Create the folder and remember it in the location file.
|
||||
pub fn choose(path: &Path, location_file: &Path) -> Result<Self, DataDirError> {
|
||||
let dir = Self::create(path)?;
|
||||
let body = json!({"dataDir": path.to_string_lossy()}).to_string();
|
||||
write_atomic(location_file, body.as_bytes()).map_err(io("could not write", location_file))?;
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
/// The folder in use: `JARVIS_DATA_DIR`, else the location file; None before the wizard finished.
|
||||
pub fn locate(location_file: &Path) -> Option<Self> {
|
||||
Self::locate_with(std::env::var("JARVIS_DATA_DIR").ok(), location_file)
|
||||
}
|
||||
|
||||
/// `locate` with the environment variable's value given.
|
||||
pub fn locate_with(env: Option<String>, location_file: &Path) -> Option<Self> {
|
||||
if let Some(p) = env.filter(|p| !p.is_empty()) {
|
||||
return Some(Self(PathBuf::from(p)));
|
||||
}
|
||||
let raw = std::fs::read_to_string(location_file).ok()?;
|
||||
let v: serde_json::Value = serde_json::from_str(raw.trim_start_matches('\u{feff}')).ok()?;
|
||||
let p = PathBuf::from(v.get("dataDir")?.as_str()?);
|
||||
p.is_absolute().then_some(Self(p))
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.0
|
||||
}
|
||||
pub fn config(&self) -> PathBuf {
|
||||
self.0.join("config")
|
||||
}
|
||||
pub fn settings_file(&self) -> PathBuf {
|
||||
self.config().join("settings.json")
|
||||
}
|
||||
pub fn secrets_file(&self) -> PathBuf {
|
||||
self.config().join("secrets.dat")
|
||||
}
|
||||
pub fn logs(&self) -> PathBuf {
|
||||
self.0.join("logs")
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the location file lives on this OS (`%APPDATA%\JarvisBot`, else the XDG config folder).
|
||||
pub fn default_location_file() -> Option<PathBuf> {
|
||||
if cfg!(windows) {
|
||||
let appdata = std::env::var_os("APPDATA")?;
|
||||
return Some(PathBuf::from(appdata).join("JarvisBot").join("location.json"));
|
||||
}
|
||||
let base = std::env::var_os("XDG_CONFIG_HOME").map(PathBuf::from)
|
||||
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config")))?;
|
||||
Some(base.join("jarvis-bot").join("location.json"))
|
||||
}
|
||||
|
||||
/// Where the window's own browser data (WebView2) goes: `cache/webview` in the data folder once
|
||||
/// there is one. Before the first-run wizard chose a folder it is a scratch folder in the system
|
||||
/// temp folder (`JarvisBot-webview-<process id>`, `true`); WebView2 holds it until the app's process
|
||||
/// has ended, so the app removes it on its next start (`remove_scratch_webview`).
|
||||
pub fn webview_data_dir(location_file: &Path) -> (PathBuf, bool) {
|
||||
match DataDir::locate(location_file) {
|
||||
Some(d) => (d.path().join("cache").join("webview"), false),
|
||||
None => (std::env::temp_dir().join(format!("{WEBVIEW_SCRATCH}{}", std::process::id())), true),
|
||||
}
|
||||
}
|
||||
|
||||
const WEBVIEW_SCRATCH: &str = "JarvisBot-webview-";
|
||||
|
||||
/// Remove the scratch browser data folders in `temp` (all of them, or only `only`); a folder still in
|
||||
/// use stays and goes next time.
|
||||
pub fn remove_scratch_webview(temp: &Path, only: Option<&Path>) {
|
||||
let Ok(rd) = std::fs::read_dir(temp) else { return };
|
||||
for e in rd.flatten() {
|
||||
let p = e.path();
|
||||
let ours = p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with(WEBVIEW_SCRATCH));
|
||||
if ours && only.is_none_or(|o| o == p) {
|
||||
let _ = std::fs::remove_dir_all(&p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Free bytes on the disk that holds `path` (or its nearest existing parent).
|
||||
pub fn free_bytes(path: &Path) -> Result<u64, DataDirError> {
|
||||
let mut p = path;
|
||||
while !p.exists() {
|
||||
p = p.parent().ok_or_else(|| DataDirError(format!("no existing parent for {}", path.display())))?;
|
||||
}
|
||||
fs4::available_space(p).map_err(io("could not read the free space of", p))
|
||||
}
|
||||
|
||||
/// Write `bytes` to `path` through a temporary file in the same folder and a rename: a crash leaves
|
||||
/// the old file or the new one, never half of one.
|
||||
pub fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let tmp = path.with_extension(format!("tmp{}", std::process::id()));
|
||||
std::fs::write(&tmp, bytes)?;
|
||||
std::fs::rename(&tmp, path).inspect_err(|_| {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn temp_dir(tag: &str) -> PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("jarvis-{tag}-{}-{}", std::process::id(), rand::random::<u32>()));
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn create_makes_the_layout_and_keeps_what_is_there() {
|
||||
let root = temp_dir("dd").join("data");
|
||||
std::fs::create_dir_all(root.join("memory")).unwrap();
|
||||
std::fs::write(root.join("memory").join("keep.db"), b"x").unwrap();
|
||||
let d = DataDir::create(&root).unwrap();
|
||||
for sub in LAYOUT {
|
||||
assert!(root.join(sub).is_dir(), "{sub}");
|
||||
}
|
||||
assert_eq!(std::fs::read(root.join("memory").join("keep.db")).unwrap(), b"x");
|
||||
assert_eq!(d.settings_file(), root.join("config").join("settings.json"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_window_data_goes_in_the_data_folder_or_a_scratch_folder_that_is_removed() {
|
||||
let root = temp_dir("wv");
|
||||
let loc = root.join("cfg").join("location.json");
|
||||
let (dir, scratch) = webview_data_dir(&loc);
|
||||
assert!(scratch && dir.starts_with(std::env::temp_dir()), "no folder chosen yet: scratch in temp");
|
||||
DataDir::choose(&root.join("data"), &loc).unwrap();
|
||||
if std::env::var("JARVIS_DATA_DIR").is_err() {
|
||||
assert_eq!(webview_data_dir(&loc), (root.join("data").join("cache").join("webview"), false));
|
||||
}
|
||||
// Scratch folders of earlier runs are removed; other folders stay.
|
||||
let temp = root.join("tmp");
|
||||
for n in ["JarvisBot-webview-1", "JarvisBot-webview-2", "other"] {
|
||||
std::fs::create_dir_all(temp.join(n).join("EBWebView")).unwrap();
|
||||
}
|
||||
remove_scratch_webview(&temp, Some(&temp.join("JarvisBot-webview-2")));
|
||||
assert!(temp.join("JarvisBot-webview-1").exists() && !temp.join("JarvisBot-webview-2").exists());
|
||||
remove_scratch_webview(&temp, None);
|
||||
assert!(!temp.join("JarvisBot-webview-1").exists() && temp.join("other").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_relative_path_or_a_file_is_refused() {
|
||||
assert!(DataDir::create(Path::new("relative/data")).unwrap_err().0.contains("absolute"));
|
||||
let f = temp_dir("ddf").join("a-file");
|
||||
std::fs::write(&f, b"x").unwrap();
|
||||
assert!(DataDir::create(&f).unwrap_err().0.contains("file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn choose_remembers_the_folder_and_locate_finds_it() {
|
||||
let base = temp_dir("ddl");
|
||||
let loc = base.join("cfg").join("location.json");
|
||||
assert_eq!(DataDir::locate_with(None, &loc), None, "no wizard yet");
|
||||
let data = base.join("데이터");
|
||||
DataDir::choose(&data, &loc).unwrap();
|
||||
assert_eq!(DataDir::locate_with(None, &loc), Some(DataDir(data)));
|
||||
// The environment variable wins (tests, development); an empty one does not count.
|
||||
assert_eq!(DataDir::locate_with(Some("/elsewhere".into()), &loc).unwrap().path(), Path::new("/elsewhere"));
|
||||
assert!(DataDir::locate_with(Some(String::new()), &loc).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn free_space_is_read_from_the_nearest_existing_parent() {
|
||||
let base = temp_dir("ddfree");
|
||||
let n = free_bytes(&base.join("not").join("yet")).unwrap();
|
||||
assert!(n > 0);
|
||||
}
|
||||
}
|
||||
1482
app/src/discord.rs
Normal file
1482
app/src/discord.rs
Normal file
File diff suppressed because it is too large
Load Diff
266
app/src/discord_core.spec.md
Normal file
266
app/src/discord_core.spec.md
Normal file
@@ -0,0 +1,266 @@
|
||||
# Discord voice core spec (v2 M2)
|
||||
|
||||
The Discord voice core is the part of the Windows app (Rust; the GUI arrives in M5 as a Tauri 2 shell)
|
||||
that connects the bot to Discord: it joins a voice channel, hears each person separately, streams their
|
||||
audio to the voice worker for transcription, plays the worker's speech back into the channel, and reads
|
||||
and writes the registered chat channels. It has no assistant logic. It reports what it heard and does
|
||||
what it is told; the brain (M3) and call detection and response policies (M4) sit on top of it.
|
||||
Design background: `docs/v2/architecture.md`, `docs/v2/conversation.md`, `docs/v2/privacy.md`. The
|
||||
worker side of the link is `src/voice_worker/voice_worker.spec.md`.
|
||||
|
||||
## Principles
|
||||
|
||||
- **Only where the bot is.** The bot hears only the voice channel it is in and reads only the chat
|
||||
channels registered in settings with reading on. Every join speaks the transcription notice in the
|
||||
voice channel.
|
||||
- **Chat is data.** Messages read from chat channels are handed on as data, never acted on as
|
||||
commands. Text the bot sends never pings anyone (empty allowed mentions).
|
||||
- **No text in logs.** Logs carry ids, sizes, counts and timings, never what anyone said or wrote, or
|
||||
what the bot says. Debug logs (`debug_log!`, on with `JARVIS_DEBUG=1`) go at joins and leaves,
|
||||
command use, finals, speech requests and stops, chat events and sends, and worker connection changes.
|
||||
- **Language-agnostic.** Nothing in the core matches words. Wake words go to the STT as a hint; call
|
||||
detection (`conversation.spec.md`) is the only place they are matched.
|
||||
- **Testable without Discord.** The pieces with rules (receive window, playback queue, speaker limit,
|
||||
chat access and splitting, settings, worker link, worker process) are plain types tested with fakes;
|
||||
the Discord glue is tested on real Discord by an opt-in live test.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
app/
|
||||
├── Cargo.toml # crate jarvis-app: library + the jarvis-app binary
|
||||
├── src/main.rs # jarvis-app <data folder>: runs the services until Ctrl+C / SIGTERM
|
||||
├── src/services.rs # settings -> voice worker process -> worker client -> Discord core
|
||||
├── src/discord.rs # Discord client (serenity), voice (songbird), slash command, chat channels
|
||||
├── src/receive.rs # per-speaker reorder window and Opus decoding
|
||||
├── src/playback.rs # speech queue (TTS requests -> sentence clips -> player)
|
||||
├── src/earcon.rs # the earcon sound, made in code
|
||||
├── src/speakers.rs # who is transcribed right now (max speakers)
|
||||
├── src/chat.rs # chat channel access rules and message splitting
|
||||
├── src/worker_client.rs # WebSocket client for the voice worker
|
||||
├── src/worker_process.rs # starts and restarts the Python voice worker
|
||||
├── src/frames.rs # binary frame layout shared with voice_worker/protocol.py
|
||||
├── src/settings.rs # settings.json loading and validation
|
||||
└── src/log.rs # debug_log! / error_log!
|
||||
```
|
||||
|
||||
## Starting
|
||||
|
||||
`start_services(data_dir)` reads `<data_dir>/config/settings.json`, starts the voice worker process,
|
||||
connects to it once it prints `READY`, and starts the Discord core. The bot token comes from the
|
||||
stored secrets (`config/secrets.dat`, `admin.spec.md`), else from `JARVIS_DISCORD_TOKEN`; the binary
|
||||
takes the data folder from the first argument or `JARVIS_DATA_DIR`, installs the log hub (daily files
|
||||
in `logs/`) and stops every service on Ctrl+C or SIGTERM.
|
||||
|
||||
## Settings
|
||||
|
||||
`<data folder>/config/settings.json`. Every key is optional; missing keys take the defaults. A bad
|
||||
value stops the app with a message naming the key. Unknown keys are ignored. Discord IDs are JSON
|
||||
strings (numbers would lose precision).
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `discord.guildId` | `null` | The one server. The slash command is registered there; commands and channels from other servers are ignored. |
|
||||
| `discord.voiceChannelId` | `null` | Join this voice channel on start ("자동 입장"). |
|
||||
| `discord.commandName` | `"자비스"` | Slash command name (Discord rule: 1-32 lowercase letters, digits, `-`, `_`). |
|
||||
| `discord.ephemeralReplies` | `true` | Command replies only the caller sees. |
|
||||
| `voice.maxSpeakers` | `10` | Most people transcribed at once, 1-10. |
|
||||
| `voice.listenToBots` | `false` | Also transcribe other bots (music bots are ignored by default). |
|
||||
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Spoken in the voice channel on every join. Must not be empty. |
|
||||
| `voice.wakeWords` | `["자비스"]` | Sent to the worker as STT `hotwords`, and the voice wake words of call detection. |
|
||||
| `voice.stt` | `{}` | Other worker STT settings (`partials`, `end_silence_ms`, `final_beam`, `language`, ...); the worker validates them. |
|
||||
| `chatChannels` | `[]` | `{id, name, description, read (true), send (true), edit (true), react (true), delete (false), pin (false), answer ("wakeWord"), syncDiscord (true)}` per channel; `answer` is in `conversation.spec.md`, `syncDiscord` (the channel's Discord name and topic follow `name` and `description`) in `admin.spec.md`. |
|
||||
| `voice.tts.speed` | `1.5` | How fast the voice speaks, 0.5-2.0 (1.0 is MeloTTS's natural pace; MeloTTS speaks up to 1.25 itself and above that the worker stretches the speech and keeps its pitch, `voice_worker.spec.md`, "TTS"). Passed to the worker as `--melo-speed` when it speaks (`--melo-python`) and its arguments set no speed. A change saved while the services run goes to the worker at once (`tts_speed`) and applies from the next sentence, without a restart (saved while they start, it goes once they run, since the start may have read the settings before); it is sent again whenever the worker connects again. MeloTTS's Korean model has one voice, so there is no voice choice. |
|
||||
| `voiceWorker.python` | `python3` (`python.exe` on Windows) | Python of the worker venv. |
|
||||
| `voiceWorker.cwd` | `python-src` next to the app's executable when it is there (the Windows build), else the repo's `src/` | Folder holding the `voice_worker` package. |
|
||||
| `voiceWorker.args` | `[]` | Extra worker arguments, e.g. `["--melo-python", "<MeloTTS venv python>"]`. |
|
||||
| `memory` | `{}` | The worker's memory settings (`voice_worker.spec.md`, "Memory settings"), sent in its `hello`; the settings screens check them (`admin.spec.md`). |
|
||||
|
||||
The `call` and `response` sections (call detection and response policies, M4) are in
|
||||
`conversation.spec.md`.
|
||||
|
||||
## Voice worker process
|
||||
|
||||
The app starts `python -m voice_worker --host 127.0.0.1 --port <port> <args>` in `voiceWorker.cwd`
|
||||
(no console window on Windows). Each launch makes a new random token (32 bytes) and passes it only in
|
||||
`JARVIS_VOICE_TOKEN`. The port is a free loopback port chosen once at start. If the worker exits, it is
|
||||
started again on the same port with the same token after 1 s, doubling to at most 30 s while it keeps
|
||||
failing (reset once a run printed `READY`). Stopping kills it; MeloTTS's child process ends when its
|
||||
input closes. A stop (and a restart) first logs everything the exited worker wrote, waiting at most
|
||||
5 s (a child of the worker may hold its output open): a stop returns only after its last lines were
|
||||
masked with the secrets that run was started with (`admin.spec.md`, "Logs"). Each run's lines are
|
||||
masked with the values masked when it started too, so a line still being logged after that wait
|
||||
(and after the next start replaced the values in use) stays masked.
|
||||
|
||||
## Worker link
|
||||
|
||||
One WebSocket to the worker, `hello` with the token, the Discord PCM format both ways
|
||||
(48 kHz stereo), and the STT settings (`voice.stt` with `hotwords` = `voice.wakeWords`). If the
|
||||
connection drops, the client reconnects with backoff (0.5 s doubling to 5 s) and says hello again with
|
||||
the same settings, including changes made since with `configure()`. While disconnected, audio frames
|
||||
are dropped (not queued), and every TTS request in flight ends as failed, so playback moves on. A TTS
|
||||
request the worker refuses (`tts_unavailable`) also ends as failed. TTS audio for request ids the client
|
||||
did not open is ignored.
|
||||
|
||||
### History reporting (M3)
|
||||
|
||||
The core tells the worker what its history store needs (`voice_worker.spec.md`, "History and brain"):
|
||||
|
||||
- `voice_join` with the channel after every join or move (also when someone else moves the bot), and
|
||||
`member_join` with the display name for everyone already in the channel that the bot hears (people;
|
||||
bots only with `voice.listenToBots`). Again whenever the worker link becomes ready, because messages
|
||||
sent while disconnected are dropped.
|
||||
- `member_join` / `member_leave` as heard people enter and leave the channel.
|
||||
- `voice_leave` whenever a session ends (leave, move, disconnect).
|
||||
- For every message in a readable registered channel, the bot's own included: `chat_message` with
|
||||
Discord's time; on the bot's own message that replies to someone, `to` is that person. Edits
|
||||
(`chat_edit`), deletes and bulk deletes (`chat_delete`) follow. Chat sent while the worker is
|
||||
disconnected is not recorded.
|
||||
- Finals are recorded by the worker itself; the core does not echo them.
|
||||
|
||||
## Voice
|
||||
|
||||
One voice session at a time, in the configured server. Joining another channel ends the current
|
||||
session (`Left` with `moved`). Within the same server the existing songbird connection is pointed at
|
||||
the new channel and its event handlers are replaced: leaving and joining at once races the gateway,
|
||||
and the late "left" update cancelled the new join (seen on real Discord).
|
||||
|
||||
- **Join** (slash command `join`, auto-join, or `join(channel_id)`): connect through songbird with DAVE
|
||||
end-to-end encryption, not deafened, then speak `voice.joinNotice` in the voice channel (a speech
|
||||
request like any other, nothing is posted in chat). The notice is owed from the join until some of it
|
||||
was played or someone stopped it. When it cannot be spoken (the worker is not connected yet, as at
|
||||
auto-join while the worker still starts), it stays owed and is spoken as soon as the worker is ready;
|
||||
this is logged as an error and the bot stays. Leaving drops it; a worker restart later in the session
|
||||
does not repeat a notice that was heard.
|
||||
- **Hearing**: songbird only decrypts (transport and DAVE). Its own Opus decoding is not used: in
|
||||
songbird 0.6.1 it decodes the payload without trimming what DAVE decryption left at its end, which
|
||||
garbled, repeated or dropped about a third of the frames (measured with a test tone that changes
|
||||
pitch every 20 ms). The core takes each decrypted RTP packet with songbird's exact payload bounds:
|
||||
- Packets go through a per-speaker reorder window in sequence-number order. A missing packet is
|
||||
waited for until two later packets have arrived (about 40 ms), then rebuilt from the next packet's
|
||||
Opus in-band FEC (the last missing frame) or concealed (any earlier ones). Late and duplicate
|
||||
packets are dropped; a jump of 64 or more starts the window over.
|
||||
- Frames are decoded with libopus to 48 kHz stereo PCM and sent to the worker in 20 ms frames under
|
||||
the user's ID. Decoding goes on for people who are not heard, so their decoder stays continuous.
|
||||
- The bot's own audio is never heard; other bots are skipped unless `voice.listenToBots`.
|
||||
- At most `voice.maxSpeakers` people are transcribed at once. Someone who starts while the limit is
|
||||
reached is not heard until their next start after a place frees up.
|
||||
- No packet for 100 ms ends a person's speaking: what their window still holds is played out, then
|
||||
`speaking_end` is sent.
|
||||
- When a person leaves the channel (gateway voice state or songbird's client disconnect), the bot
|
||||
sends `speaker_leave`, so their utterance finishes and the worker frees their place.
|
||||
- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID,
|
||||
server display name and voice session. Each `Joined` carries a new session number (from 1); an
|
||||
utterance belongs to the session Jarvis was in when its `speech_start` came (none outside a
|
||||
session, and none once that session ended or the worker restarted), and its `Partial` and
|
||||
`Final` carry that number even when they arrive later; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent;
|
||||
`SpeakStart` and `SpeakEnd`; `Stopped` when all speech is stopped (`stop()`, the slash command or the
|
||||
session ending), sent before the `SpeakEnd`s it causes; `Joined` and `Left` (with a reason: `command`,
|
||||
`moved`, `disconnected`, `shutdown`); `Addressed` and `Ranked` with the brain's answers to `ask_addressed` and `ask_rank`. `WorkerLost` when the voice worker connection ends: the finals of utterances in progress will not
|
||||
come, and the restarted worker numbers utterances afresh. `AnswerAsked { id, user, ended_at,
|
||||
final_at }` when `answer(user, text, heard)` queued speech `id` for a request whose speech ended
|
||||
and whose final arrived at those times (the dashboard times answers by it, `admin.spec.md`).
|
||||
`voice_members()` lists the people in Jarvis's voice channel that it hears (ID and display name),
|
||||
the same set reported to the worker.
|
||||
- **Speaking**: `say(text, more)` queues a TTS request and returns its id (`None` when not in a voice
|
||||
channel); `continue_text(id, text, more)` adds text to an open request. Each sentence the worker
|
||||
sends becomes one clip (a 48 kHz stereo WAV played through songbird without resampling); clips play
|
||||
in sentence order as soon as they arrive, and requests play first come, first served. A sentence that
|
||||
failed is skipped and the rest still play. `SpeakEnd` reports how many sentences played and whether
|
||||
the request was cancelled or failed; the worker is told the same (`tts_played`, with `last_ms`, how
|
||||
long the clip playing when it was stopped had played, timed from when it was handed to the player)
|
||||
so its voice recordings keep only what was heard (`voice_worker.spec.md`, "Recordings").
|
||||
- **Answering** (M4): `answer(user, text)` queues a request like `say` whose speech the worker makes
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. No answer
|
||||
from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed.
|
||||
`cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of
|
||||
an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and
|
||||
`ask_rank(id, candidates)` pass the conversation's questions to the brain. `prepare(user, text)` sends `prepare` (a
|
||||
call still being said) and expects nothing back.
|
||||
- **Earcon** (M4): `earcon()` queues the app's own earcon clip (`app/src/earcon.rs`, 48 kHz stereo,
|
||||
peak 0.3 of full scale) as a request of its own: no worker round trip, `SpeakStart`/`SpeakEnd` like
|
||||
any request, dropped by a stop. None when not in a voice channel.
|
||||
- **Stop** (slash command `stop`, or `stop()`): the current clip stops at the next 20 ms mix, every
|
||||
queued request is dropped, and unfinished requests are cancelled at the worker; each ends with
|
||||
`cancelled: true`. Stopping by voice ("자비스, 멈춰") is call detection's stop (`conversation.spec.md`),
|
||||
which does the same.
|
||||
- **Leave** (slash command `leave`, or `leave()`): stop speaking, send `speaker_leave` for everyone
|
||||
heard, and disconnect. When songbird's driver gives up the connection (kicked, channel deleted, long
|
||||
network loss) the session ends with `disconnected`. A driver disconnect without a reason is one the
|
||||
core asked for (leave or move) and is ignored, and a disconnect only ends the session its handler
|
||||
was attached to, so an old connection's late event never ends a newer session. When someone moves the bot to another channel,
|
||||
the session follows it.
|
||||
|
||||
## Slash command
|
||||
|
||||
One guild command, `/<commandName>` with subcommands `join` (the caller's voice channel), `leave`
|
||||
and `stop`, registered on every start with a create-or-update by name, so the app's other commands in
|
||||
the server are left alone. A failed registration (for example a bot invited without the
|
||||
`applications.commands` scope) is logged and the bot runs on without it. Usable only in the configured
|
||||
server. Replies are in Korean, ephemeral by default. Renaming `commandName` leaves the old command
|
||||
registered until it is removed (M5 setup).
|
||||
|
||||
## Chat channels
|
||||
|
||||
- The bot asks for the message intents only when some registered channel has reading on. Reading
|
||||
needs the Message Content intent, which is checked in the application's flags before login. When it
|
||||
is off, the bot still logs in, voice works, and an error explains how to turn it on; chat reads then
|
||||
fail with a clear error instead of returning empty messages.
|
||||
- **Live**: every new message in a readable registered channel, except the bot's own, is emitted as a
|
||||
`Chat` event: channel, message and author IDs, author display name, whether the author is a bot, the
|
||||
text, and the time.
|
||||
- **History**: `read_chat(channel_id, limit)` returns the latest 1-100 messages, oldest first, with
|
||||
`from_self` marking the bot's own.
|
||||
- **Send**: `send_chat(channel_id, text)` sends to a registered channel with sending on, split at
|
||||
Discord's 2,000-character limit at the last newline or space (never inside a character), and returns
|
||||
the message ids.
|
||||
- **Answer** (M4): `send_chat_reply(channel_id, message_id, text)` sends like Send, with the first
|
||||
part as a Discord reply to the message; nobody is pinged, not even its author. `chat_answer(id, user,
|
||||
text)` asks the brain for a chat answer (`reply` with `via: "chat"`); its text comes as `ReplyText`.
|
||||
`reply_delivered(id, ok)` tells the worker whether that answer was posted (`reply_delivered`), so a
|
||||
delete asked in an answer the person never saw cannot be confirmed (`voice_worker.spec.md`).
|
||||
- A channel that is not registered, or has the needed switch off, is refused with an access error.
|
||||
- **Chat tool actions** (M3, `chat_channels.spec.md`): when the worker link is ready and after login,
|
||||
the core sends `chat_channels` (every registered channel with its switches, and the bot's user ID).
|
||||
A `chat_action` from the worker runs `chat_action()` and answers `chat_action_result` with the
|
||||
message ids or the error text: `send` (as Send above), `edit` (only the bot's own messages, one
|
||||
message long, pings nobody), `react` (a Unicode or custom emoji), `delete`, `pin`, `unpin`. Each
|
||||
needs its channel switch. Discord's own refusals come back as errors, for example a voice channel's
|
||||
chat cannot pin.
|
||||
|
||||
## Tests
|
||||
|
||||
- `cargo test`: unit tests for the receive window (with a tagging fake and real libopus), playback
|
||||
order, gaps, stop and cancel, the earcon clip (before an answer, dropped by a stop; its sound short,
|
||||
silent at both ends, below its peak), speaker limit, chat access and splitting, settings, frames; the worker
|
||||
link against a fake server (hello, events, reconnect, failed TTS); the worker process with a
|
||||
stand-in `voice_worker` package run by a real Python (token only in the environment, restart on the
|
||||
same port, stop, the last output logged before a stop returns, a line logged after the stop gave up masked with its
|
||||
run's secrets).
|
||||
- `JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol` runs the worker link and
|
||||
playback against the real Python worker server with fake engines (`tests/fixtures/fake_engine_worker.py`).
|
||||
- `tests/live_discord.rs` (opt-in, `JARVIS_LIVE_DISCORD=1`, see the file header): two bots in one
|
||||
voice channel with the real worker on the GPU. It refuses to run if a person is in the channel or
|
||||
either bot is already in a voice channel, and cleans up even when a step fails. It checks command
|
||||
registration, transcription of recorded sentences played into the channel, speech the other bot
|
||||
hears and the STT reads back, stop, chat events, history, an edit and a delete, sending without pings, moving to a second
|
||||
voice channel and back (the session survives each move and only the two moves end one), the speaker
|
||||
leaving and coming back three times (each a DAVE group change), and leaving. The worker runs with
|
||||
`--history`; at the end the store must hold the speaker's finals under its display name and channel,
|
||||
its chat message with the edit and without the deleted one, Jarvis's own messages, a presence
|
||||
interval per join, and no presence left open. The chat tool's actions run on the channel too:
|
||||
editing its own message, refusing to edit the speaker's, a reaction, pin (Discord refuses it in a
|
||||
voice channel's chat, which must come back as an error), delete, and refusing an unregistered
|
||||
channel. It needs a second voice
|
||||
channel of the server (`JARVIS_LIVE_VOICE_CHANNEL_2`) and refuses if a person is in either.
|
||||
Measured on this server's RTX 5050 (2026-11-11): five sentences all heard (character error rate 0%
|
||||
for three, 5% and 12.5% for two), 0.71-0.90 s from the end of speech to the final, Jarvis's speech
|
||||
read back at 3.4-6.9%, sound gone 0.03-0.08 s after stop, and the sentence heard exactly after each
|
||||
of three rejoins. With history (2026-11-11): 8 voice records under the speaker's name, the edited
|
||||
chat message only, 3 of Jarvis's own messages, 5 presence intervals, none open.
|
||||
|
||||
## Known risks
|
||||
|
||||
- songbird issue #310 (open): receive can stall after a DAVE (MLS) group change on some joins. The live
|
||||
test covers four changes (the second bot joining, then leaving and rejoining three times) without a
|
||||
stall; a channel where many people come and go over hours still needs a longer check before M4.
|
||||
106
app/src/discord_setup.rs
Normal file
106
app/src/discord_setup.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! The setup wizard's Discord steps (`app/src/admin.spec.md`, "Admin commands"): check a bot
|
||||
//! token, the Message Content intent and the invite link, and list servers and channels for the
|
||||
//! pickers. REST only: nothing here connects to the gateway or joins anything.
|
||||
|
||||
use serde::Serialize;
|
||||
use serenity::all::{ApplicationFlags, ChannelType, GuildId, Http, Permissions};
|
||||
|
||||
/// What the bot needs: read and post in chat channels (reply, react, delete, pin), keep channel
|
||||
/// names and topics in step (desktop-app.md), and talk in voice.
|
||||
pub fn needed_permissions() -> Permissions {
|
||||
Permissions::VIEW_CHANNEL | Permissions::SEND_MESSAGES | Permissions::READ_MESSAGE_HISTORY
|
||||
| Permissions::ADD_REACTIONS | Permissions::MANAGE_MESSAGES | Permissions::MANAGE_CHANNELS
|
||||
| Permissions::CONNECT | Permissions::SPEAK | Permissions::USE_VAD
|
||||
}
|
||||
|
||||
/// The link that adds the bot to a server with those permissions and its slash command.
|
||||
pub fn invite_url(application_id: u64) -> String {
|
||||
format!("https://discord.com/oauth2/authorize?client_id={application_id}&scope=bot%20applications.commands&permissions={}",
|
||||
needed_permissions().bits())
|
||||
}
|
||||
|
||||
/// The Message Content intent is on (either the full or the limited flag of unverified bots).
|
||||
pub fn message_content_on(flags: Option<ApplicationFlags>) -> bool {
|
||||
flags.is_some_and(|f| f.intersects(ApplicationFlags::GATEWAY_MESSAGE_CONTENT
|
||||
| ApplicationFlags::GATEWAY_MESSAGE_CONTENT_LIMITED))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TokenCheck {
|
||||
pub bot_tag: String,
|
||||
pub bot_id: String,
|
||||
pub application_id: String,
|
||||
pub message_content: bool,
|
||||
pub invite_url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
pub struct Named {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
pub struct Channels {
|
||||
pub voice: Vec<Named>,
|
||||
pub text: Vec<Named>,
|
||||
}
|
||||
|
||||
fn err(e: serenity::Error) -> String {
|
||||
match e {
|
||||
serenity::Error::Http(h) if h.status_code().is_some_and(|s| s.as_u16() == 401) =>
|
||||
"디스코드가 이 봇 토큰을 받아들이지 않았습니다".into(),
|
||||
e => format!("디스코드에 연결하지 못했습니다: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check a token without storing it.
|
||||
pub async fn check_token(token: &str) -> Result<TokenCheck, String> {
|
||||
let http = Http::new(token.trim());
|
||||
let me = http.get_current_user().await.map_err(err)?;
|
||||
let app = http.get_current_application_info().await.map_err(err)?;
|
||||
Ok(TokenCheck { bot_tag: me.tag(), bot_id: me.id.to_string(), application_id: app.id.to_string(),
|
||||
message_content: message_content_on(app.flags), invite_url: invite_url(app.id.get()) })
|
||||
}
|
||||
|
||||
/// The servers the bot is in.
|
||||
pub async fn guilds(token: &str) -> Result<Vec<Named>, String> {
|
||||
let list = Http::new(token).get_guilds(None, Some(200)).await.map_err(err)?;
|
||||
Ok(list.into_iter().map(|g| Named { id: g.id.to_string(), name: g.name }).collect())
|
||||
}
|
||||
|
||||
/// A server's voice and text channels, in their server order.
|
||||
pub async fn channels(token: &str, guild_id: u64) -> Result<Channels, String> {
|
||||
let mut list = Http::new(token).get_channels(GuildId::new(guild_id)).await.map_err(err)?;
|
||||
list.sort_by_key(|c| c.position);
|
||||
let pick = |kinds: &[ChannelType]| list.iter().filter(|c| kinds.contains(&c.kind))
|
||||
.map(|c| Named { id: c.id.to_string(), name: c.name.clone() }).collect();
|
||||
Ok(Channels { voice: pick(&[ChannelType::Voice, ChannelType::Stage]), text: pick(&[ChannelType::Text]) })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_invite_asks_for_the_bot_its_command_and_its_permissions() {
|
||||
let url = invite_url(123456789012345678);
|
||||
assert!(url.contains("client_id=123456789012345678"));
|
||||
assert!(url.contains("scope=bot%20applications.commands"));
|
||||
let bits: u64 = url.rsplit("permissions=").next().unwrap().parse().unwrap();
|
||||
let p = Permissions::from_bits_truncate(bits);
|
||||
for need in [Permissions::CONNECT, Permissions::SPEAK, Permissions::SEND_MESSAGES, Permissions::MANAGE_CHANNELS] {
|
||||
assert!(p.contains(need), "{need:?}");
|
||||
}
|
||||
assert!(!p.contains(Permissions::ADMINISTRATOR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_content_counts_when_full_or_limited() {
|
||||
assert!(message_content_on(Some(ApplicationFlags::GATEWAY_MESSAGE_CONTENT)));
|
||||
assert!(message_content_on(Some(ApplicationFlags::GATEWAY_MESSAGE_CONTENT_LIMITED)));
|
||||
assert!(!message_content_on(Some(ApplicationFlags::GATEWAY_GUILD_MEMBERS)));
|
||||
assert!(!message_content_on(None));
|
||||
}
|
||||
}
|
||||
253
app/src/download.rs
Normal file
253
app/src/download.rs
Normal file
@@ -0,0 +1,253 @@
|
||||
//! Downloads for the installer (`app/src/admin.spec.md`, "Install"): resumed where they stopped, and
|
||||
//! checked against a pinned SHA-256 before they are used.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use futures_util::StreamExt;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DownloadError(pub String);
|
||||
|
||||
impl std::fmt::Display for DownloadError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for DownloadError {}
|
||||
|
||||
/// Await `fut`, giving up when `cancel` is set (checked every 200 ms): a server that keeps the
|
||||
/// connection open and sends nothing cannot hold a cancel.
|
||||
async fn or_cancel<T>(fut: impl std::future::Future<Output = T>, cancel: &AtomicBool) -> Result<T, DownloadError> {
|
||||
tokio::pin!(fut);
|
||||
loop {
|
||||
tokio::select! {
|
||||
v = &mut fut => return Ok(v),
|
||||
_ = tokio::time::sleep(std::time::Duration::from_millis(200)) => {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(DownloadError("cancelled".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn e(what: &str) -> impl FnOnce(std::io::Error) -> DownloadError + '_ {
|
||||
move |err| DownloadError(format!("{what}: {err}"))
|
||||
}
|
||||
|
||||
pub fn sha256_file(path: &Path) -> std::io::Result<String> {
|
||||
let mut f = std::fs::File::open(path)?;
|
||||
let mut h = Sha256::new();
|
||||
std::io::copy(&mut f, &mut h)?;
|
||||
Ok(format!("{:x}", h.finalize()))
|
||||
}
|
||||
|
||||
/// Download `url` to `dest`, checked against `sha256` (lowercase hex). The bytes go to
|
||||
/// `dest.part` first; an interrupted download continues from there (HTTP Range), a server that
|
||||
/// ignores the range starts it over. A wrong checksum deletes the part file, so the next try starts
|
||||
/// clean. `progress(done, total)` is called as bytes arrive; `cancel` stops it between chunks (the
|
||||
/// part file stays for the next try). An existing `dest` with the right checksum is kept as it is.
|
||||
pub async fn download(url: &str, dest: &Path, sha256: &str, progress: &(dyn Fn(u64, Option<u64>) + Sync),
|
||||
cancel: &AtomicBool) -> Result<(), DownloadError> {
|
||||
if dest.exists() && sha256_file(dest).map_err(e("could not read the download"))? == sha256 {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(dir) = dest.parent() {
|
||||
tokio::fs::create_dir_all(dir).await.map_err(e("could not create the download folder"))?;
|
||||
}
|
||||
let part = PathBuf::from(format!("{}.part", dest.display()));
|
||||
let have = tokio::fs::metadata(&part).await.map(|m| m.len()).unwrap_or(0);
|
||||
let client = reqwest::Client::builder().build().map_err(|x| DownloadError(x.to_string()))?;
|
||||
let mut req = client.get(url);
|
||||
if have > 0 {
|
||||
req = req.header(reqwest::header::RANGE, format!("bytes={have}-"));
|
||||
}
|
||||
let resp = or_cancel(req.send(), cancel).await?
|
||||
.map_err(|x| DownloadError(format!("could not download {url}: {x}")))?;
|
||||
let status = resp.status();
|
||||
if status.as_u16() == 416 && have > 0 {
|
||||
// The part file already holds every byte: check it below.
|
||||
} else if !status.is_success() {
|
||||
return Err(DownloadError(format!("download of {url} failed: HTTP {status}")));
|
||||
}
|
||||
let resumed = status.as_u16() == 206;
|
||||
let total = resp.content_length().map(|n| if resumed { n + have } else { n });
|
||||
let mut file = tokio::fs::OpenOptions::new().create(true).write(true).append(resumed).truncate(!resumed && status.as_u16() != 416)
|
||||
.open(&part).await.map_err(e("could not write the download"))?;
|
||||
let mut done = if resumed || status.as_u16() == 416 { have } else { 0 };
|
||||
progress(done, total);
|
||||
if status.as_u16() != 416 {
|
||||
let mut stream = resp.bytes_stream();
|
||||
loop {
|
||||
let next = match or_cancel(stream.next(), cancel).await {
|
||||
Ok(n) => n,
|
||||
Err(c) => {
|
||||
file.flush().await.ok(); // what arrived stays for the next try
|
||||
return Err(c);
|
||||
}
|
||||
};
|
||||
let Some(chunk) = next else { break };
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
file.flush().await.ok();
|
||||
return Err(DownloadError("cancelled".into()));
|
||||
}
|
||||
let chunk = chunk.map_err(|x| DownloadError(format!("download of {url} broke off: {x}")))?;
|
||||
file.write_all(&chunk).await.map_err(e("could not write the download"))?;
|
||||
done += chunk.len() as u64;
|
||||
progress(done, total);
|
||||
}
|
||||
}
|
||||
file.flush().await.map_err(e("could not write the download"))?;
|
||||
drop(file);
|
||||
let got = sha256_file(&part).map_err(e("could not read the download"))?;
|
||||
if got != sha256 {
|
||||
let _ = tokio::fs::remove_file(&part).await;
|
||||
return Err(DownloadError(format!("the download of {url} is damaged (checksum {got}, expected {sha256})")));
|
||||
}
|
||||
tokio::fs::rename(&part, dest).await.map_err(e("could not move the download into place"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// A file server that honours `Range: bytes=N-` (unless `ranges` is off) and can stop after
|
||||
/// `cut` bytes of a response, like a broken connection. Records the ranges asked for.
|
||||
async fn serve(body: Vec<u8>, ranges: bool, cut: Option<usize>) -> (String, Arc<Mutex<Vec<String>>>) {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
|
||||
let asked = Arc::new(Mutex::new(vec![]));
|
||||
let a2 = asked.clone();
|
||||
let cut = Arc::new(Mutex::new(cut));
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((mut s, _)) = l.accept().await else { return };
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let n = s.read(&mut buf).await.unwrap_or(0);
|
||||
let head = String::from_utf8_lossy(&buf[..n]).to_string();
|
||||
let from = head.lines().find_map(|l| l.strip_prefix("range: bytes=").or(l.strip_prefix("Range: bytes=")))
|
||||
.and_then(|r| r.trim_end_matches('-').parse::<usize>().ok());
|
||||
a2.lock().unwrap().push(from.map(|f| f.to_string()).unwrap_or_default());
|
||||
let (status, slice) = match from {
|
||||
Some(f) if ranges && f >= body.len() => ("416 Range Not Satisfiable", &body[0..0]),
|
||||
Some(f) if ranges => ("206 Partial Content", &body[f..]),
|
||||
_ => ("200 OK", &body[..]),
|
||||
};
|
||||
let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", slice.len());
|
||||
let _ = s.write_all(head.as_bytes()).await;
|
||||
let send = cut.lock().unwrap().take().map_or(slice.len(), |c| c.min(slice.len()));
|
||||
let _ = s.write_all(&slice[..send]).await;
|
||||
}
|
||||
});
|
||||
(url, asked)
|
||||
}
|
||||
|
||||
fn data() -> (Vec<u8>, String) {
|
||||
let b: Vec<u8> = (0..200_000u32).map(|i| (i * 31 % 251) as u8).collect();
|
||||
let h = format!("{:x}", Sha256::digest(&b));
|
||||
(b, h)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_broken_download_resumes_where_it_stopped() {
|
||||
let (body, sum) = data();
|
||||
let (url, asked) = serve(body.clone(), true, Some(70_000)).await;
|
||||
let dest = temp_dir("dl1").join("f.bin");
|
||||
let never = AtomicBool::new(false);
|
||||
assert!(download(&url, &dest, &sum, &|_, _| {}, &never).await.is_err(), "the first try breaks off");
|
||||
assert!(!dest.exists());
|
||||
let seen = Mutex::new(vec![]);
|
||||
download(&url, &dest, &sum, &|d, t| seen.lock().unwrap().push((d, t)), &never).await.unwrap();
|
||||
assert_eq!(std::fs::read(&dest).unwrap(), body);
|
||||
assert_eq!(asked.lock().unwrap().as_slice(), ["", "70000"]);
|
||||
let seen = seen.lock().unwrap();
|
||||
assert_eq!(seen.first(), Some(&(70_000, Some(200_000))));
|
||||
assert_eq!(seen.last(), Some(&(200_000, Some(200_000))));
|
||||
// Present and right: nothing is fetched again.
|
||||
download(&url, &dest, &sum, &|_, _| {}, &never).await.unwrap();
|
||||
assert_eq!(asked.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_server_without_ranges_starts_over_and_a_bad_checksum_is_thrown_away() {
|
||||
let (body, sum) = data();
|
||||
let (url, _) = serve(body.clone(), false, Some(50_000)).await;
|
||||
let dest = temp_dir("dl2").join("f.bin");
|
||||
let never = AtomicBool::new(false);
|
||||
let _ = download(&url, &dest, &sum, &|_, _| {}, &never).await;
|
||||
download(&url, &dest, &sum, &|_, _| {}, &never).await.unwrap();
|
||||
assert_eq!(std::fs::read(&dest).unwrap(), body);
|
||||
|
||||
let wrong = "0".repeat(64);
|
||||
let dest2 = temp_dir("dl3").join("f.bin");
|
||||
let err = download(&url, &dest2, &wrong, &|_, _| {}, &never).await.unwrap_err();
|
||||
assert!(err.0.contains("damaged"));
|
||||
assert!(!dest2.exists() && !PathBuf::from(format!("{}.part", dest2.display())).exists());
|
||||
}
|
||||
|
||||
/// Accepts, optionally sends headers and the first `bytes`, then holds the connection silently.
|
||||
async fn stall(headers: bool, bytes: usize) -> String {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
let mut held = vec![];
|
||||
while let Ok((mut s, _)) = l.accept().await {
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let _ = s.read(&mut buf).await;
|
||||
if headers {
|
||||
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n").await;
|
||||
let _ = s.write_all(&vec![7u8; bytes]).await;
|
||||
}
|
||||
held.push(s);
|
||||
}
|
||||
});
|
||||
url
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_silent_server_does_not_hold_a_cancel() {
|
||||
for (headers, bytes) in [(false, 0), (true, 5000)] {
|
||||
let url = stall(headers, bytes).await;
|
||||
let dest = temp_dir("dlstall").join("f.bin");
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let c2 = cancel.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
||||
c2.store(true, Ordering::Relaxed);
|
||||
});
|
||||
let t = std::time::Instant::now();
|
||||
let err = download(&url, &dest, &"0".repeat(64), &|_, _| {}, &cancel).await.unwrap_err();
|
||||
assert_eq!(err.0, "cancelled", "headers={headers}");
|
||||
assert!(t.elapsed() < std::time::Duration::from_secs(3), "{:?}", t.elapsed());
|
||||
if headers {
|
||||
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
|
||||
assert_eq!(part, bytes as u64, "what arrived stays for the next try");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_keeps_the_part_for_the_next_try() {
|
||||
let (body, sum) = data();
|
||||
let (url, asked) = serve(body.clone(), true, None).await;
|
||||
let dest = temp_dir("dl4").join("f.bin");
|
||||
let cancel = AtomicBool::new(false);
|
||||
let err = download(&url, &dest, &sum, &|d, _| if d > 0 { cancel.store(true, Ordering::Relaxed) }, &cancel)
|
||||
.await.unwrap_err();
|
||||
assert_eq!(err.0, "cancelled");
|
||||
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
|
||||
assert!(part > 0 && part < body.len() as u64);
|
||||
cancel.store(false, Ordering::Relaxed);
|
||||
download(&url, &dest, &sum, &|_, _| {}, &cancel).await.unwrap();
|
||||
assert_eq!(std::fs::read(&dest).unwrap(), body);
|
||||
assert_eq!(asked.lock().unwrap()[1], part.to_string());
|
||||
}
|
||||
}
|
||||
65
app/src/earcon.rs
Normal file
65
app/src/earcon.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
//! The short "I heard you" sound (`app/src/conversation.spec.md`, "Earcon"): two rising bell tones,
|
||||
//! made here so the app ships no audio file. 48 kHz stereo PCM16, the format the player takes.
|
||||
|
||||
use std::f32::consts::TAU;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const RATE: f32 = 48_000.0;
|
||||
/// Peak level, well below full scale so the cue never sounds louder than speech.
|
||||
pub const PEAK: f32 = 0.3;
|
||||
|
||||
/// (start s, frequency Hz, length s) of each tone: "띠" then "링".
|
||||
const TONES: [(f32, f32, f32); 2] = [(0.0, 1318.5, 0.11), (0.09, 1975.5, 0.22)];
|
||||
|
||||
/// The earcon as 48 kHz stereo PCM16 (little-endian), made once.
|
||||
pub fn pcm() -> &'static [u8] {
|
||||
static PCM: OnceLock<Vec<u8>> = OnceLock::new();
|
||||
PCM.get_or_init(make)
|
||||
}
|
||||
|
||||
fn make() -> Vec<u8> {
|
||||
let total = TONES.iter().map(|(s, _, l)| s + l).fold(0.0, f32::max);
|
||||
let frames = (total * RATE) as usize;
|
||||
let mut out = Vec::with_capacity(frames * 4);
|
||||
for i in 0..frames {
|
||||
let t = i as f32 / RATE;
|
||||
let mut v = 0.0;
|
||||
for (start, freq, len) in TONES {
|
||||
let u = t - start;
|
||||
if !(0.0..len).contains(&u) {
|
||||
continue;
|
||||
}
|
||||
// 5 ms rise and a bell-like decay that reaches silence at the tone's end: no clicks.
|
||||
let rise = (u / 0.005).min(1.0);
|
||||
let fall = (1.0 - u / len).powi(2);
|
||||
let tone = (TAU * freq * u).sin() + 0.25 * (TAU * 2.0 * freq * u).sin();
|
||||
v += rise * fall * tone / 1.25;
|
||||
}
|
||||
let s = (v.clamp(-1.0, 1.0) * PEAK * i16::MAX as f32) as i16;
|
||||
let b = s.to_le_bytes();
|
||||
out.extend_from_slice(&b);
|
||||
out.extend_from_slice(&b);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn samples() -> Vec<i16> {
|
||||
pcm().chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_short_stereo_audio_that_starts_and_ends_silent_and_stays_below_its_peak() {
|
||||
let s = samples();
|
||||
let seconds = s.len() as f32 / 2.0 / RATE;
|
||||
assert!((0.2..0.5).contains(&seconds), "{seconds} s");
|
||||
assert!(s.chunks_exact(2).all(|lr| lr[0] == lr[1]), "both channels carry the same sound");
|
||||
let limit = (PEAK * i16::MAX as f32) as u16 + 1;
|
||||
let loudest = s.iter().map(|v| v.unsigned_abs()).max().unwrap();
|
||||
assert!(loudest <= limit && loudest > limit / 2, "loudest {loudest}");
|
||||
assert!(s[..4].iter().all(|v| v.abs() < 200) && s[s.len() - 4..].iter().all(|v| v.abs() < 200));
|
||||
}
|
||||
}
|
||||
160
app/src/features.rs
Normal file
160
app/src/features.rs
Normal file
@@ -0,0 +1,160 @@
|
||||
//! The brain's features and MCP servers (`app/src/admin.spec.md`, "Features and MCP servers"):
|
||||
//! what the app writes into the brain's settings, and the MCP servers' environment values, which
|
||||
//! are secrets and reach the worker only through its environment.
|
||||
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use crate::secrets::{Secrets, SecretsError};
|
||||
use crate::settings::{FeatureSettings, McpServerSettings, Settings};
|
||||
|
||||
/// The worker's environment variable with the MCP servers' environment values
|
||||
/// (`voice_worker/brain.py`, `MCP_ENV_VAR`).
|
||||
pub const MCP_ENV_VAR: &str = "JARVIS_MCP_ENV";
|
||||
|
||||
/// The brain's built-in tools behind each feature switch (`jarvis/tools/registry.py`).
|
||||
pub const FEATURE_TOOLS: &[(&str, &[&str])] = &[
|
||||
("weather", &["getWeather"]),
|
||||
("webSearch", &["webSearch", "fetchWebPage"]),
|
||||
("meals", &["logMeal", "fetchMeals", "deleteMeal"]),
|
||||
("browser", &["botBrowser"]),
|
||||
("schedule", &["schedule"]),
|
||||
("notes", &["personalNotes"]),
|
||||
("collect", &["collectedNews"]),
|
||||
];
|
||||
|
||||
/// The built-in tools of the features that are off, and `screenView` unless screen viewing is
|
||||
/// available (admin.spec.md, "Screen viewing").
|
||||
pub fn disabled_tools(s: &Settings) -> Vec<&'static str> {
|
||||
let mut off = feature_tools_off(&s.features);
|
||||
if !crate::screen::available(s) {
|
||||
off.push("screenView");
|
||||
}
|
||||
off
|
||||
}
|
||||
|
||||
fn feature_tools_off(f: &FeatureSettings) -> Vec<&'static str> {
|
||||
let on = |key: &str| match key {
|
||||
"weather" => f.weather,
|
||||
"webSearch" => f.web_search,
|
||||
"browser" => f.browser,
|
||||
"schedule" => f.schedule,
|
||||
"notes" => f.notes,
|
||||
"collect" => f.collect,
|
||||
_ => f.meals,
|
||||
};
|
||||
FEATURE_TOOLS.iter().filter(|(k, _)| !on(k)).flat_map(|(_, tools)| tools.iter().copied()).collect()
|
||||
}
|
||||
|
||||
/// The brain's `mcps`: the enabled servers, without their environment values.
|
||||
pub fn brain_mcps(servers: &[McpServerSettings]) -> Value {
|
||||
Value::Object(servers.iter().filter(|s| s.enabled)
|
||||
.map(|s| (s.name.clone(), json!({"transport": "stdio", "command": s.command, "args": s.args})))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Where one environment value of a server is kept in the secrets.
|
||||
pub fn secret_name(server: &str, var: &str) -> String {
|
||||
format!("mcp:{server}:{var}")
|
||||
}
|
||||
|
||||
/// `{server: {NAME: value}}` for the enabled servers, with the values that are stored.
|
||||
pub fn mcp_env(secrets: &Secrets, servers: &[McpServerSettings]) -> Result<Value, SecretsError> {
|
||||
let mut out = Map::new();
|
||||
for s in servers.iter().filter(|s| s.enabled) {
|
||||
let mut vars = Map::new();
|
||||
for var in &s.env {
|
||||
if let Some(v) = secrets.get(&secret_name(&s.name, var))? {
|
||||
vars.insert(var.clone(), Value::String(v));
|
||||
}
|
||||
}
|
||||
if !vars.is_empty() {
|
||||
out.insert(s.name.clone(), Value::Object(vars));
|
||||
}
|
||||
}
|
||||
Ok(Value::Object(out))
|
||||
}
|
||||
|
||||
/// Which environment values are stored: `{server: {NAME: true|false}}` (never the values).
|
||||
pub fn env_status(secrets: &Secrets, servers: &[McpServerSettings]) -> Result<Value, SecretsError> {
|
||||
let names = secrets.names()?;
|
||||
Ok(Value::Object(servers.iter().map(|s| {
|
||||
let vars = s.env.iter().map(|v| (v.clone(), Value::Bool(names.contains(&secret_name(&s.name, v))))).collect();
|
||||
(s.name.clone(), Value::Object(vars))
|
||||
}).collect()))
|
||||
}
|
||||
|
||||
/// Remove the stored values of servers and names no longer listed.
|
||||
pub fn prune(secrets: &Secrets, servers: &[McpServerSettings]) -> Result<(), SecretsError> {
|
||||
let keep: Vec<String> = servers.iter().flat_map(|s| s.env.iter().map(|v| secret_name(&s.name, v))).collect();
|
||||
for name in secrets.names()? {
|
||||
if name.starts_with("mcp:") && !keep.contains(&name) {
|
||||
secrets.remove(&name)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use crate::settings::parse_settings;
|
||||
|
||||
fn servers(v: Value) -> Vec<McpServerSettings> {
|
||||
parse_settings(&json!({"mcpServers": v})).unwrap().mcp_servers
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_feature_tools_are_the_brains_tool_names() {
|
||||
let registry = std::fs::read_to_string(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../src/jarvis/tools/registry.py")).unwrap();
|
||||
for (_, tools) in FEATURE_TOOLS {
|
||||
for t in *tools {
|
||||
assert!(registry.contains(&format!("\"{t}\":")), "{t} is not a built-in tool");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn features_that_are_off_turn_their_tools_off() {
|
||||
let mut s = Settings::default();
|
||||
s.pc_control.allowed_users = vec![1];
|
||||
let screen = if cfg!(windows) { vec![] } else { vec!["screenView"] };
|
||||
assert_eq!(disabled_tools(&s), screen, "all on by default");
|
||||
s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true };
|
||||
let mut want = vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"];
|
||||
want.extend(&screen);
|
||||
assert_eq!(disabled_tools(&s), want);
|
||||
// Screen viewing needs someone registered for PC control (and Windows).
|
||||
assert!(disabled_tools(&Settings::default()).contains(&"screenView"));
|
||||
s.features.screen = false;
|
||||
assert!(disabled_tools(&s).contains(&"screenView"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_enabled_servers_reach_the_brain_and_their_values_only_through_the_environment() {
|
||||
let s = servers(json!([
|
||||
{"name": "brave", "command": "npx", "args": ["-y", "brave"], "env": ["BRAVE_API_KEY", "UNSET"]},
|
||||
{"name": "off", "command": "uvx", "env": ["K"], "enabled": false}]));
|
||||
assert_eq!(brain_mcps(&s), json!({"brave": {"transport": "stdio", "command": "npx", "args": ["-y", "brave"]}}));
|
||||
let secrets = Secrets::new(&temp_dir("mcpenv").join("secrets.dat"));
|
||||
secrets.set(&secret_name("brave", "BRAVE_API_KEY"), "key-1").unwrap();
|
||||
secrets.set(&secret_name("off", "K"), "key-2").unwrap();
|
||||
assert_eq!(mcp_env(&secrets, &s).unwrap(), json!({"brave": {"BRAVE_API_KEY": "key-1"}}));
|
||||
assert_eq!(env_status(&secrets, &s).unwrap(),
|
||||
json!({"brave": {"BRAVE_API_KEY": true, "UNSET": false}, "off": {"K": true}}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_of_removed_servers_and_names_are_deleted() {
|
||||
let secrets = Secrets::new(&temp_dir("mcpprune").join("secrets.dat"));
|
||||
secrets.set(crate::secrets::DISCORD_TOKEN, "tok").unwrap();
|
||||
for (srv, var) in [("a", "K1"), ("a", "K2"), ("b", "K")] {
|
||||
secrets.set(&secret_name(srv, var), "v").unwrap();
|
||||
}
|
||||
prune(&secrets, &servers(json!([{"name": "a", "command": "x", "env": ["K1"]}]))).unwrap();
|
||||
let mut left = secrets.names().unwrap();
|
||||
left.sort();
|
||||
assert_eq!(left, vec![crate::secrets::DISCORD_TOKEN.to_string(), secret_name("a", "K1")]);
|
||||
}
|
||||
}
|
||||
73
app/src/frames.rs
Normal file
73
app/src/frames.rs
Normal file
@@ -0,0 +1,73 @@
|
||||
//! Binary frame layout shared with the Python voice worker (`src/voice_worker/protocol.py`).
|
||||
|
||||
pub const KIND_AUDIO: u8 = 0x01;
|
||||
pub const KIND_TTS: u8 = 0x02;
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct TtsFrame {
|
||||
pub id: String,
|
||||
pub seq: u32,
|
||||
pub pcm: Vec<u8>,
|
||||
}
|
||||
|
||||
/// `0x01 | u8 n | speaker id | PCM16LE` in the hello "audio" format. None if the id does not fit.
|
||||
pub fn encode_audio_frame(speaker: &str, pcm: &[i16]) -> Option<Vec<u8>> {
|
||||
let id = speaker.as_bytes();
|
||||
if id.is_empty() || id.len() > 255 {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity(2 + id.len() + pcm.len() * 2);
|
||||
out.push(KIND_AUDIO);
|
||||
out.push(id.len() as u8);
|
||||
out.extend_from_slice(id);
|
||||
for s in pcm {
|
||||
out.extend_from_slice(&s.to_le_bytes());
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// `0x02 | u8 n | request id | u32 LE seq | PCM16LE` in the hello "tts" format. None if malformed.
|
||||
pub fn decode_tts_frame(frame: &[u8]) -> Option<TtsFrame> {
|
||||
if frame.len() < 2 || frame[0] != KIND_TTS {
|
||||
return None;
|
||||
}
|
||||
let n = frame[1] as usize;
|
||||
let end = 2 + n;
|
||||
if n == 0 || frame.len() < end + 4 {
|
||||
return None;
|
||||
}
|
||||
let id = std::str::from_utf8(&frame[2..end]).ok()?.to_owned();
|
||||
let seq = u32::from_le_bytes(frame[end..end + 4].try_into().ok()?);
|
||||
Some(TtsFrame { id, seq, pcm: frame[end + 4..].to_vec() })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn audio_frame_is_kind_id_length_id_then_pcm() {
|
||||
let f = encode_audio_frame("유저", &[1, -2]).unwrap();
|
||||
let id = "유저".as_bytes();
|
||||
let mut want = vec![KIND_AUDIO, id.len() as u8];
|
||||
want.extend_from_slice(id);
|
||||
want.extend_from_slice(&[1, 0, 0xfe, 0xff]);
|
||||
assert_eq!(f, want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_must_fit_in_a_byte() {
|
||||
assert!(encode_audio_frame("", &[0]).is_none());
|
||||
assert!(encode_audio_frame(&"x".repeat(256), &[0]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tts_frames_decode_and_truncated_ones_are_refused() {
|
||||
let mut f = vec![KIND_TTS, 2, b'r', b'1'];
|
||||
f.extend_from_slice(&7u32.to_le_bytes());
|
||||
f.extend_from_slice(&[9, 9]);
|
||||
assert_eq!(decode_tts_frame(&f), Some(TtsFrame { id: "r1".into(), seq: 7, pcm: vec![9, 9] }));
|
||||
assert_eq!(decode_tts_frame(&f[..5]), None);
|
||||
assert_eq!(decode_tts_frame(&[KIND_AUDIO, 1, b'a', 0, 0, 0, 0]), None);
|
||||
}
|
||||
}
|
||||
702
app/src/game.rs
Normal file
702
app/src/game.rs
Normal file
@@ -0,0 +1,702 @@
|
||||
//! Game detection (`docs/v2/pc-features.md`, "게임 감지 기준"; admin.spec.md, "Game detection"):
|
||||
//! whether a game runs, from the foreground window, Windows' full-screen state and where the
|
||||
//! program lives. It drives the performance stages (M7). Nothing here looks at the game's screen.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::settings::GameSettings;
|
||||
|
||||
/// How often the foreground is looked at.
|
||||
const EVERY: Duration = Duration::from_secs(2);
|
||||
/// A borderless window counts as a game when its process uses at least this much of the GPU's 3D
|
||||
/// engine (the M0 `games` check's threshold).
|
||||
const MIN_GPU_3D: f64 = 10.0;
|
||||
|
||||
/// Programs that are never games: browsers (also the bot's Chrome), video players, the desktop,
|
||||
/// Discord, launchers' own windows and this app.
|
||||
const NOT_GAMES: &[&str] = &[
|
||||
"chrome.exe", "msedge.exe", "firefox.exe", "opera.exe", "brave.exe", "whale.exe", "vivaldi.exe",
|
||||
"msedgewebview2.exe", "vlc.exe", "potplayer.exe", "potplayermini.exe", "potplayermini64.exe", "mpc-hc.exe",
|
||||
"mpc-hc64.exe", "mpc-be64.exe", "wmplayer.exe", "video.ui.exe", "explorer.exe", "discord.exe", "jarvis.exe",
|
||||
"steam.exe", "steamwebhelper.exe", "epicgameslauncher.exe", "epicwebhelper.exe", "riotclientservices.exe",
|
||||
"riot client.exe", "riotclientux.exe", "riotclientuxrender.exe", "leagueclient.exe", "leagueclientux.exe",
|
||||
"leagueclientuxrender.exe", "battle.net.exe", "agent.exe",
|
||||
];
|
||||
|
||||
/// Launcher game folders, as the path parts before the game's own folder: a program under one
|
||||
/// counts as a game whatever its window looks like, unless that folder is one of the launcher's
|
||||
/// own (its client, shared runtimes, or a non-game app such as Wallpaper Engine, which runs all
|
||||
/// the time). Battle.net installs anywhere, so its games come from their install folders
|
||||
/// (`InstalledGame`).
|
||||
const LAUNCHERS: &[(&[&str], &str, &[&str])] = &[
|
||||
(&["steamapps", "common"], "Steam", &["steamworks shared", "wallpaper_engine"]),
|
||||
(&["epic games"], "Epic", &["launcher", "epic online services"]),
|
||||
(&["riot games"], "Riot", &["riot client"]),
|
||||
];
|
||||
|
||||
/// A game a launcher installed somewhere of its own choosing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct InstalledGame {
|
||||
pub dir: String,
|
||||
pub name: String,
|
||||
pub launcher: &'static str,
|
||||
}
|
||||
|
||||
/// A Battle.net game from its Windows uninstall entry: Battle.net writes one per game, removed by
|
||||
/// its "Blizzard Uninstaller" (also for other publishers' games); Blizzard's own entries count too.
|
||||
/// Battle.net itself and entries without a folder are not games.
|
||||
pub fn battle_net_game(name: &str, publisher: &str, uninstall: &str, dir: &str) -> Option<InstalledGame> {
|
||||
let by_battle_net = uninstall.to_ascii_lowercase().contains("blizzard uninstaller")
|
||||
|| publisher.trim().eq_ignore_ascii_case("blizzard entertainment");
|
||||
let dir = dir.trim().trim_end_matches(['\\', '/']);
|
||||
(by_battle_net && !name.trim().eq_ignore_ascii_case("battle.net") && !dir.is_empty())
|
||||
.then(|| InstalledGame { dir: dir.to_owned(), name: name.trim().to_owned(), launcher: "Battle.net" })
|
||||
}
|
||||
|
||||
/// A path's parts (folders and file name), whichever slash it uses.
|
||||
fn parts(path: &str) -> Vec<&str> {
|
||||
path.split(['\\', '/']).filter(|p| !p.is_empty()).collect()
|
||||
}
|
||||
|
||||
/// The launcher and game name of a program under a launcher's game folder or an installed game's
|
||||
/// folder. Parts are compared without case (ASCII), never cut by byte positions.
|
||||
fn launcher_game(path: &str, installed: &[InstalledGame]) -> Option<(&'static str, String)> {
|
||||
let p = parts(path);
|
||||
let same = |a: &[&str], b: &[&str]| a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.eq_ignore_ascii_case(y));
|
||||
for g in installed {
|
||||
let d = parts(&g.dir);
|
||||
if p.len() > d.len() && same(&p[..d.len()], &d) {
|
||||
return Some((g.launcher, g.name.clone()));
|
||||
}
|
||||
}
|
||||
for (marker, launcher, own) in LAUNCHERS {
|
||||
// The game's folder follows the marker, and the program is inside it.
|
||||
if let Some(i) = (0..p.len().saturating_sub(marker.len() + 1)).find(|&i| same(&p[i..i + marker.len()], marker)) {
|
||||
let folder = p[i + marker.len()];
|
||||
return (!own.iter().any(|o| folder.eq_ignore_ascii_case(o))).then(|| (*launcher, folder.to_owned()));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The foreground window as Windows reports it.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Foreground {
|
||||
pub pid: u32,
|
||||
/// The program's full path.
|
||||
pub path: String,
|
||||
/// The window covers one monitor completely.
|
||||
pub covers_monitor: bool,
|
||||
/// No title bar and no sizing border.
|
||||
pub no_frame: bool,
|
||||
/// The process's use of the GPU's 3D engine, in percent.
|
||||
pub gpu_3d: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct Snapshot {
|
||||
pub foreground: Option<Foreground>,
|
||||
/// Windows says a full-screen Direct3D program runs (exclusive full screen).
|
||||
pub d3d_fullscreen: bool,
|
||||
/// Games installed outside a launcher's game folder (Battle.net).
|
||||
pub installed: Vec<InstalledGame>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(tag = "kind", content = "launcher", rename_all = "camelCase")]
|
||||
pub enum Reason {
|
||||
Fullscreen,
|
||||
Borderless,
|
||||
Launcher(&'static str),
|
||||
Registered,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct Detected {
|
||||
#[serde(skip)]
|
||||
pub pid: u32,
|
||||
pub name: String,
|
||||
pub reason: Reason,
|
||||
}
|
||||
|
||||
pub(crate) fn file_name(path: &str) -> String {
|
||||
path.rsplit(['\\', '/']).next().unwrap_or(path).to_lowercase()
|
||||
}
|
||||
|
||||
/// Whether `name` (a program's file name, lower case) is in `list`; list entries may leave out ".exe".
|
||||
pub(crate) fn listed(name: &str, list: &[String]) -> bool {
|
||||
list.iter().map(|e| e.trim().to_lowercase()).any(|e| e == name || format!("{e}.exe") == name)
|
||||
}
|
||||
|
||||
/// Why the foreground program is a game, or `None`.
|
||||
pub fn classify(fg: &Foreground, d3d_fullscreen: bool, s: &GameSettings, installed: &[InstalledGame]) -> Option<Reason> {
|
||||
let name = file_name(&fg.path);
|
||||
if listed(&name, &s.not_games) {
|
||||
return None;
|
||||
}
|
||||
if listed(&name, &s.games) {
|
||||
return Some(Reason::Registered);
|
||||
}
|
||||
if NOT_GAMES.contains(&name.as_str()) {
|
||||
return None;
|
||||
}
|
||||
if let Some((launcher, _)) = launcher_game(&fg.path, installed) {
|
||||
return Some(Reason::Launcher(launcher));
|
||||
}
|
||||
if d3d_fullscreen {
|
||||
return Some(Reason::Fullscreen);
|
||||
}
|
||||
(fg.covers_monitor && fg.no_frame && fg.gpu_3d >= MIN_GPU_3D).then_some(Reason::Borderless)
|
||||
}
|
||||
|
||||
/// The first of these programs (full paths) that is a game by what it is: the person's list or a
|
||||
/// launcher's game folder. Used once as the services start, for games behind other windows (how a
|
||||
/// window looks is not known there, so those rules do not apply).
|
||||
pub fn game_running_among(paths: &[String], s: &GameSettings, installed: &[InstalledGame]) -> Option<String> {
|
||||
paths.iter().find(|p| {
|
||||
let fg = Foreground { pid: 0, path: (*p).clone(), covers_monitor: false, no_frame: false, gpu_3d: 0.0 };
|
||||
classify(&fg, false, s, installed).is_some()
|
||||
}).cloned()
|
||||
}
|
||||
|
||||
/// A game's name for the dashboard: its name from the launcher (the folder under a launcher's game
|
||||
/// folder, or an installed game's name), else the program's name without ".exe".
|
||||
pub fn game_name(path: &str, installed: &[InstalledGame]) -> String {
|
||||
if let Some((_, name)) = launcher_game(path, installed) {
|
||||
return name;
|
||||
}
|
||||
let name = parts(path).last().copied().unwrap_or(path);
|
||||
match name.len().checked_sub(4) {
|
||||
Some(i) if name.is_char_boundary(i) && name[i..].eq_ignore_ascii_case(".exe") => name[..i].to_owned(),
|
||||
_ => name.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The game mode: once a game is seen it stays until its process ends, also while another window
|
||||
/// is in front (the game still holds its video memory).
|
||||
#[derive(Default)]
|
||||
pub struct GameState {
|
||||
current: Option<Detected>,
|
||||
}
|
||||
|
||||
impl GameState {
|
||||
/// Look at `snap`; `alive(pid)` says whether a process still runs. Returns the new state when
|
||||
/// it changed.
|
||||
pub fn update(&mut self, snap: &Snapshot, s: &GameSettings, alive: impl Fn(u32) -> bool) -> Option<Option<Detected>> {
|
||||
if !s.detect {
|
||||
return self.current.take().map(|_| None);
|
||||
}
|
||||
if self.current.as_ref().is_some_and(|d| alive(d.pid)) {
|
||||
return None;
|
||||
}
|
||||
let next = snap.foreground.as_ref().and_then(|fg| {
|
||||
classify(fg, snap.d3d_fullscreen, s, &snap.installed)
|
||||
.map(|reason| Detected { pid: fg.pid, name: game_name(&fg.path, &snap.installed), reason })
|
||||
});
|
||||
if next == self.current {
|
||||
return None;
|
||||
}
|
||||
self.current = next.clone();
|
||||
Some(next)
|
||||
}
|
||||
|
||||
pub fn current(&self) -> Option<&Detected> {
|
||||
self.current.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Watches for games every two seconds while the services run (Windows only; elsewhere it never
|
||||
/// sees one).
|
||||
pub struct GameWatch {
|
||||
state: Arc<Mutex<GameState>>,
|
||||
/// A game process ran (in front or behind other windows) when the watch started.
|
||||
background: bool,
|
||||
stop: Arc<AtomicBool>,
|
||||
thread: Mutex<Option<std::thread::JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl GameWatch {
|
||||
/// Start watching; the first look happens before this returns, so a game already running is
|
||||
/// known at once (`on`).
|
||||
pub fn start(settings: GameSettings) -> Self {
|
||||
let (state, stop) = (Arc::new(Mutex::new(GameState::default())), Arc::new(AtomicBool::new(false)));
|
||||
let mut background = false;
|
||||
if cfg!(windows) && settings.detect {
|
||||
let snap = probe::Probe::new().snapshot();
|
||||
state.lock().unwrap().update(&snap, &settings, probe::alive);
|
||||
if let Some(path) = game_running_among(&probe::process_paths(), &settings, &snap.installed) {
|
||||
// Which program, so a wrong one can be put on the "not games" list.
|
||||
crate::info_log!("game", "a game runs as the services start: {}", file_name(&path));
|
||||
background = true;
|
||||
}
|
||||
}
|
||||
let thread = (cfg!(windows) && settings.detect).then(|| {
|
||||
let (state, stop) = (state.clone(), stop.clone());
|
||||
std::thread::spawn(move || {
|
||||
let mut probe = probe::Probe::new();
|
||||
while !stop.load(Ordering::SeqCst) {
|
||||
let snap = probe.snapshot();
|
||||
let changed = state.lock().unwrap().update(&snap, &settings, probe::alive);
|
||||
match changed {
|
||||
Some(Some(d)) => {
|
||||
crate::info_log!("game", "game mode on: {} ({:?})", d.name, d.reason);
|
||||
}
|
||||
Some(None) => {
|
||||
crate::info_log!("game", "game mode off");
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
// Short sleeps, so a stop does not wait for the whole interval.
|
||||
for _ in 0..(EVERY.as_millis() / 100) {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
Self { state, background, stop, thread: Mutex::new(thread) }
|
||||
}
|
||||
|
||||
/// Whether a game process ran when the watch started, also one behind other windows (found by
|
||||
/// what the program is: the person's list or a launcher's game folder).
|
||||
pub fn in_background_at_start(&self) -> bool {
|
||||
self.background
|
||||
}
|
||||
|
||||
/// Whether game mode is on.
|
||||
pub fn on(&self) -> bool {
|
||||
self.state.lock().unwrap().current().is_some()
|
||||
}
|
||||
|
||||
/// The game for the dashboard: `{name, reason}` or null.
|
||||
pub fn view(&self) -> serde_json::Value {
|
||||
serde_json::json!(self.state.lock().unwrap().current())
|
||||
}
|
||||
|
||||
pub fn stop(&self) {
|
||||
self.stop.store(true, Ordering::SeqCst);
|
||||
if let Some(t) = self.thread.lock().unwrap().take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GameWatch {
|
||||
/// A start that fails part way drops the watch: its thread ends too.
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
mod probe {
|
||||
use super::Snapshot;
|
||||
|
||||
pub struct Probe;
|
||||
|
||||
impl Probe {
|
||||
pub(super) fn new() -> Self {
|
||||
Probe
|
||||
}
|
||||
|
||||
pub fn snapshot(&mut self) -> Snapshot {
|
||||
Snapshot::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn alive(_pid: u32) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub fn process_paths() -> Vec<String> {
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) mod probe {
|
||||
use super::{battle_net_game, Foreground, InstalledGame, Snapshot};
|
||||
use windows_sys::Win32::System::Registry::{
|
||||
RegCloseKey, RegEnumKeyExW, RegGetValueW, RegOpenKeyExW, HKEY, HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_32KEY,
|
||||
KEY_WOW64_64KEY, RRF_RT_REG_EXPAND_SZ, RRF_RT_REG_SZ,
|
||||
};
|
||||
use windows_sys::Win32::Foundation::{CloseHandle, HWND, RECT, WAIT_TIMEOUT};
|
||||
use windows_sys::Win32::Graphics::Gdi::{GetMonitorInfoW, MonitorFromWindow, MONITORINFO, MONITOR_DEFAULTTONEAREST};
|
||||
use windows_sys::Win32::System::Performance::*;
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
OpenProcess, QueryFullProcessImageNameW, WaitForSingleObject, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION,
|
||||
PROCESS_SYNCHRONIZE,
|
||||
};
|
||||
use windows_sys::Win32::UI::Shell::{SHQueryUserNotificationState, QUNS_RUNNING_D3D_FULL_SCREEN};
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
||||
|
||||
/// The GPU 3D engine counters (PDH), kept open: the rate needs two samples.
|
||||
pub struct Probe {
|
||||
query: PDH_HQUERY,
|
||||
counter: PDH_HCOUNTER,
|
||||
installed: Vec<InstalledGame>,
|
||||
}
|
||||
|
||||
impl Probe {
|
||||
pub(super) fn new() -> Self {
|
||||
let mut query: PDH_HQUERY = std::ptr::null_mut();
|
||||
let mut counter: PDH_HCOUNTER = std::ptr::null_mut();
|
||||
let path: Vec<u16> = "\\GPU Engine(*engtype_3D)\\Utilization Percentage".encode_utf16().chain(Some(0)).collect();
|
||||
// SAFETY: plain PDH calls with out pointers to locals; a failure leaves zeros, which
|
||||
// `gpu_3d` treats as "no counters".
|
||||
unsafe {
|
||||
if PdhOpenQueryW(std::ptr::null(), 0, &mut query) == 0 {
|
||||
PdhAddEnglishCounterW(query, path.as_ptr(), 0, &mut counter);
|
||||
PdhCollectQueryData(query);
|
||||
}
|
||||
}
|
||||
Self { query, counter, installed: battle_net_games() }
|
||||
}
|
||||
|
||||
/// 3D engine use of process `pid` in percent, summed over its engines.
|
||||
fn gpu_3d(&mut self, pid: u32) -> f64 {
|
||||
if self.query.is_null() || self.counter.is_null() {
|
||||
return 0.0;
|
||||
}
|
||||
let (mut size, mut count) = (0u32, 0u32);
|
||||
// SAFETY: the documented two-call pattern: the first call asks for the buffer size,
|
||||
// the second fills a buffer of that many bytes, aligned for the items.
|
||||
unsafe {
|
||||
if PdhCollectQueryData(self.query) != 0 {
|
||||
return 0.0;
|
||||
}
|
||||
PdhGetFormattedCounterArrayW(self.counter, PDH_FMT_DOUBLE, &mut size, &mut count, std::ptr::null_mut());
|
||||
if size == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let mut buf = vec![0u64; (size as usize).div_ceil(8)];
|
||||
let items = buf.as_mut_ptr() as *mut PDH_FMT_COUNTERVALUE_ITEM_W;
|
||||
if PdhGetFormattedCounterArrayW(self.counter, PDH_FMT_DOUBLE, &mut size, &mut count, items) != 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let prefix = format!("pid_{pid}_");
|
||||
(0..count as usize).map(|i| &*items.add(i))
|
||||
.filter(|it| wide_str(it.szName).starts_with(&prefix))
|
||||
.map(|it| it.FmtValue.Anonymous.doubleValue)
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&mut self) -> Snapshot {
|
||||
// SAFETY: plain query.
|
||||
let mut state = 0;
|
||||
let d3d_fullscreen = unsafe { SHQueryUserNotificationState(&mut state) } == 0 && state == QUNS_RUNNING_D3D_FULL_SCREEN;
|
||||
let foreground = foreground().map(|(pid, path, covers_monitor, no_frame)| {
|
||||
Foreground { pid, path, covers_monitor, no_frame, gpu_3d: self.gpu_3d(pid) }
|
||||
});
|
||||
Snapshot { foreground, d3d_fullscreen, installed: self.installed.clone() }
|
||||
}
|
||||
}
|
||||
|
||||
/// Battle.net's games from the machine's uninstall entries (64- and 32-bit views), read once
|
||||
/// when the watch starts.
|
||||
fn battle_net_games() -> Vec<InstalledGame> {
|
||||
const KEY: &str = "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall";
|
||||
let key: Vec<u16> = KEY.encode_utf16().chain(Some(0)).collect();
|
||||
let mut out: Vec<InstalledGame> = vec![];
|
||||
for view in [KEY_WOW64_64KEY, KEY_WOW64_32KEY] {
|
||||
let mut h: HKEY = std::ptr::null_mut();
|
||||
// SAFETY: opening a key for reading into a local handle, closed below.
|
||||
if unsafe { RegOpenKeyExW(HKEY_LOCAL_MACHINE, key.as_ptr(), 0, KEY_READ | view, &mut h) } != 0 {
|
||||
continue;
|
||||
}
|
||||
for i in 0.. {
|
||||
let mut name = [0u16; 256];
|
||||
let mut len = name.len() as u32;
|
||||
// SAFETY: `name` holds `len` characters; the loop ends at the first index past the list.
|
||||
let r = unsafe {
|
||||
RegEnumKeyExW(h, i, name.as_mut_ptr(), &mut len, std::ptr::null(), std::ptr::null_mut(), std::ptr::null_mut(),
|
||||
std::ptr::null_mut())
|
||||
};
|
||||
if r != 0 {
|
||||
break;
|
||||
}
|
||||
let sub = &name[..=len as usize];
|
||||
let value = |v: &str| -> String {
|
||||
let v: Vec<u16> = v.encode_utf16().chain(Some(0)).collect();
|
||||
let mut buf = [0u16; 1024];
|
||||
let mut size = (buf.len() * 2) as u32;
|
||||
// SAFETY: valid NUL-terminated names; `buf` holds `size` bytes.
|
||||
let ok = unsafe {
|
||||
RegGetValueW(h, sub.as_ptr(), v.as_ptr(), RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ, std::ptr::null_mut(),
|
||||
buf.as_mut_ptr().cast(), &mut size)
|
||||
} == 0;
|
||||
if !ok {
|
||||
return String::new();
|
||||
}
|
||||
let n = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
|
||||
String::from_utf16_lossy(&buf[..n])
|
||||
};
|
||||
if let Some(g) = battle_net_game(&value("DisplayName"), &value("Publisher"), &value("UninstallString"),
|
||||
&value("InstallLocation")) {
|
||||
if !out.contains(&g) {
|
||||
out.push(g);
|
||||
}
|
||||
}
|
||||
}
|
||||
// SAFETY: the key opened above.
|
||||
unsafe { RegCloseKey(h) };
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
impl Drop for Probe {
|
||||
fn drop(&mut self) {
|
||||
if !self.query.is_null() {
|
||||
// SAFETY: the query this probe opened.
|
||||
unsafe { PdhCloseQuery(self.query) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn wide_str(p: *const u16) -> String {
|
||||
if p.is_null() {
|
||||
return String::new();
|
||||
}
|
||||
let mut n = 0;
|
||||
// SAFETY: PDH gives NUL-terminated names that live as long as the buffer.
|
||||
while unsafe { *p.add(n) } != 0 {
|
||||
n += 1;
|
||||
}
|
||||
String::from_utf16_lossy(unsafe { std::slice::from_raw_parts(p, n) })
|
||||
}
|
||||
|
||||
/// The foreground window's process, program path, whether it covers its monitor and whether
|
||||
/// it has no title bar or sizing border.
|
||||
fn foreground() -> Option<(u32, String, bool, bool)> {
|
||||
// SAFETY: plain window queries on the foreground window; buffers are sized as passed and
|
||||
// the process handle is closed.
|
||||
unsafe {
|
||||
let hwnd: HWND = GetForegroundWindow();
|
||||
if hwnd.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut pid = 0;
|
||||
GetWindowThreadProcessId(hwnd, &mut pid);
|
||||
let path = image_path(pid)?;
|
||||
let mut r: RECT = std::mem::zeroed();
|
||||
GetWindowRect(hwnd, &mut r);
|
||||
let mut mi = MONITORINFO { cbSize: std::mem::size_of::<MONITORINFO>() as u32, ..Default::default() };
|
||||
GetMonitorInfoW(MonitorFromWindow(hwnd, MONITOR_DEFAULTTONEAREST), &mut mi);
|
||||
let m = mi.rcMonitor;
|
||||
let covers = r.left <= m.left && r.top <= m.top && r.right >= m.right && r.bottom >= m.bottom;
|
||||
let style = GetWindowLongPtrW(hwnd, GWL_STYLE) as u32;
|
||||
let no_frame = style & WS_CAPTION != WS_CAPTION && style & WS_THICKFRAME == 0;
|
||||
Some((pid, path, covers, no_frame))
|
||||
}
|
||||
}
|
||||
|
||||
/// Process `pid`'s program path.
|
||||
pub(crate) fn image_path(pid: u32) -> Option<String> {
|
||||
// SAFETY: a query-only handle, closed at once; the buffer is sized as passed.
|
||||
unsafe {
|
||||
let h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid);
|
||||
if h.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut buf = [0u16; 1024];
|
||||
let mut len = buf.len() as u32;
|
||||
let ok = QueryFullProcessImageNameW(h, PROCESS_NAME_WIN32, buf.as_mut_ptr(), &mut len) != 0;
|
||||
CloseHandle(h);
|
||||
ok.then(|| String::from_utf16_lossy(&buf[..len as usize]))
|
||||
}
|
||||
}
|
||||
|
||||
/// Every running program's path this user may read.
|
||||
pub fn process_paths() -> Vec<String> {
|
||||
use windows_sys::Win32::System::Diagnostics::ToolHelp::{
|
||||
CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS,
|
||||
};
|
||||
let mut out = vec![];
|
||||
// SAFETY: a process snapshot walked with a sized entry, then closed.
|
||||
unsafe {
|
||||
let snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
|
||||
if snap == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE {
|
||||
return out;
|
||||
}
|
||||
let mut e: PROCESSENTRY32W = std::mem::zeroed();
|
||||
e.dwSize = std::mem::size_of::<PROCESSENTRY32W>() as u32;
|
||||
let mut more = Process32FirstW(snap, &mut e) != 0;
|
||||
while more {
|
||||
if let Some(p) = image_path(e.th32ProcessID) {
|
||||
out.push(p);
|
||||
}
|
||||
more = Process32NextW(snap, &mut e) != 0;
|
||||
}
|
||||
CloseHandle(snap);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether process `pid` still runs.
|
||||
pub fn alive(pid: u32) -> bool {
|
||||
// SAFETY: a handle only to wait on, closed at once.
|
||||
unsafe {
|
||||
let h = OpenProcess(PROCESS_SYNCHRONIZE, 0, pid);
|
||||
if h.is_null() {
|
||||
return false;
|
||||
}
|
||||
let running = WaitForSingleObject(h, 0) == WAIT_TIMEOUT;
|
||||
CloseHandle(h);
|
||||
running
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn settings() -> GameSettings {
|
||||
GameSettings { detect: true, games: vec![], not_games: vec![], other_room_gb: 1.0, reserve_gb: 8.0, revert_seconds: 30, pause_in_game: false }
|
||||
}
|
||||
|
||||
fn fg(path: &str, covers: bool, no_frame: bool, gpu: f64) -> Foreground {
|
||||
Foreground { pid: 7, path: path.into(), covers_monitor: covers, no_frame, gpu_3d: gpu }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_borderless_game_is_a_game_and_a_maximised_program_or_video_is_not() {
|
||||
let s = settings();
|
||||
// A borderless game outside any launcher folder (the M0 check: covers, no frame, GPU >= 10%).
|
||||
assert_eq!(classify(&fg(r"D:\Games\Hades\Hades.exe", true, true, 45.0), false, &s, &[]), Some(Reason::Borderless));
|
||||
// Too little 3D work: a borderless tool, not a game.
|
||||
assert_eq!(classify(&fg(r"D:\Tools\kiosk.exe", true, true, 3.0), false, &s, &[]), None);
|
||||
// A maximised program has a title bar.
|
||||
assert_eq!(classify(&fg(r"C:\Windows\notepad.exe", true, false, 0.0), false, &s, &[]), None);
|
||||
// A browser's full-screen video, even with the GPU busy.
|
||||
assert_eq!(classify(&fg(r"C:\Program Files\Google\Chrome\Application\chrome.exe", true, true, 60.0), false, &s, &[]), None);
|
||||
// Exclusive full screen.
|
||||
assert_eq!(classify(&fg(r"D:\Games\Old\old.exe", true, true, 0.0), true, &s, &[]), Some(Reason::Fullscreen));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn launcher_folders_make_a_game_whatever_the_window_and_name_it_by_its_folder() {
|
||||
let s = settings();
|
||||
let valorant = r"C:\Riot Games\VALORANT\live\ShooterGame\Binaries\Win64\VALORANT-Win64-Shipping.exe";
|
||||
assert_eq!(classify(&fg(valorant, false, false, 0.0), false, &s, &[]), Some(Reason::Launcher("Riot")));
|
||||
assert_eq!(game_name(valorant, &[]), "VALORANT");
|
||||
let steam = r"D:\SteamLibrary\steamapps\common\Stardew Valley\Stardew Valley.exe";
|
||||
assert_eq!(classify(&fg(steam, false, false, 0.0), false, &s, &[]), Some(Reason::Launcher("Steam")));
|
||||
assert_eq!(game_name(steam, &[]), "Stardew Valley");
|
||||
// The launchers' own windows are not games.
|
||||
assert_eq!(classify(&fg(r"C:\Riot Games\Riot Client\RiotClientServices.exe", true, true, 20.0), false, &s, &[]), None);
|
||||
assert_eq!(game_name(r"D:\Games\Hades\Hades.exe", &[]), "Hades");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_letters_in_a_path_are_fine() {
|
||||
// Lower-casing "İ" changes its length: matching must not cut the path by those positions.
|
||||
let path = "D:\\İ\\steamapps\\common\\하데스\\game.exe";
|
||||
assert_eq!(classify(&fg(path, false, false, 0.0), false, &settings(), &[]), Some(Reason::Launcher("Steam")));
|
||||
assert_eq!(game_name(path, &[]), "하데스");
|
||||
assert_eq!(game_name("D:\\İ\\게임.exe", &[]), "게임");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn battle_net_games_are_found_where_they_are_installed() {
|
||||
let wow = InstalledGame { dir: r"D:\Games\World of Warcraft".into(), name: "World of Warcraft".into(), launcher: "Battle.net" };
|
||||
let installed = [wow];
|
||||
let exe = r"D:\Games\world of warcraft\_retail_\Wow.exe";
|
||||
// A plain window: the install folder alone makes it a game.
|
||||
assert_eq!(classify(&fg(exe, false, false, 0.0), false, &settings(), &installed), Some(Reason::Launcher("Battle.net")));
|
||||
assert_eq!(game_name(exe, &installed), "World of Warcraft");
|
||||
// A folder that only starts with the same letters is not inside it.
|
||||
assert_eq!(classify(&fg(r"D:\Games\World of Warcraft Tools\x.exe", false, false, 0.0), false, &settings(), &installed), None);
|
||||
// The uninstall entries Battle.net writes: its games (Blizzard's, or any it uninstalls), not itself.
|
||||
let entry = |name: &str, publisher: &str, uninstall: &str, dir: &str| battle_net_game(name, publisher, uninstall, dir);
|
||||
assert_eq!(entry("World of Warcraft", "Blizzard Entertainment", r#""C:\ProgramData\Battle.net\Agent\Blizzard Uninstaller.exe" --lang=koKR --uid=wow"#, r"D:\Games\World of Warcraft"),
|
||||
Some(InstalledGame { dir: r"D:\Games\World of Warcraft".into(), name: "World of Warcraft".into(), launcher: "Battle.net" }));
|
||||
assert!(entry("Call of Duty", "Activision", r#""C:\ProgramData\Battle.net\Agent\Blizzard Uninstaller.exe" --uid=fore"#, r"C:\Games\Call of Duty").is_some());
|
||||
assert_eq!(entry("Battle.net", "Blizzard Entertainment", r"C:\Battle.net\Uninstall.exe", r"C:\Program Files (x86)\Battle.net"), None);
|
||||
assert_eq!(entry("Notepad++", "Notepad++ Team", r"C:\npp\uninstall.exe", r"C:\npp"), None);
|
||||
assert_eq!(entry("World of Warcraft", "Blizzard Entertainment", "x", ""), None, "no folder, nothing to match");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_game_running_in_the_background_is_found_by_its_program() {
|
||||
let s = settings();
|
||||
let wow = InstalledGame { dir: r"D:\Games\World of Warcraft".into(), name: "World of Warcraft".into(), launcher: "Battle.net" };
|
||||
// Behind other windows: only what the program is (where it lives, or the person's list).
|
||||
assert_eq!(game_running_among(&[r"C:\Windows\explorer.exe".into(), r"C:\Riot Games\VALORANT\live\VALORANT.exe".into()], &s, &[]),
|
||||
Some(r"C:\Riot Games\VALORANT\live\VALORANT.exe".into()), "the program found, for the log");
|
||||
assert!(game_running_among(&[r"D:\Games\World of Warcraft\_retail_\Wow.exe".into()], &s, &[wow]).is_some());
|
||||
assert!(game_running_among(&[r"C:\Windows\explorer.exe".into(), r"C:\Riot Games\Riot Client\RiotClientServices.exe".into(),
|
||||
r"C:\Program Files\Google\Chrome\Application\chrome.exe".into()], &s, &[]).is_none(), "launchers and browsers are not games");
|
||||
let mut mine = settings();
|
||||
mine.games = vec!["mygame".into()];
|
||||
assert!(game_running_among(&[r"D:\x\MyGame.exe".into()], &mine, &[]).is_some());
|
||||
mine.not_games = vec!["VALORANT.exe".into()];
|
||||
assert!(game_running_among(&[r"C:\Riot Games\VALORANT\live\VALORANT.exe".into()], &mine, &[]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn programs_in_a_launchers_own_folders_are_not_games() {
|
||||
let s = settings();
|
||||
// Always running on the game PC (2026-11-17): Wallpaper Engine is a Steam app, and the Riot
|
||||
// client's crash handler sits in the client's folder.
|
||||
for p in [r"C:\Program Files (x86)\Steam\steamapps\common\wallpaper_engine\wallpaper64.exe",
|
||||
r"C:\Program Files (x86)\Steam\steamapps\common\Steamworks Shared\_CommonRedist\vcredist\x.exe",
|
||||
r"C:\Riot Games\Riot Client\RiotClientCrashHandler.exe",
|
||||
r"C:\Program Files\Epic Games\Launcher\Portal\Binaries\Win64\EpicGamesLauncher.exe",
|
||||
r"C:\Program Files (x86)\Epic Games\Epic Online Services\service\EpicOnlineServices.exe"] {
|
||||
assert_eq!(classify(&fg(p, false, false, 0.0), false, &s, &[]), None, "{p}");
|
||||
}
|
||||
assert_eq!(game_running_among(&[r"C:\Riot Games\Riot Client\RiotClientCrashHandler.exe".into(),
|
||||
r"C:\Program Files (x86)\Steam\steamapps\common\wallpaper_engine\wallpaper64.exe".into()], &s, &[]), None);
|
||||
// The games beside them still are.
|
||||
assert_eq!(classify(&fg(r"C:\Program Files\Epic Games\Fortnite\FortniteClient.exe", false, false, 0.0), false, &s, &[]),
|
||||
Some(Reason::Launcher("Epic")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_persons_lists_come_first() {
|
||||
let mut s = settings();
|
||||
s.games = vec!["MyGame".into()];
|
||||
s.not_games = vec!["Hades.exe".into()];
|
||||
assert_eq!(classify(&fg(r"D:\x\mygame.exe", false, false, 0.0), false, &s, &[]), Some(Reason::Registered));
|
||||
assert_eq!(classify(&fg(r"D:\Games\Hades\Hades.exe", true, true, 90.0), true, &s, &[]), None, "not a game, whatever it does");
|
||||
s.games = vec!["chrome".into()];
|
||||
assert_eq!(classify(&fg(r"C:\c\chrome.exe", false, false, 0.0), false, &s, &[]), Some(Reason::Registered));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn game_mode_stays_until_the_game_ends_and_detection_off_turns_it_off() {
|
||||
let mut s = settings();
|
||||
let mut g = GameState::default();
|
||||
let game = Snapshot { foreground: Some(fg(r"D:\Games\Hades\Hades.exe", true, true, 50.0)), d3d_fullscreen: false, installed: vec![] };
|
||||
let desk = Snapshot { foreground: Some(fg(r"C:\Windows\explorer.exe", true, true, 0.0)), d3d_fullscreen: false, installed: vec![] };
|
||||
let on = g.update(&game, &s, |_| true).expect("changed").expect("a game");
|
||||
assert_eq!((on.name.as_str(), on.reason.clone()), ("Hades", Reason::Borderless));
|
||||
// Alt+Tab to the desktop: still on while the game runs.
|
||||
assert_eq!(g.update(&desk, &s, |_| true), None);
|
||||
assert!(g.current().is_some());
|
||||
// The game ended: off.
|
||||
assert_eq!(g.update(&desk, &s, |_| false), Some(None));
|
||||
assert_eq!(g.update(&desk, &s, |_| false), None, "no change, no event");
|
||||
// Detection off: nothing, and a game on turns off.
|
||||
g.update(&game, &s, |_| true);
|
||||
s.detect = false;
|
||||
assert_eq!(g.update(&game, &s, |_| true), Some(None));
|
||||
assert_eq!(g.update(&game, &s, |_| true), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_dashboard_gets_the_name_and_the_reason() {
|
||||
let d = Detected { pid: 1, name: "VALORANT".into(), reason: Reason::Launcher("Riot") };
|
||||
assert_eq!(serde_json::to_value(&d).unwrap(),
|
||||
serde_json::json!({"name": "VALORANT", "reason": {"kind": "launcher", "launcher": "Riot"}}));
|
||||
assert_eq!(serde_json::to_value(Reason::Borderless).unwrap(), serde_json::json!({"kind": "borderless"}));
|
||||
}
|
||||
}
|
||||
612
app/src/history_view.rs
Normal file
612
app/src/history_view.rs
Normal file
@@ -0,0 +1,612 @@
|
||||
//! The 대화 screen's reads (`app/src/admin.spec.md`, "Conversations"): what people said, what Jarvis
|
||||
//! answered and how each answer was made, from the worker's history store
|
||||
//! (`memory/history.sqlite`, schema owned by `src/jarvis/memory/history.py`). Read only; the
|
||||
//! screen is the only reader, and nothing it reads goes to a log.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params_from_iter, types::Value as Sql, Connection, OpenFlags};
|
||||
use serde_json::{json, Value};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
pub const DEFAULT_LIMIT: usize = 200;
|
||||
pub const MAX_LIMIT: usize = 2000;
|
||||
|
||||
/// What the screen asks for. `None` filters match everything.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Query {
|
||||
/// People's words (voice transcripts and chat messages).
|
||||
pub said: bool,
|
||||
/// Jarvis's answers and its own chat messages.
|
||||
pub answered: bool,
|
||||
/// How each answer was made (the traces).
|
||||
pub thought: bool,
|
||||
/// `voice`, `chat` or `task` (a task has traces only; its answer is a chat message).
|
||||
pub via: Option<String>,
|
||||
/// A person's records: written by them or answers to them.
|
||||
pub person: Option<String>,
|
||||
pub channel: Option<String>,
|
||||
/// Text the record (or the trace's question, answer or steps) contains, case folded.
|
||||
pub find: Option<String>,
|
||||
/// Unix seconds, inclusive / exclusive.
|
||||
pub since: Option<f64>,
|
||||
pub until: Option<f64>,
|
||||
/// Only items older than this one (paging back) / this one and newer (the window shown).
|
||||
pub before: Option<Cursor>,
|
||||
pub from: Option<Cursor>,
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
/// An item's place in the order: time, then records before traces, then id. Items are ordered by
|
||||
/// it, so paging by it never skips or repeats one, even when two share a time.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct Cursor {
|
||||
pub at: f64,
|
||||
/// 0: a record (`h<id>`), 1: a trace (`t<id>`).
|
||||
pub src: u8,
|
||||
pub id: i64,
|
||||
}
|
||||
|
||||
impl Cursor {
|
||||
/// From `{at, key}`, as the screen sends an item back.
|
||||
pub fn of(v: &Value) -> Option<Self> {
|
||||
let at = v.get("at")?.as_f64()?;
|
||||
let key = v.get("key")?.as_str()?;
|
||||
let src = match key.get(..1)? { "h" => 0, "t" => 1, _ => return None };
|
||||
Some(Self { at, src, id: key[1..].parse().ok()? })
|
||||
}
|
||||
|
||||
/// SQL for "a row of table `src` comes before this cursor", over columns `{p}ts` and `{p}id`.
|
||||
fn before_sql(&self, src: u8, p: &str) -> (String, Vec<Sql>) {
|
||||
use std::cmp::Ordering::*;
|
||||
match src.cmp(&self.src) {
|
||||
Less => (format!("{p}ts <= ?"), vec![Sql::Real(self.at)]),
|
||||
Greater => (format!("{p}ts < ?"), vec![Sql::Real(self.at)]),
|
||||
Equal => (format!("({p}ts < ? OR ({p}ts = ? AND {p}id < ?))"),
|
||||
vec![Sql::Real(self.at), Sql::Real(self.at), Sql::Integer(self.id)]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The cursor conditions of `q` for table `src`, appended to `sql` and `args`.
|
||||
fn cursor_sql(q: &Query, src: u8, p: &str, sql: &mut String, args: &mut Vec<Sql>) {
|
||||
if let Some(c) = &q.before {
|
||||
let (cond, a) = c.before_sql(src, p);
|
||||
*sql += &format!(" AND {cond}");
|
||||
args.extend(a);
|
||||
}
|
||||
if let Some(c) = &q.from {
|
||||
let (cond, a) = c.before_sql(src, p);
|
||||
*sql += &format!(" AND NOT {cond}");
|
||||
args.extend(a);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Query {
|
||||
fn default() -> Self {
|
||||
Self { said: true, answered: true, thought: true, via: None, person: None, channel: None, find: None,
|
||||
since: None, until: None, before: None, from: None, limit: DEFAULT_LIMIT }
|
||||
}
|
||||
}
|
||||
|
||||
/// Jarvis's Discord user id, from its bot token: the token's first part is the id in base64.
|
||||
pub fn bot_id_of_token(token: &str) -> Option<String> {
|
||||
use base64::Engine;
|
||||
let first = token.trim().split('.').next()?.trim_end_matches('=');
|
||||
let raw = base64::engine::general_purpose::STANDARD_NO_PAD.decode(first)
|
||||
.or_else(|_| base64::engine::general_purpose::URL_SAFE_NO_PAD.decode(first)).ok()?;
|
||||
let id = String::from_utf8(raw).ok()?;
|
||||
(!id.is_empty() && id.len() <= 20 && id.bytes().all(|b| b.is_ascii_digit())).then_some(id)
|
||||
}
|
||||
|
||||
/// The history store's own text folding (`history.py`, `_fold`): NFKC, then lower case.
|
||||
fn fold(text: &str) -> String {
|
||||
text.nfkc().collect::<String>().to_lowercase()
|
||||
}
|
||||
|
||||
fn like(text: &str) -> String {
|
||||
let escaped = text.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");
|
||||
format!("%{escaped}%")
|
||||
}
|
||||
|
||||
fn open(path: &Path) -> Result<Connection, String> {
|
||||
// Read-write without create: a WAL database needs its shared-memory file, which a read-only
|
||||
// connection may not make. Only SELECTs run (query_only).
|
||||
let c = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX)
|
||||
.map_err(|e| format!("대화 기록을 열 수 없습니다: {e}"))?;
|
||||
c.busy_timeout(std::time::Duration::from_secs(3)).map_err(|e| e.to_string())?;
|
||||
c.pragma_update(None, "query_only", true).map_err(|e| e.to_string())?;
|
||||
Ok(c)
|
||||
}
|
||||
|
||||
fn has_traces(c: &Connection) -> bool {
|
||||
c.query_row("SELECT 1 FROM sqlite_master WHERE type='table' AND name='traces'", [], |_| Ok(())).is_ok()
|
||||
}
|
||||
|
||||
/// The records and traces `q` asks for, newest `q.limit` of them in time order, with the people
|
||||
/// and channels the filters can pick from. `bot`: Jarvis's Discord user id, when known (its chat
|
||||
/// messages count as answers; without it only those that answer someone do, and the authors of those).
|
||||
pub fn query(path: &Path, bot: Option<&str>, q: &Query) -> Result<Value, String> {
|
||||
if !path.is_file() {
|
||||
return Ok(json!({"items": [], "more": false, "people": [], "channels": [], "missing": true}));
|
||||
}
|
||||
let c = open(path)?;
|
||||
let limit = q.limit.clamp(1, MAX_LIMIT);
|
||||
let bot = bot.unwrap_or("");
|
||||
let is_bot = "(h.to_id IS NOT NULL OR h.author_id = ?1 OR h.author_id IN \
|
||||
(SELECT author_id FROM history WHERE kind = 'chat' AND to_id IS NOT NULL))";
|
||||
let err = |e: rusqlite::Error| format!("대화 기록을 읽지 못했습니다: {e}");
|
||||
|
||||
let names: HashMap<String, String> = {
|
||||
let mut st = c.prepare("SELECT author_id, author_name, MAX(ts) FROM history \
|
||||
WHERE kind IN ('voice', 'chat') AND author_id != '' GROUP BY author_id").map_err(err)?;
|
||||
let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).map_err(err)?;
|
||||
rows.collect::<Result<_, _>>().map_err(err)?
|
||||
};
|
||||
let name_of = |id: &str| names.get(id).cloned().unwrap_or_default();
|
||||
|
||||
let mut items: Vec<((f64, u8, i64), Value)> = Vec::new();
|
||||
let mut more = false;
|
||||
if q.said || q.answered {
|
||||
let mut sql = format!("SELECT h.id, h.ts, h.kind, h.channel_id, h.author_id, h.author_name, h.to_id, h.text, \
|
||||
({is_bot} AND h.kind = 'chat') OR h.kind = 'reply' FROM history h WHERE 1");
|
||||
let mut args: Vec<Sql> = vec![Sql::Text(bot.into())];
|
||||
match (q.said, q.answered) {
|
||||
(true, false) => sql += &format!(" AND (h.kind = 'voice' OR (h.kind = 'chat' AND NOT {is_bot}))"),
|
||||
(false, true) => sql += &format!(" AND (h.kind = 'reply' OR (h.kind = 'chat' AND {is_bot}))"),
|
||||
_ => {}
|
||||
}
|
||||
match q.via.as_deref() {
|
||||
Some("voice") => sql += " AND h.kind IN ('voice', 'reply')",
|
||||
Some("chat") => sql += " AND h.kind = 'chat'",
|
||||
Some(_) => sql += " AND 0",
|
||||
None => {}
|
||||
}
|
||||
if let Some(p) = &q.person {
|
||||
sql += " AND (h.author_id = ? OR h.to_id = ?)";
|
||||
args.extend([Sql::Text(p.clone()), Sql::Text(p.clone())]);
|
||||
}
|
||||
if let Some(ch) = &q.channel {
|
||||
sql += " AND h.channel_id = ?";
|
||||
args.push(Sql::Text(ch.clone()));
|
||||
}
|
||||
if let Some(f) = q.find.as_deref().map(str::trim).filter(|f| !f.is_empty()) {
|
||||
sql += " AND h.norm LIKE ? ESCAPE '\\'";
|
||||
args.push(Sql::Text(like(&fold(f))));
|
||||
}
|
||||
if let Some(t) = q.since { sql += " AND h.ts >= ?"; args.push(Sql::Real(t)); }
|
||||
if let Some(t) = q.until { sql += " AND h.ts < ?"; args.push(Sql::Real(t)); }
|
||||
cursor_sql(q, 0, "h.", &mut sql, &mut args);
|
||||
sql += &format!(" ORDER BY h.ts DESC, h.id DESC LIMIT {}", limit + 1);
|
||||
let mut st = c.prepare(&sql).map_err(err)?;
|
||||
let rows = st.query_map(params_from_iter(args), |r| {
|
||||
let (id, ts, kind): (i64, f64, String) = (r.get(0)?, r.get(1)?, r.get(2)?);
|
||||
let (author, author_name, to): (String, String, Option<String>) = (r.get(4)?, r.get(5)?, r.get(6)?);
|
||||
let answer: bool = r.get(8)?;
|
||||
Ok(((ts, 0, id), json!({
|
||||
"key": format!("h{id}"), "type": if answer { "answered" } else { "said" }, "at": ts,
|
||||
"via": if kind == "chat" { "chat" } else { "voice" }, "channel": r.get::<_, String>(3)?,
|
||||
"who": {"id": author, "name": author_name},
|
||||
"to": to.map(|t| json!({"id": t, "name": name_of(&t)})),
|
||||
"text": r.get::<_, String>(7)?,
|
||||
})))
|
||||
}).map_err(err)?;
|
||||
let got = rows.collect::<Result<Vec<_>, _>>().map_err(err)?;
|
||||
more |= got.len() > limit;
|
||||
items.extend(got.into_iter().take(limit));
|
||||
}
|
||||
if q.thought && has_traces(&c) {
|
||||
let mut sql = String::from("SELECT id, ts, via, channel_id, user_id, question, answer, ms, steps FROM traces WHERE 1");
|
||||
let mut args: Vec<Sql> = Vec::new();
|
||||
if let Some(v) = &q.via { sql += " AND via = ?"; args.push(Sql::Text(v.clone())); }
|
||||
if let Some(p) = &q.person { sql += " AND user_id = ?"; args.push(Sql::Text(p.clone())); }
|
||||
if let Some(ch) = &q.channel { sql += " AND channel_id = ?"; args.push(Sql::Text(ch.clone())); }
|
||||
if let Some(f) = q.find.as_deref().map(str::trim).filter(|f| !f.is_empty()) {
|
||||
sql += " AND (question LIKE ? ESCAPE '\\' OR answer LIKE ? ESCAPE '\\' OR steps LIKE ? ESCAPE '\\')";
|
||||
args.extend(std::iter::repeat_n(Sql::Text(like(f)), 3));
|
||||
}
|
||||
if let Some(t) = q.since { sql += " AND ts >= ?"; args.push(Sql::Real(t)); }
|
||||
if let Some(t) = q.until { sql += " AND ts < ?"; args.push(Sql::Real(t)); }
|
||||
cursor_sql(q, 1, "", &mut sql, &mut args);
|
||||
let sql = format!("{sql} ORDER BY ts DESC, id DESC LIMIT {}", limit + 1);
|
||||
let mut st = c.prepare(&sql).map_err(err)?;
|
||||
let rows = st.query_map(params_from_iter(args), |r| {
|
||||
let (id, ts): (i64, f64) = (r.get(0)?, r.get(1)?);
|
||||
let user: String = r.get(4)?;
|
||||
let steps: String = r.get(8)?;
|
||||
let ms: i64 = r.get(7)?;
|
||||
Ok(((ts, 1, id), json!({
|
||||
"key": format!("t{id}"), "type": "thought", "at": ts, "via": r.get::<_, String>(2)?,
|
||||
"channel": r.get::<_, Option<String>>(3)?,
|
||||
"who": {"id": user.clone(), "name": name_of(&user)},
|
||||
"question": r.get::<_, String>(5)?, "answer": r.get::<_, String>(6)?, "ms": ms.max(0),
|
||||
"pending": ms < 0,
|
||||
"steps": serde_json::from_str::<Value>(&steps).unwrap_or(json!([])),
|
||||
})))
|
||||
}).map_err(err)?;
|
||||
let got = rows.collect::<Result<Vec<_>, _>>().map_err(err)?;
|
||||
more |= got.len() > limit;
|
||||
items.extend(got.into_iter().take(limit));
|
||||
}
|
||||
items.sort_by(|a, b| b.0.0.total_cmp(&a.0.0).then(b.0.1.cmp(&a.0.1)).then(b.0.2.cmp(&a.0.2)));
|
||||
more |= items.len() > limit;
|
||||
items.truncate(limit);
|
||||
items.reverse();
|
||||
|
||||
let mut people: Vec<Value> = {
|
||||
let mut st = c.prepare(&format!("SELECT h.author_id, h.author_name, MAX(h.ts) FROM history h \
|
||||
WHERE (h.kind = 'voice' OR (h.kind = 'chat' AND NOT {is_bot})) AND h.author_id != '' GROUP BY h.author_id"))
|
||||
.map_err(err)?;
|
||||
let rows = st.query_map([bot], |r| Ok(json!({"id": r.get::<_, String>(0)?, "name": r.get::<_, String>(1)?})))
|
||||
.map_err(err)?;
|
||||
rows.collect::<Result<_, _>>().map_err(err)?
|
||||
};
|
||||
people.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str()));
|
||||
let channels: Vec<Value> = {
|
||||
let mut st = c.prepare("SELECT channel_id, MAX(kind = 'chat'), MAX(ts) FROM history GROUP BY channel_id \
|
||||
ORDER BY MAX(ts) DESC").map_err(err)?;
|
||||
let rows = st.query_map([], |r| Ok(json!({"id": r.get::<_, String>(0)?,
|
||||
"via": if r.get::<_, bool>(1)? { "chat" } else { "voice" }}))).map_err(err)?;
|
||||
rows.collect::<Result<_, _>>().map_err(err)?
|
||||
};
|
||||
Ok(json!({"items": items.into_iter().map(|(_, v)| v).collect::<Vec<_>>(), "more": more,
|
||||
"people": people, "channels": channels}))
|
||||
}
|
||||
|
||||
/// Every item between `since` (inclusive) and `until` (exclusive) as plain text, in time order:
|
||||
/// what people said, Jarvis's answers, and each trace in full (the records and notes it was given,
|
||||
/// the tools offered, the plan, the model's thinking, every tool call with its arguments and result,
|
||||
/// the answer), for the 로그 screen's 대화 export. `names`: chat channel id to the name given it.
|
||||
/// Returns the text and how many items it holds.
|
||||
pub fn export_text(path: &Path, bot: Option<&str>, since: f64, until: f64, names: &HashMap<String, String>)
|
||||
-> Result<(String, usize), String> {
|
||||
let mut pages: Vec<Vec<Value>> = vec![];
|
||||
let mut before = None;
|
||||
loop {
|
||||
let q = Query { since: Some(since), until: Some(until), before, limit: MAX_LIMIT, ..Query::default() };
|
||||
let v = query(path, bot, &q)?;
|
||||
let items = v["items"].as_array().cloned().unwrap_or_default();
|
||||
before = items.first().and_then(Cursor::of);
|
||||
let more = v["more"].as_bool().unwrap_or(false);
|
||||
pages.push(items);
|
||||
if !more || before.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let items: Vec<Value> = pages.into_iter().rev().flatten().collect();
|
||||
let mut out = String::new();
|
||||
for i in &items {
|
||||
export_item(&mut out, i, names);
|
||||
}
|
||||
Ok((out, items.len()))
|
||||
}
|
||||
|
||||
fn export_item(out: &mut String, i: &Value, names: &HashMap<String, String>) {
|
||||
use std::fmt::Write;
|
||||
let s = |v: &Value| v.as_str().unwrap_or("").to_string();
|
||||
let at = chrono::DateTime::from_timestamp_millis((i["at"].as_f64().unwrap_or(0.0) * 1000.0) as i64)
|
||||
.map(|t| t.with_timezone(&chrono::Local).format("%Y-%m-%d %H:%M:%S").to_string()).unwrap_or_default();
|
||||
let via = match i["via"].as_str() { Some("voice") => "음성", Some("chat") => "채팅", Some("task") => "정기 작업", _ => "" };
|
||||
let channel = match i["channel"].as_str() {
|
||||
Some(id) if !id.is_empty() => match names.get(id) {
|
||||
Some(n) => format!(" · {n}"),
|
||||
None => format!(" · {via} 채널 {id}"),
|
||||
},
|
||||
_ => String::new(),
|
||||
};
|
||||
let person = |w: &Value| {
|
||||
let (id, name) = (s(&w["id"]), s(&w["name"]));
|
||||
if name.is_empty() { id } else if id.is_empty() { name } else { format!("{name}({id})") }
|
||||
};
|
||||
let indent = |text: &str, pad: &str| text.lines().map(|l| format!("{pad}{l}\n")).collect::<String>();
|
||||
match i["type"].as_str() {
|
||||
Some("thought") => {
|
||||
let tools = i["steps"].as_array().map_or(0, |st| st.iter()
|
||||
.filter(|x| x["kind"] == "tool" && x.get("skipped").is_none_or(Value::is_null)).count());
|
||||
let took = if i["pending"].as_bool() == Some(true) { "답을 만드는 중".to_string() }
|
||||
else { format!("{:.1}초", i["ms"].as_f64().unwrap_or(0.0) / 1000.0) };
|
||||
let _ = writeln!(out, "[{at}] 자비스 생각 · {}의 말에 답하며 · 도구 {tools}번 · {took} · {via}{channel}", person(&i["who"]));
|
||||
let _ = writeln!(out, " 받은 말: {}", s(&i["question"]));
|
||||
for st in i["steps"].as_array().into_iter().flatten() {
|
||||
export_step(out, st, &indent);
|
||||
}
|
||||
let answer = s(&i["answer"]);
|
||||
let _ = writeln!(out, " 답: {}", if i["pending"].as_bool() == Some(true) { "(만드는 중)".into() }
|
||||
else if answer.is_empty() { "(답하지 못함)".into() } else { answer });
|
||||
}
|
||||
Some(kind) => {
|
||||
let who = if kind == "answered" {
|
||||
match i.get("to").filter(|t| !t.is_null()) { Some(t) => format!("자비스 → {}", person(t)), None => "자비스".into() }
|
||||
} else { person(&i["who"]) };
|
||||
let _ = writeln!(out, "[{at}] {who} · {via}{channel}");
|
||||
out.push_str(&indent(&s(&i["text"]), " "));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
fn export_step(out: &mut String, st: &Value, indent: &dyn Fn(&str, &str) -> String) {
|
||||
use std::fmt::Write;
|
||||
let list = |k: &str| st[k].as_array().map(|a| a.iter().filter_map(Value::as_str).map(String::from).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
match st["kind"].as_str() {
|
||||
Some("records") => {
|
||||
let (recent, matches) = (list("recent"), list("matches"));
|
||||
let notes = st["notes"].as_u64().unwrap_or(0);
|
||||
let _ = writeln!(out, " 참고한 기록: 최근 대화 {}줄 · 관련 기록 {}줄 · 사람별 지침 {notes}개", recent.len(), matches.len());
|
||||
for (title, xs) in [("최근 대화", recent), ("관련 기록", matches)] {
|
||||
if !xs.is_empty() {
|
||||
let _ = writeln!(out, " {title}:");
|
||||
out.push_str(&indent(&xs.join("\n"), " "));
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("tools") => { let _ = writeln!(out, " 쓸 수 있게 고른 도구: {}", list("names").join(", ")); }
|
||||
Some("plan") => {
|
||||
let _ = writeln!(out, " 계획:");
|
||||
for (n, x) in list("steps").iter().enumerate() {
|
||||
let _ = writeln!(out, " {}. {x}", n + 1);
|
||||
}
|
||||
}
|
||||
Some("think") => {
|
||||
let _ = writeln!(out, " 모델 생각:");
|
||||
out.push_str(&indent(st["text"].as_str().unwrap_or(""), " "));
|
||||
}
|
||||
Some("tool") => {
|
||||
let state = match st["skipped"].as_str() {
|
||||
Some("duplicate") => "건너뜀: 같은 호출을 되풀이해 부르지 않음".to_string(),
|
||||
Some("not_allowed") => "건너뜀: 허용되지 않은 도구라 부르지 않음".to_string(),
|
||||
Some(other) => format!("건너뜀: {other}"),
|
||||
None => (if st["ok"].as_bool() == Some(true) { "성공" } else { "실패" }).to_string(),
|
||||
};
|
||||
let _ = writeln!(out, " 도구 호출: {} ({state})", st["name"].as_str().unwrap_or(""));
|
||||
if st["args"].as_object().is_some_and(|a| !a.is_empty()) {
|
||||
let _ = writeln!(out, " 인자:");
|
||||
out.push_str(&indent(&serde_json::to_string_pretty(&st["args"]).unwrap_or_default(), " "));
|
||||
}
|
||||
if let Some(r) = st["result"].as_str().filter(|r| !r.is_empty()) {
|
||||
let _ = writeln!(out, " 결과:");
|
||||
out.push_str(&indent(r, " "));
|
||||
}
|
||||
}
|
||||
other => { let _ = writeln!(out, " {}: {st}", other.unwrap_or("?")); }
|
||||
}
|
||||
}
|
||||
|
||||
/// A cheap mark of what the 대화 screen shows that changes when something is said or a question
|
||||
/// arrives (the newest record and trace), and as an answer is made (its steps) and ends, so the
|
||||
/// screen can poll it often and read the items again only when it changed.
|
||||
pub fn stamp(path: &Path) -> Result<String, String> {
|
||||
if !path.is_file() {
|
||||
return Ok("missing".into());
|
||||
}
|
||||
let c = open(path)?;
|
||||
let err = |e: rusqlite::Error| format!("대화 기록을 읽지 못했습니다: {e}");
|
||||
let said: Option<i64> = c.query_row("SELECT MAX(id) FROM history", [], |r| r.get(0)).map_err(err)?;
|
||||
let thought: (Option<i64>, i64, Option<i64>) = if has_traces(&c) {
|
||||
c.query_row("SELECT MAX(id), (SELECT COUNT(*) FROM traces WHERE ms < 0), \
|
||||
(SELECT SUM(LENGTH(steps)) FROM traces WHERE ms < 0) FROM traces", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
|
||||
.map_err(err)?
|
||||
} else {
|
||||
(None, 0, None)
|
||||
};
|
||||
Ok(format!("{said:?}/{thought:?}"))
|
||||
}
|
||||
|
||||
/// The history store's tables as `src/jarvis/memory/history.py` makes them, for tests.
|
||||
#[cfg(test)]
|
||||
pub(crate) const TEST_SCHEMA: &str = "
|
||||
CREATE TABLE history (id INTEGER PRIMARY KEY, ts REAL NOT NULL, kind TEXT NOT NULL, channel_id TEXT NOT NULL,
|
||||
author_id TEXT NOT NULL, author_name TEXT NOT NULL, to_id TEXT, message_id TEXT, text TEXT NOT NULL,
|
||||
norm TEXT NOT NULL);
|
||||
CREATE TABLE traces (id INTEGER PRIMARY KEY, ts REAL NOT NULL, via TEXT NOT NULL, channel_id TEXT,
|
||||
user_id TEXT NOT NULL, question TEXT NOT NULL, answer TEXT NOT NULL, ms INTEGER NOT NULL, steps TEXT NOT NULL);";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn db(tag: &str) -> (PathBuf, Connection) {
|
||||
let dir = std::env::temp_dir().join(format!("jarvis-history-view-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("history.sqlite");
|
||||
let c = Connection::open(&path).unwrap();
|
||||
c.pragma_update(None, "journal_mode", "WAL").unwrap();
|
||||
c.execute_batch(TEST_SCHEMA).unwrap();
|
||||
(path, c)
|
||||
}
|
||||
|
||||
fn rec(c: &Connection, ts: f64, kind: &str, ch: &str, author: &str, name: &str, to: Option<&str>, text: &str) {
|
||||
c.execute("INSERT INTO history(ts, kind, channel_id, author_id, author_name, to_id, text, norm) VALUES (?,?,?,?,?,?,?,?)",
|
||||
rusqlite::params![ts, kind, ch, author, name, to, text, fold(text)]).unwrap();
|
||||
}
|
||||
|
||||
fn trace(c: &Connection, ts: f64, via: &str, ch: Option<&str>, user: &str, q: &str, a: &str, steps: &str) {
|
||||
c.execute("INSERT INTO traces(ts, via, channel_id, user_id, question, answer, ms, steps) VALUES (?,?,?,?,?,?,?,?)",
|
||||
rusqlite::params![ts, via, ch, user, q, a, 900, steps]).unwrap();
|
||||
}
|
||||
|
||||
/// Alice speaks and Jarvis answers by voice; Bob writes in chat and Jarvis (bot 900) answers there
|
||||
/// and also posts a reminder of its own.
|
||||
fn sample(tag: &str) -> PathBuf {
|
||||
let (path, c) = db(tag);
|
||||
rec(&c, 10.0, "voice", "vc", "1", "Alice", None, "서울 날씨 어때");
|
||||
trace(&c, 11.0, "voice", Some("vc"), "1", "서울 날씨 어때", "맑습니다",
|
||||
r#"[{"kind":"tools","names":["getWeather"]},{"kind":"think","text":"날씨 도구를 쓰자"}]"#);
|
||||
rec(&c, 12.0, "reply", "vc", "", "Jarvis", Some("1"), "맑습니다");
|
||||
rec(&c, 20.0, "chat", "ch", "2", "Bob", None, "Hello THERE");
|
||||
trace(&c, 21.0, "chat", Some("ch"), "2", "Hello THERE", "안녕하세요", "[]");
|
||||
rec(&c, 22.0, "chat", "ch", "900", "Jarvis", Some("2"), "안녕하세요");
|
||||
rec(&c, 30.0, "chat", "ch", "900", "Jarvis", None, "약속 알림");
|
||||
path
|
||||
}
|
||||
|
||||
fn kinds(v: &Value) -> Vec<(String, String)> {
|
||||
v["items"].as_array().unwrap().iter()
|
||||
.map(|i| (i["type"].as_str().unwrap().into(), i["text"].as_str().or(i["question"].as_str()).unwrap().into()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn everything_comes_in_time_order_with_who_said_it() {
|
||||
let path = sample("all");
|
||||
let v = query(&path, None, &Query::default()).unwrap();
|
||||
let k = kinds(&v);
|
||||
assert_eq!(k.iter().map(|(t, _)| t.as_str()).collect::<Vec<_>>(),
|
||||
["said", "thought", "answered", "said", "thought", "answered", "answered"]);
|
||||
let items = v["items"].as_array().unwrap();
|
||||
assert_eq!(items[2]["to"], json!({"id": "1", "name": "Alice"}));
|
||||
assert_eq!(items[1]["who"]["name"], "Alice");
|
||||
assert_eq!(items[1]["steps"][1]["text"], "날씨 도구를 쓰자");
|
||||
assert_eq!(items[1]["ms"], 900);
|
||||
// Jarvis's reminder counts as its own words because it answered in chat before.
|
||||
assert_eq!(k[6], ("answered".into(), "약속 알림".into()));
|
||||
assert_eq!(v["people"], json!([{"id": "1", "name": "Alice"}, {"id": "2", "name": "Bob"}]));
|
||||
assert_eq!(v["channels"], json!([{"id": "ch", "via": "chat"}, {"id": "vc", "via": "voice"}]));
|
||||
assert_eq!(v["more"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_filters_pick_kinds_people_places_text_and_time() {
|
||||
let path = sample("filters");
|
||||
let only = |q: Query| kinds(&query(&path, Some("900"), &q).unwrap());
|
||||
assert_eq!(only(Query { answered: false, thought: false, ..Query::default() }),
|
||||
[("said".into(), "서울 날씨 어때".into()), ("said".into(), "Hello THERE".into())]);
|
||||
assert_eq!(only(Query { said: false, answered: false, ..Query::default() }).len(), 2);
|
||||
assert_eq!(only(Query { via: Some("voice".into()), ..Query::default() }).len(), 3);
|
||||
assert_eq!(only(Query { via: Some("task".into()), ..Query::default() }), Vec::<(String, String)>::new());
|
||||
assert_eq!(only(Query { person: Some("2".into()), ..Query::default() }).len(), 3);
|
||||
assert_eq!(only(Query { channel: Some("vc".into()), ..Query::default() }).len(), 3);
|
||||
// Text search ignores case, and finds a trace by what it thought.
|
||||
assert_eq!(only(Query { find: Some("hello there".into()), thought: false, ..Query::default() }),
|
||||
[("said".into(), "Hello THERE".into())]);
|
||||
assert_eq!(only(Query { find: Some("도구를".into()), ..Query::default() }),
|
||||
[("thought".into(), "서울 날씨 어때".into())]);
|
||||
assert_eq!(only(Query { find: Some("100%".into()), ..Query::default() }).len(), 0);
|
||||
assert_eq!(only(Query { since: Some(20.0), until: Some(30.0), ..Query::default() }).len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_newest_come_first_when_there_are_more_than_asked() {
|
||||
let path = sample("limit");
|
||||
let v = query(&path, None, &Query { limit: 2, ..Query::default() }).unwrap();
|
||||
assert_eq!(kinds(&v), [("answered".into(), "안녕하세요".into()), ("answered".into(), "약속 알림".into())]);
|
||||
assert_eq!(v["more"], true);
|
||||
}
|
||||
|
||||
/// Every item, walking back page by page with the cursor of the oldest one shown.
|
||||
#[test]
|
||||
fn pages_walk_back_through_everything_past_any_limit_even_at_equal_times() {
|
||||
let (path, c) = db("pages");
|
||||
for i in 0..7 {
|
||||
// Records and traces at the same moments, so a page edge falls inside equal times.
|
||||
rec(&c, (i / 2) as f64, "voice", "vc", "1", "Alice", None, &format!("말 {i}"));
|
||||
trace(&c, (i / 2) as f64, "voice", Some("vc"), "1", &format!("생각 {i}"), "", "[]");
|
||||
}
|
||||
let all = query(&path, None, &Query { limit: MAX_LIMIT, ..Query::default() }).unwrap()["items"].clone();
|
||||
let all: Vec<Value> = all.as_array().unwrap().clone();
|
||||
assert_eq!(all.len(), 14);
|
||||
let cursor = |i: &Value| Cursor::of(&json!({"at": i["at"], "key": i["key"]})).unwrap();
|
||||
let mut got: Vec<Value> = Vec::new();
|
||||
let mut before = None;
|
||||
loop {
|
||||
let v = query(&path, None, &Query { limit: 3, before, ..Query::default() }).unwrap();
|
||||
let page = v["items"].as_array().unwrap().clone();
|
||||
before = page.first().map(cursor);
|
||||
got.splice(0..0, page);
|
||||
if v["more"] == false { break; }
|
||||
}
|
||||
assert_eq!(got, all);
|
||||
// From a cursor on: that item and everything newer, whatever the limit was before.
|
||||
let from = query(&path, None, &Query { limit: MAX_LIMIT, from: Some(cursor(&all[5])), ..Query::default() }).unwrap();
|
||||
assert_eq!(from["items"].as_array().unwrap()[..], all[5..]);
|
||||
assert!(Cursor::of(&json!({"at": 1.0, "key": "x1"})).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_trace_still_being_made_reads_as_pending_with_its_steps_so_far() {
|
||||
let (path, c) = db("pending");
|
||||
rec(&c, 10.0, "voice", "vc", "1", "Alice", None, "서울 날씨 어때");
|
||||
c.execute("INSERT INTO traces(ts, via, channel_id, user_id, question, answer, ms, steps) VALUES (?,?,?,?,?,?,?,?)",
|
||||
rusqlite::params![10.5, "voice", "vc", "1", "서울 날씨 어때", "", -1, r#"[{"kind":"tools","names":["getWeather"]}]"#])
|
||||
.unwrap();
|
||||
let v = query(&path, None, &Query::default()).unwrap();
|
||||
let t = &v["items"][1];
|
||||
assert_eq!((t["type"].as_str(), t["pending"].as_bool()), (Some("thought"), Some(true)));
|
||||
assert_eq!(t["steps"][0]["names"], json!(["getWeather"]));
|
||||
let done = query(&sample("pending-done"), None, &Query::default()).unwrap();
|
||||
assert_eq!(done["items"][1]["pending"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_stamp_changes_when_a_question_is_answered_step_by_step_and_only_then() {
|
||||
let (path, c) = db("stamp");
|
||||
let now = || stamp(&path).unwrap();
|
||||
let empty = now();
|
||||
assert_eq!(now(), empty, "nothing changed");
|
||||
rec(&c, 10.0, "voice", "vc", "1", "Alice", None, "서울 날씨 어때");
|
||||
let said = now();
|
||||
assert_ne!(said, empty);
|
||||
c.execute("INSERT INTO traces(ts, via, channel_id, user_id, question, answer, ms, steps) \
|
||||
VALUES (10.5, 'voice', 'vc', '1', '서울 날씨 어때', '', -1, '[]')", []).unwrap();
|
||||
let asked = now();
|
||||
assert_ne!(asked, said, "a trace still being made");
|
||||
c.execute("UPDATE traces SET steps = '[{\"kind\":\"tools\",\"names\":[\"getWeather\"]}]'", []).unwrap();
|
||||
let step = now();
|
||||
assert_ne!(step, asked, "a step added");
|
||||
c.execute("UPDATE traces SET answer = '맑습니다', ms = 900", []).unwrap();
|
||||
assert_ne!(now(), step, "the answer");
|
||||
let dir = std::env::temp_dir().join(format!("jarvis-history-view-nostamp-{}", std::process::id()));
|
||||
assert!(stamp(&dir.join("history.sqlite")).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_export_holds_every_item_of_the_days_in_order_with_each_trace_in_full() {
|
||||
let path = sample("export");
|
||||
let c = Connection::open(&path).unwrap();
|
||||
trace(&c, 25.0, "voice", Some("vc"), "1", "아이유 신곡 틀어 줘", "",
|
||||
r#"[{"kind":"records","recent":["[14:44] Alice: 안녕"],"matches":[],"notes":2},
|
||||
{"kind":"plan","steps":["botBrowser open"]},
|
||||
{"kind":"tool","name":"botBrowser","args":{"action":"open","url":"https://youtube.com"},"ok":true,"result":"Title: YouTube\nMedia: paused"},
|
||||
{"kind":"tool","name":"webSearch","args":{"q":"x"},"ok":false,"skipped":"duplicate"}]"#);
|
||||
drop(c);
|
||||
let names = HashMap::from([("ch".to_string(), "잡담".to_string())]);
|
||||
let (text, n) = export_text(&path, Some("900"), 0.0, 100.0, &names).unwrap();
|
||||
assert_eq!(n, 8);
|
||||
let order: Vec<usize> = ["서울 날씨 어때", " 답: 맑습니다", "자비스 → Alice(1) · 음성", "Bob(2) · 채팅 · 잡담",
|
||||
"아이유 신곡 틀어 줘", "약속 알림"].iter().map(|x| text.find(x).unwrap_or_else(|| panic!("{x} in\n{text}"))).collect();
|
||||
assert!(order.windows(2).all(|w| w[0] < w[1]), "time order:\n{text}");
|
||||
for want in ["쓸 수 있게 고른 도구: getWeather", "모델 생각:\n 날씨 도구를 쓰자", " [14:44] Alice: 안녕",
|
||||
"사람별 지침 2개", " 1. botBrowser open", "도구 호출: botBrowser (성공)", "\"url\": \"https://youtube.com\"",
|
||||
" 결과:\n Title: YouTube\n Media: paused", "도구 호출: webSearch (건너뜀: 같은 호출을 되풀이해 부르지 않음)",
|
||||
"도구 1번", " 답: (답하지 못함)"] {
|
||||
assert!(text.contains(want), "{want:?} in\n{text}");
|
||||
}
|
||||
// Only the days asked for.
|
||||
let (text, n) = export_text(&path, Some("900"), 20.0, 23.0, &names).unwrap();
|
||||
assert_eq!(n, 3);
|
||||
assert!(!text.contains("서울 날씨"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_bot_id_comes_from_its_token() {
|
||||
assert_eq!(bot_id_of_token("MTIzNDU2Nzg5MDEyMzQ1Njc4.GxYzAb.abcdef").as_deref(), Some("123456789012345678"));
|
||||
assert_eq!(bot_id_of_token("bm90IGFuIGlk.x.y"), None);
|
||||
assert_eq!(bot_id_of_token(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_store_or_one_without_traces_reads_as_empty() {
|
||||
let dir = std::env::temp_dir().join(format!("jarvis-history-view-none-{}", std::process::id()));
|
||||
let v = query(&dir.join("history.sqlite"), None, &Query::default()).unwrap();
|
||||
assert_eq!((v["items"].clone(), v["missing"].clone()), (json!([]), json!(true)));
|
||||
let (path, c) = db("old");
|
||||
c.execute_batch("DROP TABLE traces").unwrap();
|
||||
rec(&c, 1.0, "voice", "vc", "1", "Alice", None, "안녕");
|
||||
assert_eq!(kinds(&query(&path, None, &Query::default()).unwrap()), [("said".into(), "안녕".into())]);
|
||||
}
|
||||
}
|
||||
919
app/src/install.rs
Normal file
919
app/src/install.rs
Normal file
@@ -0,0 +1,919 @@
|
||||
//! Runtime and model install into the data folder (`app/src/admin.spec.md`, "Install").
|
||||
//!
|
||||
//! The steps run one after another and each leaves a marker (`runtime/.done/<id>` holding its
|
||||
//! version) once it finished, so a later run skips what is done and redoes a step whose pinned
|
||||
//! version changed. The same recipe the M0 kit used on the game PC (`scripts/m0`): a pinned `uv`
|
||||
//! installs Python and the venvs, Strata's own installer puts it in the folder. Every cache, temp
|
||||
//! file and model goes inside the data folder.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Serialize;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
|
||||
/// Pinned `uv` (the version the M0 kit used), per platform: archive URL and SHA-256 from the
|
||||
/// release's `.sha256` files.
|
||||
pub const UV_VERSION: &str = "0.11.20";
|
||||
#[cfg(windows)]
|
||||
const UV_ARCHIVE: (&str, &str) = ("uv-x86_64-pc-windows-msvc.zip",
|
||||
"4c281f1b6b9ccb14d9567ebde9a218a7cabfb35e5ffe8c1d4ef5e2821a655474");
|
||||
#[cfg(not(windows))]
|
||||
const UV_ARCHIVE: (&str, &str) = ("uv-x86_64-unknown-linux-gnu.tar.gz",
|
||||
"5de211d9278af365497d387e25316907b3b4a9f25b4476dd6dbf238d6f85cff3");
|
||||
|
||||
pub const STRATA_INSTALL: &str = "https://github.com/Niko1221/Strata/releases/latest/download/install";
|
||||
/// MeloTTS and the PyTorch it runs on (scripts/m0/m0lib/envs.py, "melo").
|
||||
const CU128_INDEX: &str = "https://download.pytorch.org/whl/cu128";
|
||||
const TORCH: [&str; 2] = ["torch==2.11.0", "torchaudio==2.11.0"];
|
||||
const TORCH_PIN: &str = "torch==2.11.0+cu128\ntorchaudio==2.11.0+cu128\n";
|
||||
const MELO_URL: &str = "https://github.com/myshell-ai/MeloTTS/archive/209145371cff8fc3bd60d7be902ea69cbdb7965a.tar.gz";
|
||||
pub const STT_MODEL: &str = "large-v3-turbo";
|
||||
/// The bot browser (admin.spec.md, "Bot browser"): Chrome for Testing, per platform the archive
|
||||
/// and its SHA-256 (computed from the downloads; Google publishes none).
|
||||
pub const CHROME_VERSION: &str = "157.0.8103.78";
|
||||
#[cfg(windows)]
|
||||
const CHROME_ARCHIVE: (&str, &str) = ("win64/chrome-win64.zip",
|
||||
"5f6ec40206407112702e163bf694d07f9fdc8ccbd346c70ce61fa62f716493f9");
|
||||
#[cfg(not(windows))]
|
||||
const CHROME_ARCHIVE: (&str, &str) = ("linux64/chrome-linux64.zip",
|
||||
"9223832977a13db3646201b1341c1f9cf08afa3ceae92540116d7b0c062364fb");
|
||||
|
||||
/// Where everything goes, from the data folder and the app's Python sources (`voice_worker`, `jarvis`).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Layout {
|
||||
pub data: PathBuf,
|
||||
pub src: PathBuf,
|
||||
}
|
||||
|
||||
impl Layout {
|
||||
pub fn runtime(&self) -> PathBuf {
|
||||
self.data.join("runtime")
|
||||
}
|
||||
pub fn uv(&self) -> PathBuf {
|
||||
self.runtime().join("uv").join(if cfg!(windows) { "uv.exe" } else { "uv" })
|
||||
}
|
||||
pub fn venv(&self, name: &str) -> PathBuf {
|
||||
self.runtime().join("venvs").join(name)
|
||||
}
|
||||
pub fn venv_python(&self, name: &str) -> PathBuf {
|
||||
let v = self.venv(name);
|
||||
if cfg!(windows) { v.join("Scripts").join("python.exe") } else { v.join("bin").join("python") }
|
||||
}
|
||||
pub fn strata_home(&self) -> PathBuf {
|
||||
self.runtime().join("strata")
|
||||
}
|
||||
pub fn strata_exe(&self) -> PathBuf {
|
||||
self.strata_home().join("terminal").join("current").join("bin")
|
||||
.join(if cfg!(windows) { "strata.cmd" } else { "strata" })
|
||||
}
|
||||
pub fn chrome_exe(&self) -> PathBuf {
|
||||
let dir = self.runtime().join("chrome");
|
||||
if cfg!(windows) { dir.join("chrome-win64").join("chrome.exe") } else { dir.join("chrome-linux64").join("chrome") }
|
||||
}
|
||||
/// Hugging Face models (Whisper, MeloTTS and its BERT); the worker gets the same `HF_HOME`.
|
||||
pub fn hf_home(&self) -> PathBuf {
|
||||
self.data.join("models").join("hf")
|
||||
}
|
||||
fn done_marker(&self, id: &str) -> PathBuf {
|
||||
self.runtime().join(".done").join(id)
|
||||
}
|
||||
/// The environment every install command and the voice worker run with: caches, temp files and
|
||||
/// models inside the data folder, Python from uv only.
|
||||
pub fn env(&self) -> Vec<(String, String)> {
|
||||
let s = |p: PathBuf| p.to_string_lossy().into_owned();
|
||||
let tmp = s(self.data.join("cache").join("tmp"));
|
||||
let mut env = vec![
|
||||
("UV_CACHE_DIR".into(), s(self.data.join("cache").join("uv"))),
|
||||
("UV_PYTHON_INSTALL_DIR".into(), s(self.runtime().join("python"))),
|
||||
("UV_PYTHON_PREFERENCE".into(), "only-managed".into()),
|
||||
("UV_LINK_MODE".into(), "copy".into()),
|
||||
("PYTHONUTF8".into(), "1".into()),
|
||||
("PYTHONIOENCODING".into(), "utf-8".into()),
|
||||
("HF_HOME".into(), s(self.hf_home())),
|
||||
("NLTK_DATA".into(), s(self.data.join("models").join("nltk"))),
|
||||
("TMP".into(), tmp.clone()),
|
||||
("TEMP".into(), tmp.clone()),
|
||||
("TMPDIR".into(), tmp),
|
||||
("STRATA_DATA_DIR".into(), s(self.strata_home())),
|
||||
("STRATA_NO_PATH".into(), "1".into()),
|
||||
("STRATA_HOME".into(), s(self.strata_home().join("home"))),
|
||||
("STRATA_COMPONENTS_DIR".into(), s(self.strata_home().join("components"))),
|
||||
];
|
||||
if cfg!(windows) {
|
||||
// Strata keeps state under %APPDATA%\Strata and the NVIDIA driver its shader cache under
|
||||
// %APPDATA% too, whatever STRATA_* says (game PC, 2026-11-12): both go in the data folder.
|
||||
env.push(("LOCALAPPDATA".into(), s(self.strata_home().join("localappdata"))));
|
||||
env.push(("APPDATA".into(), s(self.strata_home().join("appdata"))));
|
||||
} else {
|
||||
// On Windows the worker adds the pip CUDA wheels' DLL folders itself; on Linux the loader
|
||||
// needs them in LD_LIBRARY_PATH before the process starts.
|
||||
let libs = self.cuda_lib_dirs();
|
||||
if !libs.is_empty() {
|
||||
let mut path = libs.iter().map(|p| s(p.clone())).collect::<Vec<_>>().join(":");
|
||||
if let Ok(old) = std::env::var("LD_LIBRARY_PATH") {
|
||||
path = format!("{path}:{old}");
|
||||
}
|
||||
env.push(("LD_LIBRARY_PATH".into(), path));
|
||||
}
|
||||
}
|
||||
env
|
||||
}
|
||||
|
||||
/// `nvidia/*/lib` folders of the worker venv's CUDA wheels (cuBLAS, cuDNN).
|
||||
fn cuda_lib_dirs(&self) -> Vec<PathBuf> {
|
||||
let mut out = vec![];
|
||||
let Ok(libs) = std::fs::read_dir(self.venv("worker").join("lib")) else { return out };
|
||||
for py in libs.flatten() {
|
||||
let nvidia = py.path().join("site-packages").join("nvidia");
|
||||
if let Ok(rd) = std::fs::read_dir(&nvidia) {
|
||||
out.extend(rd.flatten().map(|e| e.path().join("lib")).filter(|p| p.is_dir()));
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum StepState {
|
||||
Pending,
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StepStatus {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
pub size_hint: String,
|
||||
pub state: StepState,
|
||||
/// Bytes done and total of a download, when the step has one.
|
||||
pub bytes: Option<(u64, Option<u64>)>,
|
||||
/// The step's last output line or its error (no secrets: install output holds none).
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
/// What a step gets: the layout, a way to report, and the cancel flag.
|
||||
pub struct Ctx {
|
||||
pub layout: Layout,
|
||||
status: Arc<Mutex<Vec<StepStatus>>>,
|
||||
index: usize,
|
||||
pub cancel: Arc<AtomicBool>,
|
||||
/// The log tab its command lines go to.
|
||||
tab: &'static str,
|
||||
}
|
||||
|
||||
impl Ctx {
|
||||
/// A context for commands outside the install (Strata's start and stop, `strata.rs`): their
|
||||
/// lines go to log tab `tab`, the last one is `last_note`.
|
||||
pub fn standalone(layout: Layout, tab: &'static str, cancel: Arc<AtomicBool>) -> Self {
|
||||
let status = vec![StepStatus { id: tab.into(), label: String::new(), size_hint: String::new(),
|
||||
state: StepState::Running, bytes: None, note: None }];
|
||||
Self { layout, status: Arc::new(Mutex::new(status)), index: 0, cancel, tab }
|
||||
}
|
||||
pub fn last_note(&self) -> Option<String> {
|
||||
self.status.lock().unwrap()[self.index].note.clone()
|
||||
}
|
||||
pub fn note(&self, line: &str) {
|
||||
let line: String = line.chars().take(300).collect();
|
||||
self.status.lock().unwrap()[self.index].note = Some(line);
|
||||
}
|
||||
pub fn bytes(&self, done: u64, total: Option<u64>) {
|
||||
self.status.lock().unwrap()[self.index].bytes = Some((done, total));
|
||||
}
|
||||
|
||||
/// Run a program with the layout's environment; each output line becomes the step's note and an
|
||||
/// `install` log line. Fails on a non-zero exit, with the last line.
|
||||
pub async fn run(&self, program: &Path, args: &[&str]) -> Result<(), String> {
|
||||
self.run_in(program, args, None).await
|
||||
}
|
||||
|
||||
/// `run` in folder `cwd` (paths never go into a command string, so no quoting can break).
|
||||
/// The program and everything it starts form one process group (a job on Windows) so a cancel
|
||||
/// ends all of them, not only the first.
|
||||
pub async fn run_in(&self, program: &Path, args: &[&str], cwd: Option<&Path>) -> Result<(), String> {
|
||||
let mut cmd = tokio::process::Command::new(program);
|
||||
cmd.args(args).envs(self.layout.env()).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
if let Some(dir) = cwd {
|
||||
cmd.current_dir(dir);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
|
||||
#[cfg(unix)]
|
||||
cmd.process_group(0);
|
||||
let _ = std::fs::create_dir_all(self.layout.data.join("cache").join("tmp"));
|
||||
#[cfg(windows)]
|
||||
for d in ["appdata", "localappdata"] {
|
||||
let _ = std::fs::create_dir_all(self.layout.strata_home().join(d));
|
||||
}
|
||||
let name = program.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
crate::info_log!(self.tab, "run {name} {}", args.first().copied().unwrap_or(""));
|
||||
let mut child = cmd.spawn().map_err(|e| format!("{name}을(를) 실행하지 못했습니다: {e}"))?;
|
||||
let (out, err) = (child.stdout.take().unwrap(), child.stderr.take().unwrap());
|
||||
let last = Arc::new(Mutex::new(String::new()));
|
||||
let pump = |r: Box<dyn tokio::io::AsyncRead + Unpin + Send>| {
|
||||
let (last, status, index, tab) = (last.clone(), self.status.clone(), self.index, self.tab);
|
||||
tokio::spawn(async move {
|
||||
let mut reader = BufReader::new(r);
|
||||
let mut buf = Vec::new();
|
||||
loop {
|
||||
buf.clear();
|
||||
match reader.read_until(b'\n', &mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(_) => {}
|
||||
}
|
||||
let l = decode_line(&buf).trim().to_owned();
|
||||
if l.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(hub) = crate::logs::installed() {
|
||||
hub.push(crate::logs::Level::Info, tab, &l);
|
||||
}
|
||||
status.lock().unwrap()[index].note = Some(l.chars().take(300).collect());
|
||||
*last.lock().unwrap() = l;
|
||||
}
|
||||
})
|
||||
};
|
||||
let (a, b) = (pump(Box::new(out)), pump(Box::new(err)));
|
||||
let status = loop {
|
||||
tokio::select! {
|
||||
s = child.wait() => break Some(s.map_err(|e| e.to_string())?),
|
||||
_ = tokio::time::sleep(std::time::Duration::from_millis(200)) => {
|
||||
if self.cancel.load(Ordering::Relaxed) {
|
||||
kill_tree(&mut child).await;
|
||||
break None;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// The pipes close once every process holding them is gone.
|
||||
let _ = tokio::time::timeout(std::time::Duration::from_secs(5), async { let _ = (a.await, b.await); }).await;
|
||||
let Some(status) = status else { return Err("cancelled".into()) };
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{name}이(가) 실패했습니다 ({status}): {}", last.lock().unwrap()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One output line as text: UTF-8 when it is, else (Windows without the UTF-8 option) the PC's OEM
|
||||
/// code page, which console programs print in; never an error, so the reading goes on.
|
||||
pub fn decode_line(bytes: &[u8]) -> String {
|
||||
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||
return s.to_owned();
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Win32::Globalization::{MultiByteToWideChar, CP_OEMCP};
|
||||
let len = i32::try_from(bytes.len()).unwrap_or(i32::MAX);
|
||||
// SAFETY: the input pointer and length describe `bytes`; the first call only measures, the
|
||||
// second writes at most `n` UTF-16 units into a buffer of that size.
|
||||
unsafe {
|
||||
let n = MultiByteToWideChar(CP_OEMCP, 0, bytes.as_ptr(), len, std::ptr::null_mut(), 0);
|
||||
if n > 0 {
|
||||
let mut wide = vec![0u16; n as usize];
|
||||
if MultiByteToWideChar(CP_OEMCP, 0, bytes.as_ptr(), len, wide.as_mut_ptr(), n) == n {
|
||||
return String::from_utf16_lossy(&wide);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String::from_utf8_lossy(bytes).into_owned()
|
||||
}
|
||||
|
||||
/// End `child` and every process it started, and wait for it.
|
||||
async fn kill_tree(child: &mut tokio::process::Child) {
|
||||
if let Some(pid) = child.id() {
|
||||
#[cfg(unix)]
|
||||
// SAFETY: a plain signal to the child's own process group (made with process_group(0)).
|
||||
unsafe {
|
||||
libc::kill(-(pid as i32), libc::SIGKILL);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let mut tk = tokio::process::Command::new("taskkill");
|
||||
tk.args(["/T", "/F", "/PID", &pid.to_string()]).stdin(Stdio::null()).stdout(Stdio::null())
|
||||
.stderr(Stdio::null()).creation_flags(0x0800_0000);
|
||||
let _ = tk.status().await;
|
||||
}
|
||||
}
|
||||
let _ = child.kill().await;
|
||||
let _ = child.wait().await;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Step: Send + Sync {
|
||||
fn id(&self) -> &'static str;
|
||||
fn label(&self) -> &'static str;
|
||||
fn size_hint(&self) -> &'static str;
|
||||
/// Changing it (a new pin, new requirement files) makes the step run again.
|
||||
fn version(&self, layout: &Layout) -> String;
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String>;
|
||||
}
|
||||
|
||||
pub struct Installer {
|
||||
layout: Layout,
|
||||
steps: Vec<Box<dyn Step>>,
|
||||
status: Arc<Mutex<Vec<StepStatus>>>,
|
||||
running: AtomicBool,
|
||||
cancel: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl Installer {
|
||||
pub fn new(layout: Layout, steps: Vec<Box<dyn Step>>) -> Arc<Self> {
|
||||
let status = steps.iter().map(|s| StepStatus { id: s.id().into(), label: s.label().into(),
|
||||
size_hint: s.size_hint().into(), state: StepState::Pending, bytes: None, note: None }).collect();
|
||||
let me = Arc::new(Self { layout, steps, status: Arc::new(Mutex::new(status)), running: AtomicBool::new(false),
|
||||
cancel: Arc::new(AtomicBool::new(false)) });
|
||||
me.mark_done();
|
||||
me
|
||||
}
|
||||
|
||||
fn is_done(&self, s: &dyn Step) -> bool {
|
||||
std::fs::read_to_string(self.layout.done_marker(s.id())).is_ok_and(|v| v == s.version(&self.layout))
|
||||
}
|
||||
|
||||
fn mark_done(&self) {
|
||||
let mut st = self.status.lock().unwrap();
|
||||
for (i, s) in self.steps.iter().enumerate() {
|
||||
if st[i].state != StepState::Running {
|
||||
st[i].state = if self.is_done(s.as_ref()) { StepState::Done } else if st[i].state == StepState::Failed {
|
||||
StepState::Failed } else { StepState::Pending };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn status(&self) -> serde_json::Value {
|
||||
self.mark_done();
|
||||
let steps = self.status.lock().unwrap().clone();
|
||||
let all_done = steps.iter().all(|s| s.state == StepState::Done);
|
||||
serde_json::json!({"running": self.running.load(Ordering::SeqCst), "done": all_done, "steps": steps})
|
||||
}
|
||||
|
||||
/// Whether the step `id` is installed at its current version.
|
||||
pub fn done(&self, id: &str) -> bool {
|
||||
self.steps.iter().any(|s| s.id() == id && self.is_done(s.as_ref()))
|
||||
}
|
||||
|
||||
pub fn all_done(&self) -> bool {
|
||||
self.steps.iter().all(|s| self.is_done(s.as_ref()))
|
||||
}
|
||||
|
||||
/// Run the steps not done yet, in order, in the background; false if already running.
|
||||
pub fn start(self: &Arc<Self>) -> bool {
|
||||
if self.running.swap(true, Ordering::SeqCst) {
|
||||
return false;
|
||||
}
|
||||
self.cancel.store(false, Ordering::SeqCst);
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move {
|
||||
me.run_all().await;
|
||||
me.running.store(false, Ordering::SeqCst);
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// The same, awaited (tests and the command line).
|
||||
pub async fn run_all(self: &Arc<Self>) -> bool {
|
||||
let mut ran = false;
|
||||
for (i, step) in self.steps.iter().enumerate() {
|
||||
if self.is_done(step.as_ref()) {
|
||||
continue;
|
||||
}
|
||||
ran = true;
|
||||
{
|
||||
let mut st = self.status.lock().unwrap();
|
||||
st[i].state = StepState::Running;
|
||||
st[i].note = None;
|
||||
st[i].bytes = None;
|
||||
}
|
||||
crate::info_log!("install", "step {} started", step.id());
|
||||
let ctx = Ctx { layout: self.layout.clone(), status: self.status.clone(), index: i, cancel: self.cancel.clone(),
|
||||
tab: "install" };
|
||||
let result = step.run(&ctx).await;
|
||||
let mut st = self.status.lock().unwrap();
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let marker = self.layout.done_marker(step.id());
|
||||
let _ = crate::datadir::write_atomic(&marker, step.version(&self.layout).as_bytes());
|
||||
st[i].state = StepState::Done;
|
||||
crate::info_log!("install", "step {} done", step.id());
|
||||
}
|
||||
Err(e) => {
|
||||
st[i].state = StepState::Failed;
|
||||
st[i].note = Some(if e == "cancelled" { "취소했습니다. 다시 시도하면 이어서 받습니다.".into() } else { e.clone() });
|
||||
crate::error_log!("install", "step {} failed: {e}", step.id());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// The venvs hold copies (UV_LINK_MODE=copy): the download cache (about 12 GB after the
|
||||
// MeloTTS venv) is no longer needed once everything is in.
|
||||
let uv = self.layout.uv();
|
||||
if ran && uv.exists() {
|
||||
let mut cmd = tokio::process::Command::new(&uv);
|
||||
cmd.args(["cache", "clean"]).envs(self.layout.env()).stdin(Stdio::null()).stdout(Stdio::null())
|
||||
.stderr(Stdio::null());
|
||||
#[cfg(windows)]
|
||||
cmd.creation_flags(0x0800_0000);
|
||||
let ok = cmd.status().await.is_ok_and(|s| s.success());
|
||||
crate::info_log!("install", "download cache cleaned: {ok}");
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn cancel(&self) {
|
||||
self.cancel.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn running(&self) -> bool {
|
||||
self.running.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the real steps -------------------------------------------------------------------------------
|
||||
|
||||
struct UvStep;
|
||||
#[async_trait]
|
||||
impl Step for UvStep {
|
||||
fn id(&self) -> &'static str { "uv" }
|
||||
fn label(&self) -> &'static str { "설치 도구(uv)" }
|
||||
fn size_hint(&self) -> &'static str { "약 25MB" }
|
||||
fn version(&self, _: &Layout) -> String { UV_VERSION.into() }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let (file, sum) = UV_ARCHIVE;
|
||||
let url = format!("https://github.com/astral-sh/uv/releases/download/{UV_VERSION}/{file}");
|
||||
let archive = ctx.layout.data.join("cache").join("downloads").join(file);
|
||||
crate::download::download(&url, &archive, sum, &|d, t| ctx.bytes(d, t), &ctx.cancel).await.map_err(|e| e.0)?;
|
||||
let dir = ctx.layout.uv().parent().unwrap().to_path_buf();
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
if file.ends_with(".zip") {
|
||||
let f = std::fs::File::open(&archive).map_err(|e| e.to_string())?;
|
||||
let mut z = zip::ZipArchive::new(f).map_err(|e| e.to_string())?;
|
||||
for i in 0..z.len() {
|
||||
let mut entry = z.by_index(i).map_err(|e| e.to_string())?;
|
||||
let Some(name) = entry.enclosed_name().and_then(|p| p.file_name().map(PathBuf::from)) else { continue };
|
||||
if entry.is_file() {
|
||||
let mut out = std::fs::File::create(dir.join(name)).map_err(|e| e.to_string())?;
|
||||
std::io::copy(&mut entry, &mut out).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx.run(Path::new("tar"), &["-xzf", &archive.to_string_lossy(), "-C", &dir.to_string_lossy(),
|
||||
"--strip-components=1"]).await?;
|
||||
}
|
||||
ctx.run(&ctx.layout.uv(), &["--version"]).await
|
||||
}
|
||||
}
|
||||
|
||||
/// `uv venv` then `uv pip install` steps for one venv.
|
||||
async fn venv(ctx: &Ctx, name: &str, python: &str) -> Result<PathBuf, String> {
|
||||
let uv = ctx.layout.uv();
|
||||
ctx.run(&uv, &["venv", "--clear", "--python", python, &ctx.layout.venv(name).to_string_lossy()]).await?;
|
||||
Ok(ctx.layout.venv_python(name))
|
||||
}
|
||||
|
||||
struct WorkerEnvStep;
|
||||
#[async_trait]
|
||||
impl Step for WorkerEnvStep {
|
||||
fn id(&self) -> &'static str { "worker_env" }
|
||||
fn label(&self) -> &'static str { "음성 워커·두뇌 Python 환경" }
|
||||
fn size_hint(&self) -> &'static str { "약 1.5GB" }
|
||||
fn version(&self, layout: &Layout) -> String {
|
||||
// The requirement files ship with the app: new pins in them install again.
|
||||
let read = |f: &str| std::fs::read(layout.src.join("voice_worker").join(f)).unwrap_or_default();
|
||||
let mut h = <sha2::Sha256 as sha2::Digest>::new();
|
||||
sha2::Digest::update(&mut h, read("requirements.txt"));
|
||||
sha2::Digest::update(&mut h, read("requirements-brain.txt"));
|
||||
format!("3.12|{:x}", sha2::Digest::finalize(h))
|
||||
}
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let py = venv(ctx, "worker", "3.12").await?;
|
||||
let reqs = |f: &str| ctx.layout.src.join("voice_worker").join(f).to_string_lossy().into_owned();
|
||||
ctx.run(&ctx.layout.uv(), &["pip", "install", "--python", &py.to_string_lossy(),
|
||||
"-r", &reqs("requirements.txt"), "-r", &reqs("requirements-brain.txt")]).await
|
||||
}
|
||||
}
|
||||
|
||||
struct MeloEnvStep;
|
||||
#[async_trait]
|
||||
impl Step for MeloEnvStep {
|
||||
fn id(&self) -> &'static str { "melo_env" }
|
||||
fn label(&self) -> &'static str { "음성(MeloTTS) Python 환경" }
|
||||
fn size_hint(&self) -> &'static str { "약 4GB" }
|
||||
fn version(&self, _: &Layout) -> String { format!("3.11|{}|{MELO_URL}", TORCH.join(",")) }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let uv = ctx.layout.uv();
|
||||
let py = venv(ctx, "melo", "3.11").await?;
|
||||
let p = py.to_string_lossy().into_owned();
|
||||
ctx.run(&uv, &["pip", "install", "--python", &p, TORCH[0], TORCH[1], "--index-url", CU128_INDEX]).await?;
|
||||
// Later installs must not swap the CUDA build of torch for another.
|
||||
let pin = ctx.layout.venv("melo").join("torch-pin.txt");
|
||||
std::fs::write(&pin, TORCH_PIN).map_err(|e| e.to_string())?;
|
||||
let pin = pin.to_string_lossy().into_owned();
|
||||
let keep = ["-c", pin.as_str(), "--extra-index-url", CU128_INDEX, "--index-strategy", "unsafe-best-match"];
|
||||
let mut args = vec!["pip", "install", "--python", &p, MELO_URL];
|
||||
args.extend(keep);
|
||||
ctx.run(&uv, &args).await?;
|
||||
// mecab-python3 (Japanese, folder "MeCab") and python-mecab-ko (folder "mecab") are the same
|
||||
// folder on Windows: remove the Japanese one first (melo_child stubs its tagger).
|
||||
ctx.run(&uv, &["pip", "uninstall", "--python", &p, "mecab-python3"]).await?;
|
||||
remove_package_dir(&ctx.layout.venv("melo"), "mecab");
|
||||
let mut args = vec!["pip", "install", "--python", &p, "python-mecab-ko==1.3.7", "python-mecab-ko-dic==2.1.1.post2"];
|
||||
args.extend(keep);
|
||||
ctx.run(&uv, &args).await?;
|
||||
// The full `unidic` ships without its dictionary; removed, the fallback unidic_lite is used.
|
||||
ctx.run(&uv, &["pip", "uninstall", "--python", &p, "unidic"]).await
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_package_dir(venv: &Path, name: &str) {
|
||||
let mut sites = vec![venv.join("Lib").join("site-packages")];
|
||||
if let Ok(rd) = std::fs::read_dir(venv.join("lib")) {
|
||||
sites.extend(rd.flatten().map(|e| e.path().join("site-packages")));
|
||||
}
|
||||
for site in sites {
|
||||
if let Ok(rd) = std::fs::read_dir(&site) {
|
||||
for e in rd.flatten() {
|
||||
if e.file_name().to_string_lossy().eq_ignore_ascii_case(name) && e.path().is_dir() {
|
||||
let _ = std::fs::remove_dir_all(e.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SttModelStep;
|
||||
#[async_trait]
|
||||
impl Step for SttModelStep {
|
||||
fn id(&self) -> &'static str { "stt_model" }
|
||||
fn label(&self) -> &'static str { "받아 적기 모델(Whisper)" }
|
||||
fn size_hint(&self) -> &'static str { "약 1.6GB" }
|
||||
fn version(&self, _: &Layout) -> String { STT_MODEL.into() }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let code = format!("from faster_whisper.utils import download_model; print(download_model({STT_MODEL:?}))");
|
||||
ctx.run(&ctx.layout.venv_python("worker"), &["-c", &code]).await
|
||||
}
|
||||
}
|
||||
|
||||
struct TtsModelStep;
|
||||
#[async_trait]
|
||||
impl Step for TtsModelStep {
|
||||
fn id(&self) -> &'static str { "tts_model" }
|
||||
fn label(&self) -> &'static str { "한국어 음성 모델(MeloTTS)" }
|
||||
fn size_hint(&self) -> &'static str { "약 1GB" }
|
||||
fn version(&self, _: &Layout) -> String { "KR".into() }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
// The worker's own MeloTTS child loads the Korean voice (downloading it) and says one
|
||||
// warm-up sentence, then exits with no input: exit 0 means the voice works.
|
||||
// MeloTTS's English g2p (g2p_en) fetches NLTK data into the user's home folder on first use;
|
||||
// fetched here into the data folder's NLTK_DATA instead, where it then finds them.
|
||||
let nltk = ctx.layout.data.join("models").join("nltk");
|
||||
std::fs::create_dir_all(&nltk).map_err(|e| e.to_string())?;
|
||||
// NLTK refuses a data folder other accounts may write (a group-writable umask on Linux).
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let _ = std::fs::set_permissions(&nltk, std::fs::Permissions::from_mode(0o755));
|
||||
}
|
||||
let code = format!("import nltk\nfor p in ('averaged_perceptron_tagger_eng', 'averaged_perceptron_tagger', 'cmudict'):\n \
|
||||
assert nltk.download(p, download_dir={:?}, quiet=True), p", nltk.to_string_lossy());
|
||||
ctx.run(&ctx.layout.venv_python("melo"), &["-c", &code]).await?;
|
||||
let child = ctx.layout.src.join("voice_worker").join("melo_child.py");
|
||||
ctx.run(&ctx.layout.venv_python("melo"), &[&child.to_string_lossy(), "--device", "cpu", "--language", "KR"]).await
|
||||
}
|
||||
}
|
||||
|
||||
struct StrataStep;
|
||||
#[async_trait]
|
||||
impl Step for StrataStep {
|
||||
fn id(&self) -> &'static str { "strata" }
|
||||
fn label(&self) -> &'static str { "LLM 엔진(Strata)" }
|
||||
fn size_hint(&self) -> &'static str { "약 1GB" }
|
||||
fn version(&self, _: &Layout) -> String { "latest".into() }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let home = ctx.layout.strata_home();
|
||||
std::fs::create_dir_all(&home).map_err(|e| e.to_string())?;
|
||||
// The folder goes in as the working folder, never inside the script text.
|
||||
if cfg!(windows) {
|
||||
let script = format!("irm {STRATA_INSTALL}.ps1 | iex");
|
||||
ctx.run_in(Path::new("powershell"), &["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &script], Some(&home)).await?;
|
||||
} else {
|
||||
let script = format!("curl -fsSL {STRATA_INSTALL}.sh -o install.sh && sh install.sh");
|
||||
ctx.run_in(Path::new("sh"), &["-c", &script], Some(&home)).await?;
|
||||
}
|
||||
if !ctx.layout.strata_exe().exists() {
|
||||
return Err("Strata 설치가 끝났지만 실행 파일이 없습니다".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct BrowserStep;
|
||||
#[async_trait]
|
||||
impl Step for BrowserStep {
|
||||
fn id(&self) -> &'static str { "browser" }
|
||||
fn label(&self) -> &'static str { "봇용 크롬" }
|
||||
fn size_hint(&self) -> &'static str { "약 200MB" }
|
||||
fn version(&self, _: &Layout) -> String { CHROME_VERSION.into() }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let (path, sum) = CHROME_ARCHIVE;
|
||||
let url = format!("https://storage.googleapis.com/chrome-for-testing-public/{CHROME_VERSION}/{path}");
|
||||
let archive = ctx.layout.data.join("cache").join("downloads").join(path.rsplit('/').next().unwrap());
|
||||
crate::download::download(&url, &archive, sum, &|d, t| ctx.bytes(d, t), &ctx.cancel).await.map_err(|e| e.0)?;
|
||||
let dir = ctx.layout.runtime().join("chrome");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
unzip(&archive, &dir)?;
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
if !ctx.layout.chrome_exe().exists() {
|
||||
return Err("크롬 압축을 풀었지만 실행 파일이 없습니다".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Unpack a zip into `dir`, keeping its folders and (on Unix) its executable bits.
|
||||
fn unzip(archive: &Path, dir: &Path) -> Result<(), String> {
|
||||
let f = std::fs::File::open(archive).map_err(|e| e.to_string())?;
|
||||
let mut z = zip::ZipArchive::new(f).map_err(|e| e.to_string())?;
|
||||
for i in 0..z.len() {
|
||||
let mut entry = z.by_index(i).map_err(|e| e.to_string())?;
|
||||
let Some(rel) = entry.enclosed_name() else { continue };
|
||||
let out = dir.join(rel);
|
||||
if entry.is_dir() {
|
||||
std::fs::create_dir_all(&out).map_err(|e| e.to_string())?;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = out.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
|
||||
}
|
||||
let mut w = std::fs::File::create(&out).map_err(|e| e.to_string())?;
|
||||
std::io::copy(&mut entry, &mut w).map_err(|e| e.to_string())?;
|
||||
#[cfg(unix)]
|
||||
if let Some(mode) = entry.unix_mode() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(&out, std::fs::Permissions::from_mode(mode & 0o777)).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Point the services at what was installed: the worker venv's Python and the arguments for the
|
||||
/// MeloTTS venv, the history store and the brain (`voiceWorker` in `config/settings.json`; other
|
||||
/// keys are kept).
|
||||
pub fn apply_runtime_settings(settings_file: &Path, layout: &Layout) -> Result<(), String> {
|
||||
let mut raw: serde_json::Map<String, serde_json::Value> = match std::fs::read_to_string(settings_file) {
|
||||
Ok(t) => serde_json::from_str(t.trim_start_matches('\u{feff}')).map_err(|e| format!("설정 파일을 읽지 못했습니다: {e}"))?,
|
||||
Err(_) => serde_json::Map::new(),
|
||||
};
|
||||
let s = |p: PathBuf| p.to_string_lossy().into_owned();
|
||||
let worker = raw.entry("voiceWorker").or_insert_with(|| serde_json::json!({}));
|
||||
let w = worker.as_object_mut().ok_or("voiceWorker가 객체가 아닙니다")?;
|
||||
w.insert("python".into(), serde_json::json!(s(layout.venv_python("worker"))));
|
||||
w.insert("cwd".into(), serde_json::json!(s(layout.src.clone())));
|
||||
w.insert("args".into(), serde_json::json!(["--melo-python", s(layout.venv_python("melo")),
|
||||
"--history", s(layout.data.join("memory").join("history.sqlite")), "--brain"]));
|
||||
let mut text = serde_json::to_string_pretty(&serde_json::Value::Object(raw)).unwrap();
|
||||
text.push('\n');
|
||||
crate::datadir::write_atomic(settings_file, text.as_bytes()).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// The last step: the settings point at the runtime; a failure shows and can be retried like any step.
|
||||
struct SettingsStep;
|
||||
#[async_trait]
|
||||
impl Step for SettingsStep {
|
||||
fn id(&self) -> &'static str { "settings" }
|
||||
fn label(&self) -> &'static str { "설정 반영" }
|
||||
fn size_hint(&self) -> &'static str { "" }
|
||||
fn version(&self, layout: &Layout) -> String {
|
||||
format!("{}|{}|{}", layout.venv_python("worker").display(), layout.venv_python("melo").display(), layout.src.display())
|
||||
}
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
apply_runtime_settings(&ctx.layout.data.join("config").join("settings.json"), &ctx.layout)
|
||||
}
|
||||
}
|
||||
|
||||
/// The install steps, in order.
|
||||
pub fn steps() -> Vec<Box<dyn Step>> {
|
||||
vec![Box::new(UvStep), Box::new(WorkerEnvStep), Box::new(MeloEnvStep), Box::new(SttModelStep),
|
||||
Box::new(TtsModelStep), Box::new(StrataStep), Box::new(BrowserStep), Box::new(SettingsStep)]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use std::sync::atomic::AtomicU32;
|
||||
|
||||
struct Fake {
|
||||
id: &'static str,
|
||||
version: String,
|
||||
runs: Arc<AtomicU32>,
|
||||
fail: bool,
|
||||
}
|
||||
#[async_trait]
|
||||
impl Step for Fake {
|
||||
fn id(&self) -> &'static str { self.id }
|
||||
fn label(&self) -> &'static str { "fake" }
|
||||
fn size_hint(&self) -> &'static str { "" }
|
||||
fn version(&self, _: &Layout) -> String { self.version.clone() }
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
self.runs.fetch_add(1, Ordering::SeqCst);
|
||||
ctx.note("working");
|
||||
if self.fail { Err("broke".into()) } else { Ok(()) }
|
||||
}
|
||||
}
|
||||
|
||||
fn layout(tag: &str) -> Layout {
|
||||
Layout { data: temp_dir(tag), src: PathBuf::from("src") }
|
||||
}
|
||||
|
||||
fn fakes(runs: &[Arc<AtomicU32>], versions: [&str; 3], fail_second: bool) -> Vec<Box<dyn Step>> {
|
||||
let ids = ["a", "b", "c"];
|
||||
(0..3).map(|i| Box::new(Fake { id: ids[i], version: versions[i].into(), runs: runs[i].clone(),
|
||||
fail: fail_second && i == 1 }) as Box<dyn Step>).collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn steps_run_in_order_stop_at_a_failure_and_resume_after_it() {
|
||||
let l = layout("inst1");
|
||||
let runs: Vec<Arc<AtomicU32>> = (0..3).map(|_| Arc::new(AtomicU32::new(0))).collect();
|
||||
let i = Installer::new(l.clone(), fakes(&runs, ["1", "1", "1"], true));
|
||||
assert!(!i.run_all().await);
|
||||
let s = i.status();
|
||||
let states: Vec<_> = s["steps"].as_array().unwrap().iter().map(|x| x["state"].as_str().unwrap().to_owned()).collect();
|
||||
assert_eq!(states, ["done", "failed", "pending"]);
|
||||
assert_eq!(s["steps"][1]["note"], "broke");
|
||||
assert_eq!(runs.iter().map(|r| r.load(Ordering::SeqCst)).collect::<Vec<_>>(), [1, 1, 0]);
|
||||
// A new run (a fresh app start) skips the finished step and carries on.
|
||||
let i = Installer::new(l, fakes(&runs, ["1", "1", "1"], false));
|
||||
assert!(i.run_all().await);
|
||||
assert_eq!(runs.iter().map(|r| r.load(Ordering::SeqCst)).collect::<Vec<_>>(), [1, 2, 1]);
|
||||
assert!(i.all_done() && i.status()["done"] == true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_new_pinned_version_runs_its_step_again() {
|
||||
let l = layout("inst2");
|
||||
let runs: Vec<Arc<AtomicU32>> = (0..3).map(|_| Arc::new(AtomicU32::new(0))).collect();
|
||||
assert!(Installer::new(l.clone(), fakes(&runs, ["1", "1", "1"], false)).run_all().await);
|
||||
let i = Installer::new(l, fakes(&runs, ["1", "2", "1"], false));
|
||||
assert!(!i.all_done());
|
||||
assert!(i.run_all().await);
|
||||
assert_eq!(runs.iter().map(|r| r.load(Ordering::SeqCst)).collect::<Vec<_>>(), [1, 2, 1]);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_command_reports_its_lines_fails_on_exit_codes_and_stops_on_cancel() {
|
||||
let l = layout("inst3");
|
||||
let i = Installer::new(l.clone(), vec![]);
|
||||
let status = Arc::new(Mutex::new(vec![StepStatus { id: "x".into(), label: "".into(), size_hint: "".into(),
|
||||
state: StepState::Running, bytes: None, note: None }]));
|
||||
let ctx = Ctx { layout: l.clone(), status: status.clone(), index: 0, cancel: i.cancel.clone(), tab: "install" };
|
||||
ctx.run(Path::new("sh"), &["-c", "echo one; echo two; echo $UV_CACHE_DIR"]).await.unwrap();
|
||||
let note = status.lock().unwrap()[0].note.clone().unwrap();
|
||||
assert!(note.ends_with("cache/uv") && note.starts_with(&l.data.to_string_lossy().to_string()), "{note}");
|
||||
let err = ctx.run(Path::new("sh"), &["-c", "echo nope; exit 3"]).await.unwrap_err();
|
||||
assert!(err.contains("nope"), "{err}");
|
||||
let cancel = ctx.cancel.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
});
|
||||
let t = std::time::Instant::now();
|
||||
assert_eq!(ctx.run(Path::new("sh"), &["-c", "sleep 30"]).await.unwrap_err(), "cancelled");
|
||||
assert!(t.elapsed() < std::time::Duration::from_secs(5));
|
||||
}
|
||||
|
||||
/// A Windows PC without the UTF-8 option prints in its code page (CP949 for Korean): such a line
|
||||
/// must not end the reading, or every later line and the error are lost.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn output_that_is_not_utf8_does_not_stop_the_reading() {
|
||||
let l = layout("inst9");
|
||||
let status = Arc::new(Mutex::new(vec![StepStatus { id: "x".into(), label: "".into(), size_hint: "".into(),
|
||||
state: StepState::Running, bytes: None, note: None }]));
|
||||
let ctx = Ctx { layout: l, status: status.clone(), index: 0, cancel: Arc::new(AtomicBool::new(false)), tab: "install" };
|
||||
// "설치 중" in CP949, then two plain lines and a failure.
|
||||
let script = r"printf '\274\263\304\241 \301\337\\n'; echo 'step two'; echo 'last words'; exit 2";
|
||||
let err = ctx.run(Path::new("sh"), &["-c", script]).await.unwrap_err();
|
||||
assert!(err.contains("last words"), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_cancel_ends_the_command_and_everything_it_started() {
|
||||
let l = layout("inst6");
|
||||
let status = Arc::new(Mutex::new(vec![StepStatus { id: "x".into(), label: "".into(), size_hint: "".into(),
|
||||
state: StepState::Running, bytes: None, note: None }]));
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let ctx = Ctx { layout: l.clone(), status, index: 0, cancel: cancel.clone(), tab: "install" };
|
||||
let pidfile = l.data.join("child.pid");
|
||||
let script = format!("sleep 60 & echo $! > {}; wait", pidfile.display());
|
||||
let c2 = cancel.clone();
|
||||
let pf = pidfile.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..50 {
|
||||
if pf.exists() { break; }
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
c2.store(true, Ordering::SeqCst);
|
||||
});
|
||||
assert_eq!(ctx.run(Path::new("sh"), &["-c", &script]).await.unwrap_err(), "cancelled");
|
||||
let pid: i32 = std::fs::read_to_string(&pidfile).unwrap().trim().parse().unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
// SAFETY: signal 0 only asks whether the process exists.
|
||||
let alive = unsafe { libc::kill(pid, 0) } == 0;
|
||||
assert!(!alive, "the child started by the command is gone too");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_folder_with_a_quote_is_passed_as_the_working_folder() {
|
||||
let l = layout("inst7");
|
||||
let dir = l.data.join("it's here");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let status = Arc::new(Mutex::new(vec![StepStatus { id: "x".into(), label: "".into(), size_hint: "".into(),
|
||||
state: StepState::Running, bytes: None, note: None }]));
|
||||
let ctx = Ctx { layout: l.clone(), status: status.clone(), index: 0, cancel: Arc::new(AtomicBool::new(false)), tab: "install" };
|
||||
ctx.run_in(Path::new("sh"), &["-c", "touch made && pwd"], Some(&dir)).await.unwrap();
|
||||
assert!(dir.join("made").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_settings_step_is_part_of_the_install_and_can_fail_and_be_retried() {
|
||||
let l = layout("inst8");
|
||||
let settings = l.data.join("config").join("settings.json");
|
||||
std::fs::create_dir_all(settings.parent().unwrap()).unwrap();
|
||||
std::fs::write(&settings, "{ broken").unwrap();
|
||||
let i = Installer::new(l.clone(), vec![Box::new(SettingsStep)]);
|
||||
assert!(!i.run_all().await);
|
||||
assert_eq!(i.status()["steps"][0]["state"], "failed");
|
||||
assert!(!i.all_done(), "not done while the settings could not be written");
|
||||
std::fs::write(&settings, "{}").unwrap();
|
||||
assert!(i.run_all().await && i.all_done());
|
||||
assert!(std::fs::read_to_string(&settings).unwrap().contains("--brain"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_settings_point_at_the_installed_runtime_and_keep_the_rest() {
|
||||
let l = layout("inst5");
|
||||
let f = l.data.join("config").join("settings.json");
|
||||
std::fs::create_dir_all(f.parent().unwrap()).unwrap();
|
||||
std::fs::write(&f, r#"{"call": {"earcon": false}, "voiceWorker": {"python": "old"}}"#).unwrap();
|
||||
apply_runtime_settings(&f, &l).unwrap();
|
||||
let s = crate::settings::load_settings(&f).unwrap();
|
||||
assert_eq!(std::path::Path::new(&s.voice_worker.python), l.venv_python("worker"));
|
||||
assert_eq!(s.voice_worker.args[0..2], ["--melo-python".to_string(), l.venv_python("melo").to_string_lossy().into_owned()]);
|
||||
assert!(s.voice_worker.args.contains(&"--brain".to_string()));
|
||||
assert!(!s.call.earcon, "other settings are kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn everything_goes_inside_the_data_folder() {
|
||||
let l = layout("inst4");
|
||||
for (k, v) in l.env() {
|
||||
if v.contains('/') || v.contains('\\') {
|
||||
assert!(v.starts_with(&l.data.to_string_lossy().to_string()), "{k}={v}");
|
||||
}
|
||||
}
|
||||
assert!(l.uv().starts_with(&l.data) && l.venv_python("worker").starts_with(&l.data));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_unpacks_with_its_folders_and_executable_bits() {
|
||||
use std::io::Write;
|
||||
let d = temp_dir("unzip");
|
||||
let archive = d.join("a.zip");
|
||||
let mut w = zip::ZipWriter::new(std::fs::File::create(&archive).unwrap());
|
||||
let exe = zip::write::SimpleFileOptions::default().unix_permissions(0o755);
|
||||
w.add_directory("top/sub/", zip::write::SimpleFileOptions::default()).unwrap();
|
||||
w.start_file("top/chrome", exe).unwrap();
|
||||
w.write_all(b"#!/bin/sh\n").unwrap();
|
||||
w.start_file("top/sub/data.pak", zip::write::SimpleFileOptions::default().unix_permissions(0o644)).unwrap();
|
||||
w.write_all(b"data").unwrap();
|
||||
w.start_file("../escape", zip::write::SimpleFileOptions::default()).unwrap();
|
||||
w.write_all(b"no").unwrap();
|
||||
w.finish().unwrap();
|
||||
let out = d.join("out");
|
||||
unzip(&archive, &out).unwrap();
|
||||
assert_eq!(std::fs::read(out.join("top/sub/data.pak")).unwrap(), b"data");
|
||||
assert!(!d.join("escape").exists());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
assert_eq!(std::fs::metadata(out.join("top/chrome")).unwrap().permissions().mode() & 0o777, 0o755);
|
||||
}
|
||||
}
|
||||
}
|
||||
47
app/src/lib.rs
Normal file
47
app/src/lib.rs
Normal file
@@ -0,0 +1,47 @@
|
||||
//! Jarvis v2 app core (see `app/src/discord_core.spec.md`).
|
||||
|
||||
pub mod log;
|
||||
pub mod activity;
|
||||
pub mod admin_api;
|
||||
pub mod browser;
|
||||
pub mod calls;
|
||||
pub mod channel_sync;
|
||||
pub mod chat;
|
||||
pub mod collect;
|
||||
pub mod collect_notify;
|
||||
pub mod control;
|
||||
pub mod conversation;
|
||||
pub mod dashboard;
|
||||
pub mod datadir;
|
||||
pub mod discord;
|
||||
pub mod discord_setup;
|
||||
pub mod download;
|
||||
pub mod earcon;
|
||||
pub mod features;
|
||||
pub mod history_view;
|
||||
pub mod game;
|
||||
pub mod frames;
|
||||
pub mod install;
|
||||
pub mod logs;
|
||||
pub mod notes;
|
||||
pub mod playback;
|
||||
pub mod perf;
|
||||
pub mod permissions;
|
||||
pub mod placement;
|
||||
pub mod policy;
|
||||
pub mod receive;
|
||||
pub mod recordings;
|
||||
pub mod schedule;
|
||||
pub mod screen;
|
||||
pub mod secrets;
|
||||
pub mod services;
|
||||
pub mod settings;
|
||||
pub mod settings_edit;
|
||||
pub mod setup_lock;
|
||||
pub mod speakers;
|
||||
pub mod strata;
|
||||
pub mod system;
|
||||
pub mod update;
|
||||
pub mod vdisplay;
|
||||
pub mod worker_client;
|
||||
pub mod worker_process;
|
||||
72
app/src/log.rs
Normal file
72
app/src/log.rs
Normal file
@@ -0,0 +1,72 @@
|
||||
//! Debug logging, the Rust side of `src/jarvis/debug.py`.
|
||||
//!
|
||||
//! On only when `JARVIS_DEBUG=1`. Messages carry ids, sizes, counts and timings, never what anyone
|
||||
//! said or wrote (no transcripts, no chat text, no reply text).
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
pub fn debug_enabled() -> bool {
|
||||
static ON: OnceLock<bool> = OnceLock::new();
|
||||
*ON.get_or_init(|| std::env::var("JARVIS_DEBUG").as_deref() == Ok("1"))
|
||||
}
|
||||
|
||||
/// `[ category ] message`, the category centred in 10 columns like the Python `debug_log`.
|
||||
pub fn line(category: &str, message: &str) -> String {
|
||||
format!("[{category:^10}] {message}")
|
||||
}
|
||||
|
||||
/// Print a line and give it to the log hub when one is installed (`app/src/admin.spec.md`).
|
||||
pub fn emit(level: crate::logs::Level, category: &str, message: &str) {
|
||||
match level {
|
||||
crate::logs::Level::Debug => eprintln!("{}", line(category, message)),
|
||||
crate::logs::Level::Info => eprintln!("[{category}] {message}"),
|
||||
crate::logs::Level::Warn => eprintln!("[{category}] WARN {message}"),
|
||||
crate::logs::Level::Error => eprintln!("[{category}] ERROR {message}"),
|
||||
}
|
||||
if let Some(hub) = crate::logs::installed() {
|
||||
hub.push(level, category, message);
|
||||
}
|
||||
}
|
||||
|
||||
/// Only when `JARVIS_DEBUG=1`.
|
||||
#[macro_export]
|
||||
macro_rules! debug_log {
|
||||
($cat:expr, $($arg:tt)*) => {
|
||||
if $crate::log::debug_enabled() {
|
||||
$crate::log::emit($crate::logs::Level::Debug, $cat, &format!($($arg)*));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Always logged: events the log screen shows without debug logging (start, stop, joins).
|
||||
#[macro_export]
|
||||
macro_rules! info_log {
|
||||
($cat:expr, $($arg:tt)*) => {
|
||||
$crate::log::emit($crate::logs::Level::Info, $cat, &format!($($arg)*));
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! warn_log {
|
||||
($cat:expr, $($arg:tt)*) => {
|
||||
$crate::log::emit($crate::logs::Level::Warn, $cat, &format!($($arg)*));
|
||||
};
|
||||
}
|
||||
|
||||
/// Errors are always printed; the same rule applies: no transcript or chat text in the message.
|
||||
#[macro_export]
|
||||
macro_rules! error_log {
|
||||
($cat:expr, $($arg:tt)*) => {
|
||||
$crate::log::emit($crate::logs::Level::Error, $cat, &format!($($arg)*));
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn category_is_centred_like_python() {
|
||||
// Python: f"[{'voice':^10}]" == "[ voice ]"
|
||||
assert_eq!(super::line("voice", "x"), "[ voice ] x");
|
||||
assert_eq!(super::line("worker", "x"), "[ worker ] x");
|
||||
}
|
||||
}
|
||||
621
app/src/logs.rs
Normal file
621
app/src/logs.rs
Normal file
@@ -0,0 +1,621 @@
|
||||
//! The log hub behind the log screen (`app/src/admin.spec.md`, "Logs"): every line the app logs,
|
||||
//! by service tab, kept in memory for the screen and in daily files under `logs/`.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use chrono::{DateTime, Local, NaiveDate};
|
||||
|
||||
/// Lines kept in memory for the screen.
|
||||
pub const KEPT_LINES: usize = 5_000;
|
||||
/// Log files older than this are removed.
|
||||
pub const RETENTION_DAYS: i64 = 14;
|
||||
/// Discord's message limit: a longer error report is offered as a download.
|
||||
pub const DISCORD_MESSAGE_CHARS: usize = 2_000;
|
||||
/// Discord's file limit for the zip export.
|
||||
pub const ZIP_MAX_BYTES: usize = 10 * 1024 * 1024;
|
||||
const REPORT_BEFORE: usize = 30;
|
||||
const REPORT_AFTER: usize = 10;
|
||||
pub const SECRET_MARK: &str = "[비밀값]";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Level {
|
||||
Debug,
|
||||
Info,
|
||||
Warn,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
|
||||
pub struct LogLine {
|
||||
pub seq: u64,
|
||||
/// Milliseconds since the Unix epoch.
|
||||
pub at_ms: i64,
|
||||
pub level: Level,
|
||||
pub tab: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// The screen's tab for a log category (worker lines: by the worker's own category, see `push_worker`).
|
||||
pub fn tab_of(category: &str) -> &'static str {
|
||||
match category {
|
||||
"voice" | "receive" | "play" | "discord" => "discord",
|
||||
"chat" => "chat",
|
||||
"conversation" | "schedule" => "conversation",
|
||||
"worker" | "voice_worker" => "worker",
|
||||
"stt" => "stt",
|
||||
"llm" => "llm",
|
||||
"tts" => "tts",
|
||||
"browser" => "browser",
|
||||
"capture" => "capture",
|
||||
"game" => "game",
|
||||
"install" => "install",
|
||||
_ => "app",
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
/// Where the daily files go; None: memory only (no data folder yet).
|
||||
dir: Option<PathBuf>,
|
||||
lines: VecDeque<LogLine>,
|
||||
next_seq: u64,
|
||||
files: HashMap<(String, NaiveDate), std::fs::File>,
|
||||
secrets: Vec<String>,
|
||||
/// The values the running services were started with (`set_secrets_in_use`).
|
||||
in_use: Vec<String>,
|
||||
last_cleanup: Option<NaiveDate>,
|
||||
}
|
||||
|
||||
impl State {
|
||||
/// Every value to mask: the stored secrets and those the running services use.
|
||||
fn masked(&self) -> Vec<String> {
|
||||
let mut all = self.secrets.clone();
|
||||
all.extend(self.in_use.iter().filter(|v| !self.secrets.contains(v)).cloned());
|
||||
all
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LogHub {
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
impl LogHub {
|
||||
/// A hub writing daily files to `dir` (None: memory only).
|
||||
pub fn new(dir: Option<&Path>) -> Arc<Self> {
|
||||
let hub = Arc::new(Self { state: Mutex::new(State { dir: dir.map(Path::to_path_buf), ..State::default() }) });
|
||||
hub.cleanup(Local::now().date_naive());
|
||||
hub
|
||||
}
|
||||
|
||||
/// Start writing daily files to `dir` (the data folder was just chosen); lines kept in memory
|
||||
/// stay there, the next ones also go to the files.
|
||||
pub fn set_dir(&self, dir: &Path) {
|
||||
{
|
||||
let mut s = self.state.lock().unwrap();
|
||||
s.dir = Some(dir.to_path_buf());
|
||||
s.files.clear();
|
||||
}
|
||||
self.cleanup(Local::now().date_naive());
|
||||
}
|
||||
|
||||
/// The stored secrets' values, masked in every line from now on (and again in copies and exports).
|
||||
pub fn set_secrets(&self, values: Vec<String>) {
|
||||
self.state.lock().unwrap().secrets = values;
|
||||
}
|
||||
|
||||
/// The secret values the running services were started with (the bot token, the MCP servers'
|
||||
/// values). They stay masked until the next start, also after one is replaced or removed in
|
||||
/// the store: the running worker and MCP servers keep using them until they are restarted.
|
||||
pub fn set_secrets_in_use(&self, values: Vec<String>) {
|
||||
self.state.lock().unwrap().in_use = values;
|
||||
}
|
||||
|
||||
pub fn push(&self, level: Level, category: &str, text: &str) {
|
||||
self.push_at(Local::now(), level, tab_of(category), text);
|
||||
}
|
||||
|
||||
/// A line of the voice worker's output: its `[ category ]` prefix names the tab (stt, llm, tts,
|
||||
/// else worker); a Python warning or traceback line is a warning.
|
||||
/// Every value masked right now (stored and in use). A worker run keeps these for its lines
|
||||
/// (`push_worker_masked`), so a line it writes late is masked even after the next start.
|
||||
pub fn masked_values(&self) -> Vec<String> {
|
||||
self.state.lock().unwrap().masked()
|
||||
}
|
||||
|
||||
pub fn push_worker(&self, line: &str) {
|
||||
self.push_worker_masked(line, &[]);
|
||||
}
|
||||
|
||||
/// `push_worker`, also masking `extra` (the values of the run that wrote the line).
|
||||
pub fn push_worker_masked(&self, line: &str, extra: &[String]) {
|
||||
let (cat, text) = match line.strip_prefix('[').and_then(|r| r.split_once(']')) {
|
||||
Some((c, rest)) => (c.trim(), rest.trim_start()),
|
||||
None => ("worker", line),
|
||||
};
|
||||
let tab = match tab_of(cat) {
|
||||
t @ ("stt" | "llm" | "tts") => t,
|
||||
_ => "worker",
|
||||
};
|
||||
let level = if text.contains("Error") || text.contains("Traceback") { Level::Error }
|
||||
else if text.contains("Warning") { Level::Warn } else { Level::Info };
|
||||
self.push_masked(Local::now(), level, tab, text, extra);
|
||||
}
|
||||
|
||||
fn push_at(&self, at: DateTime<Local>, level: Level, tab: &str, text: &str) {
|
||||
self.push_masked(at, level, tab, text, &[]);
|
||||
}
|
||||
|
||||
fn push_masked(&self, at: DateTime<Local>, level: Level, tab: &str, text: &str, extra: &[String]) {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
// Masked before it is kept or written, so a secret replaced or removed later (the hub then
|
||||
// no longer knows the old value) never shows on the screen, in a copy, a file or an export.
|
||||
let mut masks = s.masked();
|
||||
masks.extend(extra.iter().cloned());
|
||||
let text = redact(text, &masks);
|
||||
let line = LogLine { seq: s.next_seq, at_ms: at.timestamp_millis(), level, tab: tab.into(), text };
|
||||
s.next_seq += 1;
|
||||
if let Some(dir) = s.dir.clone() {
|
||||
let day = at.date_naive();
|
||||
let key = (tab.to_owned(), day);
|
||||
if !s.files.contains_key(&key) {
|
||||
s.files.retain(|(_, d), _| *d == day); // yesterday's files are done
|
||||
let path = dir.join(format!("{tab}-{day}.log"));
|
||||
if let Ok(f) = std::fs::OpenOptions::new().create(true).append(true).open(path) {
|
||||
s.files.insert(key.clone(), f);
|
||||
}
|
||||
}
|
||||
if let Some(f) = s.files.get_mut(&key) {
|
||||
let _ = writeln!(f, "{}", format_line(&line));
|
||||
}
|
||||
}
|
||||
s.lines.push_back(line);
|
||||
if s.lines.len() > KEPT_LINES {
|
||||
s.lines.pop_front();
|
||||
}
|
||||
let today = at.date_naive();
|
||||
let due = s.last_cleanup != Some(today);
|
||||
drop(s);
|
||||
if due {
|
||||
self.cleanup(today);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove log files older than the retention.
|
||||
fn cleanup(&self, today: NaiveDate) {
|
||||
let dir = {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
s.last_cleanup = Some(today);
|
||||
s.dir.clone()
|
||||
};
|
||||
let Some(dir) = dir else { return };
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return };
|
||||
for e in entries.flatten() {
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
if let Some(day) = file_day(&name) {
|
||||
if (today - day).num_days() > RETENTION_DAYS {
|
||||
let _ = std::fs::remove_file(e.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lines for the screen: one tab (None: all), this level and above, containing `find`, after
|
||||
/// sequence number `after` (live follow), the last `limit`.
|
||||
pub fn query(&self, tab: Option<&str>, min_level: Level, find: Option<&str>, after: Option<u64>, limit: usize)
|
||||
-> Vec<LogLine> {
|
||||
let s = self.state.lock().unwrap();
|
||||
let find = find.map(str::to_lowercase);
|
||||
let mut out: Vec<LogLine> = s.lines.iter()
|
||||
.filter(|l| tab.is_none_or(|t| l.tab == t) && l.level >= min_level
|
||||
&& after.is_none_or(|a| l.seq > a)
|
||||
&& find.as_deref().is_none_or(|f| l.text.to_lowercase().contains(f)))
|
||||
.cloned().collect();
|
||||
if out.len() > limit {
|
||||
out.drain(..out.len() - limit);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The last error with the lines around it, the app version and a PC summary, masked; None
|
||||
/// without an error.
|
||||
pub fn recent_error_report(&self, app_version: &str) -> Option<String> {
|
||||
let s = self.state.lock().unwrap();
|
||||
let i = s.lines.iter().rposition(|l| l.level == Level::Error)?;
|
||||
let from = i.saturating_sub(REPORT_BEFORE);
|
||||
let to = (i + 1 + REPORT_AFTER).min(s.lines.len());
|
||||
let mut text = format!("Jarvis {app_version} | {}\n", pc_summary());
|
||||
for l in s.lines.range(from..to) {
|
||||
text.push_str(&format_line(l));
|
||||
text.push('\n');
|
||||
}
|
||||
Some(redact(&text, &s.masked()))
|
||||
}
|
||||
|
||||
/// The log files of days `from..=to` in a zip at `out`, masked, at most `ZIP_MAX_BYTES`: when
|
||||
/// larger, the oldest lines are left out and NOTE.txt says how many. Returns the lines left out.
|
||||
pub fn export_zip(&self, from: NaiveDate, to: NaiveDate, out: &Path) -> std::io::Result<usize> {
|
||||
let (secrets, dir) = {
|
||||
let s = self.state.lock().unwrap();
|
||||
(s.masked(), s.dir.clone())
|
||||
};
|
||||
let Some(dir) = dir else { return Err(std::io::Error::other("no log folder")) };
|
||||
let mut files: Vec<(NaiveDate, String, Vec<String>)> = vec![];
|
||||
for e in std::fs::read_dir(&dir)?.flatten() {
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
let Some(day) = file_day(&name) else { continue };
|
||||
if day < from || day > to {
|
||||
continue;
|
||||
}
|
||||
let text = std::fs::read_to_string(e.path()).unwrap_or_default();
|
||||
files.push((day, name, redact(&text, &secrets).lines().map(str::to_owned).collect()));
|
||||
}
|
||||
files.sort();
|
||||
let mut dropped = 0usize;
|
||||
loop {
|
||||
let bytes = build_zip(&files, dropped)?;
|
||||
if bytes.len() <= ZIP_MAX_BYTES {
|
||||
std::fs::write(out, bytes)?;
|
||||
return Ok(dropped);
|
||||
}
|
||||
let total: usize = files.iter().map(|f| f.2.len()).sum();
|
||||
if dropped >= total {
|
||||
return Err(std::io::Error::other("the logs do not fit in the zip limit"));
|
||||
}
|
||||
// Drop a fifth more of the oldest lines and try again.
|
||||
dropped = (dropped + total.div_ceil(5)).min(total);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The zip with the `dropped` oldest lines left out: oldest by the time each line was logged, across
|
||||
/// every tab's file (a line's time is its first 23 characters, `YYYY-MM-DD HH:MM:SS.mmm`; lines
|
||||
/// without one count as oldest).
|
||||
fn build_zip(files: &[(NaiveDate, String, Vec<String>)], dropped: usize) -> std::io::Result<Vec<u8>> {
|
||||
let mut order: Vec<(&str, usize, usize)> = files.iter().enumerate()
|
||||
.flat_map(|(fi, (_, _, lines))| lines.iter().enumerate()
|
||||
.map(move |(li, l)| (l.get(..23).filter(|t| line_time_ok(t)).unwrap_or(""), fi, li)))
|
||||
.collect();
|
||||
order.sort();
|
||||
let gone: std::collections::HashSet<(usize, usize)> = order.iter().take(dropped).map(|&(_, f, l)| (f, l)).collect();
|
||||
let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
|
||||
let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
|
||||
for (fi, (_, name, lines)) in files.iter().enumerate() {
|
||||
let kept: Vec<&String> = lines.iter().enumerate().filter(|(li, _)| !gone.contains(&(fi, *li))).map(|(_, l)| l).collect();
|
||||
if kept.is_empty() {
|
||||
continue;
|
||||
}
|
||||
zip.start_file(name.as_str(), opts).map_err(std::io::Error::other)?;
|
||||
for l in kept {
|
||||
zip.write_all(l.as_bytes())?;
|
||||
zip.write_all(b"\n")?;
|
||||
}
|
||||
}
|
||||
if dropped > 0 {
|
||||
zip.start_file("NOTE.txt", opts).map_err(std::io::Error::other)?;
|
||||
zip.write_all(format!("The oldest {dropped} lines were left out to stay under 10 MB.\n").as_bytes())?;
|
||||
}
|
||||
Ok(zip.finish().map_err(std::io::Error::other)?.into_inner())
|
||||
}
|
||||
|
||||
fn line_time_ok(t: &str) -> bool {
|
||||
chrono::NaiveDateTime::parse_from_str(t, "%Y-%m-%d %H:%M:%S%.3f").is_ok()
|
||||
}
|
||||
|
||||
fn file_day(name: &str) -> Option<NaiveDate> {
|
||||
let stem = name.strip_suffix(".log")?;
|
||||
let date = stem.get(stem.len().checked_sub(10)?..)?;
|
||||
NaiveDate::parse_from_str(date, "%Y-%m-%d").ok()
|
||||
}
|
||||
|
||||
fn format_line(l: &LogLine) -> String {
|
||||
let at = DateTime::from_timestamp_millis(l.at_ms).map(|t| t.with_timezone(&Local).format("%Y-%m-%d %H:%M:%S%.3f").to_string())
|
||||
.unwrap_or_default();
|
||||
let level = match l.level { Level::Debug => "DEBUG", Level::Info => "INFO", Level::Warn => "WARN", Level::Error => "ERROR" };
|
||||
format!("{at} {level:5} [{}] {}", l.tab, l.text)
|
||||
}
|
||||
|
||||
fn token_char(c: char) -> bool {
|
||||
c.is_ascii_alphanumeric() || c == '_' || c == '-'
|
||||
}
|
||||
|
||||
/// A Discord bot token: three base64url parts joined by dots (id, time, signature).
|
||||
fn looks_like_token(word: &str) -> bool {
|
||||
let parts: Vec<&str> = word.split('.').collect();
|
||||
parts.len() == 3 && parts[0].len() >= 20 && parts[1].len() >= 5 && parts[2].len() >= 25
|
||||
&& parts.iter().all(|p| p.chars().all(token_char))
|
||||
}
|
||||
|
||||
/// Mask each stored secret and anything shaped like a Discord bot token.
|
||||
pub fn redact(text: &str, secrets: &[String]) -> String {
|
||||
let mut out = text.to_owned();
|
||||
for s in secrets.iter().filter(|s| s.len() >= 6) {
|
||||
out = out.replace(s.as_str(), SECRET_MARK);
|
||||
}
|
||||
let mut result = String::with_capacity(out.len());
|
||||
let mut word = String::new();
|
||||
// A word may carry sentence dots around it ("token=<token>."): they are not part of the token.
|
||||
let flush = |word: &mut String, result: &mut String| {
|
||||
let core = word.trim_matches('.');
|
||||
if looks_like_token(core) {
|
||||
let lead = word.len() - word.trim_start_matches('.').len();
|
||||
let trail = word.len() - word.trim_end_matches('.').len();
|
||||
result.push_str(&word[..lead]);
|
||||
result.push_str(SECRET_MARK);
|
||||
result.push_str(&word[word.len() - trail..]);
|
||||
} else {
|
||||
result.push_str(word)
|
||||
}
|
||||
word.clear();
|
||||
};
|
||||
for c in out.chars() {
|
||||
if token_char(c) || c == '.' {
|
||||
word.push(c);
|
||||
} else {
|
||||
flush(&mut word, &mut result);
|
||||
result.push(c);
|
||||
}
|
||||
}
|
||||
flush(&mut word, &mut result);
|
||||
result
|
||||
}
|
||||
|
||||
/// OS, CPU threads and memory, for error reports.
|
||||
pub fn pc_summary() -> String {
|
||||
let cpus = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(0);
|
||||
let mem = total_memory_bytes().map(|b| format!("{:.1} GB RAM", b as f64 / 1e9)).unwrap_or_default();
|
||||
format!("{} {} | {cpus} CPU threads | {mem}", std::env::consts::OS, std::env::consts::ARCH)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn total_memory_bytes() -> Option<u64> {
|
||||
let s = std::fs::read_to_string("/proc/meminfo").ok()?;
|
||||
let kb: u64 = s.lines().find(|l| l.starts_with("MemTotal:"))?.split_whitespace().nth(1)?.parse().ok()?;
|
||||
Some(kb * 1024)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
fn total_memory_bytes() -> Option<u64> {
|
||||
use windows_sys::Win32::System::SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX};
|
||||
// SAFETY: the struct is sized and zeroed as the API asks.
|
||||
let mut m: MEMORYSTATUSEX = unsafe { std::mem::zeroed() };
|
||||
m.dwLength = std::mem::size_of::<MEMORYSTATUSEX>() as u32;
|
||||
(unsafe { GlobalMemoryStatusEx(&mut m) } != 0).then_some(m.ullTotalPhys)
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
fn total_memory_bytes() -> Option<u64> {
|
||||
None
|
||||
}
|
||||
|
||||
static HUB: OnceLock<Arc<LogHub>> = OnceLock::new();
|
||||
|
||||
/// Make the log macros feed `hub` too (once per process).
|
||||
pub fn install(hub: Arc<LogHub>) {
|
||||
let _ = HUB.set(hub);
|
||||
}
|
||||
|
||||
pub fn installed() -> Option<&'static Arc<LogHub>> {
|
||||
HUB.get()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use chrono::TimeZone;
|
||||
|
||||
fn at(y: i32, m: u32, d: u32) -> DateTime<Local> {
|
||||
Local.with_ymd_and_hms(y, m, d, 12, 0, 0).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categories_map_to_tabs_and_worker_lines_to_their_own() {
|
||||
let hub = LogHub::new(None);
|
||||
hub.push(Level::Debug, "play", "say id=r1");
|
||||
hub.push(Level::Error, "nonsense", "x");
|
||||
hub.push_worker("[ voice_worker ] brain reply ms=600");
|
||||
hub.push_worker("[ stt ] batch=3");
|
||||
hub.push_worker("/x.py:9: FutureWarning: deprecated");
|
||||
let tabs: Vec<_> = hub.query(None, Level::Debug, None, None, 99).into_iter().map(|l| (l.tab, l.level)).collect();
|
||||
assert_eq!(tabs, vec![("discord".into(), Level::Debug), ("app".into(), Level::Error),
|
||||
("worker".into(), Level::Info), ("stt".into(), Level::Info), ("worker".into(), Level::Warn)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_filters_by_tab_level_text_and_follows() {
|
||||
let hub = LogHub::new(None);
|
||||
for i in 0..10 {
|
||||
hub.push(if i % 2 == 0 { Level::Debug } else { Level::Error }, "chat", &format!("line {i}"));
|
||||
}
|
||||
hub.push(Level::Error, "voice", "joined");
|
||||
assert_eq!(hub.query(Some("chat"), Level::Error, None, None, 99).len(), 5);
|
||||
assert_eq!(hub.query(None, Level::Debug, Some("LINE 3"), None, 99)[0].text, "line 3");
|
||||
let last = hub.query(None, Level::Debug, None, None, 3);
|
||||
assert_eq!(last.iter().map(|l| l.seq).collect::<Vec<_>>(), vec![8, 9, 10]);
|
||||
assert_eq!(hub.query(None, Level::Debug, None, Some(9), 99).len(), 1, "only lines after seq 9");
|
||||
for i in 0..KEPT_LINES {
|
||||
hub.push(Level::Debug, "app", &i.to_string());
|
||||
}
|
||||
assert_eq!(hub.query(None, Level::Debug, None, None, usize::MAX).len(), KEPT_LINES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lines_go_to_daily_files_and_old_files_are_removed() {
|
||||
let dir = temp_dir("logs");
|
||||
std::fs::write(dir.join("chat-2026-01-01.log"), "old\n").unwrap();
|
||||
std::fs::write(dir.join("notes.txt"), "not a log").unwrap();
|
||||
let hub = LogHub::new(Some(&dir));
|
||||
hub.push_at(at(2026, 1, 10), Level::Info, "chat", "kept");
|
||||
hub.push_at(at(2026, 1, 20), Level::Info, "chat", "next day");
|
||||
assert!(std::fs::read_to_string(dir.join("chat-2026-01-10.log")).unwrap().contains("kept"));
|
||||
assert!(std::fs::read_to_string(dir.join("chat-2026-01-20.log")).unwrap().contains("next day"));
|
||||
// The 20th is more than 14 days after the 1st.
|
||||
assert!(!dir.join("chat-2026-01-01.log").exists());
|
||||
assert!(dir.join("notes.txt").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_error_report_holds_the_last_error_and_its_neighbours_masked() {
|
||||
let hub = LogHub::new(None);
|
||||
assert_eq!(hub.recent_error_report("0.1.0"), None);
|
||||
hub.set_secrets(vec!["s3cr3t-value".into()]);
|
||||
for i in 0..50 {
|
||||
hub.push(Level::Debug, "app", &format!("before {i}"));
|
||||
}
|
||||
hub.push(Level::Error, "discord", "login failed with s3cr3t-value");
|
||||
for i in 0..20 {
|
||||
hub.push(Level::Debug, "app", &format!("after {i}"));
|
||||
}
|
||||
let r = hub.recent_error_report("0.1.0").unwrap();
|
||||
assert!(r.starts_with("Jarvis 0.1.0 | "));
|
||||
assert!(r.contains("before 49") && r.contains("before 20") && !r.contains("before 19"));
|
||||
assert!(r.contains("after 9") && !r.contains("after 10"));
|
||||
assert!(r.contains("login failed with [비밀값]") && !r.contains("s3cr3t"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redaction_masks_secrets_and_token_shaped_text() {
|
||||
let token = format!("{}.{}.{}", "M".repeat(24), "GabcdE", "x".repeat(30));
|
||||
let out = redact(&format!("token={token}, ok id=123456789012345678 a.b.c"), &[]);
|
||||
assert_eq!(out, "token=[비밀값], ok id=123456789012345678 a.b.c");
|
||||
// Sentence dots around it do not hide it.
|
||||
assert_eq!(redact(&format!("token={token}. next ...{token}..."), &[]), "token=[비밀값]. next ...[비밀값]...");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_zip_leaves_out_the_oldest_lines_by_time_across_tabs() {
|
||||
let at_hm = |d: u32, h: u32, m: u32| Local.with_ymd_and_hms(2026, 5, d, h, m, 0).unwrap();
|
||||
let dir = temp_dir("logorder");
|
||||
let hub = LogHub::new(Some(&dir));
|
||||
hub.push_at(at_hm(1, 0, 1), Level::Info, "chat", "first chat line");
|
||||
hub.push_at(at_hm(1, 23, 59), Level::Info, "app", "late on the first");
|
||||
hub.push_at(at_hm(2, 0, 1), Level::Info, "chat", "early on the second");
|
||||
let mut files: Vec<(NaiveDate, String, Vec<String>)> = vec![];
|
||||
for e in std::fs::read_dir(&dir).unwrap().flatten() {
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
files.push((file_day(&name).unwrap(), name, std::fs::read_to_string(e.path()).unwrap().lines().map(str::to_owned).collect()));
|
||||
}
|
||||
files.sort();
|
||||
// Leave out one line: the 00:01 chat line of the 1st, not a line of the app file.
|
||||
let z = build_zip(&files, 1).unwrap();
|
||||
let mut z = zip::ZipArchive::new(std::io::Cursor::new(z)).unwrap();
|
||||
let mut all = String::new();
|
||||
for i in 0..z.len() {
|
||||
std::io::Read::read_to_string(&mut z.by_index(i).unwrap(), &mut all).unwrap();
|
||||
}
|
||||
assert!(!all.contains("first chat line"));
|
||||
assert!(all.contains("late on the first") && all.contains("early on the second"));
|
||||
// Two: then the 23:59 app line goes and the next day's chat line stays.
|
||||
let z = build_zip(&files, 2).unwrap();
|
||||
let mut z = zip::ZipArchive::new(std::io::Cursor::new(z)).unwrap();
|
||||
let mut all = String::new();
|
||||
for i in 0..z.len() {
|
||||
std::io::Read::read_to_string(&mut z.by_index(i).unwrap(), &mut all).unwrap();
|
||||
}
|
||||
assert!(!all.contains("late on the first") && all.contains("early on the second"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_is_masked_when_the_line_is_written_so_removing_or_replacing_it_later_shows_nothing() {
|
||||
let dir = temp_dir("logwrite");
|
||||
let hub = LogHub::new(Some(&dir));
|
||||
hub.set_secrets(vec!["old-api-key-1".into()]);
|
||||
hub.push_at(Local::now(), Level::Error, "llm", "server refused old-api-key-1");
|
||||
// The key is replaced (or its server removed): the hub no longer knows the old value.
|
||||
hub.set_secrets(vec!["new-api-key-2".into()]);
|
||||
let shown: String = hub.query(None, Level::Debug, None, None, 100).iter().map(|l| l.text.clone()).collect();
|
||||
assert!(shown.contains("server refused [비밀값]") && !shown.contains("old-api-key-1"), "{shown}");
|
||||
assert!(!hub.recent_error_report("0.1.0").unwrap().contains("old-api-key-1"));
|
||||
// After a restart a new hub reads the files: the value was never written.
|
||||
// Today's file (a new hub removes files older than the retention).
|
||||
let today = Local::now().date_naive();
|
||||
let file = std::fs::read_to_string(dir.join(format!("llm-{today}.log"))).unwrap();
|
||||
assert!(!file.contains("old-api-key-1"), "{file}");
|
||||
let out = dir.join("after.zip");
|
||||
LogHub::new(Some(&dir)).export_zip(today, today, &out).unwrap();
|
||||
let mut z = zip::ZipArchive::new(std::fs::File::open(&out).unwrap()).unwrap();
|
||||
let mut text = String::new();
|
||||
std::io::Read::read_to_string(&mut z.by_name(&format!("llm-{today}.log")).unwrap(), &mut text).unwrap();
|
||||
assert!(!text.contains("old-api-key-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_key_replaced_while_the_services_run_stays_masked_in_their_late_lines_until_the_next_start() {
|
||||
let dir = temp_dir("loginuse");
|
||||
let hub = LogHub::new(Some(&dir));
|
||||
let today = Local::now().date_naive();
|
||||
// The services start with the old key.
|
||||
hub.set_secrets(vec!["old-api-key-1".into()]);
|
||||
hub.set_secrets_in_use(vec!["old-api-key-1".into()]);
|
||||
// While they run the key is replaced (or its server removed): only the new one is stored.
|
||||
hub.set_secrets(vec!["new-api-key-2".into()]);
|
||||
// A late line of the running worker still carries the old key.
|
||||
hub.push_worker("[ llm ] Error: MCP server brave failed: bad key old-api-key-1");
|
||||
let shown: String = hub.query(None, Level::Debug, None, None, 100).iter().map(|l| l.text.clone()).collect();
|
||||
assert!(shown.contains("bad key [비밀값]") && !shown.contains("old-api-key-1"), "{shown}");
|
||||
assert!(!hub.recent_error_report("0.1.0").unwrap().contains("old-api-key-1"));
|
||||
let file = std::fs::read_to_string(dir.join(format!("llm-{today}.log"))).unwrap();
|
||||
assert!(!file.contains("old-api-key-1"), "{file}");
|
||||
let out = dir.join("inuse.zip");
|
||||
hub.export_zip(today, today, &out).unwrap();
|
||||
let mut z = zip::ZipArchive::new(std::fs::File::open(&out).unwrap()).unwrap();
|
||||
let mut text = String::new();
|
||||
std::io::Read::read_to_string(&mut z.by_name(&format!("llm-{today}.log")).unwrap(), &mut text).unwrap();
|
||||
assert!(!text.contains("old-api-key-1"));
|
||||
// The next start gives the services only the new key: the old one is no longer held.
|
||||
hub.set_secrets_in_use(vec!["new-api-key-2".into()]);
|
||||
hub.push(Level::Info, "app", "after restart old-api-key-1 new-api-key-2");
|
||||
let last = hub.query(None, Level::Debug, None, None, 1).pop().unwrap().text;
|
||||
assert_eq!(last, "after restart old-api-key-1 [비밀값]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_zip_stays_under_the_limit_by_leaving_out_the_oldest_lines() {
|
||||
let dir = temp_dir("logzip");
|
||||
let hub = LogHub::new(Some(&dir));
|
||||
hub.set_secrets(vec!["hidden-token".into()]);
|
||||
hub.push_at(at(2026, 3, 1), Level::Info, "app", "day one hidden-token");
|
||||
hub.push_at(at(2026, 3, 2), Level::Info, "app", "day two");
|
||||
hub.push_at(at(2026, 3, 5), Level::Info, "app", "outside the range");
|
||||
let out = dir.join("small.zip");
|
||||
assert_eq!(hub.export_zip(NaiveDate::from_ymd_opt(2026, 3, 1).unwrap(),
|
||||
NaiveDate::from_ymd_opt(2026, 3, 2).unwrap(), &out).unwrap(), 0);
|
||||
let mut z = zip::ZipArchive::new(std::fs::File::open(&out).unwrap()).unwrap();
|
||||
let mut names: Vec<String> = (0..z.len()).map(|i| z.by_index(i).unwrap().name().unwrap().to_string()).collect();
|
||||
names.sort();
|
||||
assert_eq!(names, vec!["app-2026-03-01.log", "app-2026-03-02.log"]);
|
||||
let mut first = String::new();
|
||||
std::io::Read::read_to_string(&mut z.by_name("app-2026-03-01.log").unwrap(), &mut first).unwrap();
|
||||
assert!(first.contains("day one [비밀값]"));
|
||||
|
||||
// Incompressible lines past 10 MB: the oldest go, the newest stay.
|
||||
let big = temp_dir("logbig");
|
||||
let hub = LogHub::new(Some(&big));
|
||||
let mut seed = 1u64;
|
||||
for day in 1..=3 {
|
||||
for _ in 0..40_000 {
|
||||
let mut s = String::new();
|
||||
for _ in 0..140 {
|
||||
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
s.push(char::from(b'!' + (seed >> 58) as u8));
|
||||
}
|
||||
hub.push_at(at(2026, 4, day), Level::Info, "app", &s);
|
||||
}
|
||||
}
|
||||
hub.push_at(at(2026, 4, 3), Level::Info, "app", "the newest line");
|
||||
let out = big.join("big.zip");
|
||||
let dropped = hub.export_zip(NaiveDate::from_ymd_opt(2026, 4, 1).unwrap(),
|
||||
NaiveDate::from_ymd_opt(2026, 4, 3).unwrap(), &out).unwrap();
|
||||
assert!(dropped > 0);
|
||||
assert!(std::fs::metadata(&out).unwrap().len() as usize <= ZIP_MAX_BYTES);
|
||||
let mut z = zip::ZipArchive::new(std::fs::File::open(&out).unwrap()).unwrap();
|
||||
let mut last = String::new();
|
||||
std::io::Read::read_to_string(&mut z.by_name("app-2026-04-03.log").unwrap(), &mut last).unwrap();
|
||||
assert!(last.contains("the newest line"));
|
||||
let mut note = String::new();
|
||||
std::io::Read::read_to_string(&mut z.by_name("NOTE.txt").unwrap(), &mut note).unwrap();
|
||||
assert!(note.contains(&dropped.to_string()));
|
||||
}
|
||||
}
|
||||
43
app/src/main.rs
Normal file
43
app/src/main.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
//! Jarvis app (headless until the Tauri GUI arrives in M5): jarvis-app <data folder>
|
||||
//! The data folder can also come from JARVIS_DATA_DIR; the bot token from the stored secrets, else
|
||||
//! JARVIS_DISCORD_TOKEN. The services run through the same controller the dashboard uses.
|
||||
|
||||
use jarvis_app::control::{Controller, ServicesHost};
|
||||
use jarvis_app::error_log;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let data_dir = std::env::args().nth(1).or_else(|| std::env::var("JARVIS_DATA_DIR").ok());
|
||||
let Some(data_dir) = data_dir else {
|
||||
error_log!("app", "usage: jarvis-app <data folder> (or set JARVIS_DATA_DIR)");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let data_dir = std::path::PathBuf::from(data_dir);
|
||||
// The log hub: daily files in <data folder>/logs, stored secrets masked in exports.
|
||||
let logs_dir = data_dir.join("logs");
|
||||
let _ = std::fs::create_dir_all(&logs_dir);
|
||||
let hub = jarvis_app::logs::LogHub::new(Some(&logs_dir));
|
||||
hub.set_secrets(jarvis_app::secrets::Secrets::new(&data_dir.join("config").join("secrets.dat")).values());
|
||||
jarvis_app::logs::install(hub);
|
||||
let controller = Controller::new(ServicesHost::new(data_dir));
|
||||
if controller.start().await.is_err() {
|
||||
std::process::exit(1);
|
||||
}
|
||||
shutdown_signal().await;
|
||||
let _ = controller.stop().await;
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("signal");
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {}
|
||||
_ = term.recv() => {}
|
||||
}
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
}
|
||||
}
|
||||
193
app/src/notes.rs
Normal file
193
app/src/notes.rs
Normal file
@@ -0,0 +1,193 @@
|
||||
//! Each person's standing instructions for how Jarvis answers them (`app/src/admin.spec.md`,
|
||||
//! "Personal notes"; `docs/v2/proactive.md` 제안 4): the store in the data folder, the brain's
|
||||
//! `note_action`, and the notes sent with each of that person's replies.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::worker_client::NoteAction;
|
||||
|
||||
pub const MAX_TEXT_CHARS: usize = 300;
|
||||
/// Notes one person may have (the worker takes at most this many with a reply).
|
||||
pub const PER_PERSON: usize = 20;
|
||||
|
||||
static FILE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Note {
|
||||
pub id: u64,
|
||||
pub user: u64,
|
||||
pub name: String,
|
||||
pub text: String,
|
||||
pub created: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct File {
|
||||
next_id: u64,
|
||||
items: Vec<Note>,
|
||||
}
|
||||
|
||||
pub struct Notes {
|
||||
file: PathBuf,
|
||||
}
|
||||
|
||||
fn clean(text: &str) -> Result<String, String> {
|
||||
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if text.is_empty() || text.chars().count() > MAX_TEXT_CHARS {
|
||||
return Err(format!("a note must be 1-{MAX_TEXT_CHARS} characters"));
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
impl Notes {
|
||||
pub fn new(data: &Path) -> Self {
|
||||
Self { file: data.join("config").join("notes.json") }
|
||||
}
|
||||
|
||||
fn load(&self) -> Result<File, String> {
|
||||
match std::fs::read_to_string(&self.file) {
|
||||
Ok(t) => serde_json::from_str(&t).map_err(|e| format!("지침 파일을 읽을 수 없습니다: {e}")),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(File::default()),
|
||||
Err(e) => Err(format!("지침 파일을 읽을 수 없습니다: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn change<T>(&self, f: impl FnOnce(&mut File) -> Result<T, String>) -> Result<T, String> {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut file = self.load()?;
|
||||
let out = f(&mut file)?;
|
||||
let mut text = serde_json::to_string_pretty(&file).unwrap();
|
||||
text.push('\n');
|
||||
crate::datadir::write_atomic(&self.file, text.as_bytes()).map_err(|e| format!("지침 파일을 쓸 수 없습니다: {e}"))?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Every note, by person then age.
|
||||
pub fn all(&self) -> Result<Vec<Note>, String> {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut items = self.load()?.items;
|
||||
items.sort_by_key(|n| (n.user, n.id));
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// One person's notes, oldest first.
|
||||
pub fn of(&self, user: u64) -> Result<Vec<Note>, String> {
|
||||
Ok(self.all()?.into_iter().filter(|n| n.user == user).collect())
|
||||
}
|
||||
|
||||
/// The texts sent with that person's replies; a store that cannot be read sends none.
|
||||
pub fn texts_of(&self, user: u64) -> Vec<String> {
|
||||
match self.of(user) {
|
||||
Ok(n) => n.into_iter().map(|n| n.text).take(PER_PERSON).collect(),
|
||||
Err(e) => {
|
||||
crate::warn_log!("conversation", "Warning: {e}");
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(&self, user: u64, name: &str, text: &str, now: i64) -> Result<Note, String> {
|
||||
let text = clean(text)?;
|
||||
self.change(|f| {
|
||||
if f.items.iter().filter(|n| n.user == user).count() >= PER_PERSON {
|
||||
return Err(format!("at most {PER_PERSON} notes per person; delete one first"));
|
||||
}
|
||||
f.next_id += 1;
|
||||
let n = Note { id: f.next_id, user, name: name.to_owned(), text, created: now };
|
||||
f.items.push(n.clone());
|
||||
Ok(n)
|
||||
})
|
||||
}
|
||||
|
||||
/// Change a note's text (the screen).
|
||||
pub fn update(&self, id: u64, text: &str) -> Result<Note, String> {
|
||||
let text = clean(text)?;
|
||||
self.change(|f| {
|
||||
let n = f.items.iter_mut().find(|n| n.id == id).ok_or("no such note")?;
|
||||
n.text = text;
|
||||
Ok(n.clone())
|
||||
})
|
||||
}
|
||||
|
||||
/// Delete a note; with `user`, only one of theirs.
|
||||
pub fn delete(&self, id: u64, user: Option<u64>) -> Result<Note, String> {
|
||||
self.change(|f| {
|
||||
let pos = f.items.iter().position(|n| n.id == id && user.is_none_or(|u| u == n.user))
|
||||
.ok_or_else(|| "no such note of this person".to_string())?;
|
||||
Ok(f.items.remove(pos))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn brain_item(n: &Note) -> Value {
|
||||
json!({"id": n.id.to_string(), "text": n.text})
|
||||
}
|
||||
|
||||
/// Do what the brain's personalNotes tool asked for the person in `a.user` (named `name`).
|
||||
pub fn act(notes: &Notes, a: &NoteAction, name: &str, now: i64) -> Result<Value, String> {
|
||||
let user: u64 = a.user.parse().map_err(|_| "bad user id".to_string())?;
|
||||
match a.action.as_str() {
|
||||
"list" => Ok(json!({"items": notes.of(user)?.iter().map(brain_item).collect::<Vec<_>>()})),
|
||||
"add" => {
|
||||
let n = notes.add(user, name, a.text.as_deref().unwrap_or(""), now)?;
|
||||
crate::info_log!("conversation", "note {} added by {user}", n.id);
|
||||
Ok(json!({"item": brain_item(&n)}))
|
||||
}
|
||||
"delete" => {
|
||||
let id: u64 = a.note.as_deref().and_then(|s| s.trim().parse().ok()).ok_or("delete needs the note's id")?;
|
||||
Ok(json!({"item": brain_item(¬es.delete(id, Some(user))?)}))
|
||||
}
|
||||
other => Err(format!("unknown note action {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn notes() -> Notes {
|
||||
Notes::new(&crate::datadir::temp_dir("notes"))
|
||||
}
|
||||
|
||||
fn action(action: &str, user: &str, extra: Value) -> NoteAction {
|
||||
let mut v = json!({"id": "x", "action": action, "user": user, "via": "voice"});
|
||||
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
|
||||
serde_json::from_value(v).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_persons_notes_are_theirs_alone() {
|
||||
let n = notes();
|
||||
let added = act(&n, &action("add", "1", json!({"text": " 반말로 해 "})), "가", 10).unwrap();
|
||||
assert_eq!(added["item"]["text"], "반말로 해");
|
||||
act(&n, &action("add", "2", json!({"text": "섭씨로"})), "나", 10).unwrap();
|
||||
assert_eq!(n.texts_of(1), vec!["반말로 해".to_string()]);
|
||||
assert!(!act(&n, &action("list", "2", json!({})), "나", 10).unwrap().to_string().contains("반말"));
|
||||
let id = added["item"]["id"].as_str().unwrap().to_owned();
|
||||
assert!(act(&n, &action("delete", "2", json!({"note": id})), "나", 10).is_err());
|
||||
assert!(act(&n, &action("delete", "1", json!({"note": id})), "가", 10).is_ok());
|
||||
assert!(n.texts_of(1).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_notes_and_too_many_are_refused_and_the_screen_can_edit_any() {
|
||||
let n = notes();
|
||||
assert!(n.add(1, "가", " ", 0).is_err());
|
||||
assert!(n.add(1, "가", &"x".repeat(MAX_TEXT_CHARS + 1), 0).is_err());
|
||||
for i in 0..PER_PERSON {
|
||||
n.add(1, "가", &format!("지침 {i}"), 0).unwrap();
|
||||
}
|
||||
assert!(n.add(1, "가", "하나 더", 0).is_err());
|
||||
let first = n.of(1).unwrap()[0].clone();
|
||||
assert_eq!(n.update(first.id, "고친 지침").unwrap().text, "고친 지침");
|
||||
assert!(n.update(first.id, "").is_err());
|
||||
assert_eq!(n.texts_of(1)[0], "고친 지침");
|
||||
assert_eq!(n.delete(first.id, None).unwrap().id, first.id);
|
||||
}
|
||||
}
|
||||
136
app/src/perf.rs
Normal file
136
app/src/perf.rs
Normal file
@@ -0,0 +1,136 @@
|
||||
//! The performance stages (`docs/v2/pc-features.md`, "자동 성능 조절" and "전환 규칙"; admin.spec.md,
|
||||
//! "Performance stages"): which placement the LLM should run in, from game mode and time. The
|
||||
//! switching itself (waiting for the answer in progress, restarting the LLM) is `services.rs`.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Stage {
|
||||
Normal,
|
||||
Gaming,
|
||||
/// The LLM is stopped while a game runs (chosen, or no placement leaves the game room).
|
||||
Paused,
|
||||
}
|
||||
|
||||
/// Picks the stage: a game means the game stage (or pause when the person chose it) at once; after
|
||||
/// the game ends the normal stage comes back only once `revert_after` has passed, so quitting and
|
||||
/// starting a game again does not move the model back and forth.
|
||||
#[derive(Debug)]
|
||||
pub struct Perf {
|
||||
revert_after: Duration,
|
||||
pause_in_game: bool,
|
||||
game_since_off: Option<Instant>,
|
||||
current: Stage,
|
||||
}
|
||||
|
||||
impl Perf {
|
||||
pub fn new(revert_after: Duration, pause_in_game: bool) -> Self {
|
||||
Self { revert_after, pause_in_game, game_since_off: None, current: Stage::Normal }
|
||||
}
|
||||
|
||||
/// The stage wanted now, given whether a game runs.
|
||||
pub fn target(&mut self, game: bool, now: Instant) -> Stage {
|
||||
if game {
|
||||
self.game_since_off = None;
|
||||
self.current = if self.pause_in_game { Stage::Paused } else { Stage::Gaming };
|
||||
} else if self.current != Stage::Normal {
|
||||
let since = *self.game_since_off.get_or_insert(now);
|
||||
if now.duration_since(since) >= self.revert_after {
|
||||
self.current = Stage::Normal;
|
||||
self.game_since_off = None;
|
||||
}
|
||||
}
|
||||
self.current
|
||||
}
|
||||
}
|
||||
|
||||
/// How often a stage that failed to start is tried again (then not until the wanted stage changes).
|
||||
pub const MAX_TRIES: u32 = 3;
|
||||
|
||||
/// The limited retry of a stage whose restart failed: after a failure it waits `after`, and after
|
||||
/// `MAX_TRIES` failures of the same stage it stops until another stage is wanted.
|
||||
#[derive(Debug)]
|
||||
pub struct Retry {
|
||||
after: Duration,
|
||||
stage: Option<Stage>,
|
||||
tries: u32,
|
||||
next: Option<Instant>,
|
||||
}
|
||||
|
||||
impl Retry {
|
||||
pub fn new(after: Duration) -> Self {
|
||||
Self { after, stage: None, tries: 0, next: None }
|
||||
}
|
||||
|
||||
/// Whether switching to `want` may be tried now.
|
||||
pub fn allowed(&self, want: Stage, now: Instant) -> bool {
|
||||
self.stage != Some(want) || (self.tries < MAX_TRIES && self.next.is_none_or(|n| now >= n))
|
||||
}
|
||||
|
||||
pub fn failed(&mut self, want: Stage, now: Instant) {
|
||||
if self.stage == Some(want) {
|
||||
self.tries += 1;
|
||||
} else {
|
||||
(self.stage, self.tries) = (Some(want), 1);
|
||||
}
|
||||
self.next = Some(now + self.after);
|
||||
}
|
||||
|
||||
pub fn succeeded(&mut self) {
|
||||
(self.stage, self.tries, self.next) = (None, 0, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_game_switches_at_once_and_its_end_only_after_the_wait() {
|
||||
let t = Instant::now();
|
||||
let s = |n: u64| t + Duration::from_secs(n);
|
||||
let mut p = Perf::new(Duration::from_secs(30), false);
|
||||
assert_eq!(p.target(false, s(0)), Stage::Normal);
|
||||
assert_eq!(p.target(true, s(1)), Stage::Gaming);
|
||||
// The game ends: still the game stage for 30 s.
|
||||
assert_eq!(p.target(false, s(2)), Stage::Gaming);
|
||||
assert_eq!(p.target(false, s(31)), Stage::Gaming);
|
||||
// Started again within the wait: nothing moves, and the wait starts over at the next end.
|
||||
assert_eq!(p.target(true, s(32)), Stage::Gaming);
|
||||
assert_eq!(p.target(false, s(40)), Stage::Gaming);
|
||||
assert_eq!(p.target(false, s(69)), Stage::Gaming);
|
||||
assert_eq!(p.target(false, s(70)), Stage::Normal);
|
||||
assert_eq!(p.target(false, s(500)), Stage::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_stage_is_tried_again_after_a_wait_three_times_at_most() {
|
||||
let t = Instant::now();
|
||||
let s = |n: u64| t + Duration::from_secs(n);
|
||||
let mut r = Retry::new(Duration::from_secs(60));
|
||||
assert!(r.allowed(Stage::Gaming, s(0)));
|
||||
r.failed(Stage::Gaming, s(0));
|
||||
assert!(!r.allowed(Stage::Gaming, s(59)), "waits");
|
||||
assert!(r.allowed(Stage::Normal, s(1)), "another stage may be tried at once");
|
||||
assert!(r.allowed(Stage::Gaming, s(60)));
|
||||
r.failed(Stage::Gaming, s(60));
|
||||
r.failed(Stage::Gaming, s(120));
|
||||
assert!(!r.allowed(Stage::Gaming, s(1000)), "three failures: no more tries");
|
||||
// Another stage wanted (and failing) starts its own count; a success clears it.
|
||||
r.failed(Stage::Normal, s(1000));
|
||||
assert!(r.allowed(Stage::Gaming, s(1000)));
|
||||
r.succeeded();
|
||||
assert!(r.allowed(Stage::Normal, s(1000)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_pause_chosen_a_game_pauses_the_llm() {
|
||||
let t = Instant::now();
|
||||
let mut p = Perf::new(Duration::ZERO, true);
|
||||
assert_eq!(p.target(true, t), Stage::Paused);
|
||||
assert_eq!(p.target(false, t), Stage::Normal);
|
||||
}
|
||||
}
|
||||
110
app/src/permissions.rs
Normal file
110
app/src/permissions.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
//! What Jarvis may do on this PC, and for whom (`app/src/admin.spec.md`, "PC control permissions";
|
||||
//! `docs/v2/proactive.md` 제안 3). The PC control module asks `decide` before every action.
|
||||
|
||||
use crate::settings::{Level, PcControlSettings};
|
||||
|
||||
/// A kind of action on the PC.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Action {
|
||||
/// See the screen, a monitor or a window, and list the windows.
|
||||
View,
|
||||
/// Press a button, a menu item or a point on the screen.
|
||||
Click,
|
||||
/// Put text or keys into a program.
|
||||
Type,
|
||||
/// Start a program.
|
||||
Run,
|
||||
/// Pay or buy anything.
|
||||
Pay,
|
||||
/// Read, type or change a password or another secret.
|
||||
Password,
|
||||
/// Delete files.
|
||||
DeleteFiles,
|
||||
/// Anything that needs administrator rights.
|
||||
Admin,
|
||||
}
|
||||
|
||||
/// Why an action is refused (the PC control module tells the person).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
/// The person is not in `pcControl.allowedUsers`.
|
||||
NotAllowed,
|
||||
/// The action's level is "안 함".
|
||||
TurnedOff,
|
||||
/// Never done, whatever the settings say.
|
||||
Never,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Decision {
|
||||
Allow,
|
||||
/// Allowed once the person agrees in their next message.
|
||||
Ask,
|
||||
Refuse(Refusal),
|
||||
}
|
||||
|
||||
/// Whether `user` may have Jarvis do `action`. Only registered people may ask for anything (screen
|
||||
/// viewing too: the screen may show anything); payments, passwords, deleting files and
|
||||
/// administrator rights are never done.
|
||||
pub fn decide(s: &PcControlSettings, user: u64, action: Action) -> Decision {
|
||||
if !s.allowed_users.contains(&user) {
|
||||
return Decision::Refuse(Refusal::NotAllowed);
|
||||
}
|
||||
let level = match action {
|
||||
Action::View => s.view,
|
||||
Action::Click => s.click,
|
||||
Action::Type => s.type_text,
|
||||
Action::Run => s.run,
|
||||
Action::Pay | Action::Password | Action::DeleteFiles | Action::Admin => return Decision::Refuse(Refusal::Never),
|
||||
};
|
||||
match level {
|
||||
Level::Auto => Decision::Allow,
|
||||
Level::Ask => Decision::Ask,
|
||||
Level::Never => Decision::Refuse(Refusal::TurnedOff),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const OWNER: u64 = 123456789012345678;
|
||||
|
||||
fn settings() -> PcControlSettings {
|
||||
PcControlSettings { allowed_users: vec![OWNER], ..crate::settings::Settings::default().pc_control }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn by_default_viewing_is_automatic_and_operating_asks_first() {
|
||||
let s = settings();
|
||||
assert_eq!(decide(&s, OWNER, Action::View), Decision::Allow);
|
||||
for a in [Action::Click, Action::Type, Action::Run] {
|
||||
assert_eq!(decide(&s, OWNER, a), Decision::Ask, "{a:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_registered_people_may_ask_for_anything() {
|
||||
let s = settings();
|
||||
for a in [Action::View, Action::Click, Action::Type, Action::Run, Action::Pay] {
|
||||
assert_eq!(decide(&s, 42, a), Decision::Refuse(Refusal::NotAllowed), "{a:?}");
|
||||
}
|
||||
// Nobody is registered by default.
|
||||
let none = crate::settings::Settings::default().pc_control;
|
||||
assert_eq!(decide(&none, OWNER, Action::View), Decision::Refuse(Refusal::NotAllowed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_levels_follow_the_settings_but_some_actions_are_never_done() {
|
||||
let s = PcControlSettings { view: Level::Never, click: Level::Auto, type_text: Level::Never, run: Level::Auto,
|
||||
..settings() };
|
||||
assert_eq!(decide(&s, OWNER, Action::View), Decision::Refuse(Refusal::TurnedOff));
|
||||
assert_eq!(decide(&s, OWNER, Action::Click), Decision::Allow);
|
||||
assert_eq!(decide(&s, OWNER, Action::Type), Decision::Refuse(Refusal::TurnedOff));
|
||||
let all_auto = PcControlSettings { view: Level::Auto, click: Level::Auto, type_text: Level::Auto, run: Level::Auto,
|
||||
..settings() };
|
||||
for a in [Action::Pay, Action::Password, Action::DeleteFiles, Action::Admin] {
|
||||
assert_eq!(decide(&all_auto, OWNER, a), Decision::Refuse(Refusal::Never), "{a:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
152
app/src/placement.rs
Normal file
152
app/src/placement.rs
Normal file
@@ -0,0 +1,152 @@
|
||||
//! Where the LLM's routed experts go (`docs/v2/pc-features.md`, "PC마다 값을 계산"; admin.spec.md,
|
||||
//! "LLM placement"): the one model is placed by the M0 kit's formula, so the speech-to-text worker
|
||||
//! and other programs keep their room on the graphics card. Strata's automatic placement fills the
|
||||
//! card with experts and left the STT no room (M0 `gamemode_stt`), and its qwen36 engine takes no
|
||||
//! VRAM reserve, so the app sets `--expert-vram` itself.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What Strata's qwen36 engine keeps free itself when it places experts (M0 kit).
|
||||
pub const MARGIN_MIB: i64 = 768;
|
||||
/// The STT worker's share: its model (1300 MiB) and what ten speakers at once add (1440 MiB),
|
||||
/// measured on the game PC (M0 `stt`). The services start the LLM before the worker, so both count.
|
||||
pub const STT_MIB: i64 = STT_MODEL_MIB + 1440;
|
||||
/// The STT worker's model alone (M0 `stt`).
|
||||
pub const STT_MODEL_MIB: i64 = 1300;
|
||||
/// The TTS (MeloTTS, Korean) on the card after its warm-up sentence, measured on the game PC
|
||||
/// (M0 `tts_melo`). It loads after the LLM starts, so it is kept beside like the STT's share.
|
||||
pub const TTS_MIB: i64 = 1005;
|
||||
|
||||
/// What the voice worker's models hold on the card now: the STT model and the TTS once loaded.
|
||||
/// Taken off what the card holds, so they are not counted again as other programs.
|
||||
pub fn worker_models_mib(stt_loaded: bool, tts_loaded: bool) -> i64 {
|
||||
(if stt_loaded { STT_MODEL_MIB } else { 0 }) + if tts_loaded { TTS_MIB } else { 0 }
|
||||
}
|
||||
/// The smallest expert budget in a game: it holds the long-prompt staging (about 1.1 GiB) first,
|
||||
/// so 2 GiB is the least that loads experts, and it was 1.5-2.1 times faster than none in a real
|
||||
/// game (user decision, 2026-11-10).
|
||||
pub const MIN_GAME_EXPERT_GIB: f64 = 2.0;
|
||||
/// The context in a game (16K keeps room for the history, tools and results; 8K saved only 0.4 GB).
|
||||
pub const GAME_CONTEXT: u32 = 16384;
|
||||
|
||||
/// What one start without experts at `context` showed on this PC (the first-run self-check).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Probe {
|
||||
pub context: u32,
|
||||
/// Requests answered at once, which changes the engine's buffers.
|
||||
pub parallel: u32,
|
||||
/// The card measured (its NVIDIA UUID): another card means measuring again.
|
||||
pub gpu: String,
|
||||
/// What the LLM held without experts.
|
||||
pub model_mib: i64,
|
||||
/// The long-prompt staging the expert budget holds first (engine log, "x.xx GiB staging").
|
||||
pub staging_gib: f64,
|
||||
}
|
||||
|
||||
/// The expert budget (GiB, down to 0.1) that leaves `room_mib` free besides what the card already
|
||||
/// holds (`used_mib`), the LLM without experts, the STT's and TTS's shares and the engine's margin. The budget
|
||||
/// first holds the staging, which the start without experts already has (M0 `expert_budget_gib`).
|
||||
pub fn expert_budget_gib(total_mib: i64, used_mib: i64, p: &Probe, room_mib: i64) -> f64 {
|
||||
let spare = total_mib - used_mib - p.model_mib - STT_MIB - TTS_MIB - MARGIN_MIB - room_mib;
|
||||
if spare <= 0 {
|
||||
return 0.0;
|
||||
}
|
||||
((p.staging_gib * 1024.0 + spare as f64) / 1024.0 * 10.0).floor() / 10.0
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct GamePlacement {
|
||||
pub expert_gib: f64,
|
||||
/// What a game really gets when the reserve does not fit beside the smallest placement.
|
||||
pub game_room_mib: i64,
|
||||
/// Not even the smallest placement leaves a game any room: the LLM pauses in a game.
|
||||
pub paused: bool,
|
||||
}
|
||||
|
||||
/// The normal placement: experts get what is left after the room for other programs opened later.
|
||||
pub fn normal(total_mib: i64, used_mib: i64, p: &Probe, other_room_mib: i64) -> f64 {
|
||||
expert_budget_gib(total_mib, used_mib, p, other_room_mib)
|
||||
}
|
||||
|
||||
/// The game placement: experts get what is left after the game's reserve, but never less than the
|
||||
/// smallest placement (2 GiB); then the game gets what that leaves (shown on the dashboard). When
|
||||
/// 2 GiB leaves a game nothing, no experts on the card (0 GiB) is the last step; with no room even
|
||||
/// then the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
|
||||
pub fn game(total_mib: i64, used_mib: i64, p: &Probe, reserve_mib: i64) -> GamePlacement {
|
||||
let expert = expert_budget_gib(total_mib, used_mib, p, reserve_mib);
|
||||
if expert >= MIN_GAME_EXPERT_GIB {
|
||||
return GamePlacement { expert_gib: expert, game_room_mib: reserve_mib, paused: false };
|
||||
}
|
||||
// Beside the model without experts (which already holds the staging): what 2 GiB adds.
|
||||
let free = total_mib - used_mib - p.model_mib - STT_MIB - TTS_MIB - MARGIN_MIB;
|
||||
let extra = ((MIN_GAME_EXPERT_GIB - p.staging_gib) * 1024.0).max(0.0) as i64;
|
||||
if free - extra > 0 {
|
||||
return GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: free - extra, paused: false };
|
||||
}
|
||||
// The last step before pausing: no experts on the card (slower, but the game gets room).
|
||||
GamePlacement { expert_gib: 0.0, game_room_mib: free.max(0), paused: free <= 0 }
|
||||
}
|
||||
|
||||
/// The staging the engine log reports ("(1.10 GiB staging)"), else the usual 1.1 GiB.
|
||||
pub fn staging_gib(lines: &str) -> f64 {
|
||||
lines.lines().rev().find_map(|l| {
|
||||
let i = l.find(" GiB staging")?;
|
||||
l[..i].rsplit(|c: char| !(c.is_ascii_digit() || c == '.')).next()?.parse().ok()
|
||||
}).unwrap_or(1.1)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The game PC in M0 `gamemode_tune` (RTX 5070 Ti 16 GB): 2760 MiB held before the LLM there
|
||||
/// included the STT model (1300 MiB), which the app starts after the LLM. M0 ran no TTS.
|
||||
const TOTAL: i64 = 16303;
|
||||
const OTHERS: i64 = 2760 - 1300;
|
||||
fn probe32() -> Probe {
|
||||
Probe { context: 32768, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_normal_placement_is_the_one_m0_measured_with_room_for_the_stt() {
|
||||
// M0 got 1.5 GiB (the STT model counted as already held); the app counts it in the STT share.
|
||||
// M0 ran no TTS: the app keeps its share too, so the same 1.5 GiB needs that much less held.
|
||||
assert_eq!(normal(TOTAL, OTHERS - TTS_MIB, &probe32(), 1024), 1.5);
|
||||
// More room for other programs, fewer experts; no room at all, none.
|
||||
assert!(normal(TOTAL, OTHERS, &probe32(), 2048) < 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS + 8000, &probe32(), 1024), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_game_gets_its_reserve_or_what_the_smallest_placement_leaves_or_the_llm_pauses() {
|
||||
// 16K saves about 0.7 GB of conversation memory (M0).
|
||||
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..probe32() };
|
||||
// 8 GB does not fit on 16 GB: the smallest placement, and the game gets what is left.
|
||||
let g = game(TOTAL, OTHERS, &probe16, 8192);
|
||||
assert_eq!((g.expert_gib, g.paused), (MIN_GAME_EXPERT_GIB, false));
|
||||
let expected = TOTAL - OTHERS - probe16.model_mib - STT_MIB - TTS_MIB - MARGIN_MIB - ((2.0 - 1.1) * 1024.0) as i64;
|
||||
assert_eq!(g.game_room_mib, expected);
|
||||
assert!(g.game_room_mib > 0 && g.game_room_mib < 8192);
|
||||
// A bigger card keeps the whole reserve, with more experts than the minimum.
|
||||
let big = game(24 * 1024, OTHERS, &probe16, 8192);
|
||||
assert_eq!(big.game_room_mib, 8192);
|
||||
assert!(big.expert_gib > MIN_GAME_EXPERT_GIB && !big.paused);
|
||||
// No room beside 2 GiB but some beside none: the 0 GiB placement, before pausing.
|
||||
let tight = game(TOTAL, 1955, &Probe { model_mib: 9124, ..probe16.clone() }, 8192);
|
||||
assert_eq!((tight.expert_gib, tight.paused), (0.0, false));
|
||||
assert_eq!(tight.game_room_mib, TOTAL - 1955 - 9124 - STT_MIB - TTS_MIB - MARGIN_MIB);
|
||||
// Other programs already hold the card: no room for a game, the LLM pauses in one.
|
||||
let full = game(TOTAL, OTHERS + 4000, &probe16, 8192);
|
||||
assert!(full.paused);
|
||||
assert_eq!(full.game_room_mib, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_staging_comes_from_the_engine_log() {
|
||||
assert_eq!(staging_gib("strata-q36: hybrid experts: 5 of 256 per layer in VRAM (0.37 GiB), bigger prompt chunks streamed (1.10 GiB staging); 7.1 s\n"), 1.1);
|
||||
assert_eq!(staging_gib("x\nstrata-q36: budget 2.00 GiB (1.25 GiB staging)"), 1.25);
|
||||
assert_eq!(staging_gib("no such line"), 1.1);
|
||||
}
|
||||
}
|
||||
483
app/src/playback.rs
Normal file
483
app/src/playback.rs
Normal file
@@ -0,0 +1,483 @@
|
||||
//! The bot's mouth: turns text into queued speech, one request after another.
|
||||
//!
|
||||
//! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence.
|
||||
//! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still
|
||||
//! being made. Requests play first come, first served. `stop()` silences everything at once and
|
||||
//! cancels what the worker has not made yet.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
|
||||
use crate::worker_client::TtsDone;
|
||||
|
||||
pub trait TtsBackend {
|
||||
/// Send text for request `id`; false when the worker is not connected (the request then fails).
|
||||
fn tts(&self, id: &str, text: &str, more: bool) -> bool;
|
||||
fn tts_cancel(&self, id: &str);
|
||||
/// Request `id` ended after `played` of its sentences started playing, the last of them for
|
||||
/// `last_ms` before it was stopped (None: played to its end). The worker's recording keeps that
|
||||
/// much (voice_worker.spec.md, "Recordings").
|
||||
fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option<u32>);
|
||||
}
|
||||
|
||||
/// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes.
|
||||
pub trait ClipSink {
|
||||
fn play(&mut self, pcm: Vec<u8>);
|
||||
fn stop(&mut self);
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PlaybackEnd {
|
||||
pub id: String,
|
||||
/// Sentences that were played.
|
||||
pub played: u32,
|
||||
pub cancelled: bool,
|
||||
pub failed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PlaybackEvent {
|
||||
Start(String),
|
||||
End(PlaybackEnd),
|
||||
}
|
||||
|
||||
struct Request {
|
||||
id: String,
|
||||
chunks: BTreeMap<u32, Vec<u8>>,
|
||||
next: u32,
|
||||
done: bool,
|
||||
failed: bool,
|
||||
played: u32,
|
||||
started: bool,
|
||||
}
|
||||
|
||||
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
||||
backend: B,
|
||||
sink: S,
|
||||
queue: VecDeque<Request>,
|
||||
playing: bool,
|
||||
counter: u64,
|
||||
events: Vec<PlaybackEvent>,
|
||||
/// When the clip now playing started.
|
||||
clip_started: Option<std::time::Instant>,
|
||||
}
|
||||
|
||||
impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
pub fn new(backend: B, sink: S) -> Self {
|
||||
Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![], clip_started: None }
|
||||
}
|
||||
|
||||
/// True while a clip is playing or a request is waiting for its audio.
|
||||
pub fn busy(&self) -> bool {
|
||||
self.playing || !self.queue.is_empty()
|
||||
}
|
||||
|
||||
/// Start and end events since the last call.
|
||||
pub fn take_events(&mut self) -> Vec<PlaybackEvent> {
|
||||
std::mem::take(&mut self.events)
|
||||
}
|
||||
|
||||
/// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id.
|
||||
pub fn say(&mut self, text: &str, more: bool) -> String {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
||||
self.backend.tts(&id, text, more);
|
||||
id
|
||||
}
|
||||
|
||||
/// Queue a request whose speech the worker makes from the brain's answer (`reply` with `speak`),
|
||||
/// under the returned id. Nothing is sent here.
|
||||
pub fn adopt(&mut self) -> String {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
||||
id
|
||||
}
|
||||
|
||||
/// Queue a clip the app already has (the earcon) as a request of its own. Returns its id.
|
||||
pub fn clip(&mut self, pcm: Vec<u8>) -> String {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
||||
failed: false, played: 0, started: false });
|
||||
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
||||
self.pump();
|
||||
id
|
||||
}
|
||||
|
||||
/// End a request that will get no audio (the brain gave no answer).
|
||||
pub fn fail(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.done) {
|
||||
r.done = true;
|
||||
r.failed = true;
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
||||
/// queue goes on.
|
||||
pub fn cancel(&mut self, id: &str) {
|
||||
let Some(pos) = self.queue.iter().position(|r| r.id == id) else { return };
|
||||
let r = self.queue.remove(pos).expect("found");
|
||||
if !r.done {
|
||||
self.backend.tts_cancel(&r.id);
|
||||
}
|
||||
let mut last_ms = None;
|
||||
if pos == 0 && self.playing {
|
||||
self.playing = false;
|
||||
last_ms = self.clip_elapsed_ms();
|
||||
self.sink.stop();
|
||||
}
|
||||
crate::debug_log!("play", "cancel id={} played={}", r.id, r.played);
|
||||
self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: true, failed: false }, last_ms);
|
||||
self.pump();
|
||||
}
|
||||
|
||||
/// More text for an open request (an LLM answer that is still streaming).
|
||||
pub fn continue_text(&mut self, id: &str, text: &str, more: bool) {
|
||||
if self.queue.iter().any(|r| r.id == id && !r.done) {
|
||||
self.backend.tts(id, text, more);
|
||||
}
|
||||
}
|
||||
|
||||
/// A sentence of audio arrived from the worker.
|
||||
pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec<u8>) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) {
|
||||
r.chunks.insert(seq, pcm);
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// The worker finished a request (all sentences sent, cancelled, or failed).
|
||||
pub fn done(&mut self, d: &TtsDone) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) {
|
||||
r.done = true;
|
||||
r.failed = d.failed;
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// The sink finished playing the current clip.
|
||||
pub fn clip_ended(&mut self) {
|
||||
self.playing = false;
|
||||
self.pump();
|
||||
}
|
||||
|
||||
/// Silence now: stop the clip, drop every queued request, cancel the worker's work.
|
||||
pub fn stop(&mut self) -> usize {
|
||||
let stopped: Vec<Request> = self.queue.drain(..).collect();
|
||||
let was_playing = self.playing;
|
||||
let head_ms = if was_playing { self.clip_elapsed_ms() } else { None };
|
||||
self.playing = false;
|
||||
for r in stopped.iter().filter(|r| !r.done) {
|
||||
self.backend.tts_cancel(&r.id);
|
||||
}
|
||||
if was_playing {
|
||||
self.sink.stop();
|
||||
}
|
||||
for (i, r) in stopped.iter().enumerate() {
|
||||
let last_ms = if i == 0 { head_ms } else { None };
|
||||
self.ended(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true, failed: false }, last_ms);
|
||||
}
|
||||
crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len());
|
||||
stopped.len()
|
||||
}
|
||||
|
||||
/// A request ended: tell the worker how much of it was played, then the owner.
|
||||
fn ended(&mut self, end: PlaybackEnd, last_ms: Option<u32>) {
|
||||
self.backend.tts_played(&end.id, end.played, end.cancelled, last_ms);
|
||||
self.events.push(PlaybackEvent::End(end));
|
||||
}
|
||||
|
||||
/// How long the clip now playing has played.
|
||||
fn clip_elapsed_ms(&self) -> Option<u32> {
|
||||
self.clip_started.map(|t| t.elapsed().as_millis().min(u32::MAX as u128) as u32)
|
||||
}
|
||||
|
||||
fn pump(&mut self) {
|
||||
while !self.playing {
|
||||
let Some(head) = self.queue.front_mut() else { return };
|
||||
if !head.chunks.contains_key(&head.next) && head.done {
|
||||
// A sentence failed: skip its gap and play what did arrive.
|
||||
if let Some(&first) = head.chunks.keys().next() {
|
||||
head.next = first;
|
||||
}
|
||||
}
|
||||
if let Some(pcm) = head.chunks.remove(&head.next) {
|
||||
head.next += 1;
|
||||
head.played += 1;
|
||||
if !head.started {
|
||||
head.started = true;
|
||||
self.events.push(PlaybackEvent::Start(head.id.clone()));
|
||||
}
|
||||
self.playing = true;
|
||||
self.clip_started = Some(std::time::Instant::now());
|
||||
self.sink.play(pcm);
|
||||
return;
|
||||
}
|
||||
if !head.done {
|
||||
return;
|
||||
}
|
||||
let r = self.queue.pop_front().expect("head exists");
|
||||
crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed);
|
||||
self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: false, failed: r.failed }, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{cell::RefCell, rc::Rc};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Log {
|
||||
sent: Vec<(String, String, bool)>,
|
||||
cancelled: Vec<String>,
|
||||
played: Vec<String>,
|
||||
stops: usize,
|
||||
reported: Vec<(String, u32, bool, Option<u32>)>,
|
||||
}
|
||||
#[derive(Clone, Default)]
|
||||
struct Fake(Rc<RefCell<Log>>);
|
||||
impl TtsBackend for Fake {
|
||||
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
||||
self.0.borrow_mut().sent.push((id.into(), text.into(), more));
|
||||
true
|
||||
}
|
||||
fn tts_cancel(&self, id: &str) {
|
||||
self.0.borrow_mut().cancelled.push(id.into());
|
||||
}
|
||||
fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option<u32>) {
|
||||
self.0.borrow_mut().reported.push((id.into(), played, cancelled, last_ms));
|
||||
}
|
||||
}
|
||||
impl ClipSink for Fake {
|
||||
fn play(&mut self, pcm: Vec<u8>) {
|
||||
self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap());
|
||||
}
|
||||
fn stop(&mut self) {
|
||||
self.0.borrow_mut().stops += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn setup() -> (Playback<Fake, Fake>, Fake) {
|
||||
let f = Fake::default();
|
||||
(Playback::new(f.clone(), f.clone()), f)
|
||||
}
|
||||
fn done(id: &str, failed: bool) -> TtsDone {
|
||||
TtsDone { id: id.into(), chunks: 0, cancelled: false, failed }
|
||||
}
|
||||
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent {
|
||||
PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed })
|
||||
}
|
||||
fn b(s: &str) -> Vec<u8> {
|
||||
s.as_bytes().to_vec()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.say("하나. 둘.", false);
|
||||
assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]);
|
||||
p.audio(&id, 0, b("s0"));
|
||||
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
||||
p.audio(&id, 1, b("s1"));
|
||||
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
||||
p.done(&done(&id, false));
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
||||
p.clip_ended();
|
||||
assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]);
|
||||
assert!(!p.busy());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_worker_is_told_how_much_of_each_request_was_played() {
|
||||
let (mut p, f) = setup();
|
||||
let a = p.say("하나. 둘.", false);
|
||||
p.audio(&a, 0, b("a0"));
|
||||
p.audio(&a, 1, b("a1"));
|
||||
p.done(&done(&a, false));
|
||||
p.clip_ended();
|
||||
p.clip_ended();
|
||||
let c = p.say("셋. 넷.", false);
|
||||
p.audio(&c, 0, b("c0"));
|
||||
p.audio(&c, 1, b("c1"));
|
||||
p.done(&done(&c, false));
|
||||
std::thread::sleep(std::time::Duration::from_millis(60));
|
||||
p.cancel(&c); // stopped about 60 ms into its first sentence
|
||||
let e = p.say("여섯.", false);
|
||||
p.audio(&e, 0, b("e0"));
|
||||
let d = p.say("다섯.", false);
|
||||
p.stop(); // e stopped at once, d before any sound
|
||||
let r = f.0.borrow().reported.clone();
|
||||
assert_eq!(r[0], (a, 2, false, None), "played to its end");
|
||||
let (id, n, cancelled, ms) = r[1].clone();
|
||||
assert_eq!((id, n, cancelled), (c, 1, true));
|
||||
assert!(ms.is_some_and(|ms| (50..1000).contains(&ms)), "how long the stopped sentence played: {ms:?}");
|
||||
assert_eq!((r[2].0.clone(), r[2].1, r[2].2), (e, 1, true));
|
||||
assert!(r[2].3.is_some_and(|ms| ms < 50));
|
||||
assert_eq!(r[3], (d, 0, true, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requests_play_in_the_order_they_were_made() {
|
||||
let (mut p, f) = setup();
|
||||
let a = p.say("a", false);
|
||||
let c = p.say("b", false);
|
||||
p.audio(&c, 0, b("b0"));
|
||||
assert!(f.0.borrow().played.is_empty());
|
||||
p.audio(&a, 0, b("a0"));
|
||||
p.done(&done(&a, false));
|
||||
p.done(&done(&c, false));
|
||||
p.clip_ended();
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
||||
let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect();
|
||||
assert_eq!(ends, vec![a, c]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sentences_stay_in_order_when_they_arrive_out_of_order() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.say("x", true);
|
||||
p.audio(&id, 1, b("s1"));
|
||||
assert!(f.0.borrow().played.is_empty());
|
||||
p.audio(&id, 0, b("s0"));
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_sentence_is_skipped_and_the_rest_still_play() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.say("a. b. c.", false);
|
||||
p.audio(&id, 0, b("s0"));
|
||||
p.audio(&id, 2, b("s2"));
|
||||
p.done(&done(&id, true));
|
||||
p.clip_ended();
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["s0", "s2"]);
|
||||
assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_silences_cancels_unfinished_and_drops_the_queue() {
|
||||
let (mut p, f) = setup();
|
||||
let a = p.say("a", true);
|
||||
let c = p.say("b", false);
|
||||
p.audio(&a, 0, b("a0"));
|
||||
p.done(&done(&c, false));
|
||||
p.take_events();
|
||||
assert_eq!(p.stop(), 2);
|
||||
assert_eq!(f.0.borrow().stops, 1);
|
||||
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
||||
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]);
|
||||
p.audio(&a, 1, b("a1"));
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
||||
assert!(!p.busy());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_with_nothing_to_say_touches_nothing() {
|
||||
let (mut p, f) = setup();
|
||||
assert_eq!(p.stop(), 0);
|
||||
assert_eq!(f.0.borrow().stops, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_can_be_added_to_an_open_request_but_not_a_finished_one() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.say("첫 문장.", true);
|
||||
p.continue_text(&id, " 둘째 문장.", false);
|
||||
p.done(&done(&id, false));
|
||||
p.continue_text(&id, "늦은 글", false);
|
||||
let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect();
|
||||
assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_adopted_answer_plays_the_brains_audio_without_sending_text() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.adopt();
|
||||
assert!(f.0.borrow().sent.is_empty(), "the brain speaks it under this id");
|
||||
p.audio(&id, 0, b("a0"));
|
||||
p.done(&done(&id, false));
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone()), end(&id, 1, false, false)]);
|
||||
assert_ne!(p.adopt(), id, "ids never repeat");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_with_no_text_fails_and_the_next_request_plays() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.adopt();
|
||||
let c = p.say("b", false);
|
||||
p.audio(&c, 0, b("b0"));
|
||||
assert!(f.0.borrow().played.is_empty());
|
||||
p.fail(&id);
|
||||
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
||||
assert_eq!(p.take_events()[0], end(&id, 0, false, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_stops_one_request_and_leaves_the_others() {
|
||||
let (mut p, f) = setup();
|
||||
let a = p.say("a", true);
|
||||
let c = p.say("b", false);
|
||||
p.audio(&a, 0, b("a0"));
|
||||
p.audio(&c, 0, b("b0"));
|
||||
p.take_events();
|
||||
p.cancel(&a);
|
||||
assert_eq!(f.0.borrow().stops, 1);
|
||||
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0", "b0"], "the next request plays at once");
|
||||
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), PlaybackEvent::Start(c.clone())]);
|
||||
// A queued request that has not started is cancelled without touching the clip.
|
||||
let d = p.say("d", false);
|
||||
p.cancel(&d);
|
||||
assert_eq!(f.0.borrow().stops, 1);
|
||||
assert_eq!(p.take_events(), vec![end(&d, 0, true, false)]);
|
||||
p.cancel("nope");
|
||||
assert!(p.take_events().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() {
|
||||
let (mut p, f) = setup();
|
||||
let a = p.say("a", false);
|
||||
let c = p.say("b", false);
|
||||
p.done(&done(&a, true));
|
||||
p.audio(&c, 0, b("b0"));
|
||||
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
||||
assert_eq!(p.take_events()[0], end(&a, 0, false, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_clip_plays_at_once_then_the_answer_queued_after_it() {
|
||||
let (mut p, f) = setup();
|
||||
let cue = p.clip(b("ding"));
|
||||
let id = p.adopt();
|
||||
assert_eq!(f.0.borrow().played, vec!["ding"], "the clip needs no worker");
|
||||
assert!(f.0.borrow().sent.is_empty());
|
||||
p.audio(&id, 0, b("a0"));
|
||||
assert_eq!(f.0.borrow().played, vec!["ding"]);
|
||||
p.clip_ended();
|
||||
assert_eq!(f.0.borrow().played, vec!["ding", "a0"]);
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(cue.clone()), end(&cue, 1, false, false),
|
||||
PlaybackEvent::Start(id.clone())]);
|
||||
// A stop drops a clip still waiting its turn, without asking the worker to cancel it.
|
||||
let cue2 = p.clip(b("ding"));
|
||||
p.stop();
|
||||
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
||||
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
||||
}
|
||||
}
|
||||
800
app/src/policy.rs
Normal file
800
app/src/policy.rs
Normal file
@@ -0,0 +1,800 @@
|
||||
//! Response policy: which call Jarvis answers, in what order, and when an answer stops
|
||||
//! (`app/src/conversation.spec.md`).
|
||||
//!
|
||||
//! A state machine driven by events with the time passed in. It never plays or asks anything itself:
|
||||
//! every call returns the actions for the caller to carry out.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PolicyKind {
|
||||
/// 처음 대화만: new requests are dropped while an answer runs (default).
|
||||
First,
|
||||
/// 들어온 순서대로: answered one after another in end-time order.
|
||||
InOrder,
|
||||
/// 마지막 대화만: the newest request replaces everything else, even a running answer.
|
||||
Latest,
|
||||
/// 제일 중요한 대화만: candidates are collected and the best one is answered.
|
||||
Important,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PolicySettings {
|
||||
pub kind: PolicyKind,
|
||||
pub queue_max: usize,
|
||||
pub collect: Duration,
|
||||
pub settle: Duration,
|
||||
pub priorities: HashMap<u64, i64>,
|
||||
/// How long the brain's importance ranking is waited for.
|
||||
pub rank_timeout: Duration,
|
||||
/// `latest` and `important`: a new request may stop the running answer. Off: the answer is
|
||||
/// spoken to its end and the new request waits for it.
|
||||
pub interrupt: bool,
|
||||
}
|
||||
|
||||
impl Default for PolicySettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
kind: PolicyKind::Latest,
|
||||
queue_max: 5,
|
||||
collect: Duration::from_millis(700),
|
||||
settle: Duration::from_millis(800),
|
||||
priorities: HashMap::new(),
|
||||
rank_timeout: Duration::from_millis(1500),
|
||||
interrupt: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Request {
|
||||
pub id: u64,
|
||||
pub speaker: u64,
|
||||
pub text: String,
|
||||
/// Called by a wake word.
|
||||
pub direct: bool,
|
||||
/// When the utterance ended.
|
||||
pub ended_at: Instant,
|
||||
/// When its final arrived.
|
||||
pub arrived_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MissReason {
|
||||
/// `first`: an answer was running or another caller finished first.
|
||||
Busy,
|
||||
/// `inOrder`: the queue was full.
|
||||
QueueFull,
|
||||
/// `latest`: a newer request took its place.
|
||||
Replaced,
|
||||
/// The answer was stopped for a newer or more important request.
|
||||
Interrupted,
|
||||
/// `important`: another candidate was chosen.
|
||||
LowerScore,
|
||||
/// A stop command.
|
||||
Stopped,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Action {
|
||||
/// Start answering this request.
|
||||
Answer(Request),
|
||||
/// Stop the running answer with this id (its speech and its reply).
|
||||
StopAnswer(u64),
|
||||
/// Not answered, or stopped: for the "놓친 요청" list.
|
||||
Missed(Request, MissReason),
|
||||
/// Ask the brain to rank these tied candidates by importance; answer with `ranked(id, ..)`. A newer
|
||||
/// `Rank` replaces an older one, whose answer is then ignored.
|
||||
Rank { id: u64, candidates: Vec<Request> },
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Open {
|
||||
/// When the speaker stopped sending audio, if they have.
|
||||
stopped_at: Option<Instant>,
|
||||
}
|
||||
|
||||
pub struct Policy {
|
||||
s: PolicySettings,
|
||||
current: Option<Request>,
|
||||
waiting: Vec<Request>,
|
||||
open: HashMap<u64, Open>,
|
||||
/// `important`: no choice before this.
|
||||
collect_until: Option<Instant>,
|
||||
/// `important`: a ranking the brain was asked for.
|
||||
ranking: Option<Ranking>,
|
||||
next_rank_id: u64,
|
||||
}
|
||||
|
||||
struct Ranking {
|
||||
rank_id: u64,
|
||||
ids: HashSet<u64>,
|
||||
deadline: Instant,
|
||||
/// Asked about interrupting the running answer with this id (it is one of `ids`).
|
||||
against: Option<u64>,
|
||||
}
|
||||
|
||||
impl Policy {
|
||||
pub fn new(s: PolicySettings) -> Self {
|
||||
Self { s, current: None, waiting: vec![], open: HashMap::new(), collect_until: None, ranking: None,
|
||||
next_rank_id: 0 }
|
||||
}
|
||||
|
||||
pub fn settings(&self) -> &PolicySettings {
|
||||
&self.s
|
||||
}
|
||||
|
||||
/// The id of the running answer.
|
||||
pub fn current(&self) -> Option<u64> {
|
||||
self.current.as_ref().map(|r| r.id)
|
||||
}
|
||||
|
||||
pub fn waiting(&self) -> usize {
|
||||
self.waiting.len()
|
||||
}
|
||||
|
||||
/// An utterance started (`speech_start`).
|
||||
pub fn utterance_open(&mut self, speaker: u64) {
|
||||
self.open.insert(speaker, Open::default());
|
||||
}
|
||||
|
||||
/// The speaker stopped sending audio; their final is still to come.
|
||||
pub fn speaking_stopped(&mut self, speaker: u64, at: Instant) {
|
||||
if let Some(o) = self.open.get_mut(&speaker) {
|
||||
o.stopped_at.get_or_insert(at);
|
||||
}
|
||||
}
|
||||
|
||||
/// The speaker's final arrived and was not a call (or the speaker left).
|
||||
pub fn utterance_closed(&mut self, speaker: u64, now: Instant) -> Vec<Action> {
|
||||
self.open.remove(&speaker);
|
||||
self.choose(now, None)
|
||||
}
|
||||
|
||||
fn score(&self, r: &Request) -> (i64, bool) {
|
||||
(self.s.priorities.get(&r.speaker).copied().unwrap_or(0), r.direct)
|
||||
}
|
||||
|
||||
/// The speaker's final was a call.
|
||||
pub fn request(&mut self, req: Request, now: Instant) -> Vec<Action> {
|
||||
self.open.remove(&req.speaker);
|
||||
let mut out = vec![];
|
||||
if let Some(cur) = self.current.clone() {
|
||||
match self.s.kind {
|
||||
PolicyKind::First => out.push(Action::Missed(req, MissReason::Busy)),
|
||||
PolicyKind::InOrder => self.enqueue(req, &mut out),
|
||||
PolicyKind::Latest => {
|
||||
// Newer than the running answer and than every waiting request (only possible
|
||||
// without interrupting: otherwise the newest is already the running one).
|
||||
let newest = self.waiting.iter().map(|r| r.ended_at).max().map_or(cur.ended_at, |w| w.max(cur.ended_at));
|
||||
if req.ended_at > newest {
|
||||
if self.s.interrupt {
|
||||
self.interrupt(&mut out);
|
||||
}
|
||||
for r in self.waiting.drain(..) {
|
||||
out.push(Action::Missed(r, MissReason::Replaced));
|
||||
}
|
||||
self.waiting.push(req);
|
||||
} else {
|
||||
out.push(Action::Missed(req, MissReason::Replaced));
|
||||
}
|
||||
}
|
||||
PolicyKind::Important => {
|
||||
let (new, running) = (self.score(&req), self.score(&cur));
|
||||
self.enqueue(req, &mut out);
|
||||
if !self.s.interrupt {
|
||||
// Kept as a candidate; the best one is answered when the running answer ends.
|
||||
} else if new > running {
|
||||
self.interrupt(&mut out);
|
||||
self.collect_until = None;
|
||||
self.ranking = None;
|
||||
} else if new == running {
|
||||
// A tie on priority and wake word: the brain's importance decides, over the
|
||||
// running answer and every candidate tied with it (a pending ranking is replaced,
|
||||
// keeping its deadline).
|
||||
let mut cands = vec![cur.clone()];
|
||||
cands.extend(self.waiting.iter().filter(|r| self.score(r) == running).cloned());
|
||||
let deadline = self.ranking.as_ref().map_or(now + self.s.rank_timeout, |r| r.deadline);
|
||||
out.push(self.ask_rank(cands, deadline, Some(cur.id)));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if self.s.kind == PolicyKind::Important && self.waiting.is_empty() && self.ranking.is_none() {
|
||||
self.collect_until = Some(now + self.s.collect);
|
||||
}
|
||||
let score = self.score(&req);
|
||||
self.enqueue(req, &mut out);
|
||||
// A candidate arriving while tied candidates are being ranked: a tie joins a new ranking
|
||||
// (same deadline); a better one makes the ranking moot.
|
||||
if let Some(r) = &self.ranking {
|
||||
let ranked_score = self.waiting.iter().find(|q| r.ids.contains(&q.id)).map(|q| self.score(q));
|
||||
let deadline = r.deadline;
|
||||
match ranked_score {
|
||||
Some(t) if score == t => {
|
||||
let cands: Vec<Request> = self.waiting.iter().filter(|q| self.score(q) == t).cloned().collect();
|
||||
out.push(self.ask_rank(cands, deadline, None));
|
||||
}
|
||||
Some(t) if score > t => self.ranking = None,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.extend(self.choose(now, None));
|
||||
out
|
||||
}
|
||||
|
||||
fn enqueue(&mut self, req: Request, out: &mut Vec<Action>) {
|
||||
let at = self.waiting.partition_point(|r| r.ended_at <= req.ended_at);
|
||||
self.waiting.insert(at, req);
|
||||
if self.s.kind == PolicyKind::InOrder {
|
||||
while self.waiting.len() > self.s.queue_max.max(1) {
|
||||
out.push(Action::Missed(self.waiting.remove(0), MissReason::QueueFull));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn ask_rank(&mut self, candidates: Vec<Request>, deadline: Instant, against: Option<u64>) -> Action {
|
||||
self.next_rank_id += 1;
|
||||
self.ranking = Some(Ranking {
|
||||
rank_id: self.next_rank_id,
|
||||
ids: candidates.iter().map(|r| r.id).collect(),
|
||||
deadline,
|
||||
against,
|
||||
});
|
||||
Action::Rank { id: self.next_rank_id, candidates }
|
||||
}
|
||||
|
||||
fn interrupt(&mut self, out: &mut Vec<Action>) {
|
||||
if let Some(cur) = self.current.take() {
|
||||
out.push(Action::StopAnswer(cur.id));
|
||||
out.push(Action::Missed(cur, MissReason::Interrupted));
|
||||
}
|
||||
}
|
||||
|
||||
/// A stop command: stop the answer and drop everything waiting.
|
||||
pub fn stop(&mut self) -> Vec<Action> {
|
||||
let mut out = vec![];
|
||||
if let Some(cur) = self.current.take() {
|
||||
out.push(Action::StopAnswer(cur.id));
|
||||
out.push(Action::Missed(cur, MissReason::Stopped));
|
||||
}
|
||||
for r in self.waiting.drain(..) {
|
||||
out.push(Action::Missed(r, MissReason::Stopped));
|
||||
}
|
||||
self.collect_until = None;
|
||||
self.ranking = None;
|
||||
out
|
||||
}
|
||||
|
||||
/// The answer's speech ended, failed or was stopped.
|
||||
pub fn answer_done(&mut self, id: u64, now: Instant) -> Vec<Action> {
|
||||
if self.current.as_ref().map(|r| r.id) != Some(id) {
|
||||
return vec![];
|
||||
}
|
||||
self.current = None;
|
||||
// An interruption ranking is moot once the answer it was about has ended.
|
||||
if self.ranking.as_ref().is_some_and(|r| r.against.is_some()) {
|
||||
self.ranking = None;
|
||||
}
|
||||
self.choose(now, None)
|
||||
}
|
||||
|
||||
/// The brain's answer to ranking `rank_id`, best first. An answer to any other ranking (replaced or
|
||||
/// dropped) is ignored; one after the deadline counts as a timeout.
|
||||
pub fn ranked(&mut self, rank_id: u64, order: &[u64], now: Instant) -> Vec<Action> {
|
||||
if self.ranking.as_ref().map(|r| r.rank_id) != Some(rank_id) {
|
||||
return vec![];
|
||||
}
|
||||
let r = self.ranking.take().expect("checked");
|
||||
let winner = if now >= r.deadline { None } else { order.iter().copied().find(|id| r.ids.contains(id)) };
|
||||
self.resolve(r, winner, now)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, r: Ranking, winner: Option<u64>, now: Instant) -> Vec<Action> {
|
||||
let Some(running) = r.against else { return self.choose(now, Some(winner)) };
|
||||
// Interrupt only when the ranking put a waiting request above the running answer; a timeout
|
||||
// keeps the running answer, which ended first.
|
||||
let mut out = vec![];
|
||||
match winner {
|
||||
Some(w) if w != running && self.current() == Some(running) && self.waiting.iter().any(|q| q.id == w) => {
|
||||
self.interrupt(&mut out);
|
||||
self.collect_until = None;
|
||||
out.extend(self.choose(now, Some(Some(w))));
|
||||
}
|
||||
_ => out.extend(self.choose(now, None)),
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Timers: settling, collecting and the ranking timeout.
|
||||
pub fn tick(&mut self, now: Instant) -> Vec<Action> {
|
||||
if self.ranking.as_ref().is_some_and(|r| now >= r.deadline) {
|
||||
let r = self.ranking.take().expect("checked");
|
||||
return self.resolve(r, None, now);
|
||||
}
|
||||
self.choose(now, None)
|
||||
}
|
||||
|
||||
/// When `tick` should run next.
|
||||
pub fn next_deadline(&self) -> Option<Instant> {
|
||||
if let Some(r) = &self.ranking {
|
||||
return Some(r.deadline);
|
||||
}
|
||||
if self.current.is_some() || self.waiting.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// Nothing can start before the collect timer ends.
|
||||
if let Some(t) = self.collect_until {
|
||||
return Some(t);
|
||||
}
|
||||
// The settle deadline matters only while an ending utterance holds the choice up.
|
||||
if self.open.values().any(|o| o.stopped_at.is_some()) {
|
||||
return self.waiting.iter().map(|r| r.arrived_at).min().map(|t| t + self.s.settle);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Wait for an ending utterance that may still be a call that should come first.
|
||||
fn blocked(&self, now: Instant) -> bool {
|
||||
let ending: Vec<Instant> = self.open.values().filter_map(|o| o.stopped_at).collect();
|
||||
if ending.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Some(first_arrival) = self.waiting.iter().map(|r| r.arrived_at).min() else { return false };
|
||||
if now >= first_arrival + self.s.settle {
|
||||
return false;
|
||||
}
|
||||
match self.s.kind {
|
||||
PolicyKind::Latest => false,
|
||||
PolicyKind::First | PolicyKind::InOrder => {
|
||||
let head = self.waiting[0].ended_at;
|
||||
ending.iter().any(|t| *t < head)
|
||||
}
|
||||
PolicyKind::Important => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the next answer if the policy allows one now. `rank`: `Some(winner)` once a ranking
|
||||
/// finished or gave up.
|
||||
fn choose(&mut self, now: Instant, rank: Option<Option<u64>>) -> Vec<Action> {
|
||||
let mut out = vec![];
|
||||
if self.current.is_some() || self.waiting.is_empty() || (self.ranking.is_some() && rank.is_none()) {
|
||||
return out;
|
||||
}
|
||||
if rank.is_none() {
|
||||
if let Some(t) = self.collect_until {
|
||||
if now < t {
|
||||
return out;
|
||||
}
|
||||
self.collect_until = None;
|
||||
}
|
||||
if self.blocked(now) {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
let winner = match self.s.kind {
|
||||
PolicyKind::First | PolicyKind::InOrder => 0,
|
||||
PolicyKind::Latest => self.waiting.len() - 1,
|
||||
PolicyKind::Important => {
|
||||
let best = self.waiting.iter().map(|r| self.score(r)).max().expect("not empty");
|
||||
let tied: Vec<usize> = (0..self.waiting.len()).filter(|&i| self.score(&self.waiting[i]) == best).collect();
|
||||
match rank {
|
||||
Some(Some(id)) if tied.iter().any(|&i| self.waiting[i].id == id) => {
|
||||
tied.into_iter().find(|&i| self.waiting[i].id == id).expect("present")
|
||||
}
|
||||
// A ranking that failed, timed out, or named nobody still tied: the earliest-ended.
|
||||
Some(_) => tied[0],
|
||||
None if tied.len() >= 2 => {
|
||||
let cands: Vec<Request> = tied.iter().map(|&i| self.waiting[i].clone()).collect();
|
||||
let deadline = now + self.s.rank_timeout;
|
||||
out.push(self.ask_rank(cands, deadline, None));
|
||||
return out;
|
||||
}
|
||||
None => tied[0],
|
||||
}
|
||||
}
|
||||
};
|
||||
let chosen = self.waiting.remove(winner);
|
||||
let others = match self.s.kind {
|
||||
PolicyKind::InOrder => vec![],
|
||||
PolicyKind::First => std::mem::take(&mut self.waiting).into_iter().map(|r| (r, MissReason::Busy)).collect(),
|
||||
PolicyKind::Latest => std::mem::take(&mut self.waiting).into_iter().map(|r| (r, MissReason::Replaced)).collect(),
|
||||
PolicyKind::Important => {
|
||||
std::mem::take(&mut self.waiting).into_iter().map(|r| (r, MissReason::LowerScore)).collect()
|
||||
}
|
||||
};
|
||||
for (r, why) in others {
|
||||
out.push(Action::Missed(r, why));
|
||||
}
|
||||
self.collect_until = None;
|
||||
self.current = Some(chosen.clone());
|
||||
out.push(Action::Answer(chosen));
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const A: u64 = 1;
|
||||
const B: u64 = 2;
|
||||
const C: u64 = 3;
|
||||
|
||||
struct T {
|
||||
p: Policy,
|
||||
t0: Instant,
|
||||
next_id: u64,
|
||||
}
|
||||
|
||||
impl T {
|
||||
fn new(kind: PolicyKind) -> Self {
|
||||
Self::with(PolicySettings { kind, ..PolicySettings::default() })
|
||||
}
|
||||
fn with(s: PolicySettings) -> Self {
|
||||
Self { p: Policy::new(s), t0: Instant::now(), next_id: 0 }
|
||||
}
|
||||
fn at(&self, ms: u64) -> Instant {
|
||||
self.t0 + Duration::from_millis(ms)
|
||||
}
|
||||
/// A call from `speaker` that ended at `ended` ms and arrived at `arrived` ms.
|
||||
fn call(&mut self, speaker: u64, direct: bool, ended: u64, arrived: u64) -> (u64, Vec<Action>) {
|
||||
self.next_id += 1;
|
||||
let r = Request { id: self.next_id, speaker, text: String::new(), direct, ended_at: self.at(ended),
|
||||
arrived_at: self.at(arrived) };
|
||||
let now = self.at(arrived);
|
||||
(self.next_id, self.p.request(r, now))
|
||||
}
|
||||
}
|
||||
|
||||
fn answered(a: &[Action]) -> Vec<u64> {
|
||||
a.iter().filter_map(|x| if let Action::Answer(r) = x { Some(r.id) } else { None }).collect()
|
||||
}
|
||||
fn missed(a: &[Action]) -> Vec<(u64, MissReason)> {
|
||||
a.iter().filter_map(|x| if let Action::Missed(r, w) = x { Some((r.id, *w)) } else { None }).collect()
|
||||
}
|
||||
/// Every ranking asked for: (ranking id, candidate ids).
|
||||
fn ranks(a: &[Action]) -> Vec<(u64, Vec<u64>)> {
|
||||
a.iter().filter_map(|x| if let Action::Rank { id, candidates } = x {
|
||||
Some((*id, candidates.iter().map(|r| r.id).collect())) } else { None }).collect()
|
||||
}
|
||||
fn rank_of(a: &[Action]) -> (u64, Vec<u64>) {
|
||||
let r = ranks(a);
|
||||
assert_eq!(r.len(), 1, "one ranking asked for: {a:?}");
|
||||
r[0].clone()
|
||||
}
|
||||
fn stopped(a: &[Action]) -> Vec<u64> {
|
||||
a.iter().filter_map(|x| if let Action::StopAnswer(id) = x { Some(*id) } else { None }).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_answers_at_once_and_drops_what_comes_while_busy() {
|
||||
let mut t = T::new(PolicyKind::First);
|
||||
let (a, out) = t.call(A, true, 0, 200);
|
||||
assert_eq!(answered(&out), vec![a]);
|
||||
let (b, out) = t.call(B, true, 300, 500);
|
||||
assert_eq!(missed(&out), vec![(b, MissReason::Busy)]);
|
||||
assert!(t.p.answer_done(a, t.at(2000)).is_empty());
|
||||
let (c, out) = t.call(C, true, 2100, 2300);
|
||||
assert_eq!(answered(&out), vec![c]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_waits_for_an_earlier_ended_utterance_and_answers_it_if_it_is_a_call() {
|
||||
let mut t = T::new(PolicyKind::First);
|
||||
// B stopped talking at 100 ms but is still being transcribed; A ended at 150 ms.
|
||||
t.p.utterance_open(B);
|
||||
t.p.speaking_stopped(B, t.at(100));
|
||||
let (a, out) = t.call(A, true, 150, 300);
|
||||
assert!(answered(&out).is_empty(), "waits for B");
|
||||
let (b, out) = t.call(B, true, 100, 500);
|
||||
assert_eq!(answered(&out), vec![b], "B finished speaking first");
|
||||
assert_eq!(missed(&out), vec![(a, MissReason::Busy)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settling_ends_when_the_earlier_utterance_is_not_a_call_or_takes_too_long() {
|
||||
let mut t = T::new(PolicyKind::First);
|
||||
t.p.utterance_open(B);
|
||||
t.p.speaking_stopped(B, t.at(100));
|
||||
let (a, _) = t.call(A, true, 150, 300);
|
||||
assert_eq!(answered(&t.p.utterance_closed(B, t.at(400))), vec![a]);
|
||||
|
||||
let mut t = T::new(PolicyKind::First);
|
||||
t.p.utterance_open(B);
|
||||
t.p.speaking_stopped(B, t.at(100));
|
||||
let (a, _) = t.call(A, true, 150, 300);
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(300 + 800)));
|
||||
assert!(answered(&t.p.tick(t.at(1000))).is_empty());
|
||||
assert_eq!(answered(&t.p.tick(t.at(1100))), vec![a]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_utterance_still_being_spoken_or_ended_later_does_not_hold_up_the_answer() {
|
||||
let mut t = T::new(PolicyKind::First);
|
||||
t.p.utterance_open(B); // still talking
|
||||
t.p.utterance_open(C);
|
||||
t.p.speaking_stopped(C, t.at(400)); // ended after A
|
||||
let (a, out) = t.call(A, true, 150, 300);
|
||||
assert_eq!(answered(&out), vec![a]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_order_follows_end_times_not_arrival_and_drops_the_oldest_when_full() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::InOrder, queue_max: 2, ..PolicySettings::default() });
|
||||
let (a, out) = t.call(A, true, 0, 100);
|
||||
assert_eq!(answered(&out), vec![a]);
|
||||
let (c, _) = t.call(C, true, 900, 1000); // ended later, arrived first
|
||||
let (b, _) = t.call(B, true, 600, 1500); // long utterance: ended earlier, arrived later
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![b]);
|
||||
assert_eq!(answered(&t.p.answer_done(b, t.at(3000))), vec![c]);
|
||||
|
||||
let (_, out) = t.call(A, true, 3100, 3200);
|
||||
assert!(answered(&out).is_empty());
|
||||
let (e, _) = t.call(B, true, 3300, 3400);
|
||||
let (_f, out) = t.call(C, true, 3500, 3600);
|
||||
assert_eq!(t.p.waiting(), 2);
|
||||
assert_eq!(missed(&out).len(), 1, "third waiting request pushes the oldest out");
|
||||
assert_eq!(answered(&t.p.answer_done(c, t.at(4000))), vec![e]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latest_interrupts_for_a_newer_request_and_drops_older_ones() {
|
||||
let mut t = T::new(PolicyKind::Latest);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 500, 700);
|
||||
assert_eq!(stopped(&out), vec![a]);
|
||||
assert_eq!(missed(&out), vec![(a, MissReason::Interrupted)]);
|
||||
assert_eq!(answered(&out), vec![b]);
|
||||
// A late final from before B's end is not newer.
|
||||
let (c, out) = t.call(C, true, 400, 900);
|
||||
assert_eq!(missed(&out), vec![(c, MissReason::Replaced)]);
|
||||
assert!(stopped(&out).is_empty());
|
||||
// The stopped answer's late "done" changes nothing.
|
||||
assert!(t.p.answer_done(a, t.at(1000)).is_empty());
|
||||
assert_eq!(t.p.current(), Some(b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_interrupting_latest_finishes_the_answer_then_answers_the_newest_waiting() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::Latest, interrupt: false, ..PolicySettings::default() });
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 500, 700);
|
||||
assert!(stopped(&out).is_empty(), "the answer keeps playing");
|
||||
assert!(answered(&out).is_empty());
|
||||
let (c, out) = t.call(C, true, 800, 900);
|
||||
assert_eq!(missed(&out), vec![(b, MissReason::Replaced)], "a newer waiting request replaces it");
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![c]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_interrupting_latest_keeps_the_newest_waiting_request_whatever_order_finals_arrive_in() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::Latest, interrupt: false, ..PolicySettings::default() });
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, _) = t.call(B, true, 700, 800);
|
||||
// C ended before B but its final came later: B stays, C is replaced.
|
||||
let (c, out) = t.call(C, true, 500, 900);
|
||||
assert_eq!(missed(&out), vec![(c, MissReason::Replaced)]);
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![b]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_interrupting_important_waits_for_the_answer_even_for_a_better_request() {
|
||||
let s = PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO, interrupt: false,
|
||||
priorities: HashMap::from([(B, 5)]), ..PolicySettings::default() };
|
||||
let mut t = T::with(s);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 500, 700);
|
||||
assert!(stopped(&out).is_empty() && ranks(&out).is_empty(), "no stop and no ranking against the answer");
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![b]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latest_answers_the_newest_of_several_waiting() {
|
||||
let mut t = T::new(PolicyKind::Latest);
|
||||
t.p.utterance_open(C);
|
||||
let (a, out) = t.call(A, true, 0, 100);
|
||||
assert_eq!(answered(&out), vec![a], "latest never settles");
|
||||
let (b, _) = t.call(B, true, 200, 300);
|
||||
assert_eq!(t.p.current(), Some(b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn important_collects_then_answers_the_best_by_priority_then_wake_word() {
|
||||
let mut s = PolicySettings { kind: PolicyKind::Important, ..PolicySettings::default() };
|
||||
s.priorities.insert(C, 5);
|
||||
let mut t = T::with(s);
|
||||
let (a, out) = t.call(A, true, 0, 100);
|
||||
assert!(answered(&out).is_empty(), "collecting for 700 ms");
|
||||
let (b, _) = t.call(B, false, 100, 200);
|
||||
let (c, _) = t.call(C, false, 200, 300);
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(800)));
|
||||
let out = t.p.tick(t.at(800));
|
||||
assert_eq!(answered(&out), vec![c], "priority wins over the wake word");
|
||||
let mut m = missed(&out);
|
||||
m.sort_by_key(|(id, _)| *id);
|
||||
assert_eq!(m, vec![(a, MissReason::LowerScore), (b, MissReason::LowerScore)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn important_asks_for_a_ranking_only_on_a_tie_and_falls_back_on_timeout() {
|
||||
let mut t = T::new(PolicyKind::Important);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, _) = t.call(B, true, 50, 200);
|
||||
let out = t.p.tick(t.at(800));
|
||||
let (r, ids) = rank_of(&out);
|
||||
assert_eq!(ids, vec![a, b]);
|
||||
assert!(answered(&out).is_empty());
|
||||
assert_eq!(answered(&t.p.ranked(r, &[b, a], t.at(900))), vec![b]);
|
||||
|
||||
let mut t = T::new(PolicyKind::Important);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, _) = t.call(B, true, 50, 200);
|
||||
let (r, _) = rank_of(&t.p.tick(t.at(800)));
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(800 + 1500)));
|
||||
assert_eq!(answered(&t.p.tick(t.at(2300))), vec![a], "timeout: earliest-ended wins");
|
||||
assert!(t.p.ranked(r, &[b], t.at(2400)).is_empty(), "a late ranking is ignored");
|
||||
|
||||
// One candidate: no ranking, no extra delay beyond collecting.
|
||||
let mut t = T::new(PolicyKind::Important);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
assert_eq!(answered(&t.p.tick(t.at(800))), vec![a]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn important_interrupts_only_for_a_higher_score_and_otherwise_answers_after() {
|
||||
let mut s = PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
s.priorities.insert(C, 1);
|
||||
let mut t = T::with(s);
|
||||
let (a, out) = t.call(A, false, 0, 100);
|
||||
assert_eq!(answered(&out), vec![a]);
|
||||
let (b, out) = t.call(B, false, 200, 300);
|
||||
assert!(stopped(&out).is_empty(), "same score: keep going");
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(1000))), vec![b]);
|
||||
let (c, out) = t.call(C, false, 1100, 1200);
|
||||
assert_eq!(stopped(&out), vec![b]);
|
||||
assert_eq!(missed(&out), vec![(b, MissReason::Interrupted)]);
|
||||
assert_eq!(answered(&out), vec![c]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn important_asks_the_ranking_when_a_new_request_ties_the_running_answer() {
|
||||
let s = PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
// The ranking says the new one matters more: the answer stops for it.
|
||||
let mut t = T::with(s.clone());
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 200, 300);
|
||||
let (r, ids) = rank_of(&out);
|
||||
assert_eq!(ids, vec![a, b]);
|
||||
assert!(stopped(&out).is_empty(), "keeps speaking while ranking");
|
||||
let out = t.p.ranked(r, &[b, a], t.at(400));
|
||||
assert_eq!(stopped(&out), vec![a]);
|
||||
assert_eq!(missed(&out), vec![(a, MissReason::Interrupted)]);
|
||||
assert_eq!(answered(&out), vec![b]);
|
||||
// The ranking keeps the running answer: the new one waits for it.
|
||||
let mut t = T::with(s.clone());
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 200, 300);
|
||||
let (r, _) = rank_of(&out);
|
||||
assert!(stopped(&t.p.ranked(r, &[a, b], t.at(400))).is_empty());
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(1000))), vec![b]);
|
||||
// No ranking in time: the running answer ended first, so it goes on.
|
||||
let mut t = T::with(s);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, _) = t.call(B, true, 200, 300);
|
||||
assert!(stopped(&t.p.tick(t.at(300 + 1500))).is_empty());
|
||||
assert_eq!(answered(&t.p.answer_done(a, t.at(2000))), vec![b]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_ranking_after_its_deadline_counts_as_a_timeout() {
|
||||
let mut t = T::new(PolicyKind::Important);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, _) = t.call(B, true, 50, 200);
|
||||
let (r, _) = rank_of(&t.p.tick(t.at(800)));
|
||||
assert_eq!(answered(&t.p.ranked(r, &[b, a], t.at(2400))), vec![a], "late: earliest-ended wins");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn candidates_collected_while_busy_keep_end_time_order_for_the_fallback() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO,
|
||||
..PolicySettings::default() });
|
||||
let (a, _) = t.call(A, false, 0, 100);
|
||||
let (_late, _) = t.call(B, false, 900, 1000);
|
||||
let (early, _) = t.call(C, false, 600, 1200); // ended first, arrived last
|
||||
let out = t.p.answer_done(a, t.at(1300));
|
||||
assert_eq!(rank_of(&out).1, vec![early, _late]);
|
||||
assert_eq!(answered(&t.p.tick(t.at(1300 + 1500))), vec![early]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_finished_collect_timer_is_not_reported_again_while_settling() {
|
||||
let mut t = T::new(PolicyKind::Important);
|
||||
t.p.utterance_open(B);
|
||||
t.p.speaking_stopped(B, t.at(50));
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(800)));
|
||||
assert!(answered(&t.p.tick(t.at(800))).is_empty(), "still waiting for B");
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(100 + 800)), "the settle deadline, not the spent collect timer");
|
||||
assert_eq!(answered(&t.p.tick(t.at(900))), vec![a]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tie_arriving_during_a_ranking_against_the_answer_is_ranked_too() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO,
|
||||
..PolicySettings::default() });
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 200, 300);
|
||||
let (r1, _) = rank_of(&out);
|
||||
let (c, out) = t.call(C, true, 400, 500);
|
||||
let (r2, ids) = rank_of(&out);
|
||||
assert_ne!(r1, r2);
|
||||
assert_eq!(ids, vec![a, b, c], "the running answer and every tied candidate");
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(300 + 1500)), "the first ranking's deadline is kept");
|
||||
assert!(t.p.ranked(r1, &[b, a], t.at(600)).is_empty(), "the replaced ranking's result is ignored");
|
||||
let out = t.p.ranked(r2, &[c, b, a], t.at(700));
|
||||
assert_eq!(stopped(&out), vec![a]);
|
||||
assert_eq!(answered(&out), vec![c]);
|
||||
assert_eq!(missed(&out), vec![(a, MissReason::Interrupted), (b, MissReason::LowerScore)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tie_arriving_during_a_ranking_while_idle_is_ranked_too() {
|
||||
let mut t = T::new(PolicyKind::Important);
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, _) = t.call(B, true, 50, 200);
|
||||
let (r1, _) = rank_of(&t.p.tick(t.at(800)));
|
||||
let (c, out) = t.call(C, true, 850, 900);
|
||||
let (r2, ids) = rank_of(&out);
|
||||
assert_eq!(ids, vec![a, b, c]);
|
||||
assert!(t.p.ranked(r1, &[b], t.at(950)).is_empty());
|
||||
let out = t.p.ranked(r2, &[c], t.at(1000));
|
||||
assert_eq!(answered(&out), vec![c]);
|
||||
assert_eq!(missed(&out), vec![(a, MissReason::LowerScore), (b, MissReason::LowerScore)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dropped_rankings_late_result_does_not_decide_the_next_one() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::Important, collect: Duration::ZERO,
|
||||
..PolicySettings::default() });
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
let (b, out) = t.call(B, true, 200, 300);
|
||||
let (r1, _) = rank_of(&out);
|
||||
let (_c, _) = t.call(C, true, 250, 350);
|
||||
// A ends: the ranking about interrupting it is dropped, and B and C are ranked anew.
|
||||
let (r2, ids) = rank_of(&t.p.answer_done(a, t.at(400)));
|
||||
assert_eq!(ids, vec![b, _c]);
|
||||
assert!(t.p.ranked(r1, &[b, a], t.at(450)).is_empty(), "the dropped ranking's result is ignored");
|
||||
assert_eq!(answered(&t.p.ranked(r2, &[_c, b], t.at(500))), vec![_c]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_no_settle_time_the_next_deadline_is_the_collect_timer() {
|
||||
let mut t = T::with(PolicySettings { kind: PolicyKind::Important, settle: Duration::ZERO,
|
||||
..PolicySettings::default() });
|
||||
let (a, _) = t.call(A, true, 0, 100);
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(800)));
|
||||
assert!(t.p.tick(t.at(100)).is_empty());
|
||||
assert_eq!(t.p.next_deadline(), Some(t.at(800)), "moves forward, not stuck at a passed time");
|
||||
assert_eq!(answered(&t.p.tick(t.at(800))), vec![a]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_ends_the_answer_and_drops_the_queue_in_every_policy() {
|
||||
for kind in [PolicyKind::First, PolicyKind::InOrder, PolicyKind::Latest, PolicyKind::Important] {
|
||||
let mut t = T::with(PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() });
|
||||
let _ = t.call(A, true, 0, 100);
|
||||
let _ = t.call(B, true, 50, 200);
|
||||
let running = t.p.current().expect("an answer runs");
|
||||
let out = t.p.stop();
|
||||
assert_eq!(stopped(&out), vec![running], "{kind:?}");
|
||||
assert!(missed(&out).iter().any(|(id, _)| *id == running), "{kind:?}");
|
||||
assert!(missed(&out).iter().all(|(_, w)| *w == MissReason::Stopped), "{kind:?}");
|
||||
assert_eq!(t.p.current(), None);
|
||||
assert_eq!(t.p.waiting(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
271
app/src/receive.rs
Normal file
271
app/src/receive.rs
Normal file
@@ -0,0 +1,271 @@
|
||||
//! Per-speaker receive: RTP packets in arrival order -> Opus frames in sequence order -> PCM.
|
||||
//!
|
||||
//! songbird 0.6.1 decrypts (transport and DAVE) but feeds its own Opus decoder the payload without
|
||||
//! trimming what DAVE decryption left at the end, which garbles or drops about a third of the
|
||||
//! frames. The core therefore takes the decrypted packets with their exact payload bounds and
|
||||
//! decodes them itself. A small reorder window waits for at most `MAX_WAIT` later packets before a
|
||||
//! missing one is concealed (Opus in-band FEC from the next packet when it has it).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Later packets to wait for before giving up on a missing one: about 40 ms.
|
||||
pub const MAX_WAIT: usize = 2;
|
||||
/// A jump this far ahead (or any jump back past it) means the sender restarted its sequence.
|
||||
const RESYNC: i16 = 64;
|
||||
|
||||
pub trait FrameDecoder {
|
||||
/// Decode one Opus packet; `fec` decodes the previous (lost) frame from this packet's FEC data.
|
||||
fn decode(&mut self, packet: &[u8], fec: bool) -> Option<Vec<i16>>;
|
||||
/// Packet-loss concealment for one missing frame.
|
||||
fn conceal(&mut self) -> Option<Vec<i16>>;
|
||||
}
|
||||
|
||||
pub struct OpusFrames(opus2::Decoder);
|
||||
|
||||
impl OpusFrames {
|
||||
/// 48 kHz stereo, Discord's format.
|
||||
pub fn new() -> Self {
|
||||
Self(opus2::Decoder::new(48000, opus2::Channels::Stereo).expect("opus decoder"))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for OpusFrames {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameDecoder for OpusFrames {
|
||||
fn decode(&mut self, packet: &[u8], fec: bool) -> Option<Vec<i16>> {
|
||||
// FEC must ask for exactly the missing 20 ms; a normal decode gets room for 120 ms, the
|
||||
// largest Opus frame. Stereo.
|
||||
let mut out = vec![0i16; if fec { 960 * 2 } else { 5760 * 2 }];
|
||||
let n = self.0.decode(packet, &mut out, fec).ok()?;
|
||||
out.truncate(n * 2);
|
||||
Some(out)
|
||||
}
|
||||
fn conceal(&mut self) -> Option<Vec<i16>> {
|
||||
let mut out = vec![0i16; 960 * 2];
|
||||
let n = self.0.decode(&[], &mut out, false).ok()?;
|
||||
out.truncate(n * 2);
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// One speaker's reorder window and decoder.
|
||||
pub struct SpeakerStream<D: FrameDecoder> {
|
||||
dec: D,
|
||||
next: Option<u16>,
|
||||
pending: BTreeMap<i16, Vec<u8>>,
|
||||
pub lost: u64,
|
||||
pub late: u64,
|
||||
pub bad: u64,
|
||||
}
|
||||
|
||||
impl<D: FrameDecoder> SpeakerStream<D> {
|
||||
pub fn new(dec: D) -> Self {
|
||||
Self { dec, next: None, pending: BTreeMap::new(), lost: 0, late: 0, bad: 0 }
|
||||
}
|
||||
|
||||
/// A packet arrived; returns the PCM frames now ready, in order.
|
||||
pub fn push(&mut self, seq: u16, opus: Vec<u8>) -> Vec<Vec<i16>> {
|
||||
let mut out = vec![];
|
||||
let next = *self.next.get_or_insert(seq);
|
||||
let ahead = seq.wrapping_sub(next) as i16;
|
||||
if !(-RESYNC..RESYNC).contains(&ahead) {
|
||||
// The sender restarted: drop the window and start over at this packet.
|
||||
self.flush_into(&mut out);
|
||||
self.next = Some(seq);
|
||||
self.decode_into(&opus, &mut out);
|
||||
self.next = Some(seq.wrapping_add(1));
|
||||
return out;
|
||||
}
|
||||
if ahead < 0 {
|
||||
self.late += 1;
|
||||
return out;
|
||||
}
|
||||
self.pending.insert(ahead, opus);
|
||||
self.drain(&mut out, false);
|
||||
out
|
||||
}
|
||||
|
||||
/// The speaker stopped: play out what is waiting, concealing nothing at the end.
|
||||
pub fn flush(&mut self) -> Vec<Vec<i16>> {
|
||||
let mut out = vec![];
|
||||
self.flush_into(&mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn flush_into(&mut self, out: &mut Vec<Vec<i16>>) {
|
||||
self.drain(out, true);
|
||||
}
|
||||
|
||||
fn decode_into(&mut self, opus: &[u8], out: &mut Vec<Vec<i16>>) {
|
||||
match self.dec.decode(opus, false) {
|
||||
Some(pcm) if !pcm.is_empty() => out.push(pcm),
|
||||
_ => self.bad += 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Play every packet that is due; when `force`, or the window holds more than MAX_WAIT packets
|
||||
/// past a gap, conceal the gap and go on.
|
||||
fn drain(&mut self, out: &mut Vec<Vec<i16>>, force: bool) {
|
||||
loop {
|
||||
if let Some(opus) = self.pending.remove(&0) {
|
||||
self.decode_into(&opus, out);
|
||||
self.advance(1);
|
||||
continue;
|
||||
}
|
||||
let Some(&first) = self.pending.keys().next() else { return };
|
||||
if !force && self.pending.len() <= MAX_WAIT {
|
||||
return;
|
||||
}
|
||||
// Frames 0..first are missing. Recover the last one from the next packet's FEC, conceal
|
||||
// the others.
|
||||
for k in 0..first {
|
||||
let pcm = if k == first - 1 {
|
||||
let next = self.pending.get(&first).cloned().unwrap_or_default();
|
||||
self.dec.decode(&next, true)
|
||||
} else {
|
||||
self.dec.conceal()
|
||||
};
|
||||
if let Some(pcm) = pcm.filter(|p| !p.is_empty()) {
|
||||
out.push(pcm);
|
||||
}
|
||||
self.lost += 1;
|
||||
}
|
||||
self.advance(first);
|
||||
}
|
||||
}
|
||||
|
||||
fn advance(&mut self, by: i16) {
|
||||
self.next = self.next.map(|n| n.wrapping_add(by as u16));
|
||||
self.pending = std::mem::take(&mut self.pending).into_iter().map(|(k, v)| (k - by, v)).collect();
|
||||
}
|
||||
}
|
||||
|
||||
/// The RTP sequence number and SSRC of a packet, and the Opus payload inside it given songbird's
|
||||
/// payload bounds (offset into the body after the fixed header and CSRCs, bytes to drop at the end).
|
||||
pub fn opus_payload(packet: &[u8], payload_offset: usize, payload_end_pad: usize) -> Option<(u16, u32, &[u8])> {
|
||||
if packet.len() < 12 {
|
||||
return None;
|
||||
}
|
||||
let csrc = usize::from(packet[0] & 0x0f);
|
||||
let header = 12 + 4 * csrc;
|
||||
let seq = u16::from_be_bytes([packet[2], packet[3]]);
|
||||
let ssrc = u32::from_be_bytes([packet[8], packet[9], packet[10], packet[11]]);
|
||||
let start = header.checked_add(payload_offset)?;
|
||||
let end = packet.len().checked_sub(payload_end_pad)?;
|
||||
(start < end).then(|| (seq, ssrc, &packet[start..end]))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// "Decodes" a packet to one sample holding its first byte; FEC and concealment are marked.
|
||||
struct Tag;
|
||||
impl FrameDecoder for Tag {
|
||||
fn decode(&mut self, p: &[u8], fec: bool) -> Option<Vec<i16>> {
|
||||
Some(vec![if fec { 1000 + p.first().copied().unwrap_or(0) as i16 } else { p[0] as i16 }])
|
||||
}
|
||||
fn conceal(&mut self) -> Option<Vec<i16>> {
|
||||
Some(vec![-1])
|
||||
}
|
||||
}
|
||||
|
||||
fn run(s: &mut SpeakerStream<Tag>, seqs: &[u16]) -> Vec<i16> {
|
||||
seqs.iter().flat_map(|&q| s.push(q, vec![q as u8])).map(|f| f[0]).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_order_packets_play_at_once() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
assert_eq!(run(&mut s, &[5, 6, 7]), vec![5, 6, 7]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_swapped_pair_is_put_back_in_order() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
assert_eq!(run(&mut s, &[1, 3, 2, 4]), vec![1, 2, 3, 4]);
|
||||
assert_eq!((s.lost, s.late), (0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lost_packet_is_recovered_from_fec_after_the_wait() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
// 3 never comes; after MAX_WAIT + 1 later packets it is rebuilt from 4's FEC data.
|
||||
assert_eq!(run(&mut s, &[1, 2, 4, 5]), vec![1, 2]);
|
||||
assert_eq!(run(&mut s, &[6]), vec![1004, 4, 5, 6]);
|
||||
assert_eq!(s.lost, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn several_lost_packets_are_concealed_then_fec_for_the_last() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
assert_eq!(run(&mut s, &[1, 4, 5, 6]), vec![1, -1, 1004, 4, 5, 6]);
|
||||
assert_eq!(s.lost, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_and_duplicate_packets_are_dropped() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
assert_eq!(run(&mut s, &[1, 2, 3, 2, 1]), vec![1, 2, 3]);
|
||||
assert_eq!(s.late, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequence_numbers_wrap() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
let out: Vec<i16> = [65534u16, 65535, 0, 1].iter().flat_map(|&q| s.push(q, vec![(q % 7) as u8])).map(|f| f[0]).collect();
|
||||
assert_eq!(out, vec![65534 % 7, 65535 % 7, 0, 1].into_iter().map(|v| v as i16).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_far_jump_starts_over() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
assert_eq!(run(&mut s, &[10, 11, 200, 201]), vec![10, 11, 200, 201]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_plays_what_waits_behind_a_gap() {
|
||||
let mut s = SpeakerStream::new(Tag);
|
||||
assert_eq!(run(&mut s, &[1, 3]), vec![1]);
|
||||
assert_eq!(s.flush().iter().map(|f| f[0]).collect::<Vec<_>>(), vec![1003, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_opus_round_trip_through_the_window() {
|
||||
// Encode 5 frames of a tone, drop one, swap two: all 5 frames come out, in order.
|
||||
let mut enc = opus2::Encoder::new(48000, opus2::Channels::Stereo, opus2::Application::Voip).unwrap();
|
||||
enc.set_inband_fec(true).unwrap();
|
||||
enc.set_packet_loss_perc(10).unwrap();
|
||||
let frames: Vec<Vec<u8>> = (0..6).map(|k| {
|
||||
let pcm: Vec<i16> = (0..960).flat_map(|i| {
|
||||
let v = (8000.0 * (2.0 * std::f64::consts::PI * 440.0 * (k * 960 + i) as f64 / 48000.0).sin()) as i16;
|
||||
[v, v]
|
||||
}).collect();
|
||||
enc.encode_vec(&pcm, 4000).unwrap()
|
||||
}).collect();
|
||||
let mut s = SpeakerStream::new(OpusFrames::new());
|
||||
let mut out = vec![];
|
||||
for q in [0u16, 2, 1, 4, 5] {
|
||||
out.extend(s.push(q, frames[q as usize].clone()));
|
||||
}
|
||||
out.extend(s.flush());
|
||||
assert_eq!(out.len(), 6);
|
||||
assert!(out.iter().all(|f| f.len() == 1920));
|
||||
assert_eq!(s.lost, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payload_bounds_follow_the_header_and_songbirds_offsets() {
|
||||
let mut p = vec![0x80u8, 0x78, 0x01, 0x02, 0, 0, 0, 0, 0, 0, 0x30, 0x39];
|
||||
p.extend_from_slice(&[9, 9, 9, 9]); // extension (offset 4)
|
||||
p.extend_from_slice(&[1, 2, 3]); // opus
|
||||
p.extend_from_slice(&[7, 7]); // DAVE leftovers (end pad 2)
|
||||
let (seq, ssrc, opus) = opus_payload(&p, 4, 2).unwrap();
|
||||
assert_eq!((seq, ssrc, opus), (0x0102, 12345, &[1u8, 2, 3][..]));
|
||||
assert!(opus_payload(&p, 10, 9).is_none());
|
||||
}
|
||||
}
|
||||
115
app/src/recordings.rs
Normal file
115
app/src/recordings.rs
Normal file
@@ -0,0 +1,115 @@
|
||||
//! The voice recordings the worker keeps when `voice.record` is on (`voice_worker.spec.md`,
|
||||
//! "Recordings"), packed for the 로그 screen's 목소리 view (`admin.spec.md`, "Logs").
|
||||
//!
|
||||
//! `recordings/<YYYY-MM-DD>/` holds one WAV per utterance or spoken answer and `index.jsonl`, one
|
||||
//! line per file: `{file, at, ms, kind, who, name, channel, text, cancelled}`.
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use serde_json::Value;
|
||||
|
||||
/// The recordings of days `from..=to` in a zip at `out`: every day's WAV files under `<day>/`, its
|
||||
/// `index.jsonl`, and `index.txt` listing every file in time order with who said it and its text.
|
||||
/// Returns how many sound files it holds.
|
||||
pub fn export_zip(root: &Path, from: NaiveDate, to: NaiveDate, out: &Path) -> std::io::Result<usize> {
|
||||
let mut days: Vec<(NaiveDate, std::path::PathBuf)> = std::fs::read_dir(root).into_iter().flatten().flatten()
|
||||
.filter_map(|e| {
|
||||
let day = NaiveDate::parse_from_str(&e.file_name().to_string_lossy(), "%Y-%m-%d").ok()?;
|
||||
(day >= from && day <= to && e.path().is_dir()).then(|| (day, e.path()))
|
||||
})
|
||||
.collect();
|
||||
days.sort();
|
||||
let mut zip = zip::ZipWriter::new(std::fs::File::create(out)?);
|
||||
let deflate = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
|
||||
let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
let (mut listing, mut files) = (String::new(), 0usize);
|
||||
for (day, dir) in &days {
|
||||
let index = std::fs::read_to_string(dir.join("index.jsonl")).unwrap_or_default();
|
||||
let mut entries: Vec<Value> = index.lines().filter_map(|l| serde_json::from_str(l).ok()).collect();
|
||||
entries.sort_by(|a, b| a["at"].as_f64().unwrap_or(0.0).total_cmp(&b["at"].as_f64().unwrap_or(0.0)));
|
||||
for e in &entries {
|
||||
listing += &line_of(day, e);
|
||||
}
|
||||
let mut wavs: Vec<_> = std::fs::read_dir(dir)?.flatten().map(|e| e.path())
|
||||
.filter(|p| p.extension().is_some_and(|x| x == "wav")).collect();
|
||||
wavs.sort();
|
||||
for p in wavs {
|
||||
let name = format!("{day}/{}", p.file_name().unwrap_or_default().to_string_lossy());
|
||||
zip.start_file(name, stored).map_err(std::io::Error::other)?;
|
||||
zip.write_all(&std::fs::read(&p)?)?;
|
||||
files += 1;
|
||||
}
|
||||
if !index.is_empty() {
|
||||
zip.start_file(format!("{day}/index.jsonl"), deflate).map_err(std::io::Error::other)?;
|
||||
zip.write_all(index.as_bytes())?;
|
||||
}
|
||||
}
|
||||
zip.start_file("index.txt", deflate).map_err(std::io::Error::other)?;
|
||||
zip.write_all(if listing.is_empty() { "저장된 목소리가 없습니다.\n".as_bytes() } else { listing.as_bytes() })?;
|
||||
zip.finish().map_err(std::io::Error::other)?;
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn line_of(day: &NaiveDate, e: &Value) -> String {
|
||||
let s = |k: &str| e[k].as_str().unwrap_or("").to_string();
|
||||
let at = chrono::DateTime::from_timestamp_millis((e["at"].as_f64().unwrap_or(0.0) * 1000.0) as i64)
|
||||
.map(|t| t.with_timezone(&chrono::Local).format("%H:%M:%S%.3f").to_string()).unwrap_or_default();
|
||||
let who = if s("kind") == "spoke" {
|
||||
format!("자비스{}", if e["cancelled"].as_bool() == Some(true) { " (중간에 멈춤)" } else { "" })
|
||||
} else if s("name").is_empty() || s("name") == s("who") {
|
||||
s("who")
|
||||
} else {
|
||||
format!("{}({})", s("name"), s("who"))
|
||||
};
|
||||
let text = s("text");
|
||||
format!("{day} {at} · {:.1}초 · {day}/{} · {who}: {}\n", e["ms"].as_f64().unwrap_or(0.0) / 1000.0, s("file"),
|
||||
if text.trim().is_empty() { "(받아 적은 글 없음)".into() } else { text })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
#[test]
|
||||
fn the_zip_holds_the_days_asked_for_with_a_listing_of_who_said_what() {
|
||||
let root = std::env::temp_dir().join(format!("jarvis-recordings-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
for (day, lines) in [
|
||||
("2026-11-15", vec![r#"{"file":"235959-000-said-1.wav","at":1.0,"ms":800,"kind":"said","who":"1","name":"Alice","text":"어제 말"}"#]),
|
||||
("2026-11-16", vec![
|
||||
r#"{"file":"144843-500-jarvis.wav","at":20.0,"ms":2500,"kind":"spoke","who":"jarvis","text":"알려드리겠습니다.","cancelled":true}"#,
|
||||
r#"{"file":"144842-100-said-1.wav","at":10.0,"ms":1200,"kind":"said","who":"1","name":"Alice","text":"자비스 내일 알려 줘"}"#,
|
||||
r#"{"file":"144850-000-said-1.wav","at":30.0,"ms":300,"kind":"said","who":"1","name":"Alice","text":""}"#]),
|
||||
] {
|
||||
let dir = root.join(day);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("index.jsonl"), lines.join("\n") + "\n").unwrap();
|
||||
for l in lines {
|
||||
let v: Value = serde_json::from_str(l).unwrap();
|
||||
std::fs::write(dir.join(v["file"].as_str().unwrap()), b"RIFF").unwrap();
|
||||
}
|
||||
}
|
||||
let out = root.join("voice.zip");
|
||||
let d = |s: &str| NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap();
|
||||
assert_eq!(export_zip(&root, d("2026-11-16"), d("2026-11-16"), &out).unwrap(), 3);
|
||||
let mut z = zip::ZipArchive::new(std::fs::File::open(&out).unwrap()).unwrap();
|
||||
let mut names: Vec<String> = (0..z.len()).map(|i| z.by_index(i).unwrap().name().unwrap().to_string()).collect();
|
||||
names.sort();
|
||||
assert_eq!(names, ["2026-11-16/144842-100-said-1.wav", "2026-11-16/144843-500-jarvis.wav",
|
||||
"2026-11-16/144850-000-said-1.wav", "2026-11-16/index.jsonl", "index.txt"]);
|
||||
let mut listing = String::new();
|
||||
z.by_name("index.txt").unwrap().read_to_string(&mut listing).unwrap();
|
||||
let lines: Vec<&str> = listing.lines().collect();
|
||||
assert_eq!(lines.len(), 3);
|
||||
assert!(lines[0].ends_with("1.2초 · 2026-11-16/144842-100-said-1.wav · Alice(1): 자비스 내일 알려 줘"), "{listing}");
|
||||
assert!(lines[1].ends_with("2026-11-16/144843-500-jarvis.wav · 자비스 (중간에 멈춤): 알려드리겠습니다."), "{listing}");
|
||||
assert!(lines[2].ends_with("Alice(1): (받아 적은 글 없음)"), "{listing}");
|
||||
// No recordings at all: a zip that says so.
|
||||
let none = root.join("none");
|
||||
assert_eq!(export_zip(&none, d("2026-11-16"), d("2026-11-16"), &out).unwrap(), 0);
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
}
|
||||
673
app/src/schedule.rs
Normal file
673
app/src/schedule.rs
Normal file
@@ -0,0 +1,673 @@
|
||||
//! Reminders and scheduled tasks (`app/src/admin.spec.md`, "Schedules"): the store in the data
|
||||
//! folder, the next times, the brain's `schedule_action`, and the loop that fires and delivers.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{Datelike, Local, NaiveDateTime, TimeZone, Weekday};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::worker_client::ScheduleAction;
|
||||
|
||||
/// A schedule later than this when its time comes (the app was off, the PC asleep) is not delivered.
|
||||
pub const GRACE_SECS: i64 = 15 * 60;
|
||||
/// Schedules one person may have.
|
||||
pub const PER_PERSON: usize = 50;
|
||||
pub const MAX_TEXT_CHARS: usize = 500;
|
||||
/// How often the loop looks for schedules that are due.
|
||||
pub const TICK: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The store's file is read and written whole; one change at a time in this process (the services'
|
||||
/// loop and the settings screen both use it).
|
||||
static FILE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Outcome {
|
||||
pub at: i64,
|
||||
pub ok: bool,
|
||||
/// Why it failed, for the screen (no text of the schedule).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Schedule {
|
||||
pub id: u64,
|
||||
pub user: u64,
|
||||
pub name: String,
|
||||
/// `remind` or `task`.
|
||||
pub kind: String,
|
||||
pub text: String,
|
||||
/// `none`, `daily`, `weekdays` or `weekly`.
|
||||
pub repeat: String,
|
||||
/// When it fires next (Unix seconds); None: it will not fire again (kept to show its outcome).
|
||||
pub next: Option<i64>,
|
||||
/// `voice` or `chat`: where it was asked.
|
||||
pub via: String,
|
||||
/// The chat channel it was asked in (via chat).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub channel: Option<u64>,
|
||||
pub created: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub last: Option<Outcome>,
|
||||
/// Handed out by `take_due` and not finished yet; still set at the next start means the app
|
||||
/// stopped while delivering it (`recover`).
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub firing: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct File {
|
||||
next_id: u64,
|
||||
items: Vec<Schedule>,
|
||||
}
|
||||
|
||||
pub struct Store {
|
||||
file: PathBuf,
|
||||
}
|
||||
|
||||
/// The local time `t` (Unix seconds) one day, a weekday or a week on, at the same wall time.
|
||||
fn step(t: i64, repeat: &str) -> i64 {
|
||||
let local = Local.timestamp_opt(t, 0).single().unwrap_or_else(Local::now);
|
||||
let wall = local.naive_local();
|
||||
let mut day = wall.date();
|
||||
loop {
|
||||
day = day.succ_opt().unwrap_or(day);
|
||||
let ok = match repeat {
|
||||
"weekdays" => !matches!(day.weekday(), Weekday::Sat | Weekday::Sun),
|
||||
"weekly" => day.weekday() == wall.weekday() && day != wall.date(),
|
||||
_ => true,
|
||||
};
|
||||
if ok {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let at = NaiveDateTime::new(day, wall.time());
|
||||
// A wall time a clock change skips is taken an hour later.
|
||||
Local.from_local_datetime(&at).earliest()
|
||||
.or_else(|| Local.from_local_datetime(&(at + chrono::Duration::hours(1))).earliest())
|
||||
.map(|d| d.timestamp()).unwrap_or(t + 86_400)
|
||||
}
|
||||
|
||||
fn is_weekend(t: i64) -> bool {
|
||||
Local.timestamp_opt(t, 0).single().is_some_and(|d| matches!(d.weekday(), Weekday::Sat | Weekday::Sun))
|
||||
}
|
||||
|
||||
/// The first time a schedule asked for `at` fires: `at` itself, the next weekday for `weekdays`,
|
||||
/// and for a repeating one that has passed, its first time after `now`.
|
||||
pub fn first_time(at: i64, repeat: &str, now: i64) -> i64 {
|
||||
let mut t = at;
|
||||
if repeat == "weekdays" && is_weekend(t) {
|
||||
t = step(t, repeat);
|
||||
}
|
||||
while repeat != "none" && t <= now {
|
||||
t = step(t, repeat);
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
/// What `take_due` hands out to deliver.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Due {
|
||||
pub item: Schedule,
|
||||
/// The time it was due (the item's `next` has moved on for a repeating one).
|
||||
pub due: i64,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn new(data: &Path) -> Self {
|
||||
Self { file: data.join("config").join("schedules.json") }
|
||||
}
|
||||
|
||||
fn load(&self) -> Result<File, String> {
|
||||
match std::fs::read_to_string(&self.file) {
|
||||
Ok(t) => serde_json::from_str(&t).map_err(|e| format!("예약 파일을 읽을 수 없습니다: {e}")),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(File::default()),
|
||||
Err(e) => Err(format!("예약 파일을 읽을 수 없습니다: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn save(&self, f: &File) -> Result<(), String> {
|
||||
let mut text = serde_json::to_string_pretty(f).unwrap();
|
||||
text.push('\n');
|
||||
crate::datadir::write_atomic(&self.file, text.as_bytes()).map_err(|e| format!("예약 파일을 쓸 수 없습니다: {e}"))
|
||||
}
|
||||
|
||||
fn change<T>(&self, f: impl FnOnce(&mut File) -> Result<T, String>) -> Result<T, String> {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut file = self.load()?;
|
||||
let out = f(&mut file)?;
|
||||
self.save(&file)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Every schedule, soonest first (those that will not fire again last).
|
||||
pub fn all(&self) -> Result<Vec<Schedule>, String> {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut items = self.load()?.items;
|
||||
items.sort_by_key(|i| (i.next.is_none(), i.next, i.id));
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// One person's schedules, soonest first.
|
||||
pub fn of(&self, user: u64) -> Result<Vec<Schedule>, String> {
|
||||
Ok(self.all()?.into_iter().filter(|i| i.user == user).collect())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn add(&self, user: u64, name: &str, kind: &str, text: &str, at: i64, repeat: &str, via: &str,
|
||||
channel: Option<u64>, now: i64) -> Result<Schedule, String> {
|
||||
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if text.is_empty() || text.chars().count() > MAX_TEXT_CHARS {
|
||||
return Err(format!("text must be 1-{MAX_TEXT_CHARS} characters"));
|
||||
}
|
||||
if !["remind", "task"].contains(&kind) {
|
||||
return Err("kind must be remind or task".into());
|
||||
}
|
||||
if !["none", "daily", "weekdays", "weekly"].contains(&repeat) {
|
||||
return Err("repeat must be none, daily, weekdays or weekly".into());
|
||||
}
|
||||
let channel = match via {
|
||||
"voice" => None,
|
||||
"chat" => Some(channel.ok_or("a chat schedule needs the channel it was asked in")?),
|
||||
_ => return Err("via must be voice or chat".into()),
|
||||
};
|
||||
if repeat == "none" && at <= now {
|
||||
return Err("that time has already passed".into());
|
||||
}
|
||||
let next = first_time(at, repeat, now);
|
||||
self.change(|f| {
|
||||
if f.items.iter().filter(|i| i.user == user && i.next.is_some()).count() >= PER_PERSON {
|
||||
return Err(format!("at most {PER_PERSON} schedules per person; delete one first"));
|
||||
}
|
||||
f.next_id += 1;
|
||||
let item = Schedule { id: f.next_id, user, name: name.to_owned(), kind: kind.into(), text,
|
||||
repeat: repeat.into(), next: Some(next), via: via.into(), channel, created: now, last: None,
|
||||
firing: false };
|
||||
f.items.push(item.clone());
|
||||
Ok(item)
|
||||
})
|
||||
}
|
||||
|
||||
/// Delete a schedule; with `user`, only one of theirs.
|
||||
pub fn delete(&self, id: u64, user: Option<u64>) -> Result<Schedule, String> {
|
||||
self.change(|f| {
|
||||
let pos = f.items.iter().position(|i| i.id == id && user.is_none_or(|u| u == i.user))
|
||||
.ok_or_else(|| "no such schedule of this person".to_string())?;
|
||||
Ok(f.items.remove(pos))
|
||||
})
|
||||
}
|
||||
|
||||
/// The schedules due at `now`. A repeating one is judged by its latest time up to `now` (the
|
||||
/// ones before it, while the app was off, are passed over) and moves on to its next time; a
|
||||
/// one-off one stops (`next` None). Each one handed out is `firing` until `finished` records how
|
||||
/// it went. One more than `GRACE_SECS` late is not handed out and is recorded as missed.
|
||||
pub fn take_due(&self, now: i64) -> Result<Vec<Due>, String> {
|
||||
let _g = FILE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut file = self.load()?;
|
||||
let mut due = vec![];
|
||||
let mut changed = false;
|
||||
for item in file.items.iter_mut() {
|
||||
let Some(mut t) = item.next.filter(|t| *t <= now) else { continue };
|
||||
changed = true;
|
||||
if item.repeat != "none" {
|
||||
loop {
|
||||
let n = step(t, &item.repeat);
|
||||
if n > now {
|
||||
break;
|
||||
}
|
||||
t = n;
|
||||
}
|
||||
}
|
||||
item.next = (item.repeat != "none").then(|| first_time(t, &item.repeat, now));
|
||||
if now - t > GRACE_SECS {
|
||||
item.last = Some(Outcome { at: now, ok: false, note: Some("앱이 꺼져 있어 제때 전하지 못함".into()) });
|
||||
crate::warn_log!("schedule", "Warning: schedule {} missed ({} s late)", item.id, now - t);
|
||||
} else {
|
||||
item.firing = true;
|
||||
due.push(Due { item: item.clone(), due: t });
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
self.save(&file)?;
|
||||
}
|
||||
Ok(due)
|
||||
}
|
||||
|
||||
/// Record how a delivery went. A one-off schedule delivered is removed; one that failed stays
|
||||
/// (not firing again) so the screen shows it.
|
||||
pub fn finished(&self, id: u64, outcome: Outcome) -> Result<(), String> {
|
||||
self.change(|f| {
|
||||
if let Some(pos) = f.items.iter().position(|i| i.id == id) {
|
||||
if outcome.ok && f.items[pos].next.is_none() {
|
||||
f.items.remove(pos);
|
||||
} else {
|
||||
f.items[pos].last = Some(outcome);
|
||||
f.items[pos].firing = false;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// At start: the schedules a previous run handed out and never finished (it stopped while
|
||||
/// delivering them) get a failed outcome. How many.
|
||||
pub fn recover(&self, now: i64) -> Result<usize, String> {
|
||||
self.change(|f| {
|
||||
let mut n = 0;
|
||||
for i in f.items.iter_mut().filter(|i| i.firing) {
|
||||
i.firing = false;
|
||||
i.last = Some(Outcome { at: now, ok: false, note: Some("전하는 도중 앱이 멈춤".into()) });
|
||||
crate::warn_log!("schedule", "Warning: schedule {} was being delivered when the app stopped", i.id);
|
||||
n += 1;
|
||||
}
|
||||
Ok(n)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A schedule as the brain's tool sees it.
|
||||
pub fn brain_item(s: &Schedule) -> Value {
|
||||
json!({"id": s.id.to_string(), "kind": s.kind, "text": s.text, "repeat": s.repeat,
|
||||
"next": s.next.map(|t| t as f64)})
|
||||
}
|
||||
|
||||
/// Do what the brain's schedule tool asked for the person in `a.user` (named `name`): the item(s)
|
||||
/// for its result, or why not.
|
||||
pub fn act(store: &Store, a: &ScheduleAction, name: &str, now: i64) -> Result<Value, String> {
|
||||
let user: u64 = a.user.parse().map_err(|_| "bad user id".to_string())?;
|
||||
match a.action.as_str() {
|
||||
"list" => Ok(json!({"items": store.of(user)?.iter().filter(|s| s.next.is_some()).map(brain_item).collect::<Vec<_>>()})),
|
||||
"delete" => {
|
||||
let id: u64 = a.schedule.as_deref().and_then(|s| s.trim().parse().ok()).ok_or("delete needs the schedule's id")?;
|
||||
Ok(json!({"item": brain_item(&store.delete(id, Some(user))?)}))
|
||||
}
|
||||
"add" => {
|
||||
let at = a.at.filter(|t| t.is_finite()).ok_or("add needs at")?;
|
||||
let channel = a.channel.as_deref().and_then(|c| c.parse().ok());
|
||||
let s = store.add(user, name, a.kind.as_deref().unwrap_or("remind"), a.text.as_deref().unwrap_or(""),
|
||||
at.round() as i64, a.repeat.as_deref().unwrap_or("none"), &a.via, channel, now)?;
|
||||
crate::info_log!("schedule", "schedule {} added: {} {} by {user}", s.id, s.kind, s.repeat);
|
||||
Ok(json!({"item": brain_item(&s)}))
|
||||
}
|
||||
other => Err(format!("unknown schedule action {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the loop needs to deliver (the Discord core; a fake in tests).
|
||||
#[async_trait::async_trait]
|
||||
pub trait Deliver: Send + Sync {
|
||||
/// The person is in Jarvis's voice channel.
|
||||
fn hears(&self, user: u64) -> bool;
|
||||
/// Speak in the voice channel and wait until it was heard to the end; false when not in one,
|
||||
/// or when the speech failed or was stopped.
|
||||
async fn say(&self, text: &str) -> bool;
|
||||
/// Post in a registered chat channel (never pinging anyone).
|
||||
async fn post(&self, channel: u64, text: &str) -> Result<(), String>;
|
||||
/// Run a task in the brain (read-only tools) and wait for its answer.
|
||||
async fn task(&self, id: &str, user: u64, text: &str) -> Option<String>;
|
||||
}
|
||||
|
||||
/// Deliver one due schedule: a task runs first; then by voice when it was asked by voice and the
|
||||
/// person is in Jarvis's voice channel, else in the chat channel it was asked in, or for voice the
|
||||
/// fallback channel (`schedule.chatChannel`).
|
||||
pub async fn deliver(d: &dyn Deliver, due: &Due, fallback: Option<u64>) -> Outcome {
|
||||
let s = &due.item;
|
||||
let now = || chrono::Utc::now().timestamp();
|
||||
let fail = |note: &str| Outcome { at: now(), ok: false, note: Some(note.into()) };
|
||||
let body = if s.kind == "task" {
|
||||
match d.task(&format!("s{}-{}", s.id, due.due), s.user, &s.text).await {
|
||||
Some(a) if !a.trim().is_empty() => a,
|
||||
_ => return fail("작업 결과를 만들지 못함"),
|
||||
}
|
||||
} else {
|
||||
s.text.clone()
|
||||
};
|
||||
let text = format!("{}님, {}", s.name, body.trim());
|
||||
// Speech that did not finish goes to the chat channel like a person who is not there.
|
||||
if s.via == "voice" && d.hears(s.user) && d.say(&text).await {
|
||||
crate::debug_log!("schedule", "schedule {} spoken chars={}", s.id, text.chars().count());
|
||||
return Outcome { at: now(), ok: true, note: None };
|
||||
}
|
||||
let Some(channel) = (if s.via == "chat" { s.channel } else { fallback }) else {
|
||||
return fail("음성 채널에 없고 알림 채팅 채널이 정해지지 않음");
|
||||
};
|
||||
match d.post(channel, &text).await {
|
||||
Ok(()) => {
|
||||
crate::debug_log!("schedule", "schedule {} posted in {channel} chars={}", s.id, text.chars().count());
|
||||
Outcome { at: now(), ok: true, note: None }
|
||||
}
|
||||
Err(e) => {
|
||||
crate::warn_log!("schedule", "Warning: schedule {} could not be posted in {channel}: {e}", s.id);
|
||||
fail("채팅 채널에 보내지 못함")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record how a delivery went: in the store (its outcome) and in the activity log (kind, id, who
|
||||
/// asked and the outcome; never the text).
|
||||
pub fn finish(store: &Store, activity: &crate::activity::Activity, due: &Due, out: Outcome) {
|
||||
let s = &due.item;
|
||||
crate::info_log!("schedule", "schedule {} {}", s.id, if out.ok { "delivered" } else { "not delivered" });
|
||||
activity.record(&format!("schedule.{}", s.kind), &format!("예약 {}", s.id), Some(format!("{} {}", s.user, s.name)),
|
||||
out.ok, out.note.clone());
|
||||
if let Err(e) = store.finished(s.id, out) {
|
||||
crate::warn_log!("schedule", "Warning: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Fire due schedules until the task is aborted. Each delivery runs on its own, so a slow task does
|
||||
/// not hold up a reminder due meanwhile; aborting the loop aborts the deliveries it started (they
|
||||
/// are recorded as interrupted at the next start, `recover`).
|
||||
pub async fn run(store: std::sync::Arc<Store>, activity: std::sync::Arc<crate::activity::Activity>,
|
||||
d: std::sync::Arc<dyn Deliver>, fallback: Option<u64>) {
|
||||
if let Err(e) = store.recover(chrono::Utc::now().timestamp()) {
|
||||
crate::warn_log!("schedule", "Warning: {e}");
|
||||
}
|
||||
let mut deliveries = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
while deliveries.try_join_next().is_some() {}
|
||||
match store.take_due(chrono::Utc::now().timestamp()) {
|
||||
Ok(due) => for item in due {
|
||||
let (store, activity, d) = (store.clone(), activity.clone(), d.clone());
|
||||
deliveries.spawn(async move {
|
||||
let out = deliver(d.as_ref(), &item, fallback).await;
|
||||
finish(&store, &activity, &item, out);
|
||||
});
|
||||
},
|
||||
Err(e) => {
|
||||
crate::warn_log!("schedule", "Warning: {e}");
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(TICK).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Timelike;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn local(y: i32, mo: u32, d: u32, h: u32, mi: u32) -> i64 {
|
||||
Local.with_ymd_and_hms(y, mo, d, h, mi, 0).earliest().unwrap().timestamp()
|
||||
}
|
||||
|
||||
fn store() -> Store {
|
||||
Store::new(&crate::datadir::temp_dir("schedule"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeats_keep_the_wall_time_and_skip_what_they_leave_out() {
|
||||
// 2026-11-13 is a Friday.
|
||||
let fri = local(2026, 11, 13, 9, 0);
|
||||
assert_eq!(step(fri, "daily"), local(2026, 11, 14, 9, 0));
|
||||
assert_eq!(step(fri, "weekdays"), local(2026, 11, 16, 9, 0));
|
||||
assert_eq!(step(fri, "weekly"), local(2026, 11, 20, 9, 0));
|
||||
// Asked for a Saturday on weekdays: Monday. A daily one that passed: its next time.
|
||||
assert_eq!(first_time(local(2026, 11, 14, 9, 0), "weekdays", 0), local(2026, 11, 16, 9, 0));
|
||||
assert_eq!(first_time(fri, "daily", local(2026, 11, 15, 10, 0)), local(2026, 11, 16, 9, 0));
|
||||
assert_eq!(Local.timestamp_opt(step(fri, "daily"), 0).unwrap().hour(), 9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_person_sees_and_deletes_only_their_own() {
|
||||
let s = store();
|
||||
let a = s.add(1, "가", "remind", " 회의 준비 ", 2000, "none", "voice", None, 1000).unwrap();
|
||||
let b = s.add(2, "나", "task", "날씨", 3000, "daily", "chat", Some(77), 1000).unwrap();
|
||||
assert_eq!(a.text, "회의 준비");
|
||||
assert_eq!(s.of(1).unwrap(), vec![a.clone()]);
|
||||
assert!(s.delete(b.id, Some(1)).is_err());
|
||||
assert_eq!(s.delete(b.id, None).unwrap(), b);
|
||||
assert_eq!(s.all().unwrap(), vec![a]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_adds_are_refused() {
|
||||
let s = store();
|
||||
for (kind, text, at, repeat, via, channel) in [
|
||||
("remind", "", 2000, "none", "voice", None),
|
||||
("remind", "x", 500, "none", "voice", None),
|
||||
("alarm", "x", 2000, "none", "voice", None),
|
||||
("remind", "x", 2000, "hourly", "voice", None),
|
||||
("remind", "x", 2000, "none", "chat", None),
|
||||
] {
|
||||
assert!(s.add(1, "가", kind, text, at, repeat, via, channel, 1000).is_err(), "{kind} {text} {repeat} {via}");
|
||||
}
|
||||
assert!(s.add(1, "가", "remind", &"x".repeat(MAX_TEXT_CHARS + 1), 2000, "none", "voice", None, 1000).is_err());
|
||||
for _ in 0..PER_PERSON {
|
||||
s.add(1, "가", "remind", "x", 2000, "none", "voice", None, 1000).unwrap();
|
||||
}
|
||||
assert!(s.add(1, "가", "remind", "x", 2000, "none", "voice", None, 1000).is_err());
|
||||
assert!(s.add(2, "나", "remind", "x", 2000, "none", "voice", None, 1000).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn due_schedules_are_handed_out_once_and_late_ones_are_recorded_as_missed() {
|
||||
let s = store();
|
||||
let once = s.add(1, "가", "remind", "a", 2000, "none", "voice", None, 1000).unwrap();
|
||||
let daily = s.add(1, "가", "remind", "b", local(2026, 11, 13, 9, 0), "daily", "voice", None, 0).unwrap();
|
||||
let late = s.add(1, "가", "remind", "c", 1500, "none", "voice", None, 1000).unwrap();
|
||||
assert!(s.take_due(1499).unwrap().is_empty());
|
||||
let now = local(2026, 11, 13, 9, 1);
|
||||
let got: Vec<u64> = s.take_due(now).unwrap().into_iter().map(|d| d.item.id).collect();
|
||||
// `once` and `late` were due long before `now`: missed. The daily one moves to tomorrow.
|
||||
assert_eq!(got, vec![daily.id]);
|
||||
assert!(s.take_due(now).unwrap().is_empty());
|
||||
let all = s.all().unwrap();
|
||||
let find = |id| all.iter().find(|i| i.id == id).unwrap().clone();
|
||||
assert_eq!(find(daily.id).next, Some(local(2026, 11, 14, 9, 0)));
|
||||
for id in [once.id, late.id] {
|
||||
let i = find(id);
|
||||
assert_eq!(i.next, None);
|
||||
assert!(!i.last.unwrap().ok);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_delivered_one_off_schedule_is_removed_and_a_failed_one_kept() {
|
||||
let s = store();
|
||||
let a = s.add(1, "가", "remind", "a", 2000, "none", "voice", None, 1000).unwrap();
|
||||
let b = s.add(1, "가", "remind", "b", 2000, "none", "voice", None, 1000).unwrap();
|
||||
assert_eq!(s.take_due(2000).unwrap().len(), 2);
|
||||
s.finished(a.id, Outcome { at: 2001, ok: true, note: None }).unwrap();
|
||||
s.finished(b.id, Outcome { at: 2001, ok: false, note: Some("x".into()) }).unwrap();
|
||||
let all = s.all().unwrap();
|
||||
assert_eq!(all.len(), 1);
|
||||
assert_eq!(all[0].id, b.id);
|
||||
assert!(!all[0].last.as_ref().unwrap().ok);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Fake {
|
||||
hears: bool,
|
||||
/// The speech does not finish (it failed or was stopped).
|
||||
speech_fails: bool,
|
||||
/// task() never answers; `hung` is set while it waits and cleared when it is dropped.
|
||||
hang: bool,
|
||||
hung: Arc<std::sync::atomic::AtomicBool>,
|
||||
said: Mutex<Vec<String>>,
|
||||
posted: Mutex<Vec<(u64, String)>>,
|
||||
post_fails: bool,
|
||||
answer: Option<String>,
|
||||
tasks: Mutex<Vec<(u64, String)>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Deliver for Fake {
|
||||
fn hears(&self, _user: u64) -> bool {
|
||||
self.hears
|
||||
}
|
||||
async fn say(&self, text: &str) -> bool {
|
||||
self.said.lock().unwrap().push(text.into());
|
||||
!self.speech_fails
|
||||
}
|
||||
async fn post(&self, channel: u64, text: &str) -> Result<(), String> {
|
||||
if self.post_fails {
|
||||
return Err("no".into());
|
||||
}
|
||||
self.posted.lock().unwrap().push((channel, text.into()));
|
||||
Ok(())
|
||||
}
|
||||
async fn task(&self, _id: &str, user: u64, text: &str) -> Option<String> {
|
||||
self.tasks.lock().unwrap().push((user, text.into()));
|
||||
if self.hang {
|
||||
struct Clear(Arc<std::sync::atomic::AtomicBool>);
|
||||
impl Drop for Clear {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
self.hung.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let _clear = Clear(self.hung.clone());
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
self.answer.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn speech_that_does_not_finish_falls_back_to_the_chat_channel_or_fails() {
|
||||
let f = Fake { hears: true, speech_fails: true, ..Default::default() };
|
||||
assert!(deliver(&f, &due("remind", "voice", None), Some(9)).await.ok);
|
||||
assert_eq!(*f.posted.lock().unwrap(), vec![(9, "철수님, 회의".to_string())]);
|
||||
let out = deliver(&f, &due("remind", "voice", None), None).await;
|
||||
assert!(!out.ok && out.note.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_repeating_schedule_the_app_was_off_for_still_fires_its_latest_time_when_recent() {
|
||||
let s = store();
|
||||
// Set on the 12th; the app was off until a minute after 9:00 on the 14th.
|
||||
let d = s.add(1, "가", "remind", "b", local(2026, 11, 12, 9, 0), "daily", "voice", None,
|
||||
local(2026, 11, 12, 8, 0)).unwrap();
|
||||
let due = s.take_due(local(2026, 11, 14, 9, 1)).unwrap();
|
||||
assert_eq!(due.iter().map(|d| (d.item.id, d.due)).collect::<Vec<_>>(), vec![(d.id, local(2026, 11, 14, 9, 0))]);
|
||||
assert_eq!(s.all().unwrap()[0].next, Some(local(2026, 11, 15, 9, 0)));
|
||||
// Off past the grace: that day's is missed, the next one stays.
|
||||
let late = s.take_due(local(2026, 11, 15, 9, 30)).unwrap();
|
||||
assert!(late.is_empty());
|
||||
let item = s.all().unwrap()[0].clone();
|
||||
assert_eq!(item.next, Some(local(2026, 11, 16, 9, 0)));
|
||||
assert!(!item.last.unwrap().ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_schedule_taken_but_never_finished_is_recorded_as_interrupted_at_the_next_start() {
|
||||
let s = store();
|
||||
let a = s.add(1, "가", "remind", "a", 2000, "none", "voice", None, 1000).unwrap();
|
||||
let b = s.add(1, "가", "remind", "b", 2000, "none", "voice", None, 1000).unwrap();
|
||||
assert_eq!(s.take_due(2000).unwrap().len(), 2);
|
||||
s.finished(b.id, Outcome { at: 2001, ok: false, note: Some("x".into()) }).unwrap();
|
||||
// The app quit while `a` was being delivered.
|
||||
assert_eq!(s.recover(3000).unwrap(), 1);
|
||||
let all = s.all().unwrap();
|
||||
let find = |id| all.iter().find(|i| i.id == id).unwrap().clone();
|
||||
let last = find(a.id).last.unwrap();
|
||||
assert!(!last.ok && last.note.is_some() && last.at == 3000);
|
||||
assert_eq!(find(b.id).last.unwrap().note.as_deref(), Some("x"));
|
||||
assert_eq!(s.recover(3001).unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_delivery_is_recorded_in_the_activity_without_its_text() {
|
||||
let data = crate::datadir::temp_dir("schedule-activity");
|
||||
let s = Store::new(&data);
|
||||
let a = crate::activity::Activity::new(&data);
|
||||
let item = s.add(5, "철수", "remind", "비밀 회의", 2000, "none", "voice", None, 1000).unwrap();
|
||||
let due = s.take_due(2000).unwrap().remove(0);
|
||||
finish(&s, &a, &due, Outcome { at: 2001, ok: false, note: Some("채팅 채널에 보내지 못함".into()) });
|
||||
let got = a.recent(10);
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!((got[0].kind.as_str(), got[0].ok), ("schedule.remind", false));
|
||||
assert_eq!(got[0].by.as_deref(), Some("5 철수"));
|
||||
assert!(got[0].target.contains(&item.id.to_string()));
|
||||
assert!(!serde_json::to_string(&got[0]).unwrap().contains("비밀"), "no text in the activity");
|
||||
assert!(!s.all().unwrap()[0].last.as_ref().unwrap().ok, "the outcome is kept too");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopping_the_loop_stops_the_deliveries_it_started() {
|
||||
let data = crate::datadir::temp_dir("schedule-run");
|
||||
let s = Arc::new(Store::new(&data));
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
s.add(1, "가", "task", "x", now + 1, "none", "chat", Some(77), now).unwrap();
|
||||
let f = Arc::new(Fake { hang: true, ..Default::default() });
|
||||
let run = tokio::spawn(run(s.clone(), Arc::new(crate::activity::Activity::new(&data)), f.clone(), None));
|
||||
let hung = f.hung.clone();
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !hung.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}).await.expect("the task started");
|
||||
run.abort();
|
||||
let _ = run.await;
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(!hung.load(std::sync::atomic::Ordering::SeqCst), "the delivery was not stopped");
|
||||
assert!(f.posted.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
fn due(kind: &str, via: &str, channel: Option<u64>) -> Due {
|
||||
Due { item: Schedule { id: 1, user: 5, name: "철수".into(), kind: kind.into(), text: "회의".into(),
|
||||
repeat: "none".into(), next: None, via: via.into(), channel, created: 0, last: None, firing: false }, due: 10 }
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_voice_reminder_is_spoken_to_someone_there_and_posted_otherwise() {
|
||||
let f = Fake { hears: true, ..Default::default() };
|
||||
assert!(deliver(&f, &due("remind", "voice", None), Some(9)).await.ok);
|
||||
assert_eq!(*f.said.lock().unwrap(), vec!["철수님, 회의".to_string()]);
|
||||
let away = Fake::default();
|
||||
assert!(deliver(&away, &due("remind", "voice", None), Some(9)).await.ok);
|
||||
assert_eq!(*away.posted.lock().unwrap(), vec![(9, "철수님, 회의".to_string())]);
|
||||
// Nowhere to go: not delivered, and says why.
|
||||
let out = deliver(&away, &due("remind", "voice", None), None).await;
|
||||
assert!(!out.ok && out.note.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_chat_reminder_goes_to_its_channel_even_when_the_person_is_in_voice() {
|
||||
let f = Fake { hears: true, ..Default::default() };
|
||||
assert!(deliver(&f, &due("remind", "chat", Some(77)), Some(9)).await.ok);
|
||||
assert!(f.said.lock().unwrap().is_empty());
|
||||
assert_eq!(*f.posted.lock().unwrap(), vec![(77, "철수님, 회의".to_string())]);
|
||||
let fails = Fake { post_fails: true, ..Default::default() };
|
||||
assert!(!deliver(&fails, &due("remind", "chat", Some(77)), None).await.ok);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_task_delivers_the_brains_answer_and_fails_without_one() {
|
||||
let f = Fake { answer: Some("맑아요".into()), ..Default::default() };
|
||||
assert!(deliver(&f, &due("task", "chat", Some(77)), None).await.ok);
|
||||
assert_eq!(*f.tasks.lock().unwrap(), vec![(5, "회의".to_string())]);
|
||||
assert_eq!(*f.posted.lock().unwrap(), vec![(77, "철수님, 맑아요".to_string())]);
|
||||
let none = Fake::default();
|
||||
let out = deliver(&none, &due("task", "chat", Some(77)), None).await;
|
||||
assert!(!out.ok && none.posted.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_brains_actions_are_done_for_the_asker_only() {
|
||||
let s = store();
|
||||
let a = |action: &str, user: &str, extra: Value| -> ScheduleAction {
|
||||
let mut v = json!({"id": "x", "action": action, "user": user, "via": "voice"});
|
||||
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
|
||||
serde_json::from_value(v).unwrap()
|
||||
};
|
||||
let added = act(&s, &a("add", "1", json!({"kind": "remind", "text": "물", "at": 2000.4, "repeat": "none"})), "가", 1000).unwrap();
|
||||
assert_eq!(added["item"]["id"], "1");
|
||||
assert_eq!(added["item"]["next"], 2000.0);
|
||||
assert_eq!(act(&s, &a("list", "2", json!({})), "나", 1000).unwrap()["items"], json!([]));
|
||||
assert!(act(&s, &a("delete", "2", json!({"schedule": "1"})), "나", 1000).is_err());
|
||||
assert_eq!(act(&s, &a("list", "1", json!({})), "가", 1000).unwrap()["items"].as_array().unwrap().len(), 1);
|
||||
assert!(act(&s, &a("delete", "1", json!({"schedule": "1"})), "가", 1000).is_ok());
|
||||
assert!(act(&s, &a("add", "1", json!({"text": "x", "at": 2000.0, "via": "chat"})), "가", 1000).is_err());
|
||||
}
|
||||
}
|
||||
830
app/src/screen.rs
Normal file
830
app/src/screen.rs
Normal file
@@ -0,0 +1,830 @@
|
||||
//! Screen viewing for the brain's `screenView` tool (admin.spec.md, "Screen viewing";
|
||||
//! `docs/v2/pc-features.md`, "화면 보기"): who may see, which monitor or window, the pictures it
|
||||
//! gets (scaled, excluded programs painted black, JPEG). The capture itself is Windows Graphics
|
||||
//! Capture (`win`); everything else is plain code the tests drive with a fake screen.
|
||||
|
||||
use base64::Engine as _;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::permissions::{decide, Action, Decision, Refusal};
|
||||
use crate::settings::{Level, Settings};
|
||||
use crate::worker_client::ScreenAction;
|
||||
|
||||
/// Long side of a `fast` and a `normal` picture, and of each `detailed` part.
|
||||
pub const FAST_PX: u32 = 768;
|
||||
pub const NORMAL_PX: u32 = 1280;
|
||||
/// A picture whose long side is at most this stays one `detailed` part at full size.
|
||||
pub const ONE_PART_MAX_PX: u32 = 1600;
|
||||
pub const JPEG_QUALITY: u8 = 80;
|
||||
/// Pictures per answer (base64 grows them by a third; the worker takes 4 MiB messages).
|
||||
pub const MAX_BYTES: usize = 3 * 1024 * 1024;
|
||||
/// Windows in the list.
|
||||
pub const MAX_WINDOWS: usize = 30;
|
||||
|
||||
/// Whether the brain gets the tool and the LLM reads pictures: Windows, the feature on, someone
|
||||
/// registered for PC control and viewing not turned off.
|
||||
pub fn available(s: &Settings) -> bool {
|
||||
cfg!(windows) && s.features.screen && !s.pc_control.allowed_users.is_empty() && s.pc_control.view != Level::Never
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Rect {
|
||||
pub x: i32,
|
||||
pub y: i32,
|
||||
pub w: i32,
|
||||
pub h: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Monitor {
|
||||
/// As Windows numbers it (`\\.\DISPLAY<n>`).
|
||||
pub number: u32,
|
||||
pub rect: Rect,
|
||||
pub main: bool,
|
||||
/// The opaque handle the capture takes.
|
||||
pub handle: isize,
|
||||
}
|
||||
|
||||
/// A visible top-level window, front first.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Window {
|
||||
pub handle: isize,
|
||||
/// Program file name, lower case (`chrome.exe`).
|
||||
pub program: String,
|
||||
pub title: String,
|
||||
pub rect: Rect,
|
||||
pub minimised: bool,
|
||||
/// Owned windows and tool windows (pop-ups, palettes): never listed or picked, but painted
|
||||
/// black when their program is excluded.
|
||||
pub secondary: bool,
|
||||
/// This app's own windows.
|
||||
pub own: bool,
|
||||
}
|
||||
|
||||
/// A captured picture: top-down BGRA rows.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Image {
|
||||
pub w: u32,
|
||||
pub h: u32,
|
||||
pub bgra: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Image {
|
||||
pub fn new(w: u32, h: u32) -> Self {
|
||||
Image { w, h, bgra: vec![0; (w * h * 4) as usize] }
|
||||
}
|
||||
}
|
||||
|
||||
/// What the screen offers; Windows in `win`, a fake in the tests.
|
||||
pub trait Screen {
|
||||
/// The monitors without the bot browser's virtual one.
|
||||
fn monitors(&self) -> Result<Vec<Monitor>, String>;
|
||||
fn windows(&self) -> Result<Vec<Window>, String>;
|
||||
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String>;
|
||||
fn capture_window(&self, w: &Window) -> Result<Image, String>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Detail {
|
||||
Fast,
|
||||
Normal,
|
||||
Detailed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Target {
|
||||
Screen,
|
||||
Monitor(u32),
|
||||
Window(String),
|
||||
}
|
||||
|
||||
/// The activity log's target: never a title.
|
||||
pub fn activity_target(t: &Target) -> String {
|
||||
match t {
|
||||
Target::Screen => "전체 화면".into(),
|
||||
Target::Monitor(n) => format!("모니터 {n}"),
|
||||
Target::Window(_) => "창".into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The request in a `capture` action.
|
||||
pub fn target_of(a: &ScreenAction) -> Result<(Target, Detail), String> {
|
||||
let detail = match a.detail.as_deref() {
|
||||
Some("fast") => Detail::Fast,
|
||||
Some("detailed") => Detail::Detailed,
|
||||
_ => Detail::Normal,
|
||||
};
|
||||
let target = match a.target.as_deref() {
|
||||
Some("monitor") => Target::Monitor(a.monitor.filter(|n| *n >= 1).ok_or("모니터 번호가 필요합니다")?),
|
||||
Some("window") => {
|
||||
let w = a.window.as_deref().map(str::trim).filter(|w| !w.is_empty()).ok_or("창 이름이 필요합니다")?;
|
||||
Target::Window(w.chars().take(200).collect())
|
||||
}
|
||||
_ => Target::Screen,
|
||||
};
|
||||
Ok((target, detail))
|
||||
}
|
||||
|
||||
/// Why the asker may not see the screen, or `None` when they may.
|
||||
pub fn refusal(s: &Settings, user: &str) -> Option<String> {
|
||||
let Ok(id) = user.parse::<u64>() else { return Some(not_registered()) };
|
||||
match decide(&s.pc_control, id, Action::View) {
|
||||
Decision::Allow => None,
|
||||
Decision::Refuse(Refusal::NotAllowed) => Some(not_registered()),
|
||||
// `ask` needs the next-message consent the PC control tool brings (M7, next step).
|
||||
Decision::Ask => Some("화면 보기가 '먼저 묻기'로 되어 있어 지금은 볼 수 없습니다(PC 제어 화면에서 '자동'으로 바꾸면 됩니다)".into()),
|
||||
Decision::Refuse(_) => Some("화면 보기가 꺼져 있습니다".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn not_registered() -> String {
|
||||
"PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다".into()
|
||||
}
|
||||
|
||||
/// Excluded: on the list, or a program that could not be told (its process refused the query), so
|
||||
/// no excluded program slips through unnamed.
|
||||
fn excluded(s: &Settings, program: &str) -> bool {
|
||||
program.is_empty() || crate::game::listed(program, &s.pc_control.screen_exclude)
|
||||
}
|
||||
|
||||
/// Where excluded windows show now (minimised ones show nowhere).
|
||||
fn hidden_rects(s: &Settings, all: &[Window]) -> Vec<Rect> {
|
||||
all.iter().filter(|w| !w.minimised && excluded(s, &w.program)).map(|w| w.rect).collect()
|
||||
}
|
||||
|
||||
/// Captures of monitors and screens tried while excluded windows keep moving.
|
||||
const STILL_TRIES: usize = 3;
|
||||
|
||||
/// The windows the list shows and a window request may pick: not secondary, not this app's, not
|
||||
/// excluded; front first.
|
||||
fn listable<'a>(s: &'a Settings, all: &'a [Window]) -> impl Iterator<Item = &'a Window> + 'a {
|
||||
all.iter().filter(move |w| !w.secondary && !w.own && !w.title.trim().is_empty() && !excluded(s, &w.program))
|
||||
}
|
||||
|
||||
/// The window nearest the front whose title or program name contains `text` (any case).
|
||||
pub fn pick_window<'a>(s: &'a Settings, all: &'a [Window], text: &str) -> Option<&'a Window> {
|
||||
let t = text.to_lowercase();
|
||||
let stem = |p: &str| p.strip_suffix(".exe").unwrap_or(p).to_owned();
|
||||
listable(s, all).find(|w| w.title.to_lowercase().contains(&t) || stem(&w.program).contains(&t))
|
||||
}
|
||||
|
||||
/// Whether nothing shows (protected video, exclusive full screen): every pixel near black.
|
||||
pub fn all_black(img: &Image) -> bool {
|
||||
img.bgra.chunks_exact(4).all(|p| p[0] <= 8 && p[1] <= 8 && p[2] <= 8)
|
||||
}
|
||||
|
||||
/// Paint `rects` (screen coordinates) black on a picture of `area`; the picture may have another
|
||||
/// size than `area` (display scaling), so the rectangles are scaled to it.
|
||||
pub fn paint_black(img: &mut Image, area: Rect, rects: &[Rect]) {
|
||||
if area.w <= 0 || area.h <= 0 {
|
||||
return;
|
||||
}
|
||||
let (sx, sy) = (img.w as f64 / area.w as f64, img.h as f64 / area.h as f64);
|
||||
for r in rects {
|
||||
let x0 = (((r.x - area.x) as f64 * sx).floor().max(0.0) as u32).min(img.w);
|
||||
let y0 = (((r.y - area.y) as f64 * sy).floor().max(0.0) as u32).min(img.h);
|
||||
let x1 = (((r.x + r.w - area.x) as f64 * sx).ceil().max(0.0) as u32).min(img.w);
|
||||
let y1 = (((r.y + r.h - area.y) as f64 * sy).ceil().max(0.0) as u32).min(img.h);
|
||||
for y in y0..y1 {
|
||||
let row = (y * img.w) as usize * 4;
|
||||
img.bgra[row + x0 as usize * 4..row + x1 as usize * 4].fill(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `img` scaled down (area average) so its long side is at most `max`; never up.
|
||||
pub fn scale(img: &Image, max: u32) -> Image {
|
||||
let long = img.w.max(img.h);
|
||||
if long <= max || img.w == 0 || img.h == 0 {
|
||||
return img.clone();
|
||||
}
|
||||
let (w, h) = (((img.w as u64 * max as u64) / long as u64).max(1) as u32, ((img.h as u64 * max as u64) / long as u64).max(1) as u32);
|
||||
let mut out = Image::new(w, h);
|
||||
let (fx, fy) = (img.w as f64 / w as f64, img.h as f64 / h as f64);
|
||||
for oy in 0..h {
|
||||
let (y0, y1) = ((oy as f64 * fy) as u32, (((oy + 1) as f64 * fy).ceil() as u32).min(img.h));
|
||||
for ox in 0..w {
|
||||
let (x0, x1) = ((ox as f64 * fx) as u32, (((ox + 1) as f64 * fx).ceil() as u32).min(img.w));
|
||||
let mut sum = [0u32; 3];
|
||||
for y in y0..y1 {
|
||||
let row = (y * img.w) as usize * 4;
|
||||
for x in x0..x1 {
|
||||
let p = &img.bgra[row + x as usize * 4..];
|
||||
sum[0] += p[0] as u32;
|
||||
sum[1] += p[1] as u32;
|
||||
sum[2] += p[2] as u32;
|
||||
}
|
||||
}
|
||||
let n = ((y1 - y0) * (x1 - x0)).max(1);
|
||||
let o = ((oy * w + ox) * 4) as usize;
|
||||
out.bgra[o..o + 4].copy_from_slice(&[(sum[0] / n) as u8, (sum[1] / n) as u8, (sum[2] / n) as u8, 255]);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn crop(img: &Image, x: u32, y: u32, w: u32, h: u32) -> Image {
|
||||
let mut out = Image::new(w, h);
|
||||
for row in 0..h {
|
||||
let src = (((y + row) * img.w + x) * 4) as usize;
|
||||
let dst = (row * w * 4) as usize;
|
||||
out.bgra[dst..dst + (w * 4) as usize].copy_from_slice(&img.bgra[src..src + (w * 4) as usize]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The pictures of a request: one scaled picture, or for `detailed` four parts (left to right,
|
||||
/// top to bottom) of at most `NORMAL_PX` each.
|
||||
pub fn parts(img: &Image, detail: Detail) -> Vec<Image> {
|
||||
match detail {
|
||||
Detail::Fast => vec![scale(img, FAST_PX)],
|
||||
Detail::Normal => vec![scale(img, NORMAL_PX)],
|
||||
Detail::Detailed if img.w.max(img.h) <= ONE_PART_MAX_PX => vec![img.clone()],
|
||||
Detail::Detailed => {
|
||||
let (hw, hh) = (img.w / 2, img.h / 2);
|
||||
[(0, 0, hw, hh), (hw, 0, img.w - hw, hh), (0, hh, hw, img.h - hh), (hw, hh, img.w - hw, img.h - hh)]
|
||||
.into_iter().map(|(x, y, w, h)| scale(&crop(img, x, y, w, h), NORMAL_PX)).collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn jpeg(img: &Image) -> Result<Vec<u8>, String> {
|
||||
let mut out = Vec::new();
|
||||
jpeg_encoder::Encoder::new(&mut out, JPEG_QUALITY)
|
||||
.encode(&img.bgra, img.w as u16, img.h as u16, jpeg_encoder::ColorType::Bgra)
|
||||
.map_err(|e| format!("그림을 만들지 못했습니다: {e}"))?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn bounds(ms: &[Monitor]) -> Rect {
|
||||
let x0 = ms.iter().map(|m| m.rect.x).min().unwrap_or(0);
|
||||
let y0 = ms.iter().map(|m| m.rect.y).min().unwrap_or(0);
|
||||
let x1 = ms.iter().map(|m| m.rect.x + m.rect.w).max().unwrap_or(0);
|
||||
let y1 = ms.iter().map(|m| m.rect.y + m.rect.h).max().unwrap_or(0);
|
||||
Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }
|
||||
}
|
||||
|
||||
/// One picture of `ms` as they are placed; each monitor's capture is fitted to its place.
|
||||
fn whole(screen: &dyn Screen, ms: &[Monitor], excluded_rects: &[Rect]) -> Result<(Image, bool), String> {
|
||||
let b = bounds(ms);
|
||||
let mut out = Image::new(b.w.max(1) as u32, b.h.max(1) as u32);
|
||||
let mut black = true;
|
||||
for m in ms {
|
||||
let shot = screen.capture_monitor(m)?;
|
||||
black &= all_black(&shot);
|
||||
let mut fit = if shot.w as i32 == m.rect.w && shot.h as i32 == m.rect.h { shot } else { stretch(&shot, m.rect) };
|
||||
paint_black(&mut fit, m.rect, excluded_rects);
|
||||
let (ox, oy) = ((m.rect.x - b.x) as u32, (m.rect.y - b.y) as u32);
|
||||
for row in 0..fit.h.min(out.h - oy) {
|
||||
let w = fit.w.min(out.w - ox) as usize * 4;
|
||||
let src = (row * fit.w * 4) as usize;
|
||||
let dst = (((oy + row) * out.w + ox) * 4) as usize;
|
||||
out.bgra[dst..dst + w].copy_from_slice(&fit.bgra[src..src + w]);
|
||||
}
|
||||
}
|
||||
Ok((out, black))
|
||||
}
|
||||
|
||||
/// `img` resized to exactly `r`'s size (nearest pixel): monitors whose capture differs from their
|
||||
/// placed size (display scaling) still fill their place.
|
||||
fn stretch(img: &Image, r: Rect) -> Image {
|
||||
let (w, h) = (r.w.max(1) as u32, r.h.max(1) as u32);
|
||||
let mut out = Image::new(w, h);
|
||||
for y in 0..h {
|
||||
let sy = (y as u64 * img.h as u64 / h as u64) as u32;
|
||||
for x in 0..w {
|
||||
let sx = (x as u64 * img.w as u64 / w as u64) as u32;
|
||||
let s = ((sy * img.w + sx) * 4) as usize;
|
||||
let d = ((y * w + x) * 4) as usize;
|
||||
out.bgra[d..d + 4].copy_from_slice(&img.bgra[s..s + 4]);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Answer one `screen_action` for its asker: `{item, images}` or why not.
|
||||
pub fn act(s: &Settings, screen: &dyn Screen, a: &ScreenAction) -> Result<Value, String> {
|
||||
if let Some(why) = refusal(s, &a.user) {
|
||||
return Err(why);
|
||||
}
|
||||
match a.action.as_str() {
|
||||
"list" => {
|
||||
let monitors: Vec<Value> = screen.monitors()?.iter().map(|m| json!({"number": m.number,
|
||||
"width": m.rect.w, "height": m.rect.h, "main": m.main})).collect();
|
||||
let all = screen.windows()?;
|
||||
let windows: Vec<Value> = listable(s, &all).take(MAX_WINDOWS).map(|w| json!({
|
||||
"program": w.program.strip_suffix(".exe").unwrap_or(&w.program), "title": w.title,
|
||||
"minimised": w.minimised})).collect();
|
||||
Ok(json!({"item": {"monitors": monitors, "windows": windows}}))
|
||||
}
|
||||
"capture" => {
|
||||
let (target, detail) = target_of(a)?;
|
||||
let (img, black, what) = match &target {
|
||||
Target::Window(text) => {
|
||||
let all = screen.windows()?;
|
||||
let w = pick_window(s, &all, text).ok_or("그런 창을 찾지 못했습니다")?;
|
||||
if w.minimised {
|
||||
return Err("그 창은 최소화되어 있어 볼 수 없습니다".into());
|
||||
}
|
||||
let img = screen.capture_window(w)?;
|
||||
let black = all_black(&img);
|
||||
(img, black, format!("창 {}", w.program.strip_suffix(".exe").unwrap_or(&w.program)))
|
||||
}
|
||||
_ => {
|
||||
let ms = screen.monitors()?;
|
||||
let chosen: Vec<Monitor> = match target {
|
||||
Target::Monitor(n) => vec![ms.iter().find(|m| m.number == n).cloned()
|
||||
.ok_or_else(|| format!("{n}번 모니터가 없습니다 (있는 모니터: {})",
|
||||
ms.iter().map(|m| m.number.to_string()).collect::<Vec<_>>().join(", ")))?],
|
||||
_ => ms,
|
||||
};
|
||||
if chosen.is_empty() {
|
||||
return Err("모니터를 찾지 못했습니다".into());
|
||||
}
|
||||
// Excluded windows are read before and after the capture; if one moved, was
|
||||
// brought back or opened meanwhile, the black areas may miss it: that capture
|
||||
// is thrown away.
|
||||
let mut tries = 0;
|
||||
let (img, black) = loop {
|
||||
let before = hidden_rects(s, &screen.windows()?);
|
||||
let shot = whole(screen, &chosen, &before)?;
|
||||
if hidden_rects(s, &screen.windows()?) == before {
|
||||
break shot;
|
||||
}
|
||||
tries += 1;
|
||||
if tries == STILL_TRIES {
|
||||
return Err("보면 안 되는 창이 움직이는 중이라 지금은 볼 수 없습니다".into());
|
||||
}
|
||||
};
|
||||
(img, black, activity_target(&target))
|
||||
}
|
||||
};
|
||||
if black {
|
||||
return Err("화면이 검게 나와 볼 수 없습니다 (보호된 영상이나 전체 화면 게임일 수 있습니다)".into());
|
||||
}
|
||||
let pics = parts(&img, detail);
|
||||
let mut images = Vec::with_capacity(pics.len());
|
||||
let mut total = 0;
|
||||
for p in &pics {
|
||||
let j = jpeg(p)?;
|
||||
total += j.len();
|
||||
images.push(base64::engine::general_purpose::STANDARD.encode(j));
|
||||
}
|
||||
if total > MAX_BYTES {
|
||||
return Err("그림이 너무 커서 보낼 수 없습니다 (화질을 낮춰 다시 보세요)".into());
|
||||
}
|
||||
Ok(json!({"item": {"what": what, "width": img.w, "height": img.h, "parts": images.len()}, "images": images}))
|
||||
}
|
||||
other => Err(format!("모르는 화면 동작입니다: {}", other.chars().take(20).collect::<String>())),
|
||||
}
|
||||
}
|
||||
|
||||
/// The real screen (Windows), or none elsewhere.
|
||||
pub fn system_screen() -> Option<Box<dyn Screen + Send>> {
|
||||
#[cfg(windows)]
|
||||
return Some(Box::new(win::System));
|
||||
#[cfg(not(windows))]
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
mod win {
|
||||
use super::{Image, Monitor, Rect, Screen, Window};
|
||||
use std::time::{Duration, Instant};
|
||||
use windows::core::Interface;
|
||||
use windows::Graphics::Capture::{Direct3D11CaptureFramePool, GraphicsCaptureItem};
|
||||
use windows::Graphics::DirectX::Direct3D11::IDirect3DDevice;
|
||||
use windows::Graphics::DirectX::DirectXPixelFormat;
|
||||
use windows::Win32::Foundation::{HMODULE, HWND};
|
||||
use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP};
|
||||
use windows::Win32::Graphics::Direct3D11::*;
|
||||
use windows::Win32::Graphics::Dxgi::IDXGIDevice;
|
||||
use windows::Win32::Graphics::Gdi::HMONITOR;
|
||||
use windows::Win32::System::WinRT::Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess};
|
||||
use windows::Win32::System::WinRT::Graphics::Capture::IGraphicsCaptureItemInterop;
|
||||
use windows::Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED};
|
||||
use windows_sys::core::BOOL;
|
||||
use windows_sys::Win32::Foundation::{LPARAM, RECT};
|
||||
use windows_sys::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED};
|
||||
use windows_sys::Win32::Graphics::Gdi::{EnumDisplayMonitors, GetMonitorInfoW, HDC, MONITORINFOEXW};
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
||||
|
||||
/// The first frame of a capture.
|
||||
const FRAME_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
pub struct System;
|
||||
|
||||
fn text(buf: &[u16]) -> String {
|
||||
String::from_utf16_lossy(&buf[..buf.iter().position(|&c| c == 0).unwrap_or(buf.len())])
|
||||
}
|
||||
|
||||
unsafe extern "system" fn add_monitor(h: windows_sys::Win32::Graphics::Gdi::HMONITOR, _: HDC, _: *mut RECT, lp: LPARAM) -> BOOL {
|
||||
// SAFETY: `lp` is the Vec `monitors` passed, alive for the whole enumeration.
|
||||
let out = unsafe { &mut *(lp as *mut Vec<(isize, String, Rect, bool)>) };
|
||||
let mut mi: MONITORINFOEXW = unsafe { std::mem::zeroed() };
|
||||
mi.monitorInfo.cbSize = std::mem::size_of::<MONITORINFOEXW>() as u32;
|
||||
// SAFETY: `mi` is a MONITORINFOEXW with its size set.
|
||||
if unsafe { GetMonitorInfoW(h, &mut mi as *mut MONITORINFOEXW as *mut _) } != 0 {
|
||||
let r = mi.monitorInfo.rcMonitor;
|
||||
out.push((h as isize, text(&mi.szDevice),
|
||||
Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top },
|
||||
mi.monitorInfo.dwFlags & MONITORINFOF_PRIMARY != 0));
|
||||
}
|
||||
1
|
||||
}
|
||||
|
||||
unsafe extern "system" fn add_window(hwnd: windows_sys::Win32::Foundation::HWND, lp: LPARAM) -> BOOL {
|
||||
// SAFETY: `lp` is the Vec `windows` passed, alive for the whole enumeration.
|
||||
let out = unsafe { &mut *(lp as *mut Vec<windows_sys::Win32::Foundation::HWND>) };
|
||||
out.push(hwnd);
|
||||
1
|
||||
}
|
||||
|
||||
fn window(hwnd: windows_sys::Win32::Foundation::HWND, own: u32) -> Option<Window> {
|
||||
// SAFETY: plain queries on a window handle from EnumWindows; a stale one only fails them.
|
||||
unsafe {
|
||||
if IsWindowVisible(hwnd) == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut cloaked: u32 = 0;
|
||||
DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED as u32, (&mut cloaked as *mut u32).cast(), 4);
|
||||
if cloaked != 0 {
|
||||
return None;
|
||||
}
|
||||
let mut buf = [0u16; 512];
|
||||
let n = GetWindowTextW(hwnd, buf.as_mut_ptr(), buf.len() as i32).max(0) as usize;
|
||||
let mut pid = 0;
|
||||
GetWindowThreadProcessId(hwnd, &mut pid);
|
||||
let mut r: RECT = std::mem::zeroed();
|
||||
GetWindowRect(hwnd, &mut r);
|
||||
let ex = GetWindowLongPtrW(hwnd, GWL_EXSTYLE) as u32;
|
||||
let program = crate::game::probe::image_path(pid).map(|p| crate::game::file_name(&p)).unwrap_or_default();
|
||||
Some(Window { handle: hwnd as isize, program, title: String::from_utf16_lossy(&buf[..n]),
|
||||
rect: Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top },
|
||||
minimised: IsIconic(hwnd) != 0,
|
||||
secondary: !GetWindow(hwnd, GW_OWNER).is_null() || ex & WS_EX_TOOLWINDOW != 0,
|
||||
own: pid == own })
|
||||
}
|
||||
}
|
||||
|
||||
fn device() -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext, IDirect3DDevice)> {
|
||||
let make = |kind: D3D_DRIVER_TYPE| -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext)> {
|
||||
let (mut d, mut c) = (None, None);
|
||||
// SAFETY: out pointers to locals; no adapter, no software module.
|
||||
unsafe { D3D11CreateDevice(None, kind, HMODULE::default(), D3D11_CREATE_DEVICE_BGRA_SUPPORT, None,
|
||||
D3D11_SDK_VERSION, Some(&mut d), None, Some(&mut c))? };
|
||||
Ok((d.unwrap(), c.unwrap()))
|
||||
};
|
||||
// Without a graphics card (a virtual machine) WARP still captures.
|
||||
let (d, c) = make(D3D_DRIVER_TYPE_HARDWARE).or_else(|_| make(D3D_DRIVER_TYPE_WARP))?;
|
||||
let dxgi: IDXGIDevice = d.cast()?;
|
||||
// SAFETY: a valid DXGI device.
|
||||
let winrt: IDirect3DDevice = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi)? }.cast()?;
|
||||
Ok((d, c, winrt))
|
||||
}
|
||||
|
||||
fn capture(item: GraphicsCaptureItem) -> Result<Image, String> {
|
||||
let e = |e: windows::core::Error| format!("화면을 캡처하지 못했습니다 ({:#x})", e.code().0);
|
||||
// SAFETY: a second call on a thread already initialised only reports it.
|
||||
let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) };
|
||||
let (dev, ctx, winrt) = device().map_err(e)?;
|
||||
let size = item.Size().map_err(e)?;
|
||||
let pool = Direct3D11CaptureFramePool::CreateFreeThreaded(&winrt, DirectXPixelFormat::B8G8R8A8UIntNormalized, 1, size)
|
||||
.map_err(e)?;
|
||||
let session = pool.CreateCaptureSession(&item).map_err(e)?;
|
||||
let _ = session.SetIsCursorCaptureEnabled(false);
|
||||
// Windows 11 lets the app leave out the yellow border; older Windows refuses, which is fine.
|
||||
let _ = session.SetIsBorderRequired(false);
|
||||
session.StartCapture().map_err(e)?;
|
||||
let started = Instant::now();
|
||||
let frame = loop {
|
||||
if let Ok(f) = pool.TryGetNextFrame() {
|
||||
break f;
|
||||
}
|
||||
if started.elapsed() > FRAME_TIMEOUT {
|
||||
let _ = session.Close();
|
||||
let _ = pool.Close();
|
||||
return Err("화면이 제때 오지 않았습니다".into());
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
};
|
||||
let result = (|| -> windows::core::Result<Image> {
|
||||
let content = frame.ContentSize()?;
|
||||
let access: IDirect3DDxgiInterfaceAccess = frame.Surface()?.cast()?;
|
||||
// SAFETY: the frame's surface is a D3D11 texture.
|
||||
let tex: ID3D11Texture2D = unsafe { access.GetInterface()? };
|
||||
let mut desc = D3D11_TEXTURE2D_DESC::default();
|
||||
// SAFETY: `desc` is the out parameter.
|
||||
unsafe { tex.GetDesc(&mut desc) };
|
||||
let (w, h) = ((content.Width.max(1) as u32).min(desc.Width), (content.Height.max(1) as u32).min(desc.Height));
|
||||
desc.Usage = D3D11_USAGE_STAGING;
|
||||
desc.BindFlags = 0;
|
||||
desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32;
|
||||
desc.MiscFlags = 0;
|
||||
let mut staging = None;
|
||||
// SAFETY: a staging copy of the frame's texture, read on this thread only.
|
||||
unsafe {
|
||||
dev.CreateTexture2D(&desc, None, Some(&mut staging))?;
|
||||
let staging = staging.unwrap();
|
||||
ctx.CopyResource(&staging, &tex);
|
||||
let mut map = D3D11_MAPPED_SUBRESOURCE::default();
|
||||
ctx.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut map))?;
|
||||
let mut img = Image::new(w, h);
|
||||
for y in 0..h as usize {
|
||||
let src = std::slice::from_raw_parts((map.pData as *const u8).add(y * map.RowPitch as usize), w as usize * 4);
|
||||
img.bgra[y * w as usize * 4..(y + 1) * w as usize * 4].copy_from_slice(src);
|
||||
}
|
||||
ctx.Unmap(&staging, 0);
|
||||
Ok(img)
|
||||
}
|
||||
})();
|
||||
let _ = frame.Close();
|
||||
let _ = session.Close();
|
||||
let _ = pool.Close();
|
||||
result.map_err(e)
|
||||
}
|
||||
|
||||
fn interop() -> Result<IGraphicsCaptureItemInterop, String> {
|
||||
windows::core::factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()
|
||||
.map_err(|e| format!("화면 캡처를 쓸 수 없습니다 ({:#x})", e.code().0))
|
||||
}
|
||||
|
||||
impl Screen for System {
|
||||
fn monitors(&self) -> Result<Vec<Monitor>, String> {
|
||||
let mut found: Vec<(isize, String, Rect, bool)> = vec![];
|
||||
// SAFETY: the callback only pushes into `found`, which outlives the call.
|
||||
unsafe { EnumDisplayMonitors(std::ptr::null_mut(), std::ptr::null(), Some(add_monitor), &mut found as *mut _ as LPARAM) };
|
||||
let bot = crate::vdisplay::virtual_display_name();
|
||||
Ok(found.into_iter().filter(|(_, name, _, _)| Some(name) != bot.as_ref()).map(|(h, name, rect, main)| Monitor {
|
||||
number: name.trim_start_matches(r"\\.\DISPLAY").parse().unwrap_or(0), rect, main, handle: h }).collect())
|
||||
}
|
||||
|
||||
fn windows(&self) -> Result<Vec<Window>, String> {
|
||||
let mut hwnds = vec![];
|
||||
// SAFETY: the callback only pushes into `hwnds`, which outlives the call.
|
||||
unsafe { EnumWindows(Some(add_window), &mut hwnds as *mut _ as LPARAM) };
|
||||
let own = std::process::id();
|
||||
Ok(hwnds.into_iter().filter_map(|h| window(h, own)).collect())
|
||||
}
|
||||
|
||||
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String> {
|
||||
// SAFETY: a monitor handle from EnumDisplayMonitors.
|
||||
let item = unsafe { interop()?.CreateForMonitor::<GraphicsCaptureItem>(HMONITOR(m.handle as *mut _)) }
|
||||
.map_err(|e| format!("모니터를 캡처하지 못했습니다 ({:#x})", e.code().0))?;
|
||||
capture(item)
|
||||
}
|
||||
|
||||
fn capture_window(&self, w: &Window) -> Result<Image, String> {
|
||||
// SAFETY: a window handle from EnumWindows; a closed window only fails the call.
|
||||
let item = unsafe { interop()?.CreateForWindow::<GraphicsCaptureItem>(HWND(w.handle as *mut _)) }
|
||||
.map_err(|e| format!("창을 캡처하지 못했습니다 ({:#x})", e.code().0))?;
|
||||
capture(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::cell::RefCell;
|
||||
|
||||
const OWNER: &str = "123456789012345678";
|
||||
|
||||
fn settings() -> Settings {
|
||||
let mut s = Settings::default();
|
||||
s.pc_control.allowed_users = vec![OWNER.parse().unwrap()];
|
||||
s
|
||||
}
|
||||
|
||||
fn solid(w: u32, h: u32, v: u8) -> Image {
|
||||
Image { w, h, bgra: [v, v, v, 255].repeat((w * h) as usize) }
|
||||
}
|
||||
|
||||
struct Fake {
|
||||
monitors: Vec<Monitor>,
|
||||
windows: Vec<Window>,
|
||||
/// What the next reads of the windows give, one each, before `windows` again: a window
|
||||
/// that moves while the screen is captured.
|
||||
later: RefCell<Vec<Vec<Window>>>,
|
||||
shot: Image,
|
||||
captured: RefCell<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Screen for Fake {
|
||||
fn monitors(&self) -> Result<Vec<Monitor>, String> {
|
||||
Ok(self.monitors.clone())
|
||||
}
|
||||
fn windows(&self) -> Result<Vec<Window>, String> {
|
||||
let mut later = self.later.borrow_mut();
|
||||
Ok(if later.is_empty() { self.windows.clone() } else { later.remove(0) })
|
||||
}
|
||||
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String> {
|
||||
self.captured.borrow_mut().push(format!("monitor {}", m.number));
|
||||
Ok(solid(m.rect.w as u32, m.rect.h as u32, 200))
|
||||
}
|
||||
fn capture_window(&self, w: &Window) -> Result<Image, String> {
|
||||
self.captured.borrow_mut().push(format!("window {}", w.program));
|
||||
Ok(self.shot.clone())
|
||||
}
|
||||
}
|
||||
|
||||
fn win(program: &str, title: &str, rect: Rect) -> Window {
|
||||
Window { handle: 1, program: program.into(), title: title.into(), rect, minimised: false, secondary: false, own: false }
|
||||
}
|
||||
|
||||
fn fake() -> Fake {
|
||||
Fake {
|
||||
monitors: vec![
|
||||
Monitor { number: 1, rect: Rect { x: 0, y: 0, w: 400, h: 300 }, main: true, handle: 1 },
|
||||
Monitor { number: 2, rect: Rect { x: 400, y: 0, w: 200, h: 100 }, main: false, handle: 2 },
|
||||
],
|
||||
windows: vec![
|
||||
win("keepassxc.exe", "Passwords", Rect { x: 10, y: 10, w: 100, h: 50 }),
|
||||
win("chrome.exe", "메일 - Chrome", Rect { x: 0, y: 0, w: 300, h: 200 }),
|
||||
Window { secondary: true, ..win("chrome.exe", "pop-up", Rect { x: 0, y: 0, w: 10, h: 10 }) },
|
||||
win("notepad.exe", "메모", Rect { x: 0, y: 0, w: 50, h: 50 }),
|
||||
Window { own: true, ..win("jarvis.exe", "Jarvis 관리", Rect { x: 0, y: 0, w: 50, h: 50 }) },
|
||||
],
|
||||
later: RefCell::new(vec![]),
|
||||
shot: solid(800, 600, 120),
|
||||
captured: RefCell::new(vec![]),
|
||||
}
|
||||
}
|
||||
|
||||
fn action(action: &str, user: &str, extra: Value) -> ScreenAction {
|
||||
let mut v = json!({"id": "s1", "action": action, "user": user});
|
||||
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
|
||||
serde_json::from_value(v).unwrap()
|
||||
}
|
||||
|
||||
fn decode(v: &Value) -> Vec<Vec<u8>> {
|
||||
v["images"].as_array().unwrap().iter()
|
||||
.map(|i| base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_registered_people_see_the_screen_and_only_when_viewing_is_automatic() {
|
||||
let f = fake();
|
||||
let mut s = settings();
|
||||
for user in ["42", "not a number"] {
|
||||
let e = act(&s, &f, &action("capture", user, json!({}))).unwrap_err();
|
||||
assert!(e.contains("등록된 사람만"), "{e}");
|
||||
}
|
||||
s.pc_control.view = Level::Ask;
|
||||
assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("먼저 묻기"));
|
||||
s.pc_control.view = Level::Never;
|
||||
assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("꺼져"));
|
||||
assert!(f.captured.borrow().is_empty(), "nothing is captured for a refusal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tool_is_available_only_with_someone_registered_and_viewing_not_off() {
|
||||
let mut s = settings();
|
||||
assert_eq!(available(&s), cfg!(windows));
|
||||
s.features.screen = false;
|
||||
assert!(!available(&s));
|
||||
let mut s = settings();
|
||||
s.pc_control.view = Level::Never;
|
||||
assert!(!available(&s));
|
||||
assert!(!available(&Settings::default()), "nobody is registered by default");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_whole_screen_is_one_picture_with_excluded_programs_painted_black() {
|
||||
let f = fake();
|
||||
let s = settings();
|
||||
let v = act(&s, &f, &action("capture", OWNER, json!({"target": "screen", "detail": "detailed"}))).unwrap();
|
||||
assert_eq!(*f.captured.borrow(), vec!["monitor 1", "monitor 2"]);
|
||||
assert_eq!((v["item"]["width"].as_u64(), v["item"]["height"].as_u64()), (Some(600), Some(300)));
|
||||
assert_eq!((v["item"]["what"].as_str(), v["item"]["parts"].as_u64()), (Some("전체 화면"), Some(1)));
|
||||
assert!(decode(&v).iter().all(|j| j.starts_with(&[0xFF, 0xD8])), "JPEG pictures");
|
||||
// The picture before encoding: the password manager's area is black, the rest is not; the
|
||||
// corner below monitor 2 that no monitor covers is black too.
|
||||
let hidden = vec![f.windows[0].rect];
|
||||
let (img, black) = whole(&f, &f.monitors, &hidden).unwrap();
|
||||
let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize];
|
||||
assert!(!black);
|
||||
assert_eq!(at(50, 30), 0, "the excluded window is painted black");
|
||||
assert!(at(200, 150) == 200 && at(500, 50) == 200, "the monitors show");
|
||||
assert_eq!(at(500, 250), 0, "nothing outside the monitors");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_excluded_window_that_moves_or_comes_back_during_the_capture_never_shows() {
|
||||
let s = settings();
|
||||
let moved = |x: i32, minimised: bool| {
|
||||
let mut w = fake().windows;
|
||||
w[0].rect.x = x;
|
||||
w[0].minimised = minimised;
|
||||
w
|
||||
};
|
||||
// Moved between the reading before and after the capture: that capture is thrown away and
|
||||
// the next one, with the window still, is sent.
|
||||
let f = fake();
|
||||
*f.later.borrow_mut() = vec![moved(10, false), moved(200, false)];
|
||||
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "screen"}))).is_ok());
|
||||
assert_eq!(f.captured.borrow().len(), 4, "captured twice");
|
||||
// Brought back from the task bar during the capture: the same.
|
||||
let f = fake();
|
||||
*f.later.borrow_mut() = vec![moved(10, true), moved(10, false)];
|
||||
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 1}))).is_ok());
|
||||
assert_eq!(f.captured.borrow().len(), 2, "captured twice");
|
||||
// Moving all the time: nothing is sent.
|
||||
let f = fake();
|
||||
*f.later.borrow_mut() = (0..20).map(|i| moved(i * 7, false)).collect();
|
||||
let e = act(&s, &f, &action("capture", OWNER, json!({"target": "screen"}))).unwrap_err();
|
||||
assert!(e.contains("움직"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_whose_program_cannot_be_told_is_treated_as_excluded() {
|
||||
let s = settings();
|
||||
let mut f = fake();
|
||||
f.windows = vec![win("", "Vault - unlocked", Rect { x: 10, y: 10, w: 100, h: 50 })];
|
||||
assert!(pick_window(&s, &f.windows, "vault").is_none());
|
||||
let v = act(&s, &f, &action("list", OWNER, json!({}))).unwrap();
|
||||
assert_eq!(v["item"]["windows"], json!([]));
|
||||
let hidden = hidden_rects(&s, &f.windows);
|
||||
assert_eq!(hidden, vec![f.windows[0].rect], "painted black on the screen");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excluded_areas_follow_a_monitor_captured_at_another_scale() {
|
||||
// A picture twice the placed size (display scaling): the black area doubles with it.
|
||||
let mut img = solid(800, 600, 200);
|
||||
paint_black(&mut img, Rect { x: 100, y: 0, w: 400, h: 300 }, &[Rect { x: 110, y: 10, w: 50, h: 20 }]);
|
||||
let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize];
|
||||
assert_eq!((at(20, 20), at(119, 59)), (0, 0));
|
||||
assert_eq!((at(121, 20), at(20, 61), at(19, 20)), (200, 200, 200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_monitor_by_its_number_and_a_missing_one_names_those_there_are() {
|
||||
let f = fake();
|
||||
let v = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 2}))).unwrap();
|
||||
assert_eq!(*f.captured.borrow(), vec!["monitor 2"]);
|
||||
assert_eq!((v["item"]["what"].as_str(), v["item"]["width"].as_u64()), (Some("모니터 2"), Some(200)));
|
||||
let e = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 3}))).unwrap_err();
|
||||
assert!(e.contains("3번") && e.contains("1, 2"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_is_picked_by_title_or_program_front_first_never_an_excluded_or_own_one() {
|
||||
let s = settings();
|
||||
let f = fake();
|
||||
let v = act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "CHROME"}))).unwrap();
|
||||
assert_eq!(*f.captured.borrow(), vec!["window chrome.exe"]);
|
||||
assert_eq!(v["item"]["what"], "창 chrome");
|
||||
assert_eq!(pick_window(&s, &f.windows, "메모").map(|w| w.program.as_str()), Some("notepad.exe"));
|
||||
for hidden in ["keepass", "passwords", "jarvis", "pop-up"] {
|
||||
assert!(pick_window(&s, &f.windows, hidden).is_none(), "{hidden}");
|
||||
}
|
||||
let mut f = fake();
|
||||
f.windows[3].minimised = true;
|
||||
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "메모"}))).unwrap_err()
|
||||
.contains("최소화"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_black_picture_is_refused_and_pictures_are_scaled_to_the_detail() {
|
||||
let s = settings();
|
||||
let mut f = fake();
|
||||
f.shot = solid(800, 600, 0);
|
||||
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "chrome"}))).unwrap_err()
|
||||
.contains("검게"));
|
||||
let big = solid(3840, 2160, 90);
|
||||
assert_eq!(parts(&big, Detail::Fast).iter().map(|p| (p.w, p.h)).collect::<Vec<_>>(), vec![(768, 432)]);
|
||||
assert_eq!(parts(&big, Detail::Normal).iter().map(|p| (p.w, p.h)).collect::<Vec<_>>(), vec![(1280, 720)]);
|
||||
let four = parts(&big, Detail::Detailed);
|
||||
assert_eq!(four.len(), 4);
|
||||
assert!(four.iter().all(|p| p.w.max(p.h) <= NORMAL_PX));
|
||||
assert_eq!(parts(&solid(1600, 900, 90), Detail::Detailed).len(), 1);
|
||||
// Small pictures are never scaled up.
|
||||
assert_eq!(parts(&solid(300, 200, 90), Detail::Normal)[0].w, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_list_shows_monitors_and_windows_without_excluded_secondary_or_own_ones() {
|
||||
let v = act(&settings(), &fake(), &action("list", OWNER, json!({}))).unwrap();
|
||||
assert_eq!(v["item"]["monitors"], json!([{"number": 1, "width": 400, "height": 300, "main": true},
|
||||
{"number": 2, "width": 200, "height": 100, "main": false}]));
|
||||
let programs: Vec<&str> = v["item"]["windows"].as_array().unwrap().iter().map(|w| w["program"].as_str().unwrap()).collect();
|
||||
assert_eq!(programs, vec!["chrome", "notepad"]);
|
||||
assert!(v.get("images").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scaling_averages_the_area() {
|
||||
let mut img = solid(4, 2, 0);
|
||||
for y in 0..2 {
|
||||
for x in 0..2 {
|
||||
let o = ((y * 4 + x) * 4) as usize;
|
||||
img.bgra[o..o + 3].fill(200);
|
||||
}
|
||||
}
|
||||
let half = scale(&img, 2);
|
||||
assert_eq!((half.w, half.h), (2, 1));
|
||||
assert_eq!(&half.bgra[..3], &[200, 200, 200]);
|
||||
assert_eq!(&half.bgra[4..7], &[0, 0, 0]);
|
||||
}
|
||||
}
|
||||
188
app/src/secrets.rs
Normal file
188
app/src/secrets.rs
Normal file
@@ -0,0 +1,188 @@
|
||||
//! Stored secrets, `config/secrets.dat` (`app/src/admin.spec.md`): DPAPI-encrypted for the Windows
|
||||
//! user; plain JSON with owner-only permissions elsewhere (development only).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::datadir::write_atomic;
|
||||
|
||||
/// The Discord bot token's name in the store.
|
||||
pub const DISCORD_TOKEN: &str = "discordToken";
|
||||
/// The KMA API hub key (기상청 API허브) for the weather inside Korea.
|
||||
pub const KMA_KEY: &str = "kmaApiKey";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SecretsError(pub String);
|
||||
|
||||
impl std::fmt::Display for SecretsError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for SecretsError {}
|
||||
|
||||
pub struct Secrets {
|
||||
file: PathBuf,
|
||||
}
|
||||
|
||||
impl Secrets {
|
||||
pub fn new(file: &Path) -> Self {
|
||||
Self { file: file.to_path_buf() }
|
||||
}
|
||||
|
||||
fn load(&self) -> Result<Map<String, Value>, SecretsError> {
|
||||
let bytes = match std::fs::read(&self.file) {
|
||||
Ok(b) => b,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Map::new()),
|
||||
Err(e) => return Err(SecretsError(format!("could not read the secrets: {e}"))),
|
||||
};
|
||||
let plain = unprotect(&bytes)?;
|
||||
match serde_json::from_slice(&plain) {
|
||||
Ok(Value::Object(m)) => Ok(m),
|
||||
_ => Err(SecretsError("the secrets file is damaged".into())),
|
||||
}
|
||||
}
|
||||
|
||||
fn save(&self, m: &Map<String, Value>) -> Result<(), SecretsError> {
|
||||
let plain = serde_json::to_vec(m).expect("a map serialises");
|
||||
let bytes = protect(&plain)?;
|
||||
write_atomic(&self.file, &bytes).map_err(|e| SecretsError(format!("could not write the secrets: {e}")))?;
|
||||
restrict(&self.file);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get(&self, name: &str) -> Result<Option<String>, SecretsError> {
|
||||
Ok(self.load()?.get(name).and_then(Value::as_str).map(str::to_owned))
|
||||
}
|
||||
|
||||
pub fn set(&self, name: &str, value: &str) -> Result<(), SecretsError> {
|
||||
let mut m = self.load()?;
|
||||
m.insert(name.into(), Value::String(value.into()));
|
||||
self.save(&m)
|
||||
}
|
||||
|
||||
pub fn remove(&self, name: &str) -> Result<(), SecretsError> {
|
||||
let mut m = self.load()?;
|
||||
if m.remove(name).is_some() {
|
||||
self.save(&m)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The names of the stored secrets.
|
||||
pub fn names(&self) -> Result<Vec<String>, SecretsError> {
|
||||
Ok(self.load()?.keys().cloned().collect())
|
||||
}
|
||||
|
||||
/// Every stored value, for log redaction.
|
||||
pub fn values(&self) -> Vec<String> {
|
||||
self.load().map(|m| m.values().filter_map(Value::as_str).map(str::to_owned).collect()).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Only the last 4 characters, for the screens: `••••abcd`.
|
||||
pub fn masked(value: &str) -> String {
|
||||
let chars: Vec<char> = value.chars().collect();
|
||||
let tail: String = chars[chars.len().saturating_sub(4)..].iter().collect();
|
||||
format!("••••{tail}")
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn restrict(path: &Path) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600));
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
fn restrict(_: &Path) {}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn protect(plain: &[u8]) -> Result<Vec<u8>, SecretsError> {
|
||||
Ok(plain.to_vec())
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
fn unprotect(bytes: &[u8]) -> Result<Vec<u8>, SecretsError> {
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn dpapi(input: &[u8], encrypt: bool) -> Result<Vec<u8>, SecretsError> {
|
||||
use windows_sys::Win32::Foundation::LocalFree;
|
||||
use windows_sys::Win32::Security::Cryptography::{
|
||||
CryptProtectData, CryptUnprotectData, CRYPTPROTECT_UI_FORBIDDEN, CRYPT_INTEGER_BLOB,
|
||||
};
|
||||
let data_in = CRYPT_INTEGER_BLOB { cbData: input.len() as u32, pbData: input.as_ptr() as *mut u8 };
|
||||
let mut out = CRYPT_INTEGER_BLOB { cbData: 0, pbData: std::ptr::null_mut() };
|
||||
// SAFETY: the input blob points at `input` for the call; the output is freed with LocalFree.
|
||||
let ok = unsafe {
|
||||
if encrypt {
|
||||
CryptProtectData(&data_in, std::ptr::null(), std::ptr::null(), std::ptr::null(), std::ptr::null(),
|
||||
CRYPTPROTECT_UI_FORBIDDEN, &mut out)
|
||||
} else {
|
||||
CryptUnprotectData(&data_in, std::ptr::null_mut(), std::ptr::null(), std::ptr::null(), std::ptr::null(),
|
||||
CRYPTPROTECT_UI_FORBIDDEN, &mut out)
|
||||
}
|
||||
};
|
||||
if ok == 0 {
|
||||
return Err(SecretsError(if encrypt { "could not encrypt the secrets".into() } else {
|
||||
"could not decrypt the secrets (another Windows user or PC?)".into() }));
|
||||
}
|
||||
// SAFETY: DPAPI returned `cbData` bytes at `pbData`.
|
||||
let v = unsafe { std::slice::from_raw_parts(out.pbData, out.cbData as usize).to_vec() };
|
||||
unsafe { LocalFree(out.pbData as _) };
|
||||
Ok(v)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
fn protect(plain: &[u8]) -> Result<Vec<u8>, SecretsError> {
|
||||
dpapi(plain, true)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
fn unprotect(bytes: &[u8]) -> Result<Vec<u8>, SecretsError> {
|
||||
dpapi(bytes, false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
|
||||
#[test]
|
||||
fn secrets_round_trip_and_are_removed() {
|
||||
let s = Secrets::new(&temp_dir("sec").join("config").join("secrets.dat"));
|
||||
assert_eq!(s.get(DISCORD_TOKEN).unwrap(), None);
|
||||
s.set(DISCORD_TOKEN, "abc.def.ghij").unwrap();
|
||||
s.set("other", "x").unwrap();
|
||||
assert_eq!(s.get(DISCORD_TOKEN).unwrap().as_deref(), Some("abc.def.ghij"));
|
||||
let mut v = s.values();
|
||||
v.sort();
|
||||
assert_eq!(v, vec!["abc.def.ghij", "x"]);
|
||||
s.remove(DISCORD_TOKEN).unwrap();
|
||||
assert_eq!(s.get(DISCORD_TOKEN).unwrap(), None);
|
||||
assert_eq!(s.get("other").unwrap().as_deref(), Some("x"));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn the_file_is_readable_by_its_owner_only() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let f = temp_dir("secp").join("secrets.dat");
|
||||
Secrets::new(&f).set(DISCORD_TOKEN, "t").unwrap();
|
||||
assert_eq!(std::fs::metadata(&f).unwrap().permissions().mode() & 0o777, 0o600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_file_is_reported_not_overwritten() {
|
||||
let f = temp_dir("secd").join("secrets.dat");
|
||||
std::fs::write(&f, b"not json").unwrap();
|
||||
let s = Secrets::new(&f);
|
||||
assert!(s.get(DISCORD_TOKEN).is_err());
|
||||
assert!(s.set(DISCORD_TOKEN, "t").is_err());
|
||||
assert_eq!(std::fs::read(&f).unwrap(), b"not json");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn masking_shows_only_the_last_four() {
|
||||
assert_eq!(masked("MTIz.abcd.wxyz"), "••••wxyz");
|
||||
assert_eq!(masked("ab"), "••••ab");
|
||||
}
|
||||
}
|
||||
613
app/src/services.rs
Normal file
613
app/src/services.rs
Normal file
@@ -0,0 +1,613 @@
|
||||
//! Starts everything the app runs: the LLM (`strata.rs`), the voice worker process, the connection
|
||||
//! to it, and the Discord core. Used by the app binary (and the Tauri shell from M5).
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::conversation::{self, Conversation};
|
||||
use crate::discord::{CoreEvent, DiscordCore};
|
||||
use crate::settings::load_settings;
|
||||
use crate::strata::Llm;
|
||||
use crate::worker_client::{WorkerClient, WorkerClientOptions};
|
||||
use crate::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
|
||||
|
||||
pub struct Services {
|
||||
pub core: DiscordCore,
|
||||
pub events: mpsc::UnboundedReceiver<CoreEvent>,
|
||||
pub worker: WorkerClient,
|
||||
pub worker_process: WorkerProcess,
|
||||
/// The conversation's missed requests, for the dashboard.
|
||||
pub missed: conversation::MissedLog,
|
||||
pub llm: Arc<Llm>,
|
||||
pub browser: BrowserRun,
|
||||
/// Game detection while the services run (admin.spec.md, "Game detection").
|
||||
pub game: Arc<crate::game::GameWatch>,
|
||||
/// Moves the LLM between the performance stages as games come and go.
|
||||
pub perf: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
/// Fires the reminders and scheduled tasks (admin.spec.md, "Schedules").
|
||||
pub schedules: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
/// Announces new collected items (admin.spec.md, "News notifications").
|
||||
pub news: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
/// The bot browser of a run (admin.spec.md, "Bot browser").
|
||||
pub enum BrowserRun {
|
||||
/// Not started: the feature is off, or (with a note) it is not installed.
|
||||
Off(Option<String>),
|
||||
Failed(String),
|
||||
/// Running; the note says why it runs without the virtual monitor that was asked for.
|
||||
Running(Arc<crate::browser::BotBrowser>, Option<String>),
|
||||
}
|
||||
|
||||
impl BrowserRun {
|
||||
/// `{state, note}` for the dashboard.
|
||||
pub fn view(&self) -> Value {
|
||||
let (state, note) = match self {
|
||||
BrowserRun::Off(note) => ("off", note.clone()),
|
||||
BrowserRun::Failed(e) => ("failed", Some(e.clone())),
|
||||
BrowserRun::Running(b, note) if b.running() => ("ready", note.clone()),
|
||||
BrowserRun::Running(_, note) => ("exited", note.clone()),
|
||||
};
|
||||
serde_json::json!({"state": state, "note": note})
|
||||
}
|
||||
|
||||
/// Stop the browser, then take the virtual monitor off the desktop (its window first, so
|
||||
/// Windows never moves it onto a real monitor).
|
||||
pub async fn stop(&self) {
|
||||
if let BrowserRun::Running(b, _) = self {
|
||||
b.stop().await;
|
||||
crate::vdisplay::detach();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Services {
|
||||
/// Stop everything; an error means the LLM's server may still run.
|
||||
pub async fn stop(&self) -> Result<(), String> {
|
||||
let perf = self.perf.lock().unwrap().take();
|
||||
if let Some(p) = perf {
|
||||
p.abort();
|
||||
let _ = p.await;
|
||||
}
|
||||
let schedules = self.schedules.lock().unwrap().take();
|
||||
if let Some(s) = schedules {
|
||||
s.abort();
|
||||
let _ = s.await;
|
||||
}
|
||||
let news = self.news.lock().unwrap().take();
|
||||
if let Some(n) = news {
|
||||
n.abort();
|
||||
let _ = n.await;
|
||||
}
|
||||
self.game.stop();
|
||||
self.core.close().await;
|
||||
self.worker.close();
|
||||
self.worker_process.stop().await;
|
||||
self.browser.stop().await;
|
||||
self.llm.stop().await
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the bot browser when its feature is on and it is installed, on the virtual monitor when
|
||||
/// sharing is on; a failure leaves the services running without it (or without the monitor).
|
||||
async fn start_browser(data_dir: &Path, settings: &crate::settings::Settings) -> BrowserRun {
|
||||
// A monitor an earlier run left on goes off first.
|
||||
crate::vdisplay::detach();
|
||||
if !settings.features.browser {
|
||||
return BrowserRun::Off(None);
|
||||
}
|
||||
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: std::path::PathBuf::new() };
|
||||
if !crate::install::Installer::new(layout.clone(), crate::install::steps()).done("browser") {
|
||||
crate::warn_log!("browser", "the bot browser is not installed");
|
||||
return BrowserRun::Off(Some("설치되지 않음".into()));
|
||||
}
|
||||
let (exe, profile) = (layout.chrome_exe(), crate::browser::profile_dir(data_dir));
|
||||
let mut note = None;
|
||||
if settings.features.browser_share {
|
||||
let started = match crate::vdisplay::attach() {
|
||||
Ok(r) => crate::browser::BotBrowser::start(&exe, &profile, Some(r)).await,
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
match started {
|
||||
Ok(b) => return BrowserRun::Running(Arc::new(b), None),
|
||||
Err(e) => {
|
||||
crate::vdisplay::detach();
|
||||
crate::warn_log!("browser", "Warning: no virtual monitor for the bot browser: {e}");
|
||||
note = Some(format!("가상 모니터 없이 실행: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
match crate::browser::BotBrowser::start(&exe, &profile, None).await {
|
||||
Ok(b) => BrowserRun::Running(Arc::new(b), note),
|
||||
Err(e) => {
|
||||
crate::warn_log!("browser", "Warning: the bot browser did not start: {e}");
|
||||
BrowserRun::Failed(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the services; the LLM first (its first start downloads the model), so nothing answers
|
||||
/// before the brain can. A failure stops what already started.
|
||||
pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let settings = load_settings(&data_dir.join("config").join("settings.json"))?;
|
||||
// Game detection first: a game already running is not taken for other programs' memory.
|
||||
let game = Arc::new(crate::game::GameWatch::start(settings.game.clone()));
|
||||
llm.game_at_start(game.on() || game.in_background_at_start());
|
||||
let perf_settings = settings.clone();
|
||||
// The stored token (the setup wizard keeps it encrypted, admin.spec.md), else the environment.
|
||||
let secrets = crate::secrets::Secrets::new(&data_dir.join("config").join("secrets.dat"));
|
||||
let stored = secrets.get(crate::secrets::DISCORD_TOKEN)?;
|
||||
// The MCP servers' environment values stay encrypted on disk: they reach the brain only in the
|
||||
// worker's environment (admin.spec.md, "Features and MCP servers").
|
||||
let mcp_env = crate::features::mcp_env(&secrets, &settings.mcp_servers)?;
|
||||
let kma_key = secrets.get(crate::secrets::KMA_KEY)?;
|
||||
let token = match stored {
|
||||
Some(t) => t,
|
||||
None => std::env::var("JARVIS_DISCORD_TOKEN").map_err(|_| "no bot token: set it in the app or JARVIS_DISCORD_TOKEN")?,
|
||||
};
|
||||
|
||||
// What these services are given stays masked in the logs until the next start, also if it is
|
||||
// replaced or removed in the store meanwhile (admin.spec.md, "Logs").
|
||||
if let Some(hub) = crate::logs::installed() {
|
||||
hub.set_secrets_in_use(secrets_in_use(&token, &mcp_env, kma_key.as_deref()));
|
||||
}
|
||||
|
||||
let (base_url, model) = llm.start(&settings).await?;
|
||||
let brain_config = match crate::strata::write_brain_config(data_dir, &base_url, &model, &settings) {
|
||||
Ok(f) => f,
|
||||
Err(e) => return Err(after_llm_stop(&llm, format!("could not write the brain settings: {e}")).await.into()),
|
||||
};
|
||||
let browser = start_browser(data_dir, &settings).await;
|
||||
|
||||
let w = &settings.voice_worker;
|
||||
let mut wopts = WorkerProcessOptions::new(&w.python, &w.cwd, settings.worker_args());
|
||||
// With the installed runtime, the worker's models, caches and temp files stay in the data folder.
|
||||
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: w.cwd.clone() };
|
||||
if crate::install::Installer::new(layout.clone(), crate::install::steps()).done("settings") {
|
||||
wopts.env.extend(layout.env());
|
||||
}
|
||||
wopts.env.insert("JARVIS_CONFIG_PATH".into(), brain_config.to_string_lossy().into_owned());
|
||||
wopts.env.insert(crate::features::MCP_ENV_VAR.into(), mcp_env.to_string());
|
||||
if let Some(k) = &kma_key {
|
||||
wopts.env.insert(KMA_ENV_VAR.into(), k.clone());
|
||||
}
|
||||
if let BrowserRun::Running(b, _) = &browser {
|
||||
wopts.env.insert("JARVIS_BROWSER_CDP".into(), b.cdp_url());
|
||||
}
|
||||
if let Some(dir) = recordings_dir(data_dir, &settings) {
|
||||
wopts.env.insert(RECORDINGS_ENV_VAR.into(), dir.to_string_lossy().into_owned());
|
||||
}
|
||||
// Background collection (admin.spec.md): the enabled targets, and the keep days in any case.
|
||||
match crate::collect::write_config(data_dir, &settings) {
|
||||
Ok(f) => { wopts.env.insert(crate::collect::COLLECT_ENV_VAR.into(), f.to_string_lossy().into_owned()); }
|
||||
Err(e) => return Err(after_stop(&llm, &browser, format!("could not write the collection settings: {e}")).await.into()),
|
||||
}
|
||||
let worker_process = match WorkerProcess::new(wopts) {
|
||||
Ok(p) => p,
|
||||
Err(e) => return Err(after_stop(&llm, &browser, e.to_string()).await.into()),
|
||||
};
|
||||
let mut process_events = worker_process.subscribe();
|
||||
worker_process.start();
|
||||
let mut opts = WorkerClientOptions::new(worker_process.url(), worker_process.token.clone());
|
||||
opts.stt = settings.voice.stt.clone();
|
||||
opts.stt.insert("hotwords".into(), Value::from(settings.voice.wake_words.clone()));
|
||||
opts.memory = settings.memory.clone();
|
||||
let (worker, worker_events) = WorkerClient::new(opts);
|
||||
// The worker loads its models before it listens: connect once it says READY (after restarts the
|
||||
// client keeps retrying on its own).
|
||||
let w2 = worker.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match process_events.recv().await {
|
||||
Ok(ProcessEvent::Ready) => return w2.start(),
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {}
|
||||
Err(_) => return,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let (calls, response) = (settings.call.clone(), settings.response.clone());
|
||||
let chat_calls = conversation::ChatCalls::from_settings(&settings);
|
||||
let schedule_channel = settings.schedule_channel;
|
||||
let news_settings = settings.clone();
|
||||
let (core, core_events) = DiscordCore::new(settings, token, worker.clone(), worker_events);
|
||||
let schedule_store = Arc::new(crate::schedule::Store::new(data_dir));
|
||||
core.set_schedules(schedule_store.clone());
|
||||
let activity = Arc::new(crate::activity::Activity::new(data_dir));
|
||||
core.set_activity(activity.clone());
|
||||
core.set_notes(Arc::new(crate::notes::Notes::new(data_dir)));
|
||||
// Call detection and the response policy answer what is heard; every event still reaches `events`.
|
||||
let (out, events) = mpsc::unbounded_channel();
|
||||
let conv = Conversation::new(core.clone(), calls, response).with_chat(chat_calls);
|
||||
let missed = conv.missed_handle();
|
||||
tokio::spawn(conversation::run(conv, core_events, out));
|
||||
if let Err(e) = core.start().await {
|
||||
worker.close();
|
||||
worker_process.stop().await;
|
||||
return Err(after_stop(&llm, &browser, e.to_string()).await.into());
|
||||
}
|
||||
crate::info_log!("app", "services started");
|
||||
let schedules = std::sync::Mutex::new(Some(tokio::spawn(crate::schedule::run(schedule_store, activity.clone(),
|
||||
Arc::new(core.clone()), schedule_channel))));
|
||||
let news = std::sync::Mutex::new(Some(tokio::spawn(crate::collect_notify::run(data_dir.to_path_buf(), news_settings,
|
||||
Arc::new(core.clone()), activity))));
|
||||
let on = { let g = game.clone(); Arc::new(move || g.on()) };
|
||||
let perf = std::sync::Mutex::new(Some(tokio::spawn(perf_loop(perf_settings, llm.clone(), Arc::new(worker.clone()), on, RETRY_AFTER))));
|
||||
Ok(Services { core, events, worker, worker_process, missed, llm, browser, game, perf, schedules, news })
|
||||
}
|
||||
|
||||
/// The voice worker writes its recordings where this names (voice_worker.spec.md, "Recordings").
|
||||
pub const RECORDINGS_ENV_VAR: &str = "JARVIS_RECORDINGS_DIR";
|
||||
/// The stored KMA key reaches the brain's weather tool only in the worker's environment.
|
||||
pub const KMA_ENV_VAR: &str = "JARVIS_KMA_API_KEY";
|
||||
|
||||
/// Where the voice worker keeps its recordings, when `voice.record` is on.
|
||||
pub fn recordings_dir(data_dir: &Path, settings: &crate::settings::Settings) -> Option<std::path::PathBuf> {
|
||||
settings.voice.record.then(|| data_dir.join("recordings"))
|
||||
}
|
||||
|
||||
/// The brain as the performance stages see it: told the LLM's state, and asked to finish what it is
|
||||
/// answering (the worker client; a fake in tests).
|
||||
#[async_trait::async_trait]
|
||||
pub trait LlmBrain: Send + Sync {
|
||||
/// Announce a switch; its number for `wait_llm_idle`.
|
||||
fn llm_switching(&self, notice: Option<&str>) -> u64;
|
||||
/// `ready` (with whether a game runs) or `paused`.
|
||||
fn llm_state(&self, state: &str, game: bool) -> bool;
|
||||
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool;
|
||||
/// What the worker's loaded models (STT, TTS) hold on the graphics card, in MiB.
|
||||
fn worker_models_mib(&self) -> i64;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LlmBrain for WorkerClient {
|
||||
fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
||||
WorkerClient::llm_switching(self, notice)
|
||||
}
|
||||
fn llm_state(&self, state: &str, game: bool) -> bool {
|
||||
if state == "ready" { WorkerClient::llm_ready(self, game) } else { WorkerClient::llm_state(self, state) }
|
||||
}
|
||||
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool {
|
||||
WorkerClient::wait_llm_idle(self, id, t).await
|
||||
}
|
||||
fn worker_models_mib(&self) -> i64 {
|
||||
self.hello().map_or(0, |h| crate::placement::worker_models_mib(h.stt_ready, h.tts_ready))
|
||||
}
|
||||
}
|
||||
|
||||
/// What a request that waits for a switch hears once (voice_worker.spec.md, "LLM switches").
|
||||
fn switch_notice(stage: crate::perf::Stage) -> Option<&'static str> {
|
||||
match stage {
|
||||
crate::perf::Stage::Gaming => Some("잠깐만요, 게임 모드로 바꾸는 중이에요"),
|
||||
crate::perf::Stage::Normal => Some("잠깐만요, 평소 모드로 돌아가는 중이에요"),
|
||||
crate::perf::Stage::Paused => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// How long after a failed switch the stage is tried again (`perf::Retry`).
|
||||
const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
/// The performance stages (admin.spec.md, "Performance stages"): every second the stage wanted
|
||||
/// from game mode (`perf.rs`); a change tells the brain the LLM switches, waits until the answers
|
||||
/// in progress are written (at most 2 minutes), restarts the LLM in that stage (or stops it to
|
||||
/// pause) and tells the brain it is ready (or paused). A restart that fails brings back the stage
|
||||
/// that ran before; the failed one is tried again after `retry_after`, three times at most. With
|
||||
/// `llm.url` nothing moves.
|
||||
async fn perf_loop(settings: crate::settings::Settings, llm: Arc<Llm>, brain: Arc<dyn LlmBrain>,
|
||||
game: Arc<dyn Fn() -> bool + Send + Sync>, retry_after: std::time::Duration) {
|
||||
use crate::perf::{Perf, Retry, Stage};
|
||||
if settings.llm.url.is_some() {
|
||||
return;
|
||||
}
|
||||
let mut perf = Perf::new(std::time::Duration::from_secs(settings.game.revert_seconds.into()), settings.game.pause_in_game);
|
||||
let mut retry = Retry::new(retry_after);
|
||||
// The stage that runs (a game stage that had to pause counts as the game stage, so it is not
|
||||
// tried again while the game runs).
|
||||
let mut actual = Stage::Normal;
|
||||
let mut last_want = Stage::Normal;
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
let now = std::time::Instant::now();
|
||||
let want = perf.target(game(), now);
|
||||
// Another stage wanted (also back to the one that runs): earlier failures no longer count.
|
||||
if want != last_want {
|
||||
retry.succeeded();
|
||||
last_want = want;
|
||||
}
|
||||
if want == actual || !retry.allowed(want, now) {
|
||||
continue;
|
||||
}
|
||||
crate::info_log!("perf", "performance stage: {want:?}");
|
||||
let id = brain.llm_switching(switch_notice(want));
|
||||
if !brain.wait_llm_idle(id, std::time::Duration::from_secs(120)).await {
|
||||
crate::warn_log!("perf", "Warning: the brain did not finish its answers in 2 minutes; switching anyway");
|
||||
}
|
||||
let ended = match llm.switch(&settings, want, brain.worker_models_mib(), game()).await {
|
||||
Ok(ended) => {
|
||||
retry.succeeded();
|
||||
actual = want;
|
||||
ended
|
||||
}
|
||||
Err(e) => {
|
||||
crate::error_log!("perf", "the LLM did not start in the {want:?} stage: {e}; going back to {actual:?}");
|
||||
let back = match llm.switch(&settings, actual, brain.worker_models_mib(), game()).await {
|
||||
Ok(ended) => ended,
|
||||
Err(e) => {
|
||||
crate::error_log!("perf", "the LLM did not come back in the {actual:?} stage either: {e}");
|
||||
// Nothing answers: requests fail at once instead of waiting.
|
||||
actual = Stage::Paused;
|
||||
Stage::Paused
|
||||
}
|
||||
};
|
||||
// The wait counts from when that is over, not from when the switch began.
|
||||
retry.failed(want, std::time::Instant::now());
|
||||
back
|
||||
}
|
||||
};
|
||||
// In the game stage partial passes wait while an answer is written (M0 gamemode_tune).
|
||||
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }, ended == Stage::Gaming);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// The secret values the services get: the bot token and the MCP servers' environment values.
|
||||
pub fn secrets_in_use(token: &str, mcp_env: &Value, kma_key: Option<&str>) -> Vec<String> {
|
||||
let mut v = vec![token.to_owned()];
|
||||
v.extend(kma_key.map(str::to_owned));
|
||||
for vars in mcp_env.as_object().into_iter().flat_map(|m| m.values()) {
|
||||
v.extend(vars.as_object().into_iter().flat_map(|m| m.values()).filter_map(Value::as_str).map(str::to_owned));
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// Stop the bot browser and the LLM after a failed start.
|
||||
async fn after_stop(llm: &Llm, browser: &BrowserRun, error: String) -> String {
|
||||
browser.stop().await;
|
||||
after_llm_stop(llm, error).await
|
||||
}
|
||||
|
||||
/// Stop the LLM after a failed start; the error also says if its server may still run.
|
||||
async fn after_llm_stop(llm: &Llm, error: String) -> String {
|
||||
match llm.stop().await {
|
||||
Ok(()) => error,
|
||||
Err(stop) => format!("{error} ({stop})"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod perf_tests {
|
||||
use super::*;
|
||||
use crate::strata::tests::{card, fake_placed, launches, layout, server_after_launch};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeBrain(Mutex<Vec<String>>);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LlmBrain for FakeBrain {
|
||||
fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
||||
self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default()));
|
||||
1
|
||||
}
|
||||
fn llm_state(&self, state: &str, game: bool) -> bool {
|
||||
self.0.lock().unwrap().push(if game { format!("{state} (game)") } else { state.into() });
|
||||
true
|
||||
}
|
||||
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
|
||||
self.0.lock().unwrap().push("waited".into());
|
||||
true
|
||||
}
|
||||
fn worker_models_mib(&self) -> i64 {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_game_moves_the_llm_to_the_game_stage_and_its_end_moves_it_back() {
|
||||
let l = layout("perf-loop");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)));
|
||||
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
let brain = Arc::new(FakeBrain::default());
|
||||
let game = Arc::new(AtomicBool::new(false));
|
||||
let g = game.clone();
|
||||
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), RETRY_AFTER));
|
||||
std::fs::remove_file(l.data.join("calls")).unwrap();
|
||||
game.store(true, Ordering::SeqCst);
|
||||
let until = |want: &'static str| {
|
||||
let brain = brain.clone();
|
||||
async move {
|
||||
for _ in 0..200 {
|
||||
if brain.0.lock().unwrap().iter().filter(|m| m.as_str() == want).count() > 0 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
panic!("never {want}: {:?}", brain.0.lock().unwrap());
|
||||
}
|
||||
};
|
||||
until("ready (game)").await;
|
||||
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready (game)"]);
|
||||
assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l));
|
||||
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
|
||||
// The game ends (no wait set): back to the normal stage.
|
||||
brain.0.lock().unwrap().clear();
|
||||
game.store(false, Ordering::SeqCst);
|
||||
until("ready").await;
|
||||
assert_eq!(brain.0.lock().unwrap()[0], "switching: 잠깐만요, 평소 모드로 돌아가는 중이에요");
|
||||
assert!(launches(&l).last().unwrap().contains("--context 32768"));
|
||||
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
|
||||
task.abort();
|
||||
}
|
||||
|
||||
/// A Strata that cannot start at the game's context (taking `fail_secs` to fail).
|
||||
#[cfg(unix)]
|
||||
fn strata_failing_in_games(l: &crate::install::Layout, fail_secs: f64) -> std::path::PathBuf {
|
||||
let log = l.strata_home().join("x.engine.log");
|
||||
crate::strata::tests::fake_strata(l, &format!(
|
||||
"case \"$*\" in *'--context 16384'*) sleep {fail_secs}; echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'",
|
||||
l.data.join("gone").display(), log.display()))
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_new_game_tries_the_game_stage_again_after_three_failures_with_the_last_one() {
|
||||
let l = layout("perf-loop-new-game");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), strata_failing_in_games(&l, 0.0)).with_gpu(card(&l, "GPU-test", 0)));
|
||||
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
let brain = Arc::new(FakeBrain::default());
|
||||
let game = Arc::new(AtomicBool::new(true));
|
||||
let g = game.clone();
|
||||
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), Duration::from_millis(200)));
|
||||
let switchings = |b: &FakeBrain| b.0.lock().unwrap().iter().filter(|m| m.starts_with("switching")).count();
|
||||
tokio::time::sleep(Duration::from_secs(6)).await;
|
||||
assert_eq!(switchings(&brain), 3, "three tries");
|
||||
// The game ends (the normal stage runs already) and a new one starts: tried again.
|
||||
game.store(false, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
game.store(true, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
assert!(switchings(&brain) > 3, "{:?}", brain.0.lock().unwrap());
|
||||
task.abort();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn the_wait_before_the_next_try_counts_from_when_the_failed_switch_is_over() {
|
||||
// The failing start takes 1.5 s, longer than the 1 s wait: the next try still waits 1 s
|
||||
// after the normal stage came back.
|
||||
let l = layout("perf-loop-wait");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), strata_failing_in_games(&l, 1.5)).with_gpu(card(&l, "GPU-test", 0)));
|
||||
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
let brain = Arc::new(TimedBrain::default());
|
||||
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_secs(1)));
|
||||
tokio::time::sleep(Duration::from_secs(7)).await;
|
||||
task.abort();
|
||||
let got = brain.0.lock().unwrap().clone();
|
||||
let at = |what: &str, n: usize| got.iter().filter(|(m, _)| m.starts_with(what)).nth(n).map(|(_, t)| *t).expect(what);
|
||||
let gap = at("switching", 1).duration_since(at("ready", 0));
|
||||
assert!(gap >= Duration::from_secs(1), "the next try came {gap:?} after the normal stage was back");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn going_back_to_normal_while_the_game_runs_keeps_the_reading_from_before_the_game() {
|
||||
// 24 GB card; other programs `OTHERS` at the start. A game (2500 MiB) starts; the first
|
||||
// game start fails and the normal stage comes back while the game still runs; the second
|
||||
// try works. It must count other programs from the reading before the game, not one that
|
||||
// has the game in it.
|
||||
let l = layout("perf-loop-keep-reading");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let log = l.strata_home().join("x.engine.log");
|
||||
let failed_once = l.data.join("failed-once");
|
||||
let exe = crate::strata::tests::fake_strata(&l, &format!(
|
||||
"case \"$*\" in *'--context 16384'*'expert-vram=0'*) ;; *'--context 16384'*) if [ ! -e '{fo}' ]; then touch '{fo}'; exit 4; fi;; esac; rm -f '{g}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{log}'; echo 'Strata is serving.'",
|
||||
fo = failed_once.display(), g = l.data.join("gone").display(), log = log.display()));
|
||||
let others = Arc::new(std::sync::atomic::AtomicI64::new(crate::strata::tests::OTHERS));
|
||||
let calls = l.data.join("calls");
|
||||
let o2 = others.clone();
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(Arc::new(move || {
|
||||
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
Some(crate::strata::tests::gpu("GPU-test", o2.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, 24576))
|
||||
})));
|
||||
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
others.store(crate::strata::tests::OTHERS + 2500, Ordering::SeqCst);
|
||||
let brain = Arc::new(FakeBrain::default());
|
||||
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300)));
|
||||
for _ in 0..100 {
|
||||
if llm.stage_view()["stage"] == serde_json::json!("gaming") { break; }
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
task.abort();
|
||||
assert!(failed_once.exists(), "the first game start failed");
|
||||
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
|
||||
let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 };
|
||||
let right = crate::placement::game(24576, crate::strata::tests::OTHERS, &probe16, 8192).expert_gib;
|
||||
let wrong = crate::placement::game(24576, crate::strata::tests::OTHERS + 2500, &probe16, 8192).expert_gib;
|
||||
assert_ne!(right, wrong);
|
||||
assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={right} ")), "{:?}", launches(&l));
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TimedBrain(Mutex<Vec<(String, std::time::Instant)>>);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LlmBrain for TimedBrain {
|
||||
fn llm_switching(&self, _notice: Option<&str>) -> u64 {
|
||||
self.0.lock().unwrap().push(("switching".into(), std::time::Instant::now()));
|
||||
1
|
||||
}
|
||||
fn llm_state(&self, state: &str, _game: bool) -> bool {
|
||||
self.0.lock().unwrap().push((state.into(), std::time::Instant::now()));
|
||||
true
|
||||
}
|
||||
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
|
||||
true
|
||||
}
|
||||
fn worker_models_mib(&self) -> i64 {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_game_stage_that_fails_to_start_goes_back_and_is_tried_three_times() {
|
||||
let l = layout("perf-loop-fail");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
// This Strata cannot start at the game's context.
|
||||
let log = l.strata_home().join("x.engine.log");
|
||||
let exe = crate::strata::tests::fake_strata(&l, &format!(
|
||||
"case \"$*\" in *'--context 16384'*) echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'",
|
||||
l.data.join("gone").display(), log.display()));
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0)));
|
||||
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
let brain = Arc::new(FakeBrain::default());
|
||||
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300)));
|
||||
tokio::time::sleep(Duration::from_secs(8)).await;
|
||||
let got = brain.0.lock().unwrap().clone();
|
||||
// Each try: switching, waited, the game stage fails, the normal stage comes back, ready.
|
||||
assert_eq!(got.iter().filter(|m| m.starts_with("switching")).count(), 3, "{got:?}");
|
||||
assert_eq!(got.iter().filter(|m| m.as_str() == "ready").count(), 3, "{got:?}");
|
||||
assert!(!got.iter().any(|m| m == "paused"));
|
||||
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
|
||||
assert_eq!(llm.status().state, crate::strata::LlmState::Ready, "the normal stage runs again");
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_services_secrets_are_the_token_every_mcp_value_and_the_kma_key() {
|
||||
let env = serde_json::json!({"brave": {"BRAVE_API_KEY": "k1"}, "other": {"A": "k2", "B": "k3"}});
|
||||
let mut v = secrets_in_use("tok", &env, Some("kma"));
|
||||
v.sort();
|
||||
assert_eq!(v, vec!["k1", "k2", "k3", "kma", "tok"]);
|
||||
assert_eq!(secrets_in_use("tok", &env, None).len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_worker_gets_a_recordings_folder_in_the_data_folder_only_with_voice_record_on() {
|
||||
let off = crate::settings::parse_settings(&serde_json::json!({})).unwrap();
|
||||
assert_eq!(recordings_dir(Path::new("/d"), &off), None);
|
||||
let on = crate::settings::parse_settings(&serde_json::json!({"voice": {"record": true}})).unwrap();
|
||||
assert_eq!(recordings_dir(Path::new("/d"), &on), Some(Path::new("/d").join("recordings")));
|
||||
}
|
||||
}
|
||||
1181
app/src/settings.rs
Normal file
1181
app/src/settings.rs
Normal file
File diff suppressed because it is too large
Load Diff
387
app/src/settings_edit.rs
Normal file
387
app/src/settings_edit.rs
Normal file
@@ -0,0 +1,387 @@
|
||||
//! Editing `config/settings.json` for the settings screens (`app/src/admin.spec.md`).
|
||||
//!
|
||||
//! The screens are built from `fields()`. `apply` changes only the keys it is given, writes only
|
||||
//! values that differ from the default, keeps keys the app does not know, and writes nothing unless
|
||||
//! the whole result passes the app's own check (`parse_settings`).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use crate::datadir::write_atomic;
|
||||
use crate::settings::{parse_settings, DEFAULT_JOIN_NOTICE};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Screen {
|
||||
Bot,
|
||||
Channels,
|
||||
Chat,
|
||||
Memory,
|
||||
Model,
|
||||
Features,
|
||||
Collect,
|
||||
Game,
|
||||
Schedule,
|
||||
PcControl,
|
||||
App,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
|
||||
#[serde(tag = "type", rename_all = "camelCase")]
|
||||
pub enum Kind {
|
||||
Bool,
|
||||
Int { min: i64, max: i64 },
|
||||
Number { min: f64, max: f64 },
|
||||
/// `nullable`: null is allowed (it means "not set").
|
||||
NullableNumber { min_exclusive: f64 },
|
||||
Text { max: usize, non_empty: bool },
|
||||
/// `nullable`: null means "use another setting" (chat wake words: the voice ones).
|
||||
TextList { max_items: usize, nullable: bool },
|
||||
Choice { values: &'static [&'static str] },
|
||||
/// A language code (2-8 letters), or null for "detect it".
|
||||
LanguageCode,
|
||||
/// A Discord ID as a JSON string, or null.
|
||||
Id,
|
||||
IdList,
|
||||
/// Discord user ID -> integer (response priorities).
|
||||
IdNumberMap,
|
||||
/// The registered chat channels (their own rules are checked by `parse_settings`).
|
||||
ChatChannels,
|
||||
/// The MCP servers (their own rules are checked by `parse_settings`; values are secrets).
|
||||
McpServers,
|
||||
/// The background collection targets (their own rules are checked by `parse_settings`).
|
||||
CollectTargets,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
|
||||
pub struct Field {
|
||||
pub path: &'static str,
|
||||
pub screen: Screen,
|
||||
#[serde(flatten)]
|
||||
pub kind: Kind,
|
||||
pub default: Value,
|
||||
pub label: &'static str,
|
||||
}
|
||||
|
||||
fn f(path: &'static str, screen: Screen, kind: Kind, default: Value, label: &'static str) -> Field {
|
||||
Field { path, screen, kind, default, label }
|
||||
}
|
||||
|
||||
/// The levels of a PC control action (`settings::Level`).
|
||||
const LEVELS: &[&str] = &["auto", "ask", "never"];
|
||||
|
||||
/// Every key the screens edit, in screen order.
|
||||
pub fn fields() -> Vec<Field> {
|
||||
use Kind::*;
|
||||
use Screen::*;
|
||||
vec![
|
||||
f("discord.commandName", Bot, Text { max: 32, non_empty: true }, json!("자비스"), "슬래시 명령 이름"),
|
||||
f("discord.ephemeralReplies", Bot, Bool, json!(true), "명령 응답은 부른 사람만 보기"),
|
||||
f("discord.guildId", Channels, Id, Value::Null, "서버"),
|
||||
f("discord.voiceChannelId", Channels, Id, Value::Null, "자동으로 들어갈 음성 채널"),
|
||||
f("voice.maxSpeakers", Channels, Int { min: 1, max: 10 }, json!(10), "동시에 받아 적을 최대 인원"),
|
||||
f("voice.listenToBots", Channels, Bool, json!(false), "다른 봇의 말도 받아 적기"),
|
||||
f("voice.wakeWords", Channels, TextList { max_items: 20, nullable: false }, json!(["자비스"]), "호출어"),
|
||||
f("call.mode", Channels, Choice { values: &["wakeWord", "addressed", "everything"] }, json!("wakeWord"),
|
||||
"호출 방식"),
|
||||
f("call.similarity", Channels, Number { min: 0.0, max: 0.5 }, json!(0.2), "비슷한 발음 허용 정도"),
|
||||
f("call.designated", Channels, IdList, json!([]), "지정한 사람"),
|
||||
f("call.designatedOnly", Channels, Bool, json!(false), "지정한 사람에게만 응답"),
|
||||
f("call.designatedNoWakeWord", Channels, Bool, json!(false), "지정한 사람은 호출어 없이 대답"),
|
||||
f("call.stopWords", Channels, TextList { max_items: 20, nullable: false },
|
||||
json!(["멈춰", "그만", "정지", "stop"]), "정지 명령"),
|
||||
f("call.followUpMs", Channels, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.earcon", Channels, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Channels, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
json!("latest"), "동시 대화 응답 정책"),
|
||||
f("response.queueMax", Channels, Int { min: 1, max: 20 }, json!(5), "기다리는 요청 최대 개수"),
|
||||
f("response.collectMs", Channels, Int { min: 0, max: 3000 }, json!(700), "중요도 비교 대기(ms)"),
|
||||
f("response.settleMs", Channels, Int { min: 0, max: 3000 }, json!(800), "먼저 끝난 말 전사 대기(ms)"),
|
||||
f("response.priorities", Channels, IdNumberMap, json!({}), "사람별 우선순위"),
|
||||
f("response.interrupt", Channels, Bool, json!(true),
|
||||
"말하는 중에 새 요청이 오면 하던 말 멈추기(가장 최근 것, 중요한 것 먼저)"),
|
||||
f("voice.joinNotice", Channels, Text { max: 2000, non_empty: true }, json!(DEFAULT_JOIN_NOTICE),
|
||||
"입장 시 전사 고지 문구"),
|
||||
f("chatChannels", Chat, ChatChannels, json!([]), "채팅 채널"),
|
||||
f("call.chatWakeWords", Chat, TextList { max_items: 20, nullable: true }, Value::Null,
|
||||
"채팅 호출어(비우면 음성과 같음)"),
|
||||
// The worker's memory settings and their defaults (voice_worker.spec.md, "Memory settings").
|
||||
f("memory.chat", Memory, Bool, json!(true), "등록한 채팅 채널 참고"),
|
||||
f("memory.others", Memory, Bool, json!(true), "같은 음성 채널의 다른 사람 말 참고"),
|
||||
f("memory.off", Memory, Bool, json!(false), "지난 대화 참고 끄기"),
|
||||
f("memory.since_days", Memory, NullableNumber { min_exclusive: 0.0 }, Value::Null,
|
||||
"참고할 기간(일, 비우면 전체)"),
|
||||
f("memory.recent_minutes", Memory, Number { min: 0.0, max: 1440.0 }, json!(30), "최근 대화 원문 범위(분)"),
|
||||
f("memory.recent_records", Memory, Int { min: 0, max: 200 }, json!(20), "최근 대화 원문 개수"),
|
||||
f("memory.matches", Memory, Int { min: 0, max: 20 }, json!(5), "과거 기록 가져올 개수"),
|
||||
f("memory.budget_tokens", Memory, Number { min: 0.0, max: 100000.0 }, json!(4000), "참고 분량 상한(토큰)"),
|
||||
f("voice.record", Memory, Bool, json!(false),
|
||||
"목소리 녹음 저장(사람 말과 자비스 말, 분석용, 7일 보관, 다음 시작부터)"),
|
||||
// The worker's STT settings and their defaults (voice_worker.spec.md, "STT").
|
||||
f("voice.stt.end_silence_ms", Model, Int { min: 100, max: 2000 }, json!(300), "말 끝 판정 시간(ms)"),
|
||||
f("voice.stt.partials", Model, Bool, json!(true), "중간 전사"),
|
||||
f("voice.stt.partial_interval_ms", Model, Int { min: 100, max: 5000 }, json!(400), "중간 전사 간격(ms)"),
|
||||
f("voice.stt.final_beam", Model, Int { min: 1, max: 5 }, json!(2), "최종 전사 정확도(빔)"),
|
||||
f("voice.stt.language", Model, LanguageCode, json!("ko"), "받아 적을 언어(비우면 자동)"),
|
||||
// MeloTTS's Korean model has one voice (speaker "KR"), so only the speed is a choice.
|
||||
f("voice.tts.speed", Model, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(1.0 보통, 기본 1.5, 저장하면 바로 적용)"),
|
||||
// The LLM (admin.spec.md, "LLM"); `llm.url` (a server the app does not start) is file-only.
|
||||
f("llm.context", Model, Int { min: 4096, max: 262144 }, json!(32768), "대화 모델이 한 번에 기억하는 양(토큰)"),
|
||||
f("brain.location", Model, Text { max: 100, non_empty: false }, json!(crate::settings::DEFAULT_LOCATION),
|
||||
"기본 위치(날씨, 검색, 추천의 기준, 비우면 자동 감지)"),
|
||||
f("llm.port", Model, Int { min: 1024, max: 65535 }, json!(8080), "대화 모델 포트"),
|
||||
// The brain's features (admin.spec.md, "Features and MCP servers").
|
||||
f("features.weather", Features, Bool, json!(true), "날씨"),
|
||||
f("features.webSearch", Features, Bool, json!(true), "웹 검색"),
|
||||
f("features.meals", Features, Bool, json!(true), "식단 기록"),
|
||||
f("features.collect", Features, Bool, json!(true), "백그라운드 수집"),
|
||||
f("features.browser", Features, Bool, json!(true), "브라우저 제어(봇용 크롬)"),
|
||||
f("features.browserShare", Features, Bool, json!(false), "봇용 크롬창 화면공유 선택 표시(가상 모니터, Windows)"),
|
||||
f("features.schedule", Features, Bool, json!(true), "예약 알림과 정기 작업"),
|
||||
f("features.notes", Features, Bool, json!(true), "사람별 지침"),
|
||||
f("features.screen", Features, Bool, json!(true), "화면 보기(Windows, PC 제어에 등록한 사람만)"),
|
||||
f("mcpServers", Features, McpServers, json!([]), "MCP 서버"),
|
||||
// Background collection (admin.spec.md, "Background collection"): no targets by default.
|
||||
f("collect.targets", Collect, CollectTargets, json!([]), "수집 대상"),
|
||||
f("collect.keepDays", Collect, Int { min: 1, max: 365 }, json!(30), "보관 기간(일)"),
|
||||
f("collect.notifyChannel", Collect, Id, Value::Null, "새 글을 알릴 채팅 채널(등록한 채널, 비우면 알리지 않음)"),
|
||||
f("collect.digestHour", Collect, Int { min: 0, max: 23 }, json!(9), "하루 요약을 보낼 시각(시)"),
|
||||
// Schedules (admin.spec.md, "Schedules").
|
||||
f("schedule.chatChannel", Schedule, Id, Value::Null,
|
||||
"음성으로 맡긴 알림을 그 사람이 음성 채널에 없을 때 보낼 채팅 채널(등록한 채널)"),
|
||||
// PC control permissions (admin.spec.md, "PC control permissions").
|
||||
f("pcControl.allowedUsers", PcControl, IdList, json!([]), "PC 제어를 시킬 수 있는 사람(디스코드 사용자 ID)"),
|
||||
f("pcControl.view", PcControl, Choice { values: LEVELS }, json!("auto"), "화면 보기, 창 목록"),
|
||||
f("pcControl.click", PcControl, Choice { values: LEVELS }, json!("ask"), "누르기(버튼, 메뉴, 화면의 한 점)"),
|
||||
f("pcControl.type", PcControl, Choice { values: LEVELS }, json!("ask"), "글자·키 입력"),
|
||||
f("pcControl.run", PcControl, Choice { values: LEVELS }, json!("ask"), "프로그램 실행"),
|
||||
f("pcControl.screenExclude", PcControl, TextList { max_items: 100, nullable: false },
|
||||
json!(["1Password", "Bitwarden", "KeePass", "KeePassXC"]), "화면 보기에서 뺄 프로그램(파일 이름, 은행·비밀번호 관리자 등)"),
|
||||
// Game detection (admin.spec.md, "Game detection").
|
||||
f("game.detect", Game, Bool, json!(true), "게임 감지"),
|
||||
f("game.games", Game, TextList { max_items: 100, nullable: false }, json!([]), "게임으로 볼 프로그램(파일 이름, 예: MyGame.exe)"),
|
||||
f("game.otherRoomGb", Game, Number { min: 0.0, max: 8.0 }, json!(1.0), "평소 다른 프로그램용으로 남길 VRAM(GB)"),
|
||||
f("game.reserveGb", Game, Number { min: 0.0, max: 32.0 }, json!(8.0), "게임 중 게임용으로 비울 VRAM(GB)"),
|
||||
f("game.revertSeconds", Game, Int { min: 0, max: 600 }, json!(30), "게임이 끝난 뒤 평소 배치로 돌아가기까지(초)"),
|
||||
f("game.pauseInGame", Game, Bool, json!(false), "게임 중 일시정지(대답하지 않음)"),
|
||||
f("game.notGames", Game, TextList { max_items: 100, nullable: false }, json!([]), "게임이 아닌 프로그램(파일 이름)"),
|
||||
// The window (admin.spec.md, "Window and Windows build"): read when the close button is pressed.
|
||||
f("app.closeButton", App, Choice { values: &["quit", "tray"] }, json!("quit"), "창 닫기(X) 버튼을 누르면"),
|
||||
]
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct EditError(pub String);
|
||||
|
||||
impl std::fmt::Display for EditError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for EditError {}
|
||||
|
||||
fn load_raw(file: &Path) -> Result<Map<String, Value>, EditError> {
|
||||
let text = match std::fs::read_to_string(file) {
|
||||
Ok(t) => t,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Map::new()),
|
||||
Err(e) => return Err(EditError(format!("could not read {}: {e}", file.display()))),
|
||||
};
|
||||
match serde_json::from_str(text.trim_start_matches('\u{feff}')) {
|
||||
Ok(Value::Object(m)) => Ok(m),
|
||||
Ok(_) => Err(EditError(format!("{} is not a JSON object", file.display()))),
|
||||
Err(e) => Err(EditError(format!("{} is not valid JSON: {e}", file.display()))),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_path<'a>(raw: &'a Map<String, Value>, path: &str) -> Option<&'a Value> {
|
||||
let mut parts = path.split('.');
|
||||
let mut cur = raw.get(parts.next()?)?;
|
||||
for p in parts {
|
||||
cur = cur.as_object()?.get(p)?;
|
||||
}
|
||||
Some(cur)
|
||||
}
|
||||
|
||||
fn set_path(raw: &mut Map<String, Value>, path: &str, value: Value) -> Result<(), EditError> {
|
||||
let parts: Vec<&str> = path.split('.').collect();
|
||||
let mut cur = raw;
|
||||
for p in &parts[..parts.len() - 1] {
|
||||
let entry = cur.entry(p.to_string()).or_insert_with(|| Value::Object(Map::new()));
|
||||
cur = entry.as_object_mut().ok_or_else(|| EditError(format!("{p} in the file is not an object")))?;
|
||||
}
|
||||
cur.insert(parts[parts.len() - 1].into(), value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove `path`, then every parent object it leaves empty.
|
||||
fn remove_path(raw: &mut Map<String, Value>, path: &str) {
|
||||
fn go(m: &mut Map<String, Value>, parts: &[&str]) {
|
||||
if parts.len() == 1 {
|
||||
m.remove(parts[0]);
|
||||
return;
|
||||
}
|
||||
if let Some(Value::Object(child)) = m.get_mut(parts[0]) {
|
||||
go(child, &parts[1..]);
|
||||
if child.is_empty() {
|
||||
m.remove(parts[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
go(raw, &path.split('.').collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
fn is_id(v: &Value) -> bool {
|
||||
v.as_str().is_some_and(|s| (17..=20).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_digit()))
|
||||
}
|
||||
|
||||
fn check(field: &Field, v: &Value) -> Result<(), String> {
|
||||
let ok = match &field.kind {
|
||||
Kind::Bool => v.is_boolean(),
|
||||
Kind::Int { min, max } => v.as_i64().is_some_and(|n| (*min..=*max).contains(&n)) && !v.is_f64(),
|
||||
Kind::Number { min, max } => v.as_f64().is_some_and(|n| (*min..=*max).contains(&n)),
|
||||
Kind::NullableNumber { min_exclusive } => v.is_null() || v.as_f64().is_some_and(|n| n > *min_exclusive),
|
||||
Kind::Text { max, non_empty } => v.as_str().is_some_and(|s| {
|
||||
let n = s.trim().chars().count();
|
||||
n <= *max && (!non_empty || n > 0)
|
||||
}),
|
||||
Kind::TextList { max_items, nullable } => (*nullable && v.is_null())
|
||||
|| v.as_array().is_some_and(|a| a.len() <= *max_items && a.iter().all(|w| w.as_str().is_some_and(|s| {
|
||||
let n = s.trim().chars().count();
|
||||
(1..=64).contains(&n)
|
||||
}))),
|
||||
Kind::Choice { values } => v.as_str().is_some_and(|s| values.contains(&s)),
|
||||
Kind::LanguageCode => v.is_null()
|
||||
|| v.as_str().is_some_and(|s| (2..=8).contains(&s.len()) && s.chars().all(|c| c.is_ascii_alphabetic() || c == '-')),
|
||||
Kind::Id => v.is_null() || is_id(v),
|
||||
Kind::IdList => v.as_array().is_some_and(|a| a.iter().all(is_id)),
|
||||
Kind::IdNumberMap => v.as_object().is_some_and(|m| m.iter()
|
||||
.all(|(k, n)| is_id(&Value::String(k.clone())) && n.as_i64().is_some())),
|
||||
Kind::ChatChannels | Kind::McpServers | Kind::CollectTargets => v.is_array(),
|
||||
};
|
||||
if ok { Ok(()) } else { Err(format!("{} has a value it cannot take", field.path)) }
|
||||
}
|
||||
|
||||
/// Every field's value now: the file's, else the default.
|
||||
pub fn view(settings_file: &Path) -> Result<Map<String, Value>, EditError> {
|
||||
let raw = load_raw(settings_file)?;
|
||||
Ok(fields().into_iter()
|
||||
.map(|f| (f.path.to_owned(), get_path(&raw, f.path).cloned().unwrap_or(f.default)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Apply `{path: value}` to the file; all or nothing.
|
||||
pub fn apply(settings_file: &Path, changes: &Map<String, Value>) -> Result<(), EditError> {
|
||||
let mut raw = load_raw(settings_file)?;
|
||||
let all = fields();
|
||||
for (path, value) in changes {
|
||||
let field = all.iter().find(|f| f.path == path)
|
||||
.ok_or_else(|| EditError(format!("{path} is not a setting the screens edit")))?;
|
||||
check(field, value).map_err(EditError)?;
|
||||
if *value == field.default {
|
||||
remove_path(&mut raw, path);
|
||||
} else {
|
||||
set_path(&mut raw, path, value.clone())?;
|
||||
}
|
||||
}
|
||||
let doc = Value::Object(raw);
|
||||
parse_settings(&doc).map_err(|e| EditError(e.0))?;
|
||||
let mut text = serde_json::to_string_pretty(&doc).expect("JSON serialises");
|
||||
text.push('\n');
|
||||
write_atomic(settings_file, text.as_bytes())
|
||||
.map_err(|e| EditError(format!("could not write {}: {e}", settings_file.display())))?;
|
||||
crate::debug_log!("admin", "settings changed: {}", changes.keys().cloned().collect::<Vec<_>>().join(", "));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use crate::settings::Settings;
|
||||
|
||||
fn file(tag: &str) -> std::path::PathBuf {
|
||||
temp_dir(tag).join("settings.json")
|
||||
}
|
||||
fn changes(v: Value) -> Map<String, Value> {
|
||||
v.as_object().unwrap().clone()
|
||||
}
|
||||
fn read(p: &Path) -> Value {
|
||||
serde_json::from_str(&std::fs::read_to_string(p).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_default_is_the_apps_own_default() {
|
||||
// Writing a field's default out explicitly must give the same settings as leaving it out.
|
||||
for f in fields() {
|
||||
let mut raw = Map::new();
|
||||
set_path(&mut raw, f.path, f.default.clone()).unwrap();
|
||||
let parsed = parse_settings(&Value::Object(raw)).unwrap_or_else(|e| panic!("{}: {e}", f.path));
|
||||
let mut d = Settings::default();
|
||||
// memory and stt pass to the worker as given; their defaults are the worker's.
|
||||
d.memory = parsed.memory.clone();
|
||||
d.voice.stt = parsed.voice.stt.clone();
|
||||
assert_eq!(parsed, d, "{}", f.path);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn view_shows_the_file_over_the_defaults() {
|
||||
let p = file("sev");
|
||||
std::fs::write(&p, r#"{"call": {"mode": "addressed"}, "unknown": 1}"#).unwrap();
|
||||
let v = view(&p).unwrap();
|
||||
assert_eq!(v["call.mode"], json!("addressed"));
|
||||
assert_eq!(v["voice.maxSpeakers"], json!(10));
|
||||
assert_eq!(v.len(), fields().len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_writes_only_non_defaults_and_keeps_unknown_keys() {
|
||||
let p = file("sea");
|
||||
std::fs::write(&p, r#"{"voiceWorker": {"python": "/venv/python"}, "mine": {"x": 1}}"#).unwrap();
|
||||
apply(&p, &changes(json!({"call.mode": "everything", "voice.maxSpeakers": 10,
|
||||
"memory.recent_records": 5, "voice.stt.end_silence_ms": 400}))).unwrap();
|
||||
assert_eq!(read(&p), json!({"voiceWorker": {"python": "/venv/python"}, "mine": {"x": 1},
|
||||
"call": {"mode": "everything"}, "memory": {"recent_records": 5}, "voice": {"stt": {"end_silence_ms": 400}}}));
|
||||
// Back to the default: the key goes, and so do sections it leaves empty.
|
||||
apply(&p, &changes(json!({"call.mode": "wakeWord", "voice.stt.end_silence_ms": 300}))).unwrap();
|
||||
assert_eq!(read(&p), json!({"voiceWorker": {"python": "/venv/python"}, "mine": {"x": 1},
|
||||
"memory": {"recent_records": 5}}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_value_changes_nothing() {
|
||||
let p = file("seb");
|
||||
std::fs::write(&p, r#"{"call": {"mode": "addressed"}}"#).unwrap();
|
||||
let before = std::fs::read(&p).unwrap();
|
||||
for (c, key) in [
|
||||
(json!({"voice.maxSpeakers": 11}), "voice.maxSpeakers"),
|
||||
(json!({"voice.maxSpeakers": 2.5}), "voice.maxSpeakers"),
|
||||
(json!({"call.mode": "sometimes"}), "call.mode"),
|
||||
(json!({"discord.guildId": 123}), "discord.guildId"),
|
||||
(json!({"voice.joinNotice": " "}), "voice.joinNotice"),
|
||||
(json!({"memory.since_days": 0}), "memory.since_days"),
|
||||
(json!({"voice.stt.language": ""}), "voice.stt.language"),
|
||||
(json!({"nope": 1}), "nope"),
|
||||
// Passes the field check, refused by the app's own check: a bad command name.
|
||||
(json!({"discord.commandName": "Big Name"}), "discord.commandName"),
|
||||
// A chat channel without a valid ID.
|
||||
(json!({"chatChannels": [{"id": "1"}]}), "chatChannels[0].id"),
|
||||
(json!({"collect.targets": [{"name": "nul", "kind": "rss", "value": "https://a"}]}), "collect.targets[0].name"),
|
||||
] {
|
||||
let e = apply(&p, &changes(c.clone()).into_iter().chain([("call.earcon".into(), json!(false))]).collect())
|
||||
.unwrap_err();
|
||||
assert!(e.0.contains(key), "{c} -> {e}");
|
||||
assert_eq!(std::fs::read(&p).unwrap(), before, "{c}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_does_not_parse_is_reported_not_overwritten() {
|
||||
let p = file("sep");
|
||||
std::fs::write(&p, "{ not json").unwrap();
|
||||
assert!(view(&p).is_err());
|
||||
assert!(apply(&p, &changes(json!({"call.earcon": false}))).is_err());
|
||||
assert_eq!(std::fs::read_to_string(&p).unwrap(), "{ not json");
|
||||
}
|
||||
}
|
||||
41
app/src/setup_lock.rs
Normal file
41
app/src/setup_lock.rs
Normal file
@@ -0,0 +1,41 @@
|
||||
//! The installer's lock (admin.spec.md, "Installer"): while Jarvis is installed, updated or
|
||||
//! removed, the installer holds the named mutex `Global\JarvisBotSetup`, and the app does not
|
||||
//! start meanwhile (it would hold files being replaced, or start services on a driver being
|
||||
//! removed).
|
||||
|
||||
pub const MUTEX: &str = "Global\\JarvisBotSetup";
|
||||
|
||||
/// Whether an installer holds the lock (also when this user may not open it: it exists).
|
||||
#[cfg(windows)]
|
||||
pub fn held() -> bool {
|
||||
use windows_sys::Win32::Foundation::{CloseHandle, GetLastError, ERROR_ACCESS_DENIED};
|
||||
use windows_sys::Win32::System::Threading::{OpenMutexW, SYNCHRONIZATION_SYNCHRONIZE};
|
||||
let name: Vec<u16> = MUTEX.encode_utf16().chain(Some(0)).collect();
|
||||
// SAFETY: opening a named object by a NUL-terminated name; the handle is closed at once.
|
||||
unsafe {
|
||||
let h = OpenMutexW(SYNCHRONIZATION_SYNCHRONIZE, 0, name.as_ptr());
|
||||
if !h.is_null() {
|
||||
CloseHandle(h);
|
||||
return true;
|
||||
}
|
||||
GetLastError() == ERROR_ACCESS_DENIED
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn held() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Say so in a message box (the app has no window yet).
|
||||
#[cfg(windows)]
|
||||
pub fn tell() {
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::{MessageBoxW, MB_ICONINFORMATION, MB_OK};
|
||||
let text: Vec<u16> = "Jarvis를 설치하거나 제거하는 중입니다. 끝난 뒤 다시 실행해 주세요.".encode_utf16().chain(Some(0)).collect();
|
||||
let title: Vec<u16> = "Jarvis".encode_utf16().chain(Some(0)).collect();
|
||||
// SAFETY: NUL-terminated strings; no owner window.
|
||||
unsafe { MessageBoxW(std::ptr::null_mut(), text.as_ptr(), title.as_ptr(), MB_OK | MB_ICONINFORMATION) };
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn tell() {}
|
||||
70
app/src/speakers.rs
Normal file
70
app/src/speakers.rs
Normal file
@@ -0,0 +1,70 @@
|
||||
//! Who is being transcribed right now. At most `max` people at once (settings `voice.maxSpeakers`);
|
||||
//! someone who starts while the limit is reached is not heard until their next start after the
|
||||
//! others free a place.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub struct SpeakerGate {
|
||||
max: usize,
|
||||
active: HashSet<u64>,
|
||||
}
|
||||
|
||||
impl SpeakerGate {
|
||||
pub fn new(max: usize) -> Self {
|
||||
Self { max, active: HashSet::new() }
|
||||
}
|
||||
|
||||
/// `id` started speaking. True if their audio goes to the STT.
|
||||
pub fn start(&mut self, id: u64) -> bool {
|
||||
if self.active.contains(&id) {
|
||||
return true;
|
||||
}
|
||||
if self.active.len() >= self.max {
|
||||
return false;
|
||||
}
|
||||
self.active.insert(id);
|
||||
true
|
||||
}
|
||||
|
||||
/// `id` stopped (or left). True if they were being transcribed.
|
||||
pub fn end(&mut self, id: u64) -> bool {
|
||||
self.active.remove(&id)
|
||||
}
|
||||
|
||||
pub fn is_active(&self, id: u64) -> bool {
|
||||
self.active.contains(&id)
|
||||
}
|
||||
|
||||
pub fn count(&self) -> usize {
|
||||
self.active.len()
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.active.clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hears_up_to_the_limit_and_lets_the_next_in_when_someone_stops() {
|
||||
let mut g = SpeakerGate::new(2);
|
||||
assert!(g.start(1) && g.start(2));
|
||||
assert!(!g.start(3));
|
||||
assert!(!g.is_active(3));
|
||||
assert!(g.end(1));
|
||||
assert!(g.start(3));
|
||||
assert_eq!(g.count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_heard_stays_heard_and_ending_a_stranger_is_a_no_op() {
|
||||
let mut g = SpeakerGate::new(1);
|
||||
assert!(g.start(1) && g.start(1));
|
||||
assert!(!g.end(9));
|
||||
assert!(g.end(1));
|
||||
assert!(!g.end(1));
|
||||
}
|
||||
}
|
||||
1405
app/src/strata.rs
Normal file
1405
app/src/strata.rs
Normal file
File diff suppressed because it is too large
Load Diff
124
app/src/system.rs
Normal file
124
app/src/system.rs
Normal file
@@ -0,0 +1,124 @@
|
||||
//! Memory and GPU use for the dashboard (`app/src/admin.spec.md`, "Dashboard data").
|
||||
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Gpu {
|
||||
pub name: String,
|
||||
pub use_percent: u32,
|
||||
pub vram_used_mb: u64,
|
||||
pub vram_total_mb: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Usage {
|
||||
pub ram_used_mb: Option<u64>,
|
||||
pub ram_total_mb: Option<u64>,
|
||||
pub gpus: Vec<Gpu>,
|
||||
}
|
||||
|
||||
/// One line per GPU of `nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total
|
||||
/// --format=csv,noheader,nounits`; lines that do not parse are skipped.
|
||||
pub fn parse_nvidia_smi(out: &str) -> Vec<Gpu> {
|
||||
out.lines().filter_map(|l| {
|
||||
let f: Vec<&str> = l.split(',').map(str::trim).collect();
|
||||
let [name, util, used, total] = f.as_slice() else { return None };
|
||||
Some(Gpu { name: (*name).into(), use_percent: util.parse().ok()?, vram_used_mb: used.parse().ok()?,
|
||||
vram_total_mb: total.parse().ok()? })
|
||||
}).collect()
|
||||
}
|
||||
|
||||
/// The GPUs `nvidia-smi` reports; empty without one (or when it does not answer within 2 s).
|
||||
pub async fn gpus() -> Vec<Gpu> {
|
||||
let mut cmd = tokio::process::Command::new("nvidia-smi");
|
||||
cmd.args(["--query-gpu=name,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"])
|
||||
.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()).kill_on_drop(true);
|
||||
#[cfg(windows)]
|
||||
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
|
||||
match tokio::time::timeout(Duration::from_secs(2), cmd.output()).await {
|
||||
Ok(Ok(o)) if o.status.success() => parse_nvidia_smi(&String::from_utf8_lossy(&o.stdout)),
|
||||
_ => vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// The NVIDIA driver version (`581.57`), None without nvidia-smi.
|
||||
pub async fn driver_version() -> Option<String> {
|
||||
let mut cmd = tokio::process::Command::new("nvidia-smi");
|
||||
cmd.args(["--query-gpu=driver_version", "--format=csv,noheader"])
|
||||
.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()).kill_on_drop(true);
|
||||
#[cfg(windows)]
|
||||
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
|
||||
match tokio::time::timeout(Duration::from_secs(2), cmd.output()).await {
|
||||
Ok(Ok(o)) if o.status.success() => String::from_utf8_lossy(&o.stdout).lines().next().map(|l| l.trim().to_owned()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// CUDA 13 needs driver 580 or newer (desktop-app.md, "PC 점검").
|
||||
pub const MIN_DRIVER_MAJOR: u32 = 580;
|
||||
|
||||
pub fn driver_ok(version: &str) -> bool {
|
||||
version.split('.').next().and_then(|m| m.parse::<u32>().ok()).is_some_and(|m| m >= MIN_DRIVER_MAJOR)
|
||||
}
|
||||
|
||||
pub async fn usage() -> Usage {
|
||||
let (used, total) = memory_mb();
|
||||
Usage { ram_used_mb: used, ram_total_mb: total, gpus: gpus().await }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn memory_mb() -> (Option<u64>, Option<u64>) {
|
||||
let Ok(s) = std::fs::read_to_string("/proc/meminfo") else { return (None, None) };
|
||||
let kb = |key: &str| s.lines().find(|l| l.starts_with(key))
|
||||
.and_then(|l| l.split_whitespace().nth(1)?.parse::<u64>().ok());
|
||||
let (total, avail) = (kb("MemTotal:"), kb("MemAvailable:"));
|
||||
(total.zip(avail).map(|(t, a)| (t - a) / 1024), total.map(|t| t / 1024))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
fn memory_mb() -> (Option<u64>, Option<u64>) {
|
||||
use windows_sys::Win32::System::SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX};
|
||||
// SAFETY: the struct is sized and zeroed as the API asks.
|
||||
let mut m: MEMORYSTATUSEX = unsafe { std::mem::zeroed() };
|
||||
m.dwLength = std::mem::size_of::<MEMORYSTATUSEX>() as u32;
|
||||
if unsafe { GlobalMemoryStatusEx(&mut m) } == 0 {
|
||||
return (None, None);
|
||||
}
|
||||
let mb = 1024 * 1024;
|
||||
(Some((m.ullTotalPhys - m.ullAvailPhys) / mb), Some(m.ullTotalPhys / mb))
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
fn memory_mb() -> (Option<u64>, Option<u64>) {
|
||||
(None, None)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn nvidia_smi_lines_become_gpus_and_junk_is_skipped() {
|
||||
let out = "NVIDIA GeForce RTX 5050, 37, 5120, 8151\nNVIDIA GeForce RTX 4090, [N/A], 1, 24564\n\n";
|
||||
assert_eq!(parse_nvidia_smi(out), vec![Gpu { name: "NVIDIA GeForce RTX 5050".into(), use_percent: 37,
|
||||
vram_used_mb: 5120, vram_total_mb: 8151 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drivers_from_580_run_cuda_13() {
|
||||
assert!(driver_ok("581.57") && driver_ok("595.91.07") && driver_ok("580.0"));
|
||||
assert!(!driver_ok("572.16") && !driver_ok("") && !driver_ok("abc"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_reads_this_machines_memory() {
|
||||
let u = usage().await;
|
||||
if cfg!(any(target_os = "linux", windows)) {
|
||||
let (used, total) = (u.ram_used_mb.unwrap(), u.ram_total_mb.unwrap());
|
||||
assert!(used > 0 && used <= total);
|
||||
}
|
||||
}
|
||||
}
|
||||
438
app/src/update.rs
Normal file
438
app/src/update.rs
Normal file
@@ -0,0 +1,438 @@
|
||||
//! App updates (admin.spec.md, "Updates"): the app looks for a newer release on the project's
|
||||
//! Gitea, downloads its installer into the data folder checked against the release's SHA-256, and
|
||||
//! on request stops what runs and starts the installer silently (`/S /RUN`), which waits for the
|
||||
//! app to close, replaces it and starts it again.
|
||||
//!
|
||||
//! A release of this app is tagged `app-v<version>` and carries `Jarvis-Setup-<version>.exe` and
|
||||
//! `Jarvis-Setup-<version>.exe.sha256` (what `app/shell/installer.sh` writes). Other releases of the
|
||||
//! repository (v1's `v1.x` tags) are not this app's and are skipped.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// The project's releases (Gitea API, newest first).
|
||||
pub const SOURCE: &str = "https://git.tkrmagid.kr/api/v1/repos/tkrmagid/javis_bot/releases";
|
||||
pub const TAG_PREFIX: &str = "app-v";
|
||||
|
||||
/// Where to look: `JARVIS_UPDATE_SOURCE` when set (a test server), else `SOURCE`.
|
||||
pub fn source() -> String {
|
||||
std::env::var("JARVIS_UPDATE_SOURCE").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| SOURCE.into())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Release {
|
||||
pub version: String,
|
||||
pub notes: String,
|
||||
pub file: String,
|
||||
pub size: u64,
|
||||
#[serde(skip)]
|
||||
pub installer_url: String,
|
||||
#[serde(skip)]
|
||||
pub checksum_url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Phase {
|
||||
Idle,
|
||||
Checking,
|
||||
UpToDate,
|
||||
Available,
|
||||
Downloading,
|
||||
Ready,
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Dotted numbers ("0.2.0"), padded to four parts so "1.0" equals "1.0.0"; anything else is `None`.
|
||||
fn version(s: &str) -> Option<[u64; 4]> {
|
||||
let parts: Vec<u64> = s.split('.').map(|p| p.parse().ok()).collect::<Option<_>>()?;
|
||||
if parts.is_empty() || parts.len() > 4 {
|
||||
return None;
|
||||
}
|
||||
let mut v = [0; 4];
|
||||
v[..parts.len()].copy_from_slice(&parts);
|
||||
Some(v)
|
||||
}
|
||||
|
||||
/// "https://host[:port]" of a URL: downloads come only from where the release list came from.
|
||||
fn origin(url: &str) -> Option<&str> {
|
||||
let rest = url.strip_prefix("https://").or_else(|| url.strip_prefix("http://"))?;
|
||||
Some(&url[..url.len() - rest.len() + rest.find('/').unwrap_or(rest.len())])
|
||||
}
|
||||
|
||||
/// HTTPS only; plain HTTP only to this PC (a test server).
|
||||
fn allowed(url: &str) -> bool {
|
||||
url.starts_with("https://") || url.starts_with("http://127.0.0.1:") || url.starts_with("http://localhost:")
|
||||
}
|
||||
|
||||
/// The newest release of this app above `current` in a Gitea release list, if any. Drafts,
|
||||
/// prereleases, other tags and releases without both assets on `source`'s origin are skipped.
|
||||
pub fn pick(list: &Value, current: &str, source: &str) -> Result<Option<Release>, String> {
|
||||
let list = list.as_array().ok_or("릴리스 목록을 읽지 못했습니다")?;
|
||||
let current = version(current).ok_or("현재 버전을 읽지 못했습니다")?;
|
||||
let home = origin(source);
|
||||
let mut best: Option<([u64; 4], Release)> = None;
|
||||
for r in list {
|
||||
if r["draft"] == json!(true) || r["prerelease"] == json!(true) {
|
||||
continue;
|
||||
}
|
||||
let Some(v) = r["tag_name"].as_str().and_then(|t| t.strip_prefix(TAG_PREFIX)) else { continue };
|
||||
let Some(parsed) = version(v) else { continue };
|
||||
if parsed <= current || best.as_ref().is_some_and(|(b, _)| parsed <= *b) {
|
||||
continue;
|
||||
}
|
||||
let file = format!("Jarvis-Setup-{v}.exe");
|
||||
let asset = |name: &str| r["assets"].as_array()?.iter().find(|a| a["name"].as_str() == Some(name)).cloned();
|
||||
let (Some(exe), Some(sum)) = (asset(&file), asset(&format!("{file}.sha256"))) else { continue };
|
||||
let url = |a: &Value| a["browser_download_url"].as_str().map(str::to_owned)
|
||||
.filter(|u| allowed(u) && origin(u).is_some() && origin(u) == home);
|
||||
let (Some(installer_url), Some(checksum_url)) = (url(&exe), url(&sum)) else { continue };
|
||||
best = Some((parsed, Release { version: v.into(), notes: r["body"].as_str().unwrap_or("").into(), file,
|
||||
size: exe["size"].as_u64().unwrap_or(0), installer_url, checksum_url }));
|
||||
}
|
||||
Ok(best.map(|(_, r)| r))
|
||||
}
|
||||
|
||||
/// The SHA-256 in a `.sha256` file ("<hex> <file>"): 64 hex digits, for `file` when it names one.
|
||||
pub fn checksum(text: &str, file: &str) -> Result<String, String> {
|
||||
let mut words = text.split_whitespace();
|
||||
let hex = words.next().unwrap_or("").to_ascii_lowercase();
|
||||
let named = words.next().map(|n| n.trim_start_matches('*'));
|
||||
if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) || named.is_some_and(|n| n != file) {
|
||||
return Err("릴리스의 SHA-256 파일이 올바르지 않습니다".into());
|
||||
}
|
||||
Ok(hex)
|
||||
}
|
||||
|
||||
struct State {
|
||||
phase: Phase,
|
||||
latest: Option<Release>,
|
||||
error: Option<String>,
|
||||
done: u64,
|
||||
total: Option<u64>,
|
||||
sha256: Option<String>,
|
||||
}
|
||||
|
||||
/// The update state for the screens: checks, the download in the background, and the ready file.
|
||||
pub struct Updater {
|
||||
source: String,
|
||||
current: String,
|
||||
dir: PathBuf,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
impl Updater {
|
||||
/// `dir`: where installers are downloaded (the data folder's `cache/updates`).
|
||||
pub fn new(source: String, current: &str, dir: PathBuf) -> Arc<Self> {
|
||||
Arc::new(Self { source, current: current.into(), dir, state: Mutex::new(State {
|
||||
phase: Phase::Idle, latest: None, error: None, done: 0, total: None, sha256: None }) })
|
||||
}
|
||||
|
||||
pub fn status(&self) -> Value {
|
||||
let s = self.state.lock().unwrap();
|
||||
json!({"current": self.current, "phase": s.phase, "latest": s.latest, "error": s.error,
|
||||
"done": s.done, "total": s.total})
|
||||
}
|
||||
|
||||
fn client() -> Result<reqwest::Client, String> {
|
||||
reqwest::Client::builder().timeout(std::time::Duration::from_secs(30)).build().map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn get_text(url: &str) -> Result<String, String> {
|
||||
let r = Self::client()?.get(url).header("accept", "application/json").send().await
|
||||
.map_err(|e| format!("업데이트 서버에 연결하지 못했습니다: {e}"))?;
|
||||
if !r.status().is_success() {
|
||||
return Err(format!("업데이트 서버가 HTTP {}로 답했습니다", r.status()));
|
||||
}
|
||||
r.text().await.map_err(|e| format!("업데이트 정보를 받지 못했습니다: {e}"))
|
||||
}
|
||||
|
||||
fn fail(&self, e: String) -> String {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
(s.phase, s.error) = (Phase::Failed, Some(e.clone()));
|
||||
crate::warn_log!("app", "update: {e}");
|
||||
e
|
||||
}
|
||||
|
||||
/// Look for a newer release; not while downloading.
|
||||
pub async fn check(&self) -> Result<Value, String> {
|
||||
let was = {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
if matches!(s.phase, Phase::Checking | Phase::Downloading) {
|
||||
return Ok(json!({"current": self.current, "phase": s.phase, "latest": s.latest}));
|
||||
}
|
||||
std::mem::replace(&mut s.phase, Phase::Checking)
|
||||
};
|
||||
self.state.lock().unwrap().error = None;
|
||||
if !allowed(&self.source) {
|
||||
return Err(self.fail(format!("업데이트 주소는 https여야 합니다: {}", self.source)));
|
||||
}
|
||||
let url = format!("{}?limit=50", self.source);
|
||||
let found = match Self::get_text(&url).await
|
||||
.and_then(|t| serde_json::from_str::<Value>(&t).map_err(|_| "릴리스 목록을 읽지 못했습니다".into()))
|
||||
.and_then(|v| pick(&v, &self.current, &self.source)) {
|
||||
Ok(f) => f,
|
||||
Err(e) => return Err(self.fail(e)),
|
||||
};
|
||||
let mut s = self.state.lock().unwrap();
|
||||
// The release already downloaded stays ready.
|
||||
if was == Phase::Ready && found.is_some() && s.latest == found {
|
||||
s.phase = Phase::Ready;
|
||||
} else {
|
||||
s.phase = if found.is_some() { Phase::Available } else { Phase::UpToDate };
|
||||
(s.done, s.total, s.sha256) = (0, None, None);
|
||||
}
|
||||
crate::info_log!("app", "update check: {}", found.as_ref().map_or("up to date".into(), |r| format!("{} available", r.version)));
|
||||
s.latest = found;
|
||||
Ok(json!({"current": self.current, "phase": s.phase, "latest": s.latest}))
|
||||
}
|
||||
|
||||
/// Download the release found by `check` in the background (resumed where an earlier try stopped).
|
||||
pub fn download(self: &Arc<Self>) -> Result<(), String> {
|
||||
let release = {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
let release = match (s.phase, &s.latest) {
|
||||
(Phase::Available | Phase::Failed, Some(r)) => r.clone(),
|
||||
(Phase::Downloading | Phase::Ready, Some(_)) => return Ok(()),
|
||||
_ => return Err("먼저 새 버전을 확인해 주세요".into()),
|
||||
};
|
||||
(s.phase, s.error, s.done, s.total) = (Phase::Downloading, None, 0, Some(release.size));
|
||||
release
|
||||
};
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move {
|
||||
let dest = me.dir.join(&release.file);
|
||||
let got = async {
|
||||
let sha = checksum(&Self::get_text(&release.checksum_url).await?, &release.file)?;
|
||||
std::fs::create_dir_all(&me.dir).map_err(|e| format!("업데이트 폴더를 만들지 못했습니다: {e}"))?;
|
||||
let never = AtomicBool::new(false);
|
||||
let progress = |done: u64, total: Option<u64>| {
|
||||
let mut s = me.state.lock().unwrap();
|
||||
(s.done, s.total) = (done, total.or(s.total));
|
||||
};
|
||||
crate::download::download(&release.installer_url, &dest, &sha, &progress, &never).await
|
||||
.map_err(|e| format!("설치 파일을 받지 못했습니다: {e}"))?;
|
||||
Ok::<_, String>(sha)
|
||||
}.await;
|
||||
match got {
|
||||
Ok(sha) => {
|
||||
// Earlier installers and broken parts go; this one stays until it is installed.
|
||||
let keep = [release.file.clone(), format!("{}.part", release.file)];
|
||||
for e in std::fs::read_dir(&me.dir).into_iter().flatten().flatten() {
|
||||
if !keep.iter().any(|k| e.file_name() == k.as_str()) {
|
||||
let _ = std::fs::remove_file(e.path());
|
||||
}
|
||||
}
|
||||
let mut s = me.state.lock().unwrap();
|
||||
(s.phase, s.sha256) = (Phase::Ready, Some(sha));
|
||||
crate::info_log!("app", "update {} downloaded", release.version);
|
||||
}
|
||||
Err(e) => { me.fail(e); }
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The downloaded installer, checked again right before it is started.
|
||||
pub fn ready_installer(&self) -> Result<PathBuf, String> {
|
||||
let s = self.state.lock().unwrap();
|
||||
let (Phase::Ready, Some(r), Some(sha)) = (s.phase, &s.latest, &s.sha256) else {
|
||||
return Err("설치할 업데이트를 먼저 받아 주세요".into());
|
||||
};
|
||||
let path = self.dir.join(&r.file);
|
||||
if crate::download::sha256_file(&path).ok().as_deref() != Some(sha.as_str()) {
|
||||
return Err("받은 설치 파일이 바뀌었습니다. 다시 받아 주세요".into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the installer silently, as administrator (Windows asks), without waiting: it waits for
|
||||
/// this app to close, installs, and starts the app again.
|
||||
#[cfg(windows)]
|
||||
pub fn launch_installer(path: &Path) -> Result<(), String> {
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use windows_sys::Win32::Foundation::{GetLastError, ERROR_CANCELLED};
|
||||
use windows_sys::Win32::UI::Shell::{ShellExecuteExW, SHELLEXECUTEINFOW};
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::SW_SHOWNORMAL;
|
||||
let wide = |s: &std::ffi::OsStr| s.encode_wide().chain(Some(0)).collect::<Vec<u16>>();
|
||||
let (verb, file, params) = (wide("runas".as_ref()), wide(path.as_os_str()), wide("/S /RUN".as_ref()));
|
||||
// SAFETY: SHELLEXECUTEINFOW is plain data; all zero is its documented empty state.
|
||||
let mut info: SHELLEXECUTEINFOW = unsafe { std::mem::zeroed() };
|
||||
info.cbSize = std::mem::size_of::<SHELLEXECUTEINFOW>() as u32;
|
||||
info.lpVerb = verb.as_ptr();
|
||||
info.lpFile = file.as_ptr();
|
||||
info.lpParameters = params.as_ptr();
|
||||
info.nShow = SW_SHOWNORMAL;
|
||||
// SAFETY: `info` is filled as documented and its strings live through the call; no process
|
||||
// handle is asked for.
|
||||
if unsafe { ShellExecuteExW(&mut info) } == 0 {
|
||||
let e = unsafe { GetLastError() };
|
||||
return Err(if e == ERROR_CANCELLED { "관리자 권한을 받지 못해 업데이트를 설치하지 않았습니다".into() }
|
||||
else { format!("설치 파일을 실행하지 못했습니다 ({e})") });
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn launch_installer(_path: &Path) -> Result<(), String> {
|
||||
Err("업데이트 설치는 Windows에서만 됩니다".into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::datadir::temp_dir;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
fn both(v: &str) -> Vec<String> {
|
||||
vec![format!("Jarvis-Setup-{v}.exe"), format!("Jarvis-Setup-{v}.exe.sha256")]
|
||||
}
|
||||
|
||||
fn rel(tag: &str, base: &str, files: &[String]) -> Value {
|
||||
json!({"tag_name": tag, "draft": false, "prerelease": false, "body": format!("notes {tag}"),
|
||||
"assets": files.iter().map(|f| json!({"name": f, "size": 10,
|
||||
"browser_download_url": format!("{base}/download/{tag}/{f}")})).collect::<Vec<_>>()})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_newest_release_of_this_app_above_the_current_version_is_picked() {
|
||||
let src = "https://git.example/api/v1/repos/o/r/releases";
|
||||
let base = "https://git.example/o/r/releases";
|
||||
let mut pre = rel("app-v0.4.0", base, &both("0.4.0"));
|
||||
pre["prerelease"] = json!(true);
|
||||
let list = json!([
|
||||
rel("v1.10.0", base, &[]), // v1's release
|
||||
rel("app-v0.3.0", base, &both("0.3.0")[..1]), // no checksum
|
||||
pre,
|
||||
rel("app-v0.2.1", "https://elsewhere.example/o/r", &both("0.2.1")), // another server
|
||||
rel("app-v0.2.0", base, &both("0.2.0")),
|
||||
rel("app-v0.1.5", base, &both("0.1.5")),
|
||||
]);
|
||||
let r = pick(&list, "0.1.0", src).unwrap().unwrap();
|
||||
assert_eq!((r.version.as_str(), r.file.as_str(), r.notes.as_str()), ("0.2.0", "Jarvis-Setup-0.2.0.exe", "notes app-v0.2.0"));
|
||||
assert_eq!(r.installer_url, format!("{base}/download/app-v0.2.0/Jarvis-Setup-0.2.0.exe"));
|
||||
assert_eq!(pick(&list, "0.2.0", src).unwrap(), None, "nothing newer");
|
||||
assert_eq!(pick(&list, "0.2", src).unwrap(), None, "0.2 is 0.2.0");
|
||||
assert_eq!(pick(&json!([]), "0.1.0", src).unwrap(), None);
|
||||
assert!(pick(&json!({"message": "not found"}), "0.1.0", src).is_err());
|
||||
// Plain http only to this PC.
|
||||
let http = rel("app-v0.2.0", "http://git.example/o/r", &both("0.2.0"));
|
||||
assert_eq!(pick(&json!([http]), "0.1.0", "http://git.example/api").unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_checksum_file_gives_its_hash_for_the_right_file_only() {
|
||||
let h = "AB".repeat(32);
|
||||
assert_eq!(checksum(&format!("{h} Jarvis-Setup-0.2.0.exe\n"), "Jarvis-Setup-0.2.0.exe").unwrap(), h.to_lowercase());
|
||||
assert_eq!(checksum(&format!("{h} *Jarvis-Setup-0.2.0.exe"), "Jarvis-Setup-0.2.0.exe").unwrap(), h.to_lowercase());
|
||||
assert_eq!(checksum(&h, "x.exe").unwrap(), h.to_lowercase());
|
||||
assert!(checksum(&format!("{h} other.exe"), "Jarvis-Setup-0.2.0.exe").is_err());
|
||||
assert!(checksum("abc Jarvis-Setup-0.2.0.exe", "Jarvis-Setup-0.2.0.exe").is_err());
|
||||
assert!(checksum("", "x").is_err());
|
||||
}
|
||||
|
||||
/// A fake Gitea on this PC: the release list and the two assets of `app-v<version>`.
|
||||
pub(crate) async fn gitea(version: &str, installer: Vec<u8>, sum: String) -> String {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", l.local_addr().unwrap());
|
||||
let file = format!("Jarvis-Setup-{version}.exe");
|
||||
let list = json!([rel("v1.10.0", &base, &[]), rel(&format!("app-v{version}"), &base, &both(version))]).to_string();
|
||||
let sum_text = format!("{sum} {file}\n");
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut s, _)) = l.accept().await {
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let n = s.read(&mut buf).await.unwrap_or(0);
|
||||
let path = String::from_utf8_lossy(&buf[..n]).split_whitespace().nth(1).unwrap_or("").to_owned();
|
||||
let body: Vec<u8> = if path.starts_with("/releases") { list.clone().into_bytes() }
|
||||
else if path.ends_with(".sha256") { sum_text.clone().into_bytes() }
|
||||
else if path.ends_with(".exe") { installer.clone() } else { vec![] };
|
||||
let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
|
||||
let _ = s.write_all(head.as_bytes()).await;
|
||||
let _ = s.write_all(&body).await;
|
||||
}
|
||||
});
|
||||
base
|
||||
}
|
||||
|
||||
async fn settle(u: &Updater) -> Value {
|
||||
for _ in 0..200 {
|
||||
let s = u.status();
|
||||
if s["phase"] != json!("downloading") {
|
||||
return s;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
panic!("the download did not end");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_newer_release_is_found_downloaded_checked_and_handed_over() {
|
||||
let installer: Vec<u8> = (0..50_000u32).map(|i| (i % 253) as u8).collect();
|
||||
let sum = format!("{:x}", Sha256::digest(&installer));
|
||||
let base = gitea("0.2.0", installer.clone(), sum).await;
|
||||
let dir = temp_dir("upd1").join("cache/updates");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("Jarvis-Setup-0.1.5.exe"), b"old").unwrap();
|
||||
let u = Updater::new(format!("{base}/releases"), "0.1.0", dir.clone());
|
||||
assert!(u.ready_installer().is_err(), "nothing to install yet");
|
||||
assert!(u.download().is_err(), "check first");
|
||||
let c = u.check().await.unwrap();
|
||||
assert_eq!((c["phase"].clone(), c["latest"]["version"].clone()), (json!("available"), json!("0.2.0")));
|
||||
u.download().unwrap();
|
||||
let s = settle(&u).await;
|
||||
assert_eq!((s["phase"].clone(), s["done"].clone()), (json!("ready"), json!(installer.len())));
|
||||
let path = u.ready_installer().unwrap();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), installer);
|
||||
assert!(!dir.join("Jarvis-Setup-0.1.5.exe").exists(), "the earlier installer is gone");
|
||||
// Checking again keeps the ready file; a file changed after the download is refused.
|
||||
assert_eq!(u.check().await.unwrap()["phase"], json!("ready"));
|
||||
std::fs::write(&path, b"changed").unwrap();
|
||||
assert!(u.ready_installer().unwrap_err().contains("바뀌었"));
|
||||
// The same version as the release: up to date.
|
||||
let same = Updater::new(format!("{base}/releases"), "0.2.0", temp_dir("upd1b"));
|
||||
assert_eq!(same.check().await.unwrap()["phase"], json!("upToDate"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_download_that_does_not_match_its_checksum_is_not_offered_for_install() {
|
||||
let base = gitea("0.2.0", b"tampered".to_vec(), "0".repeat(64)).await;
|
||||
let dir = temp_dir("upd2");
|
||||
let u = Updater::new(format!("{base}/releases"), "0.1.0", dir.clone());
|
||||
u.check().await.unwrap();
|
||||
u.download().unwrap();
|
||||
let s = settle(&u).await;
|
||||
assert_eq!(s["phase"], json!("failed"));
|
||||
assert!(s["error"].as_str().unwrap().contains("damaged"));
|
||||
assert!(u.ready_installer().is_err());
|
||||
assert!(!dir.join("Jarvis-Setup-0.2.0.exe").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_server_that_cannot_be_reached_or_is_not_https_fails_with_a_reason() {
|
||||
let u = Updater::new("http://127.0.0.1:9/releases".into(), "0.1.0", temp_dir("upd3"));
|
||||
assert!(u.check().await.unwrap_err().contains("연결하지 못했습니다"));
|
||||
assert_eq!(u.status()["phase"], json!("failed"));
|
||||
let plain = Updater::new("http://git.example/releases".into(), "0.1.0", temp_dir("upd4"));
|
||||
assert!(plain.check().await.unwrap_err().contains("https"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_app_and_its_installer_carry_the_same_version() {
|
||||
// installer.sh names the setup after the shell's version; the app compares its own; the
|
||||
// program file's version comes from the shell's Cargo.toml when tauri.conf.json sets none.
|
||||
let shell = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/shell/Cargo.toml")).unwrap();
|
||||
let v = shell.lines().find_map(|l| l.strip_prefix("version = \"")).unwrap().trim_end_matches('"');
|
||||
assert_eq!(v, env!("CARGO_PKG_VERSION"));
|
||||
let conf: Value = serde_json::from_str(&std::fs::read_to_string(
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/shell/tauri.conf.json")).unwrap()).unwrap();
|
||||
assert_eq!(conf.get("version"), None, "one version, in the Cargo.toml files");
|
||||
}
|
||||
}
|
||||
701
app/src/vdisplay.rs
Normal file
701
app/src/vdisplay.rs
Normal file
@@ -0,0 +1,701 @@
|
||||
//! The virtual monitor the bot browser's window sits on, so the user can share it in Discord's
|
||||
//! "Screen" tab while no real monitor shows it (admin.spec.md, "Virtual monitor";
|
||||
//! `docs/v2/pc-features.md`, "봇용 크롬창과 디스코드 화면공유").
|
||||
//!
|
||||
//! The monitor comes from the open-source Virtual Display Driver (MIT, signed by the SignPath
|
||||
//! Foundation), shipped next to `jarvis.exe` in `display-driver/`. Installing it needs the
|
||||
//! administrator once (`--display-driver install`, started elevated by the app); turning the monitor
|
||||
//! on and off does not: the app attaches it to the desktop only while the bot browser uses it.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
/// The monitor's mode (the driver offers only this one): small enough to share cheaply.
|
||||
pub const WIDTH: i32 = 1280;
|
||||
pub const HEIGHT: i32 = 720;
|
||||
pub const REFRESH: u32 = 30;
|
||||
/// The adapter name the driver's INF gives its monitors.
|
||||
pub const ADAPTER_NAME: &str = "Virtual Display Driver";
|
||||
pub const HARDWARE_ID: &str = "Root\\MttVDD";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
pub struct Rect {
|
||||
pub x: i32,
|
||||
pub y: i32,
|
||||
pub w: i32,
|
||||
pub h: i32,
|
||||
}
|
||||
|
||||
/// Where the virtual monitor goes: right of and below the rightmost real monitor, touching it only
|
||||
/// at that monitor's bottom-right corner, so the mouse hardly ever wanders onto it and no real
|
||||
/// monitor overlaps it (nothing reaches further right).
|
||||
pub fn corner(real: &[Rect]) -> (i32, i32) {
|
||||
real.iter()
|
||||
.max_by_key(|r| (r.x + r.w, r.y + r.h))
|
||||
.map(|r| (r.x + r.w, r.y + r.h))
|
||||
.unwrap_or((0, 0))
|
||||
}
|
||||
|
||||
/// The driver's settings file: one monitor with one mode, the driver's other options at their
|
||||
/// defaults.
|
||||
pub fn settings_xml() -> String {
|
||||
format!(
|
||||
"<?xml version='1.0' encoding='utf-8'?>\r\n<vdd_settings>\r\n\
|
||||
<monitors><count>1</count></monitors>\r\n\
|
||||
<gpu><friendlyname>default</friendlyname></gpu>\r\n\
|
||||
<global><g_refresh_rate>{REFRESH}</g_refresh_rate></global>\r\n\
|
||||
<resolutions><resolution><width>{WIDTH}</width><height>{HEIGHT}</height><refresh_rate>{REFRESH}</refresh_rate></resolution></resolutions>\r\n\
|
||||
<options><CustomEdid>false</CustomEdid><PreventSpoof>false</PreventSpoof><EdidCeaOverride>false</EdidCeaOverride>\
|
||||
<HardwareCursor>true</HardwareCursor><SDR10bit>false</SDR10bit><HDRPlus>false</HDRPlus>\
|
||||
<logging>false</logging><debuglogging>false</debuglogging></options>\r\n</vdd_settings>\r\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Where the driver reads its settings: the driver runs as a system service, which cannot read a
|
||||
/// data folder in the user's profile, so this is the one place outside the data folder.
|
||||
pub fn config_dir() -> Option<PathBuf> {
|
||||
std::env::var_os("ProgramData").map(|p| PathBuf::from(p).join("JarvisBot").join("display-driver"))
|
||||
}
|
||||
|
||||
/// The driver files shipped with the app.
|
||||
pub fn driver_dir() -> Option<PathBuf> {
|
||||
std::env::current_exe().ok()?.parent().map(|d| d.join("display-driver"))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
pub struct DriverStatus {
|
||||
/// Windows only.
|
||||
pub supported: bool,
|
||||
/// Installed by this app with its monitor there: usable.
|
||||
pub installed: bool,
|
||||
/// Something this app installed is left without a usable monitor (a removal or a failed
|
||||
/// install's clean-up that did not finish): to be removed.
|
||||
pub leftover: bool,
|
||||
/// Another program's copy of the same driver is installed; the app leaves it alone.
|
||||
pub other: bool,
|
||||
/// The button the screen offers: `install`, `uninstall` (also for a leftover), or none.
|
||||
pub action: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// The status from whether the driver is ours (its registry value points at our settings), its
|
||||
/// monitor is there, and any of its devices is.
|
||||
#[cfg_attr(not(windows), allow(dead_code))]
|
||||
fn driver_status(ours: bool, monitor: bool, devices: bool) -> DriverStatus {
|
||||
let installed = ours && monitor;
|
||||
let leftover = ours && !installed;
|
||||
let other = (monitor || devices) && !ours;
|
||||
let action = if ours { Some("uninstall") } else if other { None } else { Some("install") };
|
||||
DriverStatus { supported: true, installed, leftover, other, action }
|
||||
}
|
||||
|
||||
/// After a failed install: whether to forget the driver (its registry value and settings folder)
|
||||
/// and what to report. Only an attempt that was not ours before and left no device and no package
|
||||
/// (1168: not in the driver store) is forgotten; otherwise it stays ours so the screen can remove
|
||||
/// what is left.
|
||||
#[cfg_attr(not(windows), allow(dead_code))]
|
||||
fn after_failed_install(failure: String, was_ours: bool, devices_left: usize, package: Result<(), u32>) -> (bool, String) {
|
||||
match removal_done(devices_left, package) {
|
||||
Ok(()) => (!was_ours, failure),
|
||||
Err(e) => (false, format!("{failure}. 정리하지 못한 것이 남아 있으니 제거를 눌러 주세요 ({e})")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn status() -> DriverStatus {
|
||||
DriverStatus { supported: false, installed: false, leftover: false, other: false, action: None }
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn attach() -> Result<Rect, String> {
|
||||
Err("가상 모니터는 Windows에서만 씁니다".into())
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn detach() {}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub fn settle_windows(_pid: u32) -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub async fn run_elevated(_action: &str) -> Result<(), String> {
|
||||
Err("가상 모니터는 Windows에서만 씁니다".into())
|
||||
}
|
||||
|
||||
/// The elevated half's exit code when removing finds no driver of this app's (none, or another
|
||||
/// program's copy): the uninstaller goes on. 0 is done, 1 failed (the uninstaller stops).
|
||||
pub const NOTHING_TO_REMOVE: i32 = 2;
|
||||
|
||||
/// `jarvis.exe --display-driver install|uninstall --result FILE`: the elevated half of
|
||||
/// `run_elevated`; the outcome (empty, or why it failed) goes to FILE. `None` without the option.
|
||||
pub fn elevated_main(args: &[String]) -> Option<i32> {
|
||||
let i = args.iter().position(|a| a == "--display-driver")?;
|
||||
let action = args.get(i + 1).map(String::as_str).unwrap_or("");
|
||||
let result = args.iter().position(|a| a == "--result").and_then(|j| args.get(j + 1)).map(PathBuf::from);
|
||||
if action == "uninstall" && !owns_driver() {
|
||||
if let Some(f) = &result {
|
||||
let _ = std::fs::write(f, "");
|
||||
}
|
||||
return Some(NOTHING_TO_REMOVE);
|
||||
}
|
||||
let outcome = match (action, driver_dir()) {
|
||||
(_, None) => Err("앱 폴더를 찾지 못했습니다".to_owned()),
|
||||
("install", Some(d)) => install(&d),
|
||||
("uninstall", Some(d)) => uninstall(&d),
|
||||
_ => Err(format!("알 수 없는 동작: {action}")),
|
||||
};
|
||||
if let Some(f) = result {
|
||||
let _ = std::fs::write(f, outcome.as_ref().err().map(String::as_str).unwrap_or(""));
|
||||
}
|
||||
Some(if outcome.is_ok() { 0 } else { 1 })
|
||||
}
|
||||
|
||||
/// Whether this app's driver (or something it left) is on the PC.
|
||||
fn owns_driver() -> bool {
|
||||
#[cfg(windows)]
|
||||
return win::ours();
|
||||
#[cfg(not(windows))]
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn install(_driver: &Path) -> Result<(), String> {
|
||||
Err("가상 모니터는 Windows에서만 씁니다".into())
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn uninstall(_driver: &Path) -> Result<(), String> {
|
||||
Err("가상 모니터는 Windows에서만 씁니다".into())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub use win::{attach, detach, run_elevated, settle_windows, status, virtual_display_name};
|
||||
#[cfg(windows)]
|
||||
use win::{install, uninstall};
|
||||
|
||||
#[cfg(windows)]
|
||||
mod win {
|
||||
use super::*;
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use windows_sys::core::BOOL;
|
||||
use windows_sys::Win32::Devices::DeviceAndDriverInstallation::*;
|
||||
use windows_sys::Win32::Foundation::*;
|
||||
use windows_sys::Win32::Graphics::Gdi::*;
|
||||
use windows_sys::Win32::System::Registry::{RegGetValueW, HKEY_LOCAL_MACHINE, RRF_RT_REG_SZ};
|
||||
use windows_sys::Win32::System::Threading::{GetExitCodeProcess, WaitForSingleObject, INFINITE};
|
||||
use windows_sys::Win32::UI::Shell::{ShellExecuteExW, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW};
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
||||
|
||||
const REG_KEY: &str = "SOFTWARE\\MikeTheTech\\VirtualDisplayDriver";
|
||||
|
||||
fn wide(s: &str) -> Vec<u16> {
|
||||
s.encode_utf16().chain(Some(0)).collect()
|
||||
}
|
||||
|
||||
fn wide_path(p: &Path) -> Vec<u16> {
|
||||
p.as_os_str().encode_wide().chain(Some(0)).collect()
|
||||
}
|
||||
|
||||
fn text(buf: &[u16]) -> String {
|
||||
String::from_utf16_lossy(&buf[..buf.iter().position(|&c| c == 0).unwrap_or(buf.len())])
|
||||
}
|
||||
|
||||
struct Device {
|
||||
name: [u16; 32],
|
||||
string: String,
|
||||
attached: bool,
|
||||
}
|
||||
|
||||
fn devices() -> Vec<Device> {
|
||||
let mut out = vec![];
|
||||
for i in 0.. {
|
||||
let mut dd = DISPLAY_DEVICEW { cb: std::mem::size_of::<DISPLAY_DEVICEW>() as u32, ..Default::default() };
|
||||
// SAFETY: `dd` is a DISPLAY_DEVICEW with its size set, as the call requires.
|
||||
if unsafe { EnumDisplayDevicesW(std::ptr::null(), i, &mut dd, 0) } == 0 {
|
||||
break;
|
||||
}
|
||||
out.push(Device { name: dd.DeviceName, string: text(&dd.DeviceString),
|
||||
attached: dd.StateFlags & DISPLAY_DEVICE_ATTACHED_TO_DESKTOP != 0 });
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn current(name: &[u16; 32]) -> Option<Rect> {
|
||||
let mut dm = DEVMODEW { dmSize: std::mem::size_of::<DEVMODEW>() as u16, ..Default::default() };
|
||||
// SAFETY: `name` is NUL-terminated (from EnumDisplayDevicesW) and `dm` has its size set.
|
||||
if unsafe { EnumDisplaySettingsExW(name.as_ptr(), ENUM_CURRENT_SETTINGS, &mut dm, 0) } == 0 {
|
||||
return None;
|
||||
}
|
||||
// SAFETY: for a display, the position variant of the union is the one Windows fills.
|
||||
let pos = unsafe { dm.Anonymous1.Anonymous2.dmPosition };
|
||||
Some(Rect { x: pos.x, y: pos.y, w: dm.dmPelsWidth as i32, h: dm.dmPelsHeight as i32 })
|
||||
}
|
||||
|
||||
/// Put the display at `rect`, or take it off the desktop with `None`, and keep that.
|
||||
fn set_mode(name: &[u16; 32], rect: Option<Rect>) -> Result<(), String> {
|
||||
let mut dm = DEVMODEW { dmSize: std::mem::size_of::<DEVMODEW>() as u16, ..Default::default() };
|
||||
// SAFETY: as in `current`; a failure leaves `dm` zeroed, which the fields below complete.
|
||||
unsafe { EnumDisplaySettingsExW(name.as_ptr(), ENUM_REGISTRY_SETTINGS, &mut dm, 0) };
|
||||
let r = rect.unwrap_or(Rect { x: 0, y: 0, w: 0, h: 0 });
|
||||
dm.dmFields = DM_POSITION | DM_PELSWIDTH | DM_PELSHEIGHT;
|
||||
dm.Anonymous1.Anonymous2 = DEVMODEW_0_1 { dmPosition: POINTL { x: r.x, y: r.y }, ..Default::default() };
|
||||
dm.dmPelsWidth = r.w as u32;
|
||||
dm.dmPelsHeight = r.h as u32;
|
||||
// SAFETY: valid NUL-terminated device name and DEVMODEW; null window and parameter.
|
||||
let changed = unsafe {
|
||||
ChangeDisplaySettingsExW(name.as_ptr(), &dm, std::ptr::null_mut(), CDS_UPDATEREGISTRY | CDS_NORESET, std::ptr::null())
|
||||
};
|
||||
// SAFETY: the documented "apply the stored changes" call: all null, no flags.
|
||||
let applied = unsafe { ChangeDisplaySettingsExW(std::ptr::null(), std::ptr::null(), std::ptr::null_mut(), 0, std::ptr::null()) };
|
||||
if changed != DISP_CHANGE_SUCCESSFUL || applied != DISP_CHANGE_SUCCESSFUL {
|
||||
return Err(format!("화면 설정을 바꾸지 못했습니다 ({changed}, {applied})"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn vdd_path() -> Option<String> {
|
||||
let mut buf = [0u16; 520];
|
||||
let mut size = (buf.len() * 2) as u32;
|
||||
let (key, value) = (wide(REG_KEY), wide("VDDPATH"));
|
||||
// SAFETY: valid NUL-terminated strings; `buf` holds `size` bytes.
|
||||
let r = unsafe {
|
||||
RegGetValueW(HKEY_LOCAL_MACHINE, key.as_ptr(), value.as_ptr(), RRF_RT_REG_SZ, std::ptr::null_mut(),
|
||||
buf.as_mut_ptr().cast(), &mut size)
|
||||
};
|
||||
(r == ERROR_SUCCESS).then(|| text(&buf))
|
||||
}
|
||||
|
||||
pub(super) fn ours() -> bool {
|
||||
match (vdd_path(), config_dir()) {
|
||||
(Some(p), Some(c)) => p.trim_end_matches('\\').eq_ignore_ascii_case(c.to_string_lossy().trim_end_matches('\\')),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn adapter() -> Option<Device> {
|
||||
devices().into_iter().find(|d| d.string == ADAPTER_NAME)
|
||||
}
|
||||
|
||||
/// Installed means ours with its monitor there (the driver loaded); ours without it is a
|
||||
/// leftover the screen offers to remove.
|
||||
pub fn status() -> DriverStatus {
|
||||
driver_status(ours(), adapter().is_some(), !hardware_devices().is_empty())
|
||||
}
|
||||
|
||||
/// Put the virtual monitor on the desktop at `corner` and return where it is.
|
||||
pub fn attach() -> Result<Rect, String> {
|
||||
if !ours() {
|
||||
return Err("가상 모니터 드라이버가 설치되지 않았습니다".into());
|
||||
}
|
||||
let v = adapter().ok_or("가상 모니터를 찾지 못했습니다")?;
|
||||
let real: Vec<Rect> = devices().iter().filter(|d| d.attached && d.string != ADAPTER_NAME)
|
||||
.filter_map(|d| current(&d.name)).collect();
|
||||
let (x, y) = corner(&real);
|
||||
set_mode(&v.name, Some(Rect { x, y, w: WIDTH, h: HEIGHT }))?;
|
||||
current(&v.name).filter(|r| r.w > 0).ok_or_else(|| "가상 모니터가 켜지지 않았습니다".into())
|
||||
}
|
||||
|
||||
/// The virtual monitor's display name (`\\.\DISPLAY<n>`) while it is on the desktop: screen
|
||||
/// viewing leaves it out (admin.spec.md, "Screen viewing").
|
||||
pub fn virtual_display_name() -> Option<String> {
|
||||
adapter().filter(|d| d.attached).map(|d| text(&d.name))
|
||||
}
|
||||
|
||||
/// Take the virtual monitor off the desktop (when it is ours and on it).
|
||||
pub fn detach() {
|
||||
if !ours() {
|
||||
return;
|
||||
}
|
||||
if let Some(v) = adapter().filter(|d| d.attached) {
|
||||
if let Err(e) = set_mode(&v.name, None) {
|
||||
crate::warn_log!("browser", "Warning: the virtual monitor stays on: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Found {
|
||||
pid: u32,
|
||||
windows: Vec<HWND>,
|
||||
}
|
||||
|
||||
unsafe extern "system" fn collect(hwnd: HWND, lparam: LPARAM) -> BOOL {
|
||||
// SAFETY: `lparam` is the `Found` that `settle_windows` passes and outlives EnumWindows.
|
||||
let found = unsafe { &mut *(lparam as *mut Found) };
|
||||
let mut pid = 0u32;
|
||||
let mut class = [0u16; 64];
|
||||
// SAFETY: plain queries on a window handle EnumWindows gave; buffers are sized as passed.
|
||||
unsafe {
|
||||
GetWindowThreadProcessId(hwnd, &mut pid);
|
||||
if pid == found.pid && IsWindowVisible(hwnd) != 0 && GetWindow(hwnd, GW_OWNER).is_null()
|
||||
&& GetClassNameW(hwnd, class.as_mut_ptr(), class.len() as i32) > 0 && text(&class) == "Chrome_WidgetWin_1" {
|
||||
found.windows.push(hwnd);
|
||||
}
|
||||
}
|
||||
1
|
||||
}
|
||||
|
||||
/// Make process `pid`'s new top-level Chrome windows tool windows (not in Alt+Tab, Win+Tab or
|
||||
/// the taskbar) and fill their monitor's work area (Windows may give the virtual monitor a
|
||||
/// taskbar of its own, which would cover the page's bottom), all without activating them. The
|
||||
/// bot's windows stay behind all others, the newest in front of the older ones. Returns how many
|
||||
/// windows were new.
|
||||
pub fn settle_windows(pid: u32) -> usize {
|
||||
let mut found = Found { pid, windows: vec![] };
|
||||
// SAFETY: `collect` only reads windows and writes into `found`, which lives through the call.
|
||||
unsafe { EnumWindows(Some(collect), &mut found as *mut Found as LPARAM) };
|
||||
// SAFETY: a plain query on a window handle EnumWindows gave.
|
||||
let (old, new): (Vec<HWND>, Vec<HWND>) = found.windows.into_iter()
|
||||
.partition(|&h| unsafe { GetWindowLongPtrW(h, GWL_EXSTYLE) } as u32 & WS_EX_TOOLWINDOW != 0);
|
||||
for &h in &new {
|
||||
// SAFETY: style changes on a window of our own child process; hiding and showing it
|
||||
// again is what makes the taskbar notice the new style.
|
||||
unsafe {
|
||||
let ex = GetWindowLongPtrW(h, GWL_EXSTYLE) as u32;
|
||||
ShowWindow(h, SW_HIDE);
|
||||
SetWindowLongPtrW(h, GWL_EXSTYLE, ((ex | WS_EX_TOOLWINDOW) & !WS_EX_APPWINDOW) as isize);
|
||||
ShowWindow(h, SW_SHOWNOACTIVATE);
|
||||
let mut mi = MONITORINFO { cbSize: std::mem::size_of::<MONITORINFO>() as u32, ..Default::default() };
|
||||
if GetMonitorInfoW(MonitorFromWindow(h, MONITOR_DEFAULTTONEAREST), &mut mi) != 0 {
|
||||
let w = mi.rcWork;
|
||||
SetWindowPos(h, HWND_BOTTOM, w.left, w.top, w.right - w.left, w.bottom - w.top, SWP_NOACTIVATE);
|
||||
} else {
|
||||
SetWindowPos(h, HWND_BOTTOM, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !new.is_empty() {
|
||||
// Top first, each to the bottom: they keep their order, all behind the new ones.
|
||||
for &h in &old {
|
||||
// SAFETY: a z-order change of our own child process's window, without activating it.
|
||||
unsafe { SetWindowPos(h, HWND_BOTTOM, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE) };
|
||||
}
|
||||
}
|
||||
new.len()
|
||||
}
|
||||
|
||||
/// The root devices with the driver's hardware ID, in any class (a device whose driver install
|
||||
/// was declined has none), present or not.
|
||||
fn hardware_devices() -> Vec<SP_DEVINFO_DATA> {
|
||||
let (set, list) = hardware_set();
|
||||
if set != INVALID_HANDLE_VALUE as HDEVINFO {
|
||||
// SAFETY: the list from `hardware_set`, no longer used.
|
||||
unsafe { SetupDiDestroyDeviceInfoList(set) };
|
||||
}
|
||||
list
|
||||
}
|
||||
|
||||
fn hardware_set() -> (HDEVINFO, Vec<SP_DEVINFO_DATA>) {
|
||||
let root = wide("ROOT");
|
||||
let flags = DIGCF_ALLCLASSES;
|
||||
// SAFETY: a device list of the root enumerator; destroyed by the callers that keep it.
|
||||
let set = unsafe { SetupDiGetClassDevsW(std::ptr::null(), root.as_ptr(), std::ptr::null_mut(), flags) };
|
||||
let mut out = vec![];
|
||||
if set == INVALID_HANDLE_VALUE as HDEVINFO {
|
||||
return (set, out);
|
||||
}
|
||||
for i in 0.. {
|
||||
let mut di = SP_DEVINFO_DATA { cbSize: std::mem::size_of::<SP_DEVINFO_DATA>() as u32, ..Default::default() };
|
||||
// SAFETY: `di` has its size set; the loop ends at the first index past the list.
|
||||
if unsafe { SetupDiEnumDeviceInfo(set, i, &mut di) } == 0 {
|
||||
break;
|
||||
}
|
||||
let mut buf = [0u16; 256];
|
||||
// SAFETY: `buf` holds the byte count passed.
|
||||
let ok = unsafe {
|
||||
SetupDiGetDeviceRegistryPropertyW(set, &di, SPDRP_HARDWAREID, std::ptr::null_mut(), buf.as_mut_ptr().cast(),
|
||||
(buf.len() * 2) as u32, std::ptr::null_mut())
|
||||
};
|
||||
if ok != 0 && text(&buf).eq_ignore_ascii_case(HARDWARE_ID) {
|
||||
out.push(di);
|
||||
}
|
||||
}
|
||||
(set, out)
|
||||
}
|
||||
|
||||
fn run(cmd: &str, args: &[&str]) -> Result<(), String> {
|
||||
use std::os::windows::process::CommandExt;
|
||||
let s = std::process::Command::new(cmd).args(args).creation_flags(0x0800_0000)
|
||||
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).status()
|
||||
.map_err(|e| format!("{cmd}: {e}"))?;
|
||||
if s.success() { Ok(()) } else { Err(format!("{cmd} failed ({s})")) }
|
||||
}
|
||||
|
||||
/// Elevated: settings folder, the driver's registry value, the device, and its driver. Windows
|
||||
/// asks the user once whether to install software from the SignPath Foundation. A failure (also
|
||||
/// the user saying no) undoes what this attempt made, so nothing looks installed and the next
|
||||
/// attempt starts clean.
|
||||
pub(super) fn install(driver: &Path) -> Result<(), String> {
|
||||
let inf = driver.join("MttVDD.inf");
|
||||
if !inf.is_file() {
|
||||
return Err("드라이버 파일이 없습니다 (display-driver\\MttVDD.inf)".into());
|
||||
}
|
||||
let (was_ours, had_device) = (ours(), !hardware_devices().is_empty());
|
||||
if (had_device || adapter().is_some()) && !was_ours {
|
||||
return Err("다른 프로그램이 설치한 가상 디스플레이 드라이버가 이미 있습니다".into());
|
||||
}
|
||||
let Err(failure) = install_steps(&inf, had_device) else { return Ok(()) };
|
||||
let left = if had_device { 0 } else { remove_devices() };
|
||||
let package = if was_ours || left > 0 { Ok(()) } else { uninstall_package(&inf) };
|
||||
let (forget, why) = after_failed_install(failure, was_ours, left, package);
|
||||
if forget {
|
||||
let _ = run("reg", &["delete", &format!("HKLM\\{REG_KEY}"), "/f"]);
|
||||
remove_config_dir();
|
||||
}
|
||||
Err(why)
|
||||
}
|
||||
|
||||
fn install_steps(inf: &Path, had_device: bool) -> Result<(), String> {
|
||||
let dir = config_dir().ok_or("ProgramData 폴더를 찾지 못했습니다")?;
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
std::fs::write(dir.join("vdd_settings.xml"), settings_xml()).map_err(|e| e.to_string())?;
|
||||
// Only the system, administrators, the driver's service account (LOCAL SERVICE) and users
|
||||
// reading: no user can change what the driver reads.
|
||||
let d = dir.to_string_lossy().into_owned();
|
||||
run("icacls", &[&d, "/inheritance:r", "/grant:r", "*S-1-5-18:(OI)(CI)F", "*S-1-5-32-544:(OI)(CI)F",
|
||||
"*S-1-5-19:(OI)(CI)RX", "*S-1-5-32-545:(OI)(CI)RX"])?;
|
||||
run("reg", &["add", &format!("HKLM\\{REG_KEY}"), "/v", "VDDPATH", "/t", "REG_SZ", "/d", &d, "/f"])?;
|
||||
if !had_device {
|
||||
create_device()?;
|
||||
}
|
||||
let (hw, inf_w) = (wide(HARDWARE_ID), wide_path(inf));
|
||||
let mut reboot: BOOL = 0;
|
||||
// SAFETY: valid NUL-terminated strings; no parent window.
|
||||
if unsafe { UpdateDriverForPlugAndPlayDevicesW(std::ptr::null_mut(), hw.as_ptr(), inf_w.as_ptr(), INSTALLFLAG_FORCE, &mut reboot) } == 0 {
|
||||
// SAFETY: plain call.
|
||||
let e = unsafe { GetLastError() };
|
||||
// No to Windows' question about the publisher comes back as "publisher not trusted".
|
||||
const PUBLISHER_NOT_TRUSTED: u32 = 0xE000_0243;
|
||||
return Err(if e == ERROR_CANCELLED || e == PUBLISHER_NOT_TRUSTED { "드라이버 설치를 취소했습니다".into() }
|
||||
else { format!("드라이버를 설치하지 못했습니다 ({e})") });
|
||||
}
|
||||
// The new monitor joins the desktop at once; it stays off until the bot browser uses it.
|
||||
for _ in 0..50 {
|
||||
if adapter().is_some_and(|a| a.attached) {
|
||||
detach();
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove every device with the driver's hardware ID; returns how many are left.
|
||||
fn remove_devices() -> usize {
|
||||
let (set, list) = hardware_set();
|
||||
for di in &list {
|
||||
// SAFETY: removing a device of the list `set`, which is destroyed after the loop.
|
||||
unsafe { SetupDiCallClassInstaller(DIF_REMOVE, set, di) };
|
||||
}
|
||||
if set != INVALID_HANDLE_VALUE as HDEVINFO {
|
||||
// SAFETY: the list from `hardware_set`.
|
||||
unsafe { SetupDiDestroyDeviceInfoList(set) };
|
||||
}
|
||||
hardware_devices().len()
|
||||
}
|
||||
|
||||
/// Remove the driver package from the driver store; `Err` with the Windows error code.
|
||||
fn uninstall_package(inf: &Path) -> Result<(), u32> {
|
||||
let inf = wide_path(inf);
|
||||
let mut reboot: BOOL = 0;
|
||||
// SAFETY: valid NUL-terminated path; no parent window.
|
||||
if unsafe { DiUninstallDriverW(std::ptr::null_mut(), inf.as_ptr(), 0, &mut reboot) } != 0 {
|
||||
return Ok(());
|
||||
}
|
||||
// SAFETY: plain call.
|
||||
Err(unsafe { GetLastError() })
|
||||
}
|
||||
|
||||
fn remove_config_dir() {
|
||||
if let Some(dir) = config_dir() {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
if let Some(parent) = dir.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn create_device() -> Result<(), String> {
|
||||
let (name, hw) = (wide("Display"), {
|
||||
let mut v: Vec<u16> = HARDWARE_ID.encode_utf16().collect();
|
||||
v.extend([0, 0]);
|
||||
v
|
||||
});
|
||||
// SAFETY: the devcon "install" sequence: a list, one generated device with the hardware ID,
|
||||
// registered with the class installer; the list is destroyed on every path.
|
||||
unsafe {
|
||||
let set = SetupDiCreateDeviceInfoList(&GUID_DEVCLASS_DISPLAY, std::ptr::null_mut());
|
||||
if set == INVALID_HANDLE_VALUE as HDEVINFO {
|
||||
return Err(format!("장치 목록을 만들지 못했습니다 ({})", GetLastError()));
|
||||
}
|
||||
let mut di = SP_DEVINFO_DATA { cbSize: std::mem::size_of::<SP_DEVINFO_DATA>() as u32, ..Default::default() };
|
||||
let ok = SetupDiCreateDeviceInfoW(set, name.as_ptr(), &GUID_DEVCLASS_DISPLAY, std::ptr::null(), std::ptr::null_mut(), DICD_GENERATE_ID, &mut di) != 0
|
||||
&& SetupDiSetDeviceRegistryPropertyW(set, &mut di, SPDRP_HARDWAREID, hw.as_ptr().cast(), (hw.len() * 2) as u32) != 0
|
||||
&& SetupDiCallClassInstaller(DIF_REGISTERDEVICE, set, &di) != 0;
|
||||
let e = GetLastError();
|
||||
SetupDiDestroyDeviceInfoList(set);
|
||||
if ok { Ok(()) } else { Err(format!("가상 모니터 장치를 만들지 못했습니다 ({e})")) }
|
||||
}
|
||||
}
|
||||
|
||||
/// Elevated: remove the device, the driver, the registry value and the settings folder; only
|
||||
/// when this app installed it. The registry value (what makes it ours) goes only once the
|
||||
/// device and the driver are gone, so a failed removal can be retried.
|
||||
pub(super) fn uninstall(driver: &Path) -> Result<(), String> {
|
||||
if !ours() {
|
||||
return Err("이 앱이 설치한 가상 디스플레이 드라이버가 아닙니다".into());
|
||||
}
|
||||
let left = remove_devices();
|
||||
let package = if left == 0 { uninstall_package(&driver.join("MttVDD.inf")) } else { Ok(()) };
|
||||
removal_done(left, package)?;
|
||||
run("reg", &["delete", &format!("HKLM\\{REG_KEY}"), "/f"])
|
||||
.map_err(|e| format!("드라이버 등록 정보를 지우지 못했습니다: {e}"))?;
|
||||
remove_config_dir();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run `jarvis.exe --display-driver <action>` as administrator (Windows asks the user) and wait.
|
||||
pub async fn run_elevated(action: &str) -> Result<(), String> {
|
||||
let exe = std::env::current_exe().map_err(|e| e.to_string())?;
|
||||
let result = std::env::temp_dir().join(format!("jarvis-display-{}.txt", std::process::id()));
|
||||
let _ = std::fs::remove_file(&result);
|
||||
let params = format!("--display-driver {action} --result \"{}\"", result.display());
|
||||
let code = tokio::task::spawn_blocking(move || -> Result<u32, String> {
|
||||
let (verb, file, params) = (wide("runas"), wide_path(&exe), wide(¶ms));
|
||||
// SAFETY: SHELLEXECUTEINFOW is plain data; all zero is its documented empty state.
|
||||
let mut info: SHELLEXECUTEINFOW = unsafe { std::mem::zeroed() };
|
||||
info.cbSize = std::mem::size_of::<SHELLEXECUTEINFOW>() as u32;
|
||||
info.fMask = SEE_MASK_NOCLOSEPROCESS;
|
||||
info.lpVerb = verb.as_ptr();
|
||||
info.lpFile = file.as_ptr();
|
||||
info.lpParameters = params.as_ptr();
|
||||
info.nShow = SW_HIDE;
|
||||
// SAFETY: `info` is filled as documented and its strings live through the call; the
|
||||
// process handle it returns is waited on and closed here.
|
||||
unsafe {
|
||||
if ShellExecuteExW(&mut info) == 0 {
|
||||
let e = GetLastError();
|
||||
return Err(if e == ERROR_CANCELLED { "관리자 권한을 받지 못했습니다".into() } else { format!("관리자 권한으로 실행하지 못했습니다 ({e})") });
|
||||
}
|
||||
WaitForSingleObject(info.hProcess, INFINITE);
|
||||
let mut code = 1u32;
|
||||
GetExitCodeProcess(info.hProcess, &mut code);
|
||||
CloseHandle(info.hProcess);
|
||||
Ok(code)
|
||||
}
|
||||
}).await.map_err(|e| e.to_string())??;
|
||||
let why = std::fs::read_to_string(&result).unwrap_or_default();
|
||||
let _ = std::fs::remove_file(&result);
|
||||
match code {
|
||||
0 => Ok(()),
|
||||
c if c as i32 == super::NOTHING_TO_REMOVE && action == "uninstall" => Ok(()),
|
||||
_ if !why.is_empty() => Err(why),
|
||||
c => Err(format!("드라이버 작업이 실패했습니다 (종료 코드 {c})")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a removal may forget the driver was ours: no device left, and the driver package
|
||||
/// removed or not in the driver store (Windows error 1168, ERROR_NOT_FOUND).
|
||||
#[cfg_attr(not(windows), allow(dead_code))]
|
||||
fn removal_done(devices_left: usize, package: Result<(), u32>) -> Result<(), String> {
|
||||
if devices_left > 0 {
|
||||
return Err(format!("가상 모니터 장치 {devices_left}개를 지우지 못했습니다. 다시 시도해 주세요"));
|
||||
}
|
||||
match package {
|
||||
Ok(()) | Err(1168) => Ok(()),
|
||||
Err(e) => Err(format!("드라이버 패키지를 지우지 못했습니다 ({e}). 다시 시도해 주세요")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_failed_install_forgets_the_driver_only_when_its_leftovers_are_gone() {
|
||||
let declined = || "드라이버 설치를 취소했습니다".to_owned();
|
||||
// Cleaned up: forgotten (when it was not ours before), the failure reported as it was.
|
||||
assert_eq!(after_failed_install(declined(), false, 0, Err(1168)), (true, declined()));
|
||||
// A device left: kept ours, so the leftover can be removed; both reasons reported.
|
||||
let (forget, why) = after_failed_install(declined(), false, 1, Ok(()));
|
||||
assert!(!forget, "a device left keeps the registry value");
|
||||
assert!(why.starts_with(&declined()) && why.contains("제거"), "{why}");
|
||||
// A package left: the same.
|
||||
let (forget, why) = after_failed_install(declined(), false, 0, Err(5));
|
||||
assert!(!forget);
|
||||
assert!(why.contains("제거"), "{why}");
|
||||
// Ours before the attempt: never forgotten by a failed reinstall.
|
||||
assert!(!after_failed_install(declined(), true, 0, Ok(())).0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_left_of_the_driver_can_always_be_removed_from_the_screen() {
|
||||
// Installed and working: remove.
|
||||
let s = driver_status(true, true, true);
|
||||
assert!(s.installed && !s.leftover && !s.other);
|
||||
assert_eq!(s.action, Some("uninstall"));
|
||||
// Device gone but the package (and the registry value) left after a failed removal.
|
||||
let s = driver_status(true, false, false);
|
||||
assert!(!s.installed && s.leftover);
|
||||
assert_eq!(s.action, Some("uninstall"), "the leftover can be removed again");
|
||||
// Nothing of ours: install. Another program's copy: nothing to press.
|
||||
assert_eq!(driver_status(false, false, false).action, Some("install"));
|
||||
let s = driver_status(false, true, true);
|
||||
assert!(s.other && !s.installed && !s.leftover);
|
||||
assert_eq!(s.action, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_removal_forgets_the_driver_only_when_device_and_package_are_gone() {
|
||||
assert_eq!(removal_done(0, Ok(())), Ok(()));
|
||||
assert_eq!(removal_done(0, Err(1168)), Ok(()), "a package not in the store is already gone");
|
||||
assert!(removal_done(1, Ok(())).is_err(), "a device left keeps it ours, so it can be retried");
|
||||
assert!(removal_done(0, Err(5)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_monitor_touches_the_rightmost_monitor_only_at_its_corner() {
|
||||
let one = [Rect { x: 0, y: 0, w: 1920, h: 1080 }];
|
||||
assert_eq!(corner(&one), (1920, 1080));
|
||||
// A second monitor left of the main one, and one above-right: the rightmost decides.
|
||||
let three = [Rect { x: 0, y: 0, w: 2560, h: 1440 }, Rect { x: -1920, y: 0, w: 1920, h: 1080 },
|
||||
Rect { x: 2560, y: -400, w: 1080, h: 1920 }];
|
||||
let (x, y) = corner(&three);
|
||||
assert_eq!((x, y), (3640, 1520));
|
||||
for r in &three {
|
||||
assert!(r.x + r.w <= x, "no real monitor reaches into the virtual one");
|
||||
}
|
||||
// Two equally far right: the lower corner, so neither overlaps.
|
||||
let stacked = [Rect { x: 0, y: 0, w: 1920, h: 1080 }, Rect { x: 0, y: 1080, w: 1920, h: 1080 }];
|
||||
assert_eq!(corner(&stacked), (1920, 2160));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_driver_gets_one_monitor_with_the_one_mode() {
|
||||
let x = settings_xml();
|
||||
assert!(x.contains("<count>1</count>"));
|
||||
assert!(x.contains(&format!("<width>{WIDTH}</width><height>{HEIGHT}</height><refresh_rate>{REFRESH}</refresh_rate>")));
|
||||
assert_eq!(x.matches("<resolution>").count(), 1);
|
||||
assert!(x.contains("<CustomEdid>false</CustomEdid>"), "the driver never loads an EDID file from its folder");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_display_driver_option_runs_the_elevated_half() {
|
||||
assert_eq!(elevated_main(&["jarvis".into(), "--location".into(), "x".into()]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn removing_a_driver_this_app_does_not_own_says_there_is_nothing_to_remove() {
|
||||
// The uninstaller goes on with code 2, and stops on a failure (code 1).
|
||||
let result = std::env::temp_dir().join(format!("jarvis-vdd-test-{}.txt", std::process::id()));
|
||||
let args: Vec<String> = ["jarvis", "--display-driver", "uninstall", "--result", result.to_str().unwrap()]
|
||||
.map(String::from).into();
|
||||
assert_eq!(elevated_main(&args), Some(NOTHING_TO_REMOVE));
|
||||
assert_eq!(std::fs::read_to_string(&result).unwrap(), "");
|
||||
let _ = std::fs::remove_file(&result);
|
||||
let bad: Vec<String> = ["jarvis", "--display-driver", "nope"].map(String::from).into();
|
||||
assert_eq!(elevated_main(&bad), Some(1));
|
||||
}
|
||||
}
|
||||
772
app/src/worker_client.rs
Normal file
772
app/src/worker_client.rs
Normal file
@@ -0,0 +1,772 @@
|
||||
//! WebSocket client for the Python voice worker (`src/voice_worker/voice_worker.spec.md`).
|
||||
//!
|
||||
//! Keeps one connection open, says hello with the app's token, and reconnects with backoff when the
|
||||
//! worker restarts. While disconnected, audio is dropped (the worker would not know the speaker's
|
||||
//! state anyway) and every open TTS request ends as failed, so playback never waits forever.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use tokio::sync::{mpsc, watch, Notify};
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
use crate::frames::{decode_tts_frame, encode_audio_frame, KIND_TTS};
|
||||
use crate::playback::TtsBackend;
|
||||
use crate::{debug_log, error_log};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PcmFormat {
|
||||
pub rate: u32,
|
||||
pub channels: u8,
|
||||
}
|
||||
|
||||
/// Discord voice after Opus decoding, and what the worker is asked to send back.
|
||||
pub const DISCORD_PCM: PcmFormat = PcmFormat { rate: 48000, channels: 2 };
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WorkerClientOptions {
|
||||
pub url: String,
|
||||
pub token: String,
|
||||
pub audio: PcmFormat,
|
||||
pub tts: PcmFormat,
|
||||
pub stt: Map<String, Value>,
|
||||
/// The worker's memory settings, sent in `hello` (voice_worker.spec.md).
|
||||
pub memory: Map<String, Value>,
|
||||
/// First reconnect delay; doubles up to `max_backoff`.
|
||||
pub backoff: Duration,
|
||||
pub max_backoff: Duration,
|
||||
pub hello_timeout: Duration,
|
||||
}
|
||||
|
||||
impl WorkerClientOptions {
|
||||
pub fn new(url: impl Into<String>, token: impl Into<String>) -> Self {
|
||||
Self {
|
||||
url: url.into(),
|
||||
token: token.into(),
|
||||
audio: DISCORD_PCM,
|
||||
tts: DISCORD_PCM,
|
||||
stt: Map::new(),
|
||||
memory: Map::new(),
|
||||
backoff: Duration::from_millis(500),
|
||||
max_backoff: Duration::from_secs(5),
|
||||
hello_timeout: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct HelloOk {
|
||||
pub stt_ready: bool,
|
||||
pub tts_ready: bool,
|
||||
#[serde(default)]
|
||||
pub stt: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct SpeechStart {
|
||||
pub speaker: String,
|
||||
pub utt: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct Partial {
|
||||
pub speaker: String,
|
||||
pub utt: u32,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct Final {
|
||||
pub speaker: String,
|
||||
pub utt: u32,
|
||||
pub text: String,
|
||||
pub end_reason: String,
|
||||
pub audio_ms: u64,
|
||||
pub wait_ms: u64,
|
||||
pub stt_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct TtsDone {
|
||||
pub id: String,
|
||||
pub chunks: u32,
|
||||
pub cancelled: bool,
|
||||
pub failed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct WorkerError {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
#[serde(default)]
|
||||
pub id: Option<String>,
|
||||
}
|
||||
|
||||
/// The brain's chat channel tool asks the app to act (voice_worker.spec.md, chat_channels.spec.md).
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct ChatAction {
|
||||
pub id: String,
|
||||
/// `send`, `edit`, `react`, `delete`, `pin` or `unpin`.
|
||||
pub action: String,
|
||||
pub channel: String,
|
||||
#[serde(default)]
|
||||
pub message_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
#[serde(default)]
|
||||
pub emoji: Option<String>,
|
||||
/// Who asked (for the activity log).
|
||||
#[serde(default)]
|
||||
pub user: Option<String>,
|
||||
}
|
||||
|
||||
/// The brain's schedule tool asks the app for the person asking (voice_worker.spec.md, "Schedules").
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct ScheduleAction {
|
||||
pub id: String,
|
||||
/// `add`, `list` or `delete`.
|
||||
pub action: String,
|
||||
pub user: String,
|
||||
/// `voice` or `chat`: where they asked.
|
||||
pub via: String,
|
||||
#[serde(default)]
|
||||
pub channel: Option<String>,
|
||||
#[serde(default)]
|
||||
pub kind: Option<String>,
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
#[serde(default)]
|
||||
pub at: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub repeat: Option<String>,
|
||||
/// delete: the schedule's id.
|
||||
#[serde(default)]
|
||||
pub schedule: Option<String>,
|
||||
}
|
||||
|
||||
/// The brain's personalNotes tool asks the app for the person asking (voice_worker.spec.md,
|
||||
/// "Personal notes").
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct NoteAction {
|
||||
pub id: String,
|
||||
/// `add`, `list` or `delete`.
|
||||
pub action: String,
|
||||
pub user: String,
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
/// delete: the note's id.
|
||||
#[serde(default)]
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
/// The brain's screenView tool asks the app for the person asking (voice_worker.spec.md, "Screen
|
||||
/// viewing").
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct ScreenAction {
|
||||
pub id: String,
|
||||
/// `capture` or `list`.
|
||||
pub action: String,
|
||||
pub user: String,
|
||||
/// capture: `screen`, `monitor` or `window`.
|
||||
#[serde(default)]
|
||||
pub target: Option<String>,
|
||||
#[serde(default)]
|
||||
pub monitor: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub window: Option<String>,
|
||||
/// `fast`, `normal` or `detailed`.
|
||||
#[serde(default)]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum WorkerEvent {
|
||||
Ready(HelloOk),
|
||||
Disconnected,
|
||||
SpeechStart(SpeechStart),
|
||||
Partial(Partial),
|
||||
Final(Final),
|
||||
TtsAudio { id: String, seq: u32, pcm: Vec<u8> },
|
||||
TtsDone(TtsDone),
|
||||
ChatAction(ChatAction),
|
||||
ScheduleAction(ScheduleAction),
|
||||
NoteAction(NoteAction),
|
||||
ScreenAction(ScreenAction),
|
||||
/// The brain's answer to `reply` (empty and `failed` when it gave none).
|
||||
ReplyText { id: String, text: String, failed: bool },
|
||||
/// Was the utterance said to Jarvis? (a failed judgement is a no)
|
||||
Addressed { id: String, addressed: bool },
|
||||
/// Candidate ids, most important first (empty when the ranking failed).
|
||||
Ranked { id: String, order: Vec<String> },
|
||||
Error(WorkerError),
|
||||
}
|
||||
|
||||
struct State {
|
||||
conn: Option<mpsc::UnboundedSender<Message>>,
|
||||
stt: Map<String, Value>,
|
||||
/// TTS requests sent and not yet done, so a lost connection can end them.
|
||||
open_tts: HashSet<String>,
|
||||
closed: bool,
|
||||
dropped_frames: u64,
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
opts: WorkerClientOptions,
|
||||
state: Mutex<State>,
|
||||
events: mpsc::UnboundedSender<WorkerEvent>,
|
||||
ready: watch::Sender<Option<HelloOk>>,
|
||||
wake: Notify,
|
||||
/// The last switch the brain reported idle for (`llm_idle` names it).
|
||||
llm_idle: watch::Sender<u64>,
|
||||
/// The LLM state the brain should have: sent again on every connection, so a worker that
|
||||
/// missed it (a dropped connection, a restarted worker) catches up.
|
||||
llm: Mutex<Option<Value>>,
|
||||
/// The latest speech speed set while running, sent again on every connection: a restarted worker
|
||||
/// starts with the speed of its arguments.
|
||||
tts_speed: Mutex<Option<f64>>,
|
||||
/// The number of the latest switch.
|
||||
llm_switches: std::sync::atomic::AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct WorkerClient {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
impl WorkerClient {
|
||||
/// The client and the stream of worker events. Call `start()` to connect.
|
||||
pub fn new(opts: WorkerClientOptions) -> (Self, mpsc::UnboundedReceiver<WorkerEvent>) {
|
||||
let (events, rx) = mpsc::unbounded_channel();
|
||||
let stt = opts.stt.clone();
|
||||
let inner = Arc::new(Inner {
|
||||
opts,
|
||||
state: Mutex::new(State { conn: None, stt, open_tts: HashSet::new(), closed: false, dropped_frames: 0 }),
|
||||
events,
|
||||
ready: watch::channel(None).0,
|
||||
wake: Notify::new(),
|
||||
llm_idle: watch::channel(0).0,
|
||||
llm: Mutex::new(None),
|
||||
tts_speed: Mutex::new(None),
|
||||
llm_switches: std::sync::atomic::AtomicU64::new(0),
|
||||
});
|
||||
(Self { inner }, rx)
|
||||
}
|
||||
|
||||
/// Start connecting; reconnects until `close()`.
|
||||
pub fn start(&self) {
|
||||
let inner = self.inner.clone();
|
||||
tokio::spawn(async move { run(inner).await });
|
||||
}
|
||||
|
||||
/// The worker's `hello_ok` while connected (STT and TTS readiness), else None.
|
||||
pub fn hello(&self) -> Option<HelloOk> {
|
||||
self.inner.ready.borrow().clone()
|
||||
}
|
||||
|
||||
pub fn is_ready(&self) -> bool {
|
||||
self.inner.ready.borrow().is_some()
|
||||
}
|
||||
|
||||
/// Resolve once the worker has said hello_ok, or None after `timeout`.
|
||||
pub async fn wait_ready(&self, timeout: Duration) -> Option<HelloOk> {
|
||||
let mut rx = self.inner.ready.subscribe();
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
if let Some(h) = rx.borrow_and_update().clone() {
|
||||
return h;
|
||||
}
|
||||
if rx.changed().await.is_err() {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub fn close(&self) {
|
||||
{
|
||||
let mut st = self.inner.state.lock().unwrap();
|
||||
st.closed = true;
|
||||
if let Some(tx) = st.conn.take() {
|
||||
let _ = tx.send(Message::Close(None));
|
||||
}
|
||||
}
|
||||
self.inner.wake.notify_waiters();
|
||||
self.inner.wake.notify_one();
|
||||
mark_disconnected(&self.inner);
|
||||
}
|
||||
|
||||
fn send(&self, msg: Message) -> bool {
|
||||
let st = self.inner.state.lock().unwrap();
|
||||
match &st.conn {
|
||||
Some(tx) if self.inner.ready.borrow().is_some() => tx.send(msg).is_ok(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn send_json(&self, v: Value) -> bool {
|
||||
self.send(Message::text(v.to_string()))
|
||||
}
|
||||
|
||||
/// One frame of a speaker's PCM in the hello "audio" format. Dropped while disconnected.
|
||||
pub fn send_audio(&self, speaker: &str, pcm: &[i16]) {
|
||||
let Some(frame) = encode_audio_frame(speaker, pcm) else { return };
|
||||
if !self.send(Message::binary(frame)) {
|
||||
self.inner.state.lock().unwrap().dropped_frames += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn speaking_end(&self, speaker: &str) {
|
||||
self.send_json(json!({"type": "speaking_end", "speaker": speaker}));
|
||||
}
|
||||
|
||||
pub fn speaker_leave(&self, speaker: &str) {
|
||||
self.send_json(json!({"type": "speaker_leave", "speaker": speaker}));
|
||||
}
|
||||
|
||||
// --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the
|
||||
// discord core sends the voice state again when the worker is ready. ----------------------
|
||||
|
||||
/// Jarvis is in (or moved to) this voice channel.
|
||||
pub fn voice_join(&self, channel: u64) {
|
||||
self.send_json(json!({"type": "voice_join", "channel": channel.to_string()}));
|
||||
}
|
||||
|
||||
pub fn voice_leave(&self) {
|
||||
self.send_json(json!({"type": "voice_leave"}));
|
||||
}
|
||||
|
||||
pub fn member_join(&self, user: u64, name: &str) {
|
||||
self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name}));
|
||||
}
|
||||
|
||||
pub fn member_leave(&self, user: u64) {
|
||||
self.send_json(json!({"type": "member_leave", "user": user.to_string()}));
|
||||
}
|
||||
|
||||
/// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64,
|
||||
to: Option<u64>) {
|
||||
let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(),
|
||||
"author": author.to_string(), "name": name, "text": text, "ts": ts});
|
||||
if let Some(to) = to {
|
||||
v["to"] = Value::String(to.to_string());
|
||||
}
|
||||
self.send_json(v);
|
||||
}
|
||||
|
||||
pub fn chat_edit(&self, channel: u64, message: u64, text: &str) {
|
||||
self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(),
|
||||
"text": text}));
|
||||
}
|
||||
|
||||
/// The registered chat channels, their switches and the bot's own user ID, for the chat tool.
|
||||
pub fn chat_channels(&self, bot: Option<u64>, channels: &[crate::settings::ChatChannelSettings]) {
|
||||
let list: Vec<Value> = channels.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name,
|
||||
"description": c.description, "read": c.read, "send": c.send, "edit": c.edit, "react": c.react,
|
||||
"delete": c.delete, "pin": c.pin})).collect();
|
||||
let mut v = json!({"type": "chat_channels", "channels": list});
|
||||
if let Some(b) = bot {
|
||||
v["bot"] = Value::String(b.to_string());
|
||||
}
|
||||
self.send_json(v);
|
||||
}
|
||||
|
||||
/// The outcome of a `chat_action`: the message ids it made or touched, or why it failed.
|
||||
pub fn chat_action_result(&self, id: &str, result: Result<Vec<u64>, String>) {
|
||||
let v = match result {
|
||||
Ok(ids) => json!({"type": "chat_action_result", "id": id, "ok": true,
|
||||
"message_ids": ids.iter().map(|i| i.to_string()).collect::<Vec<_>>()}),
|
||||
Err(e) => json!({"type": "chat_action_result", "id": id, "ok": false, "error": e}),
|
||||
};
|
||||
self.send_json(v);
|
||||
}
|
||||
|
||||
/// The outcome of a `schedule_action`: `{item}` or `{items}`, or why it failed.
|
||||
pub fn schedule_action_result(&self, id: &str, result: Result<Value, String>) {
|
||||
self.app_action_result("schedule_action_result", id, result);
|
||||
}
|
||||
|
||||
/// The outcome of a `note_action`: `{item}` or `{items}`, or why it failed.
|
||||
pub fn note_action_result(&self, id: &str, result: Result<Value, String>) {
|
||||
self.app_action_result("note_action_result", id, result);
|
||||
}
|
||||
|
||||
/// The outcome of a `screen_action`: `{item, images}`, or why it failed.
|
||||
pub fn screen_action_result(&self, id: &str, result: Result<Value, String>) {
|
||||
self.app_action_result("screen_action_result", id, result);
|
||||
}
|
||||
|
||||
fn app_action_result(&self, kind: &str, id: &str, result: Result<Value, String>) {
|
||||
let msg = match result {
|
||||
Ok(Value::Object(mut m)) => {
|
||||
m.insert("type".into(), json!(kind));
|
||||
m.insert("id".into(), json!(id));
|
||||
m.insert("ok".into(), json!(true));
|
||||
Value::Object(m)
|
||||
}
|
||||
Ok(_) => json!({"type": kind, "id": id, "ok": true}),
|
||||
Err(e) => json!({"type": kind, "id": id, "ok": false, "error": e}),
|
||||
};
|
||||
self.send_json(msg);
|
||||
}
|
||||
|
||||
pub fn chat_delete(&self, channel: u64, message: u64) {
|
||||
self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(),
|
||||
"message_id": message.to_string()}));
|
||||
}
|
||||
|
||||
// --- brain (voice_worker.spec.md "History and brain", conversation.spec.md) -------------------
|
||||
|
||||
/// Ask the brain to answer `user`. With `speak` the answer is spoken under `id` (its audio and
|
||||
/// `tts_done` come like a TTS request's); it ends as failed when it cannot be sent, the brain gives
|
||||
/// no answer, or the worker refuses it.
|
||||
/// `via`: `voice`, `chat` or `task` (a scheduled task: read-only tools, never spoken).
|
||||
pub fn reply(&self, id: &str, user: u64, text: &str, via: &str, speak: bool) -> bool {
|
||||
self.reply_in(id, user, text, via, speak, None, &[])
|
||||
}
|
||||
|
||||
/// `reply`, naming the chat channel a chat message was written in (the brain's schedule tool
|
||||
/// delivers there) and carrying the asker's standing instructions (`notes`, at most 20).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn reply_in(&self, id: &str, user: u64, text: &str, via: &str, speak: bool, channel: Option<u64>,
|
||||
notes: &[String]) -> bool {
|
||||
if speak {
|
||||
self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
||||
}
|
||||
let mut msg = json!({"type": "reply", "id": id, "user": user.to_string(), "text": text, "via": via,
|
||||
"speak": speak});
|
||||
if let Some(c) = channel {
|
||||
msg["channel"] = json!(c.to_string());
|
||||
}
|
||||
if !notes.is_empty() {
|
||||
msg["notes"] = json!(notes);
|
||||
}
|
||||
let sent = self.send_json(msg);
|
||||
if !sent && speak {
|
||||
self.end_failed(id);
|
||||
}
|
||||
sent
|
||||
}
|
||||
|
||||
/// Whether the chat answer `id` was posted (voice_worker.spec.md "History and brain"): a
|
||||
/// question in an answer the person never saw cannot be confirmed in their next message.
|
||||
pub fn reply_delivered(&self, id: &str, ok: bool) {
|
||||
self.send_json(json!({"type": "reply_delivered", "id": id, "ok": ok}));
|
||||
}
|
||||
|
||||
fn end_failed(&self, id: &str) {
|
||||
if self.inner.state.lock().unwrap().open_tts.remove(id) {
|
||||
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0, cancelled: false,
|
||||
failed: true }));
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask whether `text` was said to Jarvis (call mode `addressed`).
|
||||
pub fn addressed(&self, id: &str, user: u64, text: &str, people: usize, callers: usize) -> bool {
|
||||
self.send_json(json!({"type": "addressed", "id": id, "user": user.to_string(), "text": text,
|
||||
"people": people, "callers": callers}))
|
||||
}
|
||||
|
||||
/// Tell the brain the LLM switches (voice_worker.spec.md, "LLM switches"): it starts no new
|
||||
/// LLM work, says `notice` once to the first spoken request that waits, and reports idle for
|
||||
/// this switch, whose number is returned for `wait_llm_idle`.
|
||||
pub fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
||||
let id = self.inner.llm_switches.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
|
||||
let mut m = json!({"type": "llm", "state": "switching", "id": id});
|
||||
if let Some(n) = notice {
|
||||
m["notice"] = json!(n);
|
||||
}
|
||||
// Kept and sent under one lock, so a connection being set up sends this or a later state.
|
||||
let mut llm = self.inner.llm.lock().unwrap();
|
||||
*llm = Some(m.clone());
|
||||
self.send_json(m);
|
||||
id
|
||||
}
|
||||
|
||||
/// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing).
|
||||
pub fn llm_state(&self, state: &str) -> bool {
|
||||
self.set_llm(json!({"type": "llm", "state": state}))
|
||||
}
|
||||
|
||||
/// Tell the brain the LLM is ready, and whether a game runs (then partial passes wait while an
|
||||
/// answer is written).
|
||||
pub fn llm_ready(&self, game: bool) -> bool {
|
||||
self.set_llm(json!({"type": "llm", "state": "ready", "game": game}))
|
||||
}
|
||||
|
||||
/// Speak at `speed` from the next sentence on (`tts_speed`). False when not connected; it still
|
||||
/// goes out on the next connection.
|
||||
pub fn set_tts_speed(&self, speed: f64) -> bool {
|
||||
let mut s = self.inner.tts_speed.lock().unwrap();
|
||||
*s = Some(speed);
|
||||
self.send_json(json!({"type": "tts_speed", "speed": speed}))
|
||||
}
|
||||
|
||||
fn set_llm(&self, m: Value) -> bool {
|
||||
let mut llm = self.inner.llm.lock().unwrap();
|
||||
*llm = Some(m.clone());
|
||||
self.send_json(m)
|
||||
}
|
||||
|
||||
/// Wait until the brain reports idle for switch `id` (also when it came before the wait); a
|
||||
/// report for an earlier switch does not count. At most `t`.
|
||||
pub async fn wait_llm_idle(&self, id: u64, t: Duration) -> bool {
|
||||
let mut rx = self.inner.llm_idle.subscribe();
|
||||
tokio::time::timeout(t, rx.wait_for(|got| *got >= id)).await.is_ok_and(|r| r.is_ok())
|
||||
}
|
||||
|
||||
/// Tell the brain a call is being said (a partial transcript), so it gets ready to answer it.
|
||||
pub fn prepare(&self, user: u64, text: &str) -> bool {
|
||||
self.send_json(json!({"type": "prepare", "user": user.to_string(), "text": text}))
|
||||
}
|
||||
|
||||
/// Ask the brain to rank tied candidates `(id, user, text)` by importance.
|
||||
pub fn rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool {
|
||||
let list: Vec<Value> = candidates.iter()
|
||||
.map(|(cid, user, text)| json!({"id": cid, "user": user.to_string(), "text": text}))
|
||||
.collect();
|
||||
self.send_json(json!({"type": "rank", "id": id, "candidates": list}))
|
||||
}
|
||||
|
||||
/// Change STT settings now and for every later connection.
|
||||
pub fn configure(&self, changes: Map<String, Value>) {
|
||||
self.inner.state.lock().unwrap().stt.extend(changes.clone());
|
||||
let mut msg = changes;
|
||||
msg.insert("type".into(), "config".into());
|
||||
self.send_json(Value::Object(msg));
|
||||
}
|
||||
}
|
||||
|
||||
impl TtsBackend for WorkerClient {
|
||||
/// Speak `text` as request `id`; with `more` the request stays open for more text. Returns false
|
||||
/// (and the request ends as failed) when the worker is not connected.
|
||||
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
||||
let fresh = self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
||||
let sent = self.send_json(json!({"type": "tts", "id": id, "text": text, "more": more}));
|
||||
if !sent {
|
||||
let was_open = self.inner.state.lock().unwrap().open_tts.remove(id);
|
||||
if fresh || was_open {
|
||||
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0,
|
||||
cancelled: false, failed: true }));
|
||||
}
|
||||
}
|
||||
sent
|
||||
}
|
||||
|
||||
fn tts_cancel(&self, id: &str) {
|
||||
self.send_json(json!({"type": "tts_cancel", "id": id}));
|
||||
}
|
||||
|
||||
fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option<u32>) {
|
||||
let mut m = json!({"type": "tts_played", "id": id, "played": played, "cancelled": cancelled});
|
||||
if let Some(ms) = last_ms {
|
||||
m["last_ms"] = json!(ms);
|
||||
}
|
||||
self.send_json(m);
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_disconnected(inner: &Inner) {
|
||||
let was = inner.ready.send_replace(None).is_some();
|
||||
let (open, dropped) = {
|
||||
let mut st = inner.state.lock().unwrap();
|
||||
st.conn = None;
|
||||
(std::mem::take(&mut st.open_tts), st.dropped_frames)
|
||||
};
|
||||
for id in open {
|
||||
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
||||
}
|
||||
if was {
|
||||
debug_log!("worker", "voice worker disconnected (dropped audio frames so far: {dropped})");
|
||||
let _ = inner.events.send(WorkerEvent::Disconnected);
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(inner: Arc<Inner>) {
|
||||
let mut backoff = inner.opts.backoff;
|
||||
loop {
|
||||
if inner.state.lock().unwrap().closed {
|
||||
return;
|
||||
}
|
||||
if connect_once(&inner).await {
|
||||
backoff = inner.opts.backoff;
|
||||
}
|
||||
mark_disconnected(&inner);
|
||||
if inner.state.lock().unwrap().closed {
|
||||
return;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(backoff) => {}
|
||||
_ = inner.wake.notified() => {}
|
||||
}
|
||||
backoff = (backoff * 2).min(inner.opts.max_backoff);
|
||||
}
|
||||
}
|
||||
|
||||
/// One connection's life. True if it got as far as hello_ok.
|
||||
async fn connect_once(inner: &Arc<Inner>) -> bool {
|
||||
let ws = match tokio_tungstenite::connect_async(inner.opts.url.as_str()).await {
|
||||
Ok((ws, _)) => ws,
|
||||
Err(e) => {
|
||||
// Refused is normal while the worker is still loading its models.
|
||||
debug_log!("worker", "voice worker connect failed: {e}");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let (mut write, mut read) = ws.split();
|
||||
let stt = inner.state.lock().unwrap().stt.clone();
|
||||
let o = &inner.opts;
|
||||
let mut hello = json!({"type": "hello", "token": o.token,
|
||||
"audio": {"rate": o.audio.rate, "channels": o.audio.channels},
|
||||
"tts": {"rate": o.tts.rate, "channels": o.tts.channels}, "stt": stt});
|
||||
if !o.memory.is_empty() {
|
||||
hello["memory"] = Value::Object(o.memory.clone());
|
||||
}
|
||||
if write.send(Message::text(hello.to_string())).await.is_err() {
|
||||
return false;
|
||||
}
|
||||
let hello_ok = tokio::time::timeout(o.hello_timeout, async {
|
||||
while let Some(Ok(msg)) = read.next().await {
|
||||
match msg {
|
||||
Message::Text(t) => {
|
||||
let v: Value = serde_json::from_str(t.as_str()).ok()?;
|
||||
if v["type"] == "hello_ok" {
|
||||
return serde_json::from_value::<HelloOk>(v).ok();
|
||||
}
|
||||
}
|
||||
Message::Close(Some(f)) => {
|
||||
if matches!(f.code, CloseCode::Library(4400) | CloseCode::Library(4401)) {
|
||||
error_log!("worker", "voice worker refused the connection ({} {})", u16::from(f.code), f.reason);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
})
|
||||
.await;
|
||||
let Ok(Some(hello_ok)) = hello_ok else { return false };
|
||||
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
|
||||
{
|
||||
// Locks in the order llm, tts_speed, then state (as `llm_state` and `set_tts_speed` take
|
||||
// them): the latest LLM state and speed go first, and the connection is ready for sending
|
||||
// before one set meanwhile is sent.
|
||||
let llm = inner.llm.lock().unwrap();
|
||||
let speed = inner.tts_speed.lock().unwrap();
|
||||
let mut st = inner.state.lock().unwrap();
|
||||
if st.closed {
|
||||
return true;
|
||||
}
|
||||
if let Some(m) = llm.clone() {
|
||||
let _ = tx.send(Message::text(m.to_string()));
|
||||
}
|
||||
if let Some(s) = *speed {
|
||||
let _ = tx.send(Message::text(json!({"type": "tts_speed", "speed": s}).to_string()));
|
||||
}
|
||||
st.conn = Some(tx);
|
||||
inner.ready.send_replace(Some(hello_ok.clone()));
|
||||
}
|
||||
debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready);
|
||||
let _ = inner.events.send(WorkerEvent::Ready(hello_ok));
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
out = rx.recv() => match out {
|
||||
Some(Message::Close(_)) | None => { let _ = write.close().await; break; }
|
||||
Some(m) => if write.send(m).await.is_err() { break },
|
||||
},
|
||||
msg = read.next() => match msg {
|
||||
Some(Ok(Message::Binary(b))) => on_binary(inner, &b),
|
||||
Some(Ok(Message::Text(t))) => on_text(inner, t.as_str()),
|
||||
Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break,
|
||||
Some(Ok(_)) => {}
|
||||
},
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn on_binary(inner: &Inner, b: &[u8]) {
|
||||
if b.first() != Some(&KIND_TTS) {
|
||||
return;
|
||||
}
|
||||
if let Some(f) = decode_tts_frame(b) {
|
||||
if inner.state.lock().unwrap().open_tts.contains(&f.id) {
|
||||
let _ = inner.events.send(WorkerEvent::TtsAudio { id: f.id, seq: f.seq, pcm: f.pcm });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn on_text(inner: &Inner, t: &str) {
|
||||
let Ok(v) = serde_json::from_str::<Value>(t) else { return };
|
||||
let kind = v["type"].as_str().unwrap_or_default().to_owned();
|
||||
let ev = match kind.as_str() {
|
||||
"speech_start" => serde_json::from_value(v).ok().map(WorkerEvent::SpeechStart),
|
||||
"partial" => serde_json::from_value(v).ok().map(WorkerEvent::Partial),
|
||||
"final" => serde_json::from_value(v).ok().map(WorkerEvent::Final),
|
||||
"chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction),
|
||||
"schedule_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScheduleAction),
|
||||
"note_action" => serde_json::from_value(v).ok().map(WorkerEvent::NoteAction),
|
||||
"screen_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScreenAction),
|
||||
"llm_idle" => {
|
||||
if let Some(id) = v["id"].as_u64() {
|
||||
inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false });
|
||||
}
|
||||
return;
|
||||
}
|
||||
"reply_text" => {
|
||||
let id = v["id"].as_str().unwrap_or_default().to_owned();
|
||||
let failed = v["failed"].as_bool().unwrap_or(true);
|
||||
let text = v["text"].as_str().unwrap_or_default().to_owned();
|
||||
// No answer means nothing will be spoken: end the speech the app is waiting for.
|
||||
if failed && inner.state.lock().unwrap().open_tts.remove(&id) {
|
||||
let _ = inner.events.send(WorkerEvent::ReplyText { id: id.clone(), text, failed });
|
||||
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
||||
return;
|
||||
}
|
||||
Some(WorkerEvent::ReplyText { id, text, failed })
|
||||
}
|
||||
"addressed_result" => Some(WorkerEvent::Addressed {
|
||||
id: v["id"].as_str().unwrap_or_default().to_owned(),
|
||||
addressed: v["addressed"].as_bool().unwrap_or(false) && !v["failed"].as_bool().unwrap_or(false),
|
||||
}),
|
||||
"rank_result" => Some(WorkerEvent::Ranked {
|
||||
id: v["id"].as_str().unwrap_or_default().to_owned(),
|
||||
order: if v["failed"].as_bool().unwrap_or(false) { vec![] } else {
|
||||
v["order"].as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(str::to_owned)).collect())
|
||||
.unwrap_or_default()
|
||||
},
|
||||
}),
|
||||
"tts_done" => serde_json::from_value::<TtsDone>(v).ok().and_then(|d| {
|
||||
inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d))
|
||||
}),
|
||||
"error" => serde_json::from_value::<WorkerError>(v).ok().map(|e| {
|
||||
// Always logged: the log screen and its error report must show what the worker refused.
|
||||
error_log!("worker", "voice worker error {} id={}", e.code, e.id.as_deref().unwrap_or("-"));
|
||||
// A request the worker refused outright never gets a tts_done.
|
||||
if e.code == "tts_unavailable" || e.code == "brain_unavailable" {
|
||||
if let Some(id) = &e.id {
|
||||
if inner.state.lock().unwrap().open_tts.remove(id) {
|
||||
let _ = inner.events.send(WorkerEvent::Error(e.clone()));
|
||||
return WorkerEvent::TtsDone(TtsDone { id: id.clone(), chunks: 0, cancelled: false, failed: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
WorkerEvent::Error(e)
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(ev) = ev {
|
||||
let _ = inner.events.send(ev);
|
||||
}
|
||||
}
|
||||
278
app/src/worker_process.rs
Normal file
278
app/src/worker_process.rs
Normal file
@@ -0,0 +1,278 @@
|
||||
//! Starts the Python voice worker as a child process and restarts it if it dies.
|
||||
//!
|
||||
//! Each app launch makes a new random token and hands it over in the environment
|
||||
//! (`JARVIS_VOICE_TOKEN`), never on the command line. The worker listens on 127.0.0.1 only.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use base64::Engine;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::process::{Child, Command};
|
||||
use tokio::sync::{broadcast, Notify};
|
||||
|
||||
use crate::error_log;
|
||||
|
||||
/// Where the worker's output lines go (default: the app's log, `worker_line`).
|
||||
#[derive(Clone)]
|
||||
pub struct LineSink(pub Arc<dyn Fn(&str) + Send + Sync>);
|
||||
|
||||
impl std::fmt::Debug for LineSink {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("LineSink")
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker
|
||||
/// (an MCP server) may hold its output open after it is gone; then the rest is not read. A line
|
||||
/// still being logged after that is masked with its run's values all the same (`run_sink`).
|
||||
const OUTPUT_DRAIN: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WorkerProcessOptions {
|
||||
pub python: String,
|
||||
pub cwd: PathBuf,
|
||||
pub args: Vec<String>,
|
||||
/// Fixed port; 0 picks a free one once at start (kept across restarts).
|
||||
pub port: u16,
|
||||
/// Extra environment for the worker (over the app's own).
|
||||
pub env: HashMap<String, String>,
|
||||
pub restart_delay: Duration,
|
||||
pub max_restart_delay: Duration,
|
||||
/// Where output lines go; None: the app's log.
|
||||
pub line_sink: Option<LineSink>,
|
||||
}
|
||||
|
||||
impl WorkerProcessOptions {
|
||||
pub fn new(python: impl Into<String>, cwd: impl Into<PathBuf>, args: Vec<String>) -> Self {
|
||||
Self {
|
||||
python: python.into(),
|
||||
cwd: cwd.into(),
|
||||
args,
|
||||
port: 0,
|
||||
env: HashMap::new(),
|
||||
restart_delay: Duration::from_secs(1),
|
||||
max_restart_delay: Duration::from_secs(30),
|
||||
line_sink: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ProcessEvent {
|
||||
/// The worker printed READY: models are loaded and it accepts connections.
|
||||
Ready,
|
||||
Exited { code: Option<i32> },
|
||||
}
|
||||
|
||||
pub fn free_port() -> std::io::Result<u16> {
|
||||
Ok(std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
|
||||
}
|
||||
|
||||
struct Shared {
|
||||
stopping: bool,
|
||||
child_pid: Option<u32>,
|
||||
restarts: u32,
|
||||
}
|
||||
|
||||
pub struct WorkerProcess {
|
||||
opts: WorkerProcessOptions,
|
||||
pub token: String,
|
||||
port: u16,
|
||||
shared: Arc<Mutex<Shared>>,
|
||||
stop: Arc<Notify>,
|
||||
events: broadcast::Sender<ProcessEvent>,
|
||||
task: Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl WorkerProcess {
|
||||
pub fn new(opts: WorkerProcessOptions) -> std::io::Result<Self> {
|
||||
let port = if opts.port == 0 { free_port()? } else { opts.port };
|
||||
let mut raw = [0u8; 32];
|
||||
rand::fill(&mut raw);
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(raw);
|
||||
Ok(Self {
|
||||
opts,
|
||||
token,
|
||||
port,
|
||||
shared: Arc::new(Mutex::new(Shared { stopping: false, child_pid: None, restarts: 0 })),
|
||||
stop: Arc::new(Notify::new()),
|
||||
events: broadcast::channel(64).0,
|
||||
task: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn url(&self) -> String {
|
||||
format!("ws://127.0.0.1:{}", self.port)
|
||||
}
|
||||
|
||||
pub fn port(&self) -> u16 {
|
||||
self.port
|
||||
}
|
||||
|
||||
pub fn restarts(&self) -> u32 {
|
||||
self.shared.lock().unwrap().restarts
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<ProcessEvent> {
|
||||
self.events.subscribe()
|
||||
}
|
||||
|
||||
/// Spawn the worker and keep it running until `stop()`.
|
||||
pub fn start(&self) {
|
||||
let (opts, token, port) = (self.opts.clone(), self.token.clone(), self.port);
|
||||
let (shared, stop, events) = (self.shared.clone(), self.stop.clone(), self.events.clone());
|
||||
let sink = opts.line_sink.clone().unwrap_or_else(run_sink);
|
||||
let task = tokio::spawn(async move {
|
||||
let mut delay = opts.restart_delay;
|
||||
loop {
|
||||
let started = Instant::now();
|
||||
let mut child = match spawn(&opts, &token, port) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error_log!("worker", "voice worker could not start: {e}");
|
||||
let _ = events.send(ProcessEvent::Exited { code: None });
|
||||
if wait_or_stop(&stop, &shared, delay).await {
|
||||
return;
|
||||
}
|
||||
delay = (delay * 2).min(opts.max_restart_delay);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
shared.lock().unwrap().child_pid = child.id();
|
||||
crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id());
|
||||
let mut ready_rx = events.subscribe();
|
||||
let mut readers = vec![];
|
||||
// The worker's own logs hold timings and counts only (voice_worker.spec.md).
|
||||
if let Some(out) = child.stdout.take() {
|
||||
let (events, sink) = (events.clone(), sink.clone());
|
||||
readers.push(tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(out).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
if line.starts_with("READY ") {
|
||||
crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis());
|
||||
let _ = events.send(ProcessEvent::Ready);
|
||||
} else {
|
||||
(sink.0)(&line);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
if let Some(err) = child.stderr.take() {
|
||||
let sink = sink.clone();
|
||||
readers.push(tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(err).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
(sink.0)(&line);
|
||||
}
|
||||
}));
|
||||
}
|
||||
let status = tokio::select! {
|
||||
s = child.wait() => s.ok(),
|
||||
_ = stop.notified() => {
|
||||
let _ = child.kill().await;
|
||||
let code = child.wait().await.ok().and_then(|s| s.code());
|
||||
drain(readers).await;
|
||||
let _ = events.send(ProcessEvent::Exited { code });
|
||||
return;
|
||||
}
|
||||
};
|
||||
// A worker that got as far as READY resets the backoff.
|
||||
while let Ok(ev) = ready_rx.try_recv() {
|
||||
if ev == ProcessEvent::Ready {
|
||||
delay = opts.restart_delay;
|
||||
}
|
||||
}
|
||||
let code = status.and_then(|s| s.code());
|
||||
drain(readers).await;
|
||||
shared.lock().unwrap().child_pid = None;
|
||||
let _ = events.send(ProcessEvent::Exited { code });
|
||||
if shared.lock().unwrap().stopping {
|
||||
return;
|
||||
}
|
||||
error_log!("worker", "voice worker exited (code {code:?}); restarting in {} ms", delay.as_millis());
|
||||
if wait_or_stop(&stop, &shared, delay).await {
|
||||
return;
|
||||
}
|
||||
shared.lock().unwrap().restarts += 1;
|
||||
delay = (delay * 2).min(opts.max_restart_delay);
|
||||
}
|
||||
});
|
||||
*self.task.lock().unwrap() = Some(task);
|
||||
}
|
||||
|
||||
/// Stop the worker and do not restart it. Returns once it has exited and its last output is logged.
|
||||
pub async fn stop(&self) {
|
||||
self.shared.lock().unwrap().stopping = true;
|
||||
self.stop.notify_one();
|
||||
let task = self.task.lock().unwrap().take();
|
||||
if let Some(t) = task {
|
||||
let _ = t.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait until the output readers have logged everything the exited worker wrote (at most
|
||||
/// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets
|
||||
/// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped.
|
||||
async fn drain(readers: Vec<tokio::task::JoinHandle<()>>) {
|
||||
let until = tokio::time::Instant::now() + OUTPUT_DRAIN;
|
||||
for mut r in readers {
|
||||
if tokio::time::timeout_at(until, &mut r).await.is_err() {
|
||||
r.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sleep `delay` unless stopped meanwhile (true = stopped).
|
||||
async fn wait_or_stop(stop: &Notify, shared: &Mutex<Shared>, delay: Duration) -> bool {
|
||||
if shared.lock().unwrap().stopping {
|
||||
return true;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(delay) => shared.lock().unwrap().stopping,
|
||||
_ = stop.notified() => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn(opts: &WorkerProcessOptions, token: &str, port: u16) -> std::io::Result<Child> {
|
||||
let mut cmd = Command::new(&opts.python);
|
||||
cmd.args(["-m", "voice_worker", "--host", "127.0.0.1", "--port", &port.to_string()])
|
||||
.args(&opts.args)
|
||||
.current_dir(&opts.cwd)
|
||||
.envs(&opts.env)
|
||||
.env("JARVIS_VOICE_TOKEN", token)
|
||||
.env("PYTHONUNBUFFERED", "1")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
#[cfg(windows)]
|
||||
{
|
||||
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||
cmd.creation_flags(CREATE_NO_WINDOW);
|
||||
}
|
||||
cmd.spawn()
|
||||
}
|
||||
|
||||
/// The default sink of a worker started now: the app's log, masking also the values masked at
|
||||
/// this moment (this run's secrets, `logs.rs` `set_secrets_in_use`), so a line it writes late (after
|
||||
/// a stop gave up waiting for it, and the next start replaced the values in use) is still masked.
|
||||
pub fn run_sink() -> LineSink {
|
||||
let masks = crate::logs::installed().map(|h| h.masked_values()).unwrap_or_default();
|
||||
LineSink(Arc::new(move |line: &str| worker_line(line, &masks)))
|
||||
}
|
||||
|
||||
/// A line the voice worker printed: on stderr with debug logging, and always to the log hub (its
|
||||
/// own category picks the tab; warnings and tracebacks reach the log screen without debug logging).
|
||||
fn worker_line(line: &str, masks: &[String]) {
|
||||
if crate::log::debug_enabled() {
|
||||
eprintln!("{}", crate::log::line("worker", &crate::logs::redact(line, masks)));
|
||||
}
|
||||
if let Some(hub) = crate::logs::installed() {
|
||||
hub.push_worker_masked(line, masks);
|
||||
}
|
||||
}
|
||||
237
app/tests/bot_browser.rs
Normal file
237
app/tests/bot_browser.rs
Normal file
@@ -0,0 +1,237 @@
|
||||
//! The bot browser with a real Chrome (admin.spec.md, "Bot browser"). Runs only with
|
||||
//! `JARVIS_TEST_CHROME=<chrome executable>`; needs the internet for the public page. On Ubuntu also
|
||||
//! set `JARVIS_CHROME_NO_SANDBOX=1`.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use jarvis_app::browser::BotBrowser;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
async fn titles(client: &reqwest::Client, port: u16) -> Vec<(String, String)> {
|
||||
let text = client.get(format!("http://127.0.0.1:{port}/json/list")).send().await.unwrap().text().await.unwrap();
|
||||
let list: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
list.as_array().unwrap().iter()
|
||||
.map(|t| (t["url"].as_str().unwrap_or("").to_owned(), t["title"].as_str().unwrap_or("").to_owned())).collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_bot_browser_reaches_public_pages_never_this_pc_and_stops() {
|
||||
let Ok(exe) = std::env::var("JARVIS_TEST_CHROME") else {
|
||||
eprintln!("skipped: set JARVIS_TEST_CHROME");
|
||||
return;
|
||||
};
|
||||
// A page on this PC the browser must never reach.
|
||||
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let local = l.local_addr().unwrap().port();
|
||||
let hits = Arc::new(Mutex::new(0));
|
||||
let h2 = hits.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut c, _)) = l.accept().await {
|
||||
*h2.lock().unwrap() += 1;
|
||||
let mut b = [0u8; 1024];
|
||||
let _ = c.read(&mut b).await;
|
||||
let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n<title>secret local</title>xx").await;
|
||||
}
|
||||
});
|
||||
|
||||
let profile = std::env::temp_dir().join(format!("jarvis-bot-browser-{}", std::process::id()));
|
||||
let b = BotBrowser::start(std::path::Path::new(&exe), &profile, None).await.expect("chrome starts");
|
||||
assert!(b.running());
|
||||
let client = reqwest::Client::builder().no_proxy().build().unwrap();
|
||||
for url in [format!("http://127.0.0.1:{local}/"), format!("http://localhost:{local}/"), "https://example.com/".into()] {
|
||||
client.put(format!("http://127.0.0.1:{}/json/new?{url}", b.cdp_port)).send().await.unwrap();
|
||||
}
|
||||
let mut got = vec![];
|
||||
for _ in 0..40 {
|
||||
got = titles(&client, b.cdp_port).await;
|
||||
if got.iter().any(|(_, t)| t == "Example Domain") {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
eprintln!("tabs: {got:?}");
|
||||
assert!(got.iter().any(|(_, t)| t == "Example Domain"), "the public page loads through the proxy");
|
||||
assert!(!got.iter().any(|(_, t)| t.contains("secret local")));
|
||||
assert_eq!(*hits.lock().unwrap(), 0, "nothing reached this PC");
|
||||
|
||||
let port = b.cdp_port;
|
||||
let t0 = std::time::Instant::now();
|
||||
b.stop().await;
|
||||
eprintln!("stopped in {:?}", t0.elapsed());
|
||||
assert!(!b.running());
|
||||
assert!(client.get(format!("http://127.0.0.1:{port}/json/version")).send().await.is_err(), "CDP is closed");
|
||||
let _ = std::fs::remove_dir_all(&profile);
|
||||
}
|
||||
|
||||
/// The install step downloads the pinned Chrome (about 200 MB) and unpacks a runnable browser.
|
||||
/// Runs only with `JARVIS_TEST_CHROME_INSTALL=1`.
|
||||
#[tokio::test]
|
||||
async fn the_browser_step_installs_a_chrome_that_starts() {
|
||||
if std::env::var("JARVIS_TEST_CHROME_INSTALL").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_TEST_CHROME_INSTALL=1");
|
||||
return;
|
||||
}
|
||||
let data = std::env::temp_dir().join(format!("jarvis-chrome-install-{}", std::process::id()));
|
||||
let layout = jarvis_app::install::Layout { data: data.clone(), src: std::path::PathBuf::new() };
|
||||
let steps = jarvis_app::install::steps().into_iter().filter(|s| s.id() == "browser").collect();
|
||||
let installer = jarvis_app::install::Installer::new(layout.clone(), steps);
|
||||
assert!(installer.run_all().await, "{}", installer.status());
|
||||
assert!(installer.done("browser"));
|
||||
assert!(!data.join("cache").join("downloads").read_dir().map(|mut d| d.next().is_some()).unwrap_or(false),
|
||||
"the archive is removed after unpacking");
|
||||
let b = BotBrowser::start(&layout.chrome_exe(), &jarvis_app::browser::profile_dir(&data), None).await.expect("starts");
|
||||
assert!(b.running());
|
||||
b.stop().await;
|
||||
let _ = std::fs::remove_dir_all(&data);
|
||||
}
|
||||
|
||||
/// The process whose window has the keyboard focus (Windows).
|
||||
#[cfg(windows)]
|
||||
fn foreground_pid() -> u32 {
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
|
||||
let mut pid = 0;
|
||||
// SAFETY: plain queries; `pid` outlives the call.
|
||||
unsafe { GetWindowThreadProcessId(GetForegroundWindow(), &mut pid) };
|
||||
pid
|
||||
}
|
||||
|
||||
/// The visible top-level Chrome windows of process `pid`, top first: (title, tool window).
|
||||
#[cfg(windows)]
|
||||
fn chrome_windows(pid: u32) -> Vec<(String, bool)> {
|
||||
use windows_sys::Win32::Foundation::{HWND, LPARAM};
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
||||
unsafe extern "system" fn each(h: HWND, l: LPARAM) -> windows_sys::core::BOOL {
|
||||
// SAFETY: `l` is the `(pid, Vec)` below, alive through EnumWindows; plain queries on `h`.
|
||||
unsafe {
|
||||
let (pid, out) = &mut *(l as *mut (u32, Vec<(String, bool)>));
|
||||
let (mut p, mut class, mut title) = (0u32, [0u16; 64], [0u16; 256]);
|
||||
GetWindowThreadProcessId(h, &mut p);
|
||||
let n = GetClassNameW(h, class.as_mut_ptr(), 64) as usize;
|
||||
if p == *pid && IsWindowVisible(h) != 0 && GetWindow(h, GW_OWNER).is_null()
|
||||
&& String::from_utf16_lossy(&class[..n]) == "Chrome_WidgetWin_1" {
|
||||
let t = GetWindowTextW(h, title.as_mut_ptr(), 256) as usize;
|
||||
let tool = GetWindowLongPtrW(h, GWL_EXSTYLE) as u32 & WS_EX_TOOLWINDOW != 0;
|
||||
out.push((String::from_utf16_lossy(&title[..t]), tool));
|
||||
}
|
||||
}
|
||||
1
|
||||
}
|
||||
let mut found = (pid, vec![]);
|
||||
// SAFETY: `each` writes only into `found`, which outlives the call.
|
||||
unsafe { EnumWindows(Some(each), &mut found as *mut _ as LPARAM) };
|
||||
found.1
|
||||
}
|
||||
|
||||
/// One call on the browser's CDP connection.
|
||||
async fn cdp<S>(s: &mut S, id: u64, method: &str, params: serde_json::Value) -> serde_json::Value
|
||||
where S: futures_util::Sink<tokio_tungstenite::tungstenite::Message> + futures_util::Stream<Item =
|
||||
Result<tokio_tungstenite::tungstenite::Message, tokio_tungstenite::tungstenite::Error>> + Unpin,
|
||||
S::Error: std::fmt::Debug {
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
let msg = serde_json::json!({"id": id, "method": method, "params": params});
|
||||
s.send(tokio_tungstenite::tungstenite::Message::text(msg.to_string())).await.unwrap();
|
||||
loop {
|
||||
let v: serde_json::Value = serde_json::from_str(s.next().await.unwrap().unwrap().to_text().unwrap()).unwrap();
|
||||
if v["id"] == id {
|
||||
break v["result"].clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// With a screen, Chrome starts without a window and the app opens exactly one, at the screen's
|
||||
/// place and size, without taking the focus. The bot's page then opens the way the botBrowser tool
|
||||
/// opens it (a window of its own behind, at that window's place): also without taking the focus,
|
||||
/// and on Windows the app makes it a tool window in front of the first. Needs a display on Linux
|
||||
/// (`DISPLAY`); on Windows the screen is the virtual monitor (its driver installed), on the desktop
|
||||
/// only meanwhile.
|
||||
#[tokio::test]
|
||||
async fn on_a_screen_the_bot_browser_has_one_window_at_the_monitor() {
|
||||
let Ok(exe) = std::env::var("JARVIS_TEST_CHROME") else {
|
||||
eprintln!("skipped: set JARVIS_TEST_CHROME");
|
||||
return;
|
||||
};
|
||||
if cfg!(unix) && std::env::var_os("DISPLAY").is_none() {
|
||||
eprintln!("skipped: no DISPLAY");
|
||||
return;
|
||||
}
|
||||
let profile = std::env::temp_dir().join(format!("jarvis-bot-browser-screen-{}", std::process::id()));
|
||||
let screen = if cfg!(windows) {
|
||||
let r = jarvis_app::vdisplay::attach().expect("the virtual monitor goes on");
|
||||
eprintln!("virtual monitor at {r:?}");
|
||||
r
|
||||
} else {
|
||||
jarvis_app::vdisplay::Rect { x: 40, y: 30, w: 1280, h: 720 }
|
||||
};
|
||||
#[cfg(windows)]
|
||||
let focused = foreground_pid();
|
||||
let b = BotBrowser::start(std::path::Path::new(&exe), &profile, Some(screen)).await.expect("chrome starts");
|
||||
#[cfg(windows)]
|
||||
assert_eq!(foreground_pid(), focused, "opening the window leaves the focus where it was");
|
||||
let client = reqwest::Client::builder().no_proxy().build().unwrap();
|
||||
let pages = |l: serde_json::Value| l.as_array().unwrap().iter().filter(|t| t["type"] == "page")
|
||||
.map(|t| t["id"].as_str().unwrap().to_owned()).collect::<Vec<_>>();
|
||||
let list: serde_json::Value = client.get(format!("http://127.0.0.1:{}/json/list", b.cdp_port)).send().await.unwrap()
|
||||
.text().await.map(|t| serde_json::from_str(&t).unwrap()).unwrap();
|
||||
let first = pages(list);
|
||||
assert_eq!(first.len(), 1, "one page in one window");
|
||||
|
||||
let ws: serde_json::Value = client.get(format!("http://127.0.0.1:{}/json/version", b.cdp_port)).send().await.unwrap()
|
||||
.text().await.map(|t| serde_json::from_str(&t).unwrap()).unwrap();
|
||||
let (mut s, _) = tokio_tungstenite::connect_async(ws["webSocketDebuggerUrl"].as_str().unwrap()).await.unwrap();
|
||||
// As the botBrowser tool does it.
|
||||
let at = cdp(&mut s, 1, "Browser.getWindowForTarget", serde_json::json!({"targetId": first[0]})).await["bounds"].clone();
|
||||
let bot = cdp(&mut s, 2, "Target.createTarget", serde_json::json!({"url": "about:blank",
|
||||
"newWindow": true, "background": true, "left": at["left"], "top": at["top"], "width": at["width"],
|
||||
"height": at["height"]})).await["targetId"].as_str().unwrap().to_owned();
|
||||
// A page loading in it, as the tool's `open` does.
|
||||
let (mut page, _) = tokio_tungstenite::connect_async(format!("ws://127.0.0.1:{}/devtools/page/{bot}", b.cdp_port))
|
||||
.await.unwrap();
|
||||
cdp(&mut page, 1, "Page.navigate", serde_json::json!({"url": "https://example.com/"})).await;
|
||||
let mut loaded = false;
|
||||
for _ in 0..40 {
|
||||
loaded = titles(&client, b.cdp_port).await.iter().any(|(_, t)| t == "Example Domain");
|
||||
if loaded {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
assert!(loaded, "the bot's page loads in its window");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
#[cfg(windows)]
|
||||
{
|
||||
assert_eq!(foreground_pid(), focused, "the bot's window leaves the focus where it was");
|
||||
let w = chrome_windows(b.pid());
|
||||
eprintln!("chrome windows, top first: {w:?}");
|
||||
assert_eq!(w.len(), 2, "{w:?}");
|
||||
assert!(w[0].0.starts_with("Example Domain"), "the bot's window is in front: {w:?}");
|
||||
assert!(w.iter().all(|(_, tool)| *tool), "no window in the taskbar or Alt+Tab: {w:?}");
|
||||
}
|
||||
|
||||
// Each page's window over CDP: a window each, at the screen's rectangle.
|
||||
let mut windows = std::collections::HashSet::new();
|
||||
for (i, id) in [first[0].clone(), bot].iter().enumerate() {
|
||||
let r = cdp(&mut s, 10 + i as u64, "Browser.getWindowForTarget", serde_json::json!({"targetId": id})).await;
|
||||
eprintln!("window: {r}");
|
||||
windows.insert(r["windowId"].as_i64().unwrap());
|
||||
let b = &r["bounds"];
|
||||
let (left, top, width, height) = (b["left"].as_i64().unwrap(), b["top"].as_i64().unwrap(),
|
||||
b["width"].as_i64().unwrap(), b["height"].as_i64().unwrap());
|
||||
let (x, y, w, h) = (screen.x as i64, screen.y as i64, screen.w as i64, screen.h as i64);
|
||||
if cfg!(windows) {
|
||||
// Inside the monitor, filling it but for the monitor's own taskbar if Windows gave it one.
|
||||
assert!(left >= x && top >= y && left + width <= x + w && top + height <= y + h, "{b}");
|
||||
assert!(width * height * 10 >= w * h * 8, "{b}");
|
||||
} else {
|
||||
assert_eq!((left, top, width, height), (x, y, w, h));
|
||||
}
|
||||
}
|
||||
assert_eq!(windows.len(), 2, "the bot's page has a window of its own");
|
||||
if std::env::var_os("JARVIS_TEST_HOLD").is_some() {
|
||||
// Time for a look at the screen (the Windows check takes a screenshot meanwhile).
|
||||
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
|
||||
}
|
||||
b.stop().await;
|
||||
jarvis_app::vdisplay::detach();
|
||||
let _ = std::fs::remove_dir_all(&profile);
|
||||
}
|
||||
69
app/tests/fixtures/fake_engine_worker.py
vendored
Normal file
69
app/tests/fixtures/fake_engine_worker.py
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
"""The real voice worker server with fake engines, for the app's protocol tests (no GPU, no models).
|
||||
|
||||
The fake VAD reads speech probability off the amplitude; the fake transcriber names a clip by its
|
||||
pitch ("level 0.9" for a 900 Hz tone); the fake TTS makes 10 samples per character; the fake brain answers every reply with two sentences.
|
||||
Run with the repo's src/ on PYTHONPATH: python fake_engine_worker.py <port>
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
from voice_worker.server import VoiceWorker
|
||||
|
||||
|
||||
class AmplitudeVad:
|
||||
def prob(self, window):
|
||||
return float(np.max(np.abs(window)))
|
||||
|
||||
|
||||
class LevelTranscriber:
|
||||
def transcribe(self, clips, opts):
|
||||
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
||||
|
||||
|
||||
class FakeBrain:
|
||||
"""Answers with two sentences, says only "켜" (on) requests were said to it, ranks in reverse.
|
||||
"예약" adds a reminder through the schedule backend and answers with its result, "화면" captures
|
||||
monitor 2 through the screen backend; a task answers "task: <text>"."""
|
||||
|
||||
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
|
||||
screen=None, trace=None):
|
||||
if task:
|
||||
return f"task: {text}" + (f" notes={'|'.join(notes)}" if notes else "")
|
||||
if "지침" in text and notes_backend is not None:
|
||||
got = notes_backend.act("add", text="반말로 해")
|
||||
return f"noted {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
|
||||
if "화면" in text and screen is not None:
|
||||
got = screen.act("capture", target="monitor", monitor=2, detail="fast")
|
||||
sizes = ",".join(str(len(i)) for i in got.get("images") or [])
|
||||
return f"saw {sizes} {got.get('item', {}).get('what', '')} {got.get('error', '')}".strip()
|
||||
if "예약" in text and schedule is not None:
|
||||
got = schedule.act("add", kind="remind", text="물 마시기", at=4_000_000_000.0, repeat="daily")
|
||||
return f"scheduled {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
|
||||
return "첫 문장입니다. 둘째 문장입니다."
|
||||
|
||||
def addressed(self, user, text, names, people=None, callers=None):
|
||||
return "켜" in text
|
||||
|
||||
def rank(self, candidates):
|
||||
return [c["id"] for c in reversed(candidates)]
|
||||
|
||||
|
||||
class FakeTts:
|
||||
def synthesize(self, text, rate):
|
||||
return np.full(len(text) * 10, 1000, dtype=np.int16)
|
||||
|
||||
|
||||
async def main(port: int) -> None:
|
||||
worker = VoiceWorker(os.environ["JARVIS_VOICE_TOKEN"], LevelTranscriber(), AmplitudeVad, FakeTts(),
|
||||
brain=FakeBrain())
|
||||
server = await worker.serve("127.0.0.1", port)
|
||||
print(f"READY 127.0.0.1:{port}", flush=True)
|
||||
await server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main(int(sys.argv[1])))
|
||||
61
app/tests/game_watch.rs
Normal file
61
app/tests/game_watch.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
//! Game detection on Windows (admin.spec.md, "Game detection"): a program started from a launcher's
|
||||
//! game folder turns game mode on with the folder's name, and off once it ends. Runs only with
|
||||
//! `JARVIS_TEST_GAME_EXE=<a windowed program under ...\steamapps\common\<Game>\>`; elsewhere it skips.
|
||||
//! For a game installed elsewhere (Battle.net) also set `JARVIS_TEST_GAME_LAUNCHER=Battle.net` and
|
||||
//! `JARVIS_TEST_GAME_NAME=<its uninstall entry's name>`.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use jarvis_app::game::GameWatch;
|
||||
use jarvis_app::settings::GameSettings;
|
||||
|
||||
fn wait_for(w: &GameWatch, want: impl Fn(&serde_json::Value) -> bool, secs: u64) -> serde_json::Value {
|
||||
let until = Instant::now() + Duration::from_secs(secs);
|
||||
loop {
|
||||
let v = w.view();
|
||||
if want(&v) || Instant::now() > until {
|
||||
return v;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_launcher_game_turns_game_mode_on_until_it_ends() {
|
||||
let Ok(exe) = std::env::var("JARVIS_TEST_GAME_EXE") else {
|
||||
eprintln!("skipped: set JARVIS_TEST_GAME_EXE");
|
||||
return;
|
||||
};
|
||||
let w = GameWatch::start(GameSettings { detect: true, games: vec![], not_games: vec![], other_room_gb: 1.0, reserve_gb: 8.0, revert_seconds: 30, pause_in_game: false });
|
||||
assert_eq!(w.view(), serde_json::Value::Null, "no game before it starts");
|
||||
let mut game = std::process::Command::new(&exe).spawn().expect("the game starts");
|
||||
let t0 = Instant::now();
|
||||
let on = wait_for(&w, |v| !v.is_null(), 15);
|
||||
eprintln!("on after {:?}: {on}", t0.elapsed());
|
||||
let folder = std::path::Path::new(&exe).parent().unwrap().file_name().unwrap().to_string_lossy().into_owned();
|
||||
let name = std::env::var("JARVIS_TEST_GAME_NAME").unwrap_or(folder);
|
||||
let launcher = std::env::var("JARVIS_TEST_GAME_LAUNCHER").unwrap_or("Steam".into());
|
||||
assert_eq!(on, serde_json::json!({"name": name, "reason": {"kind": "launcher", "launcher": launcher}}));
|
||||
game.kill().unwrap();
|
||||
let _ = game.wait();
|
||||
let t1 = Instant::now();
|
||||
let off = wait_for(&w, |v| v.is_null(), 10);
|
||||
eprintln!("off after {:?}: {off}", t1.elapsed());
|
||||
assert_eq!(off, serde_json::Value::Null, "game mode ends with the game");
|
||||
w.stop();
|
||||
}
|
||||
|
||||
/// A game program running behind other windows when the watch starts is found by what it is.
|
||||
/// Runs only with `JARVIS_TEST_BACKGROUND_GAME=1` (a launcher-folder program runs) or `=0` (none).
|
||||
#[test]
|
||||
fn a_game_running_behind_other_windows_is_found_at_the_start() {
|
||||
let Ok(want) = std::env::var("JARVIS_TEST_BACKGROUND_GAME") else {
|
||||
eprintln!("skipped: set JARVIS_TEST_BACKGROUND_GAME");
|
||||
return;
|
||||
};
|
||||
let w = GameWatch::start(GameSettings { detect: true, games: vec![], not_games: vec![], other_room_gb: 1.0,
|
||||
reserve_gb: 8.0, revert_seconds: 30, pause_in_game: false });
|
||||
eprintln!("in front: {}, at the start: {}", w.on(), w.in_background_at_start());
|
||||
assert_eq!(w.in_background_at_start(), want == "1");
|
||||
w.stop();
|
||||
}
|
||||
496
app/tests/live_discord.rs
Normal file
496
app/tests/live_discord.rs
Normal file
@@ -0,0 +1,496 @@
|
||||
//! The Discord voice core end to end on real Discord, with the real voice worker on the GPU.
|
||||
//!
|
||||
//! Two bots in one voice channel: the "Jarvis" bot runs the core; the "speaker" bot plays recorded
|
||||
//! Korean speech into the channel, records what Jarvis says, and writes in the channel's chat.
|
||||
//! Opt-in; it refuses to run if a person is in the channel or either bot is already in voice.
|
||||
//!
|
||||
//! JARVIS_LIVE_DISCORD=1 JARVIS_LIVE_BOT_TOKEN=... JARVIS_LIVE_SPEAKER_TOKEN=... \
|
||||
//! JARVIS_LIVE_GUILD=<server id> JARVIS_LIVE_VOICE_CHANNEL=<voice channel id> \
|
||||
//! JARVIS_LIVE_VOICE_CHANNEL_2=<another voice channel of that server> \
|
||||
//! JARVIS_VOICE_PYTHON=<worker venv python> JARVIS_MELO_PYTHON=<MeloTTS venv python> \
|
||||
//! cargo test --release --test live_discord -- --nocapture
|
||||
//!
|
||||
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use jarvis_app::discord::{CoreEvent, DiscordCore};
|
||||
use jarvis_app::receive::{opus_payload, OpusFrames, SpeakerStream};
|
||||
use jarvis_app::settings::parse_settings;
|
||||
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions, WorkerEvent};
|
||||
use jarvis_app::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
|
||||
use serde_json::{json, Value};
|
||||
use serenity::all::{ChannelId, Client, Context, CreateMessage, EventHandler, GatewayIntents, GuildId, Http, Ready, UserId};
|
||||
use songbird::driver::DecodeMode;
|
||||
use songbird::input::Input;
|
||||
use songbird::{Config, CoreEvent as VoiceEv, Event, EventContext, EventHandler as VoiceHandler, SerenityInit, Songbird, TrackEvent};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const COMMAND: &str = "jarvis-live-test";
|
||||
|
||||
fn env(k: &str) -> String {
|
||||
std::env::var(k).unwrap_or_else(|_| panic!("{k} is not set"))
|
||||
}
|
||||
|
||||
fn cer(reference: &str, hyp: &str) -> f64 {
|
||||
let norm = |s: &str| s.chars().filter(|c| c.is_alphanumeric()).collect::<Vec<_>>();
|
||||
let (a, b) = (norm(reference), norm(hyp));
|
||||
let mut prev: Vec<usize> = (0..=b.len()).collect();
|
||||
for i in 1..=a.len() {
|
||||
let mut cur = vec![i];
|
||||
for j in 1..=b.len() {
|
||||
cur.push((prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + usize::from(a[i - 1] != b[j - 1])));
|
||||
}
|
||||
prev = cur;
|
||||
}
|
||||
prev[b.len()] as f64 / a.len().max(1) as f64
|
||||
}
|
||||
|
||||
/// Seconds into a 16 kHz mono PCM16 WAV where the last sound above -34 dBFS ends.
|
||||
fn speech_end_seconds(wav: &[u8]) -> f64 {
|
||||
let data = &wav[44..];
|
||||
let n = data.len() / 2;
|
||||
(0..n).rev().find(|&k| i16::from_le_bytes([data[2 * k], data[2 * k + 1]]).unsigned_abs() > 655)
|
||||
.map_or(0.0, |k| (k + 1) as f64 / 16000.0)
|
||||
}
|
||||
|
||||
struct ReadyOnce(mpsc::UnboundedSender<Ready>);
|
||||
#[async_trait]
|
||||
impl EventHandler for ReadyOnce {
|
||||
async fn ready(&self, _: Context, r: Ready) {
|
||||
let _ = self.0.send(r);
|
||||
}
|
||||
}
|
||||
|
||||
/// Jarvis's voice as the speaker bot hears it, decoded the same way the core decodes (songbird's
|
||||
/// own decoder garbles DAVE audio, see src/receive.rs).
|
||||
#[derive(Clone, Default)]
|
||||
struct Ears {
|
||||
jarvis: u64,
|
||||
ssrc: Arc<Mutex<HashMap<u32, u64>>>,
|
||||
stream: Arc<Mutex<Option<SpeakerStream<OpusFrames>>>>,
|
||||
pcm: Arc<Mutex<Vec<i16>>>,
|
||||
/// When the last audible frame arrived.
|
||||
last_packet: Arc<Mutex<Option<Instant>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceHandler for Ears {
|
||||
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event> {
|
||||
match ctx {
|
||||
EventContext::SpeakingStateUpdate(s) => {
|
||||
if let Some(u) = s.user_id {
|
||||
self.ssrc.lock().unwrap().insert(s.ssrc, u.0);
|
||||
}
|
||||
}
|
||||
EventContext::RtpPacket(p) => {
|
||||
let (seq, ssrc, opus) = opus_payload(&p.packet, p.payload_offset, p.payload_end_pad)?;
|
||||
if self.ssrc.lock().unwrap().get(&ssrc) != Some(&self.jarvis) {
|
||||
return None;
|
||||
}
|
||||
let mut st = self.stream.lock().unwrap();
|
||||
let frames = st.get_or_insert_with(|| SpeakerStream::new(OpusFrames::new())).push(seq, opus.to_vec());
|
||||
let mut pcm = self.pcm.lock().unwrap();
|
||||
for f in frames {
|
||||
// Audible: louder than -36 dBFS. The sender's closing silence frames do not count.
|
||||
let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>() / f.len().max(1) as f64).sqrt();
|
||||
if rms > 500.0 {
|
||||
*self.last_packet.lock().unwrap() = Some(Instant::now());
|
||||
}
|
||||
pcm.extend_from_slice(&f);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
struct Ended(mpsc::UnboundedSender<()>);
|
||||
#[async_trait]
|
||||
impl VoiceHandler for Ended {
|
||||
async fn act(&self, _: &EventContext<'_>) -> Option<Event> {
|
||||
let _ = self.0.send(());
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn until(what: &str, ms: u64, mut pred: impl FnMut() -> bool) {
|
||||
let deadline = Instant::now() + Duration::from_millis(ms);
|
||||
while !pred() {
|
||||
assert!(Instant::now() < deadline, "timed out waiting for {what}");
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn discord_voice_core_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_DISCORD").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_DISCORD=1");
|
||||
return;
|
||||
}
|
||||
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
||||
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
||||
let voice2: u64 = env("JARVIS_LIVE_VOICE_CHANNEL_2").parse().unwrap();
|
||||
let bot_token = env("JARVIS_LIVE_BOT_TOKEN");
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let samples = root.join("../scripts/m0/assets/ko_samples");
|
||||
|
||||
// The speaker bot first: it checks the channel is free before anything joins.
|
||||
let speaker_songbird = Songbird::serenity_from_config(Config::default().decode_mode(DecodeMode::Decrypt));
|
||||
let (rtx, mut rrx) = mpsc::unbounded_channel();
|
||||
let mut speaker = Client::builder(env("JARVIS_LIVE_SPEAKER_TOKEN"),
|
||||
GatewayIntents::GUILDS | GatewayIntents::GUILD_VOICE_STATES | GatewayIntents::GUILD_MESSAGES | GatewayIntents::MESSAGE_CONTENT)
|
||||
.event_handler(ReadyOnce(rtx)).register_songbird_with(speaker_songbird.clone()).await.unwrap();
|
||||
let (s_http, s_cache, s_shards) = (speaker.http.clone(), speaker.cache.clone(), speaker.shard_manager.clone());
|
||||
tokio::spawn(async move { speaker.start().await.unwrap() });
|
||||
let speaker_id = rrx.recv().await.unwrap().user.id.get();
|
||||
let jarvis_app_id = Http::new(&bot_token).get_current_application_info().await.unwrap().id.get();
|
||||
until("guild in cache", 10_000, || s_cache.guild(GuildId::new(guild)).is_some()).await;
|
||||
{
|
||||
let g = s_cache.guild(GuildId::new(guild)).unwrap();
|
||||
let people = g.voice_states.values().filter(|v| v.channel_id.is_some_and(|c| c.get() == voice || c.get() == voice2)
|
||||
&& !s_cache.user(v.user_id).map(|u| u.bot).unwrap_or(false)).count();
|
||||
assert_eq!(people, 0, "refusing to test: people are in the voice channels");
|
||||
for id in [speaker_id, jarvis_app_id] {
|
||||
assert!(g.voice_states.get(&UserId::new(id)).and_then(|v| v.channel_id).is_none(),
|
||||
"refusing to test: bot {id} is in a voice channel");
|
||||
}
|
||||
}
|
||||
|
||||
// The real voice worker.
|
||||
// A fresh history store: the worker records what this test says and who was there.
|
||||
let history_db = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("live-history-{}.sqlite", std::process::id()));
|
||||
let _ = std::fs::remove_file(&history_db);
|
||||
let mut args = vec!["--history".to_string(), history_db.to_string_lossy().into_owned()];
|
||||
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
||||
args.extend(["--melo-python".to_string(), m]);
|
||||
}
|
||||
let wp = WorkerProcess::new(WorkerProcessOptions::new(env("JARVIS_VOICE_PYTHON"), root.join("../src"), args)).unwrap();
|
||||
let mut pev = wp.subscribe();
|
||||
wp.start();
|
||||
tokio::time::timeout(Duration::from_secs(240), async {
|
||||
while pev.recv().await.unwrap() != ProcessEvent::Ready {}
|
||||
}).await.expect("worker ready");
|
||||
let mut wo = WorkerClientOptions::new(wp.url(), wp.token.clone());
|
||||
wo.stt = json!({"hotwords": ["자비스"]}).as_object().unwrap().clone();
|
||||
let (worker, worker_events) = WorkerClient::new(wo);
|
||||
worker.start();
|
||||
worker.wait_ready(Duration::from_secs(10)).await.expect("worker hello");
|
||||
|
||||
let settings = parse_settings(&json!({
|
||||
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string(), "commandName": COMMAND},
|
||||
// The "person" here is a bot, so bots must be heard.
|
||||
"voice": {"listenToBots": true},
|
||||
// The voice channel's own chat stands in for a registered chat channel.
|
||||
"chatChannels": [{"id": voice.to_string(), "name": "테스트", "description": "라이브 테스트",
|
||||
"delete": true, "pin": true}],
|
||||
})).unwrap();
|
||||
let (core, mut core_rx) = DiscordCore::new(settings, bot_token.clone(), worker.clone(), worker_events);
|
||||
let events: Arc<Mutex<Vec<CoreEvent>>> = Arc::default();
|
||||
let ev2 = events.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = core_rx.recv().await {
|
||||
ev2.lock().unwrap().push(e);
|
||||
}
|
||||
});
|
||||
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, speaks the notice
|
||||
assert_eq!(core.voice_channel_id(), Some(voice));
|
||||
let jarvis = core.bot_id().unwrap();
|
||||
// The transcription notice is the first speech of the join, and it is spoken, not posted.
|
||||
until("notice spoken", 60_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::SpeakEnd(_)))).await;
|
||||
let notice = events.lock().unwrap().iter().find_map(|e| match e { CoreEvent::SpeakEnd(x) => Some(x.clone()), _ => None }).unwrap();
|
||||
assert!(!notice.failed && !notice.cancelled && notice.played > 0, "{notice:?}");
|
||||
|
||||
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
|
||||
let ears = Ears { jarvis, ..Default::default() };
|
||||
{
|
||||
let mut c = call.lock().await;
|
||||
c.add_global_event(VoiceEv::SpeakingStateUpdate.into(), ears.clone());
|
||||
c.add_global_event(VoiceEv::RtpPacket.into(), ears.clone());
|
||||
}
|
||||
let mut report = serde_json::Map::new();
|
||||
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
||||
|
||||
// 1. The slash command is registered.
|
||||
let jarvis_http = Http::new(&bot_token);
|
||||
jarvis_http.set_application_id(jarvis_app_id.into());
|
||||
let cmds = jarvis_http.get_guild_commands(GuildId::new(guild)).await.unwrap();
|
||||
let cmd = cmds.iter().find(|c| c.name == COMMAND).expect("command registered");
|
||||
let mut subs: Vec<_> = cmd.options.iter().map(|o| o.name.clone()).collect();
|
||||
subs.sort();
|
||||
assert_eq!(subs, ["join", "leave", "stop"]);
|
||||
|
||||
// 2. Recorded sentences played into the channel come back as the speaker's finals.
|
||||
let refs: Value = serde_json::from_str(&std::fs::read_to_string(samples.join("refs.json")).unwrap()).unwrap();
|
||||
let mut stt = vec![];
|
||||
let mut total = 0.0;
|
||||
for r in refs["samples"].as_array().unwrap().iter().take(5) {
|
||||
let wav = std::fs::read(samples.join(r["file"].as_str().unwrap())).unwrap();
|
||||
let reference = r["text"].as_str().unwrap();
|
||||
let before = events.lock().unwrap().len();
|
||||
let (etx, mut erx) = mpsc::unbounded_channel();
|
||||
let track = call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
track.add_event(Event::Track(TrackEvent::End), Ended(etx)).unwrap();
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let started = loop {
|
||||
if track.get_info().await.is_ok_and(|i| i.position > Duration::ZERO) {
|
||||
break Instant::now();
|
||||
}
|
||||
assert!(Instant::now() < deadline, "timed out waiting for the track to play");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
let speech_end = started + Duration::from_secs_f64(speech_end_seconds(&wav));
|
||||
let mut final_at = None;
|
||||
until("final", 20_000, || {
|
||||
let got = events.lock().unwrap()[before..].iter().any(|e| matches!(e, CoreEvent::Final(f) if f.heard.speaker_id == speaker_id));
|
||||
if got && final_at.is_none() {
|
||||
final_at = Some(Instant::now());
|
||||
}
|
||||
got
|
||||
}).await;
|
||||
tokio::time::timeout(Duration::from_secs(20), erx.recv()).await.expect("track end");
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
let mine: Vec<_> = events.lock().unwrap()[before..].iter().filter_map(|e| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == speaker_id => Some(f.clone()),
|
||||
_ => None,
|
||||
}).collect();
|
||||
let hyp = mine.iter().map(|f| f.text.as_str()).collect::<Vec<_>>().join(" ");
|
||||
assert!(!mine[0].heard.speaker_name.is_empty());
|
||||
let c = cer(reference, &hyp);
|
||||
total += c;
|
||||
let latency = final_at.unwrap().saturating_duration_since(speech_end).as_millis();
|
||||
stt.push(json!({"ref": reference, "hyp": hyp, "cer": c, "speech_end_to_final_ms": latency,
|
||||
"end_reason": mine.iter().map(|f| f.end_reason.clone()).collect::<Vec<_>>()}));
|
||||
assert!(!hyp.is_empty(), "nothing heard for {reference}");
|
||||
}
|
||||
report.insert("stt".into(), Value::from(stt.clone()));
|
||||
let mean = total / (stt.len() as f64);
|
||||
assert!(mean < 0.15, "mean CER {mean}");
|
||||
|
||||
// 3. Jarvis speaks; the speaker bot hears it and the STT reads it back.
|
||||
let text = "안녕하세요, 저는 자비스입니다. 디스코드 음성 연결을 확인하고 있어요.";
|
||||
ears.pcm.lock().unwrap().clear();
|
||||
let id = core.say(text, false).expect("in voice");
|
||||
until("speech to end", 30_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::SpeakEnd(x) if x.id == id))).await;
|
||||
let end = events.lock().unwrap().iter().find_map(|e| match e { CoreEvent::SpeakEnd(x) if x.id == id => Some(x.clone()), _ => None }).unwrap();
|
||||
assert!(!end.cancelled && !end.failed && end.played == 2, "{end:?}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
let heard = ears.pcm.lock().unwrap().clone();
|
||||
let seconds = heard.len() as f64 / 96000.0;
|
||||
assert!(seconds > 2.0, "heard only {seconds} s");
|
||||
let mut eo = WorkerClientOptions::new(wp.url(), wp.token.clone());
|
||||
eo.stt = json!({"partials": false}).as_object().unwrap().clone();
|
||||
let (echo, mut echo_rx) = WorkerClient::new(eo);
|
||||
echo.start();
|
||||
echo.wait_ready(Duration::from_secs(5)).await.unwrap();
|
||||
for chunk in heard.chunks(1920) {
|
||||
echo.send_audio("echo", chunk);
|
||||
}
|
||||
for _ in 0..25 {
|
||||
echo.send_audio("echo", &[0; 1920]);
|
||||
}
|
||||
echo.speaking_end("echo");
|
||||
let mut back = vec![];
|
||||
let _ = tokio::time::timeout(Duration::from_secs(3), async {
|
||||
while let Some(e) = echo_rx.recv().await {
|
||||
if let WorkerEvent::Final(f) = e {
|
||||
back.push(f.text);
|
||||
}
|
||||
}
|
||||
}).await;
|
||||
echo.close();
|
||||
let hyp = back.join(" ");
|
||||
report.insert("tts_read_back".into(), json!({"seconds": seconds, "hyp": hyp, "cer": cer(text, &hyp)}));
|
||||
assert!(cer(text, &hyp) < 0.2, "read back: {hyp}");
|
||||
|
||||
// 4. Stop silences it within a few frames and cancels the rest.
|
||||
let long = "이 문장은 정지 명령을 시험하려고 길게 말하는 첫 번째 문장입니다. 두 번째 문장도 꽤 길게 이어집니다. \
|
||||
세 번째 문장까지 들리면 정지가 안 된 것입니다. 네 번째 문장은 절대 들리면 안 됩니다.";
|
||||
let id = core.say(long, false).unwrap();
|
||||
until("speech to start", 30_000, || events.lock().unwrap().iter().any(|e| *e == CoreEvent::SpeakStart(id.clone()))).await;
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
let stop_at = Instant::now();
|
||||
assert_eq!(core.stop(), 1);
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
let ends: Vec<_> = events.lock().unwrap().iter().filter_map(|e| match e { CoreEvent::SpeakEnd(x) if x.id == id => Some(x.clone()), _ => None }).collect();
|
||||
assert!(ends.len() == 1 && ends[0].cancelled, "{ends:?}");
|
||||
let tail = ears.last_packet.lock().unwrap().map(|t| t.saturating_duration_since(stop_at).as_millis() as i64).unwrap_or(-1);
|
||||
report.insert("last_sound_after_stop_ms".into(), json!(tail));
|
||||
assert!(tail < 400, "audio went on {tail} ms after stop");
|
||||
assert!(!core.speaking());
|
||||
|
||||
// 5. Chat: live event, history, and a send that pings nobody.
|
||||
let marker = format!("라이브 테스트 메시지 {}", stop_at.elapsed().as_nanos());
|
||||
ChannelId::new(voice).send_message(&s_http, CreateMessage::new().content(&marker)).await.unwrap();
|
||||
until("chat event", 10_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::Chat(c) if c.content == marker))).await;
|
||||
let ev = events.lock().unwrap().iter().find_map(|e| match e { CoreEvent::Chat(c) if c.content == marker => Some(c.clone()), _ => None }).unwrap();
|
||||
assert!(ev.channel_id == voice && ev.author_id == speaker_id && ev.author_is_bot && !ev.from_self);
|
||||
let history = core.read_chat(voice, 20).await.unwrap();
|
||||
assert!(history.iter().any(|m| m.content == marker && !m.from_self));
|
||||
// An edit and a delete reach the history store.
|
||||
let mut marker_msg = ChannelId::new(voice).messages(&s_http, serenity::all::GetMessages::new().limit(5)).await.unwrap()
|
||||
.into_iter().find(|m| m.content == marker).unwrap();
|
||||
let edited = format!("{marker} 수정");
|
||||
marker_msg.edit(&s_http, serenity::all::EditMessage::new().content(&edited)).await.unwrap();
|
||||
let doomed = ChannelId::new(voice).send_message(&s_http, CreateMessage::new().content("곧 지울 메시지")).await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
doomed.delete(&s_http).await.unwrap();
|
||||
let reply = format!("자비스 답장 @everyone {}", stop_at.elapsed().as_nanos());
|
||||
let ids = core.send_chat(voice, &reply).await.unwrap();
|
||||
let sent = ChannelId::new(voice).message(&s_http, serenity::all::MessageId::new(ids[0])).await.unwrap();
|
||||
assert_eq!(sent.content, reply);
|
||||
assert!(!sent.mention_everyone);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
assert!(!events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::Chat(c) if c.content == reply)));
|
||||
|
||||
// 5b. The chat tool's actions, as the worker asks for them: edit (own messages only), react, pin,
|
||||
// unpin, delete; a switch that is off is refused.
|
||||
let act = |action: &str, id: u64, text: Option<&str>, emoji: Option<&str>| jarvis_app::worker_client::ChatAction {
|
||||
id: "live".into(), action: action.into(), channel: voice.to_string(), message_id: Some(id.to_string()),
|
||||
text: text.map(Into::into), emoji: emoji.map(Into::into), user: None };
|
||||
let tool_msg = format!("자비스 도구 시험 {}", stop_at.elapsed().as_nanos());
|
||||
let tid = core.send_chat(voice, &tool_msg).await.unwrap()[0];
|
||||
let edited_tool = format!("{tool_msg} 수정");
|
||||
core.chat_action(&act("edit", tid, Some(&edited_tool), None)).await.expect("edit own message");
|
||||
let err = core.chat_action(&act("edit", marker_msg.id.get(), Some("남의 글"), None)).await.unwrap_err();
|
||||
assert!(err.0.contains("own"), "{err}");
|
||||
core.chat_action(&act("react", marker_msg.id.get(), None, Some("👍"))).await.expect("react");
|
||||
// A voice channel's chat cannot pin (Discord answers "cannot be pinned in this channel"); the
|
||||
// refusal must come back as an error the assistant can say, not a crash.
|
||||
let pinned = core.chat_action(&act("pin", tid, None, None)).await;
|
||||
let pin_report = match &pinned {
|
||||
Ok(_) => "pinned".to_string(),
|
||||
Err(e) => {
|
||||
assert!(e.0.contains("pinned"), "unexpected pin error: {e}");
|
||||
format!("refused by Discord: {}", e.0)
|
||||
}
|
||||
};
|
||||
let fetched = ChannelId::new(voice).message(&s_http, serenity::all::MessageId::new(tid)).await.unwrap();
|
||||
assert_eq!(fetched.content, edited_tool, "edit");
|
||||
assert_eq!(fetched.pinned, pinned.is_ok(), "pin state");
|
||||
let marked = ChannelId::new(voice).message(&s_http, marker_msg.id).await.unwrap();
|
||||
assert!(marked.reactions.iter().any(|r| r.count >= 1), "no reaction");
|
||||
if pinned.is_ok() {
|
||||
core.chat_action(&act("unpin", tid, None, None)).await.expect("unpin");
|
||||
}
|
||||
core.chat_action(&act("delete", tid, None, None)).await.expect("delete");
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
assert!(ChannelId::new(voice).message(&s_http, serenity::all::MessageId::new(tid)).await.is_err(), "not deleted");
|
||||
let off = jarvis_app::worker_client::ChatAction { channel: voice2.to_string(), ..act("send", tid, Some("x"), None) };
|
||||
assert!(core.chat_action(&off).await.unwrap_err().0.contains("not a registered"));
|
||||
report.insert("chat_tool".into(), json!({"edit": true, "edit_others_refused": true, "react": true,
|
||||
"pin": pin_report, "delete": true, "unregistered_refused": true}));
|
||||
|
||||
// 6. Moving to another channel and back: the old connection's shutdown must not end the new one.
|
||||
for (to, what) in [(voice2, "the second channel"), (voice, "the first channel")] {
|
||||
core.join(to).await.expect("move");
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
assert_eq!(core.voice_channel_id(), Some(to), "session lost after moving to {what}");
|
||||
let in_channel = s_cache.guild(GuildId::new(guild))
|
||||
.and_then(|g| g.voice_states.get(&UserId::new(jarvis)).and_then(|v| v.channel_id)).map(|c| c.get());
|
||||
assert_eq!(in_channel, Some(to), "Jarvis is not in {what} on Discord");
|
||||
}
|
||||
let moves: Vec<String> = events.lock().unwrap().iter().filter_map(|e| match e {
|
||||
CoreEvent::Left { reason, .. } => Some(reason.clone()),
|
||||
_ => None,
|
||||
}).collect();
|
||||
assert_eq!(moves, ["moved", "moved"], "only the two moves end a session");
|
||||
|
||||
// 7. Group changes: every join and leave changes the DAVE (MLS) group, and songbird has an open
|
||||
// report of receive stalling after one (issue #310). The speaker leaves and comes back three
|
||||
// times and must still be heard each time, under its new SSRC.
|
||||
let wav = std::fs::read(samples.join("ko_03.wav")).unwrap();
|
||||
let reference = refs["samples"][2]["text"].as_str().unwrap().to_owned();
|
||||
let mut rejoins = vec![];
|
||||
for round in 0..3 {
|
||||
speaker_songbird.remove(GuildId::new(guild)).await.unwrap();
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker rejoins");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let before = events.lock().unwrap().len();
|
||||
let (etx, mut erx) = mpsc::unbounded_channel();
|
||||
let track = call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
track.add_event(Event::Track(TrackEvent::End), Ended(etx)).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), erx.recv()).await.expect("track end");
|
||||
let got = |evs: &[CoreEvent]| evs.iter().filter_map(|e| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == speaker_id && !f.text.is_empty() => Some(f.text.clone()),
|
||||
_ => None,
|
||||
}).collect::<Vec<_>>();
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while got(&events.lock().unwrap()[before..]).is_empty() && Instant::now() < deadline {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
let hyp = got(&events.lock().unwrap()[before..]).join(" ");
|
||||
rejoins.push(json!({"round": round, "hyp": hyp, "cer": cer(&reference, &hyp)}));
|
||||
}
|
||||
report.insert("rejoins".into(), Value::from(rejoins.clone()));
|
||||
assert!(rejoins.iter().all(|r| r["cer"].as_f64().unwrap() < 0.15), "not heard after a rejoin: {rejoins:?}");
|
||||
|
||||
// 8. Leaving.
|
||||
assert!(core.leave("command").await);
|
||||
until("Jarvis to leave", 10_000, || {
|
||||
s_cache.guild(GuildId::new(guild)).and_then(|g| g.voice_states.get(&UserId::new(jarvis)).and_then(|v| v.channel_id)).is_none()
|
||||
}).await;
|
||||
assert_eq!(core.voice_channel_id(), None);
|
||||
assert!(core.say("아무도 못 듣는 말", false).is_none());
|
||||
|
||||
// 9. The history store: the speaker's finals under its name, its chat with the edit and without
|
||||
// the deleted message, Jarvis's own messages, and presence closed now that Jarvis left.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
let check = r#"
|
||||
import json, sys
|
||||
from jarvis.memory.history import HistoryStore
|
||||
s = HistoryStore(sys.argv[1]); sp, jv = sys.argv[2], sys.argv[3]
|
||||
own = s.records_of(sp)
|
||||
q = lambda sql, *a: s._conn.execute(sql, a).fetchone()[0]
|
||||
print(json.dumps({"voice": [[r.author_name, r.channel_id, r.text] for r in own if r.kind == "voice"],
|
||||
"chat": [r.text for r in own if r.kind == "chat"], "jarvis_chat": [r.text for r in s.records_of(jv) if r.kind == "chat"],
|
||||
"presence": q("select count(*) from presence where user_id = ?", sp), "open": q("select count(*) from presence where left_at is null")},
|
||||
ensure_ascii=False))
|
||||
"#;
|
||||
let out = std::process::Command::new(env("JARVIS_VOICE_PYTHON")).current_dir(root.join("../src"))
|
||||
.args(["-c", check, &history_db.to_string_lossy(), &speaker_id.to_string(), &jarvis.to_string()])
|
||||
.output().unwrap();
|
||||
assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
|
||||
let h: Value = serde_json::from_slice(&out.stdout).unwrap();
|
||||
let voice_recs = h["voice"].as_array().unwrap();
|
||||
let speaker_name = s_cache.user(UserId::new(speaker_id)).map(|u| u.display_name().to_owned()).unwrap_or_default();
|
||||
report.insert("history".into(), json!({"voice_records": voice_recs.len(), "presence_intervals": h["presence"],
|
||||
"open_presence": h["open"], "chat": h["chat"], "jarvis_chat": h["jarvis_chat"].as_array().map(|a| a.len())}));
|
||||
assert!(voice_recs.len() >= 8, "finals not stored: {h}");
|
||||
assert!(voice_recs.iter().all(|r| r[0].as_str() != Some(&speaker_id.to_string()) && !r[2].as_str().unwrap().is_empty()),
|
||||
"a final without the speaker's name: {h}");
|
||||
assert!(voice_recs.iter().any(|r| r[0].as_str() == Some(speaker_name.as_str())), "{speaker_name} not in {h}");
|
||||
assert!(voice_recs.iter().any(|r| r[1].as_str() == Some(voice.to_string().as_str())), "{h}");
|
||||
let chat: Vec<&str> = h["chat"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||
assert_eq!(chat, [edited.as_str()], "speaker chat in the history");
|
||||
let jarvis_chat: Vec<&str> = h["jarvis_chat"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
|
||||
assert!(jarvis_chat.contains(&reply.as_str()) && jarvis_chat.iter().any(|t| t.contains("받아 적습니다")), "{h}");
|
||||
assert!(!jarvis_chat.iter().any(|t| t.contains(&tool_msg)), "the deleted tool message is still stored: {h}");
|
||||
assert!(h["presence"].as_u64().unwrap() >= 4, "joins and rejoins open presence: {h}");
|
||||
assert_eq!(h["open"], 0, "presence left open after Jarvis left: {h}");
|
||||
|
||||
println!("live report {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
||||
})).await;
|
||||
|
||||
// Clean up even when a step failed: the test command, the speaker, the core, the worker.
|
||||
let http = Http::new(&bot_token);
|
||||
http.set_application_id(jarvis_app_id.into());
|
||||
for c in http.get_guild_commands(GuildId::new(guild)).await.unwrap().iter().filter(|c| c.name == COMMAND) {
|
||||
let _ = http.delete_guild_command(GuildId::new(guild), c.id).await;
|
||||
}
|
||||
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
||||
s_shards.shutdown_all().await;
|
||||
core.close().await;
|
||||
worker.close();
|
||||
wp.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
373
app/tests/live_round_trip.rs
Normal file
373
app/tests/live_round_trip.rs
Normal file
@@ -0,0 +1,373 @@
|
||||
//! M4 step 4: the whole conversation on real Discord, timed from the end of a question to Jarvis's
|
||||
//! first sound, with the real voice worker, the real brain (Strata) and the real response policy.
|
||||
//!
|
||||
//! The "speaker" bot plays recorded Korean questions that call Jarvis into the voice channel and
|
||||
//! listens for the answer. The Jarvis bot runs the app's services (`start_services`: core,
|
||||
//! conversation loop, worker process). Each question is timed at four points after the speech ends:
|
||||
//! the final transcript, the brain's answer text, playback start and the first audible frame the
|
||||
//! speaker bot receives. The rounds run again while nine more voices talk at the same time (a second
|
||||
//! worker session feeding the shared STT), which is the 10-person load.
|
||||
//! Opt-in; it refuses to run if a person is in the channel or either bot is already in voice.
|
||||
//!
|
||||
//! JARVIS_LIVE_ROUND_TRIP=1 JARVIS_LIVE_BOT_TOKEN=... JARVIS_LIVE_SPEAKER_TOKEN=... \
|
||||
//! JARVIS_LIVE_GUILD=<server id> JARVIS_LIVE_VOICE_CHANNEL=<voice channel id> \
|
||||
//! JARVIS_VOICE_PYTHON=<worker venv python> JARVIS_MELO_PYTHON=<MeloTTS venv python> \
|
||||
//! JARVIS_LIVE_LLM_URL=<Strata's /v1 address, e.g. a tunnel to the game PC> \
|
||||
//! JARVIS_LIVE_QUESTIONS=<dir with refs.json and 16 kHz mono wavs> [JARVIS_LIVE_ROUNDS=2] \
|
||||
//! cargo test --release --test live_round_trip -- --nocapture
|
||||
//!
|
||||
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use jarvis_app::discord::CoreEvent;
|
||||
use jarvis_app::receive::{opus_payload, OpusFrames, SpeakerStream};
|
||||
use jarvis_app::services::start_services;
|
||||
use jarvis_app::strata::Llm;
|
||||
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions};
|
||||
use serde_json::{json, Value};
|
||||
use serenity::all::{ChannelId, Client, Context, EventHandler, GatewayIntents, GuildId, Ready, UserId};
|
||||
use songbird::driver::DecodeMode;
|
||||
use songbird::input::Input;
|
||||
use songbird::{Config, CoreEvent as VoiceEv, Event, EventContext, EventHandler as VoiceHandler, SerenityInit, Songbird};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn env(k: &str) -> String {
|
||||
std::env::var(k).unwrap_or_else(|_| panic!("{k} is not set"))
|
||||
}
|
||||
|
||||
/// Seconds from the start of a 16 kHz mono PCM16 wav to its last sample above -34 dBFS.
|
||||
fn speech_end_seconds(wav: &[u8]) -> f64 {
|
||||
let data = &wav[44..];
|
||||
let n = data.len() / 2;
|
||||
(0..n).rev().find(|&k| i16::from_le_bytes([data[2 * k], data[2 * k + 1]]).unsigned_abs() > 655)
|
||||
.map_or(0.0, |k| (k + 1) as f64 / 16000.0)
|
||||
}
|
||||
|
||||
fn wav_samples(wav: &[u8]) -> Vec<i16> {
|
||||
wav[44..].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()
|
||||
}
|
||||
|
||||
struct HeardText {
|
||||
text: String,
|
||||
stt_ms: u64,
|
||||
}
|
||||
|
||||
struct ReadyOnce(mpsc::UnboundedSender<Ready>);
|
||||
#[async_trait]
|
||||
impl EventHandler for ReadyOnce {
|
||||
async fn ready(&self, _: Context, r: Ready) {
|
||||
let _ = self.0.send(r);
|
||||
}
|
||||
}
|
||||
|
||||
/// When Jarvis's voice first becomes audible to the speaker bot, decoded the way the core decodes
|
||||
/// (songbird's own decoder garbles DAVE audio, see src/receive.rs).
|
||||
#[derive(Clone, Default)]
|
||||
struct Ears {
|
||||
jarvis: u64,
|
||||
ssrc: Arc<Mutex<HashMap<u32, u64>>>,
|
||||
stream: Arc<Mutex<Option<SpeakerStream<OpusFrames>>>>,
|
||||
/// When each audible 20 ms frame arrived (the earcon first, then the answer).
|
||||
audible: Arc<Mutex<Vec<Instant>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceHandler for Ears {
|
||||
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event> {
|
||||
match ctx {
|
||||
EventContext::SpeakingStateUpdate(s) => {
|
||||
if let Some(u) = s.user_id {
|
||||
self.ssrc.lock().unwrap().insert(s.ssrc, u.0);
|
||||
}
|
||||
}
|
||||
EventContext::RtpPacket(p) => {
|
||||
let (seq, ssrc, opus) = opus_payload(&p.packet, p.payload_offset, p.payload_end_pad)?;
|
||||
if self.ssrc.lock().unwrap().get(&ssrc) != Some(&self.jarvis) {
|
||||
return None;
|
||||
}
|
||||
let frames = self.stream.lock().unwrap().get_or_insert_with(|| SpeakerStream::new(OpusFrames::new()))
|
||||
.push(seq, opus.to_vec());
|
||||
for f in frames {
|
||||
// Audible: louder than -36 dBFS. Silence frames do not count.
|
||||
let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>() / f.len().max(1) as f64).sqrt();
|
||||
if rms > 500.0 {
|
||||
self.audible.lock().unwrap().push(Instant::now());
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn until(what: &str, ms: u64, mut pred: impl FnMut() -> bool) {
|
||||
let deadline = Instant::now() + Duration::from_millis(ms);
|
||||
while !pred() {
|
||||
assert!(Instant::now() < deadline, "timed out waiting for {what}");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Nine more voices: a second worker session streams recorded sentences for nine speakers in real
|
||||
/// time, all talking at once with short pauses, until `stop` is set.
|
||||
fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: Arc<AtomicBool>) -> tokio::task::JoinHandle<u64> {
|
||||
tokio::spawn(async move {
|
||||
let mut o = WorkerClientOptions::new(url, token);
|
||||
o.stt = json!({"partials": true}).as_object().unwrap().clone();
|
||||
let (load, mut rx) = WorkerClient::new(o);
|
||||
load.start();
|
||||
load.wait_ready(Duration::from_secs(10)).await.expect("load session hello");
|
||||
let finals = Arc::new(Mutex::new(0u64));
|
||||
let f2 = finals.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
if matches!(e, jarvis_app::worker_client::WorkerEvent::Final(_)) {
|
||||
*f2.lock().unwrap() += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
// 20 ms of 48 kHz stereo per frame, the format the core sends.
|
||||
let mut pos: Vec<(usize, usize, usize)> = (0..9).map(|i| (i % samples.len(), i * 4000, 0)).collect();
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
tick.tick().await;
|
||||
for (i, (sample, at, pause)) in pos.iter_mut().enumerate() {
|
||||
let speaker = format!("load-{i}");
|
||||
if *pause > 0 {
|
||||
*pause -= 1;
|
||||
if *pause == 0 {
|
||||
load.speaking_end(&speaker);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let s = &samples[*sample];
|
||||
let end = (*at + 320).min(s.len());
|
||||
let frame: Vec<i16> = s[*at..end].iter().flat_map(|&x| [x; 6]).collect();
|
||||
load.send_audio(&speaker, &frame);
|
||||
*at = end;
|
||||
if end == s.len() {
|
||||
*sample = (*sample + 1) % samples.len();
|
||||
*at = 0;
|
||||
*pause = 25 + i * 3; // 0.5-1 s apart, staggered
|
||||
}
|
||||
}
|
||||
}
|
||||
load.close();
|
||||
let n = *finals.lock().unwrap();
|
||||
n
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
||||
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
env("JARVIS_LIVE_LLM_URL"); // the services point the brain at it (llm.url)
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
|
||||
// The speaker bot first: it checks the channel is free before anything joins.
|
||||
let speaker_songbird = Songbird::serenity_from_config(Config::default().decode_mode(DecodeMode::Decrypt));
|
||||
let (rtx, mut rrx) = mpsc::unbounded_channel();
|
||||
let mut speaker = Client::builder(env("JARVIS_LIVE_SPEAKER_TOKEN"), GatewayIntents::GUILDS | GatewayIntents::GUILD_VOICE_STATES)
|
||||
.event_handler(ReadyOnce(rtx)).register_songbird_with(speaker_songbird.clone()).await.unwrap();
|
||||
let (s_cache, s_shards) = (speaker.cache.clone(), speaker.shard_manager.clone());
|
||||
tokio::spawn(async move { speaker.start().await.unwrap() });
|
||||
let speaker_id = rrx.recv().await.unwrap().user.id.get();
|
||||
let bot_token = env("JARVIS_LIVE_BOT_TOKEN");
|
||||
let jarvis_app_id = serenity::all::Http::new(&bot_token).get_current_application_info().await.unwrap().id.get();
|
||||
until("guild in cache", 10_000, || s_cache.guild(GuildId::new(guild)).is_some()).await;
|
||||
{
|
||||
let g = s_cache.guild(GuildId::new(guild)).unwrap();
|
||||
let people = g.voice_states.values().filter(|v| v.channel_id.is_some_and(|c| c.get() == voice)
|
||||
&& !s_cache.user(v.user_id).map(|u| u.bot).unwrap_or(false)).count();
|
||||
assert_eq!(people, 0, "refusing to test: people are in the voice channel");
|
||||
for id in [speaker_id, jarvis_app_id] {
|
||||
assert!(g.voice_states.get(&UserId::new(id)).and_then(|v| v.channel_id).is_none(),
|
||||
"refusing to test: bot {id} is in a voice channel");
|
||||
}
|
||||
}
|
||||
|
||||
// The app's services, as the app starts them, from a fresh data folder.
|
||||
let data = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("live-round-trip-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&data);
|
||||
std::fs::create_dir_all(data.join("config")).unwrap();
|
||||
let mut args = vec!["--history".to_string(), data.join("history.sqlite").to_string_lossy().into_owned(), "--brain".into()];
|
||||
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
||||
args.extend(["--melo-python".to_string(), m]);
|
||||
}
|
||||
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
|
||||
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
|
||||
// The "person" here is a bot, so bots must be heard.
|
||||
"voice": {"listenToBots": true},
|
||||
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
|
||||
"llm": {"url": env("JARVIS_LIVE_LLM_URL")},
|
||||
})).unwrap()).unwrap();
|
||||
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
|
||||
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
.expect("services start in time").expect("services start");
|
||||
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
|
||||
let core = services.core.clone();
|
||||
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
|
||||
let ev2 = events.clone();
|
||||
let mut rx = services.events;
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
ev2.lock().unwrap().push((Instant::now(), e));
|
||||
}
|
||||
});
|
||||
until("Jarvis in voice", 20_000, || core.voice_channel_id() == Some(voice)).await;
|
||||
let jarvis = core.bot_id().unwrap();
|
||||
|
||||
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
|
||||
let ears = Ears { jarvis, ..Default::default() };
|
||||
{
|
||||
let mut c = call.lock().await;
|
||||
c.add_global_event(VoiceEv::SpeakingStateUpdate.into(), ears.clone());
|
||||
c.add_global_event(VoiceEv::RtpPacket.into(), ears.clone());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
|
||||
|
||||
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
|
||||
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
|
||||
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
|
||||
.collect();
|
||||
let load_samples: Vec<Vec<i16>> = std::fs::read_dir(root.join("../scripts/m0/assets/ko_samples")).unwrap()
|
||||
.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "wav"))
|
||||
.map(|p| wav_samples(&std::fs::read(p).unwrap())).collect();
|
||||
|
||||
let mut report = serde_json::Map::new();
|
||||
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
||||
for phase in ["alone", "with_9_more_voices"] {
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let load = (phase != "alone").then(|| {
|
||||
background_voices(services.worker_process.url(), services.worker_process.token.clone(), load_samples.clone(), stop.clone())
|
||||
});
|
||||
if load.is_some() {
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
let mut rows = vec![];
|
||||
for round in 0..rounds {
|
||||
for (text, wav) in &qs {
|
||||
// The previous answer is over before the next question.
|
||||
until("Jarvis to finish speaking", 60_000, || !core.speaking()).await;
|
||||
let before = events.lock().unwrap().len();
|
||||
ears.audible.lock().unwrap().clear();
|
||||
let track = call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let started = loop {
|
||||
if track.get_info().await.is_ok_and(|i| i.position > Duration::ZERO) {
|
||||
break Instant::now();
|
||||
}
|
||||
assert!(Instant::now() < deadline, "timed out waiting for the track to play");
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
};
|
||||
let speech_end = started + Duration::from_secs_f64(speech_end_seconds(wav));
|
||||
// The answer is over when its own playback ends (or nothing came for 30 s).
|
||||
let finished = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
{
|
||||
let ev = events.lock().unwrap();
|
||||
let id = ev[before..].iter().find_map(|(_, e)| match e {
|
||||
CoreEvent::ReplyText { id, .. } => Some(id.clone()),
|
||||
_ => None,
|
||||
});
|
||||
if id.is_some_and(|id| ev[before..].iter().any(|(_, e)| matches!(e, CoreEvent::SpeakEnd(x) if x.id == id))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}).await.is_ok();
|
||||
let ev = events.lock().unwrap()[before..].to_vec();
|
||||
let ms = |t: Option<Instant>| t.map(|t| t.saturating_duration_since(speech_end).as_millis() as i64);
|
||||
let reply = ev.iter().find_map(|(t, e)| match e {
|
||||
CoreEvent::ReplyText { text, failed, .. } => Some((*t, text.clone(), *failed)),
|
||||
_ => None,
|
||||
});
|
||||
// A question may be heard as several finals (a pause after the wake word): all of
|
||||
// them are what was heard, and the last one before the answer is the one timed.
|
||||
let finals: Vec<_> = ev.iter().filter_map(|(t, e)| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == speaker_id && !f.text.is_empty()
|
||||
&& reply.as_ref().is_none_or(|r| *t <= r.0) => Some((*t, f.clone())),
|
||||
_ => None,
|
||||
}).collect();
|
||||
let final_ev = finals.last().map(|(t, f)| (*t, HeardText {
|
||||
text: finals.iter().map(|(_, f)| f.text.as_str()).collect::<Vec<_>>().join(" | "),
|
||||
stt_ms: f.stt_ms,
|
||||
}));
|
||||
// The first sound is the earcon; the answer's own sound starts after its SpeakStart.
|
||||
let reply_id = ev.iter().find_map(|(_, e)| match e {
|
||||
CoreEvent::ReplyText { id, .. } => Some(id.clone()),
|
||||
_ => None,
|
||||
});
|
||||
let speak_start = ev.iter().find_map(|(t, e)| match e {
|
||||
CoreEvent::SpeakStart(id) if Some(id) == reply_id.as_ref() => Some(*t),
|
||||
_ => None,
|
||||
});
|
||||
let audible = ears.audible.lock().unwrap().clone();
|
||||
let first_sound = audible.first().copied();
|
||||
let answer_sound = speak_start.and_then(|s| audible.iter().copied().find(|t| *t >= s));
|
||||
rows.push(json!({
|
||||
"round": round, "question": text,
|
||||
"heard": final_ev.as_ref().map(|(_, f)| f.text.clone()),
|
||||
"stt_ms": final_ev.as_ref().map(|(_, f)| f.stt_ms),
|
||||
"answer": reply.as_ref().map(|r| r.1.clone()),
|
||||
"final_ms": ms(final_ev.as_ref().map(|f| f.0)),
|
||||
"answer_text_ms": ms(reply.as_ref().map(|r| r.0)),
|
||||
"speak_start_ms": ms(speak_start),
|
||||
"first_sound_ms": ms(first_sound),
|
||||
"answer_sound_ms": ms(answer_sound),
|
||||
"spoken": finished,
|
||||
}));
|
||||
println!("{phase} {}", rows.last().unwrap());
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let load_finals = match load { Some(h) => Some(h.await.unwrap()), None => None };
|
||||
let stats = |key: &str| {
|
||||
let mut v: Vec<i64> = rows.iter().filter_map(|r| r[key].as_i64()).collect();
|
||||
v.sort_unstable();
|
||||
let pick = |q: f64| v.get(((v.len() as f64 - 1.0) * q).round() as usize).copied();
|
||||
(v.len(), json!({"min": v.first(), "median": pick(0.5), "p90": pick(0.9), "max": v.last()}))
|
||||
};
|
||||
let (answered, answer_sound) = stats("answer_sound_ms");
|
||||
report.insert(phase.into(), json!({
|
||||
"rows": rows, "answered": answered, "asked": rows.len(),
|
||||
"first_sound_ms": stats("first_sound_ms").1,
|
||||
"answer_sound_ms": answer_sound,
|
||||
"background_finals": load_finals,
|
||||
}));
|
||||
}
|
||||
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
||||
})).await;
|
||||
|
||||
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
||||
s_shards.shutdown_all().await;
|
||||
services.core.leave("test").await;
|
||||
services.core.close().await;
|
||||
services.worker.close();
|
||||
services.worker_process.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// Every question is answered aloud.
|
||||
for phase in ["alone", "with_9_more_voices"] {
|
||||
let p = &report[phase];
|
||||
assert_eq!(p["answered"], p["asked"], "{phase}: not every question got a spoken answer: {p}");
|
||||
}
|
||||
}
|
||||
|
||||
129
app/tests/screen_windows.rs
Normal file
129
app/tests/screen_windows.rs
Normal file
@@ -0,0 +1,129 @@
|
||||
//! Screen viewing on a real Windows desktop (admin.spec.md, "Screen viewing"): the monitors and
|
||||
//! windows are found, a window, each monitor and the whole screen are captured as JPEG pictures of
|
||||
//! the expected size, and an excluded program is neither picked nor shown. Runs only on Windows
|
||||
//! with `JARVIS_TEST_SCREEN=1` (it opens Notepad); elsewhere it skips.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use base64::Engine as _;
|
||||
use jarvis_app::screen;
|
||||
use jarvis_app::settings::Settings;
|
||||
use jarvis_app::worker_client::ScreenAction;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
fn action(v: Value) -> ScreenAction {
|
||||
let mut m = json!({"id": "s1", "user": "42"});
|
||||
m.as_object_mut().unwrap().extend(v.as_object().unwrap().clone());
|
||||
serde_json::from_value(m).unwrap()
|
||||
}
|
||||
|
||||
/// The pictures of an answer, decoded: (width, height, RGB pixels).
|
||||
fn pictures(v: &Value) -> Vec<(u32, u32, Vec<u8>)> {
|
||||
v["images"].as_array().unwrap().iter().map(|i| {
|
||||
let bytes = base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap();
|
||||
let mut d = jpeg_decoder::Decoder::new(&bytes[..]);
|
||||
let px = d.decode().unwrap();
|
||||
let info = d.info().unwrap();
|
||||
(info.width as u32, info.height as u32, px)
|
||||
}).collect()
|
||||
}
|
||||
|
||||
/// The share of near-black pixels in `r` (screen coordinates) of a picture of `area`.
|
||||
fn black_share(pic: &(u32, u32, Vec<u8>), area: screen::Rect, r: screen::Rect) -> f64 {
|
||||
let (w, h, px) = pic;
|
||||
let (sx, sy) = (*w as f64 / area.w as f64, *h as f64 / area.h as f64);
|
||||
let (x0, x1) = (((r.x - area.x) as f64 * sx) as u32 + 2, ((r.x + r.w - area.x) as f64 * sx) as u32 - 2);
|
||||
let (y0, y1) = (((r.y - area.y) as f64 * sy) as u32 + 2, ((r.y + r.h - area.y) as f64 * sy) as u32 - 2);
|
||||
let (mut black, mut all) = (0, 0);
|
||||
for y in y0..y1.min(*h) {
|
||||
for x in x0..x1.min(*w) {
|
||||
let o = ((y * w + x) * 3) as usize;
|
||||
all += 1;
|
||||
if px[o] < 30 && px[o + 1] < 30 && px[o + 2] < 30 {
|
||||
black += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
black as f64 / all.max(1) as f64
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_real_screen_is_seen_and_excluded_programs_are_not() {
|
||||
if std::env::var("JARVIS_TEST_SCREEN").is_err() {
|
||||
eprintln!("skipped: set JARVIS_TEST_SCREEN=1");
|
||||
return;
|
||||
}
|
||||
let sys = screen::system_screen().expect("Windows has a screen");
|
||||
let monitors = sys.monitors().unwrap();
|
||||
println!("monitors: {:?}", monitors.iter().map(|m| (m.number, m.rect, m.main)).collect::<Vec<_>>());
|
||||
assert!(!monitors.is_empty());
|
||||
// Closed however the test ends; no inherited output handles, so a waiting shell is not held.
|
||||
struct Closed(std::process::Child);
|
||||
impl Drop for Closed {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
}
|
||||
}
|
||||
let _notepad = Closed(std::process::Command::new("notepad.exe").stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).spawn().unwrap());
|
||||
let until = Instant::now() + Duration::from_secs(15);
|
||||
let win = loop {
|
||||
if let Some(w) = sys.windows().unwrap().into_iter().find(|w| w.program == "notepad.exe" && !w.secondary) {
|
||||
break w;
|
||||
}
|
||||
assert!(Instant::now() < until, "Notepad's window did not show");
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
};
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
println!("notepad: {:?} minimised={}", win.rect, win.minimised);
|
||||
let mut s = Settings::default();
|
||||
s.pc_control.allowed_users = vec![42];
|
||||
|
||||
let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap();
|
||||
println!("list: {} monitors, {} windows", list["item"]["monitors"].as_array().unwrap().len(),
|
||||
list["item"]["windows"].as_array().unwrap().len());
|
||||
assert!(list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad"));
|
||||
|
||||
let t = Instant::now();
|
||||
let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap();
|
||||
let pics = pictures(&v);
|
||||
println!("window: {} in {} ms, picture {}x{}", v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1);
|
||||
assert_eq!(v["item"]["what"], "창 notepad");
|
||||
assert!(pics[0].0 > 100 && pics[0].1 > 100);
|
||||
|
||||
for m in &monitors {
|
||||
let t = Instant::now();
|
||||
let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "monitor", "monitor": m.number, "detail": "fast"}))).unwrap();
|
||||
let pics = pictures(&v);
|
||||
println!("monitor {}: {} in {} ms, picture {}x{}", m.number, v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1);
|
||||
assert_eq!((v["item"]["width"].as_i64(), v["item"]["height"].as_i64()), (Some(m.rect.w as i64), Some(m.rect.h as i64)));
|
||||
assert!(pics[0].0.max(pics[0].1) <= screen::FAST_PX);
|
||||
}
|
||||
|
||||
let area = screen::Rect { x: monitors.iter().map(|m| m.rect.x).min().unwrap(), y: monitors.iter().map(|m| m.rect.y).min().unwrap(),
|
||||
w: monitors.iter().map(|m| m.rect.x + m.rect.w).max().unwrap() - monitors.iter().map(|m| m.rect.x).min().unwrap(),
|
||||
h: monitors.iter().map(|m| m.rect.y + m.rect.h).max().unwrap() - monitors.iter().map(|m| m.rect.y).min().unwrap() };
|
||||
let t = Instant::now();
|
||||
let shown = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap();
|
||||
let shown_pics = pictures(&shown);
|
||||
println!("screen: {} in {} ms, {} part(s)", shown["item"], t.elapsed().as_millis(), shown_pics.len());
|
||||
let shown_black = black_share(&shown_pics[0], area, win.rect);
|
||||
println!("notepad area black when shown: {shown_black:.2}");
|
||||
|
||||
// Excluded: never picked or listed, and painted black on the screen.
|
||||
s.pc_control.screen_exclude = vec!["notepad".into()];
|
||||
let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap_err();
|
||||
println!("excluded window: {e}");
|
||||
assert!(e.contains("찾지 못했습니다"));
|
||||
let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap();
|
||||
assert!(!list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad"));
|
||||
let hidden = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap();
|
||||
let hidden_black = black_share(&pictures(&hidden)[0], area, win.rect);
|
||||
println!("notepad area black when excluded: {hidden_black:.2}");
|
||||
assert!(hidden_black > 0.99, "the excluded window is black");
|
||||
assert!(shown_black < 0.5, "Notepad shows when not excluded");
|
||||
|
||||
// Someone not registered sees nothing.
|
||||
let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "list", "user": "7"}))).unwrap_err();
|
||||
assert!(e.contains("등록된 사람만"));
|
||||
}
|
||||
48
app/tests/ui/talk.test.mjs
Normal file
48
app/tests/ui/talk.test.mjs
Normal file
@@ -0,0 +1,48 @@
|
||||
// The 대화 screen's paging (app/ui/talk.js; admin.spec.md, "Screens"). Run: node --test app/tests/ui/talk.test.mjs
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { PAGE, PAGE_MAX, compare, empty, withLive, withOlder } from "../../ui/talk.js";
|
||||
|
||||
const rec = (n) => ({ key: `h${n}`, at: n, type: "said", text: `말 ${n}` });
|
||||
const range = (a, b) => Array.from({ length: b - a }, (_, i) => rec(a + i));
|
||||
const keys = (s) => s.older.concat(s.shown).map((i) => i.key);
|
||||
|
||||
/** The screen's state after the first page and as many "이전 기록 더 보기" as the store allows. */
|
||||
function allLoaded(total) {
|
||||
const store = range(0, total);
|
||||
let s = withLive(empty(), store.slice(-PAGE), total > PAGE);
|
||||
while (s.hasOlder) {
|
||||
const oldest = (s.older[0] || s.shown[0]).at;
|
||||
const before = store.filter((i) => i.at < oldest);
|
||||
s = withOlder(s, before.slice(-PAGE), before.length > PAGE);
|
||||
}
|
||||
return { s, store };
|
||||
}
|
||||
|
||||
test("every older page joins once, past the live window", () => {
|
||||
const { s } = allLoaded(2501);
|
||||
assert.equal(s.shown.length, PAGE_MAX);
|
||||
assert.deepEqual(keys(s), range(0, 2501).map((i) => i.key));
|
||||
});
|
||||
|
||||
test("the same older page arriving twice is not added twice", () => {
|
||||
let s = withLive(empty(), range(300, 500), true);
|
||||
s = withOlder(s, range(100, 300), true);
|
||||
s = withOlder(s, range(100, 300), true);
|
||||
assert.deepEqual(keys(s), range(100, 500).map((i) => i.key));
|
||||
});
|
||||
|
||||
test("a new item on a full window keeps everything loaded, in order and once", () => {
|
||||
const { s, store } = allLoaded(2501);
|
||||
const now = store.concat(rec(2501));
|
||||
// The live answer: from the window's start, at most PAGE_MAX, newest first cut.
|
||||
const since = now.filter((i) => compare(i, s.shown[0]) >= 0);
|
||||
const next = withLive(s, since.slice(-PAGE_MAX), since.length > PAGE_MAX);
|
||||
assert.deepEqual(keys(next), range(0, 2502).map((i) => i.key));
|
||||
assert.equal(next.shown.length, PAGE_MAX);
|
||||
});
|
||||
|
||||
test("items are ordered by time, then records before traces, then id", () => {
|
||||
const items = [{ key: "t2", at: 1 }, { key: "h10", at: 1 }, { key: "h9", at: 1 }, { key: "h1", at: 0 }];
|
||||
assert.deepEqual(items.sort(compare).map((i) => i.key), ["h1", "h9", "h10", "t2"]);
|
||||
});
|
||||
31
app/tests/vdisplay_elevated.rs
Normal file
31
app/tests/vdisplay_elevated.rs
Normal file
@@ -0,0 +1,31 @@
|
||||
//! Installing and removing the virtual monitor's driver through the app's elevated half
|
||||
//! (admin.spec.md, "Virtual monitor"). Windows only, with the driver files in `display-driver/` next
|
||||
//! to this program and `JARVIS_TEST_VDISPLAY=install` or `uninstall`, or `install-declined` (the
|
||||
//! person answers no to Windows' question: the install fails and leaves nothing installed); Windows
|
||||
//! asks for the administrator (and on install whether to trust the SignPath Foundation). Elsewhere
|
||||
//! it skips.
|
||||
//! No test harness: started elevated, this same program is the elevated half.
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if let Some(code) = jarvis_app::vdisplay::elevated_main(&args) {
|
||||
std::process::exit(code);
|
||||
}
|
||||
let Ok(action) = std::env::var("JARVIS_TEST_VDISPLAY") else {
|
||||
eprintln!("skipped: set JARVIS_TEST_VDISPLAY=install|install-declined|uninstall");
|
||||
return;
|
||||
};
|
||||
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
|
||||
let declined = action == "install-declined";
|
||||
let outcome = rt.block_on(jarvis_app::vdisplay::run_elevated(if declined { "install" } else { &action }));
|
||||
let status = jarvis_app::vdisplay::status();
|
||||
println!("{action}: {outcome:?} -> {status:?}");
|
||||
if declined {
|
||||
assert!(outcome.is_err(), "a declined install fails");
|
||||
} else {
|
||||
outcome.expect("the elevated half succeeds");
|
||||
}
|
||||
assert_eq!(status.installed, action == "install");
|
||||
assert!(!status.other, "nothing this app made looks like another program's driver");
|
||||
assert!(!status.leftover, "nothing of this app's is left to clean up");
|
||||
}
|
||||
524
app/tests/worker_client.rs
Normal file
524
app/tests/worker_client.rs
Normal file
@@ -0,0 +1,524 @@
|
||||
//! The worker client against a stand-in worker server (hello, events, reconnect, failed TTS).
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use jarvis_app::frames::{encode_audio_frame, KIND_TTS};
|
||||
use jarvis_app::playback::TtsBackend;
|
||||
use jarvis_app::settings::ChatChannelSettings;
|
||||
use jarvis_app::worker_client::{ChatAction, TtsDone, WorkerClient, WorkerClientOptions, WorkerEvent};
|
||||
use serde_json::{json, Value};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
const TOKEN: &str = "t0ken";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Got {
|
||||
Json(Value),
|
||||
Bin(Vec<u8>),
|
||||
}
|
||||
|
||||
/// A fake worker: checks the hello, records what arrives, and lets the test push frames.
|
||||
struct FakeWorker {
|
||||
port: u16,
|
||||
received: Arc<Mutex<Vec<Got>>>,
|
||||
push: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
conns: Arc<Mutex<Vec<tokio::task::JoinHandle<()>>>>,
|
||||
}
|
||||
|
||||
impl FakeWorker {
|
||||
async fn start(port: u16) -> Self {
|
||||
let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let received = Arc::new(Mutex::new(vec![]));
|
||||
let push = Arc::new(Mutex::new(None));
|
||||
let conns = Arc::new(Mutex::new(vec![]));
|
||||
let (r, p, cs) = (received.clone(), push.clone(), conns.clone());
|
||||
let task = tokio::spawn(async move {
|
||||
while let Ok((tcp, _)) = listener.accept().await {
|
||||
let ws = tokio_tungstenite::accept_async(tcp).await.unwrap();
|
||||
let (mut tx, mut rx) = ws.split();
|
||||
let (ptx, mut prx) = mpsc::unbounded_channel::<Message>();
|
||||
*p.lock().unwrap() = Some(ptx.clone());
|
||||
let r = r.clone();
|
||||
cs.lock().unwrap().push(tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
m = prx.recv() => match m { Some(m) => { if tx.send(m).await.is_err() { break } } None => break },
|
||||
m = rx.next() => match m {
|
||||
Some(Ok(Message::Text(t))) => {
|
||||
let v: Value = serde_json::from_str(t.as_str()).unwrap();
|
||||
if v["type"] == "hello" {
|
||||
if v["token"] != TOKEN { let _ = tx.close().await; break; }
|
||||
let ok = json!({"type": "hello_ok", "stt_ready": true, "tts_ready": true, "stt": v["stt"]});
|
||||
let _ = ptx.send(Message::text(ok.to_string()));
|
||||
}
|
||||
r.lock().unwrap().push(Got::Json(v));
|
||||
}
|
||||
Some(Ok(Message::Binary(b))) => r.lock().unwrap().push(Got::Bin(b.to_vec())),
|
||||
Some(Ok(_)) => {}
|
||||
_ => break,
|
||||
},
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
});
|
||||
Self { port, received, push, task, conns }
|
||||
}
|
||||
|
||||
fn url(&self) -> String {
|
||||
format!("ws://127.0.0.1:{}", self.port)
|
||||
}
|
||||
|
||||
fn got(&self) -> Vec<Got> {
|
||||
self.received.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn send(&self, m: Message) {
|
||||
self.push.lock().unwrap().as_ref().unwrap().send(m).unwrap();
|
||||
}
|
||||
|
||||
/// The worker dies: stop listening and drop the connection.
|
||||
fn kill(self) -> u16 {
|
||||
self.task.abort();
|
||||
for c in self.conns.lock().unwrap().drain(..) {
|
||||
c.abort();
|
||||
}
|
||||
*self.push.lock().unwrap() = None;
|
||||
self.port
|
||||
}
|
||||
}
|
||||
|
||||
fn client(url: &str, stt: Value) -> (WorkerClient, mpsc::UnboundedReceiver<WorkerEvent>) {
|
||||
let mut o = WorkerClientOptions::new(url, TOKEN);
|
||||
o.stt = stt.as_object().cloned().unwrap_or_default();
|
||||
o.backoff = Duration::from_millis(20);
|
||||
o.max_backoff = Duration::from_millis(50);
|
||||
WorkerClient::new(o)
|
||||
}
|
||||
|
||||
async fn until(mut pred: impl FnMut() -> bool) {
|
||||
tokio::time::timeout(Duration::from_secs(3), async {
|
||||
while !pred() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("timed out");
|
||||
}
|
||||
|
||||
async fn next_matching(rx: &mut mpsc::UnboundedReceiver<WorkerEvent>, f: impl Fn(&WorkerEvent) -> bool) -> WorkerEvent {
|
||||
tokio::time::timeout(Duration::from_secs(3), async {
|
||||
loop {
|
||||
let e = rx.recv().await.expect("events closed");
|
||||
if f(&e) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("timed out")
|
||||
}
|
||||
|
||||
fn tts_frame(id: &str, seq: u32) -> Message {
|
||||
let mut f = vec![KIND_TTS, id.len() as u8];
|
||||
f.extend_from_slice(id.as_bytes());
|
||||
f.extend_from_slice(&seq.to_le_bytes());
|
||||
f.extend_from_slice(&[0; 8]);
|
||||
Message::binary(f)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hello_carries_token_formats_and_stt_then_audio_flows() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, _rx) = client(&w.url(), json!({"hotwords": ["자비스"]}));
|
||||
c.start();
|
||||
let ok = c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
assert_eq!(Value::Object(ok.stt), json!({"hotwords": ["자비스"]}));
|
||||
let pcm = vec![1i16; 1920];
|
||||
c.send_audio("123", &pcm);
|
||||
c.speaking_end("123");
|
||||
c.speaker_leave("123");
|
||||
until(|| w.got().len() >= 4).await;
|
||||
let got = w.got();
|
||||
assert_eq!(got[0], Got::Json(json!({"type": "hello", "token": TOKEN, "audio": {"rate": 48000, "channels": 2},
|
||||
"tts": {"rate": 48000, "channels": 2}, "stt": {"hotwords": ["자비스"]}})));
|
||||
assert_eq!(got[1], Got::Bin(encode_audio_frame("123", &pcm).unwrap()));
|
||||
assert_eq!(got[2..], [Got::Json(json!({"type": "speaking_end", "speaker": "123"})),
|
||||
Got::Json(json!({"type": "speaker_leave", "speaker": "123"}))]);
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hello_carries_the_memory_settings_when_there_are_any() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let mut o = WorkerClientOptions::new(w.url(), TOKEN);
|
||||
o.memory = json!({"others": false, "recent_records": 5}).as_object().cloned().unwrap();
|
||||
let (c, _rx) = WorkerClient::new(o);
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
let Got::Json(hello) = &w.got()[0] else { panic!("hello first") };
|
||||
assert_eq!(hello["memory"], json!({"others": false, "recent_records": 5}));
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn events_pass_on_and_only_tts_audio_of_open_requests() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
assert!(c.tts("r1", "안녕하세요.", false));
|
||||
until(|| w.got().iter().any(|g| matches!(g, Got::Json(v) if v["type"] == "tts"))).await;
|
||||
w.send(tts_frame("zz", 0)); // not ours
|
||||
w.send(tts_frame("r1", 0));
|
||||
w.send(Message::text(json!({"type": "final", "speaker": "1", "utt": 1, "text": "네", "end_reason": "silence",
|
||||
"audio_ms": 900, "wait_ms": 100, "stt_ms": 80}).to_string()));
|
||||
w.send(Message::text(json!({"type": "tts_done", "id": "r1", "chunks": 1, "cancelled": false, "failed": false}).to_string()));
|
||||
let mut seen = vec![];
|
||||
while seen.len() < 4 {
|
||||
let e = next_matching(&mut rx, |_| true).await;
|
||||
if !matches!(e, WorkerEvent::Ready(_)) {
|
||||
seen.push(e);
|
||||
}
|
||||
if matches!(seen.last(), Some(WorkerEvent::TtsDone(_))) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(matches!(&seen[0], WorkerEvent::TtsAudio { id, seq: 0, .. } if id == "r1"));
|
||||
assert!(matches!(&seen[1], WorkerEvent::Final(f) if f.text == "네" && f.end_reason == "silence"));
|
||||
assert_eq!(seen[2], WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 1, cancelled: false, failed: false }));
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_refused_tts_request_ends_as_failed() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
c.tts("r9", "x", false);
|
||||
until(|| w.got().len() >= 2).await;
|
||||
w.send(Message::text(json!({"type": "error", "code": "tts_unavailable", "message": "no", "id": "r9"}).to_string()));
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
||||
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r9".into(), chunks: 0, cancelled: false, failed: true }));
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_worker_error_reaches_the_log_hub_without_debug_logging() {
|
||||
assert_ne!(std::env::var("JARVIS_DEBUG").as_deref(), Ok("1"), "run without debug logging");
|
||||
let hub = jarvis_app::logs::LogHub::new(None);
|
||||
jarvis_app::logs::install(hub.clone());
|
||||
let hub = jarvis_app::logs::installed().unwrap();
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
w.send(Message::text(json!({"type": "error", "code": "tts_failed", "message": "engine broke", "id": "r7"}).to_string()));
|
||||
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Error(_))).await;
|
||||
let errors = hub.query(Some("worker"), jarvis_app::logs::Level::Error, Some("tts_failed"), None, 10);
|
||||
assert_eq!(errors.len(), 1, "{errors:?}");
|
||||
assert!(errors[0].text.contains("r7"));
|
||||
assert!(hub.recent_error_report("test").unwrap().contains("tts_failed"));
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconnects_after_a_restart_and_fails_the_tts_in_flight() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({"language": "ko"}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
c.tts("r1", "길게 말하는 중", false);
|
||||
c.configure(json!({"partials": false}).as_object().unwrap().clone());
|
||||
until(|| w.got().len() >= 3).await;
|
||||
assert_eq!(w.got()[2], Got::Json(json!({"type": "config", "partials": false})));
|
||||
let port = w.kill();
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
||||
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
|
||||
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
||||
assert!(!c.is_ready());
|
||||
c.send_audio("1", &[0, 0]); // dropped, not queued
|
||||
let w = FakeWorker::start(port).await;
|
||||
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ready(_))).await;
|
||||
let got = w.got();
|
||||
// The new hello carries the settings changed since, and no stale audio was queued.
|
||||
assert!(matches!(&got[0], Got::Json(v) if v["stt"] == json!({"language": "ko", "partials": false})));
|
||||
assert!(!got.iter().any(|g| matches!(g, Got::Bin(_))));
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tts_while_disconnected_fails_at_once() {
|
||||
let (c, mut rx) = client("ws://127.0.0.1:1", json!({}));
|
||||
assert!(!c.tts("r1", "x", false));
|
||||
let e = next_matching(&mut rx, |_| true).await;
|
||||
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn history_messages_follow_the_worker_protocol() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, _rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
c.voice_join(42);
|
||||
c.member_join(7, "Alice");
|
||||
c.member_leave(7);
|
||||
c.chat_message(9, 100, 7, "Alice", "hi there", 1_800_000_000.5, None);
|
||||
c.chat_message(9, 101, 1, "Jarvis", "hello", 1_800_000_001.0, Some(7));
|
||||
c.chat_edit(9, 100, "hi, edited");
|
||||
c.chat_delete(9, 100);
|
||||
c.voice_leave();
|
||||
until(|| w.got().len() >= 9).await;
|
||||
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
|
||||
assert_eq!(got, vec![
|
||||
json!({"type": "voice_join", "channel": "42"}),
|
||||
json!({"type": "member_join", "user": "7", "name": "Alice"}),
|
||||
json!({"type": "member_leave", "user": "7"}),
|
||||
json!({"type": "chat_message", "channel": "9", "message_id": "100", "author": "7", "name": "Alice",
|
||||
"text": "hi there", "ts": 1_800_000_000.5}),
|
||||
json!({"type": "chat_message", "channel": "9", "message_id": "101", "author": "1", "name": "Jarvis",
|
||||
"text": "hello", "ts": 1_800_000_001.0, "to": "7"}),
|
||||
json!({"type": "chat_edit", "channel": "9", "message_id": "100", "text": "hi, edited"}),
|
||||
json!({"type": "chat_delete", "channel": "9", "message_id": "100"}),
|
||||
json!({"type": "voice_leave"}),
|
||||
]);
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_channels_go_out_and_chat_actions_come_back_with_their_results() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
c.chat_channels(Some(900), &[ChatChannelSettings { id: 5, name: "일정".into(), description: "약속".into(),
|
||||
read: true, send: true, edit: true, react: true, delete: false, pin: false, answer_all: false, sync_discord: true }]);
|
||||
w.send(Message::text(json!({"type": "chat_action", "id": "a1", "action": "react", "channel": "5",
|
||||
"message_id": "77", "emoji": "👍", "user": "42"}).to_string()));
|
||||
let ev = next_matching(&mut rx, |e| matches!(e, WorkerEvent::ChatAction(_))).await;
|
||||
assert_eq!(ev, WorkerEvent::ChatAction(ChatAction { id: "a1".into(), action: "react".into(), channel: "5".into(),
|
||||
message_id: Some("77".into()), text: None, emoji: Some("👍".into()), user: Some("42".into()) }));
|
||||
c.chat_action_result("a1", Ok(vec![77]));
|
||||
c.chat_action_result("a2", Err("send is turned off".into()));
|
||||
until(|| w.got().len() >= 4).await;
|
||||
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
|
||||
assert_eq!(got, vec![
|
||||
json!({"type": "chat_channels", "bot": "900", "channels": [{"id": "5", "name": "일정", "description": "약속",
|
||||
"read": true, "send": true, "edit": true, "react": true, "delete": false, "pin": false}]}),
|
||||
json!({"type": "chat_action_result", "id": "a1", "ok": true, "message_ids": ["77"]}),
|
||||
json!({"type": "chat_action_result", "id": "a2", "ok": false, "error": "send is turned off"}),
|
||||
]);
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_spoken_reply_takes_its_audio_and_a_failed_one_ends_its_speech() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
assert!(c.reply("r1", 42, "자비스 날씨 어때", "voice", true));
|
||||
assert!(c.reply("r2", 43, "자비스 안녕", "voice", true));
|
||||
until(|| w.got().len() >= 3).await;
|
||||
assert_eq!(w.got()[1], Got::Json(json!({"type": "reply", "id": "r1", "user": "42", "text": "자비스 날씨 어때",
|
||||
"via": "voice", "speak": true})));
|
||||
// The brain's answer is spoken under the reply id, so its audio reaches the app.
|
||||
w.send(Message::text(json!({"type": "reply_text", "id": "r1", "text": "맑아요.", "failed": false}).to_string()));
|
||||
w.send(tts_frame("r1", 0));
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::ReplyText { .. })).await;
|
||||
assert_eq!(e, WorkerEvent::ReplyText { id: "r1".into(), text: "맑아요.".into(), failed: false });
|
||||
next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsAudio { id, .. } if id == "r1")).await;
|
||||
// No answer: nothing will be spoken, so its speech ends as failed.
|
||||
w.send(Message::text(json!({"type": "reply_text", "id": "r2", "text": "", "failed": true}).to_string()));
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
||||
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r2".into(), chunks: 0, cancelled: false, failed: true }));
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_reply_the_worker_cannot_answer_or_send_ends_as_failed() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
c.reply("r3", 42, "x", "voice", true);
|
||||
w.send(Message::text(json!({"type": "error", "code": "brain_unavailable", "message": "no", "id": "r3"}).to_string()));
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
||||
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r3".into(), chunks: 0, cancelled: false, failed: true }));
|
||||
c.close();
|
||||
|
||||
let (c, mut rx) = client("ws://127.0.0.1:1", json!({}));
|
||||
assert!(!c.reply("r4", 42, "x", "voice", true));
|
||||
let e = next_matching(&mut rx, |_| true).await;
|
||||
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r4".into(), chunks: 0, cancelled: false, failed: true }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn addressed_and_rank_questions_go_out_and_their_answers_come_back() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
assert!(c.addressed("q1", 42, "불 좀 꺼 줄래", 3, 1));
|
||||
assert!(c.rank("k2", &[("7".into(), 42, "날씨".into()), ("8".into(), 43, "불 꺼".into())]));
|
||||
until(|| w.got().len() >= 3).await;
|
||||
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
|
||||
assert_eq!(got, vec![
|
||||
json!({"type": "addressed", "id": "q1", "user": "42", "text": "불 좀 꺼 줄래", "people": 3, "callers": 1}),
|
||||
json!({"type": "rank", "id": "k2", "candidates": [{"id": "7", "user": "42", "text": "날씨"},
|
||||
{"id": "8", "user": "43", "text": "불 꺼"}]}),
|
||||
]);
|
||||
w.send(Message::text(json!({"type": "addressed_result", "id": "q1", "addressed": true, "failed": false}).to_string()));
|
||||
w.send(Message::text(json!({"type": "rank_result", "id": "k2", "order": ["8", "7"], "failed": false}).to_string()));
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Addressed { .. })).await;
|
||||
assert_eq!(e, WorkerEvent::Addressed { id: "q1".into(), addressed: true });
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ranked { .. })).await;
|
||||
assert_eq!(e, WorkerEvent::Ranked { id: "k2".into(), order: vec!["8".into(), "7".into()] });
|
||||
// A failed judgement is a "no", and a failed ranking has no order.
|
||||
w.send(Message::text(json!({"type": "addressed_result", "id": "q3", "addressed": true, "failed": true}).to_string()));
|
||||
w.send(Message::text(json!({"type": "rank_result", "id": "k4", "order": ["8"], "failed": true}).to_string()));
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Addressed { .. })).await;
|
||||
assert_eq!(e, WorkerEvent::Addressed { id: "q3".into(), addressed: false });
|
||||
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ranked { .. })).await;
|
||||
assert_eq!(e, WorkerEvent::Ranked { id: "k4".into(), order: vec![] });
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_chat_reply_is_not_spoken_and_its_text_comes_back() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
||||
assert!(c.reply("c1", 42, "자비스 일정", "chat", false));
|
||||
until(|| w.got().len() >= 2).await;
|
||||
assert_eq!(w.got()[1], Got::Json(json!({"type": "reply", "id": "c1", "user": "42", "text": "자비스 일정",
|
||||
"via": "chat", "speak": false})));
|
||||
w.send(tts_frame("c1", 0)); // not a speech the app asked for
|
||||
w.send(Message::text(json!({"type": "reply_text", "id": "c1", "text": "", "failed": true}).to_string()));
|
||||
let e = next_matching(&mut rx, |e| !matches!(e, WorkerEvent::Ready(_))).await;
|
||||
assert_eq!(e, WorkerEvent::ReplyText { id: "c1".into(), text: String::new(), failed: true });
|
||||
// Whether the answer was posted goes back, so a question in it counts only once it was seen.
|
||||
c.reply_delivered("c1", true);
|
||||
c.reply_delivered("c2", false);
|
||||
until(|| w.got().len() >= 4).await;
|
||||
assert_eq!(w.got()[2..], [Got::Json(json!({"type": "reply_delivered", "id": "c1", "ok": true})),
|
||||
Got::Json(json!({"type": "reply_delivered", "id": "c2", "ok": false}))]);
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_llm_switch_is_announced_and_the_client_waits_until_the_brain_is_idle_for_that_switch() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, _rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
let first = c.llm_switching(Some("잠깐만요"));
|
||||
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "switching", "id": first, "notice": "잠깐만요"})))).await;
|
||||
// Nothing idle yet: the wait gives up after its time.
|
||||
assert!(!c.wait_llm_idle(first, Duration::from_millis(100)).await);
|
||||
// The idle for this switch comes late, after the app gave up on it, and a new switch starts:
|
||||
// that late idle must not let the new switch through.
|
||||
let second = c.llm_switching(None);
|
||||
w.send(Message::text(json!({"type": "llm_idle", "id": first}).to_string()));
|
||||
assert!(!c.wait_llm_idle(second, Duration::from_millis(200)).await, "a late idle of an earlier switch");
|
||||
w.send(Message::text(json!({"type": "llm_idle", "id": second}).to_string()));
|
||||
assert!(c.wait_llm_idle(second, Duration::from_secs(2)).await);
|
||||
assert!(c.llm_state("ready"));
|
||||
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready"})))).await;
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_worker_that_missed_the_llm_state_gets_it_when_it_connects_again() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
c.llm_switching(None);
|
||||
// The worker goes away; the app sets the LLM paused meanwhile (that message is lost).
|
||||
let port = w.kill();
|
||||
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
||||
assert!(!c.llm_state("paused"), "not sent: no connection");
|
||||
let w2 = FakeWorker::start(port).await;
|
||||
until(|| w2.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "paused"})))).await;
|
||||
let got = w2.got();
|
||||
assert!(matches!(&got[0], Got::Json(v) if v["type"] == "hello"));
|
||||
assert_eq!(got[1], Got::Json(json!({"type": "llm", "state": "paused"})), "first after the hello");
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_llm_state_set_while_the_worker_connects_again_is_the_last_it_gets() {
|
||||
// The app sets "ready" over and over while a new worker connects: whatever the order, the
|
||||
// last LLM message the worker gets is "ready", never the earlier "switching" alone.
|
||||
for round in 0..20 {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
c.llm_switching(None);
|
||||
let port = w.kill();
|
||||
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
||||
let w2 = FakeWorker::start(port).await;
|
||||
// "ready" is set from the moment the new worker has the hello until the client is ready.
|
||||
let received = w2.received.clone();
|
||||
let c2 = c.clone();
|
||||
let setter = tokio::task::spawn_blocking(move || {
|
||||
while received.lock().unwrap().is_empty() {
|
||||
std::thread::yield_now();
|
||||
}
|
||||
while !c2.is_ready() {
|
||||
c2.llm_state("ready");
|
||||
}
|
||||
c2.llm_state("ready");
|
||||
});
|
||||
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ready(_))).await;
|
||||
setter.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let last = w2.got().into_iter().rev().find_map(|g| match g {
|
||||
Got::Json(v) if v["type"] == "llm" => Some(v["state"].clone()),
|
||||
_ => None,
|
||||
});
|
||||
assert_eq!(last, Some(json!("ready")), "round {round}");
|
||||
c.close();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_tells_the_worker_whether_a_game_runs() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, _rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
assert!(c.llm_ready(true));
|
||||
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready", "game": true})))).await;
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_speech_speed_goes_out_at_once_and_again_on_every_connection() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
assert!(c.set_tts_speed(1.8), "sent while connected");
|
||||
until(|| w.got().contains(&Got::Json(json!({"type": "tts_speed", "speed": 1.8})))).await;
|
||||
// A restarted worker starts with the speed of its arguments; the latest one set catches it up.
|
||||
let port = w.kill();
|
||||
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
||||
assert!(!c.set_tts_speed(1.2), "not sent: no connection");
|
||||
let w2 = FakeWorker::start(port).await;
|
||||
until(|| w2.got().iter().any(|g| matches!(g, Got::Json(v) if v["type"] == "tts_speed"))).await;
|
||||
let speeds: Vec<_> = w2.got().into_iter().filter(|g| matches!(g, Got::Json(v) if v["type"] == "tts_speed")).collect();
|
||||
assert_eq!(speeds, [Got::Json(json!({"type": "tts_speed", "speed": 1.2}))]);
|
||||
c.close();
|
||||
}
|
||||
200
app/tests/worker_process.rs
Normal file
200
app/tests/worker_process.rs
Normal file
@@ -0,0 +1,200 @@
|
||||
//! The worker process with a stand-in `voice_worker` package run by a real Python, so the spawn line
|
||||
//! is the one the app uses. JARVIS_TEST_PYTHON overrides the Python (default python3 / python).
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use jarvis_app::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
|
||||
use serde_json::Value;
|
||||
|
||||
const FAKE_MAIN: &str = r#"
|
||||
import json, os, sys, time
|
||||
out = os.environ["FAKE_OUT"]
|
||||
runs = len(os.listdir(out))
|
||||
with open(os.path.join(out, f"run{runs}.json"), "w") as f:
|
||||
json.dump({"argv": sys.argv[1:], "token": os.environ.get("JARVIS_VOICE_TOKEN")}, f)
|
||||
if os.environ.get("FAKE_CRASH_FIRST") == "1" and runs == 0:
|
||||
sys.exit(3)
|
||||
print("loading models...", flush=True)
|
||||
print("READY 127.0.0.1:" + sys.argv[sys.argv.index("--port") + 1], flush=True)
|
||||
if os.environ.get("FAKE_LAST_LINES") == "1":
|
||||
for i in range(3):
|
||||
print(f"line {i}", file=sys.stderr, flush=True)
|
||||
print("LAST bad key old-api-key-1", file=sys.stderr, flush=True)
|
||||
time.sleep(60)
|
||||
"#;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("JARVIS_TEST_PYTHON").unwrap_or_else(|_| if cfg!(windows) { "python".into() } else { "python3".into() })
|
||||
}
|
||||
|
||||
struct Fake {
|
||||
root: PathBuf,
|
||||
out: PathBuf,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
fn new(tag: &str) -> Self {
|
||||
let root = std::env::temp_dir().join(format!("jarvis-fake-worker-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
std::fs::create_dir_all(root.join("voice_worker")).unwrap();
|
||||
std::fs::write(root.join("voice_worker/__init__.py"), "").unwrap();
|
||||
std::fs::write(root.join("voice_worker/__main__.py"), FAKE_MAIN).unwrap();
|
||||
let out = root.join("out");
|
||||
std::fs::create_dir_all(&out).unwrap();
|
||||
Self { root, out }
|
||||
}
|
||||
fn run(&self, n: usize) -> Value {
|
||||
serde_json::from_str(&std::fs::read_to_string(self.out.join(format!("run{n}.json"))).unwrap()).unwrap()
|
||||
}
|
||||
fn opts(&self, crash_first: bool) -> WorkerProcessOptions {
|
||||
let mut o = WorkerProcessOptions::new(python(), &self.root, vec!["--melo-python".into(), "/x/python".into()]);
|
||||
o.env.insert("FAKE_OUT".into(), self.out.display().to_string());
|
||||
if crash_first {
|
||||
o.env.insert("FAKE_CRASH_FIRST".into(), "1".into());
|
||||
}
|
||||
o.restart_delay = Duration::from_millis(50);
|
||||
o
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fake {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.root);
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for(rx: &mut tokio::sync::broadcast::Receiver<ProcessEvent>, f: impl Fn(&ProcessEvent) -> bool) -> ProcessEvent {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let e = rx.recv().await.unwrap();
|
||||
if f(&e) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("timed out")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runs_the_worker_on_loopback_with_the_token_only_in_the_environment() {
|
||||
let fake = Fake::new("env");
|
||||
let p = WorkerProcess::new(fake.opts(false)).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
let run = fake.run(0);
|
||||
let port = p.port().to_string();
|
||||
let argv: Vec<String> = serde_json::from_value(run["argv"].clone()).unwrap();
|
||||
assert_eq!(argv, ["--host", "127.0.0.1", "--port", &port, "--melo-python", "/x/python"]);
|
||||
assert_eq!(run["token"], p.token.as_str());
|
||||
assert!(p.token.len() >= 40);
|
||||
assert!(!argv.join(" ").contains(&p.token));
|
||||
assert!(p.url().starts_with("ws://127.0.0.1:"));
|
||||
p.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_dead_worker_restarts_on_the_same_port_with_the_same_token() {
|
||||
let fake = Fake::new("restart");
|
||||
let p = WorkerProcess::new(fake.opts(true)).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
assert_eq!(wait_for(&mut rx, |e| matches!(e, ProcessEvent::Exited { .. })).await, ProcessEvent::Exited { code: Some(3) });
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
assert_eq!(p.restarts(), 1);
|
||||
assert_eq!(fake.run(1)["argv"], fake.run(0)["argv"]);
|
||||
assert_eq!(fake.run(1)["token"], fake.run(0)["token"]);
|
||||
p.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_ends_the_worker_for_good() {
|
||||
let fake = Fake::new("stop");
|
||||
let p = WorkerProcess::new(fake.opts(false)).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
p.stop().await;
|
||||
assert!(matches!(rx.recv().await.unwrap(), ProcessEvent::Exited { .. }));
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
assert!(rx.try_recv().is_err(), "no restart after stop");
|
||||
assert_eq!(p.restarts(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn stop_returns_only_after_the_last_output_is_logged() {
|
||||
// A slow log (each line takes 300 ms) and a worker whose last lines are still unread when it is
|
||||
// stopped: they must be handled before stop returns, so the next start's secrets (logs.rs,
|
||||
// set_secrets_in_use) never apply to them.
|
||||
let fake = Fake::new("drain");
|
||||
let mut o = fake.opts(false);
|
||||
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
|
||||
let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
|
||||
let s2 = seen.clone();
|
||||
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
s2.lock().unwrap().push(l.to_owned());
|
||||
})));
|
||||
let p = WorkerProcess::new(o).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while seen.lock().unwrap().is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}).await.expect("no output");
|
||||
p.stop().await;
|
||||
let lines = seen.lock().unwrap().clone();
|
||||
assert!(lines.iter().any(|l| l.contains("LAST")), "{lines:?}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn a_last_line_logged_after_the_stop_gave_up_is_masked_with_its_runs_secrets() {
|
||||
// The services started with the old key; the log of the last line is held up past the stop's
|
||||
// wait, and the next start replaces the values in use before that line is written.
|
||||
let hub = jarvis_app::logs::LogHub::new(None);
|
||||
jarvis_app::logs::install(hub.clone());
|
||||
hub.set_secrets_in_use(vec!["old-api-key-1".into()]);
|
||||
let fake = Fake::new("late");
|
||||
let mut o = fake.opts(false);
|
||||
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
|
||||
let run = jarvis_app::worker_process::run_sink();
|
||||
let (started, done) = (std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)));
|
||||
let (s2, d2) = (started.clone(), done.clone());
|
||||
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
|
||||
if l.contains("LAST") {
|
||||
s2.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
std::thread::sleep(Duration::from_secs(7));
|
||||
(run.0)(l);
|
||||
d2.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
} else {
|
||||
(run.0)(l);
|
||||
}
|
||||
})));
|
||||
let p = WorkerProcess::new(o).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !started.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}).await.expect("no last line");
|
||||
p.stop().await;
|
||||
// The next start: only the new key is in use (and stored).
|
||||
hub.set_secrets(vec!["new-api-key-2".into()]);
|
||||
hub.set_secrets_in_use(vec!["new-api-key-2".into()]);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !done.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}).await.expect("the last line was never logged");
|
||||
let lines: Vec<String> = hub.query(None, jarvis_app::logs::Level::Debug, None, None, 100)
|
||||
.into_iter().map(|l| l.text).collect();
|
||||
let last = lines.iter().find(|l| l.contains("LAST")).expect("logged");
|
||||
assert!(last.contains("[비밀값]") && !last.contains("old-api-key-1"), "{lines:?}");
|
||||
}
|
||||
250
app/tests/worker_protocol.rs
Normal file
250
app/tests/worker_protocol.rs
Normal file
@@ -0,0 +1,250 @@
|
||||
//! The worker client and playback against the real Python voice worker server with fake engines, so
|
||||
//! both sides of the protocol are checked together. Needs a Python with the worker's requirements:
|
||||
//! JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use jarvis_app::playback::{ClipSink, Playback, PlaybackEvent};
|
||||
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions, WorkerEvent};
|
||||
use jarvis_app::worker_process::free_port;
|
||||
use serde_json::json;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
|
||||
const TOKEN: &str = "protocol-test-token";
|
||||
|
||||
/// 20 ms of a 48 kHz stereo tone whose amplitude and pitch (x1000 Hz) are `level`.
|
||||
fn frame(level: f64) -> Vec<i16> {
|
||||
(0..960)
|
||||
.flat_map(|i| {
|
||||
let v = (level * (2.0 * std::f64::consts::PI * level * 1000.0 * i as f64 / 48000.0).sin() * 32767.0) as i16;
|
||||
[v, v]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Sink(Vec<usize>);
|
||||
impl ClipSink for Sink {
|
||||
fn play(&mut self, pcm: Vec<u8>) {
|
||||
self.0.push(pcm.len());
|
||||
}
|
||||
fn stop(&mut self) {}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn audio_and_tts_round_trip_through_the_python_server() {
|
||||
let Ok(python) = std::env::var("JARVIS_VOICE_PYTHON") else {
|
||||
eprintln!("skipped: set JARVIS_VOICE_PYTHON");
|
||||
return;
|
||||
};
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let port = free_port().unwrap();
|
||||
let mut child = tokio::process::Command::new(python)
|
||||
.arg(root.join("tests/fixtures/fake_engine_worker.py"))
|
||||
.arg(port.to_string())
|
||||
.env("PYTHONPATH", root.join("../src"))
|
||||
.env("JARVIS_VOICE_TOKEN", TOKEN)
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let mut lines = BufReader::new(child.stdout.take().unwrap()).lines();
|
||||
while let Some(l) = lines.next_line().await.unwrap() {
|
||||
if l.starts_with("READY") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut o = WorkerClientOptions::new(format!("ws://127.0.0.1:{port}"), TOKEN);
|
||||
o.stt = json!({"partials": false, "hotwords": ["자비스"]}).as_object().unwrap().clone();
|
||||
let (c, mut rx) = WorkerClient::new(o);
|
||||
c.start();
|
||||
let ok = c.wait_ready(Duration::from_secs(5)).await.expect("ready");
|
||||
assert!(ok.stt_ready && ok.tts_ready);
|
||||
assert_eq!(ok.stt["hotwords"], json!(["자비스"]));
|
||||
|
||||
// Two Discord-format speakers; the second leaves mid-utterance and still gets its final.
|
||||
let (a, b) = ("111111111111111111", "222222222222222222");
|
||||
for _ in 0..30 {
|
||||
c.send_audio(a, &frame(0.9));
|
||||
c.send_audio(b, &frame(0.6));
|
||||
}
|
||||
c.speaking_end(a);
|
||||
c.speaker_leave(b);
|
||||
let mut finals = std::collections::HashMap::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while finals.len() < 2 {
|
||||
if let Some(WorkerEvent::Final(f)) = rx.recv().await {
|
||||
finals.insert(f.speaker.clone(), f);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("finals");
|
||||
assert_eq!((finals[a].text.as_str(), finals[a].end_reason.as_str(), finals[a].utt), ("level 0.9", "speaking_end", 1));
|
||||
assert_eq!((finals[b].text.as_str(), finals[b].end_reason.as_str()), ("level 0.6", "speaking_end"));
|
||||
|
||||
// TTS sentences play in order through Playback; 10 samples per character, stereo, 2 bytes each.
|
||||
let mut p = Playback::new(c.clone(), Sink::default());
|
||||
p.say("첫 번째 문장입니다. 두 번째 문장은 조금 더 깁니다.", false);
|
||||
let end = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match rx.recv().await.unwrap() {
|
||||
WorkerEvent::TtsAudio { id, seq, pcm } => {
|
||||
p.audio(&id, seq, pcm);
|
||||
p.clip_ended();
|
||||
}
|
||||
WorkerEvent::TtsDone(d) => p.done(&d),
|
||||
_ => {}
|
||||
}
|
||||
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("speech end");
|
||||
assert_eq!((end.played, end.cancelled, end.failed), (2, false, false));
|
||||
|
||||
// A spoken brain answer plays under the adopted id, like a TTS request.
|
||||
let id = p.adopt();
|
||||
assert!(c.reply(&id, 42, "자비스 안녕", "voice", true));
|
||||
let end = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match rx.recv().await.unwrap() {
|
||||
WorkerEvent::TtsAudio { id, seq, pcm } => {
|
||||
p.audio(&id, seq, pcm);
|
||||
p.clip_ended();
|
||||
}
|
||||
WorkerEvent::TtsDone(d) => p.done(&d),
|
||||
_ => {}
|
||||
}
|
||||
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
|
||||
return e;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("answer end");
|
||||
assert_eq!((end.id.as_str(), end.played, end.failed), (id.as_str(), 2, false));
|
||||
|
||||
// The two judgements come back under their ids.
|
||||
assert!(c.addressed("q1", 42, "불 켜 줘", 2, 2));
|
||||
assert!(c.rank("k1", &[("1".into(), 42, "a".into()), ("2".into(), 43, "b".into())]));
|
||||
let (mut addressed, mut ranked) = (None, None);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while addressed.is_none() || ranked.is_none() {
|
||||
match rx.recv().await.unwrap() {
|
||||
WorkerEvent::Addressed { id, addressed: a } => addressed = Some((id, a)),
|
||||
WorkerEvent::Ranked { id, order } => ranked = Some((id, order)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("judgements");
|
||||
assert_eq!(addressed, Some(("q1".to_string(), true)));
|
||||
assert_eq!(ranked, Some(("k1".to_string(), vec!["2".to_string(), "1".to_string()])));
|
||||
|
||||
// The brain's schedule tool asks the app for the person and the channel they asked in; the
|
||||
// app's store answers; a scheduled task comes back as text.
|
||||
let data = std::env::temp_dir().join(format!("jarvis-proto-schedule-{}", std::process::id()));
|
||||
let store = jarvis_app::schedule::Store::new(&data);
|
||||
assert!(c.reply_in("c1", 42, "예약해 줘", "chat", false, Some(77), &[]));
|
||||
let text = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match rx.recv().await.unwrap() {
|
||||
WorkerEvent::ScheduleAction(a) => {
|
||||
assert_eq!((a.action.as_str(), a.user.as_str(), a.via.as_str(), a.channel.as_deref()),
|
||||
("add", "42", "chat", Some("77")));
|
||||
c.schedule_action_result(&a.id, jarvis_app::schedule::act(&store, &a, "가", 1000));
|
||||
}
|
||||
WorkerEvent::ReplyText { id, text, .. } if id == "c1" => return text,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("schedule round trip");
|
||||
assert_eq!(text, "scheduled 1");
|
||||
let kept = store.all().unwrap();
|
||||
assert_eq!((kept[0].user, kept[0].channel, kept[0].repeat.as_str()), (42, Some(77), "daily"));
|
||||
assert!(c.reply_in("t1", 42, "날씨", "task", false, None, &["섭씨로".to_string()]));
|
||||
let task = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let WorkerEvent::ReplyText { id, text, failed } = rx.recv().await.unwrap() {
|
||||
if id == "t1" {
|
||||
return (text, failed);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("task answer");
|
||||
assert_eq!(task, ("task: 날씨 notes=섭씨로".to_string(), false));
|
||||
// The personalNotes tool's actions come to the app's notes store for the asker.
|
||||
let notes = jarvis_app::notes::Notes::new(&data);
|
||||
assert!(c.reply("n1", 42, "지침 저장해 줘", "voice", false));
|
||||
let noted = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match rx.recv().await.unwrap() {
|
||||
WorkerEvent::NoteAction(a) => {
|
||||
assert_eq!((a.action.as_str(), a.user.as_str()), ("add", "42"));
|
||||
c.note_action_result(&a.id, jarvis_app::notes::act(¬es, &a, "가", 1000));
|
||||
}
|
||||
WorkerEvent::ReplyText { id, text, .. } if id == "n1" => return text,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("note round trip");
|
||||
assert_eq!(noted, "noted 1");
|
||||
assert_eq!(notes.texts_of(42), vec!["반말로 해".to_string()]);
|
||||
// The screenView tool's captures come to the app, which decides for the asker and sends the
|
||||
// pictures back (a large message: a 4K monitor at full detail would be about 1 MB).
|
||||
struct Grey;
|
||||
impl jarvis_app::screen::Screen for Grey {
|
||||
fn monitors(&self) -> Result<Vec<jarvis_app::screen::Monitor>, String> {
|
||||
Ok(vec![jarvis_app::screen::Monitor { number: 2, rect: jarvis_app::screen::Rect { x: 0, y: 0, w: 3840, h: 2160 },
|
||||
main: true, handle: 0 }])
|
||||
}
|
||||
fn windows(&self) -> Result<Vec<jarvis_app::screen::Window>, String> {
|
||||
Ok(vec![])
|
||||
}
|
||||
fn capture_monitor(&self, m: &jarvis_app::screen::Monitor) -> Result<jarvis_app::screen::Image, String> {
|
||||
let (w, h) = (m.rect.w as u32, m.rect.h as u32);
|
||||
Ok(jarvis_app::screen::Image { w, h, bgra: (0..w * h * 4).map(|i| (i % 251) as u8 | 0x40).collect() })
|
||||
}
|
||||
fn capture_window(&self, _: &jarvis_app::screen::Window) -> Result<jarvis_app::screen::Image, String> {
|
||||
Err("no windows".into())
|
||||
}
|
||||
}
|
||||
let mut screen_settings = jarvis_app::settings::Settings::default();
|
||||
screen_settings.pc_control.allowed_users = vec![42];
|
||||
for (rid, user, want) in [("v1", 42u64, "saw "), ("v2", 7, "등록된 사람만")] {
|
||||
assert!(c.reply(rid, user, "화면 봐 줘", "voice", false));
|
||||
let seen = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
match rx.recv().await.unwrap() {
|
||||
WorkerEvent::ScreenAction(a) => {
|
||||
assert_eq!((a.action.as_str(), a.target.as_deref(), a.monitor, a.detail.as_deref()),
|
||||
("capture", Some("monitor"), Some(2), Some("fast")));
|
||||
c.screen_action_result(&a.id, jarvis_app::screen::act(&screen_settings, &Grey, &a));
|
||||
}
|
||||
WorkerEvent::ReplyText { id, text, .. } if id == rid => return text,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("screen round trip");
|
||||
assert!(seen.contains(want), "{seen}");
|
||||
if user == 42 {
|
||||
assert!(seen.ends_with("모니터 2") && seen.split(' ').nth(1).unwrap().parse::<usize>().unwrap() > 1000, "{seen}");
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&data);
|
||||
c.close();
|
||||
let _ = child.kill().await;
|
||||
}
|
||||
1204
app/ui/app.js
Normal file
1204
app/ui/app.js
Normal file
File diff suppressed because it is too large
Load Diff
36
app/ui/index.html
Normal file
36
app/ui/index.html
Normal file
@@ -0,0 +1,36 @@
|
||||
<!doctype html>
|
||||
<html lang="ko">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Jarvis 관리</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="wizard" class="wizard" hidden></div>
|
||||
<div id="main" class="layout" hidden>
|
||||
<nav class="side">
|
||||
<div class="brand">Jarvis</div>
|
||||
<button data-screen="dashboard" class="nav">대시보드</button>
|
||||
<button data-screen="conversations" class="nav">대화</button>
|
||||
<button data-screen="bot" class="nav">봇</button>
|
||||
<button data-screen="channels" class="nav">채널·호출</button>
|
||||
<button data-screen="chat" class="nav">채팅 채널</button>
|
||||
<button data-screen="memory" class="nav">대화 기억</button>
|
||||
<button data-screen="model" class="nav">모델</button>
|
||||
<button data-screen="features" class="nav">기능</button>
|
||||
<button data-screen="collect" class="nav">백그라운드 수집</button>
|
||||
<button data-screen="schedule" class="nav">예약</button>
|
||||
<button data-screen="pcControl" class="nav">PC 제어</button>
|
||||
<button data-screen="notes" class="nav">사람별 지침</button>
|
||||
<button data-screen="game" class="nav">게임 모드</button>
|
||||
<button data-screen="logs" class="nav">로그</button>
|
||||
<button data-screen="app" class="nav" id="nav-app">앱</button>
|
||||
<div class="side-foot" id="version"></div>
|
||||
</nav>
|
||||
<main id="screen" class="screen"></main>
|
||||
</div>
|
||||
<div id="toast" class="toast" hidden></div>
|
||||
<script type="module" src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
95
app/ui/style.css
Normal file
95
app/ui/style.css
Normal file
@@ -0,0 +1,95 @@
|
||||
/* Dark admin theme after DESIGN.md (near-black canvas, charcoal panels, hairline borders, one accent). */
|
||||
:root {
|
||||
--canvas: #08090a; --s1: #0f1011; --s2: #141516; --s3: #1b1c1e;
|
||||
--line: #23252a; --line2: #34343a;
|
||||
--ink: #f7f8f8; --ink2: #d0d6e0; --ink3: #8a8f98; --ink4: #62666d;
|
||||
--accent: #5e6ad2; --accent2: #828fff; --ok: #27a644; --warn: #d7a03a; --bad: #e5484d;
|
||||
font-family: "Pretendard", "Malgun Gothic", "Segoe UI", system-ui, sans-serif;
|
||||
color: var(--ink); background: var(--canvas); font-size: 14px;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
[hidden] { display: none !important; }
|
||||
body { margin: 0; min-height: 100vh; }
|
||||
button, input, select, textarea { font: inherit; color: inherit; }
|
||||
.layout { display: grid; grid-template-columns: 200px 1fr; min-height: 100vh; }
|
||||
.side { background: var(--s1); border-right: 1px solid var(--line); padding: 16px 10px; display: flex; flex-direction: column; gap: 2px; }
|
||||
.brand { font-weight: 600; font-size: 18px; letter-spacing: -0.3px; padding: 6px 10px 16px; }
|
||||
.nav { text-align: left; background: none; border: 0; padding: 8px 10px; border-radius: 6px; color: var(--ink2); cursor: pointer; }
|
||||
.nav:hover { background: var(--s2); }
|
||||
.nav.on { background: var(--s3); color: var(--ink); }
|
||||
.side-foot { margin-top: auto; color: var(--ink4); font-size: 12px; padding: 8px 10px; }
|
||||
.screen { padding: 28px 32px; max-width: 1100px; }
|
||||
h1 { font-size: 24px; font-weight: 600; letter-spacing: -0.4px; margin: 0 0 18px; }
|
||||
h2 { font-size: 15px; font-weight: 600; margin: 0 0 10px; color: var(--ink2); }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 12px; margin-bottom: 12px; }
|
||||
.card { background: var(--s1); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; }
|
||||
.muted { color: var(--ink3); }
|
||||
.small { font-size: 12px; }
|
||||
.row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; }
|
||||
.btn { background: var(--s3); border: 1px solid var(--line2); border-radius: 7px; padding: 7px 14px; cursor: pointer; }
|
||||
.btn:hover { border-color: var(--ink4); }
|
||||
.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
.btn.primary:hover { background: var(--accent2); }
|
||||
.btn:disabled { opacity: .45; cursor: default; }
|
||||
.badge { display: inline-block; border-radius: 999px; padding: 2px 10px; font-size: 12px; border: 1px solid var(--line2); }
|
||||
.badge.ok { color: var(--ok); border-color: #1f5f30; }
|
||||
.badge.warn { color: var(--warn); border-color: #6b5222; }
|
||||
.badge.bad { color: var(--bad); border-color: #6e2a2c; }
|
||||
.kv { display: grid; grid-template-columns: max-content 1fr; gap: 6px 14px; }
|
||||
.kv .k { color: var(--ink3); }
|
||||
.field { display: grid; grid-template-columns: 260px 1fr; gap: 14px; padding: 12px 0; border-bottom: 1px solid var(--line); align-items: start; }
|
||||
.field label { color: var(--ink2); padding-top: 6px; }
|
||||
.field .hint { color: var(--ink4); font-size: 12px; margin-top: 4px; }
|
||||
input[type=text], input[type=number], input[type=password], input[type=date], select, textarea {
|
||||
background: var(--s2); border: 1px solid var(--line2); border-radius: 7px; padding: 7px 10px; width: 100%; max-width: 520px; outline: none; }
|
||||
input:focus, select:focus, textarea:focus { border-color: var(--accent); }
|
||||
textarea { min-height: 70px; resize: vertical; }
|
||||
.switch { appearance: none; width: 38px; height: 22px; border-radius: 999px; background: var(--line2); position: relative; cursor: pointer; margin-top: 4px; }
|
||||
.switch:checked { background: var(--accent); }
|
||||
.switch::after { content: ""; position: absolute; top: 3px; left: 3px; width: 16px; height: 16px; border-radius: 50%; background: #fff; transition: left .12s; }
|
||||
.switch:checked::after { left: 19px; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
|
||||
th { color: var(--ink3); font-weight: 500; font-size: 12px; }
|
||||
.captions { max-height: 260px; overflow: auto; display: flex; flex-direction: column; gap: 4px; }
|
||||
.cap { color: var(--ink2); }
|
||||
.cap.partial { color: var(--ink4); font-style: italic; }
|
||||
.cap b { color: var(--ink); font-weight: 500; margin-right: 6px; }
|
||||
.logs { background: var(--s1); border: 1px solid var(--line); border-radius: 10px; height: 60vh; overflow: auto; font-family: ui-monospace, Consolas, monospace; font-size: 12px; padding: 8px 10px; }
|
||||
.log { white-space: pre-wrap; word-break: break-all; padding: 1px 0; }
|
||||
.log.error { color: var(--bad); } .log.warn { color: var(--warn); } .log.debug { color: var(--ink4); }
|
||||
.savebar { position: sticky; bottom: 0; background: var(--canvas); padding: 14px 0; display: flex; gap: 10px; align-items: center; }
|
||||
.toast { position: fixed; right: 20px; bottom: 20px; background: var(--s3); border: 1px solid var(--line2); border-radius: 8px; padding: 10px 14px; max-width: 420px; }
|
||||
.toast.bad { border-color: var(--bad); }
|
||||
.wizard { max-width: 720px; margin: 48px auto; padding: 0 20px; }
|
||||
.steps { display: flex; gap: 6px; margin-bottom: 22px; flex-wrap: wrap; }
|
||||
.step { font-size: 12px; color: var(--ink4); border: 1px solid var(--line); border-radius: 999px; padding: 3px 10px; }
|
||||
.step.on { color: var(--ink); border-color: var(--accent); }
|
||||
.step.done { color: var(--ok); border-color: #1f5f30; }
|
||||
.wizard .card { padding: 22px 24px; }
|
||||
.wizard p { color: var(--ink2); line-height: 1.6; }
|
||||
.nav-row { display: flex; justify-content: space-between; margin-top: 20px; }
|
||||
a { color: var(--accent2); }
|
||||
.row > select, .row > input[type=date] { width: auto; }
|
||||
input[type=date] { color-scheme: dark; }
|
||||
th, .btn { white-space: nowrap; }
|
||||
.filters { display: flex; flex-direction: column; gap: 10px; }
|
||||
.filters select, .filters input[type=date] { width: auto; }
|
||||
.chip { display: inline-flex; align-items: center; gap: 6px; border: 1px solid var(--line2); border-radius: 999px; padding: 4px 12px; cursor: pointer; color: var(--ink2); }
|
||||
.chip:has(input:checked) { border-color: var(--accent); color: var(--ink); background: var(--s3); }
|
||||
.talk { background: var(--s1); border: 1px solid var(--line); border-radius: 10px; height: 62vh; overflow: auto; padding: 10px 14px; display: flex; flex-direction: column; gap: 8px; }
|
||||
.day { align-self: center; color: var(--ink4); font-size: 12px; border: 1px solid var(--line); border-radius: 999px; padding: 2px 12px; margin: 6px 0; }
|
||||
.msg { border: 1px solid var(--line); border-radius: 10px; padding: 8px 12px; max-width: 85%; background: var(--s2); }
|
||||
.msg.said { align-self: flex-start; }
|
||||
.msg.answered { align-self: flex-end; border-color: #2f3570; background: #12142a; }
|
||||
.msg.thought { align-self: stretch; max-width: none; border-style: dashed; background: var(--canvas); }
|
||||
.msg.thought > summary { cursor: pointer; }
|
||||
.msg .who { font-weight: 600; }
|
||||
.msg .text { white-space: pre-wrap; word-break: break-word; margin-top: 4px; line-height: 1.5; }
|
||||
.steps { display: flex; flex-direction: column; gap: 8px; margin-top: 10px; }
|
||||
.step-row { display: grid; grid-template-columns: 150px 1fr; gap: 12px; border-top: 1px solid var(--line); padding-top: 8px; }
|
||||
.step-row .k { color: var(--ink3); font-size: 12px; padding-top: 2px; }
|
||||
.pre { white-space: pre-wrap; word-break: break-word; font-family: ui-monospace, Consolas, monospace; font-size: 12px; background: var(--s1); border: 1px solid var(--line); border-radius: 6px; padding: 6px 8px; margin: 4px 0; max-height: 320px; overflow: auto; }
|
||||
.plan { margin: 0; padding-left: 18px; }
|
||||
.linkbtn { background: none; border: 0; color: var(--ink4); cursor: pointer; padding: 0 4px; margin-left: auto; }
|
||||
.linkbtn:hover { color: var(--ink2); }
|
||||
42
app/ui/talk.js
Normal file
42
app/ui/talk.js
Normal file
@@ -0,0 +1,42 @@
|
||||
// Paging of the 대화 screen (app/src/admin.spec.md, "Screens"). Pure, so app/tests/ui checks it.
|
||||
//
|
||||
// The screen keeps a live window, every item from the cursor `from` on (re-read on each tick, at
|
||||
// most PAGE_MAX), and before it the `older` pages kept as loaded. Items are ordered like
|
||||
// history_query: time, then records (`h<id>`) before traces (`t<id>`), then id.
|
||||
|
||||
export const PAGE = 200, PAGE_MAX = 2000;
|
||||
|
||||
export const cursorOf = (i) => ({ at: i.at, key: i.key });
|
||||
|
||||
export function compare(a, b) {
|
||||
return a.at - b.at || (a.key[0] === b.key[0] ? 0 : a.key[0] < b.key[0] ? -1 : 1)
|
||||
|| Number(a.key.slice(1)) - Number(b.key.slice(1));
|
||||
}
|
||||
|
||||
export const empty = () => ({ shown: [], older: [], from: null, hasOlder: false });
|
||||
|
||||
/** A live answer: the items from `from` on (the newest page while there is no `from`), and
|
||||
* whether there were more than it holds. */
|
||||
export function withLive(s, items, more) {
|
||||
let { older, hasOlder } = s;
|
||||
if (!s.from) hasOlder = more;
|
||||
else if (more && items.length) {
|
||||
// More new items than the window holds: the shown ones it no longer reaches join the pages
|
||||
// kept as loaded, so nothing the user had on screen disappears.
|
||||
const seen = new Set(older.map((i) => i.key));
|
||||
older = older.concat(s.shown.filter((i) => compare(i, items[0]) < 0 && !seen.has(i.key)));
|
||||
}
|
||||
return { shown: items, older, hasOlder, from: items.length ? cursorOf(items[0]) : s.from };
|
||||
}
|
||||
|
||||
/** The page before the oldest item loaded: it joins the live window while that stays within
|
||||
* PAGE_MAX, else the older pages. Items already loaded are not added again. */
|
||||
export function withOlder(s, page, more) {
|
||||
const seen = new Set(s.older.concat(s.shown).map((i) => i.key));
|
||||
const fresh = page.filter((i) => !seen.has(i.key));
|
||||
if (!s.older.length && s.shown.length + fresh.length <= PAGE_MAX) {
|
||||
const shown = fresh.concat(s.shown);
|
||||
return { ...s, shown, hasOlder: more, from: shown.length ? cursorOf(shown[0]) : s.from };
|
||||
}
|
||||
return { ...s, older: fresh.concat(s.older), hasOlder: more };
|
||||
}
|
||||
@@ -2,10 +2,11 @@
|
||||
// 9222) so the action is visible on the Go-Live broadcast, and prints a JSON
|
||||
// result on stdout for the Python `browseAndSearch` tool to wrap.
|
||||
//
|
||||
// node browse-search.mjs "<query>" [search|youtube]
|
||||
// node browse-search.mjs "<query>" [search|youtube] [index]
|
||||
//
|
||||
// - search : Google-search the query, return the top organic results.
|
||||
// - youtube : search YouTube and play the first result.
|
||||
// - youtube : search YouTube and play a result. `index` is the 1-based position
|
||||
// from the top of the result list (default 1 = first result).
|
||||
//
|
||||
// Backend selection for `search`:
|
||||
// 1. The broadcast Chrome over CDP (visible on the Go-Live stream).
|
||||
@@ -29,6 +30,9 @@ const UA =
|
||||
'(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||
const query = process.argv[2] || '';
|
||||
const mode = (process.argv[3] || 'search').toLowerCase();
|
||||
// 1-based position of the YouTube result to play, counted from the top of the
|
||||
// list. Defaults to 1 (first result). Anything <1 or non-numeric falls back to 1.
|
||||
const playIndex = Math.max(1, parseInt(process.argv[4], 10) || 1);
|
||||
const out = (o) => { process.stdout.write(JSON.stringify(o)); };
|
||||
|
||||
if (!query) { out({ ok: false, error: 'no query' }); process.exit(1); }
|
||||
@@ -105,15 +109,21 @@ try {
|
||||
await page.bringToFront().catch(() => {});
|
||||
|
||||
if (mode === 'youtube') {
|
||||
// Type into YouTube's search box like a person, then play the first result.
|
||||
// Type into YouTube's search box like a person, then play the requested
|
||||
// result (the Nth from the top of the list; default the first).
|
||||
await typeSearch('https://www.youtube.com/?hl=ko', 'input#search, input[name="search_query"]', query);
|
||||
await page.waitForSelector('ytd-video-renderer a#video-title, a#video-title', { timeout: 20000 });
|
||||
const first = page.locator('ytd-video-renderer a#video-title, a#video-title').first();
|
||||
const title = (await first.getAttribute('title').catch(() => '')) || (await first.innerText().catch(() => ''));
|
||||
await first.click();
|
||||
const results = page.locator('ytd-video-renderer a#video-title, a#video-title');
|
||||
// Clamp to what's actually on the page so "play the 5th" still plays the
|
||||
// last available result rather than failing when fewer were returned.
|
||||
const available = await results.count();
|
||||
const targetIdx = Math.min(playIndex, Math.max(available, 1)) - 1;
|
||||
const target = results.nth(targetIdx);
|
||||
const title = (await target.getAttribute('title').catch(() => '')) || (await target.innerText().catch(() => ''));
|
||||
await target.click();
|
||||
await page.waitForSelector('#movie_player', { timeout: 20000 });
|
||||
await page.evaluate(() => { const v = document.querySelector('video'); if (v && v.paused) v.play(); });
|
||||
out({ ok: true, mode, title: (title || '').trim(), url: page.url() });
|
||||
out({ ok: true, mode, index: targetIdx + 1, title: (title || '').trim(), url: page.url() });
|
||||
} else {
|
||||
// Type into Google's search box like a person, then read the results.
|
||||
await typeSearch('https://www.google.com/?hl=ko', 'textarea[name="q"], input[name="q"]', query);
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
// status | listTabs
|
||||
// navigate {url} | back | forward | refresh
|
||||
// newTab {url?} | closeTab {index?} | activateTab {index} | closePopups
|
||||
// moveMouse {selector | site} (hover the real cursor, no click)
|
||||
// click {selector} | type {text, selector?} | scroll {dir, notches?}
|
||||
// pressKey {key} | screenshot {path}
|
||||
import { chromium } from 'playwright';
|
||||
@@ -40,6 +41,15 @@ if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); }
|
||||
|
||||
const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`);
|
||||
|
||||
// Per-site homepage + search-box selector, shared by `search` and `moveMouse`.
|
||||
const SITES = {
|
||||
naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' },
|
||||
google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' },
|
||||
daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' },
|
||||
youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' },
|
||||
bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' },
|
||||
};
|
||||
|
||||
// The genuinely-active tab is the one whose document is visible. Playwright has
|
||||
// no "active page" accessor over CDP, so probe visibilityState (fixes treating
|
||||
// tab 0 as active and breaking sequential ops on a specific tab).
|
||||
@@ -103,13 +113,6 @@ try {
|
||||
const q = String(cmd.query || '').trim();
|
||||
if (!q) throw new Error('search: no query');
|
||||
const siteKey = String(cmd.site || 'google').toLowerCase();
|
||||
const SITES = {
|
||||
naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' },
|
||||
google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' },
|
||||
daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' },
|
||||
youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' },
|
||||
bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' },
|
||||
};
|
||||
const s = SITES[siteKey] || SITES.google;
|
||||
await front(page);
|
||||
// 1) Go to the homepage.
|
||||
@@ -122,23 +125,31 @@ try {
|
||||
// 2) Click the on-page search box, type the query, submit.
|
||||
const box = page.locator(s.box).first();
|
||||
await box.waitFor({ state: 'visible', timeout: 15000 }).catch(() => {});
|
||||
// Report which input path actually ran: 'human' = real xdotool cursor
|
||||
// move + char typing; 'api-fallback' = the humanClick path threw and we
|
||||
// fell back to cursor-less DOM click/fill; 'api' = no xdotool at all. This
|
||||
// makes "did the cursor really move" verifiable from the result.
|
||||
let searchInput;
|
||||
if (HAS_XDOTOOL && cmd.human !== false) {
|
||||
try {
|
||||
await human.humanClick(page, box);
|
||||
await human.humanType(q);
|
||||
await human.pressKey('Return');
|
||||
searchInput = 'human';
|
||||
} catch {
|
||||
searchInput = 'api-fallback';
|
||||
await box.click().catch(() => {});
|
||||
await box.fill(q).catch(() => {});
|
||||
await page.keyboard.press('Enter').catch(() => {});
|
||||
}
|
||||
} else {
|
||||
searchInput = 'api';
|
||||
await box.click().catch(() => {});
|
||||
await box.fill(q);
|
||||
await page.keyboard.press('Enter');
|
||||
}
|
||||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||||
out({ ok: true, site: SITES[siteKey] ? siteKey : 'google', query: q, url: page.url(), title: await page.title().catch(() => '') });
|
||||
out({ ok: true, site: SITES[siteKey] ? siteKey : 'google', query: q, url: page.url(), title: await page.title().catch(() => ''), input: searchInput });
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -205,6 +216,45 @@ try {
|
||||
break;
|
||||
}
|
||||
|
||||
case 'moveMouse': {
|
||||
// Move/hover the REAL cursor onto an element WITHOUT clicking. Target is a
|
||||
// CSS selector, or site=naver/google/... for that site's search box.
|
||||
// Only meaningful with xdotool (the visible cursor); with no xdotool there
|
||||
// is no cursor to move, so report that rather than faking success. Every
|
||||
// failure to actually move (no xdotool, selector never matches, element
|
||||
// has no on-screen box) returns ok:false — we must never claim the cursor
|
||||
// moved when it did not (the exact bug the user reported).
|
||||
const siteKey = String(cmd.site || '').toLowerCase();
|
||||
const selector = String(cmd.selector || '').trim() || (SITES[siteKey] ? SITES[siteKey].box : '');
|
||||
if (!selector) throw new Error('moveMouse: no selector or known site');
|
||||
if (!(HAS_XDOTOOL && cmd.human !== false)) {
|
||||
out({ ok: false, error: 'no xdotool: cannot move the visible cursor on this host' });
|
||||
break;
|
||||
}
|
||||
await front(page);
|
||||
let locator = page.locator(selector).first();
|
||||
let visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false);
|
||||
// A named site whose search box isn't on the current page: go to its home
|
||||
// first (real omnibox), then target the box there.
|
||||
if (!visible && SITES[siteKey]) {
|
||||
try { await human.navigateOmnibox(SITES[siteKey].home); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
|
||||
catch { await page.goto(SITES[siteKey].home, { waitUntil: 'domcontentloaded' }).catch(() => {}); }
|
||||
locator = page.locator(SITES[siteKey].box).first();
|
||||
visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false);
|
||||
}
|
||||
if (!visible) {
|
||||
out({ ok: false, error: `moveMouse: target not found (${cmd.selector || siteKey})` });
|
||||
break;
|
||||
}
|
||||
const moved = await human.humanHover(page, locator);
|
||||
if (!moved) {
|
||||
out({ ok: false, error: 'moveMouse: element has no on-screen box; cursor not moved' });
|
||||
break;
|
||||
}
|
||||
out({ ok: true, target: cmd.selector || siteKey, input: 'human' });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'click': {
|
||||
const selector = String(cmd.selector || '').trim();
|
||||
if (!selector) throw new Error('click: no selector');
|
||||
|
||||
@@ -136,14 +136,18 @@ export async function navigateOmnibox(text) {
|
||||
}
|
||||
|
||||
// Move the real cursor over an element (hover, no click) - e.g. to reveal a
|
||||
// video player's controls or to focus it for a keyboard shortcut.
|
||||
// video player's controls or to focus it for a keyboard shortcut. Returns true
|
||||
// only if the element had an on-screen box and the cursor was actually moved;
|
||||
// returns false when there is nothing to move to (so callers must not report
|
||||
// success). Brings the element into view with a real wheel scroll first.
|
||||
export async function humanHover(page, locator) {
|
||||
const box = await locator.boundingBox().catch(() => null);
|
||||
if (!box) return;
|
||||
const box = await bringIntoView(page, locator);
|
||||
if (!box) return false;
|
||||
const g = await page.evaluate(() => ({ sx: window.screenX, sy: window.screenY, ow: window.outerWidth, oh: window.outerHeight, iw: window.innerWidth, ih: window.innerHeight }));
|
||||
const bx = Math.max(0, Math.round((g.ow - g.iw) / 2));
|
||||
const oy = g.sy + Math.max(0, g.oh - g.ih - bx);
|
||||
await humanMove(Math.round(g.sx + bx + box.x + box.width * 0.5), Math.round(oy + box.y + box.height * 0.4));
|
||||
return true;
|
||||
}
|
||||
|
||||
export { sleep, rand };
|
||||
|
||||
@@ -26,7 +26,7 @@ export interface TextResult {
|
||||
|
||||
/** Full voice turn: WAV in -> {transcript, reply, reply audio}. ``broadcasting``
|
||||
* is the current live screen-share state so the brain routes search (Chrome
|
||||
* while live, Gemini when off). */
|
||||
* while live, the normal search when off). */
|
||||
export async function converse(wav: Buffer, broadcasting?: boolean): Promise<ConverseResult> {
|
||||
const qs = broadcasting === undefined ? "" : `?broadcasting=${broadcasting ? "1" : "0"}`;
|
||||
const res = await fetch(`${config.bridgeUrl}/converse${qs}`, {
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
*
|
||||
* The screen broadcast is coupled to the voice session: it auto-starts when the
|
||||
* assistant joins a voice channel (screen-share mode default), reports its live
|
||||
* state to the brain each turn (search routes Chrome while live / Gemini when
|
||||
* state to the brain each turn (search routes Chrome while live / normal search when
|
||||
* off), and the brain can toggle it by voice ("방송 켜줘 / 꺼줘"). This logic
|
||||
* used to live only in the normal-bot join handler, so the userbot path (the
|
||||
* only mode that can actually Go Live) never started a broadcast. Keeping it
|
||||
|
||||
@@ -74,7 +74,7 @@ async function handleJoin(i: ChatInputCommandInteraction) {
|
||||
|
||||
// Screen-share mode (STREAM_BROWSER=true): the broadcast is coupled to the
|
||||
// voice session. Auto-start it on join, report its live state to the brain
|
||||
// each turn (search routes Chrome while live / Gemini when off), and let the
|
||||
// each turn (search routes Chrome while live / normal search when off), and let the
|
||||
// brain toggle it via voice ("방송 켜줘 / 꺼줘"). In voice-only mode
|
||||
// (STREAM_BROWSER=false) none of this runs and the broadcast stays off.
|
||||
await maybeCoupleBroadcast(session, { guildId: i.guildId!, voiceChannelId: channel.id });
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
*
|
||||
* Does not broadcast natively into Discord. Instead it shares a noVNC web URL
|
||||
* that anyone can open in a browser to watch (and optionally control) the VNC
|
||||
* desktop live. Set NOVNC_URL in .env (e.g. http://192.168.10.9:6080/vnc.html).
|
||||
* desktop live. Set NOVNC_URL in .env to the browser host's LAN IP
|
||||
* (e.g. http://<browser-host-ip>:6080/vnc.html).
|
||||
*
|
||||
* Stand up noVNC once on the host with websockify, e.g.:
|
||||
* websockify --web=/usr/share/novnc 6080 localhost:5901
|
||||
@@ -22,7 +23,7 @@ export class NoVncStreamer implements ScreenStreamer {
|
||||
|
||||
async start(_ctx: StreamContext): Promise<string> {
|
||||
if (!this.config.novncUrl) {
|
||||
return "NOVNC_URL이 설정되지 않았습니다 (.env). 예: http://192.168.10.9:6080/vnc.html";
|
||||
return "NOVNC_URL이 설정되지 않았습니다 (.env). 예: http://<브라우저호스트IP>:6080/vnc.html";
|
||||
}
|
||||
this.active = true;
|
||||
return `🖥️ VNC 화면 실시간 보기 (브라우저): ${this.config.novncUrl}`;
|
||||
|
||||
@@ -99,7 +99,7 @@ export class VoiceSession {
|
||||
ttsEndMs?: number;
|
||||
}) => void;
|
||||
/** Live screen-share state, sent with each turn so the brain routes search
|
||||
* (Chrome while broadcasting, Gemini when off). */
|
||||
* (Chrome while broadcasting, the normal search when off). */
|
||||
getBroadcasting?: () => boolean;
|
||||
/** Apply a broadcast directive the brain requested (start/stop the stream). */
|
||||
onBroadcastAction?: (action: "start" | "stop") => void | Promise<void>;
|
||||
|
||||
@@ -21,7 +21,11 @@ nvidia-cudnn-cu12
|
||||
# --- Bridge HTTP service ---
|
||||
flask>=3.0.0
|
||||
|
||||
# --- Text-to-speech (Piper) ---
|
||||
# --- Text-to-speech ---
|
||||
# Edge TTS: the primary Korean voice (online MS neural). Lightweight (httpx);
|
||||
# emits MP3, transcoded to PCM16 by the system ffmpeg in the bridge.
|
||||
edge-tts>=6.1.0
|
||||
# Piper: offline English fallback.
|
||||
piper-tts>=1.3.0
|
||||
|
||||
# --- Built-in tools (lazily imported; needed for full functionality) ---
|
||||
|
||||
109
bridge/server.py
109
bridge/server.py
@@ -87,12 +87,22 @@ VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
||||
# Korean phrase decoded as Chinese) and shaves a little latency. Empty = auto.
|
||||
STT_LANGUAGE = os.environ.get("STT_LANGUAGE", "ko").strip() or None
|
||||
|
||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||
# Whisper decoding accuracy knobs. beam_size=1 is greedy decoding — fast but the
|
||||
# least accurate; beam search (5 is the Whisper default) explores alternatives
|
||||
# and noticeably improves recognition on short, accented, or noisy Discord-mic
|
||||
# speech. condition_on_previous_text=False stops Whisper from feeding a previous
|
||||
# clip's transcript back in as a prompt, which on isolated short utterances
|
||||
# causes repetition loops and drift rather than helping. Both are env-tunable so
|
||||
# accuracy/latency can be traded without a code change (lower STT_BEAM_SIZE for
|
||||
# speed, raise it for accuracy).
|
||||
STT_BEAM_SIZE = max(1, int(os.environ.get("STT_BEAM_SIZE", "5")))
|
||||
STT_CONDITION_ON_PREV = os.environ.get("STT_CONDITION_ON_PREV", "0") in ("1", "true", "True", "yes", "on")
|
||||
|
||||
# TTS engine: "edge" (Microsoft Edge TTS, natural Korean neural voice) is the
|
||||
# primary voice. "melo" (a warm MeloTTS worker) and "piper" remain selectable.
|
||||
def _tts_engine_setting() -> str:
|
||||
"""TTS engine: settings-UI value (runtime config JSON) wins, else env, else
|
||||
melo. Read at startup; the settings UI restarts the bridge on apply."""
|
||||
edge. Read at startup; the settings UI restarts the bridge on apply."""
|
||||
try:
|
||||
_cp = os.environ.get("JARVIS_CONFIG_PATH", "/app/config/jarvis.json")
|
||||
_v = json.loads(open(_cp, encoding="utf-8").read()).get("tts_engine")
|
||||
@@ -100,16 +110,22 @@ def _tts_engine_setting() -> str:
|
||||
return str(_v).strip().lower()
|
||||
except Exception:
|
||||
pass
|
||||
return os.environ.get("TTS_ENGINE", "melo").strip().lower()
|
||||
return os.environ.get("TTS_ENGINE", "edge").strip().lower()
|
||||
|
||||
|
||||
TTS_ENGINE = _tts_engine_setting()
|
||||
# Edge TTS (online MS neural voice). Voice + rate are env-driven so they can be
|
||||
# changed without code. Default: Korean "Hyunsu" multilingual voice at +45%
|
||||
# (≈1.45×), the chosen settings. NOTE: edge synthesis sends the reply TEXT to
|
||||
# Microsoft's servers and needs internet — an intentional privacy trade-off for
|
||||
# the more natural voice.
|
||||
EDGE_TTS_VOICE = os.environ.get("EDGE_TTS_VOICE", "ko-KR-HyunsuMultilingualNeural").strip()
|
||||
EDGE_TTS_RATE = os.environ.get("EDGE_TTS_RATE", "+45%").strip()
|
||||
MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770")
|
||||
MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30"))
|
||||
# When MeloTTS is the engine, do NOT silently fall back to the English Piper
|
||||
# voice on failure: speaking Korean text through an English voice produces
|
||||
# mangled audio. Default is melo-only (return no audio on failure); set
|
||||
# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback.
|
||||
# Do NOT silently fall back to the English Piper voice on a neural-voice failure:
|
||||
# speaking Korean through an English voice produces mangled audio. Default is
|
||||
# neural-only (return no audio on failure); set MELO_FALLBACK_PIPER=1 to opt in.
|
||||
MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -238,7 +254,12 @@ def transcribe(wav_bytes: bytes) -> dict:
|
||||
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
|
||||
return {"text": "", "language": None, "note": "음성 아님(VAD 차단)"}
|
||||
|
||||
segments, info = _whisper.transcribe(audio, beam_size=1, language=STT_LANGUAGE)
|
||||
segments, info = _whisper.transcribe(
|
||||
audio,
|
||||
beam_size=STT_BEAM_SIZE,
|
||||
language=STT_LANGUAGE,
|
||||
condition_on_previous_text=STT_CONDITION_ON_PREV,
|
||||
)
|
||||
# Second line of defence: drop non-speech / hallucinated segments by
|
||||
# Whisper's own no_speech_prob. The no_speech_prob hard cutoff (plus the VAD
|
||||
# pre-gate above) is what rejects noise/hallucinations. The avg_logprob
|
||||
@@ -265,7 +286,7 @@ def think(text: str, language: Optional[str] = None, broadcasting: Optional[bool
|
||||
|
||||
``broadcasting`` is the bot's live screen-share state for this turn; it is
|
||||
stashed in request-scoped state so the search routing can pick Chrome vs
|
||||
Gemini. If the reply engine calls the setBroadcast tool, the recorded
|
||||
the normal search chain. If the reply engine calls the setBroadcast tool, the recorded
|
||||
directive is returned as ``broadcast_action`` for the bot to act on.
|
||||
"""
|
||||
if not BRAIN_ENABLED:
|
||||
@@ -302,6 +323,54 @@ def _coerce_bool(value) -> Optional[bool]:
|
||||
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _edge_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesise via Microsoft Edge TTS (online neural voice) and return a
|
||||
16-bit PCM WAV, or None on any failure. Edge emits MP3; we transcode to
|
||||
PCM16 mono with the system ffmpeg, writing to a temp file (seekable) so the
|
||||
WAV header carries a correct length. Needs internet."""
|
||||
import asyncio
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
try:
|
||||
import edge_tts # type: ignore
|
||||
|
||||
async def _gen() -> bytes:
|
||||
comm = edge_tts.Communicate(text, EDGE_TTS_VOICE, rate=EDGE_TTS_RATE)
|
||||
buf = bytearray()
|
||||
async for chunk in comm.stream():
|
||||
if chunk.get("type") == "audio":
|
||||
buf.extend(chunk["data"])
|
||||
return bytes(buf)
|
||||
|
||||
mp3 = asyncio.run(_gen())
|
||||
if not mp3:
|
||||
print("[bridge] edge TTS returned no audio", flush=True)
|
||||
return None
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as t:
|
||||
out_path = t.name
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-i", "pipe:0", "-ac", "1", "-ar", "24000",
|
||||
"-acodec", "pcm_s16le", out_path],
|
||||
input=mp3, capture_output=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(f"[bridge] edge ffmpeg transcode failed: {proc.stderr.decode('utf-8','ignore')[:200]}", flush=True)
|
||||
return None
|
||||
with open(out_path, "rb") as f:
|
||||
return f.read()
|
||||
finally:
|
||||
try:
|
||||
os.unlink(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as e: # pragma: no cover - network / dep dependent
|
||||
print(f"[bridge] edge synth failed: {e}", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
def _melo_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean
|
||||
speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so
|
||||
@@ -361,20 +430,22 @@ def _tts_ready() -> bool:
|
||||
|
||||
|
||||
def synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS
|
||||
(Korean speaker, speed 1.5) served by the warm melo worker; Piper is a
|
||||
fallback if the worker is unavailable. Returns None if TTS is off."""
|
||||
"""Synthesize text to a 16-bit PCM WAV. The primary voice is Edge TTS (a
|
||||
natural Korean neural voice); "melo" uses the warm MeloTTS worker. For a
|
||||
neural engine, Piper (English) is only used if explicitly enabled, since
|
||||
speaking Korean through an English voice mangles it. Returns None if off."""
|
||||
if not TTS_ENABLED or not text.strip():
|
||||
return None
|
||||
if TTS_ENGINE == "melo":
|
||||
audio = _melo_synthesize(text)
|
||||
_neural = {"edge": _edge_synthesize, "melo": _melo_synthesize}.get(TTS_ENGINE)
|
||||
if _neural is not None:
|
||||
audio = _neural(text)
|
||||
if audio:
|
||||
return audio
|
||||
if not MELO_FALLBACK_PIPER:
|
||||
# Melo-only: better silent than mangled English for Korean text.
|
||||
print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True)
|
||||
# Neural-only: better silent than mangled English for Korean text.
|
||||
print(f"[bridge] {TTS_ENGINE} synth failed; no audio (Piper fallback disabled)", flush=True)
|
||||
return None
|
||||
print("[bridge] melo synth failed; falling back to Piper", flush=True)
|
||||
print(f"[bridge] {TTS_ENGINE} synth failed; falling back to Piper", flush=True)
|
||||
return _piper_synthesize(text)
|
||||
|
||||
|
||||
|
||||
@@ -22,8 +22,7 @@ from typing import Any, Dict
|
||||
FIELDS = [
|
||||
("ollama_chat_model", "LLM 모델", "model"),
|
||||
("whisper_model", "STT(Whisper) 모델", "select:tiny,base,small,medium,large,large-v3"),
|
||||
("tts_engine", "TTS 엔진", "select:melo,piper"),
|
||||
("melo_speed", "TTS 속도 (MeloTTS)", "number:0.5:2.5:0.1"),
|
||||
("tts_engine", "TTS 엔진", "select:edge,piper"),
|
||||
("output_language", "출력 언어 (비우면 사용자 언어)", "text"),
|
||||
("llm_thinking_enabled", "LLM 사고(thinking) 모드", "bool"),
|
||||
("agentic_max_turns", "에이전트 최대 반복", "number:1:12:1"),
|
||||
@@ -54,9 +53,7 @@ def _current() -> Dict[str, Any]:
|
||||
cfg = _read_config()
|
||||
out: Dict[str, Any] = {}
|
||||
for k in _KEYS:
|
||||
if k == "melo_speed":
|
||||
out[k] = cfg.get("melo_speed", os.environ.get("MELO_SPEED", "1.5"))
|
||||
elif k == "output_language":
|
||||
if k == "output_language":
|
||||
out[k] = cfg.get("output_language", os.environ.get("OUTPUT_LANGUAGE", ""))
|
||||
else:
|
||||
out[k] = cfg.get(k, "")
|
||||
@@ -78,12 +75,7 @@ def _coerce(updates: Dict[str, Any]) -> Dict[str, Any]:
|
||||
for k, v in updates.items():
|
||||
if k not in _KEYS:
|
||||
continue
|
||||
if k == "melo_speed":
|
||||
try:
|
||||
v = float(v)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
elif k == "agentic_max_turns":
|
||||
if k == "agentic_max_turns":
|
||||
try:
|
||||
v = int(v)
|
||||
except (TypeError, ValueError):
|
||||
@@ -114,15 +106,15 @@ def _save(updates: Dict[str, Any]) -> None:
|
||||
|
||||
|
||||
def _apply() -> str:
|
||||
# Restart melo + bridge AFTER this response is sent. Detached (new session)
|
||||
# so the bridge being killed mid-restart doesn't drop the restart itself,
|
||||
# and the HTTP client still receives this response.
|
||||
# Restart the bridge AFTER this response is sent. Detached (new session) so
|
||||
# the bridge being killed mid-restart doesn't drop the restart itself, and
|
||||
# the HTTP client still receives this response. (Edge TTS has no worker.)
|
||||
try:
|
||||
subprocess.Popen(
|
||||
["sh", "-c", "sleep 1; supervisorctl restart melo-worker bridge"],
|
||||
["sh", "-c", "sleep 1; supervisorctl restart bridge"],
|
||||
start_new_session=True,
|
||||
)
|
||||
return "1초 후 브리지/TTS 워커가 재시작되어 반영됩니다."
|
||||
return "1초 후 브리지가 재시작되어 반영됩니다."
|
||||
except Exception as e: # pragma: no cover
|
||||
return str(e)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user