Files
javis_bot/app/tests/fixtures/fake_engine_worker.py
javis-bot 53bccb620d feat: screen viewing, Jarvis sees the whole screen, a monitor or a window for registered people
- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its
  title or program, or the list of monitors and windows; fast, normal or detailed pictures.
- The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen
  viewing"), decides for the asker (only people registered for PC control), never picks, lists
  or shows excluded programs (pcControl.screenExclude, password managers by default), refuses
  minimised windows and black pictures, and records screen.view in the activity log.
- Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to
  OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only:
  not in the tool carryover, logs or prompt dumps.
- The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a
  game), measured apart for the placement; otherwise screenView is a disabled tool.
- Worker: screen_action / screen_action_result, 4 MiB messages.
- Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 17:02:59 +09:00

70 lines
2.8 KiB
Python

"""The real voice worker server with fake engines, for the app's protocol tests (no GPU, no models).
The fake VAD reads speech probability off the amplitude; the fake transcriber names a clip by its
pitch ("level 0.9" for a 900 Hz tone); the fake TTS makes 10 samples per character; the fake brain answers every reply with two sentences.
Run with the repo's src/ on PYTHONPATH: python fake_engine_worker.py <port>
"""
import asyncio
import os
import sys
import numpy as np
from voice_worker.server import VoiceWorker
class AmplitudeVad:
def prob(self, window):
return float(np.max(np.abs(window)))
class LevelTranscriber:
def transcribe(self, clips, opts):
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
class FakeBrain:
"""Answers with two sentences, says only "켜" (on) requests were said to it, ranks in reverse.
"예약" adds a reminder through the schedule backend and answers with its result, "화면" captures
monitor 2 through the screen backend; a task answers "task: <text>"."""
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
screen=None):
if task:
return f"task: {text}" + (f" notes={'|'.join(notes)}" if notes else "")
if "지침" in text and notes_backend is not None:
got = notes_backend.act("add", text="반말로 해")
return f"noted {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
if "화면" in text and screen is not None:
got = screen.act("capture", target="monitor", monitor=2, detail="fast")
sizes = ",".join(str(len(i)) for i in got.get("images") or [])
return f"saw {sizes} {got.get('item', {}).get('what', '')} {got.get('error', '')}".strip()
if "예약" in text and schedule is not None:
got = schedule.act("add", kind="remind", text="물 마시기", at=4_000_000_000.0, repeat="daily")
return f"scheduled {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
return "첫 문장입니다. 둘째 문장입니다."
def addressed(self, user, text, names):
return "켜" in text
def rank(self, candidates):
return [c["id"] for c in reversed(candidates)]
class FakeTts:
def synthesize(self, text, rate):
return np.full(len(text) * 10, 1000, dtype=np.int16)
async def main(port: int) -> None:
worker = VoiceWorker(os.environ["JARVIS_VOICE_TOKEN"], LevelTranscriber(), AmplitudeVad, FakeTts(),
brain=FakeBrain())
server = await worker.serve("127.0.0.1", port)
print(f"READY 127.0.0.1:{port}", flush=True)
await server.serve_forever()
if __name__ == "__main__":
asyncio.run(main(int(sys.argv[1])))