- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its title or program, or the list of monitors and windows; fast, normal or detailed pictures. - The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen viewing"), decides for the asker (only people registered for PC control), never picks, lists or shows excluded programs (pcControl.screenExclude, password managers by default), refuses minimised windows and black pictures, and records screen.view in the activity log. - Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only: not in the tool carryover, logs or prompt dumps. - The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a game), measured apart for the placement; otherwise screenView is a disabled tool. - Worker: screen_action / screen_action_result, 4 MiB messages. - Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
70 lines
2.8 KiB
Python
70 lines
2.8 KiB
Python
"""The real voice worker server with fake engines, for the app's protocol tests (no GPU, no models).
|
|
|
|
The fake VAD reads speech probability off the amplitude; the fake transcriber names a clip by its
|
|
pitch ("level 0.9" for a 900 Hz tone); the fake TTS makes 10 samples per character; the fake brain answers every reply with two sentences.
|
|
Run with the repo's src/ on PYTHONPATH: python fake_engine_worker.py <port>
|
|
"""
|
|
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
|
|
import numpy as np
|
|
|
|
from voice_worker.server import VoiceWorker
|
|
|
|
|
|
class AmplitudeVad:
|
|
def prob(self, window):
|
|
return float(np.max(np.abs(window)))
|
|
|
|
|
|
class LevelTranscriber:
|
|
def transcribe(self, clips, opts):
|
|
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
|
|
|
|
|
class FakeBrain:
|
|
"""Answers with two sentences, says only "켜" (on) requests were said to it, ranks in reverse.
|
|
"예약" adds a reminder through the schedule backend and answers with its result, "화면" captures
|
|
monitor 2 through the screen backend; a task answers "task: <text>"."""
|
|
|
|
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
|
|
screen=None):
|
|
if task:
|
|
return f"task: {text}" + (f" notes={'|'.join(notes)}" if notes else "")
|
|
if "지침" in text and notes_backend is not None:
|
|
got = notes_backend.act("add", text="반말로 해")
|
|
return f"noted {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
|
|
if "화면" in text and screen is not None:
|
|
got = screen.act("capture", target="monitor", monitor=2, detail="fast")
|
|
sizes = ",".join(str(len(i)) for i in got.get("images") or [])
|
|
return f"saw {sizes} {got.get('item', {}).get('what', '')} {got.get('error', '')}".strip()
|
|
if "예약" in text and schedule is not None:
|
|
got = schedule.act("add", kind="remind", text="물 마시기", at=4_000_000_000.0, repeat="daily")
|
|
return f"scheduled {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
|
|
return "첫 문장입니다. 둘째 문장입니다."
|
|
|
|
def addressed(self, user, text, names):
|
|
return "켜" in text
|
|
|
|
def rank(self, candidates):
|
|
return [c["id"] for c in reversed(candidates)]
|
|
|
|
|
|
class FakeTts:
|
|
def synthesize(self, text, rate):
|
|
return np.full(len(text) * 10, 1000, dtype=np.int16)
|
|
|
|
|
|
async def main(port: int) -> None:
|
|
worker = VoiceWorker(os.environ["JARVIS_VOICE_TOKEN"], LevelTranscriber(), AmplitudeVad, FakeTts(),
|
|
brain=FakeBrain())
|
|
server = await worker.serve("127.0.0.1", port)
|
|
print(f"READY 127.0.0.1:{port}", flush=True)
|
|
await server.serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main(int(sys.argv[1])))
|