M0 on Strata caught both v2 models skipping a tool they had been given: the recommended model made up "24 degrees, sunny" instead of calling the weather tool, and the fast model said "timer set" without calling the timer tool (3 of 3 each). qwen2.5:7b does the same through the engine. - Prompt: TOOL_HONESTY_RULES in the persona (live facts only from a tool result this turn, actions only claimed after a tool reports success, no answer text next to a tool call), restated at the end of the system message by tool_honesty_reminder() with the turn's real tool names. - Engine: skipped-tool guard. When a native-tool model answers before any tool has run and the plan names a callable tool as a concrete call, the answer is dropped and the plan steps are direct-executed, then the model answers from the results. Only literal, schema-valid steps run (regex fast path, no LLM repair); otherwise the held-back answer is delivered. - Planner: the AVAILABLE TOOLS lines now carry argument names (planner_tool_catalog), so concrete steps use the tool's real keys. - Eval evals/test_tool_skip_confabulation.py; on qwen2.5:7b, 3 runs each, available-tool cases went from 0/12 to 9/12 calling the tool. - M0 kit: tools_fast / tools_recommended stages (run_m0_tools.bat) rerun tool calling before and after the fix on the real models, loading the production system_prompt.py and planner.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
164 lines
7.5 KiB
Python
164 lines
7.5 KiB
Python
"""Jarvis v2 M0: measure the gaming PC once and pack the results into one zip.
|
|
|
|
Started by run_m0.bat -> run_m0.ps1 (which installs uv and Python into the
|
|
work folder). Each stage writes results/<stage>.json; a stage that already has
|
|
a result is skipped on the next run, so the script can be started again after
|
|
an interruption.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
KIT = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(KIT))
|
|
from m0lib import envs # noqa: E402
|
|
from m0lib.common import read_json, say, step, write_json # noqa: E402
|
|
|
|
# key, title, venv ("" = this Python), script, options, free disk needed (GB)
|
|
STAGES = [
|
|
("sysinfo", "PC 정보", "", "sysinfo.py", [], 0),
|
|
("winchecks", "창·디스코드·게임 확인 (직접 답해 주셔야 합니다)", "", "winchecks.py", [], 0),
|
|
("vdisplay", "가상 모니터에 봇용 창 두기 (직접 답해 주셔야 합니다)", "", "vdisplay.py", [], 0),
|
|
("tts_melo", "MeloTTS 속도", "melo", "tts_melo.py", [], 10),
|
|
("tts_cosy2", "CosyVoice2 속도", "cosy", "tts_cosy.py", ["models=cosyvoice2"], 15),
|
|
("tts_cosy3", "Fun-CosyVoice3 속도", "cosy", "tts_cosy.py", ["models=cosyvoice3"], 8),
|
|
("stt", "STT(faster-whisper turbo) 속도", "stt", "stt.py", [], 5),
|
|
("llm_fast", "LLM 빠름 모델 (Qwen3.6-35B-A3B, 약 23GB 받기)", "base", "llm.py", ["model=fast"], 35),
|
|
("llm_recommended", "LLM 권장 모델 (Qwen3.8-Flash-Next Q2_0, 약 66GB 받기)", "base", "llm.py",
|
|
["model=recommended"], 80),
|
|
# Tool calling again with Jarvis's fix (prompt rules + planner + skipped-tool guard).
|
|
# Reuses the models the two stages above downloaded.
|
|
("tools_fast", "도구 호출 다시 재기: 빠름 모델", "base", "llm.py", ["model=fast", "only=tools"], 0),
|
|
("tools_recommended", "도구 호출 다시 재기: 권장 모델", "base", "llm.py", ["model=recommended", "only=tools"], 0),
|
|
]
|
|
|
|
|
|
def free_gb(path: Path) -> float:
|
|
return shutil.disk_usage(path).free / 1e9
|
|
|
|
|
|
def run_stage(key: str, venv: str, script: str, opts: list[str], work: Path, env: dict) -> None:
|
|
out = work / "results" / f"{key}.json"
|
|
log = work / "logs" / f"env_{venv or 'main'}.log"
|
|
if venv:
|
|
py = str(envs.ensure_env(venv, work, log))
|
|
else:
|
|
py = sys.executable
|
|
if venv == "cosy":
|
|
src = envs.ensure_cosy_source(work, log)
|
|
opts = opts + [f"src={src}"]
|
|
cmd = [py, str(KIT / "m0lib" / script), "--out", str(out), "--work", str(work)]
|
|
for o in opts:
|
|
cmd += ["--opt", o]
|
|
# Inherit the console so interactive stages can ask questions; stage
|
|
# scripts keep their own heavy output in logs/.
|
|
proc = subprocess.run(cmd, env=env)
|
|
if not out.exists():
|
|
write_json(out, {"stage": key, "status": "failed", "errors": [
|
|
{"where": "process", "error": f"stage exited with {proc.returncode} without a result"}]})
|
|
|
|
|
|
def summarise(work: Path) -> str:
|
|
lines = ["# Jarvis v2 M0 결과 요약", "", f"만든 시각: {dt.datetime.now():%Y-%m-%d %H:%M}", ""]
|
|
for key, title, *_ in STAGES:
|
|
p = work / "results" / f"{key}.json"
|
|
if not p.exists():
|
|
lines.append(f"- {title}: 실행 안 함")
|
|
continue
|
|
d = read_json(p)
|
|
errs = len(d.get("errors") or [])
|
|
lines.append(f"- {title}: {d.get('status')} ({d.get('seconds', '?')}초, 오류 {errs}개)")
|
|
lines += ["", "자세한 수치는 같은 zip 안의 results/*.json에 있습니다. 이 zip 파일을 그대로 보내 주시면 됩니다."]
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def pack(work: Path) -> Path:
|
|
stamp = dt.datetime.now().strftime("%Y%m%d_%H%M")
|
|
target = KIT / f"M0_results_{stamp}.zip"
|
|
summary = summarise(work)
|
|
(work / "results" / "summary.md").write_text(summary, encoding="utf-8")
|
|
with zipfile.ZipFile(target, "w", zipfile.ZIP_DEFLATED) as z:
|
|
for p in sorted((work / "results").rglob("*")):
|
|
if p.is_file() and p.suffix in (".json", ".wav", ".md"):
|
|
z.write(p, f"results/{p.relative_to(work / 'results').as_posix()}")
|
|
for p in sorted((work / "logs").glob("*.log")):
|
|
data = p.read_bytes()
|
|
# Keep the end of long logs (download progress) so the zip stays small.
|
|
z.writestr(f"logs/{p.name}", data[-1_000_000:])
|
|
return target
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--work", required=True)
|
|
ap.add_argument("--only", default="", help="comma list of stage keys")
|
|
ap.add_argument("--skip", default="", help="comma list of stage keys")
|
|
ap.add_argument("--redo", action="store_true", help="measure again even if a result exists")
|
|
ap.add_argument("--llm-url", default="", help="measure this OpenAI-compatible server instead of Strata")
|
|
a = ap.parse_args()
|
|
|
|
work = Path(a.work).resolve()
|
|
for d in ("results", "logs", "cache/tmp"):
|
|
(work / d).mkdir(parents=True, exist_ok=True)
|
|
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", HF_HOME=str(work / "models" / "hf"),
|
|
TEMP=str(work / "cache" / "tmp"), TMP=str(work / "cache" / "tmp"),
|
|
HF_HUB_DISABLE_TELEMETRY="1", DO_NOT_TRACK="1")
|
|
only = {x for x in a.only.split(",") if x}
|
|
skip = {x for x in a.skip.split(",") if x}
|
|
|
|
say("Jarvis v2 M0 측정을 시작합니다.")
|
|
say(f"작업 폴더: {work} (남은 공간 {free_gb(work):.0f} GB)")
|
|
say("모든 파일은 이 작업 폴더 안에만 받습니다. 끝나면 결과 zip 파일 위치를 알려 드립니다.")
|
|
say("앞쪽의 '직접 답해 주셔야 합니다' 단계가 끝나면 자리를 비우셔도 됩니다.")
|
|
say("측정이 정확하도록 그동안 게임이나 무거운 프로그램은 꺼 두세요.\n")
|
|
|
|
t0 = time.time()
|
|
for key, title, venv, script, opts, need_gb in STAGES:
|
|
if (only and key not in only) or key in skip:
|
|
continue
|
|
out = work / "results" / f"{key}.json"
|
|
if out.exists() and not a.redo:
|
|
status = read_json(out).get("status")
|
|
if status in ("ok", "partial", "skipped"):
|
|
say(f"[건너뜀] {title}: 이미 측정했습니다 ({status}). 다시 재려면 --redo")
|
|
continue
|
|
step(title)
|
|
if need_gb and free_gb(work) < need_gb:
|
|
say(f" 디스크 공간이 부족합니다 (필요 약 {need_gb} GB, 남은 {free_gb(work):.0f} GB). 건너뜁니다.")
|
|
write_json(out, {"stage": key, "status": "failed",
|
|
"errors": [{"where": "disk", "error": f"free {free_gb(work):.0f} GB < {need_gb} GB"}]})
|
|
continue
|
|
stage_opts = list(opts)
|
|
if key.startswith(("llm_", "tools_")) and a.llm_url:
|
|
stage_opts.append(f"url={a.llm_url}")
|
|
try:
|
|
run_stage(key, venv, script, stage_opts, work, env)
|
|
except Exception as e:
|
|
say(f" [오류] {title}: {e}")
|
|
write_json(out, {"stage": key, "status": "failed", "errors": [{"where": "setup", "error": str(e)}]})
|
|
if key == "vdisplay":
|
|
say("\n직접 확인하는 단계는 끝났습니다. 이제 자리를 비우셔도 됩니다.")
|
|
|
|
target = pack(work)
|
|
say("\n" + summarise(work))
|
|
say(f"걸린 시간: {(time.time() - t0) / 60:.0f}분")
|
|
say(f"결과 파일: {target}")
|
|
say("이 zip 파일을 디스코드로 보내 주세요.")
|
|
if os.name == "nt":
|
|
subprocess.run(["explorer", "/select,", str(target)])
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|