tkrmagid/sephiria_inv_program
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Recognizer v0.3.6: HDR-friendly empty + lower threshold + debug dump

Browse Source 
User reports all 34 cells classified as 미인식 with score 0.00 even
when the grid was correctly cropped. Multiple compounding issues:

1. _is_empty required mean<60 (dark) AND std<14. HDR/bright captures
   produce pinkish empty slots with mean ~150-180, so even empty cells
   fell through to template matching. Drop the mean check; uniformity
   alone (std<18 grayscale, std<22 per-channel) is the real signal.

2. Score 0.00 across the board strongly suggests templates list was
   empty (only path that returns exactly 0.0). Track per-bucket load
   counts (slabs_ok/fail, artifacts_ok/fail) and surface them in the
   GUI status bar so a CDN failure is immediately visible. Currently
   no signal at all on download failure.

3. min_score 0.55 was tuned against simulator-clean renders. Real game
   captures have decorative cell borders, stack-count badges in
   corners, HDR shader effects. Lower to 0.35 and inset cell crops by
   16% on each side before matching to skip the decorative frame.

4. Add 디버그 저장 button + dump_debug() that saves screenshot.png,
   bbox_crop.png, cells/<row>-<col>.png, and report.txt with top-3
   matches per cell to %LOCALAPPDATA%/sephiria_inv/debug/<timestamp>/.
   Lets us iterate on tuning from real captures without round-tripping
   raw screenshots through chat each time.
This commit is contained in:
Claude
2026-05-16 03:18:29 +09:00
parent 3c17405a6b
commit ee82b161eb
2 changed files with 154 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
sephiria_inv/recognizer.py
View File

@@ -80,17 +80,24 @@ _TEMPLATE_CACHE: List[_Template] = []
_CACHE_BUILT = False
_LAST_LOAD_STATS: Dict[str, int] = {"slabs_ok": 0, "slabs_fail": 0,
"artifacts_ok": 0, "artifacts_fail": 0}
def _build_templates(*, include_artifacts: bool = True) -> List[_Template]:
"""Build (and cache) the full template list. Lazy because download is slow."""
global _CACHE_BUILT
if _CACHE_BUILT and _TEMPLATE_CACHE:
return _TEMPLATE_CACHE
out: List[_Template] = []
s_ok = s_fail = a_ok = a_fail = 0
# Slabs: 4 rotations for rotatable, 1 otherwise
for s in SLABS:
img = fetch_slab_image(s.image)
if img is None:
s_fail += 1
continue
s_ok += 1
base = _on_dark(img)
rotations = (0, 1, 2, 3) if s.rotate else (0,)
for r in rotations:
@@ -100,9 +107,13 @@ def _build_templates(*, include_artifacts: bool = True) -> List[_Template]:
for a in ARTIFACTS:
img = fetch_artifact_image(a.image)
if img is None:
a_fail += 1
continue
a_ok += 1
base = _on_dark(img)
out.append(_Template("artifact", a.value, 0, _to_feat(base)))
_LAST_LOAD_STATS.update({"slabs_ok": s_ok, "slabs_fail": s_fail,
"artifacts_ok": a_ok, "artifacts_fail": a_fail})
_TEMPLATE_CACHE.clear()
_TEMPLATE_CACHE.extend(out)
_CACHE_BUILT = True
@@ -117,24 +128,51 @@ def warm_templates(*, include_artifacts: bool = True) -> int:
return len(_build_templates(include_artifacts=include_artifacts))
def load_stats() -> Dict[str, int]:
"""Return last template load counts: slabs_ok, slabs_fail, artifacts_ok, artifacts_fail."""
return dict(_LAST_LOAD_STATS)
# ---------- cell classification ----------
def _is_empty(cell: Image.Image) -> bool:
"""Heuristic: empty slots are dark and ~uniform."""
"""Heuristic: empty slots are uniform color (any brightness).
Drops the dark-only assumption so HDR / bright-monitor captures with
pinkish slot backgrounds still detect as empty. Uniformity is the
actual invariant — empty slots have low std-dev whatever the hue.
"""
g = np.asarray(cell.convert("L"), dtype=np.float32)
return bool(g.mean() < 60.0 and g.std() < 14.0)
rgb = np.asarray(cell.convert("RGB"), dtype=np.float32)
chan_std = float(rgb.reshape(-1, 3).std(axis=0).mean())
return bool(g.std() < 18.0 and chan_std < 22.0)
def _inset(cell: Image.Image, ratio: float = 0.16) -> Image.Image:
"""Trim decorative borders / corner badges before template matching.
The in-game slot has chunky frame ornaments and a stack-count badge in
a corner. Templates are clean icons. Cropping ~16% off every side
aligns the comparable inner art and removes the badge area in most
games.
"""
w, h = cell.size
dx = int(w * ratio)
dy = int(h * ratio)
return cell.crop((dx, dy, w - dx, h - dy))
def _classify(
cell: Image.Image,
templates: List[_Template],
*,
min_score: float = 0.55,
min_score: float = 0.35,
) -> Tuple[str, Optional[str], int, float]:
"""Return (kind, value, rotation, score)."""
if _is_empty(cell):
return "empty", None, 0, 1.0
feat = _to_feat(cell)
inner = _inset(cell)
feat = _to_feat(inner)
# Stack template features into a matrix for one big dot-product
if not templates:
return "unknown", None, 0, 0.0
@@ -148,6 +186,27 @@ def _classify(
return t.kind, t.value, t.rotation, best
def _classify_with_top(
cell: Image.Image,
templates: List[_Template],
*,
top_k: int = 3,
) -> Tuple[str, Optional[str], int, float, List[Tuple[str, str, int, float]]]:
"""Like _classify but also returns the top-k matches for debug dumps."""
if _is_empty(cell):
return "empty", None, 0, 1.0, []
if not templates:
return "unknown", None, 0, 0.0, []
feat = _to_feat(_inset(cell))
M = np.stack([t.feat for t in templates], axis=0)
scores = M @ feat
order = np.argsort(-scores)[:top_k]
top = [(templates[i].kind, templates[i].value, templates[i].rotation,
float(scores[i])) for i in order]
kind, value, rot, score = _classify(cell, templates)
return kind, value, rot, score, top
# ---------- public API ----------
def recognize_image(
@@ -156,7 +215,7 @@ def recognize_image(
*,
slot_num: int = 34,
include_artifacts: bool = True,
min_score: float = 0.55,
min_score: float = 0.35,
) -> List[CellResult]:
"""Slice img[bbox] into a 6-col grid and classify each cell.
@@ -184,6 +243,59 @@ def recognize_image(
return out
def dump_debug(
img: Image.Image,
bbox: Tuple[int, int, int, int],
out_dir: str,
*,
slot_num: int = 34,
include_artifacts: bool = True,
) -> str:
"""Save full screenshot, bbox crop, every cell crop and a top-3 match
report to out_dir. Returns the path to the report file. Used to iterate
on recognizer tuning from real captures.
"""
import os
os.makedirs(out_dir, exist_ok=True)
img.save(os.path.join(out_dir, "screenshot.png"))
L, T, R, B = bbox
crop = img.crop((L, T, R, B)).convert("RGB")
crop.save(os.path.join(out_dir, "bbox_crop.png"))
grid = generate_grid_config(slot_num)
if not grid:
return out_dir
rows = len(grid)
cell_w = (R - L) // GRID_COLS
cell_h = (B - T) // rows
templates = _build_templates(include_artifacts=include_artifacts)
stats = load_stats()
lines = [
f"bbox: {bbox}",
f"grid: {len(grid)} rows x {GRID_COLS} cols, slot_num={slot_num}",
f"cell px: {cell_w} x {cell_h}",
f"templates loaded: total={len(templates)} stats={stats}",
"",
]
cells_dir = os.path.join(out_dir, "cells")
os.makedirs(cells_dir, exist_ok=True)
for row in grid:
y = row["rows"]
for x in range(row["cols"]):
cx0 = x * cell_w
cy0 = y * cell_h
cell = crop.crop((cx0, cy0, cx0 + cell_w, cy0 + cell_h))
cell.save(os.path.join(cells_dir, f"{y}-{x}.png"))
kind, value, rot, score, top = _classify_with_top(cell, templates)
top_s = ", ".join(f"{k}:{v}@r{r}={s:.3f}" for k, v, r, s in top)
lines.append(
f" {y}-{x}: kind={kind} value={value} rot={rot} score={score:.3f} | top: {top_s}"
)
report = os.path.join(out_dir, "report.txt")
with open(report, "w", encoding="utf-8") as fh:
fh.write("\n".join(lines))
return report
def recognize_file(
path: str,
bbox: Tuple[int, int, int, int],