Files
sephiria_inv_program/sephiria_inv/gui.py
Claude e388c965bc Add screenshot-driven inventory recognition (v0.2.0)
- New ScreenshotFrame: capture screen / load PNG, two-click bbox,
  threaded template matching, editable preview grid for corrections
- ManualFrame kept as second tab for users who prefer typing counts
- capture.py: screen grab via mss (cross-platform)
- requirements: add mss>=6.0 for screen capture support

Closes the gap from v0.1.0 where users had to manually count
every slab — now they aim, click two corners, and edit any
mis-recognized cell.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-13 22:31:27 +09:00

633 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Tkinter GUI for the Sephiria inventory optimizer.
The main window has a notebook with two input modes:
- "수동": pick each slab and its count from the tier-grouped catalog.
- "스크린샷": grab the current screen (or load a PNG), click two corners of
the inventory area, and let the app recognize the slabs by template
matching against the cached CDN images. Mis-recognized cells can be
edited inline before solving.
Both modes feed into the same basket → solver → renderer pipeline on the right.
"""
from __future__ import annotations
import os
import sys
import threading
import tkinter as tk
from tkinter import filedialog, messagebox, simpledialog, ttk
from typing import Dict, List, Optional
from PIL import Image, ImageTk
from .renderer import fetch_slab_image, render_solution
from .slabs import SLABS, SLABS_BY_VALUE, TIER_LABEL, TIER_ORDER
from .solver import solve
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _slab_thumb(value: str, size: int = 48) -> Optional[ImageTk.PhotoImage]:
"""Return a Tk image for the given slab (downloaded + cached)."""
slab = SLABS_BY_VALUE.get(value)
if not slab:
return None
img = fetch_slab_image(slab.image)
if img is None:
return None
img = img.resize((size, size))
return ImageTk.PhotoImage(img)
# ---------------------------------------------------------------------------
# Crop dialog (two-click bbox)
# ---------------------------------------------------------------------------
class CropDialog(tk.Toplevel):
"""Show an image and let the user click two corners of the inventory area."""
MAX_DISPLAY = 1100
def __init__(self, parent: tk.Tk, img: Image.Image) -> None:
super().__init__(parent)
self.title("인벤토리 영역 선택 — 좌상단과 우하단을 차례로 클릭")
self.transient(parent)
self.img_full = img
self.bbox: Optional[tuple] = None
self._clicks: List[tuple] = []
# scale to fit
w, h = img.size
self.scale = min(1.0, self.MAX_DISPLAY / max(w, h))
disp_w, disp_h = int(w * self.scale), int(h * self.scale)
self.disp_img = img.resize((disp_w, disp_h))
self.tk_img = ImageTk.PhotoImage(self.disp_img)
info = ttk.Label(
self,
text="인벤토리 그리드의 좌상단을 먼저 클릭하고, "
"우하단을 다시 클릭하세요. (그리드 바깥의 UI 프레임은 빼고)",
)
info.pack(padx=8, pady=(8, 4))
self.canvas = tk.Canvas(
self, width=disp_w, height=disp_h, highlightthickness=0
)
self.canvas.pack(padx=8, pady=4)
self.canvas.create_image(0, 0, anchor="nw", image=self.tk_img)
self.canvas.bind("<Button-1>", self._on_click)
btns = ttk.Frame(self)
btns.pack(fill="x", padx=8, pady=(4, 8))
ttk.Button(btns, text="처음부터 다시", command=self._reset).pack(side="left")
ttk.Button(btns, text="취소", command=self._cancel).pack(side="right")
self.protocol("WM_DELETE_WINDOW", self._cancel)
self.grab_set()
self.focus_set()
def _on_click(self, e: tk.Event) -> None:
self._clicks.append((e.x, e.y))
self.canvas.create_oval(
e.x - 5, e.y - 5, e.x + 5, e.y + 5, outline="#ff5050", width=2
)
if len(self._clicks) == 2:
(x1, y1), (x2, y2) = self._clicks
lx, ty = min(x1, x2), min(y1, y2)
rx, by = max(x1, x2), max(y1, y2)
self.canvas.create_rectangle(
lx, ty, rx, by, outline="#50ff80", width=2
)
# convert back to original-image coords
inv = 1.0 / self.scale
self.bbox = (int(lx * inv), int(ty * inv), int(rx * inv), int(by * inv))
self.after(120, self.destroy)
def _reset(self) -> None:
self._clicks.clear()
self.canvas.delete("all")
self.canvas.create_image(0, 0, anchor="nw", image=self.tk_img)
def _cancel(self) -> None:
self.bbox = None
self.destroy()
# ---------------------------------------------------------------------------
# Slab picker dialog (used by editable recognition grid)
# ---------------------------------------------------------------------------
class SlabPicker(tk.Toplevel):
"""Modal slab picker — returns value via .selected, None if cancelled."""
def __init__(self, parent: tk.Tk, current: Optional[str] = None) -> None:
super().__init__(parent)
self.title("석판 선택")
self.transient(parent)
self.selected: Optional[str] = current
wrap = ttk.Frame(self, padding=8)
wrap.pack(fill="both", expand=True)
ttk.Label(wrap, text="석판을 선택하세요. (빈칸으로 두려면 '비우기')").pack(
anchor="w"
)
canvas = tk.Canvas(wrap, height=400, highlightthickness=0)
scroll = ttk.Scrollbar(wrap, orient="vertical", command=canvas.yview)
inner = ttk.Frame(canvas)
inner.bind("<Configure>",
lambda e: canvas.configure(scrollregion=canvas.bbox("all")))
canvas.create_window((0, 0), window=inner, anchor="nw")
canvas.configure(yscrollcommand=scroll.set)
canvas.pack(side="left", fill="both", expand=True)
scroll.pack(side="right", fill="y")
for tier in sorted({s.tier for s in SLABS}, key=lambda t: TIER_ORDER.get(t, 99)):
ttk.Label(inner, text=TIER_LABEL.get(tier, tier),
font=("TkDefaultFont", 10, "bold")).pack(anchor="w", pady=(8, 2))
row = ttk.Frame(inner)
row.pack(fill="x")
col = 0
for s in SLABS:
if s.tier != tier:
continue
b = ttk.Button(row, text=s.ko_label, width=8,
command=lambda v=s.value: self._pick(v))
b.grid(row=0, column=col % 6, sticky="w", padx=2, pady=2)
if col % 6 == 5:
row = ttk.Frame(inner)
row.pack(fill="x")
col = 0
else:
col += 1
btns = ttk.Frame(self)
btns.pack(fill="x", padx=8, pady=8)
ttk.Button(btns, text="비우기 (빈칸)",
command=lambda: self._pick("")).pack(side="left")
ttk.Button(btns, text="취소", command=self.destroy).pack(side="right")
self.grab_set()
self.focus_set()
def _pick(self, value: str) -> None:
self.selected = value if value else None
self.destroy()
# ---------------------------------------------------------------------------
# Screenshot input frame
# ---------------------------------------------------------------------------
class ScreenshotFrame(ttk.Frame):
"""Capture → crop → recognize → editable preview → confirm."""
def __init__(self, master, on_confirmed) -> None:
super().__init__(master, padding=6)
self.on_confirmed = on_confirmed
self.slot_var = master.master.slot_var # type: ignore[attr-defined]
self.image: Optional[Image.Image] = None
self.bbox: Optional[tuple] = None
self.recognitions: List["Recognition"] = [] # noqa: F821 (forward ref)
self._thumbs: List[ImageTk.PhotoImage] = [] # keep refs
controls = ttk.Frame(self)
controls.pack(fill="x")
ttk.Button(controls, text="🖥 현재 화면 캡처",
command=self._capture).pack(side="left")
ttk.Button(controls, text="📂 스크린샷 파일 열기…",
command=self._open_file).pack(side="left", padx=4)
ttk.Button(controls, text="🔁 다시 영역 지정",
command=self._reselect_bbox).pack(side="left", padx=4)
ttk.Button(controls, text="✅ 이 목록으로 솔버 실행",
command=self._confirm).pack(side="right")
self.status = ttk.Label(
self,
text="화면을 캡처하거나 PNG 파일을 열면 인벤토리 영역을 지정한 뒤 "
"자동으로 석판을 인식합니다.",
wraplength=460, justify="left",
)
self.status.pack(fill="x", pady=(6, 4))
# editable preview grid (scrollable)
self.preview_canvas = tk.Canvas(self, height=420, highlightthickness=0)
self.preview_canvas.pack(fill="both", expand=True)
self.preview_inner = ttk.Frame(self.preview_canvas)
self.preview_canvas.create_window((0, 0), window=self.preview_inner, anchor="nw")
self.preview_inner.bind(
"<Configure>",
lambda e: self.preview_canvas.configure(
scrollregion=self.preview_canvas.bbox("all")
),
)
# ----------------------------------------------- handlers
def _capture(self) -> None:
try:
from .capture import capture_screen
self.update_idletasks()
top = self.winfo_toplevel()
top.withdraw()
top.update()
self.image = capture_screen(monitor=1)
top.deiconify()
self._pick_bbox_and_recognize()
except Exception as e:
messagebox.showerror("캡처 실패", str(e))
self.winfo_toplevel().deiconify()
def _open_file(self) -> None:
path = filedialog.askopenfilename(
title="스크린샷 PNG 선택",
filetypes=[("Images", "*.png *.jpg *.jpeg *.webp"), ("All files", "*.*")],
)
if not path:
return
try:
self.image = Image.open(path).convert("RGB")
except Exception as e:
messagebox.showerror("열기 실패", str(e))
return
self._pick_bbox_and_recognize()
def _reselect_bbox(self) -> None:
if self.image is None:
messagebox.showinfo("안내", "먼저 화면 캡처 또는 파일 열기를 해주세요.")
return
self._pick_bbox_and_recognize()
def _pick_bbox_and_recognize(self) -> None:
assert self.image is not None
dlg = CropDialog(self.winfo_toplevel(), self.image)
self.winfo_toplevel().wait_window(dlg)
if not dlg.bbox:
return
self.bbox = dlg.bbox
self.status["text"] = "석판 인식 중…"
self.update_idletasks()
threading.Thread(target=self._recognize_thread, daemon=True).start()
def _recognize_thread(self) -> None:
try:
from .screenshot import recognize
slot_num = int(round(self.slot_var.get()))
# save image to temp + call recognizer on file path (recognize expects path)
import tempfile
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f:
self.image.save(f.name) # type: ignore[union-attr]
tmp = f.name
try:
recs = recognize(tmp, self.bbox, slot_num=slot_num)
finally:
try:
os.unlink(tmp)
except OSError:
pass
self.after(0, self._show_recognitions, recs)
except Exception as e:
self.after(0, lambda: messagebox.showerror("인식 실패", str(e)))
def _show_recognitions(self, recs: list) -> None:
self.recognitions = recs
for child in self.preview_inner.winfo_children():
child.destroy()
self._thumbs.clear()
# group into 6-wide grid
from .slabs import GRID_COLS
slot_to_rec = {r.slot_id: r for r in recs}
slot_num = int(round(self.slot_var.get()))
from .slabs import generate_grid_config
grid = generate_grid_config(slot_num)
for row_cfg in grid:
y = row_cfg["rows"]
for x in range(GRID_COLS):
if x >= row_cfg["cols"]:
# filler
blank = ttk.Frame(self.preview_inner, width=64, height=64)
blank.grid(row=y, column=x, padx=2, pady=2)
continue
slot_id = f"{y}-{x}"
rec = slot_to_rec.get(slot_id)
value = rec.value if rec else None
self._make_cell(y, x, slot_id, value, rec)
recognised = sum(1 for r in recs if r.value)
self.status["text"] = (
f"인식된 석판 {recognised}개 / 슬롯 {len(recs)}개. "
"잘못 인식된 칸은 셀을 클릭해서 수정하세요."
)
def _make_cell(self, y: int, x: int, slot_id: str, value: Optional[str],
rec) -> None:
frame = tk.Frame(
self.preview_inner, bd=1, relief="solid", width=72, height=78,
bg="#2c1a2a",
)
frame.grid(row=y, column=x, padx=2, pady=2)
frame.grid_propagate(False)
score_txt = f"{rec.score:.0f}" if rec else "-"
if value:
thumb = _slab_thumb(value, size=48)
if thumb is not None:
self._thumbs.append(thumb)
lbl = tk.Label(frame, image=thumb, bg="#2c1a2a")
else:
slab = SLABS_BY_VALUE.get(value)
lbl = tk.Label(frame, text=slab.ko_label if slab else value,
fg="#fff", bg="#2c1a2a")
else:
lbl = tk.Label(frame, text="(빈칸)", fg="#888", bg="#2c1a2a")
lbl.pack(expand=True)
slab = SLABS_BY_VALUE.get(value) if value else None
cap = tk.Label(
frame,
text=f"{slab.ko_label if slab else ''} · {score_txt}",
fg="#cfcfcf", bg="#2c1a2a", font=("TkDefaultFont", 8),
)
cap.pack(fill="x")
def click(_e=None, sid=slot_id):
self._edit_cell(sid)
for w in (frame, lbl, cap):
w.bind("<Button-1>", click)
def _edit_cell(self, slot_id: str) -> None:
cur: Optional[str] = None
for r in self.recognitions:
if r.slot_id == slot_id:
cur = r.value
break
dlg = SlabPicker(self.winfo_toplevel(), current=cur)
self.winfo_toplevel().wait_window(dlg)
# apply
for r in self.recognitions:
if r.slot_id == slot_id:
r.value = dlg.selected
break
self._show_recognitions(self.recognitions)
def _confirm(self) -> None:
if not self.recognitions:
messagebox.showinfo("안내", "먼저 화면을 캡처하고 영역을 지정하세요.")
return
basket = [r.value for r in self.recognitions if r.value]
self.on_confirmed(basket)
# ---------------------------------------------------------------------------
# Manual input frame (tier-grouped +/-)
# ---------------------------------------------------------------------------
class ManualFrame(ttk.Frame):
def __init__(self, master, on_changed) -> None:
super().__init__(master, padding=4)
self.counts: Dict[str, int] = {s.value: 0 for s in SLABS}
self.count_labels: Dict[str, tk.Label] = {}
self.on_changed = on_changed
self._build()
def _build(self) -> None:
canvas = tk.Canvas(self, highlightthickness=0)
scroll = ttk.Scrollbar(self, orient="vertical", command=canvas.yview)
inner = ttk.Frame(canvas)
inner.bind("<Configure>",
lambda e: canvas.configure(scrollregion=canvas.bbox("all")))
canvas.create_window((0, 0), window=inner, anchor="nw")
canvas.configure(yscrollcommand=scroll.set)
canvas.pack(side="left", fill="both", expand=True)
scroll.pack(side="right", fill="y")
by_tier: Dict[str, List] = {}
for s in SLABS:
by_tier.setdefault(s.tier, []).append(s)
row = 0
for tier in sorted(by_tier, key=lambda t: TIER_ORDER.get(t, 99)):
ttk.Label(
inner, text=TIER_LABEL.get(tier, tier),
font=("TkDefaultFont", 10, "bold"),
).grid(row=row, column=0, columnspan=4, sticky="w", pady=(8, 2))
row += 1
for slab in by_tier[tier]:
ttk.Label(inner, text=slab.ko_label, width=8).grid(
row=row, column=0, sticky="w"
)
ttk.Button(inner, text="", width=2,
command=lambda v=slab.value: self._change(v, -1)
).grid(row=row, column=1)
lbl = ttk.Label(inner, text="0", width=3, anchor="center")
lbl.grid(row=row, column=2)
self.count_labels[slab.value] = lbl
ttk.Button(inner, text="+", width=2,
command=lambda v=slab.value: self._change(v, 1)
).grid(row=row, column=3)
row += 1
def _change(self, v: str, d: int) -> None:
self.counts[v] = max(0, self.counts.get(v, 0) + d)
self.count_labels[v]["text"] = str(self.counts[v])
self.on_changed()
def clear(self) -> None:
for v in self.counts:
self.counts[v] = 0
self.count_labels[v]["text"] = "0"
self.on_changed()
def basket(self) -> List[str]:
out: List[str] = []
for v, n in self.counts.items():
out.extend([v] * n)
return out
# ---------------------------------------------------------------------------
# Main app
# ---------------------------------------------------------------------------
class App(tk.Tk):
def __init__(self) -> None:
super().__init__()
self.title("Sephiria Inventory Optimizer")
self.geometry("1320x820")
self.minsize(1100, 640)
self.slot_var = tk.IntVar(value=34)
self.basket: List[str] = []
self.summary_var = tk.StringVar(value="선택된 석판: 0개")
self.score_var = tk.StringVar(value="score: -")
self.solving = False
self.preview_image: Optional[ImageTk.PhotoImage] = None
self.last_solution = None
self._build()
def _build(self) -> None:
root = ttk.Frame(self, padding=8)
root.pack(fill="both", expand=True)
root.columnconfigure(0, weight=2)
root.columnconfigure(1, weight=3)
root.rowconfigure(0, weight=1)
# LEFT: notebook with manual + screenshot input
nb = ttk.Notebook(root)
nb.grid(row=0, column=0, sticky="nsew", padx=(0, 6))
self.manual = ManualFrame(nb, on_changed=self._on_manual_changed)
nb.add(self.manual, text="수동 선택")
self.screenshot = ScreenshotFrame(nb, on_confirmed=self._on_screenshot_confirmed)
nb.add(self.screenshot, text="스크린샷")
self.nb = nb
# RIGHT
right = ttk.Frame(root)
right.grid(row=0, column=1, sticky="nsew")
right.columnconfigure(0, weight=1)
right.rowconfigure(2, weight=1)
ctl = ttk.LabelFrame(right, text="옵션", padding=8)
ctl.grid(row=0, column=0, sticky="ew")
ttk.Label(ctl, text="슬롯 수").grid(row=0, column=0, sticky="w")
slot_scale = ttk.Scale(ctl, from_=18, to=60, orient="horizontal",
variable=self.slot_var, command=self._on_slot)
slot_scale.grid(row=0, column=1, sticky="ew", padx=6)
ctl.columnconfigure(1, weight=1)
self.slot_label = ttk.Label(ctl, text="34")
self.slot_label.grid(row=0, column=2, sticky="w")
ttk.Button(ctl, text="비우기", command=self._clear_basket).grid(
row=1, column=0, pady=(8, 0), sticky="w"
)
self.solve_btn = ttk.Button(ctl, text="최적 배치 계산", command=self._solve)
self.solve_btn.grid(row=1, column=1, pady=(8, 0), sticky="ew", padx=6)
ttk.Button(ctl, text="이미지 저장…", command=self._save).grid(
row=1, column=2, pady=(8, 0), sticky="e"
)
summary = ttk.Frame(right)
summary.grid(row=1, column=0, sticky="ew", pady=(6, 0))
ttk.Label(summary, textvariable=self.summary_var).pack(side="left")
ttk.Label(summary, textvariable=self.score_var).pack(side="right")
preview_frame = ttk.LabelFrame(right, text="결과 미리보기", padding=4)
preview_frame.grid(row=2, column=0, sticky="nsew", pady=(6, 0))
self.preview = ttk.Label(
preview_frame, anchor="center",
text="좌측에서 석판을 추가하거나 스크린샷을 분석한 뒤 "
"'최적 배치 계산'을 눌러주세요.",
)
self.preview.pack(fill="both", expand=True)
# -------------------------------------------------------- callbacks
def _on_manual_changed(self) -> None:
self.basket = self.manual.basket()
self.summary_var.set(f"선택된 석판: {len(self.basket)}개 (수동)")
def _on_screenshot_confirmed(self, basket: List[str]) -> None:
self.basket = list(basket)
self.summary_var.set(f"선택된 석판: {len(self.basket)}개 (스크린샷)")
self.nb.select(0) # back to first tab? Keep on screenshot tab actually
messagebox.showinfo(
"확정", f"{len(self.basket)}개를 가져왔습니다. '최적 배치 계산'을 누르세요."
)
def _on_slot(self, _evt) -> None:
v = int(round(self.slot_var.get()))
self.slot_var.set(v)
self.slot_label["text"] = str(v)
def _clear_basket(self) -> None:
self.manual.clear()
self.basket = []
self.summary_var.set("선택된 석판: 0개")
def _solve(self) -> None:
if self.solving:
return
if not self.basket:
messagebox.showinfo(
"안내",
"먼저 '수동 선택'에서 석판을 추가하거나, "
"'스크린샷' 탭에서 화면을 분석하고 확정해주세요.",
)
return
slot_num = int(round(self.slot_var.get()))
basket = list(self.basket)
if len(basket) > slot_num:
if not messagebox.askyesno(
"확인",
f"슬롯({slot_num})보다 석판({len(basket)})이 많습니다.\n"
"초과분은 무시하고 계산할까요?",
):
return
self.solving = True
self.solve_btn["state"] = "disabled"
self.score_var.set("score: 계산 중…")
threading.Thread(
target=self._solve_worker, args=(basket, slot_num), daemon=True
).start()
def _solve_worker(self, basket: List[str], slot_num: int) -> None:
try:
sol = solve(basket, slot_num=slot_num, time_limit=4.0)
img = render_solution(sol, download=True)
self.after(0, self._show_result, sol, img)
except Exception as e:
self.after(0, lambda: messagebox.showerror("오류", str(e)))
finally:
self.after(0, self._solve_done)
def _solve_done(self) -> None:
self.solving = False
self.solve_btn["state"] = "normal"
def _show_result(self, sol, img: Image.Image) -> None:
self.last_solution = sol
w = max(self.preview.winfo_width(), 600)
scale = min(1.0, w / img.width)
if scale < 1.0:
img = img.resize((int(img.width * scale), int(img.height * scale)))
self.preview_image = ImageTk.PhotoImage(img)
self.preview.configure(image=self.preview_image, text="")
self.score_var.set(f"score: {sol.score}")
def _save(self) -> None:
if self.last_solution is None:
messagebox.showinfo("안내", "먼저 계산을 실행하세요.")
return
path = filedialog.asksaveasfilename(
defaultextension=".png", filetypes=[("PNG", "*.png")],
initialfile="sephiria_layout.png",
)
if not path:
return
img = render_solution(self.last_solution, download=True)
img.save(path)
messagebox.showinfo("완료", f"저장됨: {path}")
def main() -> int:
try:
app = App()
except tk.TclError as e:
print(f"GUI를 띄울 수 없습니다 ({e}). CLI 모드를 시도하세요:",
file=sys.stderr)
print(" python -m sephiria_inv --help", file=sys.stderr)
return 1
app.mainloop()
return 0
if __name__ == "__main__":
raise SystemExit(main())