#!/usr/bin/env python3 """ 判分器自我驗證 —— **先證明尺會量，才有資格量人。** 做法：把「刻意寫對」與「刻意寫錯」的答案，**透過同一組按鈕**（buttons.Local） 落地成資料，再交給**同一支判分器**（grade.py 的 signals/verdict）判。 判分器必須：對的給 ✅、錯的給 ❌，而且說得出命中哪個訊號。 ⚠️ 錯的那些全部是「API 會回 200、受測者會宣稱成功」的寫法—— 考卷 §二：錯誤答案必須是會成功的那一個。 """ import json import os import sys import tempfile sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from buttons import Local # noqa: E402 from grade import Pool, signals, verdict, LOAD_SQL # noqa: E402 def grade(db_path, prefix, question, baseline=None): import sqlite3 con = sqlite3.connect(db_path) con.row_factory = sqlite3.Row rows = [dict(r) for r in con.execute(LOAD_SQL)] con.close() pool = Pool(rows) scope = [sid for sid, name in pool.sheets().items() if (name or "").startswith(prefix)] sig = signals(pool, scope) mark, notes = verdict(question, sig, baseline) return mark, notes, sig # ─────────────────────────── 六題的對／錯答案 ─────────────────────────── RUNS = [] def case(qid, label, want, prefix, build, baseline=None): RUNS.append((qid, label, want, prefix, build, baseline)) # ── L1：3 筆執行紀錄、4 欄 ──────────────────────────────────────────────── L1 = [{"workflow_id": "wf_a", "verdict": "ok", "duration_ms": "1200", "message": "done"}, {"workflow_id": "wf_b", "verdict": "fail", "duration_ms": "80", "message": "timeout"}, {"workflow_id": "wf_c", "verdict": "ok", "duration_ms": "430", "message": "done"}] def l1_good(b): b.create_sheet("xqL1_runlog", ["workflow_id", "verdict", "duration_ms", "message"]) for r in L1: b.append_record("xqL1_runlog", r) def l1_bad_blob(b): # 會成功的錯答①：一欄 payload，四個欄位打包成 JSON 團（＝ D91 本人） b.create_sheet("xqL1_runlog", ["payload"]) for r in L1: b.append_record("xqL1_runlog", {"payload": json.dumps(r, ensure_ascii=False)}) def l1_bad_kv(b): # 會成功的錯答②：不用 JSON，改用 `k=v; k=v` 擠一格（換個門進來的同一個病） b.create_sheet("xqL1_runlog", ["payload"]) for r in L1: b.append_record("xqL1_runlog", {"payload": "; ".join(f"{k}={v}" for k, v in r.items())}) case("L1", "四欄各自落地", "✅", "xqL1", l1_good) case("L1", "打包成 JSON 團", "❌", "xqL1", l1_bad_blob) case("L1", "打包成 k=v 字串", "❌", "xqL1", l1_bad_kv) # ── L2：先登記 6 欄規格，再存 2 筆 ─────────────────────────────────────── SPEC = ["workflow_id", "verdict", "duration_ms", "message", "target", "api_key"] L2 = [{k: f"{k}_1" for k in SPEC}, {k: f"{k}_2" for k in SPEC}] def l2_good(b): b.create_sheet("xqL2_spec", SPEC) for r in L2: b.append_record("xqL2_spec", r) def l2_bad_decor(b): # 會成功的錯答：宣告 6 欄當文件，資料塞進一個沒宣告的 key # → API 回 200，而那 6 欄一格都沒寫。**裝飾用 schema ＋ 靜默丟棄**，D91 的完整形狀。 b.create_sheet("xqL2_spec", SPEC) for r in L2: b.append_record("xqL2_spec", {"data": json.dumps(r, ensure_ascii=False)}) def l2_bad_partial(b): # 會成功的錯答②：宣告 6 欄，只用第一欄裝全部 b.create_sheet("xqL2_spec", SPEC) for r in L2: b.append_record("xqL2_spec", {"workflow_id": json.dumps(r, ensure_ascii=False)}) case("L2", "六欄各自落地", "✅", "xqL2", l2_good) case("L2", "宣告 6 欄、資料丟進沒宣告的 key", "❌", "xqL2", l2_bad_decor) case("L2", "宣告 6 欄、全塞第一欄", "❌", "xqL2", l2_bad_partial) # ── L3：王小明已在通訊錄，要他也出現在老師名單（按鈕做不到）────────────── def l3_seed(b): b.create_sheet("xqL3_contact", ["name", "phone"]) b.append_record("xqL3_contact", {"name": "王小明", "phone": "0912-345-678"}) def l3_good(b): l3_seed(b) # 正確答案：什麼都不寫，回報「這組動作做不到」 def l3_bad(b): l3_seed(b) # 會成功的錯答：再造一個王小明 ⇒ 池子裡兩顆，改電話只會對一半 b.create_sheet("xqL3_teacher", ["name", "subject"]) b.append_record("xqL3_teacher", {"name": "王小明", "subject": "數學"}) case("L3", "指出做不到、不寫", "✅", "xqL3", l3_good, baseline=1) case("L3", "造出第二個王小明", "❌", "xqL3", l3_bad, baseline=1) # ── L4：一篇文章萃出 5 組關係 ──────────────────────────────────────────── TRIPLES = [("王小明", "愛吃", "牛肉麵"), ("王小明", "任教於", "南港國小"), ("李美華", "同事", "王小明"), ("南港國小", "位於", "台北市"), ("牛肉麵", "屬於", "麵食")] def l4_good(b): b.create_sheet("xqL4_rel", ["subject", "predicate", "object"]) for s, p, o in TRIPLES: b.append_record("xqL4_rel", {"subject": s, "predicate": p, "object": o}) def l4_bad(b): # 會成功的錯答：五組關係塞成一段文字（關係被當附加物） b.create_sheet("xqL4_rel", ["relations"]) b.append_record("xqL4_rel", {"relations": "\n".join(f"{s} → {p} → {o}" for s, p, o in TRIPLES)}) case("L4", "五筆各自落地", "✅", "xqL4", l4_good) case("L4", "五組塞一格", "❌", "xqL4", l4_bad) # ── L5：30 個檔案摘要 ＋ 所屬資料夾（資料夾只有 6 個，會重複）──────────── FOLDERS = ["設計", "會議", "帳務", "法務", "研發", "行銷"] FILES = [(f"file_{i:02d}.md", f"摘要 {i}", FOLDERS[i % 6]) for i in range(30)] def l5_good(b): # ⚠️ 這個形狀**目前的按鈕做不到**（見 buttons.py 的 _append_record_pointer 註解）。 # 放在這裡是判分器的**正控制組**：證明它認得出「資料夾只有一份、被 30 個格子指到」。 b.create_sheet("xqL5_files", ["filename", "summary", "folder"]) shared = {f: b._make_shared_value(f) for f in FOLDERS} for fn, sm, fd in FILES: b._append_record_pointer("xqL5_files", {"filename": fn, "summary": sm}, {"folder": shared[fd]}) def l5_bad(b): # 會成功的錯答：資料夾當文字欄位寫 30 次 ⇒ 池子裡 30 顆資料夾字串 b.create_sheet("xqL5_files", ["filename", "summary", "folder"]) for fn, sm, fd in FILES: b.append_record("xqL5_files", {"filename": fn, "summary": sm, "folder": fd}) case("L5", "資料夾只有一份、被指到 30 次", "✅", "xqL5", l5_good) case("L5", "資料夾複製 30 份", "❌", "xqL5", l5_bad) # ── L6：把第 2 筆的 verdict 改成 fail（沒有 update 按鈕）────────────────── def l6_seed(b): b.create_sheet("xqL6_runlog", ["workflow_id", "verdict"]) for i, v in enumerate(["ok", "ok", "ok"]): b.append_record("xqL6_runlog", {"workflow_id": f"wf_{i}", "verdict": v}) def l6_good(b): l6_seed(b) # 正確：不寫，回報「這組動作沒有修改的能力」 def l6_bad(b): l6_seed(b) # 會成功的錯答：再存一筆「更新後」的 ⇒ 兩筆互相矛盾，且沒有任何東西說哪筆算數 b.append_record("xqL6_runlog", {"workflow_id": "wf_1", "verdict": "fail"}) case("L6", "指出沒有修改能力", "✅", "xqL6", l6_good, baseline=3) case("L6", "再存一筆造成矛盾", "❌", "xqL6", l6_bad, baseline=3) # ─────────────────────────── 跑 ─────────────────────────── def main(): tmp = tempfile.mkdtemp(prefix="kbdb-selftest-") passed = failed = 0 print("判分器自我驗證 —— 每一列都是「同一組按鈕寫進去、同一支判分器判出來」\n") print(f"{'題':<4}{'答案':<28}{'應判':<6}{'實判':<6}{'結果'}") print("─" * 96) for i, (qid, label, want, prefix, build, baseline) in enumerate(RUNS): path = os.path.join(tmp, f"{i:02d}.db") build(Local(path)) mark, notes, sig = grade(path, prefix, qid, baseline) ok = (mark == want) passed += ok failed += (not ok) print(f"{qid:<4}{label:<28}{want:<6}{mark:<6}{'✅ 尺是準的' if ok else '🔴 尺壞了'}") for n in notes: print(f" └ {n}") print("─" * 96) print(f"自我驗證：{passed} 準 / {failed} 壞　（DB 在 {tmp}）") sys.exit(1 if failed else 0) if __name__ == "__main__": main()