#!/usr/bin/env bash # search-is-not-proof-guard.sh + kbdb-evidence-stamp.sh 的迴歸測試 # (inkstone/ISEP#30 → comment 4879) # # **離線、不打網路、不碰真庫、每次結果一樣。** 兩支都是純結構判斷,沒有語意判官。 # # 🔴 這支測試的重點在 A 群,不在 B 群。 # ISEP 的第一級缺陷是**誤攔**——leo 2026-08-17 實證:在文字層封路的閘 # 8 次誤攔、0 次正確攔截。所以「該擋有擋到」只是及格, # **「不該擋的一個都沒擋」才是這支閘能不能活下來的條件。** # # 用法:hooks/tests/search-is-not-proof-guard.test.sh [guard 路徑] [stamp 路徑] set -u HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" HOOKS="$(cd "$HERE/.." && pwd)" GUARD="${1:-$HOOKS/search-is-not-proof-guard.sh}" STAMP="${2:-$HOOKS/kbdb-evidence-stamp.sh}" SID="S-SINP-TEST" SEARCH="/tmp/.kbdb-evidence-search-$SID" WALKED="/tmp/.kbdb-evidence-walked-$SID" WARNED="/tmp/.search-is-not-proof-warned-$SID" TMPD="$(mktemp -d)" trap 'rm -rf "$TMPD"; rm -f "$SEARCH" "$WALKED" "$WARNED"' EXIT PASS=0; FAIL=0; N=0 # 真跡:Arcrun#167 comment 4865 裡真的被貼出去的那兩個值 REAL_ID="e_b77f23a1-b467-4aa4-883c-58a051ab8e66" REAL_SRC="kb://小果被AFTEE詐貸.pdf#8" clean() { rm -f "$SEARCH" "$WALKED" "$WARNED"; } # seen <值...> —— 直接寫登記檔(模擬 stamp 已經跑過) seen() { f="/tmp/.kbdb-evidence-$1-$SID"; shift for v in "$@"; do printf '%s\n' "$v" >> "$f"; done } # bash_payload bash_payload() { python3 - "$SID" "$1" <<'PY' import json, sys print(json.dumps({"session_id": sys.argv[1], "hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": {"command": sys.argv[2]}}, ensure_ascii=False)) PY } # agent_payload agent_payload() { python3 - "$SID" "$1" <<'PY' import json, sys print(json.dumps({"session_id": sys.argv[1], "hook_event_name": "PreToolUse", "tool_name": "Agent", "tool_input": {"prompt": sys.argv[2]}}, ensure_ascii=False)) PY } # post_payload [有沒有 tool_response 欄位:yes|no] post_payload() { python3 - "$SID" "$1" "$2" "${3:-yes}" <<'PY' import json, sys sid, tool, resp, has = sys.argv[1:5] d = {"session_id": sid, "hook_event_name": "PostToolUse", "tool_name": tool, "tool_input": {}} if has == "yes": d["tool_response"] = json.loads(resp) print(json.dumps(d, ensure_ascii=False)) PY } t() { want="$1"; desc="$2"; body="$3"; hook="${4:-$GUARD}" N=$((N+1)) err=$(printf '%s' "$body" | bash "$hook" 2>&1 >/dev/null); rc=$? if [ "$rc" -eq "$want" ]; then printf ' ✅ %s\n' "$desc"; PASS=$((PASS+1)) else printf ' ❌ %s —— 期望 exit=%s,實得 exit=%s\n' "$desc" "$want" "$rc" printf '%s\n' "$err" | sed -n '1,6p' | sed 's/^/ /' FAIL=$((FAIL+1)) fi } echo "── A 群:🔴 不該擋(誤攔是這個 repo 的第一級缺陷)────────────────" clean t 0 "① 這個 session 從沒用過 kbdb_search → 完全沒有立場說話" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 -F /tmp/x.md")" clean; seen search "$REAL_ID" "$REAL_SRC" t 0 "② 搜過,但只是自己看——沒有任何往外送的動作(git status)" \ "$(bash_payload "git status --short")" clean; seen search "$REAL_ID" "$REAL_SRC" printf '總管:這一輪的三張票都關了,版本 1.4.59 已上 stage。\n' > "$TMPD/plain.md" t 0 "③ 搜過,也貼了票,但**沒有引用任何搜尋出來的值**(最常見的正常情況)" \ "$(bash_payload "scripts/ticket say inkstone/ISEP#30 -F $TMPD/plain.md")" clean; seen search "$REAL_ID"; seen walked "$REAL_ID" printf '總管:這筆我用 kbdb_get_record 拉過了,id 是 %s\n' "$REAL_ID" > "$TMPD/walked.md" t 0 "④ 搜完**又走了真路徑**才引用 → 這是對的做法,不准罰" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 -F $TMPD/walked.md")" clean; seen search "$REAL_ID" t 0 "⑤ 逃生口:指令裡寫了 search-is-evidence: 的理由" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID' search-is-evidence: 只是引用一筆資料的編號")" clean; seen search "$REAL_ID" t 0 "⑥ 讀那支閘的原始碼(值只是出現在指令裡,不是往外送)" \ "$(bash_payload "cat hooks/search-is-not-proof-guard.sh | grep $REAL_ID")" clean; seen search "$REAL_ID" t 0 "⑦ 撈票/搜票這些讀取動作,不是往外送" \ "$(bash_payload "curl -s -H 'Authorization: token X' 'https://git.uncle6.me/api/v1/repos/inkstone/Arcrun/issues/167'")" clean; seen search "$REAL_SRC" t 0 "⑧ 只在 wiki 裡查東西、然後改自己 repo 的檔(Write 不歸這支管)" \ "$(bash_payload "sed -i '' 's/a/b/' system-dev/wiki/status.md")" clean; seen search "$REAL_ID" printf '【工單】inkstone/ISEP#30\n' > "$TMPD/dispatch-ok.md" t 0 "⑨ 合規派工單(只有票號,沒有引用搜尋值)" \ "$(agent_payload "$(cat "$TMPD/dispatch-ok.md")")" clean; seen search "$REAL_ID" t 2 "⑩ 第一次命中 → 擋(這一發是下一條的前置)" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")" t 0 "⑩' 同一 session 第二次同樣的動作 → 放行,不鬼打牆" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")" echo "── B 群:該擋(真跡重演)──────────────────────────────────────" clean; seen search "$REAL_ID" "$REAL_SRC" cat > "$TMPD/4865.md" <> 提及 >> 小果被AFTEE詐貸" } \`\`\` "metadata_json": "{\"source\":\"$REAL_SRC\",\"library\":\"youlinhsieh-test1\"}" EOF t 2 "⑪ 真跡重演:Arcrun#167 comment 4865 那則退回,原文照貼" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 -F $TMPD/4865.md")" clean; seen search "$REAL_ID" t 2 "⑫ 同一件事走派工單(重派一條線)" \ "$(agent_payload "【工單】inkstone/Arcrun#167 上一版沒修好,實測 $REAL_ID 那塊的出處還是 ../ 開頭")" clean; seen search "$REAL_SRC" t 2 "⑬ 只貼了 kb:// 來源位址(沒貼 id)也認得出來" \ "$(bash_payload "scripts/ticket close inkstone/Arcrun#167 --deliverable '$REAL_SRC'")" clean; seen search "$REAL_ID" t 2 "⑭ 直接打 Gitea API 貼留言(繞開 scripts/ticket)" \ "$(bash_payload "curl -X POST -d '{\"body\":\"$REAL_ID 還是壞的\"}' 'https://git.uncle6.me/api/v1/repos/inkstone/Arcrun/issues/167/comments'")" clean; seen search "$REAL_ID"; seen walked "e_00000000-0000-0000-0000-000000000000" t 2 "⑮ 走過真路徑,但走的是**別筆**資料 → 這一筆仍然只有搜尋當來源" \ "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")" echo "── C 群:訊息本身講不講得出出路 ──────────────────────────────" clean; seen search "$REAL_ID" MSG=$(printf '%s' "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")" | bash "$GUARD" 2>&1 >/dev/null) for want in "D81" "kbdb_graph_neighbors" "search-is-evidence:" "4877" "同一條嗎"; do N=$((N+1)) if printf '%s' "$MSG" | grep -q -- "$want"; then printf ' ✅ 訊息裡有「%s」\n' "$want"; PASS=$((PASS+1)) else printf ' ❌ 訊息裡找不到「%s」\n' "$want"; FAIL=$((FAIL+1)) fi done N=$((N+1)) if printf '%s' "$MSG" | grep -q '`kbdb_get_record`'; then printf ' ✅ 反引號沒有被 shell 吃掉(heredoc 有跳脫)\n'; PASS=$((PASS+1)) else printf ' ❌ 反引號被 shell 展開了\n'; FAIL=$((FAIL+1)) fi echo "── D 群:登記處(stamp)──────────────────────────────────────" clean RESP=$(python3 -c " import json print(json.dumps({'ok':True,'data':{'entries':[{'id':'$REAL_ID','metadata_json':'{\"source\":\"$REAL_SRC\"}'}]}}, ensure_ascii=False)) ") t 0 "⑯ kbdb_search 的回應 → 登記進 search 檔" \ "$(post_payload "mcp__abc__kbdb_search" "$RESP")" "$STAMP" N=$((N+1)) if grep -q "$REAL_ID" "$SEARCH" 2>/dev/null && grep -q "$REAL_SRC" "$SEARCH" 2>/dev/null; then printf ' ✅ 兩類指紋(entry id + kb:// 位址)都抓到了\n'; PASS=$((PASS+1)) else printf ' ❌ 指紋沒抓到:%s\n' "$(cat "$SEARCH" 2>/dev/null)"; FAIL=$((FAIL+1)) fi clean # 🔴 MCP 的回應**不保證**是拆好的物件——常見的是包成 content blocks, # 整份 JSON 縮成一個字串塞在 text 裡(連引號都被跳脫過)。 # stamp 對兩種包裝都要認得,否則它在真的環境裡會靜默失效。 INNER=$(python3 -c " import json print(json.dumps({'ok':True,'data':{'entries':[{'id':'$REAL_ID','metadata_json':'{\"source\":\"$REAL_SRC\"}'}]}}, ensure_ascii=False)) ") WRAPPED=$(python3 -c " import json,sys print(json.dumps({'content':[{'type':'text','text':sys.argv[1]}]}, ensure_ascii=False)) " "$INNER") t 0 "⑯' 同一份回應包成 MCP content blocks(字串裡的 JSON)→ 一樣認得出來" \ "$(post_payload "mcp__abc__kbdb_search" "$WRAPPED")" "$STAMP" N=$((N+1)) if grep -q "$REAL_ID" "$SEARCH" 2>/dev/null && grep -q "$REAL_SRC" "$SEARCH" 2>/dev/null; then printf ' ✅ content blocks 包裝下兩類指紋都抓到了\n'; PASS=$((PASS+1)) else printf ' ❌ 被包裝擋住了:%s\n' "$(cat "$SEARCH" 2>/dev/null)"; FAIL=$((FAIL+1)) fi clean t 0 "⑰ kbdb_get_record 的回應 → 登記進 walked 檔" \ "$(post_payload "mcp__abc__kbdb_get_record" "$RESP")" "$STAMP" N=$((N+1)) if grep -q "$REAL_ID" "$WALKED" 2>/dev/null && [ ! -s "$SEARCH" ]; then printf ' ✅ 走過真路徑的記在 walked,沒有污染 search\n'; PASS=$((PASS+1)) else printf ' ❌ 記錯檔了\n'; FAIL=$((FAIL+1)) fi clean t 0 "⑱ 不是 kbdb 的工具 → 什麼都不做" \ "$(post_payload "Bash" "$RESP")" "$STAMP" clean t 0 "⑲ 搜尋 0 筆(回應讀得到但沒有指紋)→ 正常,不出聲" \ "$(post_payload "mcp__abc__kbdb_search" '{"ok":true,"data":{"entries":[],"count":0}}')" "$STAMP" clean t 2 "⑳ fail-closed:payload 裡沒有 tool_response 欄位 → 出聲說自己失效了" \ "$(post_payload "mcp__abc__kbdb_search" "$RESP" "no")" "$STAMP" clean echo printf '共 %d 條:✅ %d ❌ %d\n' "$N" "$PASS" "$FAIL" [ "$FAIL" -eq 0 ] || exit 1