Leo
7de1ad6be6
擋的是**證據的出處**，不是措辭。
判準（兩個條件同時成立才響）：
① 要送出去的那份東西（票上的留言／派工單）裡，貼了一個值
——entry id 或 `kb://` 來源位址——而它這個 session 只在
`kbdb_search` 的回應裡出現過
② 這個 session 從沒用產品檢索路徑（graph／wiki 內容／query）拿過那筆
為什麼不比對措辭：驗收條件第 4 條寫死不准，而 leo 2026-08-17 已經證偽過
文字層——那天 8 次誤攔、0 次正確攔截，且方向穩定：紅線寫得越細，命中
關鍵字的機率越高 ⇒ 那些閘在懲罰謹慎。值跟動作一樣有限且可枚舉，措辭不是。
🔴 `kbdb_search` 一點都沒有變難用：查 wiki、找 record_id、看某筆在不在，
全部照放。它只在「把搜尋輸出貼出去當產品檢索壞掉的證據」那一刻才響。
實測 31 條，A 群（不該擋）11 條、B 群（該擋）5 條、C 群訊息 6 條、
D 群登記處 9 條。真跡重演＝inkstone/Arcrun#167 comment 4865 那則退回，
原文照貼會被擋；走過一次真路徑之後同一份留言就放行。
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
255 lines
10 KiB
Bash
Executable File
255 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# search-is-not-proof-guard.sh ＋ kbdb-evidence-stamp.sh 的迴歸測試
|
||
# （inkstone/ISEP#30 → comment 4879）
|
||
#
|
||
# **離線、不打網路、不碰真庫、每次結果一樣。** 兩支都是純結構判斷，沒有語意判官。
|
||
#
|
||
# 🔴 這支測試的重點在 A 群，不在 B 群。
|
||
# ISEP 的第一級缺陷是**誤攔**——leo 2026-08-17 實證：在文字層封路的閘
|
||
# 8 次誤攔、0 次正確攔截。所以「該擋有擋到」只是及格，
|
||
# **「不該擋的一個都沒擋」才是這支閘能不能活下來的條件。**
|
||
#
|
||
# 用法：hooks/tests/search-is-not-proof-guard.test.sh [guard 路徑] [stamp 路徑]
|
||
|
||
set -u
|
||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
HOOKS="$(cd "$HERE/.." && pwd)"
|
||
GUARD="${1:-$HOOKS/search-is-not-proof-guard.sh}"
|
||
STAMP="${2:-$HOOKS/kbdb-evidence-stamp.sh}"
|
||
|
||
SID="S-SINP-TEST"
|
||
SEARCH="/tmp/.kbdb-evidence-search-$SID"
|
||
WALKED="/tmp/.kbdb-evidence-walked-$SID"
|
||
WARNED="/tmp/.search-is-not-proof-warned-$SID"
|
||
TMPD="$(mktemp -d)"
|
||
trap 'rm -rf "$TMPD"; rm -f "$SEARCH" "$WALKED" "$WARNED"' EXIT
|
||
|
||
PASS=0; FAIL=0; N=0
|
||
|
||
# 真跡：Arcrun#167 comment 4865 裡真的被貼出去的那兩個值
|
||
REAL_ID="e_b77f23a1-b467-4aa4-883c-58a051ab8e66"
|
||
REAL_SRC="kb://小果被AFTEE詐貸.pdf#8"
|
||
|
||
clean() { rm -f "$SEARCH" "$WALKED" "$WARNED"; }
|
||
|
||
# seen <search|walked> <值...> —— 直接寫登記檔（模擬 stamp 已經跑過）
|
||
seen() {
|
||
f="/tmp/.kbdb-evidence-$1-$SID"; shift
|
||
for v in "$@"; do printf '%s\n' "$v" >> "$f"; done
|
||
}
|
||
|
||
# bash_payload <command>
|
||
bash_payload() {
|
||
python3 - "$SID" "$1" <<'PY'
|
||
import json, sys
|
||
print(json.dumps({"session_id": sys.argv[1], "hook_event_name": "PreToolUse",
|
||
"tool_name": "Bash", "tool_input": {"command": sys.argv[2]}},
|
||
ensure_ascii=False))
|
||
PY
|
||
}
|
||
|
||
# agent_payload <prompt>
|
||
agent_payload() {
|
||
python3 - "$SID" "$1" <<'PY'
|
||
import json, sys
|
||
print(json.dumps({"session_id": sys.argv[1], "hook_event_name": "PreToolUse",
|
||
"tool_name": "Agent", "tool_input": {"prompt": sys.argv[2]}},
|
||
ensure_ascii=False))
|
||
PY
|
||
}
|
||
|
||
# post_payload <tool_name> <tool_response JSON 字串> [有沒有 tool_response 欄位:yes|no]
|
||
post_payload() {
|
||
python3 - "$SID" "$1" "$2" "${3:-yes}" <<'PY'
|
||
import json, sys
|
||
sid, tool, resp, has = sys.argv[1:5]
|
||
d = {"session_id": sid, "hook_event_name": "PostToolUse",
|
||
"tool_name": tool, "tool_input": {}}
|
||
if has == "yes":
|
||
d["tool_response"] = json.loads(resp)
|
||
print(json.dumps(d, ensure_ascii=False))
|
||
PY
|
||
}
|
||
|
||
t() {
|
||
want="$1"; desc="$2"; body="$3"; hook="${4:-$GUARD}"
|
||
N=$((N+1))
|
||
err=$(printf '%s' "$body" | bash "$hook" 2>&1 >/dev/null); rc=$?
|
||
if [ "$rc" -eq "$want" ]; then
|
||
printf ' ✅ %s\n' "$desc"; PASS=$((PASS+1))
|
||
else
|
||
printf ' ❌ %s —— 期望 exit=%s，實得 exit=%s\n' "$desc" "$want" "$rc"
|
||
printf '%s\n' "$err" | sed -n '1,6p' | sed 's/^/ /'
|
||
FAIL=$((FAIL+1))
|
||
fi
|
||
}
|
||
|
||
echo "── A 群：🔴 不該擋（誤攔是這個 repo 的第一級缺陷）────────────────"
|
||
|
||
clean
|
||
t 0 "① 這個 session 從沒用過 kbdb_search → 完全沒有立場說話" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 -F /tmp/x.md")"
|
||
|
||
clean; seen search "$REAL_ID" "$REAL_SRC"
|
||
t 0 "② 搜過，但只是自己看——沒有任何往外送的動作（git status）" \
|
||
"$(bash_payload "git status --short")"
|
||
|
||
clean; seen search "$REAL_ID" "$REAL_SRC"
|
||
printf '總管：這一輪的三張票都關了，版本 1.4.59 已上 stage。\n' > "$TMPD/plain.md"
|
||
t 0 "③ 搜過，也貼了票，但**沒有引用任何搜尋出來的值**（最常見的正常情況）" \
|
||
"$(bash_payload "scripts/ticket say inkstone/ISEP#30 -F $TMPD/plain.md")"
|
||
|
||
clean; seen search "$REAL_ID"; seen walked "$REAL_ID"
|
||
printf '總管：這筆我用 kbdb_get_record 拉過了，id 是 %s\n' "$REAL_ID" > "$TMPD/walked.md"
|
||
t 0 "④ 搜完**又走了真路徑**才引用 → 這是對的做法，不准罰" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 -F $TMPD/walked.md")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
t 0 "⑤ 逃生口：指令裡寫了 search-is-evidence: 的理由" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID' search-is-evidence: 只是引用一筆資料的編號")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
t 0 "⑥ 讀那支閘的原始碼（值只是出現在指令裡，不是往外送）" \
|
||
"$(bash_payload "cat hooks/search-is-not-proof-guard.sh | grep $REAL_ID")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
t 0 "⑦ 撈票／搜票這些讀取動作，不是往外送" \
|
||
"$(bash_payload "curl -s -H 'Authorization: token X' 'https://git.uncle6.me/api/v1/repos/inkstone/Arcrun/issues/167'")"
|
||
|
||
clean; seen search "$REAL_SRC"
|
||
t 0 "⑧ 只在 wiki 裡查東西、然後改自己 repo 的檔（Write 不歸這支管）" \
|
||
"$(bash_payload "sed -i '' 's/a/b/' system-dev/wiki/status.md")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
printf '【工單】inkstone/ISEP#30\n' > "$TMPD/dispatch-ok.md"
|
||
t 0 "⑨ 合規派工單（只有票號，沒有引用搜尋值）" \
|
||
"$(agent_payload "$(cat "$TMPD/dispatch-ok.md")")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
t 2 "⑩ 第一次命中 → 擋（這一發是下一條的前置）" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")"
|
||
t 0 "⑩' 同一 session 第二次同樣的動作 → 放行，不鬼打牆" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")"
|
||
|
||
echo "── B 群：該擋（真跡重演）──────────────────────────────────────"
|
||
|
||
clean; seen search "$REAL_ID" "$REAL_SRC"
|
||
cat > "$TMPD/4865.md" <<EOF
|
||
# 🔴【總管複驗・退回】兩個缺陷都還在——我用 MCP 實打的
|
||
|
||
\`\`\`
|
||
\$ kbdb_search(q="小果 AFTEE 詐貸") → count 5
|
||
\`\`\`
|
||
|
||
\`\`\`json
|
||
{
|
||
"id": "$REAL_ID",
|
||
"content": "### 出處\n- \`../小果被AFTEE詐貸.pdf\` >> 提及 >> 小果被AFTEE詐貸"
|
||
}
|
||
\`\`\`
|
||
|
||
"metadata_json": "{\"source\":\"$REAL_SRC\",\"library\":\"youlinhsieh-test1\"}"
|
||
EOF
|
||
t 2 "⑪ 真跡重演：Arcrun#167 comment 4865 那則退回，原文照貼" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 -F $TMPD/4865.md")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
t 2 "⑫ 同一件事走派工單（重派一條線）" \
|
||
"$(agent_payload "【工單】inkstone/Arcrun#167
|
||
上一版沒修好，實測 $REAL_ID 那塊的出處還是 ../ 開頭")"
|
||
|
||
clean; seen search "$REAL_SRC"
|
||
t 2 "⑬ 只貼了 kb:// 來源位址（沒貼 id）也認得出來" \
|
||
"$(bash_payload "scripts/ticket close inkstone/Arcrun#167 --deliverable '$REAL_SRC'")"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
t 2 "⑭ 直接打 Gitea API 貼留言（繞開 scripts/ticket）" \
|
||
"$(bash_payload "curl -X POST -d '{\"body\":\"$REAL_ID 還是壞的\"}' 'https://git.uncle6.me/api/v1/repos/inkstone/Arcrun/issues/167/comments'")"
|
||
|
||
clean; seen search "$REAL_ID"; seen walked "e_00000000-0000-0000-0000-000000000000"
|
||
t 2 "⑮ 走過真路徑，但走的是**別筆**資料 → 這一筆仍然只有搜尋當來源" \
|
||
"$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")"
|
||
|
||
echo "── C 群：訊息本身講不講得出出路 ──────────────────────────────"
|
||
|
||
clean; seen search "$REAL_ID"
|
||
MSG=$(printf '%s' "$(bash_payload "scripts/ticket say inkstone/Arcrun#167 --body '$REAL_ID'")" | bash "$GUARD" 2>&1 >/dev/null)
|
||
for want in "D81" "kbdb_graph_neighbors" "search-is-evidence:" "4877" "同一條嗎"; do
|
||
N=$((N+1))
|
||
if printf '%s' "$MSG" | grep -q -- "$want"; then
|
||
printf ' ✅ 訊息裡有「%s」\n' "$want"; PASS=$((PASS+1))
|
||
else
|
||
printf ' ❌ 訊息裡找不到「%s」\n' "$want"; FAIL=$((FAIL+1))
|
||
fi
|
||
done
|
||
N=$((N+1))
|
||
if printf '%s' "$MSG" | grep -q '`kbdb_get_record`'; then
|
||
printf ' ✅ 反引號沒有被 shell 吃掉（heredoc 有跳脫）\n'; PASS=$((PASS+1))
|
||
else
|
||
printf ' ❌ 反引號被 shell 展開了\n'; FAIL=$((FAIL+1))
|
||
fi
|
||
|
||
echo "── D 群：登記處（stamp）──────────────────────────────────────"
|
||
|
||
clean
|
||
RESP=$(python3 -c "
|
||
import json
|
||
print(json.dumps({'ok':True,'data':{'entries':[{'id':'$REAL_ID','metadata_json':'{\"source\":\"$REAL_SRC\"}'}]}}, ensure_ascii=False))
|
||
")
|
||
t 0 "⑯ kbdb_search 的回應 → 登記進 search 檔" \
|
||
"$(post_payload "mcp__abc__kbdb_search" "$RESP")" "$STAMP"
|
||
N=$((N+1))
|
||
if grep -q "$REAL_ID" "$SEARCH" 2>/dev/null && grep -q "$REAL_SRC" "$SEARCH" 2>/dev/null; then
|
||
printf ' ✅ 兩類指紋（entry id ＋ kb:// 位址）都抓到了\n'; PASS=$((PASS+1))
|
||
else
|
||
printf ' ❌ 指紋沒抓到：%s\n' "$(cat "$SEARCH" 2>/dev/null)"; FAIL=$((FAIL+1))
|
||
fi
|
||
|
||
clean
|
||
# 🔴 MCP 的回應**不保證**是拆好的物件——常見的是包成 content blocks，
|
||
# 整份 JSON 縮成一個字串塞在 text 裡（連引號都被跳脫過）。
|
||
# stamp 對兩種包裝都要認得，否則它在真的環境裡會靜默失效。
|
||
INNER=$(python3 -c "
|
||
import json
|
||
print(json.dumps({'ok':True,'data':{'entries':[{'id':'$REAL_ID','metadata_json':'{\"source\":\"$REAL_SRC\"}'}]}}, ensure_ascii=False))
|
||
")
|
||
WRAPPED=$(python3 -c "
|
||
import json,sys
|
||
print(json.dumps({'content':[{'type':'text','text':sys.argv[1]}]}, ensure_ascii=False))
|
||
" "$INNER")
|
||
t 0 "⑯' 同一份回應包成 MCP content blocks（字串裡的 JSON）→ 一樣認得出來" \
|
||
"$(post_payload "mcp__abc__kbdb_search" "$WRAPPED")" "$STAMP"
|
||
N=$((N+1))
|
||
if grep -q "$REAL_ID" "$SEARCH" 2>/dev/null && grep -q "$REAL_SRC" "$SEARCH" 2>/dev/null; then
|
||
printf ' ✅ content blocks 包裝下兩類指紋都抓到了\n'; PASS=$((PASS+1))
|
||
else
|
||
printf ' ❌ 被包裝擋住了：%s\n' "$(cat "$SEARCH" 2>/dev/null)"; FAIL=$((FAIL+1))
|
||
fi
|
||
|
||
clean
|
||
t 0 "⑰ kbdb_get_record 的回應 → 登記進 walked 檔" \
|
||
"$(post_payload "mcp__abc__kbdb_get_record" "$RESP")" "$STAMP"
|
||
N=$((N+1))
|
||
if grep -q "$REAL_ID" "$WALKED" 2>/dev/null && [ ! -s "$SEARCH" ]; then
|
||
printf ' ✅ 走過真路徑的記在 walked，沒有污染 search\n'; PASS=$((PASS+1))
|
||
else
|
||
printf ' ❌ 記錯檔了\n'; FAIL=$((FAIL+1))
|
||
fi
|
||
|
||
clean
|
||
t 0 "⑱ 不是 kbdb 的工具 → 什麼都不做" \
|
||
"$(post_payload "Bash" "$RESP")" "$STAMP"
|
||
|
||
clean
|
||
t 0 "⑲ 搜尋 0 筆（回應讀得到但沒有指紋）→ 正常，不出聲" \
|
||
"$(post_payload "mcp__abc__kbdb_search" '{"ok":true,"data":{"entries":[],"count":0}}')" "$STAMP"
|
||
|
||
clean
|
||
t 2 "⑳ fail-closed：payload 裡沒有 tool_response 欄位 → 出聲說自己失效了" \
|
||
"$(post_payload "mcp__abc__kbdb_search" "$RESP" "no")" "$STAMP"
|
||
|
||
clean
|
||
echo
|
||
printf '共 %d 條：✅ %d ❌ %d\n' "$N" "$PASS" "$FAIL"
|
||
[ "$FAIL" -eq 0 ] || exit 1
|