Claude Code
156d339308
2026-08-28 第一版漏了這一格，而它剛好就在實測時咬到： 閘判定 pass、Telegram 卻 404（實例上找不到 notify_leo 工作流）， 退路留言貼上票之後**只看得到 pass**——真正的斷點一個字都沒進票。 ⇒ 下一個人會去修閘，而斷點根本不在那裡。 把留言內文抽成 fallback_body()（獨立成一支就是為了測得到）， 測試從「看原始碼有沒有那個字串」改成**真的產一份留言出來檢查**： 斷點原因、原文、身份欄三格都要在。36 條全綠。
212 lines
14 KiB
Bash
Executable File
212 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 催辦員的迴歸測試（inkstone/ISEP#93）
|
||
#
|
||
# 對應票上那四條驗收：
|
||
# 1 撈得出逾期的 milestone，而且訊息是白話的 → A 群
|
||
# 2 沒東西可報時要說「查過了，沒有」 → B 群
|
||
# 3 開一個新 session 要自己跑過一次 → E 群（hook 的注入）
|
||
# 4 Telegram 真的收得到 → D 群只驗得了「發不出去不會安靜」
|
||
# （「leo 手機上真的出現」機器驗不了，那一格由人驗，見 docs/TESTING.md）
|
||
#
|
||
# 另外兩群是這支的**主要失效模式**：
|
||
# C 群 不該叫的不要叫 —— 誤報比漏報更該修：每次開場都被念一串不相干的東西，
|
||
# 人就學會跳過它，那時真的有事也叫不動（本 repo 心法第 2 條）
|
||
# D 群 發不出去不會安靜 —— 「送出成功」跟「送到了」是兩件事
|
||
#
|
||
# 🔴 全程離線：資料走 --fixture、時鐘走 --now、網路用 ISEP_NOTIFY_OFFLINE=1 關掉。
|
||
# 不打 Gitea、不打實例、不留任何測試票。
|
||
set -u
|
||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ROOT="$(cd "$HERE/../.." && pwd)"
|
||
NAG="$ROOT/scripts/isep-nag"
|
||
export NOTIFY_PATH="$ROOT/scripts/isep-notify"
|
||
NOTIFY="$ROOT/scripts/isep-notify"
|
||
HOOK="$ROOT/hooks/overdue-nag-guard.sh"
|
||
|
||
TMP=$(mktemp -d)
|
||
trap 'rm -rf "$TMP"' EXIT
|
||
export ISEP_NOTIFY_STATE_DIR="$TMP/state"
|
||
export ISEP_NAG_STATE_DIR="$TMP/state"
|
||
export ISEP_NOTIFY_OFFLINE=1
|
||
mkdir -p "$ISEP_NOTIFY_STATE_DIR"
|
||
|
||
# 🔴 明著清掉：測試常常跑在一條 subagent 裡，而 hook 刻意放行子 session
|
||
# ⇒ 不清掉的話 E 群會全綠，而且是**假綠**（countdown-guard.test.sh 撞過同一個坑）。
|
||
export CLAUDE_CODE_CHILD_SESSION=
|
||
|
||
PASS=0; FAIL=0
|
||
ok(){ printf ' ✅ %s\n' "$1"; PASS=$((PASS+1)); }
|
||
no(){ printf ' ❌ %s —— %s\n' "$1" "$2"; FAIL=$((FAIL+1)); }
|
||
has(){ printf '%s' "$1" | grep -q -- "$2"; }
|
||
|
||
# 定住的時鐘：2026-08-28 00:00 UTC（＝台北 08:00），跟票上那天同一天
|
||
NOW=$(python3 -c 'import datetime as d;print(int(d.datetime(2026,8,28,0,0,tzinfo=d.timezone.utc).timestamp()))')
|
||
|
||
# ── 測資：票上點名的那五個逾期 milestone，加上一整組「不該叫」的對照 ──────
|
||
cat > "$TMP/full.json" <<'JSON'
|
||
{"milestones": [
|
||
{"repo":"inkstone/arcrun-rag","title":"不必開瀏覽器，也不怕反悔","due_on":"2026-08-24T23:59:59Z","open_issues":2,"closed_issues":0},
|
||
{"repo":"inkstone/Arcrun","title":"App 打開來就是 Arcrun，而且回得去","due_on":"2026-08-24T23:59:59Z","open_issues":3,"closed_issues":1},
|
||
{"repo":"inkstone/ISEP","title":"把管理這條線做對","due_on":"2026-08-24T23:59:59Z","open_issues":14,"closed_issues":1},
|
||
{"repo":"inkstone/Arcrun","title":"AI 問得到內文，不是只查到檔名","due_on":"2026-08-26T23:59:59Z","open_issues":1,"closed_issues":1},
|
||
{"repo":"inkstone/Arcrun","title":"寫進去，就查得到","due_on":"2026-08-26T23:59:59Z","open_issues":1,"closed_issues":6},
|
||
{"repo":"inkstone/ISEP","title":"還沒到期的","due_on":"2026-09-30T23:59:59Z","open_issues":9,"closed_issues":0},
|
||
{"repo":"inkstone/ISEP","title":"逾期但票都關完了","due_on":"2026-08-01T23:59:59Z","open_issues":0,"closed_issues":7},
|
||
{"repo":"inkstone/ISEP","title":"沒設期限的","due_on":null,"open_issues":5,"closed_issues":0}
|
||
],
|
||
"issues": [
|
||
{"number":138,"title":"App 沒辦法說「我要接哪些外部事件」","updated_at":"2026-08-17T00:00:00Z",
|
||
"repository":{"full_name":"inkstone/Arcrun"},"labels":[{"name":"Human"}],"assignees":[]},
|
||
{"number":51,"title":"leo21c 上有一批舊資源還在","updated_at":"2026-08-20T00:00:00Z",
|
||
"repository":{"full_name":"inkstone/InkStoneCo"},"labels":[],"assignees":[{"login":"leo"}]},
|
||
{"number":38,"title":"辨識碼信在 stage 寄不出去","updated_at":"2026-08-12T00:00:00Z",
|
||
"repository":{"full_name":"inkstone/arcrun-rag"},"labels":[{"name":"s/doing"},{"name":"p/high"}],"assignees":[]},
|
||
{"number":999,"title":"昨天才動過的 s/doing——還在做，不要叫","updated_at":"2026-08-27T00:00:00Z",
|
||
"repository":{"full_name":"inkstone/ISEP"},"labels":[{"name":"s/doing"}],"assignees":[]},
|
||
{"number":998,"title":"這是一個 PR，pr-verdict-guard 在管","updated_at":"2026-08-01T00:00:00Z",
|
||
"repository":{"full_name":"inkstone/ISEP"},"labels":[{"name":"s/review"}],"assignees":[],
|
||
"pull_request":{"merged":false}},
|
||
{"number":997,"title":"舊 org 的同一張票（重複，不是兩件事）","updated_at":"2026-08-01T00:00:00Z",
|
||
"repository":{"full_name":"Leo/arcrun-rag"},"labels":[{"name":"s/doing"}],"assignees":[]},
|
||
{"number":996,"title":"沒有狀態標籤，也沒人等——不是誰的事","updated_at":"2026-07-01T00:00:00Z",
|
||
"repository":{"full_name":"inkstone/ISEP"},"labels":[],"assignees":[]}
|
||
]}
|
||
JSON
|
||
echo '{"milestones": [], "issues": []}' > "$TMP/empty.json"
|
||
|
||
R=$(python3 "$NAG" --fixture "$TMP/full.json" --now "$NOW" 2>&1)
|
||
S=$(python3 "$NAG" --fixture "$TMP/full.json" --now "$NOW" --short 2>&1)
|
||
J=$(python3 "$NAG" --fixture "$TMP/full.json" --now "$NOW" --json 2>&1)
|
||
|
||
echo "── A 群：撈得出票上那五個逾期 milestone，而且是白話 ──────────────"
|
||
has "$R" "逾期的 milestone（5 個）" && ok "① 五個都撈到了（票上驗收 1）" \
|
||
|| no "① 五個都撈到了" "實得：$(printf '%s' "$R" | head -3 | tr '\n' ' ')"
|
||
n=0
|
||
for t in "不必開瀏覽器" "App 打開來就是 Arcrun" "把管理這條線做對" "AI 問得到內文" "寫進去，就查得到"; do
|
||
has "$R" "$t" && n=$((n+1))
|
||
done
|
||
[ "$n" = 5 ] && ok "② 五個名字一個不漏" || no "② 五個名字一個不漏" "只對上 $n 個"
|
||
has "$R" "逾期 3 天" && has "$R" "逾期 1 天" \
|
||
&& ok "③ 逾期幾天算得對（08-24→3 天、08-26→1 天）" \
|
||
|| no "③ 逾期幾天算得對" "實得：$(printf '%s' "$R" | grep -o '逾期 [0-9]* 天' | tr '\n' ' ')"
|
||
has "$R" "還有 14 張票沒關" && ok "④ 講得出「還有幾張票沒關」（這才是他要判斷的）" \
|
||
|| no "④ 還有幾張票沒關" "沒出現"
|
||
has "$R" "對你意味著" && ok "⑤ 每一段都講「這對你意味什麼」（白話鐵律）" \
|
||
|| no "⑤ 對你意味著" "整份報告沒有一句在講後果"
|
||
has "$R" "等了 11 天" && ok "⑥ 等 leo 的票標出等了幾天（規則五：不標天數＝不會被催）" \
|
||
|| no "⑥ 等了幾天" "實得：$(printf '%s' "$R" | grep -o '等了 [0-9]* 天' | tr '\n' ' ')"
|
||
has "$S" "回「看」" && ok "⑦ 短版給得出一個可以回的詞（leo 回一個詞就能處理）" \
|
||
|| no "⑦ 可回的詞" "短版：$S"
|
||
[ "$(printf '%s' "$S" | wc -l)" -le 6 ] && ok "⑧ 短版夠短（手機一眼掃完）" \
|
||
|| no "⑧ 短版夠短" "$(printf '%s' "$S" | wc -l) 行"
|
||
|
||
echo "── B 群：沒東西可報時，要說「查過了，沒有」────────────────────────"
|
||
E=$(python3 "$NAG" --fixture "$TMP/empty.json" --now "$NOW" 2>&1)
|
||
ES=$(python3 "$NAG" --fixture "$TMP/empty.json" --now "$NOW" --short 2>&1)
|
||
has "$E" "查過了，沒有" && ok "⑨ 長版說「查過了，沒有」（票上驗收 2）" || no "⑨ 長版" "實得：$E"
|
||
has "$ES" "查過了，沒有" && ok "⑩ 短版也說（安靜跟壞掉長得一模一樣）" || no "⑩ 短版" "實得：$ES"
|
||
BAD=$(TICKET_HOST=http://127.0.0.1:9 GITEA_TOKEN_CLAUDE_CODE= GITEA_TOKEN= \
|
||
CLAUDE_PROJECT_DIR="$TMP" python3 "$NAG" --now "$NOW" 2>&1)
|
||
has "$BAD" "這不等於" && ok "⑪ 撈不到時講「我沒查到」，不講成「沒有事情逾期」" \
|
||
|| no "⑪ 撈不到≠沒事" "實得：$(printf '%s' "$BAD" | tr '\n' ' ')"
|
||
|
||
echo "── C 群：不該叫的不要叫（誤報比漏報更該修）──────────────────────"
|
||
has "$R" "還沒到期的" && no "⑫ 還沒到期的不叫" "叫了" || ok "⑫ 還沒到期的不叫"
|
||
has "$R" "逾期但票都關完了" && no "⑬ 票關完的不叫" "叫了" || ok "⑬ 逾期但票都關完了不叫"
|
||
has "$R" "沒設期限的" && no "⑭ 沒期限的不叫" "叫了" || ok "⑭ 沒設期限的不是節拍器"
|
||
has "$R" "昨天才動過" && no "⑮ 才動過的不叫" "叫了" || ok "⑮ 3 天內動過的 s/doing 不算掉棒"
|
||
has "$R" "pr-verdict-guard 在管" && no "⑯ PR 不重複點名" "叫了" || ok "⑯ PR 交給 pr-verdict-guard，不重複"
|
||
has "$R" "舊 org 的同一張票" && no "⑰ 舊 org 的重複票不叫" "叫了" || ok "⑰ 只看 inkstone，不重複計舊 org"
|
||
has "$R" "不是誰的事" && no "⑱ 沒標籤沒人等的不叫" "叫了" || ok "⑱ 沒標籤又沒人等的不叫"
|
||
printf '%s' "$J" | python3 -c 'import json,sys;json.load(sys.stdin)' 2>/dev/null \
|
||
&& ok "⑲ --json 是合法 JSON（全清單在這裡，長版才封頂）" || no "⑲ --json" "不是合法 JSON"
|
||
|
||
echo "── D 群：發不出去不會安靜（票上驗收 4 的機器可驗那一半）──────────"
|
||
OUT=$(python3 "$NOTIFY" --text "測試訊息 ABC" 2>&1); RC=$?
|
||
[ "$RC" = 1 ] && ok "⑳ 兩條路都沒走通 ⇒ 離開碼 1（不是靜靜地成功）" \
|
||
|| no "⑳ 離開碼" "實得 $RC"
|
||
has "$OUT" "測試訊息 ABC" && ok "㉑ 原文照抄在輸出裡（看到的人才有辦法代發）" \
|
||
|| no "㉑ 原文照抄" "實得：$OUT"
|
||
has "$OUT" "leo 的手機上沒有出現這則" && ok "㉒ 明著說 leo 沒收到（不是「送出成功」）" \
|
||
|| no "㉒ 明著說沒收到" "實得：$OUT"
|
||
# 🔴 「為什麼發不出去」要跟著訊息一起留下來。2026-08-28 第一版漏了這一格：
|
||
# 閘明明 pass，退路留言裡卻只看得到 pass，真正的斷點（實例上沒有那支工作流）
|
||
# 一個字都沒進票 ⇒ 下一個人會去修閘，而斷點根本不在那裡。
|
||
FB=$(python3 - <<'PYX'
|
||
import importlib.machinery, importlib.util, os
|
||
ld = importlib.machinery.SourceFileLoader("n", os.environ["NOTIFY_PATH"])
|
||
sp = importlib.util.spec_from_loader("n", ld); m = importlib.util.module_from_spec(sp)
|
||
ld.exec_module(m)
|
||
fact = {"verdict": "pass", "version": "0.11.0", "guard": "/x/prod-write-guard.sh",
|
||
"confidence": "high", "source": "測試", "reason": "閘放行"}
|
||
body = m.fallback_body("要發的原文 XYZ", fact,
|
||
"通道本身斷了：實例上找不到 notify_leo 工作流")
|
||
ok = ("找不到 notify_leo" in body # 真正的斷點有進票
|
||
and "要發的原文 XYZ" in body # 原文有進票
|
||
and body.splitlines()[0].startswith("【身份】")) # 身份欄在第一行
|
||
print("HAS_WHY" if ok else "NO_WHY:" + body[:120])
|
||
PYX
|
||
)
|
||
[ "$FB" = "HAS_WHY" ] && ok "㉒b 退路留言帶著「為什麼發不出去」，不是只帶閘的判定" \
|
||
|| no "㉒b 退路帶原因" "實得：$FB"
|
||
|
||
echo "── E 群：閘的版本是查得到的事實，不是假設 ────────────────────────"
|
||
G=$(python3 "$NOTIFY" --gate 2>&1)
|
||
has "$G" "這個 session 的通知閘" && ok "㉓ 問得出這個 session 的閘怎麼判" || no "㉓ --gate" "$G"
|
||
has "$G" "ISEP 版本：v" && ok "㉔ 講得出版本（「哪一版」就是這件事的全部）" || no "㉔ 版本" "$G"
|
||
has "$G" "問的是哪一份" && ok "㉕ 講得出問的是哪一個檔（同機可能有兩份）" || no "㉕ 路徑" "$G"
|
||
|
||
# 舊版的閘（v0.10.0 之前）：拿真跡來測——git 歷史裡第一版 prod-write-guard
|
||
OLD="$TMP/oldplug"
|
||
mkdir -p "$OLD/hooks/lib" "$OLD/.claude-plugin"
|
||
if git -C "$ROOT" show c263866:hooks/prod-write-guard.sh > "$OLD/hooks/prod-write-guard.sh" 2>/dev/null; then
|
||
git -C "$ROOT" show c263866:.claude-plugin/plugin.json > "$OLD/.claude-plugin/plugin.json" 2>/dev/null
|
||
OG=$(ISEP_NOTIFY_GUARD="$OLD/hooks/prod-write-guard.sh" python3 "$NOTIFY" --gate 2>&1)
|
||
has "$OG" '\*\*block\*\*' && ok "㉖ 舊版的閘會擋——**用真跡測的**（c263866 那一版）" \
|
||
|| no "㉖ 舊版會擋" "實得：$(printf '%s' "$OG" | tr '\n' ' ')"
|
||
has "$OG" "開一個新的 session" && ok "㉗ 擋的時候說得出解法（更新 plugin ＋ 開新 session）" \
|
||
|| no "㉗ 說得出解法" "$OG"
|
||
OB=$(ISEP_NOTIFY_GUARD="$OLD/hooks/prod-write-guard.sh" python3 "$NOTIFY" --text "被擋的訊息" 2>&1)
|
||
has "$OB" "不繞路" && ok "㉘ 閘說會擋就不送——**不從 python 這條路繞過去**" \
|
||
|| no "㉘ 不繞路" "$OB"
|
||
has "$OB" "被擋的訊息" && ok "㉙ 被擋也照樣把原文吐出來（擋≠靜默）" || no "㉙ 原文" "$OB"
|
||
else
|
||
printf ' ⏭️ SKIP ㉖㉗㉘㉙：拿不到 c263866 的舊版閘（淺 clone？）\n'
|
||
fi
|
||
|
||
NG=$(ISEP_NOTIFY_GUARD="$TMP/沒有這支.sh" python3 "$NOTIFY" --gate 2>&1)
|
||
has "$NG" '\*\*unknown\*\*' && ok "㉚ 找不到閘 ⇒ 誠實回 unknown（不謊稱 pass）" \
|
||
|| no "㉚ unknown" "$NG"
|
||
|
||
touch /dev/null
|
||
STAMP_BAK=""
|
||
if [ -f /tmp/.prod-write-ok ]; then STAMP_BAK=1; fi
|
||
if [ -z "$STAMP_BAK" ]; then
|
||
touch /tmp/.prod-write-ok
|
||
SG=$(python3 "$NOTIFY" --gate 2>&1)
|
||
[ -f /tmp/.prod-write-ok ] \
|
||
&& ok "㉛ 戳記在場時不探針——**探一次就把總管那次授權燒掉了**" \
|
||
|| no "㉛ 不燒戳記" "探針把 /tmp/.prod-write-ok 吃掉了"
|
||
rm -f /tmp/.prod-write-ok
|
||
else
|
||
printf ' ⏭️ SKIP ㉛：這台機器上本來就有 /tmp/.prod-write-ok，不動它\n'
|
||
fi
|
||
|
||
echo "── F 群：開 session 自己跑過一次（票上驗收 3）────────────────────"
|
||
IN='{"hook_event_name":"SessionStart","session_id":"t"}'
|
||
H=$(printf '%s' "$IN" | ISEP_NAG_NO_NOTIFY=1 ISEP_NAG_FIXTURE="$TMP/full.json" \
|
||
ISEP_NAG_TIMEOUT=40 bash "$HOOK" 2>/dev/null)
|
||
printf '%s' "$H" | python3 -c 'import json,sys;json.load(sys.stdin)' 2>/dev/null \
|
||
&& ok "㉜ SessionStart 吐得出合法 JSON（吐壞的會被 harness 丟掉）" \
|
||
|| no "㉜ 合法 JSON" "實得：$(printf '%s' "$H" | head -c 200)"
|
||
has "$H" "additionalContext" && ok "㉝ 真的把報告注入眼前（不是印在誰都看不到的地方）" \
|
||
|| no "㉝ 注入" "$H"
|
||
has "$H" "catch" && no "㉞ 不該有例外堆疊" "有" || ok "㉞ 沒有噴例外"
|
||
CH=$(printf '%s' "$IN" | CLAUDE_CODE_CHILD_SESSION=1 bash "$HOOK" 2>/dev/null)
|
||
[ -z "$CH" ] && ok "㉟ 子 session 不叫（subagent 開工不是 leo 開工）" \
|
||
|| no "㉟ 子 session 不叫" "實得：$CH"
|
||
|
||
echo
|
||
printf '通過 %d 條，失敗 %d 條\n' "$PASS" "$FAIL"
|
||
[ "$FAIL" = 0 ]
|