#!/usr/bin/env bash # mainline-idle-guard.sh 的迴歸測試(inkstone/ISEP#30 → comment 4851) # # 兩個方向都要有證據: # A 群「不該擋」——回答問題、輕量回合、任何一種推進證據、未達門檻、響過之後退讓 # B 群「該擋」 ——連續乾回合(有動作、零推進)踩到門檻 # C 群「訊息承諾的出路真的走得通」——擋完歸零、門檻加倍 # # 用法:hooks/tests/mainline-idle-guard.test.sh [hooks/mainline-idle-guard.sh 的路徑] # 🔴 全程在乾淨的 TMP 底下造假 transcript 與假狀態檔,跑完自己清; # 不碰任何真 repo、不打網路、不寫 /tmp 的正式狀態檔(走 MAINLINE_IDLE_STATE_DIR)。 set -u HOOK="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/mainline-idle-guard.sh}" TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT PASS=0; FAIL=0; N=0 # append_turn ... # block spec:`T:<文字>` = assistant 文字段;`U:<工具名>` = tool_use; # `B:<指令>` = Bash tool_use(帶 command) append_turn() { python3 - "$@" <<'PY' import json, sys path, specs = sys.argv[1], sys.argv[2:] blocks = [] for s in specs: k, v = s.split(":", 1) if k == "T": blocks.append({"type": "text", "text": v}) elif k == "B": blocks.append({"type": "tool_use", "name": "Bash", "id": "t", "input": {"command": v}}) else: blocks.append({"type": "tool_use", "name": v, "id": "t", "input": {}}) with open(path, "a") as f: f.write(json.dumps({"type": "user", "message": {"content": [{"type": "text", "text": "繼續"}]}}, ensure_ascii=False) + "\n") f.write(json.dumps({"type": "assistant", "message": {"content": blocks}}, ensure_ascii=False) + "\n") PY } # fire <期望 exit> <說明> [extra-json] fire() { sid="$1"; tr="$2"; want="$3"; desc="$4"; extra="${5:-}" N=$((N+1)) payload=$(printf '{"session_id":"%s","transcript_path":"%s"%s}' "$sid" "$tr" "$extra") out=$(printf '%s' "$payload" \ | CLAUDE_PROJECT_DIR="${PROJ_OVERRIDE:-$TMP}" \ MAINLINE_IDLE_STATE_DIR="$TMP" bash "$HOOK" 2>&1); rc=$? if [ "$rc" -eq "$want" ]; then printf ' ✅ %s\n' "$desc"; PASS=$((PASS+1)) else printf ' ❌ %s —— 期望 exit=%s,實得 exit=%s\n' "$desc" "$want" "$rc" printf '%s\n' "$out" | sed -n '1,6p' | sed 's/^/ /' FAIL=$((FAIL+1)) fi } # 造一個新 session 的 transcript,並吃掉第一次的 baseline new_session() { sid="$1"; tr="$TMP/$sid.jsonl"; : > "$tr" append_turn "$tr" 'U:Read' printf '{"session_id":"%s","transcript_path":"%s"}' "$sid" "$tr" \ | CLAUDE_PROJECT_DIR="${PROJ_OVERRIDE:-$TMP}" MAINLINE_IDLE_STATE_DIR="$TMP" \ bash "$HOOK" >/dev/null 2>&1 echo "$tr" } DRY=(U:Read U:Grep B:git\ status U:Read) # 4 個動作、零推進 = 乾回合 echo "── A 群:不該擋 ────────────────────────────────────────────" tr=$(new_session s-light) fire s-light "$tr" 0 "① 第一次見到這個 session(立基準,不評分)" # 票上第 3 條驗收:回答一個不需要派工的問題 ⇒ 放行(且**不計數**) for i in 1 2 3 4 5; do append_turn "$tr" 'U:Read' 'U:Grep' "T:查了一下,答案是 X。" fire s-light "$tr" 0 "② 回答問題的輕量回合(2 個動作)第 $i 次——連 5 次都不該擋" done tr=$(new_session s-dispatch) for i in 1 2 3 4; do append_turn "$tr" "${DRY[@]}" 'U:Agent' fire s-dispatch "$tr" 0 "③ 有派工(Agent)第 $i 次 ⇒ 永遠歸零" done tr=$(new_session s-send) for i in 1 2 3 4; do append_turn "$tr" "${DRY[@]}" 'U:SendMessage' fire s-send "$tr" 0 "④ 驅動既有 agent(SendMessage)第 $i 次" done tr=$(new_session s-write) for i in 1 2 3 4; do append_turn "$tr" "${DRY[@]}" 'U:Edit' fire s-write "$tr" 0 "⑤ 有產出(Edit)第 $i 次" done tr=$(new_session s-ticket) for i in 1 2 3 4; do append_turn "$tr" 'U:Read' 'U:Grep' 'B:scripts/ticket say inkstone/ISEP#30 -F /tmp/x.md' fire s-ticket "$tr" 0 "⑥ 寫回票(scripts/ticket say)第 $i 次" done tr=$(new_session s-mcp) for i in 1 2 3 4; do append_turn "$tr" "${DRY[@]}" 'U:mcp__x__kbdb_create_record' fire s-mcp "$tr" 0 "⑦ 寫進外部系統(kbdb_create_record)第 $i 次" done tr=$(new_session s-under) for i in 1 2 3; do append_turn "$tr" "${DRY[@]}"; fire s-under "$tr" 0 "⑧ 第 $i 個乾回合——未達門檻(4),不擋" done tr=$(new_session s-active) append_turn "$tr" "${DRY[@]}" fire s-active "$tr" 0 "⑩ stop_hook_active(已被別的 Stop 閘擋過)⇒ 不計數" ',"stop_hook_active":true' append_turn "$tr" "${DRY[@]}" fire s-active "$tr" 0 "⑪ 同上,第 2 次仍不擋(不會被別的閘的擋下推著累積)" ',"stop_hook_active":true' tr=$(new_session s-dup) append_turn "$tr" "${DRY[@]}" fire s-dup "$tr" 0 "⑫ 同一個回合被叫兩次(transcript 沒長)——第 1 次" fire s-dup "$tr" 0 "⑬ 同上第 2 次:不重複計數" fire s-dup "$tr" 0 "⑭ 同上第 3 次:仍不擋(重複呼叫不該累積成擋下)" fire s-bad "$TMP/does-not-exist.jsonl" 0 "⑮ 讀不到 transcript ⇒ 放行(不亂擋)" echo "── B 群:該擋 ──────────────────────────────────────────────" tr=$(new_session s-idle) for i in 1 2 3; do append_turn "$tr" "${DRY[@]}"; fire s-idle "$tr" 0 "⑯ 乾回合 $i/4" done append_turn "$tr" "${DRY[@]}"; fire s-idle "$tr" 2 "⑰ 乾回合 4/4 ⇒ **擋下**" tr=$(new_session s-idle2) for i in 1 2 3; do append_turn "$tr" 'U:Read' 'U:Read' 'U:Read'; fire s-idle2 "$tr" 0 "⑱ 純唯讀乾回合 $i/4" done append_turn "$tr" 'U:Bash' 'U:Bash' 'U:Read'; fire s-idle2 "$tr" 2 "⑲ 唯讀 Bash 也算乾回合 ⇒ **擋下**" echo "── C 群:響過就退讓(訊息裡承諾的出路真的存在)───────────────" # 接續 s-idle:它剛在 ⑰ 被擋過 ⇒ 計數歸零、門檻 4→8 tr="$TMP/s-idle.jsonl" for i in 1 2 3 4 5 6 7; do append_turn "$tr" "${DRY[@]}" fire s-idle "$tr" 0 "⑳ 擋過之後第 $i 個乾回合——門檻已加倍(4→8),不該再響" done append_turn "$tr" "${DRY[@]}" fire s-idle "$tr" 2 "㉑ 第 8 個乾回合 ⇒ 再擋一次(門檻 8)" # 量測用的旋鈕要真的有效(檔頭那份實測靠它重跑) export MAINLINE_IDLE_THRESHOLD=2 tr=$(new_session s-th) append_turn "$tr" "${DRY[@]}"; fire s-th "$tr" 0 "㉒ MAINLINE_IDLE_THRESHOLD=2:第 1 個乾回合" append_turn "$tr" "${DRY[@]}"; fire s-th "$tr" 2 "㉓ 同上第 2 個 ⇒ 擋(證明門檻可被量測腳本改)" unset MAINLINE_IDLE_THRESHOLD echo "── D 群:工作區狀態變了就算推進(heredoc/sed 改檔吃得到)──────" REPO="$TMP/repo"; mkdir -p "$REPO" git -C "$REPO" init -q 2>/dev/null git -C "$REPO" config user.email t@t; git -C "$REPO" config user.name t echo a > "$REPO/a.txt"; git -C "$REPO" add -A; git -C "$REPO" commit -qm init PROJ_OVERRIDE="$REPO" tr=$(new_session s-git) for i in 1 2 3; do append_turn "$tr" "${DRY[@]}"; fire s-git "$tr" 0 "㉔ 乾回合 $i/4(工作區沒動)" done # 用 heredoc/sed 改檔——工具名是 Bash、指令不在白名單,只有狀態指紋看得出來 echo "changed by heredoc" >> "$REPO/a.txt" append_turn "$tr" "${DRY[@]}"; fire s-git "$tr" 0 "㉕ 第 4 個乾回合,但工作區變了 ⇒ 算推進,不擋" for i in 1 2 3; do append_turn "$tr" "${DRY[@]}"; fire s-git "$tr" 0 "㉖ 之後乾回合 $i/4(證明上一格真的歸零過)" done append_turn "$tr" "${DRY[@]}"; fire s-git "$tr" 2 "㉗ 乾回合 4/4 ⇒ **擋下**" PROJ_OVERRIDE="" echo printf '結果:%s 通過 / %s 失敗(共 %s 條)\n' "$PASS" "$FAIL" "$N" [ "$FAIL" -eq 0 ] || exit 1