/** * skill/example 工具的**誠實性**測試(2026-08-13,Arcrun#100/#109 同一族)。 * * 真實事故:`arcrun_get_skill('write_intent_workflow')` 回「skill ... **不存在**」, * 但同一台實例的 `kbdb_search` 撈得到 `page_name: "skill-write_intent_workflow"` *(`entry_type: agent-skill`,`source: installer-seed`)——**查的鍵逐字相符**。 * 真兇是 `if (!resp.ok) return null;`:一個 HTTP 401 被翻譯成「不存在」。 * * 這個謊有實害:instructions 叫 AI 第一步先讀 skill,它照做、收到「不存在」, * 於是結論「這裡沒有 skill」⇒ 去猜/grep repo/上網。 * * 判準:**讀不到(401/403/5xx/連不上)與不存在(KBDB 正常回應但沒這張卡)必須長得不一樣**, * 且讀不到時要交出還走得通的下一步(`kbdb_search`)。 */ import { describe, it, expect } from "vitest"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { Env } from "../../../src/types.js"; import { registerGetSkill, registerListSkills, registerGetExample, } from "../../../src/tools/arcrun_skills_examples.js"; import type { KnowledgeIdentity } from "../../../src/lib/portal-client.js"; const SERVICE: KnowledgeIdentity = { kind: "service" }; const PORTAL: KnowledgeIdentity = { kind: "portal", portal: { session: "sess-abc", display_name: "Leo", role: "admin", libraries: ["*"] }, }; const STALE: KnowledgeIdentity = { kind: "stale" }; type ToolHandler = (args: Record) => Promise<{ content: { type: string; text: string }[]; isError?: boolean; }>; function fakeServer() { const tools = new Map(); const server = { tool: (name: string, _desc: string, _schema: unknown, handler: ToolHandler) => { tools.set(name, handler); }, } as unknown as McpServer; return { server, tools }; } function makeEnv(respond: (url: string) => Response): Env { return { KBDB_INTERNAL_TOKEN: "", KBDB: { fetch: async (url: string) => respond(url) }, } as unknown as Env; } const parse = (res: { content: { text: string }[] }) => JSON.parse(res.content[0].text); describe("arcrun_get_skill — 401 不准講成「不存在」", () => { it("KBDB 回 401 → kbdb_unauthorized,訊息明說「讀不到不是不存在」並給 kbdb_search 這條路", async () => { const { server, tools } = fakeServer(); registerGetSkill(server, makeEnv(() => new Response("unauthorized", { status: 401 })), SERVICE); const res = await tools.get("arcrun_get_skill")!({ slug: "write_intent_workflow" }); expect(res.isError).toBe(true); const body = parse(res); expect(body.error_code).toBe("kbdb_unauthorized"); // 🔴 這句是本次事故的核心:不可以再出現舊版那句「skill "x" 不存在」的結論 expect(body.human_message).not.toMatch(/skill "[^"]*" 不存在/); expect(body.human_message).toContain("這是「讀不到」,不是「不存在」"); expect(body.human_message).toContain("讀不到"); expect(body.human_message).toContain("HTTP 401"); // 還走得通的那條路要交到 AI 手上(實測 kbdb_search 撈得到同一張卡) expect(body.next_actions.join("\n")).toContain("kbdb_search({ q: 'skill-write_intent_workflow' })"); // 並明白禁止它改口 expect(body.next_actions.join("\n")).toContain("不准把這個錯誤回報成"); }); it("以帳密登入的連線收到 401 → additionally 說清楚「不是你的帳號讀不到」", async () => { const { server, tools } = fakeServer(); registerGetSkill(server, makeEnv(() => new Response("unauthorized", { status: 401 })), PORTAL); const body = parse(await tools.get("arcrun_get_skill")!({ slug: "write_recipe" })); expect(body.human_message).toContain("服務內部憑據"); expect(body.human_message).toContain("不代表你的帳號讀不到"); }); it("KBDB 5xx → kbdb_unreachable(連不上,也不是不存在)", async () => { const { server, tools } = fakeServer(); registerGetSkill(server, makeEnv(() => new Response("boom", { status: 503 })), SERVICE); const body = parse(await tools.get("arcrun_get_skill")!({ slug: "whatever" })); expect(body.error_code).toBe("kbdb_unreachable"); expect(body.human_message).toContain("HTTP 503"); }); it("binding 直接爆炸 → 也回 kbdb_unreachable,不吞成 not_found", async () => { const { server, tools } = fakeServer(); const env = { KBDB: { fetch: async () => { throw new Error("kbdb down"); }, }, } as unknown as Env; registerGetSkill(server, env, SERVICE); const body = parse(await tools.get("arcrun_get_skill")!({ slug: "x" })); expect(body.error_code).toBe("kbdb_unreachable"); expect(body.human_message).toContain("kbdb down"); }); it("KBDB 正常回應但真的沒這張卡 → not_found,且說明是「這台實例沒 seed」不是全世界沒有", async () => { const { server, tools } = fakeServer(); registerGetSkill( server, makeEnv(() => new Response(JSON.stringify({ entries: [] }))), SERVICE, ); const body = parse(await tools.get("arcrun_get_skill")!({ slug: "INDEX" })); expect(body.error_code).toBe("not_found"); expect(body.human_message).toContain("KBDB 正常回應"); expect(body.human_message).toContain("skill-INDEX"); expect(body.next_actions.join("\n")).toContain("arcrun_list_skills()"); }); it("卡片真的在 → 照舊回內容(沒改壞正路)", async () => { const { server, tools } = fakeServer(); let seen = ""; registerGetSkill( server, makeEnv((url) => { seen = url; return new Response( JSON.stringify({ entries: [{ id: "e1", page_name: "skill-write_intent_workflow", content: "# 意圖工作流", tags_json: '["skill:core"]' }], }), ); }), SERVICE, ); const body = parse(await tools.get("arcrun_get_skill")!({ slug: "write_intent_workflow" })); expect(body.ok).toBe(true); expect(body.data.content).toBe("# 意圖工作流"); expect(seen).toContain("page_name=skill-write_intent_workflow"); }); it("舊 token(stale)→ identity_missing(誠實要求重新連線,不謊稱找不到)", async () => { const { server, tools } = fakeServer(); registerGetSkill(server, makeEnv(() => new Response("{}")), STALE); const body = parse(await tools.get("arcrun_get_skill")!({ slug: "x" })); expect(body.error_code).toBe("identity_missing"); }); }); describe("arcrun_list_skills / arcrun_get_example — 同一條規則", () => { it("list_skills 撞 401 → kbdb_unauthorized(舊版是 fetch_failed 一句技術話)", async () => { const { server, tools } = fakeServer(); registerListSkills(server, makeEnv(() => new Response("nope", { status: 401 })), SERVICE); const body = parse(await tools.get("arcrun_list_skills")!({})); expect(body.error_code).toBe("kbdb_unauthorized"); expect(body.next_actions.join("\n")).toContain("kbdb_search"); }); it("list_skills 成功時提醒「只用清單裡真的有的 slug」(別再猜 INDEX)", async () => { const { server, tools } = fakeServer(); registerListSkills( server, makeEnv(() => new Response( JSON.stringify({ entries: [{ id: "e1", page_name: "skill-write_recipe", content: "x", tags_json: "[]" }], }), ), ), SERVICE, ); const body = parse(await tools.get("arcrun_list_skills")!({})); expect(body.ok).toBe(true); expect(body.data.count).toBe(1); expect(body.hints.join("\n")).toContain("不要硬猜名字"); }); it("get_example 撞 401 → 同樣是讀不到,不是「example 不存在」", async () => { const { server, tools } = fakeServer(); registerGetExample(server, makeEnv(() => new Response("nope", { status: 401 })), SERVICE); const body = parse(await tools.get("arcrun_get_example")!({ slug: "rag-search-answer" })); expect(body.error_code).toBe("kbdb_unauthorized"); expect(body.human_message).not.toMatch(/example "[^"]*" 不存在/); expect(body.next_actions.join("\n")).toContain("kbdb_search({ q: 'example-rag-search-answer' })"); }); });