Arcrun/mcp/tests/unit/tools/skills-examples-honesty.test.ts

/**
 * skill／example 工具的**誠實性**測試（2026-08-13，Arcrun#100／#109 同一族）。
 *
 * 真實事故：`arcrun_get_skill('write_intent_workflow')` 回「skill ... **不存在**」，
 * 但同一台實例的 `kbdb_search` 撈得到 `page_name: "skill-write_intent_workflow"`
 *（`entry_type: agent-skill`，`source: installer-seed`）——**查的鍵逐字相符**。
 * 真兇是 `if (!resp.ok) return null;`：一個 HTTP 401 被翻譯成「不存在」。
 *
 * 這個謊有實害：instructions 叫 AI 第一步先讀 skill，它照做、收到「不存在」，
 * 於是結論「這裡沒有 skill」⇒ 去猜／grep repo／上網。
 *
 * 判準：**讀不到（401/403/5xx/連不上）與不存在（KBDB 正常回應但沒這張卡）必須長得不一樣**，
 * 且讀不到時要交出還走得通的下一步（`kbdb_search`）。
 */
import { describe, it, expect } from "vitest";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import type { Env } from "../../../src/types.js";
import {
  registerGetSkill,
  registerListSkills,
  registerGetExample,
} from "../../../src/tools/arcrun_skills_examples.js";
import type { KnowledgeIdentity } from "../../../src/lib/portal-client.js";

const SERVICE: KnowledgeIdentity = { kind: "service" };
const PORTAL: KnowledgeIdentity = {
  kind: "portal",
  portal: { session: "sess-abc", display_name: "Leo", role: "admin", libraries: ["*"] },
};
const STALE: KnowledgeIdentity = { kind: "stale" };

type ToolHandler = (args: Record<string, unknown>) => Promise<{
  content: { type: string; text: string }[];
  isError?: boolean;
}>;

function fakeServer() {
  const tools = new Map<string, ToolHandler>();
  const server = {
    tool: (name: string, _desc: string, _schema: unknown, handler: ToolHandler) => {
      tools.set(name, handler);
    },
  } as unknown as McpServer;
  return { server, tools };
}

function makeEnv(respond: (url: string) => Response): Env {
  return {
    KBDB_INTERNAL_TOKEN: "",
    KBDB: { fetch: async (url: string) => respond(url) },
  } as unknown as Env;
}

const parse = (res: { content: { text: string }[] }) => JSON.parse(res.content[0].text);

describe("arcrun_get_skill — 401 不准講成「不存在」", () => {
  it("KBDB 回 401 → kbdb_unauthorized，訊息明說「讀不到不是不存在」並給 kbdb_search 這條路", async () => {
    const { server, tools } = fakeServer();
    registerGetSkill(server, makeEnv(() => new Response("unauthorized", { status: 401 })), SERVICE);
    const res = await tools.get("arcrun_get_skill")!({ slug: "write_intent_workflow" });

    expect(res.isError).toBe(true);
    const body = parse(res);
    expect(body.error_code).toBe("kbdb_unauthorized");
    // 🔴 這句是本次事故的核心：不可以再出現舊版那句「skill "x" 不存在」的結論
    expect(body.human_message).not.toMatch(/skill "[^"]*" 不存在/);
    expect(body.human_message).toContain("這是「讀不到」，不是「不存在」");
    expect(body.human_message).toContain("讀不到");
    expect(body.human_message).toContain("HTTP 401");
    // 還走得通的那條路要交到 AI 手上（實測 kbdb_search 撈得到同一張卡）
    expect(body.next_actions.join("\n")).toContain("kbdb_search({ q: 'skill-write_intent_workflow' })");
    // 並明白禁止它改口
    expect(body.next_actions.join("\n")).toContain("不准把這個錯誤回報成");
  });

  it("以帳密登入的連線收到 401 → additionally 說清楚「不是你的帳號讀不到」", async () => {
    const { server, tools } = fakeServer();
    registerGetSkill(server, makeEnv(() => new Response("unauthorized", { status: 401 })), PORTAL);
    const body = parse(await tools.get("arcrun_get_skill")!({ slug: "write_recipe" }));
    expect(body.human_message).toContain("服務內部憑據");
    expect(body.human_message).toContain("不代表你的帳號讀不到");
  });

  it("KBDB 5xx → kbdb_unreachable（連不上，也不是不存在）", async () => {
    const { server, tools } = fakeServer();
    registerGetSkill(server, makeEnv(() => new Response("boom", { status: 503 })), SERVICE);
    const body = parse(await tools.get("arcrun_get_skill")!({ slug: "whatever" }));
    expect(body.error_code).toBe("kbdb_unreachable");
    expect(body.human_message).toContain("HTTP 503");
  });

  it("binding 直接爆炸 → 也回 kbdb_unreachable，不吞成 not_found", async () => {
    const { server, tools } = fakeServer();
    const env = {
      KBDB: {
        fetch: async () => {
          throw new Error("kbdb down");
        },
      },
    } as unknown as Env;
    registerGetSkill(server, env, SERVICE);
    const body = parse(await tools.get("arcrun_get_skill")!({ slug: "x" }));
    expect(body.error_code).toBe("kbdb_unreachable");
    expect(body.human_message).toContain("kbdb down");
  });

  it("KBDB 正常回應但真的沒這張卡 → not_found，且說明是「這台實例沒 seed」不是全世界沒有", async () => {
    const { server, tools } = fakeServer();
    registerGetSkill(
      server,
      makeEnv(() => new Response(JSON.stringify({ entries: [] }))),
      SERVICE,
    );
    const body = parse(await tools.get("arcrun_get_skill")!({ slug: "INDEX" }));
    expect(body.error_code).toBe("not_found");
    expect(body.human_message).toContain("KBDB 正常回應");
    expect(body.human_message).toContain("skill-INDEX");
    expect(body.next_actions.join("\n")).toContain("arcrun_list_skills()");
  });

  it("卡片真的在 → 照舊回內容（沒改壞正路）", async () => {
    const { server, tools } = fakeServer();
    let seen = "";
    registerGetSkill(
      server,
      makeEnv((url) => {
        seen = url;
        return new Response(
          JSON.stringify({
            entries: [{ id: "e1", page_name: "skill-write_intent_workflow", content: "# 意圖工作流", tags_json: '["skill:core"]' }],
          }),
        );
      }),
      SERVICE,
    );
    const body = parse(await tools.get("arcrun_get_skill")!({ slug: "write_intent_workflow" }));
    expect(body.ok).toBe(true);
    expect(body.data.content).toBe("# 意圖工作流");
    expect(seen).toContain("page_name=skill-write_intent_workflow");
  });

  it("舊 token（stale）→ identity_missing（誠實要求重新連線，不謊稱找不到）", async () => {
    const { server, tools } = fakeServer();
    registerGetSkill(server, makeEnv(() => new Response("{}")), STALE);
    const body = parse(await tools.get("arcrun_get_skill")!({ slug: "x" }));
    expect(body.error_code).toBe("identity_missing");
  });
});

describe("arcrun_list_skills / arcrun_get_example — 同一條規則", () => {
  it("list_skills 撞 401 → kbdb_unauthorized（舊版是 fetch_failed 一句技術話）", async () => {
    const { server, tools } = fakeServer();
    registerListSkills(server, makeEnv(() => new Response("nope", { status: 401 })), SERVICE);
    const body = parse(await tools.get("arcrun_list_skills")!({}));
    expect(body.error_code).toBe("kbdb_unauthorized");
    expect(body.next_actions.join("\n")).toContain("kbdb_search");
  });

  it("list_skills 成功時提醒「只用清單裡真的有的 slug」（別再猜 INDEX）", async () => {
    const { server, tools } = fakeServer();
    registerListSkills(
      server,
      makeEnv(() =>
        new Response(
          JSON.stringify({
            entries: [{ id: "e1", page_name: "skill-write_recipe", content: "x", tags_json: "[]" }],
          }),
        ),
      ),
      SERVICE,
    );
    const body = parse(await tools.get("arcrun_list_skills")!({}));
    expect(body.ok).toBe(true);
    expect(body.data.count).toBe(1);
    expect(body.hints.join("\n")).toContain("不要硬猜名字");
  });

  it("get_example 撞 401 → 同樣是讀不到，不是「example 不存在」", async () => {
    const { server, tools } = fakeServer();
    registerGetExample(server, makeEnv(() => new Response("nope", { status: 401 })), SERVICE);
    const body = parse(await tools.get("arcrun_get_example")!({ slug: "rag-search-answer" }));
    expect(body.error_code).toBe("kbdb_unauthorized");
    expect(body.human_message).not.toMatch(/example "[^"]*" 不存在/);
    expect(body.next_actions.join("\n")).toContain("kbdb_search({ q: 'example-rag-search-answer' })");
  });
});