Arcrun/kbdb/tests/search-deprecated-filter.test.ts

// daemon-beta t24（總管 0.971 親復現、t11 斷點①②）——/entries/search 服務端濾 deprecated。
// 背景：rag_takedown_direct 下架只把 metadata_json.status 標 'deprecated'（軟刪，append-only，
// KBDB 表不變鐵律），從不砍列、也不刪 Vectorize 向量。過去唯一的濾層在
// cypher-executor/src/routes/portal-data.ts（客端治標，Arcrun#46），沒部署 rag_chat 的實例
// （如 leo21c）等於完全沒濾——MCP/raw /entries/search 直接把已下架內容當現役吐出。semantic
// 甚至最高分照吐（t11 實測 0.971）。本測試覆蓋三案：keyword 濾、semantic 濾＋補位、
// include_deprecated 開關。
//
// 測試手法同 search-source-and-score.test.ts：fake D1 捕 SQL 形狀 + getEntry 依 id 回可控
// metadata_json；mock VECTORIZE 捕 query opts（驗補位 topK）並回混合 active/deprecated 命中。
// 真 SQL 語意（json_extract 對 status 欄的實際判等）由本機 miniflare/wrangler d1 跑驗（PR 驗收證據）。
import { describe, it, expect } from 'vitest';
import { Hono } from 'hono';
import { entryRoutes } from '../src/routes/entries';
import { searchEntries, isDeprecatedEntry } from '../src/actions/entry-crud';
import type { Bindings, Entry } from '../src/types';

const NOT_DEPRECATED_PREDICATE =
  "(json_extract(metadata_json, '$.status') IS NULL OR json_extract(metadata_json, '$.status') != 'deprecated')";

// ── fake D1：捕捉 prepared SQL 與 bound params；getEntry（SELECT … WHERE id = ?）依 id 從
//   ENTRY_META 查表回可控 metadata_json，讓 semantic hydrate 路徑能測到 deprecated 過濾 ──
interface Captured { sql: string; params: unknown[] }

function mkEntry(id: string, metadata_json: string | null): Entry {
  return {
    id, content: 'some content', entry_type: 'block', owner_id: 'tenant1', parent_id: null,
    page_name: null, refs_json: '[]', tags_json: '[]', task_status: null, content_hash: null,
    is_embedded: 0, confidence: null, metadata_json, src_id: null, rel_id: null, dst_id: null, created_at: 1, updated_at: 1,
  };
}

function makeCaptureDB(captured: Captured[], entryMeta: Record<string, string | null> = {}) {
  const prepare = (sql: string) => {
    const rec: Captured = { sql, params: [] };
    captured.push(rec);
    const stmt = {
      bind(...args: unknown[]) { rec.params = args; return stmt; },
      async all<T>() { return { results: [] as T[] }; },
      async first<T>() {
        if (sql.includes('WHERE id = ?')) {
          const id = String(rec.params[0]);
          const meta = id in entryMeta ? entryMeta[id] : null;
          return mkEntry(id, meta) as unknown as T;
        }
        return { total: 0, c: 0 } as unknown as T;
      },
      async run() { return { success: true }; },
    };
    return stmt;
  };
  return { prepare } as unknown as D1Database;
}

function makeApp(captured: Captured[], extraEnv: Record<string, unknown> = {}) {
  const app = new Hono<{ Bindings: Bindings }>();
  app.route('/entries', entryRoutes);
  const env = { DB: makeCaptureDB(captured, (extraEnv._entryMeta as Record<string, string | null>) ?? {}), ENVIRONMENT: 'test', ...extraEnv } as unknown as Bindings;
  return { app, env };
}

// ══ 案①：keyword 濾 ══════════════════════════════════════════════════════

describe('t24 案① — searchEntries（keyword）預設濾 deprecated', () => {
  it('預設（不帶 includeDeprecated）→ SQL 含 NOT_DEPRECATED_PREDICATE', async () => {
    const captured: Captured[] = [];
    await searchEntries(makeCaptureDB(captured), '靛藍', 'tenant1');
    expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
  });

  it('includeDeprecated=true → SQL 不含濾 deprecated 謂詞（管理面查殘留用）', async () => {
    const captured: Captured[] = [];
    await searchEntries(makeCaptureDB(captured), '靛藍', 'tenant1', undefined, undefined, undefined, undefined, true);
    expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
  });

  it('route GET /entries/search（keyword，不帶 include_deprecated）→ 濾謂詞下傳', async () => {
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured);
    const res = await app.request('/entries/search?q=靛藍', {}, env);
    expect(res.status).toBe(200);
    const body = (await res.json()) as { mode: string };
    expect(body.mode).toBe('keyword');
    expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
  });

  it('route GET /entries/search?include_deprecated=true（keyword）→ 濾謂詞不下傳', async () => {
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured);
    const res = await app.request('/entries/search?q=靛藍&include_deprecated=true', {}, env);
    expect(res.status).toBe(200);
    expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
  });

  it('semantic 模組未開＋降級 keyword → 仍套濾（不因降級洩下架內容）', async () => {
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured); // 無 VECTORIZE/AI → semanticSearch 回 null
    const res = await app.request('/entries/search?q=靛藍&mode=semantic', {}, env);
    expect(res.status).toBe(200);
    const body = (await res.json()) as { mode: string };
    expect(body.mode).toBe('keyword');
    expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
  });

  it('semantic 模組未開＋include_deprecated=true 降級 → 濾謂詞不下傳', async () => {
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured);
    const res = await app.request('/entries/search?q=靛藍&mode=semantic&include_deprecated=true', {}, env);
    expect(res.status).toBe(200);
    expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
  });
});

// ══ isDeprecatedEntry 單元測試（JS 側判準，semantic 路徑用） ══════════════

describe('t24 — isDeprecatedEntry（JS 側判準）', () => {
  it('status:"deprecated" → true', () => {
    expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ status: 'deprecated' }) })).toBe(true);
  });
  it('status 缺欄 / null metadata_json / 空字串 → false（未下架，保留）', () => {
    expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ embed: true }) })).toBe(false);
    expect(isDeprecatedEntry({ metadata_json: null })).toBe(false);
    expect(isDeprecatedEntry({ metadata_json: '' })).toBe(false);
  });
  it('status 是其他值（非 deprecated）→ false', () => {
    expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ status: 'active' }) })).toBe(false);
  });
  it('metadata_json parse 失敗（壞 JSON）→ false（治標不誤殺）', () => {
    expect(isDeprecatedEntry({ metadata_json: '{not valid json' })).toBe(false);
  });
});

// ══ 案②：semantic 濾＋補位 ═══════════════════════════════════════════════

// mock VECTORIZE：捕 query opts（驗補位 topK）；命中組合可控（含 deprecated id 前綴 dep- 供辨識）。
function makeSemanticEnv(
  queryCalls: { opts: Record<string, unknown> }[],
  matches: { id: string; score: number }[],
) {
  return {
    AI: { async run() { return { data: [[0.1, 0.2, 0.3]] }; } },
    VECTORIZE: {
      async query(_vec: number[], opts: Record<string, unknown>) {
        queryCalls.push({ opts });
        return { matches: matches.map((m) => ({ id: m.id, score: m.score, metadata: {} })) };
      },
      async upsert(v: unknown[]) { return { count: (v as unknown[]).length }; },
    },
  };
}

describe('t24 案② — semantic 濾 deprecated ＋ 補位（t11 斷點②：0.971 最高分照吐的洞）', () => {
  it('命中含已下架（最高分）→ 回應濾掉，只留現役（覆現 t11 0.971 復現案）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const entryMeta = {
      'dep-highest': JSON.stringify({ status: 'deprecated' }), // 0.971 最高分但已下架
      'e-active': null,
    };
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, {
      ...makeSemanticEnv(calls, [
        { id: 'dep-highest', score: 0.971 },
        { id: 'e-active', score: 0.6 },
      ]),
      _entryMeta: entryMeta,
    });
    const res = await app.request('/entries/search?q=靛藍風鈴石的硬度&mode=semantic', {}, env);
    expect(res.status).toBe(200);
    const body = (await res.json()) as { mode: string; count: number; entries: (Entry & { score?: number })[] };
    expect(body.mode).toBe('semantic');
    expect(body.entries.map((e) => e.id)).toEqual(['e-active']); // dep-highest 被濾掉
    expect(body.count).toBe(1);
  });

  it('補位：預設過濾生效時，Vectorize 查詢的 topK 大於 caller 要求（避免整頁被下架品吃光）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, makeSemanticEnv(calls, []));
    await app.request('/entries/search?q=x&mode=semantic&top_k=10', {}, env);
    expect(calls[0].opts.topK).toBeGreaterThan(10); // 補位餘量（實作＝×3 封頂 100）
    expect(calls[0].opts.topK).toBe(30);
  });

  it('補位 topK 封頂 100（不因 top_k 大就超過 Vectorize 上限）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, makeSemanticEnv(calls, []));
    await app.request('/entries/search?q=x&mode=semantic&top_k=50', {}, env);
    expect(calls[0].opts.topK).toBe(100);
  });

  it('補位後截斷：濾掉部分下架品後，回應筆數不超過 caller 要求的 top_k', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    // 6 筆命中，3 筆已下架 → 濾完剩 3 筆現役，均少於 top_k=5，應原樣回（不會硬湊出更多）
    const matches = [
      { id: 'a1', score: 0.9 }, { id: 'dep1', score: 0.85 }, { id: 'a2', score: 0.8 },
      { id: 'dep2', score: 0.7 }, { id: 'a3', score: 0.6 }, { id: 'dep3', score: 0.5 },
    ];
    const entryMeta: Record<string, string | null> = {
      dep1: JSON.stringify({ status: 'deprecated' }),
      dep2: JSON.stringify({ status: 'deprecated' }),
      dep3: JSON.stringify({ status: 'deprecated' }),
    };
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, { ...makeSemanticEnv(calls, matches), _entryMeta: entryMeta });
    // 顯式帶 min_score：本案要測的是「濾下架＋不硬湊」，不是分數門檻。
    // 2026-08-05 起未帶 min_score 會套相對門檻（top×0.8），0.6 的 a3 會被砍掉
    // ⇒ 那會把這個測試變成在測門檻。帶一個寬鬆的絕對值，把門檻這個變因移開。
    const res = await app.request('/entries/search?q=x&mode=semantic&top_k=5&min_score=0.4', {}, env);
    const body = (await res.json()) as { entries: Entry[]; count: number };
    expect(body.entries.map((e) => e.id)).toEqual(['a1', 'a2', 'a3']);
    expect(body.count).toBe(3);
  });

  it('include_deprecated=true → 不補位（topK=請求值）、不過濾（下架品也回傳，管理面查殘留）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const entryMeta = { 'dep-highest': JSON.stringify({ status: 'deprecated' }) };
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, {
      ...makeSemanticEnv(calls, [{ id: 'dep-highest', score: 0.971 }]),
      _entryMeta: entryMeta,
    });
    const res = await app.request('/entries/search?q=x&mode=semantic&top_k=10&include_deprecated=true', {}, env);
    expect(calls[0].opts.topK).toBe(10); // 不補位
    const body = (await res.json()) as { entries: Entry[]; count: number };
    expect(body.entries.map((e) => e.id)).toEqual(['dep-highest']); // 保留
    expect(body.count).toBe(1);
  });
});

// ── 相對門檻（2026-08-05，leo 實測「關懷型 AI」命中 20 筆、只有前 3 筆相關）────────────
//
// 這組鎖住兩件事：
//   ① 門檻跟著「這次查詢的最高分」走，不是固定值
//      （固定 0.5 放太多雜訊；固定 0.6 會把「閉環機」那種整體偏低的查詢砍成 0 命中）
//   ② 🔴 **門檻必須在濾掉下架之後才算**——否則一筆 0.971 的下架殘影會把 0.6 的正解一起帶走，
//      那正是 leo 08-05 早上撞的「0 命中」的翻版。t24 的 0.971 復現案就是這種殘影。
describe('相對門檻（08-05）— 跟著最高分走，且在濾下架之後才算', () => {
  it('低分尾被砍：0.77/0.74/0.64 留下，0.55 以下砍掉（門檻 0.77×0.8=0.616）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const matches = [
      { id: 'hit1', score: 0.77 }, { id: 'hit2', score: 0.74 }, { id: 'hit3', score: 0.64 },
      { id: 'noise1', score: 0.55 }, { id: 'noise2', score: 0.53 }, { id: 'noise3', score: 0.52 },
    ];
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, makeSemanticEnv(calls, matches));
    const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
    const body = (await res.json()) as { entries: Entry[]; count: number };
    expect(body.entries.map((e) => e.id)).toEqual(['hit1', 'hit2', 'hit3']);
  });

  it('整體偏低的查詢不會被砍光：0.638/0.603/0.588/0.552 全留（門檻 0.638×0.8=0.510）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const matches = [
      { id: 'l1', score: 0.638 }, { id: 'l2', score: 0.603 },
      { id: 'l3', score: 0.588 }, { id: 'l4', score: 0.552 }, { id: 'noise', score: 0.446 },
    ];
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, makeSemanticEnv(calls, matches));
    const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
    const body = (await res.json()) as { entries: Entry[] };
    expect(body.entries.map((e) => e.id)).toEqual(['l1', 'l2', 'l3', 'l4']);
  });

  it('🔴 下架殘影不得決定門檻：0.971 已下架 → 門檻要用倖存者的 0.6 算，正解不被帶走', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const matches = [
      { id: 'dep-ghost', score: 0.971 },  // 下架殘影，分數卻最高
      { id: 'real1', score: 0.60 }, { id: 'real2', score: 0.52 },
    ];
    const entryMeta: Record<string, string | null> = { 'dep-ghost': JSON.stringify({ status: 'deprecated' }) };
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, { ...makeSemanticEnv(calls, matches), _entryMeta: entryMeta });
    const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
    const body = (await res.json()) as { entries: Entry[] };
    // 若拿 0.971 算門檻＝0.777 ⇒ real1/real2 全被砍 ⇒ 0 命中（就是那個病）。
    // 正解：殘影先被濾掉，門檻用 0.6×0.8=0.48 算 ⇒ 兩筆都留。
    expect(body.entries.map((e) => e.id)).toEqual(['real1', 'real2']);
  });

  it('caller 顯式帶 min_score → 尊重絕對值，不再加碼相對門檻', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const matches = [{ id: 'a', score: 0.9 }, { id: 'b', score: 0.5 }, { id: 'c', score: 0.3 }];
    const captured: Captured[] = [];
    const { app, env } = makeApp(captured, makeSemanticEnv(calls, matches));
    const res = await app.request('/entries/search?q=x&mode=semantic&min_score=0.4', {}, env);
    const body = (await res.json()) as { entries: Entry[] };
    expect(body.entries.map((e) => e.id)).toEqual(['a', 'b']); // 0.3 被絕對門檻砍，0.5 留著
  });
});