fix(kbdb): /entries/search 服務端濾 deprecated（keyword+semantic），修下架不生效

daemon-beta t24（t11 斷點①②，總管 0.971 親復現）：rag_takedown_direct 下架只把 metadata_json.status 標 deprecated（軟刪，append-only），但 /entries/search 兩 mode 都不濾，已下架內容照樣回傳——semantic 甚至最高分照吐。過去唯一的濾層只在 cypher-executor/portal-data.ts 客端（Arcrun#46），沒部署 rag_chat 的實例等於完全沒濾。 - entry-crud.ts：新增 json_extract($.status) NOT_DEPRECATED_PREDICATE（同 source/library 既有 json_extract 模式，issue #5.1/#18；不用 LIKE 避免 JSON 序列化格式誤判）； searchEntries 新增 includeDeprecated 參數（尾端新增，向後相容）。新增 isDeprecatedEntry：semantic 路徑用（Vectorize metadata 沒存 status，只能 hydrate 回完整 entry 後 JS 側判斷）。 - entries.ts：/entries/search 新增 include_deprecated 開關（預設 false，濾掉；true 給管理面查殘留）。semantic 分支補位——過濾生效時先以 topK×3（封頂 100）向 Vectorize 多撈，hydrate+濾完再截斷回 caller 要求的量，避免命中大半下架時整頁被吃光（t11 ZZ-T10 實測案例）。 - Vectorize 向量殘留本 PR 不動（只做查詢時過濾，not 同步刪向量）——取捨見 PR 描述。測試：新增 search-deprecated-filter.test.ts 15 案（keyword 濾/include_deprecated 開關/ semantic 濾+補位+封頂+截斷/isDeprecatedEntry 單元）；同步更新 search-source-and-score.test.ts 3 案的 topK 期望值（route 補位改變送進 Vectorize 的實際 topK，回應對外契約不變）。 kbdb 全套 60/60 綠、tsc --noEmit 乾淨。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-24 16:35:17 +08:00
parent e28e19069f
commit 55283f4a3b
4 changed files with 304 additions and 10 deletions
@@ -0,0 +1,226 @@
+// daemon-beta t24（總管 0.971 親復現、t11 斷點①②）——/entries/search 服務端濾 deprecated。
+// 背景：rag_takedown_direct 下架只把 metadata_json.status 標 'deprecated'（軟刪，append-only，
+// KBDB 表不變鐵律），從不砍列、也不刪 Vectorize 向量。過去唯一的濾層在
+// cypher-executor/src/routes/portal-data.ts（客端治標，Arcrun#46），沒部署 rag_chat 的實例
+// （如 leo21c）等於完全沒濾——MCP/raw /entries/search 直接把已下架內容當現役吐出。semantic
+// 甚至最高分照吐（t11 實測 0.971）。本測試覆蓋三案：keyword 濾、semantic 濾＋補位、
+// include_deprecated 開關。
+//
+// 測試手法同 search-source-and-score.test.ts：fake D1 捕 SQL 形狀 + getEntry 依 id 回可控
+// metadata_json；mock VECTORIZE 捕 query opts（驗補位 topK）並回混合 active/deprecated 命中。
+// 真 SQL 語意（json_extract 對 status 欄的實際判等）由本機 miniflare/wrangler d1 跑驗（PR 驗收證據）。
+import { describe, it, expect } from 'vitest';
+import { Hono } from 'hono';
+import { entryRoutes } from '../src/routes/entries';
+import { searchEntries, isDeprecatedEntry } from '../src/actions/entry-crud';
+import type { Bindings, Entry } from '../src/types';
+
+const NOT_DEPRECATED_PREDICATE =
+  "(json_extract(metadata_json, '$.status') IS NULL OR json_extract(metadata_json, '$.status') != 'deprecated')";
+
+// ── fake D1：捕捉 prepared SQL 與 bound params；getEntry（SELECT … WHERE id = ?）依 id 從
+//   ENTRY_META 查表回可控 metadata_json，讓 semantic hydrate 路徑能測到 deprecated 過濾 ──
+interface Captured { sql: string; params: unknown[] }
+
+function mkEntry(id: string, metadata_json: string | null): Entry {
+  return {
+    id, content: 'some content', entry_type: 'block', owner_id: 'tenant1', parent_id: null,
+    page_name: null, refs_json: '[]', tags_json: '[]', task_status: null, content_hash: null,
+    is_embedded: 0, confidence: null, metadata_json, created_at: 1, updated_at: 1,
+  };
+}
+
+function makeCaptureDB(captured: Captured[], entryMeta: Record<string, string | null> = {}) {
+  const prepare = (sql: string) => {
+    const rec: Captured = { sql, params: [] };
+    captured.push(rec);
+    const stmt = {
+      bind(...args: unknown[]) { rec.params = args; return stmt; },
+      async all<T>() { return { results: [] as T[] }; },
+      async first<T>() {
+        if (sql.includes('WHERE id = ?')) {
+          const id = String(rec.params[0]);
+          const meta = id in entryMeta ? entryMeta[id] : null;
+          return mkEntry(id, meta) as unknown as T;
+        }
+        return { total: 0, c: 0 } as unknown as T;
+      },
+      async run() { return { success: true }; },
+    };
+    return stmt;
+  };
+  return { prepare } as unknown as D1Database;
+}
+
+function makeApp(captured: Captured[], extraEnv: Record<string, unknown> = {}) {
+  const app = new Hono<{ Bindings: Bindings }>();
+  app.route('/entries', entryRoutes);
+  const env = { DB: makeCaptureDB(captured, (extraEnv._entryMeta as Record<string, string | null>) ?? {}), ENVIRONMENT: 'test', ...extraEnv } as unknown as Bindings;
+  return { app, env };
+}
+
+// ══ 案①：keyword 濾 ══════════════════════════════════════════════════════
+
+describe('t24 案① — searchEntries（keyword）預設濾 deprecated', () => {
+  it('預設（不帶 includeDeprecated）→ SQL 含 NOT_DEPRECATED_PREDICATE', async () => {
+    const captured: Captured[] = [];
+    await searchEntries(makeCaptureDB(captured), '靛藍', 'tenant1');
+    expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
+  });
+
+  it('includeDeprecated=true → SQL 不含濾 deprecated 謂詞（管理面查殘留用）', async () => {
+    const captured: Captured[] = [];
+    await searchEntries(makeCaptureDB(captured), '靛藍', 'tenant1', undefined, undefined, undefined, undefined, true);
+    expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
+  });
+
+  it('route GET /entries/search（keyword，不帶 include_deprecated）→ 濾謂詞下傳', async () => {
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured);
+    const res = await app.request('/entries/search?q=靛藍', {}, env);
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { mode: string };
+    expect(body.mode).toBe('keyword');
+    expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
+  });
+
+  it('route GET /entries/search?include_deprecated=true（keyword）→ 濾謂詞不下傳', async () => {
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured);
+    const res = await app.request('/entries/search?q=靛藍&include_deprecated=true', {}, env);
+    expect(res.status).toBe(200);
+    expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
+  });
+
+  it('semantic 模組未開＋降級 keyword → 仍套濾（不因降級洩下架內容）', async () => {
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured); // 無 VECTORIZE/AI → semanticSearch 回 null
+    const res = await app.request('/entries/search?q=靛藍&mode=semantic', {}, env);
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { mode: string };
+    expect(body.mode).toBe('keyword');
+    expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
+  });
+
+  it('semantic 模組未開＋include_deprecated=true 降級 → 濾謂詞不下傳', async () => {
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured);
+    const res = await app.request('/entries/search?q=靛藍&mode=semantic&include_deprecated=true', {}, env);
+    expect(res.status).toBe(200);
+    expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
+  });
+});
+
+// ══ isDeprecatedEntry 單元測試（JS 側判準，semantic 路徑用） ══════════════
+
+describe('t24 — isDeprecatedEntry（JS 側判準）', () => {
+  it('status:"deprecated" → true', () => {
+    expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ status: 'deprecated' }) })).toBe(true);
+  });
+  it('status 缺欄 / null metadata_json / 空字串 → false（未下架，保留）', () => {
+    expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ embed: true }) })).toBe(false);
+    expect(isDeprecatedEntry({ metadata_json: null })).toBe(false);
+    expect(isDeprecatedEntry({ metadata_json: '' })).toBe(false);
+  });
+  it('status 是其他值（非 deprecated）→ false', () => {
+    expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ status: 'active' }) })).toBe(false);
+  });
+  it('metadata_json parse 失敗（壞 JSON）→ false（治標不誤殺）', () => {
+    expect(isDeprecatedEntry({ metadata_json: '{not valid json' })).toBe(false);
+  });
+});
+
+// ══ 案②：semantic 濾＋補位 ═══════════════════════════════════════════════
+
+// mock VECTORIZE：捕 query opts（驗補位 topK）；命中組合可控（含 deprecated id 前綴 dep- 供辨識）。
+function makeSemanticEnv(
+  queryCalls: { opts: Record<string, unknown> }[],
+  matches: { id: string; score: number }[],
+) {
+  return {
+    AI: { async run() { return { data: [[0.1, 0.2, 0.3]] }; } },
+    VECTORIZE: {
+      async query(_vec: number[], opts: Record<string, unknown>) {
+        queryCalls.push({ opts });
+        return { matches: matches.map((m) => ({ id: m.id, score: m.score, metadata: {} })) };
+      },
+      async upsert(v: unknown[]) { return { count: (v as unknown[]).length }; },
+    },
+  };
+}
+
+describe('t24 案② — semantic 濾 deprecated ＋ 補位（t11 斷點②：0.971 最高分照吐的洞）', () => {
+  it('命中含已下架（最高分）→ 回應濾掉，只留現役（覆現 t11 0.971 復現案）', async () => {
+    const calls: { opts: Record<string, unknown> }[] = [];
+    const entryMeta = {
+      'dep-highest': JSON.stringify({ status: 'deprecated' }), // 0.971 最高分但已下架
+      'e-active': null,
+    };
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured, {
+      ...makeSemanticEnv(calls, [
+        { id: 'dep-highest', score: 0.971 },
+        { id: 'e-active', score: 0.6 },
+      ]),
+      _entryMeta: entryMeta,
+    });
+    const res = await app.request('/entries/search?q=靛藍風鈴石的硬度&mode=semantic', {}, env);
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { mode: string; count: number; entries: (Entry & { score?: number })[] };
+    expect(body.mode).toBe('semantic');
+    expect(body.entries.map((e) => e.id)).toEqual(['e-active']); // dep-highest 被濾掉
+    expect(body.count).toBe(1);
+  });
+
+  it('補位：預設過濾生效時，Vectorize 查詢的 topK 大於 caller 要求（避免整頁被下架品吃光）', async () => {
+    const calls: { opts: Record<string, unknown> }[] = [];
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured, makeSemanticEnv(calls, []));
+    await app.request('/entries/search?q=x&mode=semantic&top_k=10', {}, env);
+    expect(calls[0].opts.topK).toBeGreaterThan(10); // 補位餘量（實作＝×3 封頂 100）
+    expect(calls[0].opts.topK).toBe(30);
+  });
+
+  it('補位 topK 封頂 100（不因 top_k 大就超過 Vectorize 上限）', async () => {
+    const calls: { opts: Record<string, unknown> }[] = [];
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured, makeSemanticEnv(calls, []));
+    await app.request('/entries/search?q=x&mode=semantic&top_k=50', {}, env);
+    expect(calls[0].opts.topK).toBe(100);
+  });
+
+  it('補位後截斷：濾掉部分下架品後，回應筆數不超過 caller 要求的 top_k', async () => {
+    const calls: { opts: Record<string, unknown> }[] = [];
+    // 6 筆命中，3 筆已下架 → 濾完剩 3 筆現役，均少於 top_k=5，應原樣回（不會硬湊出更多）
+    const matches = [
+      { id: 'a1', score: 0.9 }, { id: 'dep1', score: 0.85 }, { id: 'a2', score: 0.8 },
+      { id: 'dep2', score: 0.7 }, { id: 'a3', score: 0.6 }, { id: 'dep3', score: 0.5 },
+    ];
+    const entryMeta: Record<string, string | null> = {
+      dep1: JSON.stringify({ status: 'deprecated' }),
+      dep2: JSON.stringify({ status: 'deprecated' }),
+      dep3: JSON.stringify({ status: 'deprecated' }),
+    };
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured, { ...makeSemanticEnv(calls, matches), _entryMeta: entryMeta });
+    const res = await app.request('/entries/search?q=x&mode=semantic&top_k=5', {}, env);
+    const body = (await res.json()) as { entries: Entry[]; count: number };
+    expect(body.entries.map((e) => e.id)).toEqual(['a1', 'a2', 'a3']);
+    expect(body.count).toBe(3);
+  });
+
+  it('include_deprecated=true → 不補位（topK=請求值）、不過濾（下架品也回傳，管理面查殘留）', async () => {
+    const calls: { opts: Record<string, unknown> }[] = [];
+    const entryMeta = { 'dep-highest': JSON.stringify({ status: 'deprecated' }) };
+    const captured: Captured[] = [];
+    const { app, env } = makeApp(captured, {
+      ...makeSemanticEnv(calls, [{ id: 'dep-highest', score: 0.971 }]),
+      _entryMeta: entryMeta,
+    });
+    const res = await app.request('/entries/search?q=x&mode=semantic&top_k=10&include_deprecated=true', {}, env);
+    expect(calls[0].opts.topK).toBe(10); // 不補位
+    const body = (await res.json()) as { entries: Entry[]; count: number };
+    expect(body.entries.map((e) => e.id)).toEqual(['dep-highest']); // 保留
+    expect(body.count).toBe(1);
+  });
+});
@@ -162,26 +162,32 @@ describe('#67 — route GET /entries/search（semantic）top_k / min_score / sco
    return makeApp(captured, makeSemanticEnv(calls));
  }

-  it('?top_k=5&min_score=0.5 → topK 透傳、低分截掉、entry 附 score', async () => {
+  // daemon-beta t24（07-24 補位變更）：route 現在對 Vectorize 的實際查詢 topK 會做「補位」
+  // （預設過濾 deprecated 時 ×3 封頂 100，見 entries.ts 補位註解），不再是 top_k 原封透傳到
+  // VECTORIZE.query。route 對 caller 的回應仍會照 top_k 截斷（見 body.count/entries 斷言不變）
+  // ——這裡改的只是「送進 Vectorize 那次呼叫的 topK 參數」，非對外契約。三筆測試同步更新
+  // calls[0].opts.topK 期望值（5→15＝5×3、20→60＝20×3），其餘斷言（回應筆數/內容/score）不動。
+
+  it('?top_k=5&min_score=0.5 → Vectorize 補位 topK=15（5×3）、回應仍照 top_k 截後低分尾、entry 附 score', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const { app, env } = makeSemanticApp(calls);
    const res = await app.request('/entries/search?q=x&mode=semantic&top_k=5&min_score=0.5', {}, env);
    expect(res.status).toBe(200);
    const body = (await res.json()) as { mode: string; count: number; entries: (Entry & { score?: number })[] };
    expect(body.mode).toBe('semantic');
-    expect(calls[0].opts.topK).toBe(5);
+    expect(calls[0].opts.topK).toBe(15); // t24 補位：5 × 3
    expect(body.count).toBe(2); // 0.2 的低分尾被 min_score 截掉
    expect(body.entries.map((e) => e.id)).toEqual(['e-high', 'e-mid']);
    expect(body.entries.map((e) => e.score)).toEqual([0.9, 0.5]);
  });

-  it('不帶新參數 → topK=20、全量回傳（行為不變），entry 仍附 score（加欄不改形）', async () => {
+  it('不帶新參數 → Vectorize 補位 topK=60（預設 20×3），回應仍全量回傳（行為不變），entry 仍附 score（加欄不改形）', async () => {
    const calls: { opts: Record<string, unknown> }[] = [];
    const { app, env } = makeSemanticApp(calls);
    const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
    expect(res.status).toBe(200);
    const body = (await res.json()) as { count: number; entries: (Entry & { score?: number })[] };
-    expect(calls[0].opts.topK).toBe(20);
+    expect(calls[0].opts.topK).toBe(60); // t24 補位：預設 20 × 3
    expect(body.count).toBe(3);
    expect(body.entries[0].score).toBe(0.9);
    // 原有欄位一個不少（回應形狀向後相容）
@@ -189,14 +195,14 @@ describe('#67 — route GET /entries/search（semantic）top_k / min_score / sco
    expect(body.entries[0].entry_type).toBe('block');
  });

-  it('壞值防呆：top_k=abc / top_k=0 / min_score=-1 → 視同沒帶（回預設，不 400）', async () => {
+  it('壞值防呆：top_k=abc / top_k=0 / min_score=-1 → 視同沒帶（回預設 20，補位後 Vectorize topK=60，不 400）', async () => {
    for (const qs of ['top_k=abc', 'top_k=0', 'min_score=-1', 'top_k=abc&min_score=xyz']) {
      const calls: { opts: Record<string, unknown> }[] = [];
      const { app, env } = makeSemanticApp(calls);
      const res = await app.request(`/entries/search?q=x&mode=semantic&${qs}`, {}, env);
      expect(res.status).toBe(200);
      const body = (await res.json()) as { count: number };
-      expect(calls[0].opts.topK).toBe(20);
+      expect(calls[0].opts.topK).toBe(60); // t24 補位：預設 20 × 3
      expect(body.count).toBe(3); // 無閾值 → 全量
    }
  });