uncle6me-web
05b215c9f7
## ① 「關懷型 AI 命中 20 筆、只有前 3 筆相關」⇒ 閾值太寬（leo 判斷正確） 固定門檻兩頭都不對，因為每個查詢的分數尺度不同（實測 youlin 實例）： 關懷型 AI 正解 0.645-0.770，雜訊起於 0.547 ← 固定 0.5 放進 6 筆雜訊 閉環機 正解 0.552-0.638，雜訊起於 0.446 ← 固定 0.6 砍到剩 2/4（＝早上的 0 命中） 人力媒合系統規劃書 正解 0.842，雜訊起於 0.550 ⇒ 改**相對門檻** max(0.45, top×0.8)。五組實測：固定 0.5 混入 9 筆雜訊／ 固定 0.6 有兩組正解被砍／相對式四組雜訊 0 且正解全留。 ## 🔴 寫測試才發現的真問題：門檻不能在 Vectorize 那層算 Vectorize 的 indexed metadata 沒有 status ⇒ 那層不知道誰已下架。 若最高分是下架殘影（t24 復現案 0.971），拿它算門檻＝0.777， 會把 0.6 的正解一起砍光 ⇒ **又變成 0 命中**。 ⇒ 相對門檻移到 routes/entries.ts，接在「hydrate＋濾下架」之後； embed.ts 只留絕對下限。新增測試鎖住這個順序。 ## ② leo：「理論上它的向量也要刪掉，就不會有殘影了吧？」——對，補上 單筆真刪已接 deleteByIds（b7af622），但「移除整個庫」走軟刪、向量原地不動。 ⇒ deprecate-by-library 同時 deleteByIds ＋ is_embedded 歸零（D1 與 Vectorize 不說兩套話）； backfill 兩條路徑（含 reindex）都排除 deprecated，否則下次補嵌會把殘影養回來。 不違背 t135「資料保留可還原」：D1 那列原封不動，還原後跑 backfill 重嵌即可。 回應新增 vectors_deleted，刪失敗誠實回 0 不假裝清乾淨。 驗：kbdb 91/91 綠（新增 4 項相對門檻測試，含「下架殘影不得決定門檻」） Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
291 lines
16 KiB
TypeScript
291 lines
16 KiB
TypeScript
// daemon-beta t24（總管 0.971 親復現、t11 斷點①②）——/entries/search 服務端濾 deprecated。
|
||
// 背景：rag_takedown_direct 下架只把 metadata_json.status 標 'deprecated'（軟刪，append-only，
|
||
// KBDB 表不變鐵律），從不砍列、也不刪 Vectorize 向量。過去唯一的濾層在
|
||
// cypher-executor/src/routes/portal-data.ts（客端治標，Arcrun#46），沒部署 rag_chat 的實例
|
||
// （如 leo21c）等於完全沒濾——MCP/raw /entries/search 直接把已下架內容當現役吐出。semantic
|
||
// 甚至最高分照吐（t11 實測 0.971）。本測試覆蓋三案：keyword 濾、semantic 濾＋補位、
|
||
// include_deprecated 開關。
|
||
//
|
||
// 測試手法同 search-source-and-score.test.ts：fake D1 捕 SQL 形狀 + getEntry 依 id 回可控
|
||
// metadata_json；mock VECTORIZE 捕 query opts（驗補位 topK）並回混合 active/deprecated 命中。
|
||
// 真 SQL 語意（json_extract 對 status 欄的實際判等）由本機 miniflare/wrangler d1 跑驗（PR 驗收證據）。
|
||
import { describe, it, expect } from 'vitest';
|
||
import { Hono } from 'hono';
|
||
import { entryRoutes } from '../src/routes/entries';
|
||
import { searchEntries, isDeprecatedEntry } from '../src/actions/entry-crud';
|
||
import type { Bindings, Entry } from '../src/types';
|
||
|
||
const NOT_DEPRECATED_PREDICATE =
|
||
"(json_extract(metadata_json, '$.status') IS NULL OR json_extract(metadata_json, '$.status') != 'deprecated')";
|
||
|
||
// ── fake D1：捕捉 prepared SQL 與 bound params；getEntry（SELECT … WHERE id = ?）依 id 從
|
||
// ENTRY_META 查表回可控 metadata_json，讓 semantic hydrate 路徑能測到 deprecated 過濾 ──
|
||
interface Captured { sql: string; params: unknown[] }
|
||
|
||
function mkEntry(id: string, metadata_json: string | null): Entry {
|
||
return {
|
||
id, content: 'some content', entry_type: 'block', owner_id: 'tenant1', parent_id: null,
|
||
page_name: null, refs_json: '[]', tags_json: '[]', task_status: null, content_hash: null,
|
||
is_embedded: 0, confidence: null, metadata_json, created_at: 1, updated_at: 1,
|
||
};
|
||
}
|
||
|
||
function makeCaptureDB(captured: Captured[], entryMeta: Record<string, string | null> = {}) {
|
||
const prepare = (sql: string) => {
|
||
const rec: Captured = { sql, params: [] };
|
||
captured.push(rec);
|
||
const stmt = {
|
||
bind(...args: unknown[]) { rec.params = args; return stmt; },
|
||
async all<T>() { return { results: [] as T[] }; },
|
||
async first<T>() {
|
||
if (sql.includes('WHERE id = ?')) {
|
||
const id = String(rec.params[0]);
|
||
const meta = id in entryMeta ? entryMeta[id] : null;
|
||
return mkEntry(id, meta) as unknown as T;
|
||
}
|
||
return { total: 0, c: 0 } as unknown as T;
|
||
},
|
||
async run() { return { success: true }; },
|
||
};
|
||
return stmt;
|
||
};
|
||
return { prepare } as unknown as D1Database;
|
||
}
|
||
|
||
function makeApp(captured: Captured[], extraEnv: Record<string, unknown> = {}) {
|
||
const app = new Hono<{ Bindings: Bindings }>();
|
||
app.route('/entries', entryRoutes);
|
||
const env = { DB: makeCaptureDB(captured, (extraEnv._entryMeta as Record<string, string | null>) ?? {}), ENVIRONMENT: 'test', ...extraEnv } as unknown as Bindings;
|
||
return { app, env };
|
||
}
|
||
|
||
// ══ 案①：keyword 濾 ══════════════════════════════════════════════════════
|
||
|
||
describe('t24 案① — searchEntries（keyword）預設濾 deprecated', () => {
|
||
it('預設（不帶 includeDeprecated）→ SQL 含 NOT_DEPRECATED_PREDICATE', async () => {
|
||
const captured: Captured[] = [];
|
||
await searchEntries(makeCaptureDB(captured), '靛藍', 'tenant1');
|
||
expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
|
||
});
|
||
|
||
it('includeDeprecated=true → SQL 不含濾 deprecated 謂詞（管理面查殘留用）', async () => {
|
||
const captured: Captured[] = [];
|
||
await searchEntries(makeCaptureDB(captured), '靛藍', 'tenant1', undefined, undefined, undefined, undefined, true);
|
||
expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
|
||
});
|
||
|
||
it('route GET /entries/search（keyword，不帶 include_deprecated）→ 濾謂詞下傳', async () => {
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured);
|
||
const res = await app.request('/entries/search?q=靛藍', {}, env);
|
||
expect(res.status).toBe(200);
|
||
const body = (await res.json()) as { mode: string };
|
||
expect(body.mode).toBe('keyword');
|
||
expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
|
||
});
|
||
|
||
it('route GET /entries/search?include_deprecated=true（keyword）→ 濾謂詞不下傳', async () => {
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured);
|
||
const res = await app.request('/entries/search?q=靛藍&include_deprecated=true', {}, env);
|
||
expect(res.status).toBe(200);
|
||
expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
|
||
});
|
||
|
||
it('semantic 模組未開＋降級 keyword → 仍套濾（不因降級洩下架內容）', async () => {
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured); // 無 VECTORIZE/AI → semanticSearch 回 null
|
||
const res = await app.request('/entries/search?q=靛藍&mode=semantic', {}, env);
|
||
expect(res.status).toBe(200);
|
||
const body = (await res.json()) as { mode: string };
|
||
expect(body.mode).toBe('keyword');
|
||
expect(captured[0].sql).toContain(NOT_DEPRECATED_PREDICATE);
|
||
});
|
||
|
||
it('semantic 模組未開＋include_deprecated=true 降級 → 濾謂詞不下傳', async () => {
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured);
|
||
const res = await app.request('/entries/search?q=靛藍&mode=semantic&include_deprecated=true', {}, env);
|
||
expect(res.status).toBe(200);
|
||
expect(captured[0].sql).not.toContain(NOT_DEPRECATED_PREDICATE);
|
||
});
|
||
});
|
||
|
||
// ══ isDeprecatedEntry 單元測試（JS 側判準，semantic 路徑用） ══════════════
|
||
|
||
describe('t24 — isDeprecatedEntry（JS 側判準）', () => {
|
||
it('status:"deprecated" → true', () => {
|
||
expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ status: 'deprecated' }) })).toBe(true);
|
||
});
|
||
it('status 缺欄 / null metadata_json / 空字串 → false（未下架，保留）', () => {
|
||
expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ embed: true }) })).toBe(false);
|
||
expect(isDeprecatedEntry({ metadata_json: null })).toBe(false);
|
||
expect(isDeprecatedEntry({ metadata_json: '' })).toBe(false);
|
||
});
|
||
it('status 是其他值（非 deprecated）→ false', () => {
|
||
expect(isDeprecatedEntry({ metadata_json: JSON.stringify({ status: 'active' }) })).toBe(false);
|
||
});
|
||
it('metadata_json parse 失敗（壞 JSON）→ false（治標不誤殺）', () => {
|
||
expect(isDeprecatedEntry({ metadata_json: '{not valid json' })).toBe(false);
|
||
});
|
||
});
|
||
|
||
// ══ 案②：semantic 濾＋補位 ═══════════════════════════════════════════════
|
||
|
||
// mock VECTORIZE：捕 query opts（驗補位 topK）；命中組合可控（含 deprecated id 前綴 dep- 供辨識）。
|
||
function makeSemanticEnv(
|
||
queryCalls: { opts: Record<string, unknown> }[],
|
||
matches: { id: string; score: number }[],
|
||
) {
|
||
return {
|
||
AI: { async run() { return { data: [[0.1, 0.2, 0.3]] }; } },
|
||
VECTORIZE: {
|
||
async query(_vec: number[], opts: Record<string, unknown>) {
|
||
queryCalls.push({ opts });
|
||
return { matches: matches.map((m) => ({ id: m.id, score: m.score, metadata: {} })) };
|
||
},
|
||
async upsert(v: unknown[]) { return { count: (v as unknown[]).length }; },
|
||
},
|
||
};
|
||
}
|
||
|
||
describe('t24 案② — semantic 濾 deprecated ＋ 補位（t11 斷點②：0.971 最高分照吐的洞）', () => {
|
||
it('命中含已下架（最高分）→ 回應濾掉，只留現役（覆現 t11 0.971 復現案）', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const entryMeta = {
|
||
'dep-highest': JSON.stringify({ status: 'deprecated' }), // 0.971 最高分但已下架
|
||
'e-active': null,
|
||
};
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, {
|
||
...makeSemanticEnv(calls, [
|
||
{ id: 'dep-highest', score: 0.971 },
|
||
{ id: 'e-active', score: 0.6 },
|
||
]),
|
||
_entryMeta: entryMeta,
|
||
});
|
||
const res = await app.request('/entries/search?q=靛藍風鈴石的硬度&mode=semantic', {}, env);
|
||
expect(res.status).toBe(200);
|
||
const body = (await res.json()) as { mode: string; count: number; entries: (Entry & { score?: number })[] };
|
||
expect(body.mode).toBe('semantic');
|
||
expect(body.entries.map((e) => e.id)).toEqual(['e-active']); // dep-highest 被濾掉
|
||
expect(body.count).toBe(1);
|
||
});
|
||
|
||
it('補位：預設過濾生效時，Vectorize 查詢的 topK 大於 caller 要求（避免整頁被下架品吃光）', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, makeSemanticEnv(calls, []));
|
||
await app.request('/entries/search?q=x&mode=semantic&top_k=10', {}, env);
|
||
expect(calls[0].opts.topK).toBeGreaterThan(10); // 補位餘量（實作＝×3 封頂 100）
|
||
expect(calls[0].opts.topK).toBe(30);
|
||
});
|
||
|
||
it('補位 topK 封頂 100（不因 top_k 大就超過 Vectorize 上限）', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, makeSemanticEnv(calls, []));
|
||
await app.request('/entries/search?q=x&mode=semantic&top_k=50', {}, env);
|
||
expect(calls[0].opts.topK).toBe(100);
|
||
});
|
||
|
||
it('補位後截斷：濾掉部分下架品後，回應筆數不超過 caller 要求的 top_k', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
// 6 筆命中，3 筆已下架 → 濾完剩 3 筆現役，均少於 top_k=5，應原樣回（不會硬湊出更多）
|
||
const matches = [
|
||
{ id: 'a1', score: 0.9 }, { id: 'dep1', score: 0.85 }, { id: 'a2', score: 0.8 },
|
||
{ id: 'dep2', score: 0.7 }, { id: 'a3', score: 0.6 }, { id: 'dep3', score: 0.5 },
|
||
];
|
||
const entryMeta: Record<string, string | null> = {
|
||
dep1: JSON.stringify({ status: 'deprecated' }),
|
||
dep2: JSON.stringify({ status: 'deprecated' }),
|
||
dep3: JSON.stringify({ status: 'deprecated' }),
|
||
};
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, { ...makeSemanticEnv(calls, matches), _entryMeta: entryMeta });
|
||
// 顯式帶 min_score：本案要測的是「濾下架＋不硬湊」，不是分數門檻。
|
||
// 2026-08-05 起未帶 min_score 會套相對門檻（top×0.8），0.6 的 a3 會被砍掉
|
||
// ⇒ 那會把這個測試變成在測門檻。帶一個寬鬆的絕對值，把門檻這個變因移開。
|
||
const res = await app.request('/entries/search?q=x&mode=semantic&top_k=5&min_score=0.4', {}, env);
|
||
const body = (await res.json()) as { entries: Entry[]; count: number };
|
||
expect(body.entries.map((e) => e.id)).toEqual(['a1', 'a2', 'a3']);
|
||
expect(body.count).toBe(3);
|
||
});
|
||
|
||
it('include_deprecated=true → 不補位（topK=請求值）、不過濾（下架品也回傳，管理面查殘留）', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const entryMeta = { 'dep-highest': JSON.stringify({ status: 'deprecated' }) };
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, {
|
||
...makeSemanticEnv(calls, [{ id: 'dep-highest', score: 0.971 }]),
|
||
_entryMeta: entryMeta,
|
||
});
|
||
const res = await app.request('/entries/search?q=x&mode=semantic&top_k=10&include_deprecated=true', {}, env);
|
||
expect(calls[0].opts.topK).toBe(10); // 不補位
|
||
const body = (await res.json()) as { entries: Entry[]; count: number };
|
||
expect(body.entries.map((e) => e.id)).toEqual(['dep-highest']); // 保留
|
||
expect(body.count).toBe(1);
|
||
});
|
||
});
|
||
|
||
// ── 相對門檻（2026-08-05，leo 實測「關懷型 AI」命中 20 筆、只有前 3 筆相關）────────────
|
||
//
|
||
// 這組鎖住兩件事：
|
||
// ① 門檻跟著「這次查詢的最高分」走，不是固定值
|
||
// （固定 0.5 放太多雜訊；固定 0.6 會把「閉環機」那種整體偏低的查詢砍成 0 命中）
|
||
// ② 🔴 **門檻必須在濾掉下架之後才算**——否則一筆 0.971 的下架殘影會把 0.6 的正解一起帶走，
|
||
// 那正是 leo 08-05 早上撞的「0 命中」的翻版。t24 的 0.971 復現案就是這種殘影。
|
||
describe('相對門檻（08-05）— 跟著最高分走，且在濾下架之後才算', () => {
|
||
it('低分尾被砍：0.77/0.74/0.64 留下，0.55 以下砍掉（門檻 0.77×0.8=0.616）', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const matches = [
|
||
{ id: 'hit1', score: 0.77 }, { id: 'hit2', score: 0.74 }, { id: 'hit3', score: 0.64 },
|
||
{ id: 'noise1', score: 0.55 }, { id: 'noise2', score: 0.53 }, { id: 'noise3', score: 0.52 },
|
||
];
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, makeSemanticEnv(calls, matches));
|
||
const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
|
||
const body = (await res.json()) as { entries: Entry[]; count: number };
|
||
expect(body.entries.map((e) => e.id)).toEqual(['hit1', 'hit2', 'hit3']);
|
||
});
|
||
|
||
it('整體偏低的查詢不會被砍光：0.638/0.603/0.588/0.552 全留（門檻 0.638×0.8=0.510）', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const matches = [
|
||
{ id: 'l1', score: 0.638 }, { id: 'l2', score: 0.603 },
|
||
{ id: 'l3', score: 0.588 }, { id: 'l4', score: 0.552 }, { id: 'noise', score: 0.446 },
|
||
];
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, makeSemanticEnv(calls, matches));
|
||
const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
|
||
const body = (await res.json()) as { entries: Entry[] };
|
||
expect(body.entries.map((e) => e.id)).toEqual(['l1', 'l2', 'l3', 'l4']);
|
||
});
|
||
|
||
it('🔴 下架殘影不得決定門檻：0.971 已下架 → 門檻要用倖存者的 0.6 算，正解不被帶走', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const matches = [
|
||
{ id: 'dep-ghost', score: 0.971 }, // 下架殘影，分數卻最高
|
||
{ id: 'real1', score: 0.60 }, { id: 'real2', score: 0.52 },
|
||
];
|
||
const entryMeta: Record<string, string | null> = { 'dep-ghost': JSON.stringify({ status: 'deprecated' }) };
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, { ...makeSemanticEnv(calls, matches), _entryMeta: entryMeta });
|
||
const res = await app.request('/entries/search?q=x&mode=semantic', {}, env);
|
||
const body = (await res.json()) as { entries: Entry[] };
|
||
// 若拿 0.971 算門檻＝0.777 ⇒ real1/real2 全被砍 ⇒ 0 命中（就是那個病）。
|
||
// 正解：殘影先被濾掉，門檻用 0.6×0.8=0.48 算 ⇒ 兩筆都留。
|
||
expect(body.entries.map((e) => e.id)).toEqual(['real1', 'real2']);
|
||
});
|
||
|
||
it('caller 顯式帶 min_score → 尊重絕對值，不再加碼相對門檻', async () => {
|
||
const calls: { opts: Record<string, unknown> }[] = [];
|
||
const matches = [{ id: 'a', score: 0.9 }, { id: 'b', score: 0.5 }, { id: 'c', score: 0.3 }];
|
||
const captured: Captured[] = [];
|
||
const { app, env } = makeApp(captured, makeSemanticEnv(calls, matches));
|
||
const res = await app.request('/entries/search?q=x&mode=semantic&min_score=0.4', {}, env);
|
||
const body = (await res.json()) as { entries: Entry[] };
|
||
expect(body.entries.map((e) => e.id)).toEqual(['a', 'b']); // 0.3 被絕對門檻砍，0.5 留著
|
||
});
|
||
});
|