diff --git a/kbdb/src/actions/entry-crud.ts b/kbdb/src/actions/entry-crud.ts index d2cd35a..e443d86 100644 --- a/kbdb/src/actions/entry-crud.ts +++ b/kbdb/src/actions/entry-crud.ts @@ -137,6 +137,9 @@ function libraryPredicate(libraries: string[]): string { // D1 LIKE keyword search (base; semantic search is the optional embed module). // entry_type: optional base filter (generic — caller passes any type, base stays type-agnostic). // library: optional 多值庫 filter（portal-auth P1）；未帶＝行為與舊版一字不變（向後相容）。 +// source: metadata_json.$.source filter（issue #66——#5.1 只接了 listEntries 那半，keyword search +// 路徑 route 解析完即丟；謂詞與 listEntries 同款 json_extract，不動表）。加在參數尾端， +// 既有 positional caller 一個都不用改（向後相容）。 export async function searchEntries( db: D1Database, q: string, @@ -144,11 +147,13 @@ export async function searchEntries( entry_type?: string, limit = 50, library?: string[], + source?: string, ): Promise { const conds = ['content LIKE ?']; const params: unknown[] = [`%${q}%`]; if (owner_id) { conds.push('owner_id = ?'); params.push(owner_id); } if (entry_type) { conds.push('entry_type = ?'); params.push(entry_type); } + if (source) { conds.push("json_extract(metadata_json, '$.source') = ?"); params.push(source); } if (library && library.length > 0) { conds.push(libraryPredicate(library)); params.push(...library); } const res = await db .prepare(`SELECT * FROM entries WHERE ${conds.join(' AND ')} ORDER BY updated_at DESC LIMIT ?`) diff --git a/kbdb/src/embed.ts b/kbdb/src/embed.ts index d6c2f68..02cff22 100644 --- a/kbdb/src/embed.ts +++ b/kbdb/src/embed.ts @@ -222,11 +222,13 @@ export interface SemanticHit { * workers-types 原生 typing；design §3.3 的 fan-out fallback 不需啟用）。未帶＝行為不變。 * 註：向量 metadata 的 library 在寫入端已正規化（未標記＝'general'），故 $in 不需 NULL 處理； * 但「建 library metadata index 之前」upsert 的既有向量沒有此欄 → 部署清單強制 reindex backfill。 + * min_score（issue #67）：分數閾值——Vectorize 只會硬湊 topK 筆，低分尾全是無關內容； + * 過濾放查詢端（非 Vectorize 端，API 無此參數）。預設 0＝不過濾（行為與舊版一字不變，向後相容）。 */ export async function semanticSearch( env: Bindings, q: string, - opts: { owner_id?: string; source?: string; entry_type?: string; library?: string[]; topK?: number } = {}, + opts: { owner_id?: string; source?: string; entry_type?: string; library?: string[]; topK?: number; min_score?: number } = {}, ): Promise { if (!embedEnabled(env)) return null; const vec = await embedText(env, q); @@ -241,12 +243,15 @@ export async function semanticSearch( returnMetadata: 'indexed', ...(Object.keys(filter).length ? { filter } : {}), }); - return (res.matches ?? []).map((m) => ({ - id: m.id, - score: m.score, - owner_id: m.metadata?.owner_id as string | undefined, - entry_type: m.metadata?.entry_type as string | undefined, - source: m.metadata?.source as string | undefined, - library: m.metadata?.library as string | undefined, - })); + const minScore = opts.min_score ?? 0; + return (res.matches ?? []) + .filter((m) => m.score >= minScore) + .map((m) => ({ + id: m.id, + score: m.score, + owner_id: m.metadata?.owner_id as string | undefined, + entry_type: m.metadata?.entry_type as string | undefined, + source: m.metadata?.source as string | undefined, + library: m.metadata?.library as string | undefined, + })); } diff --git a/kbdb/src/routes/entries.ts b/kbdb/src/routes/entries.ts index c95d64e..b578c7d 100644 --- a/kbdb/src/routes/entries.ts +++ b/kbdb/src/routes/entries.ts @@ -60,6 +60,11 @@ entryRoutes.get('/', async (c) => { // - entry_type：base 通用 filter（caller 傳任意 type，如 workflow；base 不寫死語意，workflow-discovery Q4）。 // - library：多值庫 filter（逗號分隔，portal-auth P1）。keyword 走 json_extract＋NULL→general； // semantic 走 Vectorize $in。未帶＝全庫（行為不變）。 +// - source：keyword 走 json_extract 謂詞（#66——#5.1 只接了 list 那半，這裡原本解析完即丟）； +// semantic 走 Vectorize metadata filter（原本就有）。 +// - top_k / min_score（#67，semantic 專用）：topK 可調（預設 20、上限 100）＋分數閾值 +// （預設 0＝不過濾）。未帶＝行為與舊版一致（向後相容）；semantic 回應的 entry 另附 score +// 欄讓 caller 自裁（加欄不改形，keyword 路徑不受影響）。 entryRoutes.get('/search', async (c) => { const q = c.req.query('q'); if (!q) return c.json({ success: false, error: 'q required' }, 400); @@ -68,12 +73,19 @@ entryRoutes.get('/search', async (c) => { const entry_type = c.req.query('entry_type') || undefined; const library = parseLibraryParam(c.req.query('library')); const mode = c.req.query('mode') === 'semantic' ? 'semantic' : 'keyword'; + // 數字參數防呆：非數字/非正 → 當沒帶（回預設），不 400——與其他 filter「壞值靜默忽略」一致。 + const topKNum = Number(c.req.query('top_k')); + const top_k = Number.isFinite(topKNum) && topKNum > 0 ? Math.floor(topKNum) : undefined; + const minScoreNum = Number(c.req.query('min_score')); + const min_score = Number.isFinite(minScoreNum) && minScoreNum > 0 ? minScoreNum : undefined; if (mode === 'semantic') { - const hits = await semanticSearch(c.env, q, { owner_id, source, entry_type, library }); + const hits = await semanticSearch(c.env, q, { + owner_id, source, entry_type, library, topK: top_k, min_score, + }); if (hits === null) { // 模組沒開：誠實降級 keyword + 告知「叫 CC 幫你開 vectorize」（不假裝有語義）。 - const entries = await searchEntries(c.env.DB, q, owner_id, entry_type, undefined, library); + const entries = await searchEntries(c.env.DB, q, owner_id, entry_type, undefined, library, source); return c.json({ success: true, entries, @@ -85,13 +97,19 @@ entryRoutes.get('/search', async (c) => { }); } // hydrate vector hits → 完整 entry（保持回應形狀與 keyword 一致）。 - const entries = (await Promise.all(hits.map((h) => getEntry(c.env.DB, h.id)))).filter( - (e): e is NonNullable => e !== null, - ); + // #67：entry 附 score（相似分數）——加欄不改形，既有 caller 不解析多的欄位不受影響。 + const entries = ( + await Promise.all( + hits.map(async (h) => { + const e = await getEntry(c.env.DB, h.id); + return e ? { ...e, score: h.score } : null; + }), + ) + ).filter((e): e is NonNullable => e !== null); return c.json({ success: true, entries, count: entries.length, mode: 'semantic' }); } - const entries = await searchEntries(c.env.DB, q, owner_id, entry_type, undefined, library); + const entries = await searchEntries(c.env.DB, q, owner_id, entry_type, undefined, library, source); return c.json({ success: true, entries, count: entries.length, mode: 'keyword' }); }); diff --git a/kbdb/tests/search-source-and-score.test.ts b/kbdb/tests/search-source-and-score.test.ts new file mode 100644 index 0000000..4a299f7 --- /dev/null +++ b/kbdb/tests/search-source-and-score.test.ts @@ -0,0 +1,213 @@ +// Gitea #66/#67 — /entries/search 兩個檢索缺口的回歸測試。 +// #66：keyword 路徑 source 參數解析後丟棄（#5.1 只接了 listEntries 那半）→ searchEntries 補 +// json_extract 謂詞、route 傳入；含向後相容（不帶 source ＝ SQL 一字不變）。 +// #67：semantic 固定 topK=20、零分數閾值 → route 曝 top_k/min_score、hit 依 min_score 過濾、 +// 回應 entry 附 score；含向後相容（不帶新參數 ＝ 行為不變，僅多 score 資訊）。 +// 測試手法同 library-filter.test.ts：fake D1 捕 SQL 形狀、mock VECTORIZE 捕 query opts—— +// 真 SQL 語意由本機 miniflare 驗（PR 驗收證據）。 +import { describe, it, expect } from 'vitest'; +import { Hono } from 'hono'; +import { entryRoutes } from '../src/routes/entries'; +import { searchEntries } from '../src/actions/entry-crud'; +import { semanticSearch } from '../src/embed'; +import type { Bindings, Entry } from '../src/types'; + +const SOURCE_PREDICATE = "json_extract(metadata_json, '$.source') = ?"; + +// ── fake D1：捕捉 prepared SQL 與 bound params；getEntry（SELECT … WHERE id = ?）回假 entry +// 讓 semantic hydrate 路徑走得完 ── +interface Captured { sql: string; params: unknown[] } +function makeCaptureDB(captured: Captured[]) { + const prepare = (sql: string) => { + const rec: Captured = { sql, params: [] }; + captured.push(rec); + const stmt = { + bind(...args: unknown[]) { rec.params = args; return stmt; }, + async all() { return { results: [] as T[] }; }, + async first() { + if (sql.includes('WHERE id = ?')) return mkEntry(String(rec.params[0])) as unknown as T; + return { total: 0, c: 0 } as unknown as T; + }, + async run() { return { success: true }; }, + }; + return stmt; + }; + return { prepare } as unknown as D1Database; +} + +function mkEntry(id: string): Entry { + return { + id, content: 'some content', entry_type: 'block', owner_id: 'tenant1', parent_id: null, + page_name: null, refs_json: '[]', tags_json: '[]', task_status: null, content_hash: null, + is_embedded: 0, confidence: null, metadata_json: null, created_at: 1, updated_at: 1, + }; +} + +function makeApp(captured: Captured[], extraEnv: Record = {}) { + const app = new Hono<{ Bindings: Bindings }>(); + app.route('/entries', entryRoutes); + const env = { DB: makeCaptureDB(captured), ENVIRONMENT: 'test', ...extraEnv } as unknown as Bindings; + return { app, env }; +} + +// ══ #66 source filter ══════════════════════════════════════════════════════ + +describe('#66 — searchEntries source filter（SQL 形狀）', () => { + it('帶 source → LIKE＋json_extract($.source) 謂詞＋參數（與 listEntries #5.1 同款）', async () => { + const captured: Captured[] = []; + await searchEntries(makeCaptureDB(captured), '遷移', 'tenant1', undefined, undefined, undefined, 'gitea:Leo/kb@main/foo.md'); + expect(captured[0].sql).toContain('content LIKE ?'); + expect(captured[0].sql).toContain(SOURCE_PREDICATE); + expect(captured[0].params).toContain('gitea:Leo/kb@main/foo.md'); + }); + + it('不帶 source → SQL 無 $.source 謂詞（向後相容：行為一字不變）', async () => { + const captured: Captured[] = []; + await searchEntries(makeCaptureDB(captured), '遷移', 'tenant1'); + expect(captured[0].sql).not.toContain('$.source'); + }); + + it('source＋library 併用 → 兩謂詞都在、參數順序對（source 先於 library）', async () => { + const captured: Captured[] = []; + await searchEntries(makeCaptureDB(captured), '遷移', undefined, undefined, undefined, ['finance'], 'src-a'); + expect(captured[0].sql).toContain(SOURCE_PREDICATE); + expect(captured[0].sql).toContain('$.library'); + // params: [%遷移%, 'src-a', 'finance', limit] + expect(captured[0].params[1]).toBe('src-a'); + expect(captured[0].params[2]).toBe('finance'); + }); +}); + +describe('#66 — route GET /entries/search（keyword）source 下傳', () => { + it('?q=x&source=… → 謂詞下到 searchEntries（原 bug：解析完即丟）', async () => { + const captured: Captured[] = []; + const { app, env } = makeApp(captured); + const res = await app.request('/entries/search?q=x&source=gitea%3ALeo%2Fkb%40main%2Ffoo.md', {}, env); + expect(res.status).toBe(200); + expect(captured[0].sql).toContain(SOURCE_PREDICATE); + expect(captured[0].params).toContain('gitea:Leo/kb@main/foo.md'); + }); + + it('不帶 source → SQL 無 $.source（向後相容）', async () => { + const captured: Captured[] = []; + const { app, env } = makeApp(captured); + const res = await app.request('/entries/search?q=x', {}, env); + expect(res.status).toBe(200); + expect(captured[0].sql).not.toContain('$.source'); + }); + + it('semantic 模組未開＋帶 source → 降級 keyword 仍套 source filter（不因降級洩 source）', async () => { + const captured: Captured[] = []; + const { app, env } = makeApp(captured); // 無 VECTORIZE/AI → semanticSearch 回 null + const res = await app.request('/entries/search?q=x&mode=semantic&source=src-a', {}, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { mode: string }; + expect(body.mode).toBe('keyword'); + expect(captured[0].sql).toContain(SOURCE_PREDICATE); + expect(captured[0].params).toContain('src-a'); + }); +}); + +// ══ #67 top_k / min_score ══════════════════════════════════════════════════ + +// mock VECTORIZE：捕 query opts、回三筆遞減分數（0.9 / 0.5 / 0.2）供閾值截斷驗證。 +function makeSemanticEnv(queryCalls: { opts: Record }[]) { + return { + AI: { async run() { return { data: [[0.1, 0.2, 0.3]] }; } }, + VECTORIZE: { + async query(_vec: number[], opts: Record) { + queryCalls.push({ opts }); + return { + matches: [ + { id: 'e-high', score: 0.9, metadata: {} }, + { id: 'e-mid', score: 0.5, metadata: {} }, + { id: 'e-low', score: 0.2, metadata: {} }, + ], + }; + }, + async upsert(v: unknown[]) { return { count: (v as unknown[]).length }; }, + }, + }; +} + +describe('#67 — semanticSearch topK / min_score', () => { + it('不帶新參數 → topK=20、全 matches 回傳（行為與舊版一致）', async () => { + const calls: { opts: Record }[] = []; + const env = { DB: makeCaptureDB([]), ENVIRONMENT: 'test', ...makeSemanticEnv(calls) } as unknown as Bindings; + const hits = await semanticSearch(env, 'query', {}); + expect(calls[0].opts.topK).toBe(20); + expect(hits?.length).toBe(3); + expect(hits?.map((h) => h.score)).toEqual([0.9, 0.5, 0.2]); // score 帶回 + }); + + it('min_score=0.5 → 低分尾截掉（>= 閾值者留）', async () => { + const calls: { opts: Record }[] = []; + const env = { DB: makeCaptureDB([]), ENVIRONMENT: 'test', ...makeSemanticEnv(calls) } as unknown as Bindings; + const hits = await semanticSearch(env, 'query', { min_score: 0.5 }); + expect(hits?.map((h) => h.id)).toEqual(['e-high', 'e-mid']); + }); + + it('topK 透傳且封頂 100', async () => { + const calls: { opts: Record }[] = []; + const env = { DB: makeCaptureDB([]), ENVIRONMENT: 'test', ...makeSemanticEnv(calls) } as unknown as Bindings; + await semanticSearch(env, 'query', { topK: 5 }); + expect(calls[0].opts.topK).toBe(5); + await semanticSearch(env, 'query', { topK: 500 }); + expect(calls[1].opts.topK).toBe(100); + }); +}); + +describe('#67 — route GET /entries/search（semantic）top_k / min_score / score 欄', () => { + function makeSemanticApp(calls: { opts: Record }[], captured: Captured[] = []) { + return makeApp(captured, makeSemanticEnv(calls)); + } + + it('?top_k=5&min_score=0.5 → topK 透傳、低分截掉、entry 附 score', async () => { + const calls: { opts: Record }[] = []; + const { app, env } = makeSemanticApp(calls); + const res = await app.request('/entries/search?q=x&mode=semantic&top_k=5&min_score=0.5', {}, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { mode: string; count: number; entries: (Entry & { score?: number })[] }; + expect(body.mode).toBe('semantic'); + expect(calls[0].opts.topK).toBe(5); + expect(body.count).toBe(2); // 0.2 的低分尾被 min_score 截掉 + expect(body.entries.map((e) => e.id)).toEqual(['e-high', 'e-mid']); + expect(body.entries.map((e) => e.score)).toEqual([0.9, 0.5]); + }); + + it('不帶新參數 → topK=20、全量回傳（行為不變），entry 仍附 score（加欄不改形）', async () => { + const calls: { opts: Record }[] = []; + const { app, env } = makeSemanticApp(calls); + const res = await app.request('/entries/search?q=x&mode=semantic', {}, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { count: number; entries: (Entry & { score?: number })[] }; + expect(calls[0].opts.topK).toBe(20); + expect(body.count).toBe(3); + expect(body.entries[0].score).toBe(0.9); + // 原有欄位一個不少（回應形狀向後相容） + expect(body.entries[0].id).toBe('e-high'); + expect(body.entries[0].entry_type).toBe('block'); + }); + + it('壞值防呆：top_k=abc / top_k=0 / min_score=-1 → 視同沒帶（回預設，不 400）', async () => { + for (const qs of ['top_k=abc', 'top_k=0', 'min_score=-1', 'top_k=abc&min_score=xyz']) { + const calls: { opts: Record }[] = []; + const { app, env } = makeSemanticApp(calls); + const res = await app.request(`/entries/search?q=x&mode=semantic&${qs}`, {}, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { count: number }; + expect(calls[0].opts.topK).toBe(20); + expect(body.count).toBe(3); // 無閾值 → 全量 + } + }); + + it('keyword 路徑不受 top_k/min_score 影響（參數只作用於 semantic）', async () => { + const captured: Captured[] = []; + const { app, env } = makeApp(captured); + const res = await app.request('/entries/search?q=x&top_k=5&min_score=0.9', {}, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { mode: string }; + expect(body.mode).toBe('keyword'); + expect(captured[0].sql).toContain('content LIKE ?'); // SQL 形狀不變 + }); +}); diff --git a/system-dev/wiki/status.md b/system-dev/wiki/status.md index 78b4a42..aa7a9bc 100644 --- a/system-dev/wiki/status.md +++ b/system-dev/wiki/status.md @@ -15,7 +15,19 @@ metadata: ## 📍 當前位置 -> **2026-07-14 本 session（bugfix：PBKDF2 CF runtime 上限＋http_request 1042 flag，分支 `fix-pbkdf2-cf-limit`，PR 待總管審不 merge）**： +> **2026-07-19 本 session（bugfix：kbdb search 兩缺口 #66/#67，分支 +> `fix/search-source-filter-and-semantic-threshold`，雲端總管交辦）**： +> - **#66/#67 修復 PR 已開（雲端總管交辦），等審＋gated 部署**。 +> - #66：`/entries/search` keyword 路徑 source 參數解析後丟棄（#5.1 只接了 listEntries 那半）→ +> `searchEntries` 尾端加 `source?`（positional caller 全不用改）＋同款 json_extract 謂詞， +> route keyword/semantic 降級兩處傳入。 +> - #67：semantic 固定 topK=20 零閾值 → route 曝 `top_k`（預設 20、封頂 100）/`min_score` +> （預設 0＝不過濾），semanticSearch 依閾值截低分尾，回應 entry 附 `score` 欄（加欄不改形， +> 向後相容）。 +> - 驗證：kbdb vitest 33/33 綠（新增 search-source-and-score.test.ts 13 條）＋ tsc 0。 +> 不動表（D6）、不部署——merge 後需 gated redeploy kbdb worker（leo 閘）。 +> +> **2026-07-14 上一 session（bugfix：PBKDF2 CF runtime 上限＋http_request 1042 flag，分支 `fix-pbkdf2-cf-limit`，PR 待總管審不 merge）**： > - **T6-cloud 部署抓到的框架蟲**：CF Workers **正式 runtime** PBKDF2 上限 100,000 iterations—— > portal-auth 設 600k → `crypto.subtle.deriveBits` 真雲直接拒絕 → `/portal/admin/bootstrap` 500 > （uncle6 實撞，證據 arcrun-rag `docs/manual/uncle6-deploy-record.md`）。**miniflare 無此限制＝