Leo/Arcrun
1
0
Fork 0
Code Issues 49 Pull Requests 5 Actions Packages Projects Releases Wiki Activity

fix(kbdb/embed): 語意查詢 metadata 過濾根因修復（Arcrun#11）

Browse Source 
根因：Vectorize index 建了卻從沒建 metadata index。CF Vectorize v2 要對某
metadata 欄位下 filter，必須先為該欄建 metadata index，否則帶 owner_id/
entry_type/source 過濾的語意查詢一律回 0 命中（app 端 filter 接線本來就對）。
且 metadata index 只索引「建立後 upsert」的向量 → 既有向量須重推才會被收錄。

- deploy.ts：加 ensureVectorizeMetadataIndexes()，隨部署冪等建 owner_id/
  entry_type/source（string）三個 metadata index（self-host/官方帳號皆自動）。
- embed.ts / routes/embed.ts：backfillEmbeddings 加 reindex+offset，重推「所有
  embeddable（含 is_embedded=1）」既有向量，讓事後建立的 metadata index 收錄；
  POST /embed/backfill {"reindex":true} 觸發，offset 分頁到 remaining=0。
- wrangler.toml：註解補 create-metadata-index 手動步驟 + reindex 提示。
- tests：mock DB 對齊 LIMIT?/OFFSET? 與 reindex predicate；補 reindex 測試。

leo21c 已驗：owner_id=leo / entry_type 過濾修前 0→修後命中，不帶過濾不變。
已知後續（非本 bug 症狀）：source 值 89-91 bytes 超過 Vectorize string
metadata index 的 64-byte 索引上限 → source 過濾對長值失效，另案。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HJiLCRUU2o3aSpPEzVCt2o
This commit is contained in:
kbdb-cc
2026-07-06 06:03:43 +00:00
parent 4439880bbd
commit befc63cfe0
5 changed files with 102 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kbdb/tests/embed-backfill.test.ts
+38 -9
View File
@@ -4,11 +4,12 @@ import type { Bindings, Entry } from '../src/types';
// ── Minimal in-memory fakes (no Workers runtime) ─────────────────────────────
// The fake DB interprets only the 3 statement shapes backfill issues, by keyword:
// SELECT * ... LIMIT → candidate rows (embeddable & is_embedded=0 & non-empty content)
// UPDATE ... IN (...) → flip is_embedded=1 for the bound ids
// SELECT COUNT(*) → count of remaining candidates
function isCandidate(e: Entry): boolean {
if (e.is_embedded !== 0) return false;
// SELECT * ... LIMIT ? OFFSET ? → candidate rows (embeddable & non-empty content;
// +is_embedded=0 for normal backfill, any for reindex)
// UPDATE ... IN (...) → flip is_embedded=1 for the bound ids
// SELECT COUNT(*) → count of matching candidates
// embeddable = metadata.embed===true & non-empty content（reindex predicate）。
function isEmbeddable(e: Entry): boolean {
if (!e.content || e.content.trim() === '') return false;
try {
const m = JSON.parse(e.metadata_json ?? 'null');
@@ -17,21 +18,28 @@ function isCandidate(e: Entry): boolean {
return false;
}
}
// normal backfill 額外要求 is_embedded=0（漏網補嵌）。
function isCandidate(e: Entry): boolean {
return e.is_embedded === 0 && isEmbeddable(e);
}
function makeFakeDB(store: Entry[]) {
const prepare = (sql: string) => {
// reindex predicate 不含 "is_embedded = 0" → 依 SQL 判斷該用哪個 filter（對齊 embed.ts）。
const pred = /is_embedded = 0/.test(sql) ? isCandidate : isEmbeddable;
let bound: unknown[] = [];
const stmt = {
bind(...args: unknown[]) { bound = args; return stmt; },
async all<T>() {
// SELECT * ... LIMIT ? (limit is the last bound param)
const limit = Number(bound[bound.length - 1]);
const results = store.filter(isCandidate).slice(0, limit) as unknown as T[];
// SELECT * ... LIMIT ? OFFSET ? (bound tail = [..., limit, offset])
const offset = Number(bound[bound.length - 1]);
const limit = Number(bound[bound.length - 2]);
const results = store.filter(pred).slice(offset, offset + limit) as unknown as T[];
return { results };
},
async first<T>() {
// SELECT COUNT(*) as c ...
const c = store.filter(isCandidate).length;
const c = store.filter(pred).length;
return { c } as unknown as T;
},
async run() {
@@ -125,6 +133,27 @@ describe('backfillEmbeddings', () => {
expect(r2.remaining).toBe(0);
});
it('reindex: 重推所有 embeddable（含 is_embedded=1），offset 分頁到 remaining=0（Arcrun#11）', async () => {
// 三筆皆已 is_embedded=1（既有向量）：正常 backfill 不會碰（pending=0），reindex 要全部重推
// 讓事後建立的 Vectorize metadata index 收錄。
const store = [
mkEntry('a', 'x', true, 1), mkEntry('b', 'y', true, 1), mkEntry('c', 'z', true, 1),
];
const env = makeEnv(store, true);
// 正常 backfill：沒有 is_embedded=0 → 什麼都不做（證明「不重推就補不到」）。
const normal = await backfillEmbeddings(env, { limit: 100 });
expect(normal.processed).toBe(0);
// reindex 分頁：第一批 2 筆、remaining=1；第二批 1 筆、remaining=0。
const r1 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 0 });
expect(r1.processed).toBe(2);
expect(r1.remaining).toBe(1);
const r2 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 2 });
expect(r2.processed).toBe(1);
expect(r2.remaining).toBe(0);
const upserts = (env as unknown as { __upserts: { id: string }[] }).__upserts;
expect(upserts.map((u) => u.id).sort()).toEqual(['a', 'b', 'c']);
});
it('status reports pending/embedded counts', async () => {
const store = [mkEntry('e1', 'x', true), mkEntry('e2', 'y', true, 1)];
const env = makeEnv(store, true);