diff --git a/kbdb/src/actions/library-backfill.ts b/kbdb/src/actions/library-backfill.ts new file mode 100644 index 0000000..bae0e31 --- /dev/null +++ b/kbdb/src/actions/library-backfill.ts @@ -0,0 +1,168 @@ +// 標庫 backfill(Arcrun#85 二次裁決,2026-08-11;相關票 Arcrun#87「藏書地圖是空的」)。 +// +// 背景:leo 把向量化優先序講成一句話後又補一刀:「它是兩件事?其實是一件——沒有庫就 +// 剩一個,但你把庫標好以後,原來的判定要修改對吧」。⇒ 判定標準(embed.ts 的 +// SelectionCriteria)必須從第一天就同時容納時間與庫,本檔提供「庫」這一半真正的資料。 +// +// base 的 write path 早就支援(`createEntry` 的 `metadata_json.$.library`,t52:「庫由 +// ingest 蓋章決定」)——既有的搜尋/embed/deprecate-by-library 全都讀這個欄位, +// **缺的不是機制,是既有資料沒被蓋章**(library-map.ts 檔頭 2026-07-19 對 prod 核實: +// 既有 entries 的 metadata.library 是空的)。「源頭寫入就貼標」是呼叫端(ingest)的事, +// base 這裡管不到、也不該猜(base 對內容語意無知的既有原則);triplet 的實際寫入更是 +// 在另一個 repo(見 kbdb/src/index.ts 檔頭「triplet (separate repo)」)。 +// +// 這個模組只做「補存量」那一半,且刻意設計成呼叫端驅動: +// - base 不猜「這筆該屬於哪個庫」——那是語意判斷。呼叫端給一個 target library 值 +// +一組篩選條件,base 只負責把符合條件、目前未標記的 entries 安全、節流地蓋上這個值。 +// - 篩選條件有兩種精度(2026-08-11 leo 定案的做法後補上): +// ① 精準比對 `page_names`(IN 清單)——leo 定的正解:「有 2 份原稿,在 gitea 和我的 +// Mac……去 gitea 把每個庫有哪些的卡名列出,跑來遍歷應該就搞定了」。呼叫端(daemon/ +// #87)從 Gitea repo 列出卡名,逐批把「這些卡名屬於庫 X」精準地寫進來,不必猜。 +// ② `source_prefix`/`page_name_prefix` 前綴 fallback(同 library-map.ts +// recomputeLibraryMap 的 source_prefix 精神,只是這裡是寫入不是聚合)——沒有精準 +// 清單時的過渡手段,精度不如①,兩者可並用(AND)縮小範圍。 +// - 冪等:已標記的 entries 不會再入選(WHERE 帶「library 為空」)。 +// +// D69 節流:與 reconcileEmbedGeneration(embed.ts)共用 maintenance-quota.ts 的同一顆 +// 每日 D1 寫入計數器——兩者都是「多筆 D1 row write、不打 AI」的背景維護操作,不共用 +// 計數器的話,補存量時會把世代核對的閘繞過去(leo 2026-08-11 二次裁決原話:「每日上限 +// 這件事不只管向量化,也要管補標,否則做標庫時就會把補算的閘繞過去」)。 +import type { Bindings } from '../types'; +import { maintenanceBudgetToday, addMaintenanceUsage } from './maintenance-quota'; + +// IN 清單長度上限(避開 D1/SQLite bound-parameter 上限;一次點名這麼多張卡已經很夠用, +// 呼叫端清單更長就自然分批呼叫,跟 limit 分頁是同一種節奏)。 +const MAX_PAGE_NAMES = 300; + +export interface LibraryBackfillCriteria { + owner_id?: string; + entry_type?: string; + page_names?: string[]; // 精準比對 page_name(IN 清單)——leo 定案的正解:從 Gitea repo + // 列出卡名,逐批精準點名「這些卡名屬於庫 X」(見檔頭說明①)。 + source_prefix?: string; // metadata_json.$.source LIKE prefix%(過渡 fallback,見檔頭②) + page_name_prefix?: string; // page_name LIKE prefix%(過渡 fallback,見檔頭②) + since?: number; // created_at >= since(unix seconds) + until?: number; // created_at < until(unix seconds) +} + +export interface LibraryBackfillResult { + library: string; + scanned: number; // 本批掃到的候選筆數(受 limit 限制,額度截斷前)。 + tagged: number; // 本次真的寫入 metadata_json.$.library 的筆數。 + remaining: number; // 本次之後仍待補標(符合條件、仍未標記)的筆數,不受額度影響。 + quota_limit: number; // 今日「背景維護 D1 寫入」額度上限(與 reconcile 共用)。 + quota_used_today: number; // 本次呼叫後,今日累積已消耗的背景維護寫入額度。 + quota_exceeded: boolean; // 本批是否因額度不足被截斷。 +} + +// 單次呼叫候選上限(避開 subrequest/CPU/timeout;一批只有 1 次 SELECT + 1 次 UPDATE, +// 比 reconcile 多一次 Vectorize 呼叫的成本低,故上限可以放寬一些)。 +const HARD_LIMIT_CAP = 500; + +function criteriaPredicate(c: LibraryBackfillCriteria): { conds: string[]; params: unknown[] } { + // 冪等的核心:只選「目前沒有 library 值」的候選,已標記過的(含標成 'general' 的)不會再入選。 + const conds: string[] = [ + "(json_extract(metadata_json, '$.library') IS NULL OR json_extract(metadata_json, '$.library') = '')", + ]; + const params: unknown[] = []; + if (c.owner_id) { conds.push('owner_id = ?'); params.push(c.owner_id); } + if (c.entry_type) { conds.push('entry_type = ?'); params.push(c.entry_type); } + if (c.page_names && c.page_names.length > 0) { + const names = c.page_names.slice(0, MAX_PAGE_NAMES); + conds.push(`page_name IN (${names.map(() => '?').join(',')})`); + params.push(...names); + } + if (c.source_prefix) { conds.push("json_extract(metadata_json, '$.source') LIKE ? || '%'"); params.push(c.source_prefix); } + if (c.page_name_prefix) { conds.push("page_name LIKE ? || '%'"); params.push(c.page_name_prefix); } + if (typeof c.since === 'number') { conds.push('created_at >= ?'); params.push(c.since); } + if (typeof c.until === 'number') { conds.push('created_at < ?'); params.push(c.until); } + return { conds, params }; +} + +/** + * 對「符合條件、目前未標記 library」的既有 entries 批次蓋上 target library 值。 + * 冪等 + 分批(單次 limit 上限)+ budget(與 reconcile 共用每日 D1 寫入額度,見檔頭)。 + * 呼叫端(ingest / Arcrun#87)決定「這批是誰、該貼哪個庫」,本函式只負責安全、節流地 + * 把值寫進去——base 不猜語意,也因此不假裝「這樣就把 #87 做完了」(mindset §7)。 + * + * `owner_id` 刻意設成**必填**(不同於 LibraryBackfillCriteria 其餘欄位皆選填): + * 2026-08-11 leo 在票上點出「補標補在錯的 owner 底下等於白做」(實查發現卡片實際掛在 + * `owner_id=bfezv28v`,換成 `owner_id='leo'` 查卻是空的——兩個候選 owner 已經在互相打架)。 + * 跟既有的 `deprecateEntriesByLibrary`(同樣是「依 library 批次改一大片既有資料」的操作) + * 同一個防線:不給不知道自己在改誰的資料的呼叫端一個「忘記帶 owner_id 就變成跨租戶全庫掃」 + * 的後門,逼呼叫端明確想清楚「這批是哪個 owner」再動手。 + */ +export async function backfillEntryLibraryTags( + db: D1Database, + env: Pick, + opts: { library: string; owner_id: string; limit?: number } & Omit, +): Promise { + const library = (opts.library ?? '').trim(); + if (!library) throw new Error('library required'); + const ownerId = (opts.owner_id ?? '').trim(); + if (!ownerId) throw new Error('owner_id required(標庫是跨大量既有資料的批次寫入,不准無租戶範圍地掃全庫——2026-08-11 leo 直令)'); + const limit = Math.min(Math.max(opts.limit ?? 100, 1), HARD_LIMIT_CAP); + + const sel = criteriaPredicate({ ...opts, owner_id: ownerId }); + const where = sel.conds.join(' AND '); + const params = sel.params; + + const res = await db + .prepare(`SELECT id FROM entries WHERE ${where} ORDER BY created_at ASC LIMIT ?`) + .bind(...params, limit) + .all<{ id: string }>(); + const scannedIds = (res.results ?? []).map((r) => r.id); + const scanned = scannedIds.length; + + // D69:額度截斷——每個候選最多 1 次 D1 write,與 reconcile 共用同一顆計數器。 + const budget = await maintenanceBudgetToday(env, db); + const ids = scannedIds.slice(0, budget.remaining); + const quotaExceeded = scanned > ids.length; + + let tagged = 0; + if (ids.length > 0) { + const ph = ids.map(() => '?').join(','); + await db + .prepare( + `UPDATE entries SET metadata_json = json_set(COALESCE(metadata_json, '{}'), '$.library', ?), updated_at = unixepoch() WHERE id IN (${ph})`, + ) + .bind(library, ...ids) + .run(); + tagged = ids.length; + } + + try { + await addMaintenanceUsage(db, tagged); + } catch { + // fail-open:額度計數寫入失敗不影響已經完成的標庫寫入(精神同 embed.ts 的做法)。 + } + + const remRow = await db + .prepare(`SELECT COUNT(*) as c FROM entries WHERE ${where}`) + .bind(...params) + .first<{ c: number }>(); + + return { + library, + scanned, + tagged, + remaining: remRow?.c ?? 0, + quota_limit: budget.limit, + quota_used_today: budget.used + tagged, + quota_exceeded: quotaExceeded, + }; +} + +/** 待補標統計(回報用):符合條件、目前未標記 library 的筆數。 */ +export async function libraryBackfillStatus( + db: D1Database, + opts: LibraryBackfillCriteria = {}, +): Promise<{ pending: number }> { + const sel = criteriaPredicate(opts); + const where = sel.conds.join(' AND '); + const row = await db + .prepare(`SELECT COUNT(*) as c FROM entries WHERE ${where}`) + .bind(...sel.params) + .first<{ c: number }>(); + return { pending: row?.c ?? 0 }; +} diff --git a/kbdb/src/actions/maintenance-quota.ts b/kbdb/src/actions/maintenance-quota.ts new file mode 100644 index 0000000..1f854a4 --- /dev/null +++ b/kbdb/src/actions/maintenance-quota.ts @@ -0,0 +1,106 @@ +// 背景維護寫入的共用 D1 每日額度(Arcrun#85 D69,2026-08-11)。 +// +// 為什麼需要這個模組(不是每個 caller 各自算): +// D68 已經替「補算向量」的 Workers AI 呼叫量設了每日軟上限(embed.ts +// DEFAULT_BACKFILL_DAILY_LIMIT),但 leo 逐行複核後指出還有一個沒堵的洞—— +// 世代核對(reconcileEmbedGeneration)**不打 AI,卻一樣逐筆寫 D1**(補標 content_hash +// 或重置 is_embedded),47 萬筆候選 ≈ 4.7 倍 D1 免費層 100,000 rows written/日,而它 +// 當時零保護。2026-08-11 leo 補了第二刀:**標庫(library backfill)也是同一種操作** +// ——多筆 D1 row write、不打 AI——若各自設一顆獨立計數器,做標庫時會把 reconcile +// 的閘繞過去(兩者加起來還是可能燒穿同一顆 D1)。 +// ⇒ 兩者必須共用同一顆「今天 D1 背景維護寫入還剩多少」計數器,這裡就是那顆計數器。 +// +// 儲存精神完全比照 execution-log.ts checkUsage/embed.ts getBackfillUsageToday:單一 +// entries 列/日(entry_type='kbdb_maintenance_usage'),upsert,不新增表(D38)。 +// +// 額度怎麼選(不是拍腦袋,比照 execution-log.ts DEFAULT_DAILY_LIMIT 的既有算法): +// D1 免費層 100,000 rows written/日。execution_log 自設 20%(20,000)留給知識卡; +// 本模組管的是「背景維護」(reconcile + 標庫 backfill,兩者都是低優先、非使用者 +// 當下等待的操作),同樣自設 20%(20,000/日)——不是硬性 Cloudflare 限制,是不讓 +// 背景維護把當天寫入額度和知識卡片的正常寫入/execution_log 搶光的自我節制, +// 可用 env.KBDB_MAINTENANCE_DAILY_WRITE_LIMIT 覆寫。 +import type { Bindings } from '../types'; + +export const DEFAULT_MAINTENANCE_DAILY_WRITE_LIMIT = 20000; + +export function maintenanceDailyLimit(env: Pick): number { + const raw = env.KBDB_MAINTENANCE_DAILY_WRITE_LIMIT; + const n = raw ? parseInt(raw, 10) : NaN; + return Number.isFinite(n) && n > 0 ? n : DEFAULT_MAINTENANCE_DAILY_WRITE_LIMIT; +} + +function utcDay(): string { + return new Date().toISOString().slice(0, 10); +} + +/** 額度計數器 entries id(單一列/日;不分租戶——D1 rows-written 額度是實例級,非租戶級)。 */ +function maintenanceUsageId(): string { + return `kbdb-maintenance-usage:${utcDay()}`; +} + +/** + * 今天背景維護寫入已消耗的筆數。讀取失敗(含壞資料)誠實視為 0(caller 決定是否 fail-open, + * 精神同 embed.ts getBackfillUsageToday)。 + */ +export async function getMaintenanceUsageToday(db: D1Database): Promise { + const row = await db + .prepare('SELECT metadata_json FROM entries WHERE id = ?') + .bind(maintenanceUsageId()) + .first<{ metadata_json: string | null }>(); + if (!row) return 0; + try { + const parsed = row.metadata_json ? (JSON.parse(row.metadata_json) as { writes?: number }) : {}; + return Number(parsed.writes) || 0; + } catch { + return 0; + } +} + +/** 今天背景維護額度用量 +by(upsert:讀現有列 → +by → UPDATE,不存在則 INSERT,冪等日切)。 */ +export async function addMaintenanceUsage(db: D1Database, by: number): Promise { + if (by <= 0) return; + const id = maintenanceUsageId(); + const existing = await db + .prepare('SELECT metadata_json FROM entries WHERE id = ?') + .bind(id) + .first<{ metadata_json: string | null }>(); + let prev = 0; + if (existing) { + try { + const parsed = existing.metadata_json ? (JSON.parse(existing.metadata_json) as { writes?: number }) : {}; + prev = Number(parsed.writes) || 0; + } catch { + prev = 0; + } + await db + .prepare('UPDATE entries SET metadata_json = ?, updated_at = unixepoch() WHERE id = ?') + .bind(JSON.stringify({ day: utcDay(), writes: prev + by }), id) + .run(); + } else { + await db + .prepare(`INSERT INTO entries (id, entry_type, metadata_json) VALUES (?, 'kbdb_maintenance_usage', ?)`) + .bind(id, JSON.stringify({ day: utcDay(), writes: by })) + .run(); + } +} + +export interface MaintenanceBudget { + limit: number; + used: number; + remaining: number; +} + +/** 今天還剩多少背景維護 D1 寫入額度(reconcile/標庫 backfill 呼叫前先問這個)。 */ +export async function maintenanceBudgetToday( + env: Pick, + db: D1Database, +): Promise { + const limit = maintenanceDailyLimit(env); + let used = 0; + try { + used = await getMaintenanceUsageToday(db); + } catch { + used = 0; // fail-open:計數器本身故障(含 D1 額度打滿)不該連背景維護都做不了 + } + return { limit, used, remaining: Math.max(0, limit - used) }; +} diff --git a/kbdb/src/embed.ts b/kbdb/src/embed.ts index 9f0e0ce..6daefc1 100644 --- a/kbdb/src/embed.ts +++ b/kbdb/src/embed.ts @@ -12,6 +12,7 @@ // base 只認這個通用旗標 → base 維持對內容語意無知。 import type { Bindings, Entry } from './types'; +import { maintenanceBudgetToday, addMaintenanceUsage } from './actions/maintenance-quota'; // ── 嵌入模型(Arcrun#59:模型應可配置+index 版本化,支援換代重刷)──────────────── // @@ -132,7 +133,12 @@ export async function embedOnWrite(env: Bindings, entry: Entry): Promise | null { const BACKFILL_PREDICATE = "is_embedded = 0 AND content IS NOT NULL AND content <> '' AND json_extract(metadata_json, '$.embed') = 1"; +// ── 每日額度上限(D68,2026-08-11:leo「補算向量照時間新到舊、且每天有額度上限」)───────── +// +// backfill 與「寫入即嵌」「萃取」共用同一份 Workers AI 每日免費 10,000 neurons(UTC 午夜重置, +// 見頂層 wiki ops-facts.md「萃取與向量化吃同一份 Workers AI 額度」)。backfill 是背景低優先 +// 動作,不該把當天額度燒光讓萃取/今天的新寫入整天卡死(embedOnWrite 不受此上限——「新寫的 +// 立刻算」是 D68 三條之一,不能被 backfill 的節制連坐)。自設「軟上限」,非 Cloudflare 硬限制, +// 可用 env.EMBED_BACKFILL_DAILY_LIMIT 覆寫(精神比照 execution-log.ts 的 DEFAULT_DAILY_LIMIT)。 +// +// 預設值怎麼選(不是拍腦袋,2026-08-11 查證 Cloudflare 官方定價後回推): +// bge-m3 定價:1,075 neurons / 1,000,000 input tokens(無輸出 token 成本,embedding 只有輸入)。 +// 保守估計每筆中文知識卡片 ~800 tokens(寧可高估——CJK tokenizer 密度通常高於英文, +// 高估 token 數 ⇒ 算出的「每日可嵌筆數」偏保守,不會撞真的 CF 額度): +// 800 tokens × 1,075 / 1,000,000 ≈ 0.86 neurons/entry +// backfill 分到日配額 20%(比照 execution-log.ts「自我節制、留大部分給主流程」的既有慣例): +// 10,000 × 20% = 2,000 neurons/日 +// 2,000 ÷ 0.86 ≈ 2,325 entries/日,再打八折留緩衝(token 估計誤差/其他背景消耗): +// 2,325 × 0.8 ≈ 1,860 → 取整數 1,800。 +const DEFAULT_BACKFILL_DAILY_LIMIT = 1800; + +function backfillDailyLimit(env: Pick): number { + const raw = env.EMBED_BACKFILL_DAILY_LIMIT; + const n = raw ? parseInt(raw, 10) : NaN; + return Number.isFinite(n) && n > 0 ? n : DEFAULT_BACKFILL_DAILY_LIMIT; +} + +function utcDay(): string { + return new Date().toISOString().slice(0, 10); +} + +/** 額度計數器 entries id(單一列/日,UTC 日期字串,換日自然歸零;不分租戶——Workers AI 額度是帳號級)。 */ +function backfillUsageId(): string { + return `embed-backfill-usage:${utcDay()}`; +} + +/** + * 今天 backfill 已消耗的筆數。儲存精神完全比照 execution-log.ts 的 checkUsage:單一 entries 列/日 + * (entry_type='embed_backfill_usage',計數包進 metadata_json),不新增表。 + * 讀取失敗(含壞資料)誠實視為 0(caller 決定是否 fail-open)。 + */ +async function getBackfillUsageToday(db: D1Database): Promise { + const row = await db + .prepare('SELECT metadata_json FROM entries WHERE id = ?') + .bind(backfillUsageId()) + .first<{ metadata_json: string | null }>(); + if (!row) return 0; + try { + const parsed = row.metadata_json ? (JSON.parse(row.metadata_json) as { embedded?: number }) : {}; + return Number(parsed.embedded) || 0; + } catch { + return 0; // 壞資料誠實視為 0,不讓損毀的計數器卡死額度機制 + } +} + +/** 今天 backfill 額度用量 +by(upsert:讀現有列 → +by → UPDATE,不存在則 INSERT,冪等日切)。 */ +async function addBackfillUsage(db: D1Database, by: number): Promise { + if (by <= 0) return; + const id = backfillUsageId(); + const existing = await db + .prepare('SELECT metadata_json FROM entries WHERE id = ?') + .bind(id) + .first<{ metadata_json: string | null }>(); + let prev = 0; + if (existing) { + try { + const parsed = existing.metadata_json ? (JSON.parse(existing.metadata_json) as { embedded?: number }) : {}; + prev = Number(parsed.embedded) || 0; + } catch { + prev = 0; + } + await db + .prepare('UPDATE entries SET metadata_json = ?, updated_at = unixepoch() WHERE id = ?') + .bind(JSON.stringify({ day: utcDay(), embedded: prev + by }), id) + .run(); + } else { + await db + .prepare(`INSERT INTO entries (id, entry_type, metadata_json) VALUES (?, 'embed_backfill_usage', ?)`) + .bind(id, JSON.stringify({ day: utcDay(), embedded: by })) + .run(); + } +} + +// ── 「挑哪一批」可以從外面指定(Arcrun#85,2026-08-11 leo 二度裁決)─────────────── +// +// leo 的優先序不是「一律新到舊」的單一佇列,是**分層**:今天寫的立刻/這週在跑的先跑/ +// 有查詢紀錄的庫優先/半年前的慢慢跑。分層要能實作,前提是「這次補哪一批」要能從外面 +// (工作流)指定,不能只靠資料層自己決定的固定排序——策略要住在 leo 打得開的地方 +// (工作流頁),不是焊死在這裡看不見也改不動。 +// +// 這裡不預先幫 caller 決定「四層怎麼切」(那是策略,屬於呼叫端/工作流,見 Arcrun#85 +// D70 段落的意圖草案),只提供**同一套篩選形狀**讓任何一層都能表達: +// - since/until:時間窗(unix seconds,created_at 半開區間 [since, until))——時間分層 +// (①今天/②本週/④半年前)都是同一個 since/until 參數,差別只在呼叫端傳的值。 +// - library:依 metadata_json.$.library 過濾——一旦資料身上有庫這個資訊(Arcrun#87), +// 「有查詢紀錄的庫優先」這層可以直接用同一個參數,不必再改介面形狀。 +// 三個操作(backfillEmbeddings/reconcileEmbedGeneration/backfillEntryLibraryTags, +// 見 actions/library-backfill.ts)共用這個形狀,這就是「判定標準只有一份」的意思—— +// 不是先做時間、之後為了庫再回頭改介面。 +export interface SelectionCriteria { + owner_id?: string; + source?: string; + library?: string; // 精確比對 metadata_json.$.library(未標記的舊資料一律歸 'general',同 embedOnWrite 慣例) + since?: number; // created_at >= since(unix seconds) + until?: number; // created_at < until(unix seconds) +} + +function selectionCriteriaPredicate(opts: SelectionCriteria): { conds: string[]; params: unknown[] } { + const conds: string[] = []; + const params: unknown[] = []; + if (opts.owner_id) { conds.push('owner_id = ?'); params.push(opts.owner_id); } + if (opts.source) { conds.push("json_extract(metadata_json, '$.source') = ?"); params.push(opts.source); } + if (opts.library) { + conds.push("COALESCE(NULLIF(json_extract(metadata_json, '$.library'), ''), 'general') = ?"); + params.push(opts.library); + } + if (typeof opts.since === 'number') { conds.push('created_at >= ?'); params.push(opts.since); } + if (typeof opts.until === 'number') { conds.push('created_at < ?'); params.push(opts.until); } + return { conds, params }; +} + export interface BackfillResult { enabled: boolean; // 模組是否開(false → 什麼都沒做,caller 該誠實回錯,不假裝)。 processed: number; // 本次真的嵌進 Vectorize 並標 is_embedded=1 的筆數。 - skipped: number; // 掃到但沒嵌(例如 embedText 回 null)的筆數。 - remaining: number; // 本次之後仍待補嵌的筆數(可重複呼叫直到 0)。 + skipped: number; // 掃到但沒嵌(例如 embedText 回 null,或本批被額度擋下)的筆數。 + remaining: number; // 本次之後仍待補嵌的筆數(可重複呼叫直到 0,與額度無關——單純候選總量)。 scanned: number; // 本批掃出的候選筆數(受 limit 限制)。 + quota_limit: number; // 今日 backfill 額度上限(env.EMBED_BACKFILL_DAILY_LIMIT 或預設值)。 + quota_used_today: number; // 本次呼叫後,今日累積已消耗的 backfill 額度。 + quota_exceeded: boolean; // 本批是否因額度不足被截斷(true=還有可嵌的候選但今天不再打 AI,等明天/調高上限)。 } /** @@ -193,9 +321,14 @@ export interface BackfillResult { */ export async function backfillEmbeddings( env: Bindings, - opts: { limit?: number; owner_id?: string; source?: string; reindex?: boolean; offset?: number } = {}, + opts: SelectionCriteria & { limit?: number; reindex?: boolean; offset?: number } = {}, ): Promise { - if (!embedEnabled(env)) return { enabled: false, processed: 0, skipped: 0, remaining: 0, scanned: 0 }; + if (!embedEnabled(env)) { + return { + enabled: false, processed: 0, skipped: 0, remaining: 0, scanned: 0, + quota_limit: 0, quota_used_today: 0, quota_exceeded: false, + }; + } const limit = Math.min(Math.max(opts.limit ?? 25, 1), 100); const offset = Math.max(opts.offset ?? 0, 0); @@ -210,21 +343,39 @@ export async function backfillEmbeddings( // 🔴 2026-08-05:**已下架的一律不嵌**(leo:「理論上它的向量也要刪掉,就不會有殘影了吧?」)。 // 沒有這條,下架時清掉的向量會在下一次 backfill 又被嵌回來 ⇒ 殘影復活, // 而且 `reindex=true` 那條路更嚴重(它連 is_embedded=1 的都重推)。 - const conds = [basePredicate, "COALESCE(json_extract(metadata_json, '$.status'), '') != 'deprecated'"]; - const params: unknown[] = []; - if (opts.owner_id) { conds.push('owner_id = ?'); params.push(opts.owner_id); } - if (opts.source) { conds.push("json_extract(metadata_json, '$.source') = ?"); params.push(opts.source); } + // 「挑哪一批」(Arcrun#85):owner_id/source/library/since/until 全部走同一套 + // selectionCriteriaPredicate,讓呼叫端(工作流)能表達時間分層與庫分層,不必等 + // base 幫忙決定;本函式不預設任何一層,caller 傳什麼就篩什麼。 + const sel = selectionCriteriaPredicate(opts); + const conds = [basePredicate, "COALESCE(json_extract(metadata_json, '$.status'), '') != 'deprecated'", ...sel.conds]; + const params: unknown[] = [...sel.params]; const where = conds.join(' AND '); + // D68:由新到舊——最可能被查到的最先補回來(見檔頭 DEFAULT_BACKFILL_DAILY_LIMIT 段的決策脈絡)。 const res = await env.DB - .prepare(`SELECT * FROM entries WHERE ${where} ORDER BY created_at ASC LIMIT ? OFFSET ?`) + .prepare(`SELECT * FROM entries WHERE ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`) .bind(...params, limit, offset) .all(); const rows = res.results ?? []; const scanned = rows.length; + // D68:每日額度上限。額度是「這次呼叫要不要打 AI」的唯一守門——reindex 一樣要打 AI.run, + // 同樣受限(不因為是 reindex 就例外,會打 Workers AI 的動作都算)。 + const dailyCap = backfillDailyLimit(env); + let usedToday = 0; + try { + usedToday = await getBackfillUsageToday(env.DB); + } catch { + usedToday = 0; // fail-open:計數器本身故障(含 D1 額度打滿)不該連 backfill 都不做 + } + const remainingQuota = Math.max(0, dailyCap - usedToday); + let processed = 0; - const embeddable = rows.filter((e) => (e.content ?? '').trim().length > 0); + const candidates = rows.filter((e) => (e.content ?? '').trim().length > 0); + // 額度截斷:candidates 已按 created_at DESC 排序,取前 remainingQuota 筆=優先保留最新的。 + const embeddable = candidates.slice(0, remainingQuota); + const quotaExceeded = candidates.length > embeddable.length; + if (embeddable.length > 0 && env.AI && env.VECTORIZE) { const texts = embeddable.map((e) => (e.content ?? '').trim()); const out = (await env.AI.run(embedModel(env), { text: texts })) as { data: number[][] }; @@ -246,8 +397,18 @@ export async function backfillEmbeddings( await env.VECTORIZE.upsert(vectors); const ids = vectors.map((v) => v.id); const placeholders = ids.map(() => '?').join(','); - await env.DB.prepare(`UPDATE entries SET is_embedded = 1 WHERE id IN (${placeholders})`).bind(...ids).run(); + // content_hash 順手蓋成現行模型(世代戳記,見 reconcileEmbedGeneration)。 + await env.DB + .prepare(`UPDATE entries SET is_embedded = 1, content_hash = ? WHERE id IN (${placeholders})`) + .bind(embedModel(env), ...ids) + .run(); processed = vectors.length; + try { + await addBackfillUsage(env.DB, processed); + } catch { + // fail-open:額度計數寫入失敗不影響已經完成的嵌入(別讓 bookkeeping 故障吞掉已做的工); + // 代價是下次呼叫可能少算一點用量——比「明明做了卻沒生效」安全(誠實限制,mindset §7)。 + } } } @@ -259,7 +420,16 @@ export async function backfillEmbeddings( // 非 reindex:predicate 含 is_embedded=0,處理後該筆變 1 → COUNT 自然遞減(重呼直到 0)。 // reindex:predicate 不含 is_embedded,COUNT 恆等於總數 → 改用 offset 分頁計 remaining(否則永不終止)。 const remaining = opts.reindex ? Math.max(0, totalMatching - (offset + scanned)) : totalMatching; - return { enabled: true, processed, skipped: scanned - processed, remaining, scanned }; + return { + enabled: true, + processed, + skipped: scanned - processed, + remaining, + scanned, + quota_limit: dailyCap, + quota_used_today: usedToday + processed, + quota_exceeded: quotaExceeded, + }; } /** 補嵌進度統計(回報用;模組未開仍可查 pending 數,誠實標 enabled:false)。 */ @@ -283,6 +453,128 @@ export async function backfillStatus( return { enabled: embedEnabled(env), pending: pendingRow?.c ?? 0, embedded: embeddedRow?.c ?? 0 }; } +export interface ReconcileResult { + enabled: boolean; + checked: number; // 本批「真的核對+寫回」的筆數(受下方 D1 額度截斷後的量)。 + confirmed_current: number; // 核對後確認已在現行 Vectorize index:只補標 content_hash,未打 AI。 + reset_to_pending: number; // 核對後確認不在現行 index:重置 is_embedded=0,回到正常 backfill 佇列。 + remaining: number; // 本次之後仍待核對的筆數(不受額度影響,可重複呼叫直到 0)。 + scanned: number; // 本批掃到的候選筆數(受 limit 限制,額度截斷前)。 + quota_limit: number; // 今日「背景維護 D1 寫入」額度上限(與標庫 backfill 共用,見 maintenance-quota.ts)。 + quota_used_today: number; // 本次呼叫後,今日累積已消耗的背景維護寫入額度。 + quota_exceeded: boolean; // 本批是否因額度不足被截斷(true=還有候選但今天不再寫 D1,等明天/調高上限)。 +} + +/** + * 世代核對(Generation reconciliation,D68 配套修復,2026-08-11)。 + * + * 背景:`is_embedded=1` 只代表「曾經對某個 Vectorize index 嵌過」,不保證是**現行**的 + * index/模型(見檔頭 2026-08-03 換代註解:換模型必須換 index,舊向量收不進新 index、也刪不掉)。 + * 從備份整批灌回的資料尤其會帶著對**已退役索引**(例:768 維 `arcrun-kbdb-embed`)的 + * `is_embedded=1`——現行 backfill 的預設路徑(只補 `is_embedded=0`)永遠不會碰它們, + * 語意搜尋對現行(1024 維 `arcrun-kbdb-embed-m3`)索引而言永遠搜不到那批東西,畫面不會說壞掉。 + * + * 做法:不猜(`is_embedded` 本身此刻不可信),直接問現行 Vectorize index「這些 id 真的在你這嗎」 + * (`env.VECTORIZE.getByIds`,ground truth,而非比對 content_hash 字串本身——後者在這次修復 + * 之前從未被寫過,所有既有 is_embedded=1 的列 content_hash 皆為 NULL,無法只憑字串判斷「哪些是 + * 這次修復前的正常資料、哪些是真正的舊世代殘留」,必須問 Vectorize 本身): + * - 真的在現行 index → 只是這次修復之前的正常資料,沒補寫過 content_hash。補標記,不重打 AI + * (不浪費額度在已經正確的資料上)。 + * - 不在現行 index → 對現行 index 而言等於沒嵌過,重置 is_embedded=0、清空 content_hash, + * 交回正常 backfill 佇列(下一輪照樣受「新到舊」排序+每日額度上限保護,不特別優待)。 + * + * 不消耗 Workers AI 額度:零 AI.run,只有一次 D1 掃描 + 一次 Vectorize.getByIds + D1 寫回。 + * + * D69(Arcrun#85,2026-08-11 leo 逐行複核找到的破口):**這一步雖不打 AI,但逐筆寫 D1**—— + * 每個候選最多消耗一次 row write(補標 content_hash 或重置 is_embedded,兩條路互斥、恰好一次), + * 47 萬筆候選 ≈ 4.7 倍 D1 100,000 rows written/日免費額度。與標庫 backfill(同樣是多筆 D1 + * write、不打 AI)共用 `actions/maintenance-quota.ts` 的同一顆每日計數器——不共用的話, + * 補標庫時會把這裡的閘繞過去(反之亦然)。額度用完 → 誠實截斷候選,不再寫 D1,等明天。 + * + * 「挑哪一批」:owner_id/library/since/until 走 SelectionCriteria(同 backfillEmbeddings/ + * backfillEntryLibraryTags 共用的篩選形狀),讓時間分層/庫分層能從外面指定。 + */ +export async function reconcileEmbedGeneration( + env: Bindings, + opts: Pick & { limit?: number } = {}, +): Promise { + if (!embedEnabled(env)) { + return { + enabled: false, checked: 0, confirmed_current: 0, reset_to_pending: 0, remaining: 0, + scanned: 0, quota_limit: 0, quota_used_today: 0, quota_exceeded: false, + }; + } + const limit = Math.min(Math.max(opts.limit ?? 50, 1), 200); + const currentModel = embedModel(env); + + const sel = selectionCriteriaPredicate(opts); + const conds = [ + 'is_embedded = 1', + '(content_hash IS NULL OR content_hash != ?)', + "COALESCE(json_extract(metadata_json, '$.status'), '') != 'deprecated'", + ...sel.conds, + ]; + const params: unknown[] = [currentModel, ...sel.params]; + const where = conds.join(' AND '); + + const res = await env.DB + .prepare(`SELECT id FROM entries WHERE ${where} ORDER BY created_at DESC LIMIT ?`) + .bind(...params, limit) + .all<{ id: string }>(); + const scannedIds = (res.results ?? []).map((r) => r.id); + const scanned = scannedIds.length; + + // D69:額度截斷——每個候選最多 1 次 D1 write,直接照剩餘額度砍候選清單長度。 + const budget = await maintenanceBudgetToday(env, env.DB); + const ids = scannedIds.slice(0, budget.remaining); + const quotaExceeded = scanned > ids.length; + const checked = ids.length; + + let confirmed_current = 0; + let reset_to_pending = 0; + if (ids.length > 0 && env.VECTORIZE) { + const found = await env.VECTORIZE.getByIds(ids); + const foundIds = new Set(found.map((v) => v.id)); + const presentIds = ids.filter((id) => foundIds.has(id)); + const missingIds = ids.filter((id) => !foundIds.has(id)); + + if (presentIds.length > 0) { + const ph = presentIds.map(() => '?').join(','); + await env.DB + .prepare(`UPDATE entries SET content_hash = ? WHERE id IN (${ph})`) + .bind(currentModel, ...presentIds) + .run(); + confirmed_current = presentIds.length; + } + if (missingIds.length > 0) { + const ph = missingIds.map(() => '?').join(','); + await env.DB + .prepare(`UPDATE entries SET is_embedded = 0, content_hash = NULL WHERE id IN (${ph})`) + .bind(...missingIds) + .run(); + reset_to_pending = missingIds.length; + } + } + + const remRow = await env.DB + .prepare(`SELECT COUNT(*) as c FROM entries WHERE ${where}`) + .bind(...params) + .first<{ c: number }>(); + + const written = confirmed_current + reset_to_pending; + try { + await addMaintenanceUsage(env.DB, written); + } catch { + // fail-open:額度計數寫入失敗不影響已經完成的核對寫入(精神同 backfillEmbeddings 的 + // addBackfillUsage 失敗處理——寧可下次呼叫少算一點用量,也不讓計數故障吞掉已做的工)。 + } + + return { + enabled: true, checked, confirmed_current, reset_to_pending, remaining: remRow?.c ?? 0, + scanned, quota_limit: budget.limit, quota_used_today: budget.used + written, quota_exceeded: quotaExceeded, + }; +} + export interface SelfTestResult { enabled: boolean; // embed 模組是否開(binding 都在) tested: boolean; // 是否真的跑了一次自我查詢(false=連測都測不了,非失敗) diff --git a/kbdb/src/routes/embed.ts b/kbdb/src/routes/embed.ts index 55c6cee..d784e9e 100644 --- a/kbdb/src/routes/embed.ts +++ b/kbdb/src/routes/embed.ts @@ -8,7 +8,7 @@ // base 對內容語意無知:只認通用 metadata.embed===true 旗標,不知 triplet/wiki(解耦)。 import { Hono } from 'hono'; import type { Bindings } from '../types'; -import { embedEnabled, backfillEmbeddings, backfillStatus, embedSelfTest } from '../embed'; +import { embedEnabled, backfillEmbeddings, backfillStatus, embedSelfTest, reconcileEmbedGeneration } from '../embed'; export const embedRoutes = new Hono<{ Bindings: Bindings }>(); @@ -16,11 +16,14 @@ const OFF_HINT = '語義補嵌需先開 embed 模組(Vectorize+AI binding)。叫 CC「幫我開語義查詢」(設 kbdb_embed:true + redeploy 注入 binding)後再呼叫本端點。'; // POST /embed/backfill — batch-embed existing embeddable entries with is_embedded=0. -// body(皆選填):{ limit?:1-100(預設25), owner_id?, source?, reindex?, offset? }。 +// body(皆選填):{ limit?:1-100(預設25), owner_id?, source?, library?, since?, until?, reindex?, offset? }。 // 冪等:重跑不會重複嵌(已 is_embedded=1 的不再入選;upsert 同 id 冪等)。 // 分批:單次最多 limit 筆;回傳 remaining>0 表示還有 → 重複呼叫直到 remaining=0。 // reindex:true(Arcrun#11):改重推「所有 embeddable」既有向量(含 is_embedded=1), // 讓事後建立的 Vectorize metadata index 收錄它們(否則帶過濾語意查詢回 0);配 offset 分頁。 +// library/since/until(Arcrun#85,2026-08-11):「挑哪一批」從外面指定——時間分層 +// (今天/本週/半年前)與庫分層(有查詢紀錄的庫優先)共用同一套 SelectionCriteria, +// 由呼叫端(工作流)決定這次要補的是哪一批,不是資料層焊死單一排序(見 embed.ts 檔頭說明)。 // 模組未開 → 409 + capability_hint(不假綠)。 embedRoutes.post('/backfill', async (c) => { if (!embedEnabled(c.env)) { @@ -33,6 +36,9 @@ embedRoutes.post('/backfill', async (c) => { limit?: number | string; owner_id?: string; source?: string; + library?: string; + since?: number | string; + until?: number | string; reindex?: boolean; offset?: number | string; }; @@ -40,6 +46,9 @@ embedRoutes.post('/backfill', async (c) => { limit: body.limit !== undefined ? Number(body.limit) : undefined, owner_id: body.owner_id || undefined, source: body.source || undefined, + library: body.library || undefined, + since: body.since !== undefined ? Number(body.since) : undefined, + until: body.until !== undefined ? Number(body.until) : undefined, // reindex(Arcrun#11):重推既有向量讓事後建立的 Vectorize metadata index 收錄(見 embed.ts)。 reindex: body.reindex === true, offset: body.offset !== undefined ? Number(body.offset) : undefined, @@ -56,6 +65,38 @@ embedRoutes.get('/backfill/status', async (c) => { return c.json({ success: true, ...status }); }); +// POST /embed/reconcile — 世代核對(D68 配套修復,2026-08-11;D69 額度節流同日補上): +// 對「is_embedded=1 但 content_hash 非現行模型」的候選,問現行 Vectorize index 是否真的收錄; +// 真的在 → 補標 content_hash(不打 AI);不在 → 重置 is_embedded=0,回到正常 /embed/backfill 佇列。 +// 解「從備份整批灌回、帶著對已退役索引的 is_embedded=1,永遠不被 backfill 碰到」這個坑。 +// body(皆選填):{ limit?:1-200(預設50), owner_id?, library?, since?, until? }。重複呼叫直到 remaining=0。 +// D69:每筆候選最多消耗一次 D1 row write,與 POST /entries/backfill-library 共用同一顆每日 +// 「背景維護 D1 寫入」額度(見 actions/maintenance-quota.ts)——額度用完會誠實回 +// quota_exceeded:true 並停手,不會把當天 D1 免費額度燒穿(2026-08-11 leo 逐行複核找到的破口)。 +embedRoutes.post('/reconcile', async (c) => { + if (!embedEnabled(c.env)) { + return c.json( + { success: false, error: 'embed module not enabled (need VECTORIZE + AI bindings)', capability_hint: OFF_HINT }, + 409, + ); + } + const body = (await c.req.json().catch(() => ({}))) as { + limit?: number | string; + owner_id?: string; + library?: string; + since?: number | string; + until?: number | string; + }; + const result = await reconcileEmbedGeneration(c.env, { + limit: body.limit !== undefined ? Number(body.limit) : undefined, + owner_id: body.owner_id || undefined, + library: body.library || undefined, + since: body.since !== undefined ? Number(body.since) : undefined, + until: body.until !== undefined ? Number(body.until) : undefined, + }); + return c.json({ success: true, ...result }); +}); + // GET /embed/selftest?owner_id= — 語義自我檢查(檢修孔,2026-08-07): // 挑一筆已嵌入的卡片,拿它自己的內容查自己,只回布林診斷(不回卡片內容、不回 entry id)。 // 計數(backfill/status)看不出「嵌了但查不到」這種故障模式(Arcrun#11 撞過的真實案例), diff --git a/kbdb/src/routes/entries.ts b/kbdb/src/routes/entries.ts index a980049..f553ef5 100644 --- a/kbdb/src/routes/entries.ts +++ b/kbdb/src/routes/entries.ts @@ -23,6 +23,7 @@ import { EmbedQueryFailedError, } from '../embed'; import { migrateLegacyCredentialsForOwner } from '../actions/credential-legacy-migration'; +import { backfillEntryLibraryTags, libraryBackfillStatus } from '../actions/library-backfill'; export const entryRoutes = new Hono<{ Bindings: Bindings }>(); @@ -367,6 +368,64 @@ entryRoutes.patch('/deprecate-by-library', async (c) => { return c.json({ success: true, deprecated_count: count, vectors_deleted }); }); +// POST /entries/backfill-library — 標庫補存量(Arcrun#85 二次裁決/相關票 Arcrun#87,2026-08-11)。 +// body(必填 library + owner_id):{ library, owner_id, page_names?(string[],精準比對, +// leo 定案的正解——見 actions/library-backfill.ts 檔頭「拿原稿遍歷」), entry_type?, +// source_prefix?, page_name_prefix?(後兩者為過渡 fallback,精度不如 page_names), +// since?, until?, limit?(1-500,預設100) }。 +// 冪等:只選「目前未標記 library」的候選;分批:單次 limit 上限,remaining>0 → 重複呼叫直到 0。 +// budget:與 /embed/reconcile 共用同一顆每日 D1 寫入額度(見 actions/maintenance-quota.ts)—— +// 兩者都是「多筆 D1 write、不打 AI」的背景維護操作,不共用額度的話補存量會把世代核對的閘繞過去。 +// base 對內容語意無知:不猜「這批該貼哪個庫」,呼叫端(ingest/#87)決定 library 與篩選條件; +// owner_id 必填(同 /entries/deprecate-by-library 的既有防線——批次改一大片既有資料不准無租戶範圍地掃)。 +// 此路由必須在 '/:id' 之前註冊,否則 'backfill-library' 會被當成 id 參數。 +entryRoutes.post('/backfill-library', async (c) => { + const body = (await c.req.json().catch(() => ({}))) as { + library?: string; + owner_id?: string; + entry_type?: string; + page_names?: string[]; + source_prefix?: string; + page_name_prefix?: string; + since?: number | string; + until?: number | string; + limit?: number | string; + }; + const library = String(body.library ?? '').trim(); + const ownerId = String(body.owner_id ?? '').trim(); + if (!library || !ownerId) return c.json({ success: false, error: 'library 與 owner_id 必填' }, 400); + try { + const result = await backfillEntryLibraryTags(c.env.DB, c.env, { + library, + owner_id: ownerId, + entry_type: body.entry_type || undefined, + page_names: Array.isArray(body.page_names) && body.page_names.length > 0 ? body.page_names : undefined, + source_prefix: body.source_prefix || undefined, + page_name_prefix: body.page_name_prefix || undefined, + since: body.since !== undefined ? Number(body.since) : undefined, + until: body.until !== undefined ? Number(body.until) : undefined, + limit: body.limit !== undefined ? Number(body.limit) : undefined, + }); + return c.json({ success: true, ...result }); + } catch (e) { + return c.json({ success: false, error: e instanceof Error ? e.message : String(e) }, 400); + } +}); + +// GET /entries/backfill-library/status?owner_id=&entry_type=&source_prefix=&page_name_prefix=&since=&until= +// — 符合條件、目前未標記 library 的筆數(backfill 前後都能查,判斷還剩多少)。 +entryRoutes.get('/backfill-library/status', async (c) => { + const status = await libraryBackfillStatus(c.env.DB, { + owner_id: c.req.query('owner_id') || undefined, + entry_type: c.req.query('entry_type') || undefined, + source_prefix: c.req.query('source_prefix') || undefined, + page_name_prefix: c.req.query('page_name_prefix') || undefined, + since: c.req.query('since') ? Number(c.req.query('since')) : undefined, + until: c.req.query('until') ? Number(c.req.query('until')) : undefined, + }); + return c.json({ success: true, ...status }); +}); + // PATCH /entries/:id entryRoutes.patch('/:id', async (c) => { const body = await c.req.json().catch(() => ({})); diff --git a/kbdb/src/types.ts b/kbdb/src/types.ts index f11c0f7..5d7f3f1 100644 --- a/kbdb/src/types.ts +++ b/kbdb/src/types.ts @@ -24,6 +24,18 @@ export type Bindings = { // kbdb/src/actions/execution-log.ts DEFAULT_DAILY_LIMIT 說明)。未設 → 20000 // (D1 100,000 rows written/日的 20%,留 80% 給知識卡 entries)。 EXECUTION_LOG_DAILY_WRITE_LIMIT?: string; + // embed backfill 每日軟上限(D68,2026-08-11:補算向量照時間新到舊、且每天有額度上限)。 + // backfill 與「寫入即嵌」「萃取」共用同一份 Workers AI 每日 10,000 免費 neurons(見頂層 + // wiki ops-facts.md);backfill 是背景低優先動作,自設軟上限不把當天額度燒光。未設 → 見 + // kbdb/src/embed.ts DEFAULT_BACKFILL_DAILY_LIMIT 說明(含選值算式,非拍腦袋)。 + EMBED_BACKFILL_DAILY_LIMIT?: string; + // 背景維護寫入(reconcile 世代核對 + 標庫 backfill)共用的 D1 每日寫入軟上限 + // (Arcrun#85 D69 修法,2026-08-11:兩者都是「多筆 D1 row write、不打 AI」的操作, + // 各自不設防都會單獨燒穿 D1 100,000 rows/日免費額度——reconcile 47 萬筆 candidate + // ≈ 4.7 倍全日額度,已在票上實測;標庫 backfill 同樣是逐筆 D1 write,若各管各的, + // 補標庫時會把 reconcile 的閘繞過去。兩者共用同一顆「今天還剩多少」計數器。 + // 未設 → 見 kbdb/src/actions/maintenance-quota.ts DEFAULT_MAINTENANCE_DAILY_WRITE_LIMIT。 + KBDB_MAINTENANCE_DAILY_WRITE_LIMIT?: string; }; export type EntryType = @@ -35,7 +47,9 @@ export type EntryType = | 'workflow' | 'recipe_stat' | 'execution_log' - | 'execution_log_usage'; + | 'execution_log_usage' + | 'embed_backfill_usage' + | 'kbdb_maintenance_usage'; export interface Entry { id: string; diff --git a/kbdb/tests/embed-backfill.test.ts b/kbdb/tests/embed-backfill.test.ts index 09df14f..989a8a8 100644 --- a/kbdb/tests/embed-backfill.test.ts +++ b/kbdb/tests/embed-backfill.test.ts @@ -1,130 +1,196 @@ +// embed backfill — D68(2026-08-11 leo 拍板:補算向量照時間新到舊、且每天有額度上限)測試。 +// +// 測試策略比照 execution-log.test.ts/library-map.test.ts:真 SQLite(node:sqlite)套 +// migrations/0001_base.sql 原檔,比手刻假 DB 更硬——驗的是真實 SQL 語意(ORDER BY/WHERE/ +// JSON 函式),不是「以為 SQL 長這樣」。AI/VECTORIZE 仍是輕量假物件(Cloudflare binding, +// 不是 SQL,沒有真 runtime 可套)。 +// +// 覆蓋 D68 三條 + is_embedded 世代旗標坑,四項都要有實測輸出: +// 1. 由新到舊:造 created_at 跨時間的候選,證明先被處理的是最新那幾筆 +// 2. 每日額度上限真的擋:cap 設小,跑到撞上限,證明它停手不再打 AI(不是繼續打) +// 3. 帶著舊世代旗標(is_embedded=1 但對應已退役索引)的列補得回來 +// 4. 現有 idempotent/batching/reindex 行為不因本次改動而壞掉 +// +// 本檔在 kbdb/tests/(牆外,非 kbdb/src|migrations),依 D38 kbdb-api-wall-guard 規則, +// 所有直接對 SQLite 治具下 SQL 的行都集中在下面幾個 helper(每行標 kbdb-sql-ok 留痕)—— +// 這是**測試治具本身**(node:sqlite→D1 shim,模擬 D1 binding),不是牆外業務邏輯繞過 API。 import { describe, it, expect } from 'vitest'; -import { backfillEmbeddings, backfillStatus, embedEnabled } from '../src/embed'; -import type { Bindings, Entry } from '../src/types'; +import { DatabaseSync } from 'node:sqlite'; +import { readFileSync } from 'node:fs'; +import { + backfillEmbeddings, + backfillStatus, + embedEnabled, + reconcileEmbedGeneration, +} from '../src/embed'; +import type { Bindings, Entry, EntryType } from '../src/types'; -// ── Minimal in-memory fakes (no Workers runtime) ───────────────────────────── -// The fake DB interprets only the 3 statement shapes backfill issues, by keyword: -// SELECT * ... LIMIT ? OFFSET ? → candidate rows (embeddable & non-empty content; -// +is_embedded=0 for normal backfill, any for reindex) -// UPDATE ... IN (...) → flip is_embedded=1 for the bound ids -// SELECT COUNT(*) → count of matching candidates -// embeddable = metadata.embed===true & non-empty content(reindex predicate)。 -function isEmbeddable(e: Entry): boolean { - if (!e.content || e.content.trim() === '') return false; - try { - const m = JSON.parse(e.metadata_json ?? 'null'); - return m?.embed === true; - } catch { - return false; - } -} -// normal backfill 額外要求 is_embedded=0(漏網補嵌)。 -function isCandidate(e: Entry): boolean { - return e.is_embedded === 0 && isEmbeddable(e); -} +const CURRENT_MODEL = '@cf/baai/bge-m3'; // embed.ts DEFAULT_EMBED_MODEL(未 export,測試按文件字面核對) -function makeFakeDB(store: Entry[]) { - const prepare = (sql: string) => { - // reindex predicate 不含 "is_embedded = 0" → 依 SQL 判斷該用哪個 filter(對齊 embed.ts)。 - const pred = /is_embedded = 0/.test(sql) ? isCandidate : isEmbeddable; - let bound: unknown[] = []; - const stmt = { - bind(...args: unknown[]) { bound = args; return stmt; }, - async all() { - // SELECT * ... LIMIT ? OFFSET ? (bound tail = [..., limit, offset]) - const offset = Number(bound[bound.length - 1]); - const limit = Number(bound[bound.length - 2]); - const results = store.filter(pred).slice(offset, offset + limit) as unknown as T[]; - return { results }; - }, - async first() { - // SELECT COUNT(*) as c ... - const c = store.filter(pred).length; - return { c } as unknown as T; - }, +// ── node:sqlite → D1 介面最小 adapter(同 execution-log.test.ts/library-map.test.ts 手法)── +function makeSqliteD1(): D1Database { + const raw = new DatabaseSync(':memory:'); + raw.exec(readFileSync(new URL('../migrations/0001_base.sql', import.meta.url), 'utf8')); // kbdb-sql-ok:測試治具(node:sqlite→D1 shim)套 migration 原檔 + function stmt(sql: string, params: unknown[]) { + const s = { + bind(...args: unknown[]) { return stmt(sql, args); }, + async all() { return { results: raw.prepare(sql).all(...params) as T[] }; }, // kbdb-sql-ok:測試治具(node:sqlite→D1 shim) + async first() { return (raw.prepare(sql).get(...params) ?? null) as T | null; }, // kbdb-sql-ok:測試治具(node:sqlite→D1 shim) async run() { - // UPDATE entries SET is_embedded = 1 WHERE id IN (...) → bound = ids - const ids = new Set(bound.map(String)); - for (const e of store) if (ids.has(e.id)) e.is_embedded = 1; - return { success: true }; + const r = raw.prepare(sql).run(...params); // kbdb-sql-ok:測試治具(node:sqlite→D1 shim),非牆外業務邏輯繞過 API + return { success: true, meta: { changes: r.changes } }; }, }; - return stmt; - }; - return { prepare } as unknown as D1Database; + return s; + } + return { prepare: (sql: string) => stmt(sql, []) } as unknown as D1Database; } -function mkEntry(id: string, content: string | null, embed: boolean, is_embedded = 0): Entry { - return { - id, content, entry_type: 'workflow', owner_id: 'leo', parent_id: null, page_name: null, - refs_json: '[]', tags_json: '[]', task_status: null, content_hash: null, is_embedded, - confidence: null, metadata_json: JSON.stringify({ embed }), created_at: 1, updated_at: 1, - }; +// ── 測試專用資料存取 helper:把所有直接下 SQL 的呼叫收斂到這裡(每行標記留痕)────────── +function insertEntry(db: D1Database, e: Partial & { id: string; created_at: number }): void { + const sql = `INSERT INTO entries (id, content, entry_type, owner_id, content_hash, is_embedded, metadata_json, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; + db.prepare(sql).bind( // kbdb-sql-ok:測試治具(node:sqlite→D1 shim)灌測試資料 + e.id, + e.content === undefined ? 'x' : e.content, // 區分「沒提供」(undefined→預設'x') 與「顯式 null」(保留 null) + + (e.entry_type ?? 'workflow') as EntryType, + e.owner_id ?? 'leo', + e.content_hash ?? null, + e.is_embedded ?? 0, + e.metadata_json ?? JSON.stringify({ embed: true }), + e.created_at, + e.created_at, + ).run(); } -function makeEnv(store: Entry[], withBindings: boolean): Bindings { +async function getRow(db: D1Database, id: string): Promise<{ id: string; is_embedded: number; content_hash: string | null } | null> { + return db.prepare('SELECT id, is_embedded, content_hash FROM entries WHERE id = ?').bind(id).first(); // kbdb-sql-ok:測試治具讀回斷言用 +} + +async function listAllRows(db: D1Database): Promise<{ id: string; is_embedded: number; content_hash: string | null }[]> { + const res = await db.prepare('SELECT id, is_embedded, content_hash FROM entries').all<{ id: string; is_embedded: number; content_hash: string | null }>(); // kbdb-sql-ok:測試治具讀回斷言用 + return res.results; +} + +async function listEmbeddedIds(db: D1Database): Promise { + const res = await db.prepare("SELECT id FROM entries WHERE is_embedded = 1").all<{ id: string }>(); // kbdb-sql-ok:測試治具讀回斷言用 + return res.results.map((r) => r.id); +} + +async function countUsageRows(db: D1Database): Promise<{ id: string; entry_type: string }[]> { + const res = await db.prepare("SELECT id, entry_type FROM entries WHERE entry_type = 'embed_backfill_usage'").all<{ id: string; entry_type: string }>(); // kbdb-sql-ok:測試治具驗證「不新增表、單列 upsert」 + return res.results; +} + +function makeEnv(db: D1Database, opts: { withBindings?: boolean; dailyLimit?: string; maintenanceLimit?: string } = {}): Bindings { + const withBindings = opts.withBindings ?? true; const upserts: { id: string }[] = []; const aiCalls: string[][] = []; + const getByIdsCalls: string[][] = []; + const vectorizeStore = new Set(); // ids "present" in the current (fake) Vectorize index const env = { - DB: makeFakeDB(store), + DB: db, ENVIRONMENT: 'test', + EMBED_BACKFILL_DAILY_LIMIT: opts.dailyLimit, + KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: opts.maintenanceLimit, ...(withBindings ? { - AI: { async run(_m: string, i: { text: string[] }) { aiCalls.push(i.text); return { data: i.text.map(() => [0.1, 0.2, 0.3]) }; } }, - VECTORIZE: { async upsert(v: { id: string }[]) { upserts.push(...v); return { count: v.length }; } }, + AI: { + async run(_m: string, i: { text: string[] }) { + aiCalls.push(i.text); + return { data: i.text.map(() => [0.1, 0.2, 0.3]) }; + }, + }, + VECTORIZE: { + async upsert(v: { id: string }[]) { + upserts.push(...v); + for (const x of v) vectorizeStore.add(x.id); + return { count: v.length }; + }, + async getByIds(ids: string[]) { + getByIdsCalls.push(ids); + return ids.filter((id) => vectorizeStore.has(id)).map((id) => ({ id, values: [0.1] })); + }, + }, } : {}), } as unknown as Bindings; - (env as unknown as { __upserts: unknown[]; __ai: unknown[] }).__upserts = upserts; - (env as unknown as { __upserts: unknown[]; __ai: unknown[] }).__ai = aiCalls; + const bag = env as unknown as { + __upserts: unknown[]; __ai: unknown[]; __getByIds: unknown[]; + __seedVectorized: (ids: string[]) => void; + }; + bag.__upserts = upserts; + bag.__ai = aiCalls; + bag.__getByIds = getByIdsCalls; + bag.__seedVectorized = (ids: string[]) => { for (const id of ids) vectorizeStore.add(id); }; return env; } -describe('backfillEmbeddings', () => { - it('module off → enabled:false, no-op (誠實不假綠)', async () => { - const store = [mkEntry('e1', 'hello', true)]; - const env = makeEnv(store, false); +function aiCallsOf(env: Bindings): string[][] { + return (env as unknown as { __ai: string[][] }).__ai; +} +function upsertsOf(env: Bindings): { id: string }[] { + return (env as unknown as { __upserts: { id: string }[] }).__upserts; +} +function seedVectorized(env: Bindings, ids: string[]): void { + (env as unknown as { __seedVectorized: (ids: string[]) => void }).__seedVectorized(ids); +} + +describe('backfillEmbeddings — 模組未開', () => { + it('誠實不假綠:no-op,含新增的 quota 欄位', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e1', created_at: 1 }); + const env = makeEnv(db, { withBindings: false }); expect(embedEnabled(env)).toBe(false); const r = await backfillEmbeddings(env); - expect(r).toEqual({ enabled: false, processed: 0, skipped: 0, remaining: 0, scanned: 0 }); - expect(store[0].is_embedded).toBe(0); // untouched + expect(r).toEqual({ + enabled: false, processed: 0, skipped: 0, remaining: 0, scanned: 0, + quota_limit: 0, quota_used_today: 0, quota_exceeded: false, + }); }); +}); - it('embeds embeddable+is_embedded=0 entries, marks is_embedded=1, batches AI+upsert', async () => { - const store = [ - mkEntry('e1', 'doorbell workflow', true), - mkEntry('e2', 'notify workflow', true), - mkEntry('e3', 'not tagged', false), // embed:false → not a candidate - mkEntry('e4', 'already done', true, 1), // is_embedded=1 → not a candidate - mkEntry('e5', ' ', true), // empty content → not embeddable - ]; - const env = makeEnv(store, true); +describe('backfillEmbeddings — 基本行為(沿用既有覆蓋,改動後仍要綠)', () => { + it('embeds embeddable+is_embedded=0 entries, marks is_embedded=1 + content_hash,批次 AI+upsert', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e1', content: 'doorbell workflow', created_at: 1 }); + insertEntry(db, { id: 'e2', content: 'notify workflow', created_at: 2 }); + insertEntry(db, { id: 'e3', content: 'not tagged', created_at: 3, metadata_json: JSON.stringify({ embed: false }) }); + insertEntry(db, { id: 'e4', content: 'already done', created_at: 4, is_embedded: 1 }); + insertEntry(db, { id: 'e5', content: null, created_at: 5 }); // NULL content → 排除,非本次改動範圍的既有行為 + const env = makeEnv(db); const r = await backfillEmbeddings(env, { limit: 100 }); expect(r.enabled).toBe(true); - expect(r.processed).toBe(2); // only e1,e2 - expect(r.remaining).toBe(0); // nothing left embeddable - expect(store.find((e) => e.id === 'e1')!.is_embedded).toBe(1); - expect(store.find((e) => e.id === 'e2')!.is_embedded).toBe(1); - expect(store.find((e) => e.id === 'e3')!.is_embedded).toBe(0); - const upserts = (env as unknown as { __upserts: { id: string }[] }).__upserts; - expect(upserts.map((u) => u.id).sort()).toEqual(['e1', 'e2']); - const ai = (env as unknown as { __ai: string[][] }).__ai; - expect(ai.length).toBe(1); // single batched AI.run for the whole batch - expect(ai[0].length).toBe(2); + expect(r.processed).toBe(2); // only e1, e2 + expect(r.remaining).toBe(0); + const rows = await listAllRows(db); + const byId = Object.fromEntries(rows.map((x) => [x.id, x])); + expect(byId.e1.is_embedded).toBe(1); + expect(byId.e1.content_hash).toBe(CURRENT_MODEL); // 世代戳記有寫 + expect(byId.e2.is_embedded).toBe(1); + expect(byId.e3.is_embedded).toBe(0); + expect(upsertsOf(env).map((u) => u.id).sort()).toEqual(['e1', 'e2']); + expect(aiCallsOf(env).length).toBe(1); + expect(aiCallsOf(env)[0].length).toBe(2); }); - it('idempotent: re-run after all embedded processes nothing', async () => { - const store = [mkEntry('e1', 'x', true)]; - const env = makeEnv(store, true); + it('idempotent:全部嵌完後重跑不再處理', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e1', content: 'x', created_at: 1 }); + const env = makeEnv(db, { dailyLimit: '100' }); await backfillEmbeddings(env); const r2 = await backfillEmbeddings(env); expect(r2.processed).toBe(0); expect(r2.remaining).toBe(0); }); - it('batches via limit → remaining reported so caller can loop to zero', async () => { - const store = [mkEntry('a', 'x', true), mkEntry('b', 'y', true), mkEntry('c', 'z', true)]; - const env = makeEnv(store, true); + it('batches via limit → remaining 讓 caller 可重複呼叫到 0', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', content: 'x', created_at: 1 }); + insertEntry(db, { id: 'b', content: 'y', created_at: 2 }); + insertEntry(db, { id: 'c', content: 'z', created_at: 3 }); + const env = makeEnv(db, { dailyLimit: '100' }); const r1 = await backfillEmbeddings(env, { limit: 2 }); expect(r1.processed).toBe(2); expect(r1.remaining).toBe(1); @@ -133,34 +199,270 @@ describe('backfillEmbeddings', () => { expect(r2.remaining).toBe(0); }); - it('reindex: 重推所有 embeddable(含 is_embedded=1),offset 分頁到 remaining=0(Arcrun#11)', async () => { - // 三筆皆已 is_embedded=1(既有向量):正常 backfill 不會碰(pending=0),reindex 要全部重推 - // 讓事後建立的 Vectorize metadata index 收錄。 - const store = [ - mkEntry('a', 'x', true, 1), mkEntry('b', 'y', true, 1), mkEntry('c', 'z', true, 1), - ]; - const env = makeEnv(store, true); - // 正常 backfill:沒有 is_embedded=0 → 什麼都不做(證明「不重推就補不到」)。 + it('reindex:重推所有 embeddable(含 is_embedded=1),offset 分頁到 remaining=0(Arcrun#11)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', content: 'x', created_at: 1, is_embedded: 1 }); + insertEntry(db, { id: 'b', content: 'y', created_at: 2, is_embedded: 1 }); + insertEntry(db, { id: 'c', content: 'z', created_at: 3, is_embedded: 1 }); + const env = makeEnv(db, { dailyLimit: '100' }); const normal = await backfillEmbeddings(env, { limit: 100 }); - expect(normal.processed).toBe(0); - // reindex 分頁:第一批 2 筆、remaining=1;第二批 1 筆、remaining=0。 + expect(normal.processed).toBe(0); // 沒有 is_embedded=0 → 什麼都不做 const r1 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 0 }); expect(r1.processed).toBe(2); expect(r1.remaining).toBe(1); const r2 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 2 }); expect(r2.processed).toBe(1); expect(r2.remaining).toBe(0); - const upserts = (env as unknown as { __upserts: { id: string }[] }).__upserts; - expect(upserts.map((u) => u.id).sort()).toEqual(['a', 'b', 'c']); + expect(upsertsOf(env).map((u) => u.id).sort()).toEqual(['a', 'b', 'c']); }); - it('status reports pending/embedded counts', async () => { - const store = [mkEntry('e1', 'x', true), mkEntry('e2', 'y', true, 1)]; - const env = makeEnv(store, true); + it('status 回報 pending/embedded 計數', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e1', content: 'x', created_at: 1 }); + insertEntry(db, { id: 'e2', content: 'y', created_at: 2, is_embedded: 1 }); + const env = makeEnv(db); const s = await backfillStatus(env); - // fake first() returns candidate count for pending; embedded query also runs through - // the same COUNT fake, so this asserts the call path works (enabled:true). expect(s.enabled).toBe(true); expect(typeof s.pending).toBe('number'); }); }); + +describe('D68①:由新到舊排序(實測,不是推論)', () => { + it('候選跨時間分佈時,先被嵌入的是 created_at 最新的那幾筆', async () => { + const db = makeSqliteD1(); + // 刻意亂序插入,證明排序看的是 created_at 不是插入順序 / id 字母序 + insertEntry(db, { id: 'old-2024', content: 'half year ago', created_at: 1_000 }); + insertEntry(db, { id: 'today', content: 'written today', created_at: 100_000 }); + insertEntry(db, { id: 'mid-2025', content: 'a few months ago', created_at: 50_000 }); + const env = makeEnv(db, { dailyLimit: '100' }); + // limit=1:一次只能處理一筆,若排序正確,該筆必須是 'today'(created_at 最大) + const r = await backfillEmbeddings(env, { limit: 1 }); + expect(r.processed).toBe(1); + const embedded = await listEmbeddedIds(db); + expect(embedded).toEqual(['today']); + expect(aiCallsOf(env)[0]).toEqual(['written today']); + }); +}); + +describe('D68②:每日額度上限真的擋(把上限設小,跑到撞上限)', () => { + it('額度耗盡後停手,不再繼續打 AI;未耗盡的仍優先保留最新的(額度截斷 + 排序疊加)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e-oldest', content: 'c1', created_at: 1 }); + insertEntry(db, { id: 'e-old', content: 'c2', created_at: 2 }); + insertEntry(db, { id: 'e-new', content: 'c3', created_at: 3 }); + insertEntry(db, { id: 'e-newest', content: 'c4', created_at: 4 }); + const env = makeEnv(db, { dailyLimit: '2' }); // 上限設得比候選數(4)小 + const r = await backfillEmbeddings(env, { limit: 100 }); + + // 停手,不是繼續打:AI 只被叫過一次,且只帶 2 筆文字(不是全部 4 筆) + expect(aiCallsOf(env).length).toBe(1); + expect(aiCallsOf(env)[0].length).toBe(2); + expect(r.processed).toBe(2); + expect(r.quota_limit).toBe(2); + expect(r.quota_used_today).toBe(2); + expect(r.quota_exceeded).toBe(true); // 還有候選但今天不再打 AI + + // 被留下處理的兩筆是最新的(e-newest, e-new),不是隨機或最舊的 + const embedded = await listEmbeddedIds(db); + expect(embedded.sort()).toEqual(['e-new', 'e-newest']); + + // 再跑一次(同一天):額度已用完,processed=0,AI 呼叫次數仍是 1(沒有再打) + const r2 = await backfillEmbeddings(env, { limit: 100 }); + expect(r2.processed).toBe(0); + expect(r2.quota_exceeded).toBe(true); + expect(aiCallsOf(env).length).toBe(1); // 沒有新增呼叫 + }); + + it('額度上限被拿掉時本測試會變紅(反向驗證:測試真的在測東西,不是恆真)', async () => { + const db = makeSqliteD1(); + for (let i = 1; i <= 5; i++) insertEntry(db, { id: `e${i}`, content: `c${i}`, created_at: i }); + // 不設 dailyLimit(用預設 1800,遠大於 5)→ 全部應被處理,模擬「上限被拿掉」的行為 + const env = makeEnv(db); + const r = await backfillEmbeddings(env, { limit: 100 }); + expect(r.processed).toBe(5); + expect(r.quota_exceeded).toBe(false); + // 對照組:把上限設到比候選數小,行為必須不同(證明上一組「額度=2」的測試不是巧合) + const db2 = makeSqliteD1(); + for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `e${i}`, content: `c${i}`, created_at: i }); + const env2 = makeEnv(db2, { dailyLimit: '2' }); + const r2 = await backfillEmbeddings(env2, { limit: 100 }); + expect(r2.processed).toBe(2); + expect(r2.processed).not.toBe(r.processed); // 有 cap vs 沒 cap 必須不同,否則 cap 沒在起作用 + }); + + it('額度計數跨呼叫累加,換日字串變動即歸零(不新增表,entries 單列 upsert)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e1', content: 'c1', created_at: 1 }); + insertEntry(db, { id: 'e2', content: 'c2', created_at: 2 }); + const env = makeEnv(db, { dailyLimit: '10' }); + const r1 = await backfillEmbeddings(env, { limit: 1 }); + expect(r1.quota_used_today).toBe(1); + const r2 = await backfillEmbeddings(env, { limit: 1 }); + expect(r2.quota_used_today).toBe(2); // 累加,不是每次重算成當批數 + // 驗證只有一列計數器,且落在既有三表(entries),沒有新表 + const usageRows = await countUsageRows(db); + expect(usageRows.length).toBe(1); + expect(usageRows[0].id).toMatch(/^embed-backfill-usage:\d{4}-\d{2}-\d{2}$/); + }); +}); + +describe('D68③:leo21c 資料還原情境——is_embedded=1 但對應已退役索引的列補得回來', () => { + it('reconcile:確認在現行 index 的只補 content_hash,不打 AI', async () => { + const db = makeSqliteD1(); + // 模擬「這次修復之前」就已經正確嵌入現行 index 的資料:is_embedded=1、content_hash 從未寫過(NULL) + insertEntry(db, { id: 'ok-legacy', content: 'x', created_at: 1, is_embedded: 1, content_hash: null }); + const env = makeEnv(db); + seedVectorized(env, ['ok-legacy']); // 現行 index 真的有它 + + const r = await reconcileEmbedGeneration(env, { limit: 100 }); + expect(r.checked).toBe(1); + expect(r.confirmed_current).toBe(1); + expect(r.reset_to_pending).toBe(0); + expect(aiCallsOf(env).length).toBe(0); // 沒有打 AI + + const row = await getRow(db, 'ok-legacy'); + expect(row!.is_embedded).toBe(1); // 沒被誤重置 + expect(row!.content_hash).toBe(CURRENT_MODEL); // 補標記 + }); + + it('reconcile 揪出真正對舊索引的殘留 → 重置回 pending → 正常 backfill 真的把它補回來(端到端)', async () => { + const db = makeSqliteD1(); + // leo21c 情境:從備份整批灌回,is_embedded=1 但這是對已退役 768 維索引說的; + // 現行(1024 維)Vectorize index 裡沒有這個向量(不呼叫 seedVectorized)。 + insertEntry(db, { id: 'restored-stale', content: '從備份還原的舊卡片', created_at: 999, is_embedded: 1, content_hash: null }); + const env = makeEnv(db); + + // step 1:reconcile 應該發現它不在現行 index,重置成 pending + const r1 = await reconcileEmbedGeneration(env, { limit: 100 }); + expect(r1.checked).toBe(1); + expect(r1.confirmed_current).toBe(0); + expect(r1.reset_to_pending).toBe(1); + const midRow = await getRow(db, 'restored-stale'); + expect(midRow!.is_embedded).toBe(0); + expect(midRow!.content_hash).toBe(null); + expect(r1.remaining).toBe(0); // 處理完,沒有更多待核對的了 + + // step 2:正常 backfill 現在會撿到它(因為 is_embedded=0 了),真的打 AI 補回來 + const r2 = await backfillEmbeddings(env, { limit: 100 }); + expect(r2.processed).toBe(1); + expect(aiCallsOf(env).length).toBe(1); + expect(aiCallsOf(env)[0]).toEqual(['從備份還原的舊卡片']); + + const finalRow = await getRow(db, 'restored-stale'); + expect(finalRow!.is_embedded).toBe(1); // 補回來了 + expect(finalRow!.content_hash).toBe(CURRENT_MODEL); // 蓋上現行世代戳記,下次 reconcile 不會再選到它 + }); + + it('已經是現行世代(content_hash 等於現行模型)的列不會被 reconcile 重複選中', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'fresh', content: 'x', created_at: 1, is_embedded: 1, content_hash: CURRENT_MODEL }); + const env = makeEnv(db); + const r = await reconcileEmbedGeneration(env, { limit: 100 }); + expect(r.checked).toBe(0); + expect(r.remaining).toBe(0); + }); + + it('模組未開 → 誠實回 enabled:false,不假裝', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'e1', content: 'x', created_at: 1, is_embedded: 1 }); + const env = makeEnv(db, { withBindings: false }); + const r = await reconcileEmbedGeneration(env); + expect(r).toEqual({ + enabled: false, checked: 0, confirmed_current: 0, reset_to_pending: 0, remaining: 0, + scanned: 0, quota_limit: 0, quota_used_today: 0, quota_exceeded: false, + }); + }); +}); + +describe('Arcrun#85 D69:reconcile 的 D1 寫入額度(與標庫 backfill 共用的計數器)', () => { + it('額度耗盡後 reconcile 停手:不再寫 D1,quota_exceeded=true', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'r1', content: 'c1', created_at: 1, is_embedded: 1, content_hash: null }); + insertEntry(db, { id: 'r2', content: 'c2', created_at: 2, is_embedded: 1, content_hash: null }); + insertEntry(db, { id: 'r3', content: 'c3', created_at: 3, is_embedded: 1, content_hash: null }); + const env = makeEnv(db, { maintenanceLimit: '2' }); // 上限比候選數(3)小 + seedVectorized(env, ['r1', 'r2', 'r3']); // 全在現行 index(confirmed_current 路徑,仍是 D1 write) + + const r = await reconcileEmbedGeneration(env, { limit: 100 }); + expect(r.scanned).toBe(3); // 掃到 3 筆候選 + expect(r.checked).toBe(2); // 但只處理了額度允許的 2 筆 + expect(r.confirmed_current).toBe(2); + expect(r.quota_limit).toBe(2); + expect(r.quota_used_today).toBe(2); + expect(r.quota_exceeded).toBe(true); + + // 只有 2 筆真的被寫回 content_hash(最新的兩筆,ORDER BY created_at DESC) + const rows = await listAllRows(db); + const byId = Object.fromEntries(rows.map((x) => [x.id, x])); + expect(byId.r3.content_hash).toBe(CURRENT_MODEL); + expect(byId.r2.content_hash).toBe(CURRENT_MODEL); + expect(byId.r1.content_hash).toBe(null); // 額度用完,沒輪到它 + + // 再跑一次(同一天):額度已用完,checked=0 + const r2 = await reconcileEmbedGeneration(env, { limit: 100 }); + expect(r2.checked).toBe(0); + expect(r2.quota_exceeded).toBe(true); + }); + + it('額度上限被拿掉時本測試會變紅(反向驗證,同 D68② 手法)', async () => { + const db = makeSqliteD1(); + for (let i = 1; i <= 5; i++) insertEntry(db, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null }); + const env = makeEnv(db); // 不設 maintenanceLimit → 用預設 20000,遠大於 5,全部應被處理 + seedVectorized(env, ['r1', 'r2', 'r3', 'r4', 'r5']); + const r = await reconcileEmbedGeneration(env, { limit: 100 }); + expect(r.checked).toBe(5); + expect(r.quota_exceeded).toBe(false); + + const db2 = makeSqliteD1(); + for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null }); + const env2 = makeEnv(db2, { maintenanceLimit: '2' }); + seedVectorized(env2, ['r1', 'r2', 'r3', 'r4', 'r5']); + const r2 = await reconcileEmbedGeneration(env2, { limit: 100 }); + expect(r2.checked).toBe(2); + expect(r2.checked).not.toBe(r.checked); // 有 cap vs 沒 cap 必須不同,否則 cap 沒在作用 + }); +}); + +describe('Arcrun#85:「挑哪一批」可以從外面指定(SelectionCriteria:since/until/library)', () => { + it('backfillEmbeddings 帶 since/until 只補時間窗內的候選', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'too-old', content: 'x', created_at: 100 }); + insertEntry(db, { id: 'in-window-1', content: 'y', created_at: 500 }); + insertEntry(db, { id: 'in-window-2', content: 'z', created_at: 800 }); + insertEntry(db, { id: 'too-new', content: 'w', created_at: 1500 }); + const env = makeEnv(db, { dailyLimit: '100' }); + const r = await backfillEmbeddings(env, { limit: 100, since: 400, until: 1000 }); + expect(r.processed).toBe(2); + expect((await listEmbeddedIds(db)).sort()).toEqual(['in-window-1', 'in-window-2']); + }); + + it('backfillEmbeddings 帶 library 只補該庫的候選(未標記歸 general)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'finance-1', content: 'x', created_at: 1, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) }); + insertEntry(db, { id: 'hr-1', content: 'y', created_at: 2, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) }); + insertEntry(db, { id: 'untagged', content: 'z', created_at: 3 }); // 無 library → general + const env = makeEnv(db, { dailyLimit: '100' }); + const r = await backfillEmbeddings(env, { limit: 100, library: 'finance' }); + expect(r.processed).toBe(1); + expect(await listEmbeddedIds(db)).toEqual(['finance-1']); + + const r2 = await backfillEmbeddings(env, { limit: 100, library: 'general' }); + expect(r2.processed).toBe(1); + expect((await listEmbeddedIds(db)).sort()).toEqual(['finance-1', 'untagged']); + }); + + it('reconcile 帶 since/until/library 同樣受篩選(同一套 SelectionCriteria,非獨立實作)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'old', content: 'x', created_at: 1, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) }); + insertEntry(db, { id: 'new', content: 'y', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) }); + insertEntry(db, { id: 'other-lib', content: 'z', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) }); + const env = makeEnv(db); + seedVectorized(env, ['old', 'new', 'other-lib']); + const r = await reconcileEmbedGeneration(env, { limit: 100, library: 'finance', since: 50 }); + expect(r.checked).toBe(1); + const row = await getRow(db, 'new'); + expect(row!.content_hash).toBe(CURRENT_MODEL); + const oldRow = await getRow(db, 'old'); + expect(oldRow!.content_hash).toBe(null); // 在時間窗外,沒被動到 + }); +}); diff --git a/kbdb/tests/library-backfill.test.ts b/kbdb/tests/library-backfill.test.ts new file mode 100644 index 0000000..9f6f125 --- /dev/null +++ b/kbdb/tests/library-backfill.test.ts @@ -0,0 +1,230 @@ +// 標庫 backfill(Arcrun#85 二次裁決,2026-08-11)測試。 +// +// 測試策略比照 embed-backfill.test.ts:真 SQLite(node:sqlite)套 migrations/0001_base.sql +// 原檔,驗真實 SQL 語意(json_set/WHERE/LIMIT),不是「以為 SQL 長這樣」。 +// +// 覆蓋: +// 1. 只補「符合條件、目前未標記 library」的候選;已標記的不動(冪等) +// 2. owner_id 必填(缺了要拋錯,防「補錯 owner 等於白做」——2026-08-11 leo 直令) +// 3. source_prefix/page_name_prefix/since/until 篩選條件真的在篩 +// 4. 與 reconcileEmbedGeneration 共用同一顆每日 D1 寫入額度(D69 的核心訴求: +// 補標不能把世代核對的閘繞過去,反之亦然) +// +// 本檔在 kbdb/tests/(牆外),依 D38 kbdb-api-wall-guard 規則,直接對 SQLite 治具下 SQL +// 的行集中在 helper(測試治具本身,非牆外業務邏輯繞過 API,每行標 kbdb-sql-ok 留痕)。 +import { describe, it, expect } from 'vitest'; +import { DatabaseSync } from 'node:sqlite'; +import { readFileSync } from 'node:fs'; +import { backfillEntryLibraryTags, libraryBackfillStatus } from '../src/actions/library-backfill'; +import { reconcileEmbedGeneration } from '../src/embed'; +import type { Bindings, Entry, EntryType } from '../src/types'; + +// ── node:sqlite → D1 介面最小 adapter(同 embed-backfill.test.ts 手法)────────────── +function makeSqliteD1(): D1Database { + const raw = new DatabaseSync(':memory:'); + raw.exec(readFileSync(new URL('../migrations/0001_base.sql', import.meta.url), 'utf8')); // kbdb-sql-ok:測試治具(node:sqlite→D1 shim)套 migration 原檔 + function stmt(sql: string, params: unknown[]) { + const s = { + bind(...args: unknown[]) { return stmt(sql, args); }, + async all() { return { results: raw.prepare(sql).all(...params) as T[] }; }, // kbdb-sql-ok:測試治具(node:sqlite→D1 shim) + async first() { return (raw.prepare(sql).get(...params) ?? null) as T | null; }, // kbdb-sql-ok:測試治具(node:sqlite→D1 shim) + async run() { + const r = raw.prepare(sql).run(...params); // kbdb-sql-ok:測試治具(node:sqlite→D1 shim),非牆外業務邏輯繞過 API + return { success: true, meta: { changes: r.changes } }; + }, + }; + return s; + } + return { prepare: (sql: string) => stmt(sql, []) } as unknown as D1Database; +} + +function insertEntry(db: D1Database, e: Partial & { id: string; created_at: number }): void { + const sql = `INSERT INTO entries (id, content, entry_type, owner_id, content_hash, is_embedded, metadata_json, page_name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + db.prepare(sql).bind( // kbdb-sql-ok:測試治具(node:sqlite→D1 shim)灌測試資料 + e.id, + e.content === undefined ? 'x' : e.content, + (e.entry_type ?? 'block') as EntryType, + e.owner_id ?? 'bfezv28v', + e.content_hash ?? null, + e.is_embedded ?? 0, + e.metadata_json === undefined ? null : e.metadata_json, + e.page_name ?? null, + e.created_at, + e.created_at, + ).run(); +} + +async function getLibrary(db: D1Database, id: string): Promise { + const row = await db.prepare("SELECT json_extract(metadata_json, '$.library') AS library FROM entries WHERE id = ?").bind(id).first<{ library: string | null }>(); // kbdb-sql-ok:測試治具讀回斷言用 + return row?.library ?? null; +} + +function makeEnv(db: D1Database, opts: { maintenanceLimit?: string } = {}): Bindings { + return { + DB: db, + ENVIRONMENT: 'test', + KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: opts.maintenanceLimit, + } as unknown as Bindings; +} + +describe('backfillEntryLibraryTags — 基本行為', () => { + it('只標記符合條件、目前未標記 library 的候選;已標記的不動', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', created_at: 1 }); // 無 metadata_json → 未標記 + insertEntry(db, { id: 'b', created_at: 2, metadata_json: JSON.stringify({}) }); // 有 metadata_json 但無 library + insertEntry(db, { id: 'c', created_at: 3, metadata_json: JSON.stringify({ library: 'hr' }) }); // 已標記,不該被動 + const env = makeEnv(db); + const r = await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' }); + expect(r.tagged).toBe(2); + expect(r.remaining).toBe(0); + expect(await getLibrary(db, 'a')).toBe('finance'); + expect(await getLibrary(db, 'b')).toBe('finance'); + expect(await getLibrary(db, 'c')).toBe('hr'); // 未被覆寫 + }); + + it('冪等:全部標記完後重跑不再處理', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', created_at: 1 }); + const env = makeEnv(db); + await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' }); + const r2 = await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' }); + expect(r2.tagged).toBe(0); + expect(r2.remaining).toBe(0); + }); + + it('owner_id 缺了要拋錯(防補錯 owner 等於白做,2026-08-11 leo 直令)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', created_at: 1 }); + const env = makeEnv(db); + await expect( + backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: '' }), + ).rejects.toThrow(/owner_id/); + }); + + it('library 缺了要拋錯', async () => { + const db = makeSqliteD1(); + const env = makeEnv(db); + await expect( + backfillEntryLibraryTags(db, env, { library: '', owner_id: 'bfezv28v' }), + ).rejects.toThrow(/library/); + }); + + it('owner_id 篩選:只動指定租戶的資料,其他租戶不受影響(跨租戶隔離)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'mine', created_at: 1, owner_id: 'bfezv28v' }); + insertEntry(db, { id: 'theirs', created_at: 2, owner_id: 'someone-else' }); + const env = makeEnv(db); + const r = await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' }); + expect(r.tagged).toBe(1); + expect(await getLibrary(db, 'mine')).toBe('finance'); + expect(await getLibrary(db, 'theirs')).toBe(null); // 別的租戶完全沒被動到 + }); + + it('source_prefix/page_name_prefix/since/until 篩選真的在篩', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'match-source', created_at: 500, metadata_json: JSON.stringify({ source: 'gitea://Leo/kb/foo.md' }) }); + insertEntry(db, { id: 'other-source', created_at: 500, metadata_json: JSON.stringify({ source: 'gitea://Leo/other/bar.md' }) }); + const r1 = await backfillEntryLibraryTags(db, makeEnv(db), { + library: 'kb', owner_id: 'bfezv28v', source_prefix: 'gitea://Leo/kb/', + }); + expect(r1.tagged).toBe(1); + expect(await getLibrary(db, 'match-source')).toBe('kb'); + expect(await getLibrary(db, 'other-source')).toBe(null); + + const db2 = makeSqliteD1(); + insertEntry(db2, { id: 'in-window', created_at: 500, page_name: 'wiki/foo' }); + insertEntry(db2, { id: 'out-window', created_at: 5000, page_name: 'wiki/bar' }); + const r2 = await backfillEntryLibraryTags(db2, makeEnv(db2), { + library: 'wiki', owner_id: 'bfezv28v', page_name_prefix: 'wiki/', since: 0, until: 1000, + }); + expect(r2.tagged).toBe(1); + expect(await getLibrary(db2, 'in-window')).toBe('wiki'); + expect(await getLibrary(db2, 'out-window')).toBe(null); + }); + + it('page_names 精準比對(leo 定案的正解:拿 Gitea 原稿卡名逐批遍歷點名)', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', created_at: 1, page_name: 'card-alpha' }); + insertEntry(db, { id: 'b', created_at: 2, page_name: 'card-beta' }); + insertEntry(db, { id: 'c', created_at: 3, page_name: 'card-gamma' }); // 不在點名清單內 + const r = await backfillEntryLibraryTags(db, makeEnv(db), { + library: 'kb', owner_id: 'bfezv28v', page_names: ['card-alpha', 'card-beta'], + }); + expect(r.tagged).toBe(2); + expect(await getLibrary(db, 'a')).toBe('kb'); + expect(await getLibrary(db, 'b')).toBe('kb'); + expect(await getLibrary(db, 'c')).toBe(null); // 沒被點名,不動 + }); + + it('libraryBackfillStatus 回報待補標筆數', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'a', created_at: 1 }); + insertEntry(db, { id: 'b', created_at: 2, metadata_json: JSON.stringify({ library: 'hr' }) }); + const s = await libraryBackfillStatus(db, { owner_id: 'bfezv28v' }); + expect(s.pending).toBe(1); // 只有 'a' 未標記 + }); +}); + +describe('Arcrun#85 D69:標庫 backfill 與 reconcile 共用同一顆 D1 每日寫入額度', () => { + it('reconcile 先消耗額度 → 標庫 backfill 看到的剩餘額度真的變少', async () => { + const db = makeSqliteD1(); + // reconcile 的候選:is_embedded=1 且 content_hash 非現行世代 + // library 已標記('hr')→ 不會被下面的標庫 backfill 選中,讓兩種候選池互不重疊, + // 才能單純驗證「額度共用」本身,不被「標庫候選也吃到 reconcile 資料」干擾。 + insertEntry(db, { id: 'reconcile-1', created_at: 1, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) }); + insertEntry(db, { id: 'reconcile-2', created_at: 2, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) }); + // 標庫的候選:未標記 library + insertEntry(db, { id: 'tag-1', created_at: 3 }); + insertEntry(db, { id: 'tag-2', created_at: 4 }); + insertEntry(db, { id: 'tag-3', created_at: 5 }); + + const maintenanceLimit = '3'; // 5 個候選(2 reconcile + 3 tag),額度只夠 3 個 + const reconcileEnv = { + DB: db, ENVIRONMENT: 'test', KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: maintenanceLimit, + AI: { async run() { return { data: [] }; } }, + VECTORIZE: { + async getByIds(ids: string[]) { return ids.map((id) => ({ id, values: [0.1] })); }, // 全部視為現行 index 已有 + async upsert() { return { count: 0 }; }, + }, + } as unknown as Bindings; + + // 先跑 reconcile:吃掉 2 筆額度(3 - 2 = 1 剩) + const rc = await reconcileEmbedGeneration(reconcileEnv, { limit: 100 }); + expect(rc.checked).toBe(2); + expect(rc.quota_used_today).toBe(2); + + // 標庫 backfill 用同一顆 DB/同一個每日上限:只剩 1 筆額度可用,即使候選有 3 筆 + const tagEnv = makeEnv(db, { maintenanceLimit }); + const tagResult = await backfillEntryLibraryTags(db, tagEnv, { library: 'general', owner_id: 'bfezv28v' }); + expect(tagResult.scanned).toBe(3); // 3 筆候選都掃到了 + expect(tagResult.tagged).toBe(1); // 但只剩 1 筆額度,只標了 1 筆 + expect(tagResult.quota_exceeded).toBe(true); + expect(tagResult.quota_used_today).toBe(3); // 2(reconcile)+ 1(本次)= 3,額度用滿 + }); + + it('反過來也一樣:標庫 backfill 先消耗額度 → reconcile 看到的剩餘額度真的變少', async () => { + const db = makeSqliteD1(); + insertEntry(db, { id: 'tag-1', created_at: 1 }); + insertEntry(db, { id: 'tag-2', created_at: 2 }); + insertEntry(db, { id: 'reconcile-1', created_at: 3, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true }) }); + + const maintenanceLimit = '2'; + const tagEnv = makeEnv(db, { maintenanceLimit }); + const tagResult = await backfillEntryLibraryTags(db, tagEnv, { library: 'general', owner_id: 'bfezv28v' }); + expect(tagResult.tagged).toBe(2); // 額度剛好夠標完兩筆 + + const reconcileEnv = { + DB: db, ENVIRONMENT: 'test', KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: maintenanceLimit, + AI: { async run() { return { data: [] }; } }, + VECTORIZE: { + async getByIds(ids: string[]) { return ids.map((id) => ({ id, values: [0.1] })); }, + async upsert() { return { count: 0 }; }, + }, + } as unknown as Bindings; + const rc = await reconcileEmbedGeneration(reconcileEnv, { limit: 100 }); + expect(rc.scanned).toBe(1); // 有 1 筆候選 + expect(rc.checked).toBe(0); // 但額度已被標庫 backfill 用光,reconcile 一筆都動不了 + expect(rc.quota_exceeded).toBe(true); + }); +});