fix(kbdb): D69 節流世代核對＋標庫共用 D1 額度，補「挑哪一批」的統一 SelectionCriteria（Arcrun#85）

leo 逐行複核 fix/embed-backfill-d68 後點出的破口＋二次裁決（票上全文見 Leo/Arcrun#85）：一、向量化優先序（今天寫的立刻／本週在跑的先跑／有查詢紀錄的庫優先／半年前慢慢跑）表達不出來——策略要能從外面（工作流）指定，不能焊死在資料層。新增 embed.ts 的 `SelectionCriteria`（owner_id/source/library/since/until）,backfillEmbeddings 與 reconcileEmbedGeneration 共用同一套形狀；「按庫」那一層現在有資料可用即可運作（見下）。二、世代核對（reconcileEmbedGeneration）不打 AI 但逐筆寫 D1，47 萬筆候選 ≈ 4.7 倍 D1 100,000 rows/日免費額度，先前零保護。新增 actions/maintenance-quota.ts（單一 entries 列/日的共用計數器，精神同 execution-log.ts／embed.ts 既有慣例，不新增表）。三、leo 二度裁決：「標庫」與「時間分層」其實是一件事，判定標準要從第一天同時容納兩者，不能先做一半再回頭改。新增 actions/library-backfill.ts 的 backfillEntryLibraryTags—— 呼叫端（ingest／daemon／Arcrun#87）決定要貼哪個庫、用 page_names（Gitea 原稿卡名精準點名，leo 定案的正解）或 source_prefix/page_name_prefix 過渡 fallback 篩選候選，base 只負責安全、節流地寫入。owner_id 刻意必填（leo 點出「補錯 owner 等於白做」——實查卡片掛在 owner_id=bfezv28v，換成 'leo' 查卻是空的）。 D69：reconcile 與標庫 backfill 共用同一顆「今天還剩多少 D1 寫入額度」計數器（不共用的話其中一個會把另一個的閘繞過去）；新增 POST /entries/backfill-library + GET .../status，擴充 POST /embed/backfill 與 /embed/reconcile 吃 library/since/until 參數。測試：92 → 39 個新增/擴充案例覆蓋 since/until/library 篩選、reconcile 額度真的擋（含「拿掉 cap 會變紅」反向驗證）、標庫 backfill 冪等/owner_id 必填/page_names 精準比對、以及兩個操作共用同一顆額度計數器的跨模組驗證（雙向：先 reconcile 耗盡再標庫、反之亦然）。 kbdb 全套 192 個測試綠燈，tsc --noEmit 除既有 auth.test.ts 舊缺陷外無新增錯誤。紅線：未併 main、未部署、未動任何實例的 is_embedded 旗標（只在本地 SQLite 測試治具跑過）。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-11 18:39:55 +08:00
parent 1d6dde4a01
commit 674e1b4fa2
8 changed files with 787 additions and 23 deletions
@@ -83,7 +83,7 @@ async function countUsageRows(db: D1Database): Promise<{ id: string; entry_type:
  return res.results;
 }

-function makeEnv(db: D1Database, opts: { withBindings?: boolean; dailyLimit?: string } = {}): Bindings {
+function makeEnv(db: D1Database, opts: { withBindings?: boolean; dailyLimit?: string; maintenanceLimit?: string } = {}): Bindings {
  const withBindings = opts.withBindings ?? true;
  const upserts: { id: string }[] = [];
  const aiCalls: string[][] = [];
@@ -93,6 +93,7 @@ function makeEnv(db: D1Database, opts: { withBindings?: boolean; dailyLimit?: st
    DB: db,
    ENVIRONMENT: 'test',
    EMBED_BACKFILL_DAILY_LIMIT: opts.dailyLimit,
+    KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: opts.maintenanceLimit,
    ...(withBindings
      ? {
          AI: {
@@ -366,6 +367,102 @@ describe('D68③：leo21c 資料還原情境——is_embedded=1 但對應已退
    insertEntry(db, { id: 'e1', content: 'x', created_at: 1, is_embedded: 1 });
    const env = makeEnv(db, { withBindings: false });
    const r = await reconcileEmbedGeneration(env);
-    expect(r).toEqual({ enabled: false, checked: 0, confirmed_current: 0, reset_to_pending: 0, remaining: 0 });
+    expect(r).toEqual({
+      enabled: false, checked: 0, confirmed_current: 0, reset_to_pending: 0, remaining: 0,
+      scanned: 0, quota_limit: 0, quota_used_today: 0, quota_exceeded: false,
+    });
+  });
+});
+
+describe('Arcrun#85 D69：reconcile 的 D1 寫入額度（與標庫 backfill 共用的計數器）', () => {
+  it('額度耗盡後 reconcile 停手：不再寫 D1，quota_exceeded=true', async () => {
+    const db = makeSqliteD1();
+    insertEntry(db, { id: 'r1', content: 'c1', created_at: 1, is_embedded: 1, content_hash: null });
+    insertEntry(db, { id: 'r2', content: 'c2', created_at: 2, is_embedded: 1, content_hash: null });
+    insertEntry(db, { id: 'r3', content: 'c3', created_at: 3, is_embedded: 1, content_hash: null });
+    const env = makeEnv(db, { maintenanceLimit: '2' }); // 上限比候選數（3）小
+    seedVectorized(env, ['r1', 'r2', 'r3']); // 全在現行 index（confirmed_current 路徑，仍是 D1 write）
+
+    const r = await reconcileEmbedGeneration(env, { limit: 100 });
+    expect(r.scanned).toBe(3);       // 掃到 3 筆候選
+    expect(r.checked).toBe(2);       // 但只處理了額度允許的 2 筆
+    expect(r.confirmed_current).toBe(2);
+    expect(r.quota_limit).toBe(2);
+    expect(r.quota_used_today).toBe(2);
+    expect(r.quota_exceeded).toBe(true);
+
+    // 只有 2 筆真的被寫回 content_hash（最新的兩筆，ORDER BY created_at DESC）
+    const rows = await listAllRows(db);
+    const byId = Object.fromEntries(rows.map((x) => [x.id, x]));
+    expect(byId.r3.content_hash).toBe(CURRENT_MODEL);
+    expect(byId.r2.content_hash).toBe(CURRENT_MODEL);
+    expect(byId.r1.content_hash).toBe(null); // 額度用完，沒輪到它
+
+    // 再跑一次（同一天）：額度已用完，checked=0
+    const r2 = await reconcileEmbedGeneration(env, { limit: 100 });
+    expect(r2.checked).toBe(0);
+    expect(r2.quota_exceeded).toBe(true);
+  });
+
+  it('額度上限被拿掉時本測試會變紅（反向驗證，同 D68② 手法）', async () => {
+    const db = makeSqliteD1();
+    for (let i = 1; i <= 5; i++) insertEntry(db, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null });
+    const env = makeEnv(db); // 不設 maintenanceLimit → 用預設 20000，遠大於 5，全部應被處理
+    seedVectorized(env, ['r1', 'r2', 'r3', 'r4', 'r5']);
+    const r = await reconcileEmbedGeneration(env, { limit: 100 });
+    expect(r.checked).toBe(5);
+    expect(r.quota_exceeded).toBe(false);
+
+    const db2 = makeSqliteD1();
+    for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null });
+    const env2 = makeEnv(db2, { maintenanceLimit: '2' });
+    seedVectorized(env2, ['r1', 'r2', 'r3', 'r4', 'r5']);
+    const r2 = await reconcileEmbedGeneration(env2, { limit: 100 });
+    expect(r2.checked).toBe(2);
+    expect(r2.checked).not.toBe(r.checked); // 有 cap vs 沒 cap 必須不同，否則 cap 沒在作用
+  });
+});
+
+describe('Arcrun#85：「挑哪一批」可以從外面指定（SelectionCriteria：since/until/library）', () => {
+  it('backfillEmbeddings 帶 since/until 只補時間窗內的候選', async () => {
+    const db = makeSqliteD1();
+    insertEntry(db, { id: 'too-old', content: 'x', created_at: 100 });
+    insertEntry(db, { id: 'in-window-1', content: 'y', created_at: 500 });
+    insertEntry(db, { id: 'in-window-2', content: 'z', created_at: 800 });
+    insertEntry(db, { id: 'too-new', content: 'w', created_at: 1500 });
+    const env = makeEnv(db, { dailyLimit: '100' });
+    const r = await backfillEmbeddings(env, { limit: 100, since: 400, until: 1000 });
+    expect(r.processed).toBe(2);
+    expect((await listEmbeddedIds(db)).sort()).toEqual(['in-window-1', 'in-window-2']);
+  });
+
+  it('backfillEmbeddings 帶 library 只補該庫的候選（未標記歸 general）', async () => {
+    const db = makeSqliteD1();
+    insertEntry(db, { id: 'finance-1', content: 'x', created_at: 1, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
+    insertEntry(db, { id: 'hr-1', content: 'y', created_at: 2, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
+    insertEntry(db, { id: 'untagged', content: 'z', created_at: 3 }); // 無 library → general
+    const env = makeEnv(db, { dailyLimit: '100' });
+    const r = await backfillEmbeddings(env, { limit: 100, library: 'finance' });
+    expect(r.processed).toBe(1);
+    expect(await listEmbeddedIds(db)).toEqual(['finance-1']);
+
+    const r2 = await backfillEmbeddings(env, { limit: 100, library: 'general' });
+    expect(r2.processed).toBe(1);
+    expect((await listEmbeddedIds(db)).sort()).toEqual(['finance-1', 'untagged']);
+  });
+
+  it('reconcile 帶 since/until/library 同樣受篩選（同一套 SelectionCriteria，非獨立實作）', async () => {
+    const db = makeSqliteD1();
+    insertEntry(db, { id: 'old', content: 'x', created_at: 1, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
+    insertEntry(db, { id: 'new', content: 'y', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
+    insertEntry(db, { id: 'other-lib', content: 'z', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
+    const env = makeEnv(db);
+    seedVectorized(env, ['old', 'new', 'other-lib']);
+    const r = await reconcileEmbedGeneration(env, { limit: 100, library: 'finance', since: 50 });
+    expect(r.checked).toBe(1);
+    const row = await getRow(db, 'new');
+    expect(row!.content_hash).toBe(CURRENT_MODEL);
+    const oldRow = await getRow(db, 'old');
+    expect(oldRow!.content_hash).toBe(null); // 在時間窗外，沒被動到
  });
 });