Files
Arcrun/kbdb/tests/embed-backfill.test.ts
T
uncle6me-web 674e1b4fa2 fix(kbdb): D69 節流世代核對+標庫共用 D1 額度,補「挑哪一批」的統一 SelectionCriteria(Arcrun#85)
leo 逐行複核 fix/embed-backfill-d68 後點出的破口+二次裁決(票上全文見 Leo/Arcrun#85):

一、向量化優先序(今天寫的立刻/本週在跑的先跑/有查詢紀錄的庫優先/半年前慢慢跑)表達
不出來——策略要能從外面(工作流)指定,不能焊死在資料層。新增 embed.ts 的
`SelectionCriteria`(owner_id/source/library/since/until),backfillEmbeddings 與
reconcileEmbedGeneration 共用同一套形狀;「按庫」那一層現在有資料可用即可運作(見下)。

二、世代核對(reconcileEmbedGeneration)不打 AI 但逐筆寫 D1,47 萬筆候選 ≈ 4.7 倍 D1
100,000 rows/日免費額度,先前零保護。新增 actions/maintenance-quota.ts(單一 entries
列/日的共用計數器,精神同 execution-log.ts/embed.ts 既有慣例,不新增表)。

三、leo 二度裁決:「標庫」與「時間分層」其實是一件事,判定標準要從第一天同時容納兩者,
不能先做一半再回頭改。新增 actions/library-backfill.ts 的 backfillEntryLibraryTags——
呼叫端(ingest/daemon/Arcrun#87)決定要貼哪個庫、用 page_names(Gitea 原稿卡名精準
點名,leo 定案的正解)或 source_prefix/page_name_prefix 過渡 fallback 篩選候選,base
只負責安全、節流地寫入。owner_id 刻意必填(leo 點出「補錯 owner 等於白做」——實查卡片
掛在 owner_id=bfezv28v,換成 'leo' 查卻是空的)。

D69:reconcile 與標庫 backfill 共用同一顆「今天還剩多少 D1 寫入額度」計數器(不共用的話
其中一個會把另一個的閘繞過去);新增 POST /entries/backfill-library + GET .../status,
擴充 POST /embed/backfill 與 /embed/reconcile 吃 library/since/until 參數。

測試:92 → 39 個新增/擴充案例覆蓋 since/until/library 篩選、reconcile 額度真的擋
(含「拿掉 cap 會變紅」反向驗證)、標庫 backfill 冪等/owner_id 必填/page_names 精準比對、
以及兩個操作共用同一顆額度計數器的跨模組驗證(雙向:先 reconcile 耗盡再標庫、反之亦然)。
kbdb 全套 192 個測試綠燈,tsc --noEmit 除既有 auth.test.ts 舊缺陷外無新增錯誤。

紅線:未併 main、未部署、未動任何實例的 is_embedded 旗標(只在本地 SQLite 測試治具跑過)。

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-11 18:39:55 +08:00

469 lines
24 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// embed backfill — D682026-08-11 leo 拍板:補算向量照時間新到舊、且每天有額度上限)測試。
//
// 測試策略比照 execution-log.test.tslibrary-map.test.ts:真 SQLitenode:sqlite)套
// migrations/0001_base.sql 原檔,比手刻假 DB 更硬——驗的是真實 SQL 語意(ORDER BYWHERE
// JSON 函式),不是「以為 SQL 長這樣」。AI/VECTORIZE 仍是輕量假物件(Cloudflare binding
// 不是 SQL,沒有真 runtime 可套)。
//
// 覆蓋 D68 三條 + is_embedded 世代旗標坑,四項都要有實測輸出:
// 1. 由新到舊:造 created_at 跨時間的候選,證明先被處理的是最新那幾筆
// 2. 每日額度上限真的擋:cap 設小,跑到撞上限,證明它停手不再打 AI(不是繼續打)
// 3. 帶著舊世代旗標(is_embedded=1 但對應已退役索引)的列補得回來
// 4. 現有 idempotentbatchingreindex 行為不因本次改動而壞掉
//
// 本檔在 kbdb/tests/(牆外,非 kbdb/src|migrations),依 D38 kbdb-api-wall-guard 規則,
// 所有直接對 SQLite 治具下 SQL 的行都集中在下面幾個 helper(每行標 kbdb-sql-ok 留痕)——
// 這是**測試治具本身**node:sqlite→D1 shim,模擬 D1 binding),不是牆外業務邏輯繞過 API。
import { describe, it, expect } from 'vitest';
import { DatabaseSync } from 'node:sqlite';
import { readFileSync } from 'node:fs';
import {
backfillEmbeddings,
backfillStatus,
embedEnabled,
reconcileEmbedGeneration,
} from '../src/embed';
import type { Bindings, Entry, EntryType } from '../src/types';
const CURRENT_MODEL = '@cf/baai/bge-m3'; // embed.ts DEFAULT_EMBED_MODEL(未 export,測試按文件字面核對)
// ── node:sqlite → D1 介面最小 adapter(同 execution-log.test.tslibrary-map.test.ts 手法)──
function makeSqliteD1(): D1Database {
const raw = new DatabaseSync(':memory:');
raw.exec(readFileSync(new URL('../migrations/0001_base.sql', import.meta.url), 'utf8')); // kbdb-sql-ok:測試治具(node:sqlite→D1 shim)套 migration 原檔
function stmt(sql: string, params: unknown[]) {
const s = {
bind(...args: unknown[]) { return stmt(sql, args); },
async all<T>() { return { results: raw.prepare(sql).all(...params) as T[] }; }, // kbdb-sql-ok:測試治具(node:sqlite→D1 shim
async first<T>() { return (raw.prepare(sql).get(...params) ?? null) as T | null; }, // kbdb-sql-ok:測試治具(node:sqlite→D1 shim
async run() {
const r = raw.prepare(sql).run(...params); // kbdb-sql-ok:測試治具(node:sqlite→D1 shim),非牆外業務邏輯繞過 API
return { success: true, meta: { changes: r.changes } };
},
};
return s;
}
return { prepare: (sql: string) => stmt(sql, []) } as unknown as D1Database;
}
// ── 測試專用資料存取 helper:把所有直接下 SQL 的呼叫收斂到這裡(每行標記留痕)──────────
function insertEntry(db: D1Database, e: Partial<Entry> & { id: string; created_at: number }): void {
const sql = `INSERT INTO entries (id, content, entry_type, owner_id, content_hash, is_embedded, metadata_json, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
db.prepare(sql).bind( // kbdb-sql-ok:測試治具(node:sqlite→D1 shim)灌測試資料
e.id,
e.content === undefined ? 'x' : e.content, // 區分「沒提供」(undefined→預設'x') 與「顯式 null」(保留 null)
(e.entry_type ?? 'workflow') as EntryType,
e.owner_id ?? 'leo',
e.content_hash ?? null,
e.is_embedded ?? 0,
e.metadata_json ?? JSON.stringify({ embed: true }),
e.created_at,
e.created_at,
).run();
}
async function getRow(db: D1Database, id: string): Promise<{ id: string; is_embedded: number; content_hash: string | null } | null> {
return db.prepare('SELECT id, is_embedded, content_hash FROM entries WHERE id = ?').bind(id).first(); // kbdb-sql-ok:測試治具讀回斷言用
}
async function listAllRows(db: D1Database): Promise<{ id: string; is_embedded: number; content_hash: string | null }[]> {
const res = await db.prepare('SELECT id, is_embedded, content_hash FROM entries').all<{ id: string; is_embedded: number; content_hash: string | null }>(); // kbdb-sql-ok:測試治具讀回斷言用
return res.results;
}
async function listEmbeddedIds(db: D1Database): Promise<string[]> {
const res = await db.prepare("SELECT id FROM entries WHERE is_embedded = 1").all<{ id: string }>(); // kbdb-sql-ok:測試治具讀回斷言用
return res.results.map((r) => r.id);
}
async function countUsageRows(db: D1Database): Promise<{ id: string; entry_type: string }[]> {
const res = await db.prepare("SELECT id, entry_type FROM entries WHERE entry_type = 'embed_backfill_usage'").all<{ id: string; entry_type: string }>(); // kbdb-sql-ok:測試治具驗證「不新增表、單列 upsert」
return res.results;
}
function makeEnv(db: D1Database, opts: { withBindings?: boolean; dailyLimit?: string; maintenanceLimit?: string } = {}): Bindings {
const withBindings = opts.withBindings ?? true;
const upserts: { id: string }[] = [];
const aiCalls: string[][] = [];
const getByIdsCalls: string[][] = [];
const vectorizeStore = new Set<string>(); // ids "present" in the current (fake) Vectorize index
const env = {
DB: db,
ENVIRONMENT: 'test',
EMBED_BACKFILL_DAILY_LIMIT: opts.dailyLimit,
KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: opts.maintenanceLimit,
...(withBindings
? {
AI: {
async run(_m: string, i: { text: string[] }) {
aiCalls.push(i.text);
return { data: i.text.map(() => [0.1, 0.2, 0.3]) };
},
},
VECTORIZE: {
async upsert(v: { id: string }[]) {
upserts.push(...v);
for (const x of v) vectorizeStore.add(x.id);
return { count: v.length };
},
async getByIds(ids: string[]) {
getByIdsCalls.push(ids);
return ids.filter((id) => vectorizeStore.has(id)).map((id) => ({ id, values: [0.1] }));
},
},
}
: {}),
} as unknown as Bindings;
const bag = env as unknown as {
__upserts: unknown[]; __ai: unknown[]; __getByIds: unknown[];
__seedVectorized: (ids: string[]) => void;
};
bag.__upserts = upserts;
bag.__ai = aiCalls;
bag.__getByIds = getByIdsCalls;
bag.__seedVectorized = (ids: string[]) => { for (const id of ids) vectorizeStore.add(id); };
return env;
}
function aiCallsOf(env: Bindings): string[][] {
return (env as unknown as { __ai: string[][] }).__ai;
}
function upsertsOf(env: Bindings): { id: string }[] {
return (env as unknown as { __upserts: { id: string }[] }).__upserts;
}
function seedVectorized(env: Bindings, ids: string[]): void {
(env as unknown as { __seedVectorized: (ids: string[]) => void }).__seedVectorized(ids);
}
describe('backfillEmbeddings — 模組未開', () => {
it('誠實不假綠:no-op,含新增的 quota 欄位', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e1', created_at: 1 });
const env = makeEnv(db, { withBindings: false });
expect(embedEnabled(env)).toBe(false);
const r = await backfillEmbeddings(env);
expect(r).toEqual({
enabled: false, processed: 0, skipped: 0, remaining: 0, scanned: 0,
quota_limit: 0, quota_used_today: 0, quota_exceeded: false,
});
});
});
describe('backfillEmbeddings — 基本行為(沿用既有覆蓋,改動後仍要綠)', () => {
it('embeds embeddable+is_embedded=0 entries, marks is_embedded=1 + content_hash,批次 AI+upsert', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e1', content: 'doorbell workflow', created_at: 1 });
insertEntry(db, { id: 'e2', content: 'notify workflow', created_at: 2 });
insertEntry(db, { id: 'e3', content: 'not tagged', created_at: 3, metadata_json: JSON.stringify({ embed: false }) });
insertEntry(db, { id: 'e4', content: 'already done', created_at: 4, is_embedded: 1 });
insertEntry(db, { id: 'e5', content: null, created_at: 5 }); // NULL content → 排除,非本次改動範圍的既有行為
const env = makeEnv(db);
const r = await backfillEmbeddings(env, { limit: 100 });
expect(r.enabled).toBe(true);
expect(r.processed).toBe(2); // only e1, e2
expect(r.remaining).toBe(0);
const rows = await listAllRows(db);
const byId = Object.fromEntries(rows.map((x) => [x.id, x]));
expect(byId.e1.is_embedded).toBe(1);
expect(byId.e1.content_hash).toBe(CURRENT_MODEL); // 世代戳記有寫
expect(byId.e2.is_embedded).toBe(1);
expect(byId.e3.is_embedded).toBe(0);
expect(upsertsOf(env).map((u) => u.id).sort()).toEqual(['e1', 'e2']);
expect(aiCallsOf(env).length).toBe(1);
expect(aiCallsOf(env)[0].length).toBe(2);
});
it('idempotent:全部嵌完後重跑不再處理', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e1', content: 'x', created_at: 1 });
const env = makeEnv(db, { dailyLimit: '100' });
await backfillEmbeddings(env);
const r2 = await backfillEmbeddings(env);
expect(r2.processed).toBe(0);
expect(r2.remaining).toBe(0);
});
it('batches via limit → remaining 讓 caller 可重複呼叫到 0', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'a', content: 'x', created_at: 1 });
insertEntry(db, { id: 'b', content: 'y', created_at: 2 });
insertEntry(db, { id: 'c', content: 'z', created_at: 3 });
const env = makeEnv(db, { dailyLimit: '100' });
const r1 = await backfillEmbeddings(env, { limit: 2 });
expect(r1.processed).toBe(2);
expect(r1.remaining).toBe(1);
const r2 = await backfillEmbeddings(env, { limit: 2 });
expect(r2.processed).toBe(1);
expect(r2.remaining).toBe(0);
});
it('reindex:重推所有 embeddable(含 is_embedded=1),offset 分頁到 remaining=0Arcrun#11', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'a', content: 'x', created_at: 1, is_embedded: 1 });
insertEntry(db, { id: 'b', content: 'y', created_at: 2, is_embedded: 1 });
insertEntry(db, { id: 'c', content: 'z', created_at: 3, is_embedded: 1 });
const env = makeEnv(db, { dailyLimit: '100' });
const normal = await backfillEmbeddings(env, { limit: 100 });
expect(normal.processed).toBe(0); // 沒有 is_embedded=0 → 什麼都不做
const r1 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 0 });
expect(r1.processed).toBe(2);
expect(r1.remaining).toBe(1);
const r2 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 2 });
expect(r2.processed).toBe(1);
expect(r2.remaining).toBe(0);
expect(upsertsOf(env).map((u) => u.id).sort()).toEqual(['a', 'b', 'c']);
});
it('status 回報 pending/embedded 計數', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e1', content: 'x', created_at: 1 });
insertEntry(db, { id: 'e2', content: 'y', created_at: 2, is_embedded: 1 });
const env = makeEnv(db);
const s = await backfillStatus(env);
expect(s.enabled).toBe(true);
expect(typeof s.pending).toBe('number');
});
});
describe('D68①:由新到舊排序(實測,不是推論)', () => {
it('候選跨時間分佈時,先被嵌入的是 created_at 最新的那幾筆', async () => {
const db = makeSqliteD1();
// 刻意亂序插入,證明排序看的是 created_at 不是插入順序 / id 字母序
insertEntry(db, { id: 'old-2024', content: 'half year ago', created_at: 1_000 });
insertEntry(db, { id: 'today', content: 'written today', created_at: 100_000 });
insertEntry(db, { id: 'mid-2025', content: 'a few months ago', created_at: 50_000 });
const env = makeEnv(db, { dailyLimit: '100' });
// limit=1:一次只能處理一筆,若排序正確,該筆必須是 'today'created_at 最大)
const r = await backfillEmbeddings(env, { limit: 1 });
expect(r.processed).toBe(1);
const embedded = await listEmbeddedIds(db);
expect(embedded).toEqual(['today']);
expect(aiCallsOf(env)[0]).toEqual(['written today']);
});
});
describe('D68②:每日額度上限真的擋(把上限設小,跑到撞上限)', () => {
it('額度耗盡後停手,不再繼續打 AI;未耗盡的仍優先保留最新的(額度截斷 + 排序疊加)', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e-oldest', content: 'c1', created_at: 1 });
insertEntry(db, { id: 'e-old', content: 'c2', created_at: 2 });
insertEntry(db, { id: 'e-new', content: 'c3', created_at: 3 });
insertEntry(db, { id: 'e-newest', content: 'c4', created_at: 4 });
const env = makeEnv(db, { dailyLimit: '2' }); // 上限設得比候選數(4)小
const r = await backfillEmbeddings(env, { limit: 100 });
// 停手,不是繼續打:AI 只被叫過一次,且只帶 2 筆文字(不是全部 4 筆)
expect(aiCallsOf(env).length).toBe(1);
expect(aiCallsOf(env)[0].length).toBe(2);
expect(r.processed).toBe(2);
expect(r.quota_limit).toBe(2);
expect(r.quota_used_today).toBe(2);
expect(r.quota_exceeded).toBe(true); // 還有候選但今天不再打 AI
// 被留下處理的兩筆是最新的(e-newest, e-new),不是隨機或最舊的
const embedded = await listEmbeddedIds(db);
expect(embedded.sort()).toEqual(['e-new', 'e-newest']);
// 再跑一次(同一天):額度已用完,processed=0,AI 呼叫次數仍是 1(沒有再打)
const r2 = await backfillEmbeddings(env, { limit: 100 });
expect(r2.processed).toBe(0);
expect(r2.quota_exceeded).toBe(true);
expect(aiCallsOf(env).length).toBe(1); // 沒有新增呼叫
});
it('額度上限被拿掉時本測試會變紅(反向驗證:測試真的在測東西,不是恆真)', async () => {
const db = makeSqliteD1();
for (let i = 1; i <= 5; i++) insertEntry(db, { id: `e${i}`, content: `c${i}`, created_at: i });
// 不設 dailyLimit(用預設 1800,遠大於 5)→ 全部應被處理,模擬「上限被拿掉」的行為
const env = makeEnv(db);
const r = await backfillEmbeddings(env, { limit: 100 });
expect(r.processed).toBe(5);
expect(r.quota_exceeded).toBe(false);
// 對照組:把上限設到比候選數小,行為必須不同(證明上一組「額度=2」的測試不是巧合)
const db2 = makeSqliteD1();
for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `e${i}`, content: `c${i}`, created_at: i });
const env2 = makeEnv(db2, { dailyLimit: '2' });
const r2 = await backfillEmbeddings(env2, { limit: 100 });
expect(r2.processed).toBe(2);
expect(r2.processed).not.toBe(r.processed); // 有 cap vs 沒 cap 必須不同,否則 cap 沒在起作用
});
it('額度計數跨呼叫累加,換日字串變動即歸零(不新增表,entries 單列 upsert', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e1', content: 'c1', created_at: 1 });
insertEntry(db, { id: 'e2', content: 'c2', created_at: 2 });
const env = makeEnv(db, { dailyLimit: '10' });
const r1 = await backfillEmbeddings(env, { limit: 1 });
expect(r1.quota_used_today).toBe(1);
const r2 = await backfillEmbeddings(env, { limit: 1 });
expect(r2.quota_used_today).toBe(2); // 累加,不是每次重算成當批數
// 驗證只有一列計數器,且落在既有三表(entries),沒有新表
const usageRows = await countUsageRows(db);
expect(usageRows.length).toBe(1);
expect(usageRows[0].id).toMatch(/^embed-backfill-usage:\d{4}-\d{2}-\d{2}$/);
});
});
describe('D68③:leo21c 資料還原情境——is_embedded=1 但對應已退役索引的列補得回來', () => {
it('reconcile:確認在現行 index 的只補 content_hash,不打 AI', async () => {
const db = makeSqliteD1();
// 模擬「這次修復之前」就已經正確嵌入現行 index 的資料:is_embedded=1、content_hash 從未寫過(NULL
insertEntry(db, { id: 'ok-legacy', content: 'x', created_at: 1, is_embedded: 1, content_hash: null });
const env = makeEnv(db);
seedVectorized(env, ['ok-legacy']); // 現行 index 真的有它
const r = await reconcileEmbedGeneration(env, { limit: 100 });
expect(r.checked).toBe(1);
expect(r.confirmed_current).toBe(1);
expect(r.reset_to_pending).toBe(0);
expect(aiCallsOf(env).length).toBe(0); // 沒有打 AI
const row = await getRow(db, 'ok-legacy');
expect(row!.is_embedded).toBe(1); // 沒被誤重置
expect(row!.content_hash).toBe(CURRENT_MODEL); // 補標記
});
it('reconcile 揪出真正對舊索引的殘留 → 重置回 pending → 正常 backfill 真的把它補回來(端到端)', async () => {
const db = makeSqliteD1();
// leo21c 情境:從備份整批灌回,is_embedded=1 但這是對已退役 768 維索引說的;
// 現行(1024 維)Vectorize index 裡沒有這個向量(不呼叫 seedVectorized)。
insertEntry(db, { id: 'restored-stale', content: '從備份還原的舊卡片', created_at: 999, is_embedded: 1, content_hash: null });
const env = makeEnv(db);
// step 1reconcile 應該發現它不在現行 index,重置成 pending
const r1 = await reconcileEmbedGeneration(env, { limit: 100 });
expect(r1.checked).toBe(1);
expect(r1.confirmed_current).toBe(0);
expect(r1.reset_to_pending).toBe(1);
const midRow = await getRow(db, 'restored-stale');
expect(midRow!.is_embedded).toBe(0);
expect(midRow!.content_hash).toBe(null);
expect(r1.remaining).toBe(0); // 處理完,沒有更多待核對的了
// step 2:正常 backfill 現在會撿到它(因為 is_embedded=0 了),真的打 AI 補回來
const r2 = await backfillEmbeddings(env, { limit: 100 });
expect(r2.processed).toBe(1);
expect(aiCallsOf(env).length).toBe(1);
expect(aiCallsOf(env)[0]).toEqual(['從備份還原的舊卡片']);
const finalRow = await getRow(db, 'restored-stale');
expect(finalRow!.is_embedded).toBe(1); // 補回來了
expect(finalRow!.content_hash).toBe(CURRENT_MODEL); // 蓋上現行世代戳記,下次 reconcile 不會再選到它
});
it('已經是現行世代(content_hash 等於現行模型)的列不會被 reconcile 重複選中', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'fresh', content: 'x', created_at: 1, is_embedded: 1, content_hash: CURRENT_MODEL });
const env = makeEnv(db);
const r = await reconcileEmbedGeneration(env, { limit: 100 });
expect(r.checked).toBe(0);
expect(r.remaining).toBe(0);
});
it('模組未開 → 誠實回 enabled:false,不假裝', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'e1', content: 'x', created_at: 1, is_embedded: 1 });
const env = makeEnv(db, { withBindings: false });
const r = await reconcileEmbedGeneration(env);
expect(r).toEqual({
enabled: false, checked: 0, confirmed_current: 0, reset_to_pending: 0, remaining: 0,
scanned: 0, quota_limit: 0, quota_used_today: 0, quota_exceeded: false,
});
});
});
describe('Arcrun#85 D69reconcile 的 D1 寫入額度(與標庫 backfill 共用的計數器)', () => {
it('額度耗盡後 reconcile 停手:不再寫 D1quota_exceeded=true', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'r1', content: 'c1', created_at: 1, is_embedded: 1, content_hash: null });
insertEntry(db, { id: 'r2', content: 'c2', created_at: 2, is_embedded: 1, content_hash: null });
insertEntry(db, { id: 'r3', content: 'c3', created_at: 3, is_embedded: 1, content_hash: null });
const env = makeEnv(db, { maintenanceLimit: '2' }); // 上限比候選數(3)小
seedVectorized(env, ['r1', 'r2', 'r3']); // 全在現行 indexconfirmed_current 路徑,仍是 D1 write
const r = await reconcileEmbedGeneration(env, { limit: 100 });
expect(r.scanned).toBe(3); // 掃到 3 筆候選
expect(r.checked).toBe(2); // 但只處理了額度允許的 2 筆
expect(r.confirmed_current).toBe(2);
expect(r.quota_limit).toBe(2);
expect(r.quota_used_today).toBe(2);
expect(r.quota_exceeded).toBe(true);
// 只有 2 筆真的被寫回 content_hash(最新的兩筆,ORDER BY created_at DESC
const rows = await listAllRows(db);
const byId = Object.fromEntries(rows.map((x) => [x.id, x]));
expect(byId.r3.content_hash).toBe(CURRENT_MODEL);
expect(byId.r2.content_hash).toBe(CURRENT_MODEL);
expect(byId.r1.content_hash).toBe(null); // 額度用完,沒輪到它
// 再跑一次(同一天):額度已用完,checked=0
const r2 = await reconcileEmbedGeneration(env, { limit: 100 });
expect(r2.checked).toBe(0);
expect(r2.quota_exceeded).toBe(true);
});
it('額度上限被拿掉時本測試會變紅(反向驗證,同 D68② 手法)', async () => {
const db = makeSqliteD1();
for (let i = 1; i <= 5; i++) insertEntry(db, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null });
const env = makeEnv(db); // 不設 maintenanceLimit → 用預設 20000,遠大於 5,全部應被處理
seedVectorized(env, ['r1', 'r2', 'r3', 'r4', 'r5']);
const r = await reconcileEmbedGeneration(env, { limit: 100 });
expect(r.checked).toBe(5);
expect(r.quota_exceeded).toBe(false);
const db2 = makeSqliteD1();
for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null });
const env2 = makeEnv(db2, { maintenanceLimit: '2' });
seedVectorized(env2, ['r1', 'r2', 'r3', 'r4', 'r5']);
const r2 = await reconcileEmbedGeneration(env2, { limit: 100 });
expect(r2.checked).toBe(2);
expect(r2.checked).not.toBe(r.checked); // 有 cap vs 沒 cap 必須不同,否則 cap 沒在作用
});
});
describe('Arcrun#85:「挑哪一批」可以從外面指定(SelectionCriteriasince/until/library', () => {
it('backfillEmbeddings 帶 since/until 只補時間窗內的候選', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'too-old', content: 'x', created_at: 100 });
insertEntry(db, { id: 'in-window-1', content: 'y', created_at: 500 });
insertEntry(db, { id: 'in-window-2', content: 'z', created_at: 800 });
insertEntry(db, { id: 'too-new', content: 'w', created_at: 1500 });
const env = makeEnv(db, { dailyLimit: '100' });
const r = await backfillEmbeddings(env, { limit: 100, since: 400, until: 1000 });
expect(r.processed).toBe(2);
expect((await listEmbeddedIds(db)).sort()).toEqual(['in-window-1', 'in-window-2']);
});
it('backfillEmbeddings 帶 library 只補該庫的候選(未標記歸 general)', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'finance-1', content: 'x', created_at: 1, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
insertEntry(db, { id: 'hr-1', content: 'y', created_at: 2, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
insertEntry(db, { id: 'untagged', content: 'z', created_at: 3 }); // 無 library → general
const env = makeEnv(db, { dailyLimit: '100' });
const r = await backfillEmbeddings(env, { limit: 100, library: 'finance' });
expect(r.processed).toBe(1);
expect(await listEmbeddedIds(db)).toEqual(['finance-1']);
const r2 = await backfillEmbeddings(env, { limit: 100, library: 'general' });
expect(r2.processed).toBe(1);
expect((await listEmbeddedIds(db)).sort()).toEqual(['finance-1', 'untagged']);
});
it('reconcile 帶 since/until/library 同樣受篩選(同一套 SelectionCriteria,非獨立實作)', async () => {
const db = makeSqliteD1();
insertEntry(db, { id: 'old', content: 'x', created_at: 1, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
insertEntry(db, { id: 'new', content: 'y', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
insertEntry(db, { id: 'other-lib', content: 'z', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
const env = makeEnv(db);
seedVectorized(env, ['old', 'new', 'other-lib']);
const r = await reconcileEmbedGeneration(env, { limit: 100, library: 'finance', since: 50 });
expect(r.checked).toBe(1);
const row = await getRow(db, 'new');
expect(row!.content_hash).toBe(CURRENT_MODEL);
const oldRow = await getRow(db, 'old');
expect(oldRow!.content_hash).toBe(null); // 在時間窗外,沒被動到
});
});