uncle6me-web
ceb7638d74
規格：system-dev/docs/3-specs/pending-changes.md「record 要有身分」v7 定稿（leo 2026-08-15 confirm）。 模型一句話（leo）：「真身在 pool 的 entry 裡，所有的虛擬表虛擬欄位都是指向這個 entry 的指標。」 - 0007 migration：池上型別化指標欄（src/rel/dst）＋一對方向 partial index＋啟動常數 （sys_root/sys_belongs/sys_field_of）＋templates 鏡射成 sheet/field entry＋ 每筆 record 一顆身分 entry（id＝原 record_id，引用不失效）＋每格一條關係列 （id 由舊儲存格列 id 衍生 ⇒ INSERT OR IGNORE 天然冪等）＋拆 entry_values （0006 墊表→搬→拆手法）。純 INSERT、value entries 一列不動（向量索引不失效）。 - record-crud 整份改寫到關係列（#128 指標語意/共用保護/N+1 批次/租戶過濾全數保留， 驗收測試 232→236 綠）；library-map 四段縱轉橫 SQL、records triplet-stats 改查關係列。 - entry-crud：機制列隔離（未指定 entry_type 的列表/搜尋不回機制節點）；deleteEntry 接手舊 entry_values FK 的不變量（dst 被指著→拒刪）。 - 孤兒偵測重設計（v7 §5 點名）：新模型孤兒＝指標指向不存在 id 的關係列， LEFT JOIN 斷鏈掃描（承接 2026-06-24 清理事故的 FK 形狀）， GET /maintenance/relation-orphans 唯讀巡檢。 - cli deploy.ts：0007 逐句套用＋容錯 duplicate column（SQLite 無欄位級 IF NOT EXISTS， 整檔送 /query 會在重跑時假紅）。 - 測試：tree-record-migration.test.ts 驗資料零漏／雙跑冪等／孤兒掃描； 釘死三表的斷言依 confirm 後規格改口（execution-log/credential-legacy 兩處）。 遷移期雙軌（第二刀收）：templates 表仍是欄位定義真相源；六種 metadata_json 打包型 與 §7 減法封鎖（拿掉 entry_type/metadata_json 欄）留待第二刀。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
470 lines
25 KiB
TypeScript
470 lines
25 KiB
TypeScript
// embed backfill — D68（2026-08-11 leo 拍板：補算向量照時間新到舊、且每天有額度上限）測試。
|
||
//
|
||
// 測試策略比照 execution-log.test.ts／library-map.test.ts：真 SQLite（node:sqlite）套
|
||
// migrations/0001_base.sql 原檔，比手刻假 DB 更硬——驗的是真實 SQL 語意（ORDER BY／WHERE／
|
||
// JSON 函式），不是「以為 SQL 長這樣」。AI／VECTORIZE 仍是輕量假物件（Cloudflare binding，
|
||
// 不是 SQL，沒有真 runtime 可套）。
|
||
//
|
||
// 覆蓋 D68 三條 + is_embedded 世代旗標坑，四項都要有實測輸出：
|
||
// 1. 由新到舊：造 created_at 跨時間的候選，證明先被處理的是最新那幾筆
|
||
// 2. 每日額度上限真的擋：cap 設小，跑到撞上限，證明它停手不再打 AI（不是繼續打）
|
||
// 3. 帶著舊世代旗標（is_embedded=1 但對應已退役索引）的列補得回來
|
||
// 4. 現有 idempotent／batching／reindex 行為不因本次改動而壞掉
|
||
//
|
||
// 本檔在 kbdb/tests/（牆外，非 kbdb/src|migrations），依 D38 kbdb-api-wall-guard 規則，
|
||
// 所有直接對 SQLite 治具下 SQL 的行都集中在下面幾個 helper（每行標 kbdb-sql-ok 留痕）——
|
||
// 這是**測試治具本身**（node:sqlite→D1 shim，模擬 D1 binding），不是牆外業務邏輯繞過 API。
|
||
import { describe, it, expect } from 'vitest';
|
||
import { DatabaseSync } from 'node:sqlite';
|
||
import { readFileSync } from 'node:fs';
|
||
import {
|
||
backfillEmbeddings,
|
||
backfillStatus,
|
||
embedEnabled,
|
||
reconcileEmbedGeneration,
|
||
} from '../src/embed';
|
||
import type { Bindings, Entry, EntryType } from '../src/types';
|
||
|
||
const CURRENT_MODEL = '@cf/baai/bge-m3'; // embed.ts DEFAULT_EMBED_MODEL（未 export，測試按文件字面核對）
|
||
|
||
// ── node:sqlite → D1 介面最小 adapter（同 execution-log.test.ts／library-map.test.ts 手法）──
|
||
function makeSqliteD1(): D1Database {
|
||
const raw = new DatabaseSync(':memory:');
|
||
raw.exec(readFileSync(new URL('../migrations/0001_base.sql', import.meta.url), 'utf8')); // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）套 migration 原檔
|
||
raw.exec(readFileSync(new URL('../migrations/0007_tree_record_model.sql', import.meta.url), 'utf8')); // kbdb-sql-ok：測試治具套 0007（樹狀 record 模型，v7 定稿）——真 schema 就是遷移後的 schema
|
||
function stmt(sql: string, params: unknown[]) {
|
||
const s = {
|
||
bind(...args: unknown[]) { return stmt(sql, args); },
|
||
async all<T>() { return { results: raw.prepare(sql).all(...params) as T[] }; }, // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）
|
||
async first<T>() { return (raw.prepare(sql).get(...params) ?? null) as T | null; }, // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）
|
||
async run() {
|
||
const r = raw.prepare(sql).run(...params); // kbdb-sql-ok：測試治具（node:sqlite→D1 shim），非牆外業務邏輯繞過 API
|
||
return { success: true, meta: { changes: r.changes } };
|
||
},
|
||
};
|
||
return s;
|
||
}
|
||
return { prepare: (sql: string) => stmt(sql, []) } as unknown as D1Database;
|
||
}
|
||
|
||
// ── 測試專用資料存取 helper：把所有直接下 SQL 的呼叫收斂到這裡（每行標記留痕）──────────
|
||
function insertEntry(db: D1Database, e: Partial<Entry> & { id: string; created_at: number }): void {
|
||
const sql = `INSERT INTO entries (id, content, entry_type, owner_id, content_hash, is_embedded, metadata_json, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
||
db.prepare(sql).bind( // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）灌測試資料
|
||
e.id,
|
||
e.content === undefined ? 'x' : e.content, // 區分「沒提供」(undefined→預設'x') 與「顯式 null」(保留 null)
|
||
|
||
(e.entry_type ?? 'workflow') as EntryType,
|
||
e.owner_id ?? 'leo',
|
||
e.content_hash ?? null,
|
||
e.is_embedded ?? 0,
|
||
e.metadata_json ?? JSON.stringify({ embed: true }),
|
||
e.created_at,
|
||
e.created_at,
|
||
).run();
|
||
}
|
||
|
||
async function getRow(db: D1Database, id: string): Promise<{ id: string; is_embedded: number; content_hash: string | null } | null> {
|
||
return db.prepare('SELECT id, is_embedded, content_hash FROM entries WHERE id = ?').bind(id).first(); // kbdb-sql-ok：測試治具讀回斷言用
|
||
}
|
||
|
||
async function listAllRows(db: D1Database): Promise<{ id: string; is_embedded: number; content_hash: string | null }[]> {
|
||
const res = await db.prepare('SELECT id, is_embedded, content_hash FROM entries').all<{ id: string; is_embedded: number; content_hash: string | null }>(); // kbdb-sql-ok：測試治具讀回斷言用
|
||
return res.results;
|
||
}
|
||
|
||
async function listEmbeddedIds(db: D1Database): Promise<string[]> {
|
||
const res = await db.prepare("SELECT id FROM entries WHERE is_embedded = 1").all<{ id: string }>(); // kbdb-sql-ok：測試治具讀回斷言用
|
||
return res.results.map((r) => r.id);
|
||
}
|
||
|
||
async function countUsageRows(db: D1Database): Promise<{ id: string; entry_type: string }[]> {
|
||
const res = await db.prepare("SELECT id, entry_type FROM entries WHERE entry_type = 'embed_backfill_usage'").all<{ id: string; entry_type: string }>(); // kbdb-sql-ok：測試治具驗證「不新增表、單列 upsert」
|
||
return res.results;
|
||
}
|
||
|
||
function makeEnv(db: D1Database, opts: { withBindings?: boolean; dailyLimit?: string; maintenanceLimit?: string } = {}): Bindings {
|
||
const withBindings = opts.withBindings ?? true;
|
||
const upserts: { id: string }[] = [];
|
||
const aiCalls: string[][] = [];
|
||
const getByIdsCalls: string[][] = [];
|
||
const vectorizeStore = new Set<string>(); // ids "present" in the current (fake) Vectorize index
|
||
const env = {
|
||
DB: db,
|
||
ENVIRONMENT: 'test',
|
||
EMBED_BACKFILL_DAILY_LIMIT: opts.dailyLimit,
|
||
KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: opts.maintenanceLimit,
|
||
...(withBindings
|
||
? {
|
||
AI: {
|
||
async run(_m: string, i: { text: string[] }) {
|
||
aiCalls.push(i.text);
|
||
return { data: i.text.map(() => [0.1, 0.2, 0.3]) };
|
||
},
|
||
},
|
||
VECTORIZE: {
|
||
async upsert(v: { id: string }[]) {
|
||
upserts.push(...v);
|
||
for (const x of v) vectorizeStore.add(x.id);
|
||
return { count: v.length };
|
||
},
|
||
async getByIds(ids: string[]) {
|
||
getByIdsCalls.push(ids);
|
||
return ids.filter((id) => vectorizeStore.has(id)).map((id) => ({ id, values: [0.1] }));
|
||
},
|
||
},
|
||
}
|
||
: {}),
|
||
} as unknown as Bindings;
|
||
const bag = env as unknown as {
|
||
__upserts: unknown[]; __ai: unknown[]; __getByIds: unknown[];
|
||
__seedVectorized: (ids: string[]) => void;
|
||
};
|
||
bag.__upserts = upserts;
|
||
bag.__ai = aiCalls;
|
||
bag.__getByIds = getByIdsCalls;
|
||
bag.__seedVectorized = (ids: string[]) => { for (const id of ids) vectorizeStore.add(id); };
|
||
return env;
|
||
}
|
||
|
||
function aiCallsOf(env: Bindings): string[][] {
|
||
return (env as unknown as { __ai: string[][] }).__ai;
|
||
}
|
||
function upsertsOf(env: Bindings): { id: string }[] {
|
||
return (env as unknown as { __upserts: { id: string }[] }).__upserts;
|
||
}
|
||
function seedVectorized(env: Bindings, ids: string[]): void {
|
||
(env as unknown as { __seedVectorized: (ids: string[]) => void }).__seedVectorized(ids);
|
||
}
|
||
|
||
describe('backfillEmbeddings — 模組未開', () => {
|
||
it('誠實不假綠：no-op，含新增的 quota 欄位', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e1', created_at: 1 });
|
||
const env = makeEnv(db, { withBindings: false });
|
||
expect(embedEnabled(env)).toBe(false);
|
||
const r = await backfillEmbeddings(env);
|
||
expect(r).toEqual({
|
||
enabled: false, processed: 0, skipped: 0, remaining: 0, scanned: 0,
|
||
quota_limit: 0, quota_used_today: 0, quota_exceeded: false,
|
||
});
|
||
});
|
||
});
|
||
|
||
describe('backfillEmbeddings — 基本行為（沿用既有覆蓋，改動後仍要綠）', () => {
|
||
it('embeds embeddable+is_embedded=0 entries, marks is_embedded=1 + content_hash，批次 AI+upsert', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e1', content: 'doorbell workflow', created_at: 1 });
|
||
insertEntry(db, { id: 'e2', content: 'notify workflow', created_at: 2 });
|
||
insertEntry(db, { id: 'e3', content: 'not tagged', created_at: 3, metadata_json: JSON.stringify({ embed: false }) });
|
||
insertEntry(db, { id: 'e4', content: 'already done', created_at: 4, is_embedded: 1 });
|
||
insertEntry(db, { id: 'e5', content: null, created_at: 5 }); // NULL content → 排除，非本次改動範圍的既有行為
|
||
const env = makeEnv(db);
|
||
const r = await backfillEmbeddings(env, { limit: 100 });
|
||
expect(r.enabled).toBe(true);
|
||
expect(r.processed).toBe(2); // only e1, e2
|
||
expect(r.remaining).toBe(0);
|
||
const rows = await listAllRows(db);
|
||
const byId = Object.fromEntries(rows.map((x) => [x.id, x]));
|
||
expect(byId.e1.is_embedded).toBe(1);
|
||
expect(byId.e1.content_hash).toBe(CURRENT_MODEL); // 世代戳記有寫
|
||
expect(byId.e2.is_embedded).toBe(1);
|
||
expect(byId.e3.is_embedded).toBe(0);
|
||
expect(upsertsOf(env).map((u) => u.id).sort()).toEqual(['e1', 'e2']);
|
||
expect(aiCallsOf(env).length).toBe(1);
|
||
expect(aiCallsOf(env)[0].length).toBe(2);
|
||
});
|
||
|
||
it('idempotent：全部嵌完後重跑不再處理', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e1', content: 'x', created_at: 1 });
|
||
const env = makeEnv(db, { dailyLimit: '100' });
|
||
await backfillEmbeddings(env);
|
||
const r2 = await backfillEmbeddings(env);
|
||
expect(r2.processed).toBe(0);
|
||
expect(r2.remaining).toBe(0);
|
||
});
|
||
|
||
it('batches via limit → remaining 讓 caller 可重複呼叫到 0', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', content: 'x', created_at: 1 });
|
||
insertEntry(db, { id: 'b', content: 'y', created_at: 2 });
|
||
insertEntry(db, { id: 'c', content: 'z', created_at: 3 });
|
||
const env = makeEnv(db, { dailyLimit: '100' });
|
||
const r1 = await backfillEmbeddings(env, { limit: 2 });
|
||
expect(r1.processed).toBe(2);
|
||
expect(r1.remaining).toBe(1);
|
||
const r2 = await backfillEmbeddings(env, { limit: 2 });
|
||
expect(r2.processed).toBe(1);
|
||
expect(r2.remaining).toBe(0);
|
||
});
|
||
|
||
it('reindex：重推所有 embeddable（含 is_embedded=1），offset 分頁到 remaining=0（Arcrun#11）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', content: 'x', created_at: 1, is_embedded: 1 });
|
||
insertEntry(db, { id: 'b', content: 'y', created_at: 2, is_embedded: 1 });
|
||
insertEntry(db, { id: 'c', content: 'z', created_at: 3, is_embedded: 1 });
|
||
const env = makeEnv(db, { dailyLimit: '100' });
|
||
const normal = await backfillEmbeddings(env, { limit: 100 });
|
||
expect(normal.processed).toBe(0); // 沒有 is_embedded=0 → 什麼都不做
|
||
const r1 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 0 });
|
||
expect(r1.processed).toBe(2);
|
||
expect(r1.remaining).toBe(1);
|
||
const r2 = await backfillEmbeddings(env, { reindex: true, limit: 2, offset: 2 });
|
||
expect(r2.processed).toBe(1);
|
||
expect(r2.remaining).toBe(0);
|
||
expect(upsertsOf(env).map((u) => u.id).sort()).toEqual(['a', 'b', 'c']);
|
||
});
|
||
|
||
it('status 回報 pending/embedded 計數', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e1', content: 'x', created_at: 1 });
|
||
insertEntry(db, { id: 'e2', content: 'y', created_at: 2, is_embedded: 1 });
|
||
const env = makeEnv(db);
|
||
const s = await backfillStatus(env);
|
||
expect(s.enabled).toBe(true);
|
||
expect(typeof s.pending).toBe('number');
|
||
});
|
||
});
|
||
|
||
describe('D68①：由新到舊排序（實測，不是推論）', () => {
|
||
it('候選跨時間分佈時，先被嵌入的是 created_at 最新的那幾筆', async () => {
|
||
const db = makeSqliteD1();
|
||
// 刻意亂序插入，證明排序看的是 created_at 不是插入順序 / id 字母序
|
||
insertEntry(db, { id: 'old-2024', content: 'half year ago', created_at: 1_000 });
|
||
insertEntry(db, { id: 'today', content: 'written today', created_at: 100_000 });
|
||
insertEntry(db, { id: 'mid-2025', content: 'a few months ago', created_at: 50_000 });
|
||
const env = makeEnv(db, { dailyLimit: '100' });
|
||
// limit=1：一次只能處理一筆，若排序正確，該筆必須是 'today'（created_at 最大）
|
||
const r = await backfillEmbeddings(env, { limit: 1 });
|
||
expect(r.processed).toBe(1);
|
||
const embedded = await listEmbeddedIds(db);
|
||
expect(embedded).toEqual(['today']);
|
||
expect(aiCallsOf(env)[0]).toEqual(['written today']);
|
||
});
|
||
});
|
||
|
||
describe('D68②：每日額度上限真的擋（把上限設小，跑到撞上限）', () => {
|
||
it('額度耗盡後停手，不再繼續打 AI；未耗盡的仍優先保留最新的（額度截斷 + 排序疊加）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e-oldest', content: 'c1', created_at: 1 });
|
||
insertEntry(db, { id: 'e-old', content: 'c2', created_at: 2 });
|
||
insertEntry(db, { id: 'e-new', content: 'c3', created_at: 3 });
|
||
insertEntry(db, { id: 'e-newest', content: 'c4', created_at: 4 });
|
||
const env = makeEnv(db, { dailyLimit: '2' }); // 上限設得比候選數（4）小
|
||
const r = await backfillEmbeddings(env, { limit: 100 });
|
||
|
||
// 停手，不是繼續打：AI 只被叫過一次，且只帶 2 筆文字（不是全部 4 筆）
|
||
expect(aiCallsOf(env).length).toBe(1);
|
||
expect(aiCallsOf(env)[0].length).toBe(2);
|
||
expect(r.processed).toBe(2);
|
||
expect(r.quota_limit).toBe(2);
|
||
expect(r.quota_used_today).toBe(2);
|
||
expect(r.quota_exceeded).toBe(true); // 還有候選但今天不再打 AI
|
||
|
||
// 被留下處理的兩筆是最新的（e-newest, e-new），不是隨機或最舊的
|
||
const embedded = await listEmbeddedIds(db);
|
||
expect(embedded.sort()).toEqual(['e-new', 'e-newest']);
|
||
|
||
// 再跑一次（同一天）：額度已用完，processed=0，AI 呼叫次數仍是 1（沒有再打）
|
||
const r2 = await backfillEmbeddings(env, { limit: 100 });
|
||
expect(r2.processed).toBe(0);
|
||
expect(r2.quota_exceeded).toBe(true);
|
||
expect(aiCallsOf(env).length).toBe(1); // 沒有新增呼叫
|
||
});
|
||
|
||
it('額度上限被拿掉時本測試會變紅（反向驗證：測試真的在測東西，不是恆真）', async () => {
|
||
const db = makeSqliteD1();
|
||
for (let i = 1; i <= 5; i++) insertEntry(db, { id: `e${i}`, content: `c${i}`, created_at: i });
|
||
// 不設 dailyLimit（用預設 1800，遠大於 5）→ 全部應被處理，模擬「上限被拿掉」的行為
|
||
const env = makeEnv(db);
|
||
const r = await backfillEmbeddings(env, { limit: 100 });
|
||
expect(r.processed).toBe(5);
|
||
expect(r.quota_exceeded).toBe(false);
|
||
// 對照組：把上限設到比候選數小，行為必須不同（證明上一組「額度=2」的測試不是巧合）
|
||
const db2 = makeSqliteD1();
|
||
for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `e${i}`, content: `c${i}`, created_at: i });
|
||
const env2 = makeEnv(db2, { dailyLimit: '2' });
|
||
const r2 = await backfillEmbeddings(env2, { limit: 100 });
|
||
expect(r2.processed).toBe(2);
|
||
expect(r2.processed).not.toBe(r.processed); // 有 cap vs 沒 cap 必須不同，否則 cap 沒在起作用
|
||
});
|
||
|
||
it('額度計數跨呼叫累加，換日字串變動即歸零（不新增表，entries 單列 upsert）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e1', content: 'c1', created_at: 1 });
|
||
insertEntry(db, { id: 'e2', content: 'c2', created_at: 2 });
|
||
const env = makeEnv(db, { dailyLimit: '10' });
|
||
const r1 = await backfillEmbeddings(env, { limit: 1 });
|
||
expect(r1.quota_used_today).toBe(1);
|
||
const r2 = await backfillEmbeddings(env, { limit: 1 });
|
||
expect(r2.quota_used_today).toBe(2); // 累加，不是每次重算成當批數
|
||
// 驗證只有一列計數器，且落在既有三表（entries），沒有新表
|
||
const usageRows = await countUsageRows(db);
|
||
expect(usageRows.length).toBe(1);
|
||
expect(usageRows[0].id).toMatch(/^embed-backfill-usage:\d{4}-\d{2}-\d{2}$/);
|
||
});
|
||
});
|
||
|
||
describe('D68③：leo21c 資料還原情境——is_embedded=1 但對應已退役索引的列補得回來', () => {
|
||
it('reconcile：確認在現行 index 的只補 content_hash，不打 AI', async () => {
|
||
const db = makeSqliteD1();
|
||
// 模擬「這次修復之前」就已經正確嵌入現行 index 的資料：is_embedded=1、content_hash 從未寫過（NULL）
|
||
insertEntry(db, { id: 'ok-legacy', content: 'x', created_at: 1, is_embedded: 1, content_hash: null });
|
||
const env = makeEnv(db);
|
||
seedVectorized(env, ['ok-legacy']); // 現行 index 真的有它
|
||
|
||
const r = await reconcileEmbedGeneration(env, { limit: 100 });
|
||
expect(r.checked).toBe(1);
|
||
expect(r.confirmed_current).toBe(1);
|
||
expect(r.reset_to_pending).toBe(0);
|
||
expect(aiCallsOf(env).length).toBe(0); // 沒有打 AI
|
||
|
||
const row = await getRow(db, 'ok-legacy');
|
||
expect(row!.is_embedded).toBe(1); // 沒被誤重置
|
||
expect(row!.content_hash).toBe(CURRENT_MODEL); // 補標記
|
||
});
|
||
|
||
it('reconcile 揪出真正對舊索引的殘留 → 重置回 pending → 正常 backfill 真的把它補回來（端到端）', async () => {
|
||
const db = makeSqliteD1();
|
||
// leo21c 情境：從備份整批灌回，is_embedded=1 但這是對已退役 768 維索引說的；
|
||
// 現行（1024 維）Vectorize index 裡沒有這個向量（不呼叫 seedVectorized）。
|
||
insertEntry(db, { id: 'restored-stale', content: '從備份還原的舊卡片', created_at: 999, is_embedded: 1, content_hash: null });
|
||
const env = makeEnv(db);
|
||
|
||
// step 1：reconcile 應該發現它不在現行 index，重置成 pending
|
||
const r1 = await reconcileEmbedGeneration(env, { limit: 100 });
|
||
expect(r1.checked).toBe(1);
|
||
expect(r1.confirmed_current).toBe(0);
|
||
expect(r1.reset_to_pending).toBe(1);
|
||
const midRow = await getRow(db, 'restored-stale');
|
||
expect(midRow!.is_embedded).toBe(0);
|
||
expect(midRow!.content_hash).toBe(null);
|
||
expect(r1.remaining).toBe(0); // 處理完，沒有更多待核對的了
|
||
|
||
// step 2：正常 backfill 現在會撿到它（因為 is_embedded=0 了），真的打 AI 補回來
|
||
const r2 = await backfillEmbeddings(env, { limit: 100 });
|
||
expect(r2.processed).toBe(1);
|
||
expect(aiCallsOf(env).length).toBe(1);
|
||
expect(aiCallsOf(env)[0]).toEqual(['從備份還原的舊卡片']);
|
||
|
||
const finalRow = await getRow(db, 'restored-stale');
|
||
expect(finalRow!.is_embedded).toBe(1); // 補回來了
|
||
expect(finalRow!.content_hash).toBe(CURRENT_MODEL); // 蓋上現行世代戳記，下次 reconcile 不會再選到它
|
||
});
|
||
|
||
it('已經是現行世代（content_hash 等於現行模型）的列不會被 reconcile 重複選中', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'fresh', content: 'x', created_at: 1, is_embedded: 1, content_hash: CURRENT_MODEL });
|
||
const env = makeEnv(db);
|
||
const r = await reconcileEmbedGeneration(env, { limit: 100 });
|
||
expect(r.checked).toBe(0);
|
||
expect(r.remaining).toBe(0);
|
||
});
|
||
|
||
it('模組未開 → 誠實回 enabled:false，不假裝', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'e1', content: 'x', created_at: 1, is_embedded: 1 });
|
||
const env = makeEnv(db, { withBindings: false });
|
||
const r = await reconcileEmbedGeneration(env);
|
||
expect(r).toEqual({
|
||
enabled: false, checked: 0, confirmed_current: 0, reset_to_pending: 0, remaining: 0,
|
||
scanned: 0, quota_limit: 0, quota_used_today: 0, quota_exceeded: false,
|
||
});
|
||
});
|
||
});
|
||
|
||
describe('Arcrun#85 D69：reconcile 的 D1 寫入額度（與標庫 backfill 共用的計數器）', () => {
|
||
it('額度耗盡後 reconcile 停手：不再寫 D1，quota_exceeded=true', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'r1', content: 'c1', created_at: 1, is_embedded: 1, content_hash: null });
|
||
insertEntry(db, { id: 'r2', content: 'c2', created_at: 2, is_embedded: 1, content_hash: null });
|
||
insertEntry(db, { id: 'r3', content: 'c3', created_at: 3, is_embedded: 1, content_hash: null });
|
||
const env = makeEnv(db, { maintenanceLimit: '2' }); // 上限比候選數（3）小
|
||
seedVectorized(env, ['r1', 'r2', 'r3']); // 全在現行 index（confirmed_current 路徑，仍是 D1 write）
|
||
|
||
const r = await reconcileEmbedGeneration(env, { limit: 100 });
|
||
expect(r.scanned).toBe(3); // 掃到 3 筆候選
|
||
expect(r.checked).toBe(2); // 但只處理了額度允許的 2 筆
|
||
expect(r.confirmed_current).toBe(2);
|
||
expect(r.quota_limit).toBe(2);
|
||
expect(r.quota_used_today).toBe(2);
|
||
expect(r.quota_exceeded).toBe(true);
|
||
|
||
// 只有 2 筆真的被寫回 content_hash（最新的兩筆，ORDER BY created_at DESC）
|
||
const rows = await listAllRows(db);
|
||
const byId = Object.fromEntries(rows.map((x) => [x.id, x]));
|
||
expect(byId.r3.content_hash).toBe(CURRENT_MODEL);
|
||
expect(byId.r2.content_hash).toBe(CURRENT_MODEL);
|
||
expect(byId.r1.content_hash).toBe(null); // 額度用完，沒輪到它
|
||
|
||
// 再跑一次（同一天）：額度已用完，checked=0
|
||
const r2 = await reconcileEmbedGeneration(env, { limit: 100 });
|
||
expect(r2.checked).toBe(0);
|
||
expect(r2.quota_exceeded).toBe(true);
|
||
});
|
||
|
||
it('額度上限被拿掉時本測試會變紅（反向驗證，同 D68② 手法）', async () => {
|
||
const db = makeSqliteD1();
|
||
for (let i = 1; i <= 5; i++) insertEntry(db, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null });
|
||
const env = makeEnv(db); // 不設 maintenanceLimit → 用預設 20000，遠大於 5，全部應被處理
|
||
seedVectorized(env, ['r1', 'r2', 'r3', 'r4', 'r5']);
|
||
const r = await reconcileEmbedGeneration(env, { limit: 100 });
|
||
expect(r.checked).toBe(5);
|
||
expect(r.quota_exceeded).toBe(false);
|
||
|
||
const db2 = makeSqliteD1();
|
||
for (let i = 1; i <= 5; i++) insertEntry(db2, { id: `r${i}`, content: `c${i}`, created_at: i, is_embedded: 1, content_hash: null });
|
||
const env2 = makeEnv(db2, { maintenanceLimit: '2' });
|
||
seedVectorized(env2, ['r1', 'r2', 'r3', 'r4', 'r5']);
|
||
const r2 = await reconcileEmbedGeneration(env2, { limit: 100 });
|
||
expect(r2.checked).toBe(2);
|
||
expect(r2.checked).not.toBe(r.checked); // 有 cap vs 沒 cap 必須不同，否則 cap 沒在作用
|
||
});
|
||
});
|
||
|
||
describe('Arcrun#85：「挑哪一批」可以從外面指定（SelectionCriteria：since/until/library）', () => {
|
||
it('backfillEmbeddings 帶 since/until 只補時間窗內的候選', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'too-old', content: 'x', created_at: 100 });
|
||
insertEntry(db, { id: 'in-window-1', content: 'y', created_at: 500 });
|
||
insertEntry(db, { id: 'in-window-2', content: 'z', created_at: 800 });
|
||
insertEntry(db, { id: 'too-new', content: 'w', created_at: 1500 });
|
||
const env = makeEnv(db, { dailyLimit: '100' });
|
||
const r = await backfillEmbeddings(env, { limit: 100, since: 400, until: 1000 });
|
||
expect(r.processed).toBe(2);
|
||
expect((await listEmbeddedIds(db)).sort()).toEqual(['in-window-1', 'in-window-2']);
|
||
});
|
||
|
||
it('backfillEmbeddings 帶 library 只補該庫的候選（未標記歸 general）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'finance-1', content: 'x', created_at: 1, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
|
||
insertEntry(db, { id: 'hr-1', content: 'y', created_at: 2, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
|
||
insertEntry(db, { id: 'untagged', content: 'z', created_at: 3 }); // 無 library → general
|
||
const env = makeEnv(db, { dailyLimit: '100' });
|
||
const r = await backfillEmbeddings(env, { limit: 100, library: 'finance' });
|
||
expect(r.processed).toBe(1);
|
||
expect(await listEmbeddedIds(db)).toEqual(['finance-1']);
|
||
|
||
const r2 = await backfillEmbeddings(env, { limit: 100, library: 'general' });
|
||
expect(r2.processed).toBe(1);
|
||
expect((await listEmbeddedIds(db)).sort()).toEqual(['finance-1', 'untagged']);
|
||
});
|
||
|
||
it('reconcile 帶 since/until/library 同樣受篩選（同一套 SelectionCriteria，非獨立實作）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'old', content: 'x', created_at: 1, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
|
||
insertEntry(db, { id: 'new', content: 'y', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'finance' }) });
|
||
insertEntry(db, { id: 'other-lib', content: 'z', created_at: 100, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
|
||
const env = makeEnv(db);
|
||
seedVectorized(env, ['old', 'new', 'other-lib']);
|
||
const r = await reconcileEmbedGeneration(env, { limit: 100, library: 'finance', since: 50 });
|
||
expect(r.checked).toBe(1);
|
||
const row = await getRow(db, 'new');
|
||
expect(row!.content_hash).toBe(CURRENT_MODEL);
|
||
const oldRow = await getRow(db, 'old');
|
||
expect(oldRow!.content_hash).toBe(null); // 在時間窗外，沒被動到
|
||
});
|
||
});
|