uncle6me-web
ceb7638d74
規格：system-dev/docs/3-specs/pending-changes.md「record 要有身分」v7 定稿（leo 2026-08-15 confirm）。 模型一句話（leo）：「真身在 pool 的 entry 裡，所有的虛擬表虛擬欄位都是指向這個 entry 的指標。」 - 0007 migration：池上型別化指標欄（src/rel/dst）＋一對方向 partial index＋啟動常數 （sys_root/sys_belongs/sys_field_of）＋templates 鏡射成 sheet/field entry＋ 每筆 record 一顆身分 entry（id＝原 record_id，引用不失效）＋每格一條關係列 （id 由舊儲存格列 id 衍生 ⇒ INSERT OR IGNORE 天然冪等）＋拆 entry_values （0006 墊表→搬→拆手法）。純 INSERT、value entries 一列不動（向量索引不失效）。 - record-crud 整份改寫到關係列（#128 指標語意/共用保護/N+1 批次/租戶過濾全數保留， 驗收測試 232→236 綠）；library-map 四段縱轉橫 SQL、records triplet-stats 改查關係列。 - entry-crud：機制列隔離（未指定 entry_type 的列表/搜尋不回機制節點）；deleteEntry 接手舊 entry_values FK 的不變量（dst 被指著→拒刪）。 - 孤兒偵測重設計（v7 §5 點名）：新模型孤兒＝指標指向不存在 id 的關係列， LEFT JOIN 斷鏈掃描（承接 2026-06-24 清理事故的 FK 形狀）， GET /maintenance/relation-orphans 唯讀巡檢。 - cli deploy.ts：0007 逐句套用＋容錯 duplicate column（SQLite 無欄位級 IF NOT EXISTS， 整檔送 /query 會在重跑時假紅）。 - 測試：tree-record-migration.test.ts 驗資料零漏／雙跑冪等／孤兒掃描； 釘死三表的斷言依 confirm 後規格改口（execution-log/credential-legacy 兩處）。 遷移期雙軌（第二刀收）：templates 表仍是欄位定義真相源；六種 metadata_json 打包型 與 §7 減法封鎖（拿掉 entry_type/metadata_json 欄）留待第二刀。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
232 lines
12 KiB
TypeScript
232 lines
12 KiB
TypeScript
// 標庫 backfill（Arcrun#85 二次裁決，2026-08-11）測試。
|
||
//
|
||
// 測試策略比照 embed-backfill.test.ts：真 SQLite（node:sqlite）套 migrations/0001_base.sql
|
||
// 原檔，驗真實 SQL 語意（json_set／WHERE／LIMIT），不是「以為 SQL 長這樣」。
|
||
//
|
||
// 覆蓋：
|
||
// 1. 只補「符合條件、目前未標記 library」的候選；已標記的不動（冪等）
|
||
// 2. owner_id 必填（缺了要拋錯，防「補錯 owner 等於白做」——2026-08-11 leo 直令）
|
||
// 3. source_prefix／page_name_prefix／since／until 篩選條件真的在篩
|
||
// 4. 與 reconcileEmbedGeneration 共用同一顆每日 D1 寫入額度（D69 的核心訴求：
|
||
// 補標不能把世代核對的閘繞過去，反之亦然）
|
||
//
|
||
// 本檔在 kbdb/tests/（牆外），依 D38 kbdb-api-wall-guard 規則，直接對 SQLite 治具下 SQL
|
||
// 的行集中在 helper（測試治具本身，非牆外業務邏輯繞過 API，每行標 kbdb-sql-ok 留痕）。
|
||
import { describe, it, expect } from 'vitest';
|
||
import { DatabaseSync } from 'node:sqlite';
|
||
import { readFileSync } from 'node:fs';
|
||
import { backfillEntryLibraryTags, libraryBackfillStatus } from '../src/actions/library-backfill';
|
||
import { reconcileEmbedGeneration } from '../src/embed';
|
||
import type { Bindings, Entry, EntryType } from '../src/types';
|
||
|
||
// ── node:sqlite → D1 介面最小 adapter（同 embed-backfill.test.ts 手法）──────────────
|
||
function makeSqliteD1(): D1Database {
|
||
const raw = new DatabaseSync(':memory:');
|
||
raw.exec(readFileSync(new URL('../migrations/0001_base.sql', import.meta.url), 'utf8')); // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）套 migration 原檔
|
||
raw.exec(readFileSync(new URL('../migrations/0007_tree_record_model.sql', import.meta.url), 'utf8')); // kbdb-sql-ok：測試治具套 0007（樹狀 record 模型，v7 定稿）——真 schema 就是遷移後的 schema
|
||
function stmt(sql: string, params: unknown[]) {
|
||
const s = {
|
||
bind(...args: unknown[]) { return stmt(sql, args); },
|
||
async all<T>() { return { results: raw.prepare(sql).all(...params) as T[] }; }, // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）
|
||
async first<T>() { return (raw.prepare(sql).get(...params) ?? null) as T | null; }, // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）
|
||
async run() {
|
||
const r = raw.prepare(sql).run(...params); // kbdb-sql-ok：測試治具（node:sqlite→D1 shim），非牆外業務邏輯繞過 API
|
||
return { success: true, meta: { changes: r.changes } };
|
||
},
|
||
};
|
||
return s;
|
||
}
|
||
return { prepare: (sql: string) => stmt(sql, []) } as unknown as D1Database;
|
||
}
|
||
|
||
function insertEntry(db: D1Database, e: Partial<Entry> & { id: string; created_at: number }): void {
|
||
const sql = `INSERT INTO entries (id, content, entry_type, owner_id, content_hash, is_embedded, metadata_json, page_name, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
||
db.prepare(sql).bind( // kbdb-sql-ok：測試治具（node:sqlite→D1 shim）灌測試資料
|
||
e.id,
|
||
e.content === undefined ? 'x' : e.content,
|
||
(e.entry_type ?? 'block') as EntryType,
|
||
e.owner_id ?? 'bfezv28v',
|
||
e.content_hash ?? null,
|
||
e.is_embedded ?? 0,
|
||
e.metadata_json === undefined ? null : e.metadata_json,
|
||
e.page_name ?? null,
|
||
e.created_at,
|
||
e.created_at,
|
||
).run();
|
||
}
|
||
|
||
async function getLibrary(db: D1Database, id: string): Promise<string | null> {
|
||
const row = await db.prepare("SELECT json_extract(metadata_json, '$.library') AS library FROM entries WHERE id = ?").bind(id).first<{ library: string | null }>(); // kbdb-sql-ok：測試治具讀回斷言用
|
||
return row?.library ?? null;
|
||
}
|
||
|
||
function makeEnv(db: D1Database, opts: { maintenanceLimit?: string } = {}): Bindings {
|
||
return {
|
||
DB: db,
|
||
ENVIRONMENT: 'test',
|
||
KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: opts.maintenanceLimit,
|
||
} as unknown as Bindings;
|
||
}
|
||
|
||
describe('backfillEntryLibraryTags — 基本行為', () => {
|
||
it('只標記符合條件、目前未標記 library 的候選；已標記的不動', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', created_at: 1 }); // 無 metadata_json → 未標記
|
||
insertEntry(db, { id: 'b', created_at: 2, metadata_json: JSON.stringify({}) }); // 有 metadata_json 但無 library
|
||
insertEntry(db, { id: 'c', created_at: 3, metadata_json: JSON.stringify({ library: 'hr' }) }); // 已標記，不該被動
|
||
const env = makeEnv(db);
|
||
const r = await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' });
|
||
expect(r.tagged).toBe(2);
|
||
expect(r.remaining).toBe(0);
|
||
expect(await getLibrary(db, 'a')).toBe('finance');
|
||
expect(await getLibrary(db, 'b')).toBe('finance');
|
||
expect(await getLibrary(db, 'c')).toBe('hr'); // 未被覆寫
|
||
});
|
||
|
||
it('冪等：全部標記完後重跑不再處理', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', created_at: 1 });
|
||
const env = makeEnv(db);
|
||
await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' });
|
||
const r2 = await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' });
|
||
expect(r2.tagged).toBe(0);
|
||
expect(r2.remaining).toBe(0);
|
||
});
|
||
|
||
it('owner_id 缺了要拋錯（防補錯 owner 等於白做，2026-08-11 leo 直令）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', created_at: 1 });
|
||
const env = makeEnv(db);
|
||
await expect(
|
||
backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: '' }),
|
||
).rejects.toThrow(/owner_id/);
|
||
});
|
||
|
||
it('library 缺了要拋錯', async () => {
|
||
const db = makeSqliteD1();
|
||
const env = makeEnv(db);
|
||
await expect(
|
||
backfillEntryLibraryTags(db, env, { library: '', owner_id: 'bfezv28v' }),
|
||
).rejects.toThrow(/library/);
|
||
});
|
||
|
||
it('owner_id 篩選：只動指定租戶的資料，其他租戶不受影響（跨租戶隔離）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'mine', created_at: 1, owner_id: 'bfezv28v' });
|
||
insertEntry(db, { id: 'theirs', created_at: 2, owner_id: 'someone-else' });
|
||
const env = makeEnv(db);
|
||
const r = await backfillEntryLibraryTags(db, env, { library: 'finance', owner_id: 'bfezv28v' });
|
||
expect(r.tagged).toBe(1);
|
||
expect(await getLibrary(db, 'mine')).toBe('finance');
|
||
expect(await getLibrary(db, 'theirs')).toBe(null); // 別的租戶完全沒被動到
|
||
});
|
||
|
||
it('source_prefix／page_name_prefix／since／until 篩選真的在篩', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'match-source', created_at: 500, metadata_json: JSON.stringify({ source: 'gitea://Leo/kb/foo.md' }) });
|
||
insertEntry(db, { id: 'other-source', created_at: 500, metadata_json: JSON.stringify({ source: 'gitea://Leo/other/bar.md' }) });
|
||
const r1 = await backfillEntryLibraryTags(db, makeEnv(db), {
|
||
library: 'kb', owner_id: 'bfezv28v', source_prefix: 'gitea://Leo/kb/',
|
||
});
|
||
expect(r1.tagged).toBe(1);
|
||
expect(await getLibrary(db, 'match-source')).toBe('kb');
|
||
expect(await getLibrary(db, 'other-source')).toBe(null);
|
||
|
||
const db2 = makeSqliteD1();
|
||
insertEntry(db2, { id: 'in-window', created_at: 500, page_name: 'wiki/foo' });
|
||
insertEntry(db2, { id: 'out-window', created_at: 5000, page_name: 'wiki/bar' });
|
||
const r2 = await backfillEntryLibraryTags(db2, makeEnv(db2), {
|
||
library: 'wiki', owner_id: 'bfezv28v', page_name_prefix: 'wiki/', since: 0, until: 1000,
|
||
});
|
||
expect(r2.tagged).toBe(1);
|
||
expect(await getLibrary(db2, 'in-window')).toBe('wiki');
|
||
expect(await getLibrary(db2, 'out-window')).toBe(null);
|
||
});
|
||
|
||
it('page_names 精準比對（leo 定案的正解：拿 Gitea 原稿卡名逐批遍歷點名）', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', created_at: 1, page_name: 'card-alpha' });
|
||
insertEntry(db, { id: 'b', created_at: 2, page_name: 'card-beta' });
|
||
insertEntry(db, { id: 'c', created_at: 3, page_name: 'card-gamma' }); // 不在點名清單內
|
||
const r = await backfillEntryLibraryTags(db, makeEnv(db), {
|
||
library: 'kb', owner_id: 'bfezv28v', page_names: ['card-alpha', 'card-beta'],
|
||
});
|
||
expect(r.tagged).toBe(2);
|
||
expect(await getLibrary(db, 'a')).toBe('kb');
|
||
expect(await getLibrary(db, 'b')).toBe('kb');
|
||
expect(await getLibrary(db, 'c')).toBe(null); // 沒被點名，不動
|
||
});
|
||
|
||
it('libraryBackfillStatus 回報待補標筆數', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'a', created_at: 1 });
|
||
insertEntry(db, { id: 'b', created_at: 2, metadata_json: JSON.stringify({ library: 'hr' }) });
|
||
const s = await libraryBackfillStatus(db, { owner_id: 'bfezv28v' });
|
||
expect(s.pending).toBe(1); // 只有 'a' 未標記
|
||
});
|
||
});
|
||
|
||
describe('Arcrun#85 D69：標庫 backfill 與 reconcile 共用同一顆 D1 每日寫入額度', () => {
|
||
it('reconcile 先消耗額度 → 標庫 backfill 看到的剩餘額度真的變少', async () => {
|
||
const db = makeSqliteD1();
|
||
// reconcile 的候選：is_embedded=1 且 content_hash 非現行世代
|
||
// library 已標記（'hr'）→ 不會被下面的標庫 backfill 選中，讓兩種候選池互不重疊，
|
||
// 才能單純驗證「額度共用」本身，不被「標庫候選也吃到 reconcile 資料」干擾。
|
||
insertEntry(db, { id: 'reconcile-1', created_at: 1, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
|
||
insertEntry(db, { id: 'reconcile-2', created_at: 2, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true, library: 'hr' }) });
|
||
// 標庫的候選：未標記 library
|
||
insertEntry(db, { id: 'tag-1', created_at: 3 });
|
||
insertEntry(db, { id: 'tag-2', created_at: 4 });
|
||
insertEntry(db, { id: 'tag-3', created_at: 5 });
|
||
|
||
const maintenanceLimit = '3'; // 5 個候選（2 reconcile + 3 tag），額度只夠 3 個
|
||
const reconcileEnv = {
|
||
DB: db, ENVIRONMENT: 'test', KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: maintenanceLimit,
|
||
AI: { async run() { return { data: [] }; } },
|
||
VECTORIZE: {
|
||
async getByIds(ids: string[]) { return ids.map((id) => ({ id, values: [0.1] })); }, // 全部視為現行 index 已有
|
||
async upsert() { return { count: 0 }; },
|
||
},
|
||
} as unknown as Bindings;
|
||
|
||
// 先跑 reconcile：吃掉 2 筆額度（3 - 2 = 1 剩）
|
||
const rc = await reconcileEmbedGeneration(reconcileEnv, { limit: 100 });
|
||
expect(rc.checked).toBe(2);
|
||
expect(rc.quota_used_today).toBe(2);
|
||
|
||
// 標庫 backfill 用同一顆 DB／同一個每日上限：只剩 1 筆額度可用，即使候選有 3 筆
|
||
const tagEnv = makeEnv(db, { maintenanceLimit });
|
||
const tagResult = await backfillEntryLibraryTags(db, tagEnv, { library: 'general', owner_id: 'bfezv28v' });
|
||
expect(tagResult.scanned).toBe(3); // 3 筆候選都掃到了
|
||
expect(tagResult.tagged).toBe(1); // 但只剩 1 筆額度，只標了 1 筆
|
||
expect(tagResult.quota_exceeded).toBe(true);
|
||
expect(tagResult.quota_used_today).toBe(3); // 2（reconcile）+ 1（本次）＝ 3，額度用滿
|
||
});
|
||
|
||
it('反過來也一樣：標庫 backfill 先消耗額度 → reconcile 看到的剩餘額度真的變少', async () => {
|
||
const db = makeSqliteD1();
|
||
insertEntry(db, { id: 'tag-1', created_at: 1 });
|
||
insertEntry(db, { id: 'tag-2', created_at: 2 });
|
||
insertEntry(db, { id: 'reconcile-1', created_at: 3, is_embedded: 1, content_hash: null, metadata_json: JSON.stringify({ embed: true }) });
|
||
|
||
const maintenanceLimit = '2';
|
||
const tagEnv = makeEnv(db, { maintenanceLimit });
|
||
const tagResult = await backfillEntryLibraryTags(db, tagEnv, { library: 'general', owner_id: 'bfezv28v' });
|
||
expect(tagResult.tagged).toBe(2); // 額度剛好夠標完兩筆
|
||
|
||
const reconcileEnv = {
|
||
DB: db, ENVIRONMENT: 'test', KBDB_MAINTENANCE_DAILY_WRITE_LIMIT: maintenanceLimit,
|
||
AI: { async run() { return { data: [] }; } },
|
||
VECTORIZE: {
|
||
async getByIds(ids: string[]) { return ids.map((id) => ({ id, values: [0.1] })); },
|
||
async upsert() { return { count: 0 }; },
|
||
},
|
||
} as unknown as Bindings;
|
||
const rc = await reconcileEmbedGeneration(reconcileEnv, { limit: 100 });
|
||
expect(rc.scanned).toBe(1); // 有 1 筆候選
|
||
expect(rc.checked).toBe(0); // 但額度已被標庫 backfill 用光，reconcile 一筆都動不了
|
||
expect(rc.quota_exceeded).toBe(true);
|
||
});
|
||
});
|