portal-auth P1(#24 #25):KBDB library filter 地基(不 merge,待總管審) #50
@@ -58,6 +58,8 @@ export interface ListEntriesFilter {
|
||||
parent_id?: string;
|
||||
page_name?: string; // exact-match lookup (e.g. skill-/example- idempotency key)
|
||||
source?: string; // filter by metadata_json.$.source (ingest envelope source.uri). issue #5.1
|
||||
library?: string[]; // filter by metadata_json.$.library(多值 OR;portal-auth P1,#24/#25)。
|
||||
// 未帶=不過濾(向後相容硬驗收);未標記的舊資料視同 'general'(design §3.2)。
|
||||
q?: string; // keyword filter on content (LIKE). Arcrun#3 發現①:list 端點原本完全不吃
|
||||
// search/q,caller 帶了也被靜默丟棄(不是 458K 筆搜不到,是這個 filter 沒接)。
|
||||
limit?: number;
|
||||
@@ -81,6 +83,7 @@ export async function listEntries(db: D1Database, f: ListEntriesFilter = {}): Pr
|
||||
// source is queryable via SQLite json_extract on the existing metadata_json TEXT column —
|
||||
// no new column / no migration (表不變鐵律). Per issue #5.1 (頂層化 source 成可查 filter).
|
||||
if (f.source) { conds.push("json_extract(metadata_json, '$.source') = ?"); params.push(f.source); }
|
||||
if (f.library && f.library.length > 0) { conds.push(libraryPredicate(f.library)); params.push(...f.library); }
|
||||
if (f.q) { conds.push('content LIKE ?'); params.push(`%${f.q}%`); }
|
||||
const where = conds.length ? `WHERE ${conds.join(' AND ')}` : '';
|
||||
const limit = Math.min(f.limit ?? 100, 1000);
|
||||
@@ -123,19 +126,30 @@ export async function deleteEntry(db: D1Database, id: string): Promise<void> {
|
||||
await db.prepare('DELETE FROM entries WHERE id = ?').bind(id).run();
|
||||
}
|
||||
|
||||
// 「庫」filter 的 SQL 謂詞(portal-auth P1,design §3.2/§3.3;零建表,同 #5.1 source 的 json_extract 先例)。
|
||||
// COALESCE(x,'general') IN (…) ≡ SDD §3.3 寫的 (x IN (…) OR (x IS NULL AND 'general' IN (…)))——
|
||||
// 語意完全相同(未標記/無 metadata_json 的舊資料歸 'general'),但單組佔位符、不用重複綁參數。
|
||||
function libraryPredicate(libraries: string[]): string {
|
||||
const placeholders = libraries.map(() => '?').join(',');
|
||||
return `COALESCE(json_extract(metadata_json, '$.library'), 'general') IN (${placeholders})`;
|
||||
}
|
||||
|
||||
// D1 LIKE keyword search (base; semantic search is the optional embed module).
|
||||
// entry_type: optional base filter (generic — caller passes any type, base stays type-agnostic).
|
||||
// library: optional 多值庫 filter(portal-auth P1);未帶=行為與舊版一字不變(向後相容)。
|
||||
export async function searchEntries(
|
||||
db: D1Database,
|
||||
q: string,
|
||||
owner_id?: string,
|
||||
entry_type?: string,
|
||||
limit = 50,
|
||||
library?: string[],
|
||||
): Promise<Entry[]> {
|
||||
const conds = ['content LIKE ?'];
|
||||
const params: unknown[] = [`%${q}%`];
|
||||
if (owner_id) { conds.push('owner_id = ?'); params.push(owner_id); }
|
||||
if (entry_type) { conds.push('entry_type = ?'); params.push(entry_type); }
|
||||
if (library && library.length > 0) { conds.push(libraryPredicate(library)); params.push(...library); }
|
||||
const res = await db
|
||||
.prepare(`SELECT * FROM entries WHERE ${conds.join(' AND ')} ORDER BY updated_at DESC LIMIT ?`)
|
||||
.bind(...params, Math.min(limit, 200))
|
||||
|
||||
+22
-3
@@ -44,11 +44,15 @@ export async function embedOnWrite(env: Bindings, entry: Entry): Promise<boolean
|
||||
{
|
||||
id: entry.id,
|
||||
values: vec,
|
||||
// metadata 走 indexed 範圍:owner_id(租戶隔離)、entry_type、source(#5.1 過濾與語義共用)。
|
||||
// metadata 走 indexed 範圍:owner_id(租戶隔離)、entry_type、source(#5.1 過濾與語義共用)、
|
||||
// library(portal-auth P1「庫」filter)。library 在寫入端正規化:未標記='general'(design §3.2
|
||||
// 「未蓋章的舊資料視同 general」——D1 側用查詢端 COALESCE fallback,Vectorize filter 做不了
|
||||
// COALESCE,故在 upsert 時蓋 'general',查詢端單純 $in 即可)。
|
||||
metadata: {
|
||||
owner_id: entry.owner_id ?? '',
|
||||
entry_type: entry.entry_type,
|
||||
source: readSource(entry) ?? '',
|
||||
library: readLibrary(entry) ?? 'general',
|
||||
},
|
||||
},
|
||||
]);
|
||||
@@ -69,6 +73,13 @@ function readSource(entry: Entry): string | null {
|
||||
return typeof s === 'string' ? s : null;
|
||||
}
|
||||
|
||||
/** metadata_json.$.library(portal-auth P1)。非字串/空字串一律視同未標記(→ caller fallback 'general')。 */
|
||||
function readLibrary(entry: Entry): string | null {
|
||||
const meta = parseMeta(entry.metadata_json);
|
||||
const l = meta?.library;
|
||||
return typeof l === 'string' && l.trim() !== '' ? l : null;
|
||||
}
|
||||
|
||||
function parseMeta(json: string | null): Record<string, unknown> | null {
|
||||
if (!json) return null;
|
||||
try {
|
||||
@@ -150,6 +161,7 @@ export async function backfillEmbeddings(
|
||||
owner_id: x.e.owner_id ?? '',
|
||||
entry_type: x.e.entry_type,
|
||||
source: readSource(x.e) ?? '',
|
||||
library: readLibrary(x.e) ?? 'general', // 同 embedOnWrite:寫入端正規化(P1)
|
||||
},
|
||||
}));
|
||||
if (vectors.length > 0) {
|
||||
@@ -199,25 +211,31 @@ export interface SemanticHit {
|
||||
owner_id?: string;
|
||||
entry_type?: string;
|
||||
source?: string;
|
||||
library?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 語義搜尋(mode:'semantic')。模組未開 → 回 null(caller 降級 keyword + 告知缺能力)。
|
||||
* owner_id / source / entry_type 過濾走 Vectorize metadata filter(entry_type 已 index,見上 upsert metadata)。
|
||||
* entry_type 是 base 通用 filter(caller 傳任意 type,base 不寫死語意)。
|
||||
* library(portal-auth P1):多值庫 filter 走 `$in`(官方支援已核實 2026-07-14:文件明列 $in/$nin+
|
||||
* workers-types 原生 typing;design §3.3 的 fan-out fallback 不需啟用)。未帶=行為不變。
|
||||
* 註:向量 metadata 的 library 在寫入端已正規化(未標記='general'),故 $in 不需 NULL 處理;
|
||||
* 但「建 library metadata index 之前」upsert 的既有向量沒有此欄 → 部署清單強制 reindex backfill。
|
||||
*/
|
||||
export async function semanticSearch(
|
||||
env: Bindings,
|
||||
q: string,
|
||||
opts: { owner_id?: string; source?: string; entry_type?: string; topK?: number } = {},
|
||||
opts: { owner_id?: string; source?: string; entry_type?: string; library?: string[]; topK?: number } = {},
|
||||
): Promise<SemanticHit[] | null> {
|
||||
if (!embedEnabled(env)) return null;
|
||||
const vec = await embedText(env, q);
|
||||
if (!vec) return [];
|
||||
const filter: Record<string, string> = {};
|
||||
const filter: VectorizeVectorMetadataFilter = {};
|
||||
if (opts.owner_id) filter.owner_id = opts.owner_id;
|
||||
if (opts.source) filter.source = opts.source;
|
||||
if (opts.entry_type) filter.entry_type = opts.entry_type;
|
||||
if (opts.library && opts.library.length > 0) filter.library = { $in: opts.library };
|
||||
const res = await env.VECTORIZE!.query(vec, {
|
||||
topK: Math.min(opts.topK ?? 20, 100),
|
||||
returnMetadata: 'indexed',
|
||||
@@ -229,5 +247,6 @@ export async function semanticSearch(
|
||||
owner_id: m.metadata?.owner_id as string | undefined,
|
||||
entry_type: m.metadata?.entry_type as string | undefined,
|
||||
source: m.metadata?.source as string | undefined,
|
||||
library: m.metadata?.library as string | undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -13,6 +13,14 @@ import { embedEnabled, embedOnWrite, semanticSearch } from '../embed';
|
||||
|
||||
export const entryRoutes = new Hono<{ Bindings: Bindings }>();
|
||||
|
||||
// library 多值參數(逗號分隔,portal-auth P1,design §3.3)。空值/全空白 → undefined(=不過濾,
|
||||
// 行為與未帶參數一字不變——向後相容硬驗收)。
|
||||
function parseLibraryParam(raw: string | undefined): string[] | undefined {
|
||||
if (!raw) return undefined;
|
||||
const libs = raw.split(',').map((s) => s.trim()).filter(Boolean);
|
||||
return libs.length > 0 ? libs : undefined;
|
||||
}
|
||||
|
||||
// POST /entries — create (entry_type=block/value/project/workflow/...)
|
||||
entryRoutes.post('/', async (c) => {
|
||||
const body = await c.req.json().catch(() => null);
|
||||
@@ -27,6 +35,7 @@ entryRoutes.post('/', async (c) => {
|
||||
// e.g. list workflows under a project: ?parent_id=PROJECT&entry_type=workflow
|
||||
// e.g. get one by idempotency key: ?page_name=skill-rag_with_arcrun
|
||||
// e.g. filter by ingest source: ?source=logseq://vault/foo.md (issue #5.1)
|
||||
// e.g. filter by library(多值逗號分隔,portal-auth P1): ?library=finance,hr(未標記舊資料歸 general)
|
||||
// e.g. keyword filter: ?q=遷移 或 ?search=遷移(別名,Arcrun#3 發現①:caller 實測時打的是 search=,
|
||||
// 舊版完全不接這個 filter;q 與 search 兩個名字都認,避免同一個坑再踩一次)。
|
||||
// count = 本頁筆數(受 limit 影響);total = 符合條件全部筆數(不受 limit 影響,見 total 欄位)。
|
||||
@@ -37,6 +46,7 @@ entryRoutes.get('/', async (c) => {
|
||||
parent_id: c.req.query('parent_id') || undefined,
|
||||
page_name: c.req.query('page_name') || undefined,
|
||||
source: c.req.query('source') || undefined,
|
||||
library: parseLibraryParam(c.req.query('library')),
|
||||
q: c.req.query('q') || c.req.query('search') || undefined,
|
||||
limit: c.req.query('limit') ? Number(c.req.query('limit')) : undefined,
|
||||
offset: c.req.query('offset') ? Number(c.req.query('offset')) : undefined,
|
||||
@@ -44,23 +54,26 @@ entryRoutes.get('/', async (c) => {
|
||||
return c.json({ success: true, entries, count: entries.length, total });
|
||||
});
|
||||
|
||||
// GET /entries/search?q=...&owner_id=...&source=...&entry_type=...&mode=keyword|semantic
|
||||
// GET /entries/search?q=...&owner_id=...&source=...&entry_type=...&library=...&mode=keyword|semantic
|
||||
// - mode=keyword(預設):D1 LIKE(base,永遠可用)。
|
||||
// - mode=semantic:需 embed 模組開(Vectorize+AI binding)。未開 → 降級 keyword + capability_hint 告知缺能力(#7 發現閉環)。
|
||||
// - entry_type:base 通用 filter(caller 傳任意 type,如 workflow;base 不寫死語意,workflow-discovery Q4)。
|
||||
// - library:多值庫 filter(逗號分隔,portal-auth P1)。keyword 走 json_extract+NULL→general;
|
||||
// semantic 走 Vectorize $in。未帶=全庫(行為不變)。
|
||||
entryRoutes.get('/search', async (c) => {
|
||||
const q = c.req.query('q');
|
||||
if (!q) return c.json({ success: false, error: 'q required' }, 400);
|
||||
const owner_id = c.req.query('owner_id') || undefined;
|
||||
const source = c.req.query('source') || undefined;
|
||||
const entry_type = c.req.query('entry_type') || undefined;
|
||||
const library = parseLibraryParam(c.req.query('library'));
|
||||
const mode = c.req.query('mode') === 'semantic' ? 'semantic' : 'keyword';
|
||||
|
||||
if (mode === 'semantic') {
|
||||
const hits = await semanticSearch(c.env, q, { owner_id, source, entry_type });
|
||||
const hits = await semanticSearch(c.env, q, { owner_id, source, entry_type, library });
|
||||
if (hits === null) {
|
||||
// 模組沒開:誠實降級 keyword + 告知「叫 CC 幫你開 vectorize」(不假裝有語義)。
|
||||
const entries = await searchEntries(c.env.DB, q, owner_id, entry_type);
|
||||
const entries = await searchEntries(c.env.DB, q, owner_id, entry_type, undefined, library);
|
||||
return c.json({
|
||||
success: true,
|
||||
entries,
|
||||
@@ -78,7 +91,7 @@ entryRoutes.get('/search', async (c) => {
|
||||
return c.json({ success: true, entries, count: entries.length, mode: 'semantic' });
|
||||
}
|
||||
|
||||
const entries = await searchEntries(c.env.DB, q, owner_id, entry_type);
|
||||
const entries = await searchEntries(c.env.DB, q, owner_id, entry_type, undefined, library);
|
||||
return c.json({ success: true, entries, count: entries.length, mode: 'keyword' });
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// portal-auth P1 — 「庫」filter 地基(design §3.2/§3.3,Gitea #24/#25)。
|
||||
// 覆蓋:D1 filter SQL 形狀(單值/多值/NULL→general fallback 謂詞)、route 參數解析(含向後相容:
|
||||
// 不帶 library = SQL 一字不變)、semantic 路徑 Vectorize $in filter(mock VECTORIZE)、
|
||||
// embed 寫入端 library metadata 正規化(未標記→'general')。
|
||||
// D1 真實 SQL 語意(COALESCE/json_extract 實際執行)由本機 miniflare + wrangler d1 驗證(PR 驗收證據)。
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { Hono } from 'hono';
|
||||
import { entryRoutes } from '../src/routes/entries';
|
||||
import { listEntries, searchEntries } from '../src/actions/entry-crud';
|
||||
import { semanticSearch, embedOnWrite } from '../src/embed';
|
||||
import type { Bindings, Entry } from '../src/types';
|
||||
|
||||
const LIB_PREDICATE = "COALESCE(json_extract(metadata_json, '$.library'), 'general') IN";
|
||||
|
||||
// ── fake D1:只捕捉 prepared SQL 與 bound params(不解讀語意——真語意交給 miniflare 實跑)──
|
||||
interface Captured { sql: string; params: unknown[] }
|
||||
function makeCaptureDB(captured: Captured[]) {
|
||||
const prepare = (sql: string) => {
|
||||
const rec: Captured = { sql, params: [] };
|
||||
captured.push(rec);
|
||||
const stmt = {
|
||||
bind(...args: unknown[]) { rec.params = args; return stmt; },
|
||||
async all<T>() { return { results: [] as T[] }; },
|
||||
async first<T>() { return { total: 0, c: 0 } as unknown as T; },
|
||||
async run() { return { success: true }; },
|
||||
};
|
||||
return stmt;
|
||||
};
|
||||
return { prepare } as unknown as D1Database;
|
||||
}
|
||||
|
||||
function mkEntry(id: string, metadata_json: string | null): Entry {
|
||||
return {
|
||||
id, content: 'some content', entry_type: 'block', owner_id: 'tenant1', parent_id: null,
|
||||
page_name: null, refs_json: '[]', tags_json: '[]', task_status: null, content_hash: null,
|
||||
is_embedded: 0, confidence: null, metadata_json, created_at: 1, updated_at: 1,
|
||||
};
|
||||
}
|
||||
|
||||
describe('D1 library filter — SQL 形狀(entry-crud)', () => {
|
||||
it('listEntries 帶 library 多值 → COALESCE…IN (?,?) 謂詞+參數', async () => {
|
||||
const captured: Captured[] = [];
|
||||
await listEntries(makeCaptureDB(captured), { library: ['finance', 'hr'] });
|
||||
const select = captured.find((c) => c.sql.startsWith('SELECT *'))!;
|
||||
expect(select.sql).toContain(`${LIB_PREDICATE} (?,?)`);
|
||||
expect(select.params.slice(0, 2)).toEqual(['finance', 'hr']);
|
||||
// COUNT 查詢同謂詞(total 與分頁一致)
|
||||
const count = captured.find((c) => c.sql.includes('COUNT(*)'))!;
|
||||
expect(count.sql).toContain(`${LIB_PREDICATE} (?,?)`);
|
||||
});
|
||||
|
||||
it('listEntries 不帶 library → SQL 無庫謂詞(向後相容:行為一字不變)', async () => {
|
||||
const captured: Captured[] = [];
|
||||
await listEntries(makeCaptureDB(captured), { owner_id: 'tenant1' });
|
||||
for (const c of captured) expect(c.sql).not.toContain('$.library');
|
||||
});
|
||||
|
||||
it('searchEntries 帶 library → LIKE+庫謂詞;不帶 → 原樣', async () => {
|
||||
const withLib: Captured[] = [];
|
||||
await searchEntries(makeCaptureDB(withLib), '遷移', 'tenant1', undefined, undefined, ['general']);
|
||||
expect(withLib[0].sql).toContain('content LIKE ?');
|
||||
expect(withLib[0].sql).toContain(`${LIB_PREDICATE} (?)`);
|
||||
expect(withLib[0].params).toContain('general');
|
||||
|
||||
const without: Captured[] = [];
|
||||
await searchEntries(makeCaptureDB(without), '遷移', 'tenant1');
|
||||
expect(without[0].sql).not.toContain('$.library');
|
||||
});
|
||||
});
|
||||
|
||||
describe('route 參數解析(GET /entries、/entries/search)', () => {
|
||||
function makeApp(captured: Captured[]) {
|
||||
const app = new Hono<{ Bindings: Bindings }>();
|
||||
app.route('/entries', entryRoutes);
|
||||
const env = { DB: makeCaptureDB(captured), ENVIRONMENT: 'test' } as unknown as Bindings;
|
||||
return { app, env };
|
||||
}
|
||||
|
||||
it('GET /entries?library=finance,hr →(含空白容忍)庫謂詞+兩參數', async () => {
|
||||
const captured: Captured[] = [];
|
||||
const { app, env } = makeApp(captured);
|
||||
const res = await app.request('/entries?library=finance,%20hr', {}, env);
|
||||
expect(res.status).toBe(200);
|
||||
const select = captured.find((c) => c.sql.startsWith('SELECT *'))!;
|
||||
expect(select.sql).toContain(`${LIB_PREDICATE} (?,?)`);
|
||||
expect(select.params).toContain('finance');
|
||||
expect(select.params).toContain('hr');
|
||||
});
|
||||
|
||||
it('GET /entries 不帶 library / library=空 → SQL 無庫謂詞(向後相容)', async () => {
|
||||
for (const qs of ['', '?library=', '?library=%20,%20']) {
|
||||
const captured: Captured[] = [];
|
||||
const { app, env } = makeApp(captured);
|
||||
const res = await app.request(`/entries${qs}`, {}, env);
|
||||
expect(res.status).toBe(200);
|
||||
for (const c of captured) expect(c.sql).not.toContain('$.library');
|
||||
}
|
||||
});
|
||||
|
||||
it('GET /entries/search?q=x&library=finance(keyword 模式)→ 庫謂詞下到 searchEntries', async () => {
|
||||
const captured: Captured[] = [];
|
||||
const { app, env } = makeApp(captured);
|
||||
const res = await app.request('/entries/search?q=x&library=finance', {}, env);
|
||||
expect(res.status).toBe(200);
|
||||
expect(captured[0].sql).toContain(`${LIB_PREDICATE} (?)`);
|
||||
expect(captured[0].params).toContain('finance');
|
||||
});
|
||||
|
||||
it('semantic 模組未開+帶 library → 誠實降級 keyword 仍套庫 filter(不因降級洩庫)', async () => {
|
||||
const captured: Captured[] = [];
|
||||
const { app, env } = makeApp(captured); // 無 VECTORIZE/AI binding → semanticSearch 回 null
|
||||
const res = await app.request('/entries/search?q=x&mode=semantic&library=finance', {}, env);
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { mode: string; requested_mode?: string };
|
||||
expect(body.mode).toBe('keyword');
|
||||
expect(body.requested_mode).toBe('semantic');
|
||||
expect(captured[0].sql).toContain(`${LIB_PREDICATE} (?)`);
|
||||
expect(captured[0].params).toContain('finance');
|
||||
});
|
||||
});
|
||||
|
||||
describe('semantic 路徑 — Vectorize $in filter(mock VECTORIZE)', () => {
|
||||
function makeSemanticEnv(queryCalls: { vec: number[]; opts: Record<string, unknown> }[]) {
|
||||
return {
|
||||
DB: makeCaptureDB([]),
|
||||
ENVIRONMENT: 'test',
|
||||
AI: { async run() { return { data: [[0.1, 0.2, 0.3]] }; } },
|
||||
VECTORIZE: {
|
||||
async query(vec: number[], opts: Record<string, unknown>) {
|
||||
queryCalls.push({ vec, opts });
|
||||
return { matches: [{ id: 'e1', score: 0.9, metadata: { library: 'finance' } }] };
|
||||
},
|
||||
async upsert(v: unknown[]) { return { count: (v as unknown[]).length }; },
|
||||
},
|
||||
} as unknown as Bindings;
|
||||
}
|
||||
|
||||
it('帶 library 多值 → filter.library = { $in: [...] }(主路徑,不 fan-out)', async () => {
|
||||
const calls: { vec: number[]; opts: Record<string, unknown> }[] = [];
|
||||
const env = makeSemanticEnv(calls);
|
||||
const hits = await semanticSearch(env, 'query', { owner_id: 'tenant1', library: ['finance', 'hr'] });
|
||||
expect(calls.length).toBe(1); // 單次 query(非每庫 fan-out)
|
||||
const filter = calls[0].opts.filter as Record<string, unknown>;
|
||||
expect(filter.owner_id).toBe('tenant1');
|
||||
expect(filter.library).toEqual({ $in: ['finance', 'hr'] });
|
||||
expect(hits?.[0]?.library).toBe('finance'); // hit 帶回 library metadata
|
||||
});
|
||||
|
||||
it('不帶 library → filter 無 library 鍵(行為不變)', async () => {
|
||||
const calls: { vec: number[]; opts: Record<string, unknown> }[] = [];
|
||||
const env = makeSemanticEnv(calls);
|
||||
await semanticSearch(env, 'query', { owner_id: 'tenant1' });
|
||||
const filter = calls[0].opts.filter as Record<string, unknown>;
|
||||
expect('library' in filter).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('embed 寫入端 — library metadata 正規化', () => {
|
||||
function makeUpsertEnv(upserts: { id: string; metadata: Record<string, unknown> }[]) {
|
||||
const db = {
|
||||
prepare: () => {
|
||||
const stmt = { bind: () => stmt, run: async () => ({ success: true }), all: async () => ({ results: [] }), first: async () => null };
|
||||
return stmt;
|
||||
},
|
||||
} as unknown as D1Database;
|
||||
return {
|
||||
DB: db,
|
||||
ENVIRONMENT: 'test',
|
||||
AI: { async run(_m: string, i: { text: string[] }) { return { data: i.text.map(() => [0.1, 0.2]) }; } },
|
||||
VECTORIZE: { async upsert(v: { id: string; metadata: Record<string, unknown> }[]) { upserts.push(...v); return { count: v.length }; } },
|
||||
} as unknown as Bindings;
|
||||
}
|
||||
|
||||
it('metadata_json 有 library → upsert metadata.library 原值', async () => {
|
||||
const upserts: { id: string; metadata: Record<string, unknown> }[] = [];
|
||||
const env = makeUpsertEnv(upserts);
|
||||
await embedOnWrite(env, mkEntry('e1', JSON.stringify({ embed: true, library: 'finance' })));
|
||||
expect(upserts[0].metadata.library).toBe('finance');
|
||||
});
|
||||
|
||||
it('未標記 / 空字串 / 非字串 → 正規化為 general(design §3.2 舊資料歸 general)', async () => {
|
||||
for (const meta of [{ embed: true }, { embed: true, library: '' }, { embed: true, library: 42 }]) {
|
||||
const upserts: { id: string; metadata: Record<string, unknown> }[] = [];
|
||||
const env = makeUpsertEnv(upserts);
|
||||
await embedOnWrite(env, mkEntry('e1', JSON.stringify(meta)));
|
||||
expect(upserts[0].metadata.library).toBe('general');
|
||||
}
|
||||
});
|
||||
});
|
||||
+6
-2
@@ -24,8 +24,12 @@ ENVIRONMENT = "production"
|
||||
# wrangler vectorize create-metadata-index arcrun-kbdb-embed --property-name owner_id --type string
|
||||
# wrangler vectorize create-metadata-index arcrun-kbdb-embed --property-name entry_type --type string
|
||||
# wrangler vectorize create-metadata-index arcrun-kbdb-embed --property-name source --type string
|
||||
# metadata index 只收「建立後 upsert」的向量 → 既有向量須 `POST /embed/backfill {"reindex":true}` 重推。
|
||||
# deploy.ts 的 ensureVectorizeMetadataIndexes() 已把上述三個 index 隨部署冪等建好。
|
||||
# wrangler vectorize create-metadata-index arcrun-kbdb-embed --property-name library --type string
|
||||
# (library=portal-auth P1「庫」filter;upsert 端把未標記正規化成 'general',查詢走 $in)
|
||||
# metadata index 只收「建立後 upsert」的向量 → 既有向量須 `POST /embed/backfill {"reindex":true}` 重推
|
||||
# (建 library index 後同樣要 reindex,否則舊向量帶 library filter 一律 0 命中)。
|
||||
# deploy.ts 的 ensureVectorizeMetadataIndexes() 已把前三個 index 隨部署冪等建好;library 待補進該清單
|
||||
# (cli/ 屬 portal-auth P1 範圍外,見 portal-auth tasks.md 部署清單附註)。
|
||||
# 沒有這兩個 binding 時,kbdb/src/embed.ts 的 embedEnabled() 回 false → 維持 LIKE keyword、API 不變。
|
||||
#
|
||||
# [[vectorize]]
|
||||
|
||||
@@ -9,12 +9,33 @@
|
||||
|
||||
## P1 — KBDB「庫」filter 地基(design §3.2/§3.3)|觸碰:`kbdb/`
|
||||
|
||||
- [ ] 開工第一件事:實測 Vectorize metadata filter `$in` 支援與否(決定主路徑 vs fan-out fallback)
|
||||
- [ ] `/entries/search`+`/entries` 加 `library` 多值參數(D1 json_extract IN+NULL→general fallback)
|
||||
- [ ] embed upsert metadata 加 `library` 欄;semanticSearch filter 支援 library($in 或 fan-out)
|
||||
- [ ] Vectorize `library` metadata index 建立步驟+reindex backfill 寫進部署清單
|
||||
- [x] 開工第一件事:實測 Vectorize metadata filter `$in` 支援與否(決定主路徑 vs fan-out fallback)
|
||||
- **核實結論(2026-07-14)**:**支援,走主路徑 `$in`,不需 fan-out fallback**。證據:① 官方文件
|
||||
developers.cloudflare.com/vectorize/reference/metadata-filtering/ 明列 8 運算子
|
||||
`$eq/$ne/$in/$nin/$lt/$lte/$gt/$gte`,「For $in and $nin, filter object values can be arrays of
|
||||
string, number, boolean, or null values」;② 本 repo `@cloudflare/workers-types@4.20260702.1`
|
||||
原生 typing `VectorizeVectorMetadataFilterCollectionOp = "$in" | "$nin"`(index.d.ts:15782),
|
||||
wrangler 4.98.0。限制:metadata filtering 只對 2023-12-06 之後建的 index 有效(本專案 index 皆是)。
|
||||
本機無 Vectorize runtime,`$in` 的**線上實跑**併入部署排練驗(見部署清單)。
|
||||
- [x] `/entries/search`+`/entries` 加 `library` 多值參數(D1 json_extract IN+NULL→general fallback)
|
||||
- 實作註:SQL 用 `COALESCE(json_extract(metadata_json,'$.library'),'general') IN (…)`——與 design §3.3
|
||||
的 OR 形狀語意完全等價(單組佔位符較簡)。本機 miniflare+local D1 十案例實跑全過(PR 附證據)。
|
||||
- [x] embed upsert metadata 加 `library` 欄;semanticSearch filter 支援 library(**$in 主路徑**,fan-out 不需)
|
||||
- 實作註:Vectorize 端在**寫入時正規化**(未標記→`'general'`)——Vectorize filter 做不了 COALESCE,
|
||||
寫入端蓋章後查詢端單純 `$in`;與 D1 查詢端 fallback 語意對齊。
|
||||
- [x] Vectorize `library` metadata index 建立步驟+reindex backfill 寫進部署清單
|
||||
- **部署清單(待雲端排練驗——本機無 Vectorize runtime,semantic 路徑 code 完成、線上實跑未驗)**:
|
||||
1. `wrangler vectorize create-metadata-index arcrun-kbdb-embed --property-name library --type string`
|
||||
2. `POST /embed/backfill {"reindex":true}` 分批到 remaining=0(index 只收建立後 upsert 的向量)
|
||||
3. 驗收抽查:semantic 帶 `library=` 過濾命中/不帶行為不變
|
||||
4. 掛號:`cli/src/lib/deploy.ts` `ensureVectorizeMetadataIndexes()` 待補 `library`(cli/ 屬 P1
|
||||
派工範圍外「只動 kbdb」,隨 P3 或部署 PR 補——kbdb/wrangler.toml 註解已標)
|
||||
- [ ] cypher `kbdb-proxy` 透傳 `library` 參數(供 owner/admin 面用;portal 面走 P3 的注入,不經這)
|
||||
- [ ] 測試:D1 filter 單元測、NULL fallback、多值、semantic filter(mock VECTORIZE)
|
||||
- ⚠️ 範圍註(2026-07-14):P1 派工紅線「只動 kbdb、不碰 cypher-executor」與本項矛盾——照窄範圍
|
||||
執行,本項順延(一行 query 透傳,隨 P3 動 cypher 時一併)。
|
||||
- [x] 測試:D1 filter 單元測、NULL fallback、多值、semantic filter(mock VECTORIZE)
|
||||
- `kbdb/tests/library-filter.test.ts` 11 項(SQL 形狀/route 解析/向後相容/降級仍 enforce/$in 構造/
|
||||
寫入端正規化)+既有 6 項全綠(17/17);tsc exit 0。
|
||||
- **驗收**:curl `/entries/search?q=&library=finance` 只回 finance+未標記條目歸 general 可驗;semantic 同
|
||||
- **工程量**:小-中(0.5–1 個 CC 工作天)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user