feat(ingest): 機械式 wiki 卡片→KBDB ingest（Arcrun#8 Phase A，dry-run 驗證）
新 ingest 模型（無 LLM，取代舊 raw→Haiku 路）： - 來源 = system-dev/wiki/cards/**/*.md（精耕卡）+ ## 實體/## 關聯 typed-edge + [[wikilink]] - 卡片→base entry（metadata.embed=true）；typed-edge/wikilink→graph triplet - 純機械決定性，零 token 形式 = Arcrun workflow（cron drain + Gitea webhook delta）+ 新自訂零件 km_wiki_card_parse 不撞頂設計：一卡一 tick + 超大卡以 source_uri anchor 自動分段； graph fan-out 精確 = 7+4N+M+D，dry-run 對 notes 三卡上限 33 subrequest（<50）。 Phase A：不部署、不寫 live。含純核心 + dry-run 證據（3 entries/15 triplets/16 nodes）。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HJiLCRUU2o3aSpPEzVCt2o
This commit is contained in:
Arcrun CC
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env node
|
||||
// km-wiki-ingest 乾跑（dry-run）驗證器 — 不寫 live、不部署。
|
||||
// -------------------------------------------------------------
|
||||
// 對某 repo 的 system-dev/wiki/cards/**/*.md 跑機械解析 + envelope 打包，
|
||||
// 輸出「將寫入什麼」：① entry 清單（page_name / entry_type / metadata.embed / content_hash）
|
||||
// ② triplet envelope 清單（每段的 nodes / triplets / source.uri+anchor）
|
||||
// ③ 每個 graph /triplets/ingest 呼叫的 subrequest 估算（證明壓在 CF 上限下）
|
||||
// ④ 冪等鍵設計（entry=page_name+content_hash；triplet=source.uri+content_hash）。
|
||||
//
|
||||
// 用法：node dry-run.mjs --repo-path <clone路徑> [--repo Leo/notes] [--budget 40] [--json] [--full]
|
||||
// --self-test 額外跑「合成超大卡」證明分段生效。
|
||||
|
||||
import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { planCard, estimateEnvelopeSubrequests, SUBREQ_CEILING, SUBREQ_BUDGET } from './card-to-envelope.mjs';
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { repo: 'Leo/notes', budget: SUBREQ_BUDGET };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--repo-path') out.repoPath = argv[++i];
|
||||
else if (a === '--repo') out.repo = argv[++i];
|
||||
else if (a === '--budget') out.budget = Number(argv[++i]);
|
||||
else if (a === '--json') out.json = true;
|
||||
else if (a === '--full') out.full = true;
|
||||
else if (a === '--self-test') out.selfTest = true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function walkCards(dir) {
|
||||
const out = [];
|
||||
if (!existsSync(dir)) return out;
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = path.join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...walkCards(p));
|
||||
else if (e.name.endsWith('.md') && e.name !== '.gitkeep' && !e.name.startsWith('00-INDEX')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function gitCommit(repoPath) {
|
||||
try {
|
||||
return execFileSync('git', ['-C', repoPath, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (!args.repoPath) {
|
||||
console.error('用法: node dry-run.mjs --repo-path <clone路徑> [--repo Leo/notes] [--budget 40] [--json] [--full] [--self-test]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cardsRoot = path.join(args.repoPath, 'system-dev', 'wiki', 'cards');
|
||||
const cardPaths = walkCards(cardsRoot);
|
||||
const commit = gitCommit(args.repoPath);
|
||||
|
||||
const report = { repo: args.repo, commit, budget: args.budget, ceiling: SUBREQ_CEILING, cards: [], totals: {} };
|
||||
let totEntries = 0, totEnvelopes = 0, totTriplets = 0, totNodes = 0, maxSub = 0, over = 0;
|
||||
|
||||
for (const p of cardPaths) {
|
||||
const rel = path.relative(args.repoPath, p);
|
||||
const md = readFileSync(p, 'utf8');
|
||||
const plan = planCard(md, rel, args.repo, { budget: args.budget, commit });
|
||||
totEntries++;
|
||||
totEnvelopes += plan.envelopes.length;
|
||||
const cardTriplets = plan.envelopes.reduce((s, e) => s + e.triplets.length, 0);
|
||||
const cardNodes = plan.envelopes.reduce((s, e) => s + e.nodes.length, 0);
|
||||
totTriplets += cardTriplets;
|
||||
totNodes += cardNodes;
|
||||
for (const e of plan.envelopes) {
|
||||
maxSub = Math.max(maxSub, e._estSubrequests);
|
||||
if (e._estSubrequests > SUBREQ_CEILING) over++;
|
||||
}
|
||||
report.cards.push({
|
||||
relPath: rel,
|
||||
canonical: plan.meta.canonical,
|
||||
entry: {
|
||||
page_name: plan.entry.page_name,
|
||||
entry_type: plan.entry.entry_type,
|
||||
'metadata.embed': plan.entry.metadata.embed,
|
||||
content_hash: plan.entry.metadata.content_hash.slice(0, 12) + '…',
|
||||
content_bytes: Buffer.byteLength(plan.entry.content, 'utf8'),
|
||||
tags: plan.entry.tags,
|
||||
},
|
||||
envelopeCount: plan.envelopes.length,
|
||||
envelopes: plan.envelopes.map((e) => ({
|
||||
'source.uri': e.source.uri,
|
||||
'source.anchor': e.source.anchor ?? null,
|
||||
nodes: e.nodes.length,
|
||||
triplets: e.triplets.length,
|
||||
est_subrequests: e._estSubrequests,
|
||||
under_ceiling: e._estSubrequests <= SUBREQ_CEILING,
|
||||
sample_triplets: e.triplets.slice(0, args.full ? 999 : 4).map((t) => `${t.subject} >> ${t.predicate} >> ${t.object} (${t.confidence})`),
|
||||
sample_nodes: e.nodes.slice(0, args.full ? 999 : 4).map((n) => `${n.name}${n.gloss ? ' — ' + n.gloss.slice(0, 30) + '…' : ''}`),
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
report.totals = {
|
||||
cards: totEntries,
|
||||
entries_to_upsert: totEntries,
|
||||
triplet_envelopes: totEnvelopes,
|
||||
total_triplets: totTriplets,
|
||||
total_nodes: totNodes,
|
||||
max_est_subrequests_single_call: maxSub,
|
||||
ceiling: SUBREQ_CEILING,
|
||||
budget: args.budget,
|
||||
any_envelope_over_ceiling: over,
|
||||
};
|
||||
|
||||
// --- self-test：合成一張「超大卡」（20 邊 + 22 節點）證明單 envelope 會爆、分段後每段壓在預算下 ---
|
||||
if (args.selfTest) {
|
||||
const entities = [];
|
||||
const edges = [];
|
||||
for (let i = 0; i < 22; i++) entities.push(`- **實體${i}**（別名${i}）— 這是實體 ${i} 的一句描述。`);
|
||||
for (let i = 0; i < 20; i++) edges.push(`- 實體${i} >> 關聯到 >> 實體${i + 1}`);
|
||||
const bigCard = `---\ntags: [壓測]\ngloss: 合成超大卡，測分段。\n---\n# 合成超大卡\n\n← [[notes/00-INDEX]]\n\n## 實體\n${entities.join('\n')}\n\n## 關聯\n### 內文知識關係\n${edges.join('\n')}\n`;
|
||||
const plan = planCard(bigCard, 'system-dev/wiki/cards/notes/合成超大卡.md', args.repo, { budget: args.budget });
|
||||
const single = estimateEnvelopeSubrequests(plan.tripletCount, plan.nodeCount);
|
||||
report.self_test = {
|
||||
note: '合成 20 邊 / 22 節點 的超大卡',
|
||||
if_single_envelope_est_subrequests: single,
|
||||
would_crash_single: single > SUBREQ_CEILING,
|
||||
segmented_into: plan.envelopes.length,
|
||||
per_segment: plan.envelopes.map((e) => ({
|
||||
uri: e.source.uri, anchor: e.source.anchor, triplets: e.triplets.length, nodes: e.nodes.length, est_subrequests: e._estSubrequests, under_ceiling: e._estSubrequests <= SUBREQ_CEILING,
|
||||
})),
|
||||
all_segments_under_ceiling: plan.envelopes.every((e) => e._estSubrequests <= SUBREQ_CEILING),
|
||||
};
|
||||
}
|
||||
|
||||
if (args.json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --- 人類可讀輸出 ---
|
||||
const L = (s = '') => console.log(s);
|
||||
L(`\n================ km-wiki-ingest DRY-RUN（不寫 live）================`);
|
||||
L(`repo=${report.repo} commit=${(commit || '(none)').slice(0, 12)} budget=${args.budget} CF_ceiling=${SUBREQ_CEILING}`);
|
||||
L(`卡片來源根：${path.relative(args.repoPath, cardsRoot)} 找到 ${cardPaths.length} 張卡\n`);
|
||||
|
||||
for (const c of report.cards) {
|
||||
L(`── 卡片：${c.relPath}`);
|
||||
L(` ENTRY（base POST /entries 或 kbdb_upsert_block，冪等鍵 page_name）：`);
|
||||
L(` page_name = ${c.entry.page_name}`);
|
||||
L(` entry_type = ${c.entry.entry_type}`);
|
||||
L(` metadata.embed= ${c.entry['metadata.embed']} content_hash=${c.entry.content_hash} bytes=${c.entry.content_bytes}`);
|
||||
L(` tags = ${JSON.stringify(c.entry.tags)}`);
|
||||
L(` TRIPLET ENVELOPE(s)（POST graph /triplets/ingest；分段數=${c.envelopeCount}）：`);
|
||||
for (const e of c.envelopes) {
|
||||
L(` • uri=${e['source.uri']}${e['source.anchor'] ? ' anchor=' + e['source.anchor'] : ''}`);
|
||||
L(` nodes=${e.nodes} triplets=${e.triplets} est_subrequests=${e.est_subrequests} ≤ceiling? ${e.under_ceiling ? 'YES' : 'NO ⚠️'}`);
|
||||
for (const t of e.sample_triplets) L(` - ${t}`);
|
||||
if (e.sample_nodes.length) L(` nodes: ${e.sample_nodes.join(' | ')}`);
|
||||
}
|
||||
L('');
|
||||
}
|
||||
|
||||
L(`================ 彙總 ================`);
|
||||
for (const [k, v] of Object.entries(report.totals)) L(` ${k.padEnd(34)} = ${v}`);
|
||||
L(` 冪等設計：`);
|
||||
L(` entry → page_name（穩定鍵）+ metadata.content_hash（比對是否改動 → 未改 skip、改動 PATCH 重嵌）`);
|
||||
L(` triplet → source.uri + source.content_hash（graph 現役 per-source 冪等；同 hash 整包 no-op）`);
|
||||
L(` 分段 → 各段獨立 source.uri（#segNN）→ 各自獨立冪等，繞開 per-source content_hash 整包 skip`);
|
||||
|
||||
if (report.self_test) {
|
||||
L(`\n================ SELF-TEST：超大檔分段 ================`);
|
||||
const st = report.self_test;
|
||||
L(` ${st.note}`);
|
||||
L(` 若不分段（單 envelope）估算 subrequest = ${st.if_single_envelope_est_subrequests} → 會炸? ${st.would_crash_single ? 'YES（> ' + SUBREQ_CEILING + '）' : 'no'}`);
|
||||
L(` 分段後段數 = ${st.segmented_into}，每段：`);
|
||||
for (const s of st.per_segment) L(` - ${s.anchor}: triplets=${s.triplets} nodes=${s.nodes} est=${s.est_subrequests} ≤ceiling? ${s.under_ceiling ? 'YES' : 'NO ⚠️'}`);
|
||||
L(` 全部段壓在上限下？ ${st.all_segments_under_ceiling ? 'YES ✅' : 'NO ⚠️'}`);
|
||||
}
|
||||
L('');
|
||||
Reference in New Issue
Block a user