commit 4ca94c17ca044647ad72127b74e0de1ec5c7b11a Author: cloud-worker Date: Tue Jul 7 21:08:21 2026 +0000 T3: collector 骨架——watch→轉檔→git commit/push(design §4) Node 單檔服務(非 arcrun workflow,跑客戶端機器):chokidar watch 事件驅動、debounce 合併批次 commit、.md passthrough(非 md 格式誠實丟 NotImplemented 待 T4 Markitdown)、 刪檔→target repo 移除對應檔(deprecated 標記留給 ingest workflow,collector 只管檔案鏡像)。 已驗(雲端 sandbox scratch 環境,非真實客戶環境):node --check 全過;起 scratch watch dir + scratch git repo + bare remote,實測 add/change/delete 三種事件皆正確偵測、debounce 正確 合併成一次 commit、git push 真的送達 remote(git log 驗證)、非 md 格式正確跳過並警告。 端到端待本機/客戶環境驗(README.md 已列):真實 NAS/VM watch、真實 Gitea remote+憑證、 Gitea push webhook 真觸發 ingest、systemd 常駐穩定性。 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..ff18b2b --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +# collector — 收集端骨架(rag-wave1 T3) + +> design.md §4;跑在**客戶端機器**(NAS/VM,或導入者代管的 VPS),不是 arcrun workflow(arcrun 零件禁檔案系統,見 CLAUDE.md 紅線)。 +> ⚠️ **本骨架端到端(真實 NAS/VM + 真實 Gitea remote + systemd 常駐)需在本機/客戶環境驗,雲端 sandbox 沒有這些條件**——這裡只做得到:程式邏輯本身可跑、對本機臨時目錄的煙測(見下方「已驗證」)。 + +## 做什麼 + +``` +watch 知識資料夾 ──(新增/修改)──► transform(Markitdown,T4 補)──► 寫入 target repo 工作目錄 + ──(刪除) ──► target repo 移除對應檔 + │ + git add/commit/push + │ + Gitea push webhook(既有機制,非本骨架自建) + │ + arcrun ingest workflow(另案,如 km_wiki_ingest_drain 同款模式) +``` + +**關鍵設計判斷**:collector 的責任止於「commit + push」。design.md §4 說的「打 ingest webhook」= Gitea 自己的 push webhook 機制(同 `Leo/Arcrun` `registry/examples/km-wiki-ingest` 已驗證的模式:Gitea push webhook → arcrun workflow),**不是** collector 自己再打一支 HTTP webhook。collector 不需要知道 ingest 的內部細節(哪個 workflow、哪個 KBDB),它只管「客戶檔案的忠實鏡像」進 Gitea repo。 + +## 事件驅動、不輪詢 + +用 `chokidar`(Node)watch 檔案系統事件(inotify/FSEvents,非 polling),對齊「單一 repo、事件驅動」的鐵律。debounce 視窗(預設 3 秒)把同一批變更合併成一次 commit,避免每個檔案獨立 commit 洗歷史。 + +## 刪檔語意 + +檔案在來源資料夾被刪除 → collector 在 target repo 對應路徑也 `git rm` → commit → push。**deprecated 標記不是 collector 的責任**——ingest workflow 收到 Gitea push event 裡的 `removed` 檔案清單後,自己去 KBDB 把對應 entry 標 `status: deprecated`(append-only,不物理刪,見 km-wiki-ingest description.md 冪等設計)。collector 只管檔案鏡像忠實,不碰 KBDB。 + +## 轉檔(T4,依賴本骨架) + +`transform.js` 目前只處理 `.md`(直接複製,frontmatter 補 `source_path` 溯源欄位)。非 md 格式(docx/pptx/pdf)丟 `NotImplementedError` 並記警告日誌——**這是刻意的**:T4(Markitdown adapter)要接手這段,本骨架先把「watch→轉檔→寫入→commit」的骨架打通,轉檔本體留給 T4 填。 + +## 檔案 + +| 檔案 | 職責 | +|---|---| +| `index.js` | 進入點:watch+debounce+事件分派 | +| `transform.js` | 檔案 → md 轉換(T4 填 Markitdown 邏輯;現只認 `.md` passthrough) | +| `git-sync.js` | target repo 的 git add/commit/push 封裝 | +| `config.js` | 環境變數讀取(`WATCH_DIR`/`TARGET_REPO_DIR`/`DEBOUNCE_MS`) | + +## 設定(環境變數) + +| 變數 | 說明 | 預設 | +|---|---|---| +| `WATCH_DIR` | 客戶知識資料夾(來源) | 必填 | +| `TARGET_REPO_DIR` | 已 clone 好、有 push 權限的 Gitea repo 工作目錄(去向) | 必填 | +| `TARGET_SUBDIR` | 在 target repo 內落地的子目錄 | `collected/` | +| `DEBOUNCE_MS` | 合併變更的等待視窗 | `3000` | +| `GIT_AUTHOR_NAME` / `GIT_AUTHOR_EMAIL` | commit 署名 | `collector` / `collector@localhost` | + +## 部署(客戶端機器,設計稿——本輪未實際跑 systemd) + +```ini +# /etc/systemd/system/arcrun-rag-collector.service(範本,未部署未測) +[Unit] +Description=arcrun-rag collector +After=network.target + +[Service] +Environment=WATCH_DIR=/mnt/knowledge +Environment=TARGET_REPO_DIR=/opt/collector-repo +ExecStart=/usr/bin/node /opt/arcrun-rag/collector/index.js +Restart=always +User=collector + +[Install] +WantedBy=multi-user.target +``` + +`TARGET_REPO_DIR` 需事先 `git clone`+設好有 push 權限的 remote(credential 用該機器的 git credential helper 或 SSH key,不是本骨架管的事)。 + +## 已驗證(雲端 sandbox 能做到的部分) + +- `node --check` 語法檢查全過。 +- 本機臨時目錄煙測(非真實客戶環境):起一個 scratch git repo 當 target、一個 scratch 資料夾當 watch 來源,新增/修改/刪除 `.md` 檔案 → collector 正確偵測、寫入/移除、debounce 後產生一次 commit,`git log` 驗到 commit 內容與變更一致。**不含**:真實 Gitea remote push(需真實 token+repo)、Markitdown 轉檔(T4 未做)、systemd 常駐、NAS/VM 環境。 + +## 待本機/客戶環境驗(端到端) + +1. 真實客戶知識資料夾 watch(非 md 格式檔案觸發 T4 Markitdown)。 +2. 真實 Gitea remote push(含憑證管理)。 +3. push 後確認 Gitea push webhook 真觸發 ingest workflow(另案)。 +4. systemd 常駐穩定性(重開機自動起、崩潰自動重啟)。 diff --git a/config.js b/config.js new file mode 100644 index 0000000..f11fed7 --- /dev/null +++ b/config.js @@ -0,0 +1,20 @@ +function requireEnv(name) { + const v = process.env[name]; + if (!v) { + throw new Error(`collector: 缺少環境變數 ${name}(見 README.md 設定表)`); + } + return v; +} + +function loadConfig() { + return { + watchDir: requireEnv('WATCH_DIR'), + targetRepoDir: requireEnv('TARGET_REPO_DIR'), + targetSubdir: process.env.TARGET_SUBDIR || 'collected/', + debounceMs: parseInt(process.env.DEBOUNCE_MS || '3000', 10), + gitAuthorName: process.env.GIT_AUTHOR_NAME || 'collector', + gitAuthorEmail: process.env.GIT_AUTHOR_EMAIL || 'collector@localhost', + }; +} + +module.exports = { loadConfig, requireEnv }; diff --git a/git-sync.js b/git-sync.js new file mode 100644 index 0000000..b41fcd6 --- /dev/null +++ b/git-sync.js @@ -0,0 +1,41 @@ +const { execFileSync } = require('child_process'); + +/** + * target repo 的 git add/commit/push 封裝(design.md §4:git commit/push 觸發既有 + * Gitea push webhook → ingest workflow;本模組不打任何 ingest HTTP 端點)。 + */ +class GitSync { + constructor(config) { + this.repoDir = config.targetRepoDir; + this.authorName = config.gitAuthorName; + this.authorEmail = config.gitAuthorEmail; + } + + _git(args) { + return execFileSync('git', args, { + cwd: this.repoDir, + env: { + ...process.env, + GIT_AUTHOR_NAME: this.authorName, + GIT_AUTHOR_EMAIL: this.authorEmail, + GIT_COMMITTER_NAME: this.authorName, + GIT_COMMITTER_EMAIL: this.authorEmail, + }, + encoding: 'utf8', + }); + } + + /** 一批變更合併成一次 commit + push;無變更則不 commit(冪等,避免空 commit 洗歷史)。 */ + commitAndPush(summary) { + this._git(['add', '-A']); + const status = this._git(['status', '--porcelain']); + if (!status.trim()) { + return { committed: false }; + } + this._git(['commit', '-m', summary]); + this._git(['push']); + return { committed: true }; + } +} + +module.exports = { GitSync }; diff --git a/index.js b/index.js new file mode 100644 index 0000000..584de35 --- /dev/null +++ b/index.js @@ -0,0 +1,83 @@ +const fs = require('fs'); +const path = require('path'); +const chokidar = require('chokidar'); + +const { loadConfig } = require('./config'); +const { transformFile, NotImplementedError } = require('./transform'); +const { GitSync } = require('./git-sync'); + +function ensureDir(p) { + fs.mkdirSync(p, { recursive: true }); +} + +function outputPathFor(config, relOutputPath) { + return path.join(config.targetRepoDir, config.targetSubdir, relOutputPath); +} + +function handleAddOrChange(config, srcPath) { + try { + const { relOutputPath, content } = transformFile(srcPath, config.watchDir); + const outPath = outputPathFor(config, relOutputPath); + ensureDir(path.dirname(outPath)); + fs.writeFileSync(outPath, content, 'utf8'); + console.log(`[collector] 寫入 ${relOutputPath}`); + } catch (e) { + if (e instanceof NotImplementedError) { + console.warn(`[collector] 跳過(待 T4):${e.message}`); + } else { + console.error(`[collector] 轉檔失敗 ${srcPath}:${e.message}`); + } + } +} + +function handleUnlink(config, srcPath) { + const relSrc = path.relative(config.watchDir, srcPath); + const outPath = outputPathFor(config, relSrc); + if (fs.existsSync(outPath)) { + fs.unlinkSync(outPath); + console.log(`[collector] 移除 ${relSrc}(來源已刪除,deprecated 標記交給 ingest workflow)`); + } +} + +function startCollector(config, { git = new GitSync(config) } = {}) { + ensureDir(path.join(config.targetRepoDir, config.targetSubdir)); + + let pendingCount = 0; + let debounceTimer = null; + + const flush = () => { + if (pendingCount === 0) return; + const n = pendingCount; + pendingCount = 0; + const result = git.commitAndPush(`collector: 同步 ${n} 項變更`); + if (result.committed) { + console.log(`[collector] 已 commit+push(${n} 項變更)`); + } + }; + + const schedule = () => { + pendingCount += 1; + if (debounceTimer) clearTimeout(debounceTimer); + debounceTimer = setTimeout(flush, config.debounceMs); + }; + + const watcher = chokidar.watch(config.watchDir, { + ignoreInitial: false, // 首次啟動=首灌,掃過整個資料夾 + persistent: true, + }); + + watcher + .on('add', (p) => { handleAddOrChange(config, p); schedule(); }) + .on('change', (p) => { handleAddOrChange(config, p); schedule(); }) + .on('unlink', (p) => { handleUnlink(config, p); schedule(); }); + + console.log(`[collector] watch 中:${config.watchDir} → ${config.targetRepoDir}/${config.targetSubdir}`); + return watcher; +} + +if (require.main === module) { + const config = loadConfig(); + startCollector(config); +} + +module.exports = { startCollector }; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..8f1ea3a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,201 @@ +{ + "name": "arcrun-rag-collector", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "arcrun-rag-collector", + "version": "0.1.0", + "dependencies": { + "chokidar": "^3.6.0" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..2b5cd61 --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "arcrun-rag-collector", + "version": "0.1.0", + "description": "收集端骨架(rag-wave1 T3):watch 客戶知識資料夾 → 轉檔 → git commit/push", + "main": "index.js", + "private": true, + "dependencies": { + "chokidar": "^3.6.0" + } +} diff --git a/transform.js b/transform.js new file mode 100644 index 0000000..4b67246 --- /dev/null +++ b/transform.js @@ -0,0 +1,44 @@ +const fs = require('fs'); +const path = require('path'); + +class NotImplementedError extends Error {} + +const MARKDOWN_EXT = new Set(['.md', '.markdown']); + +/** + * 檔案 → md 轉換(T3 骨架只做 .md passthrough;T4 補 Markitdown 邏輯)。 + * srcPath: 來源檔案絕對路徑(客戶知識資料夾內) + * 回傳:{ relOutputPath, content }(相對 target repo 子目錄的輸出路徑 + md 內容) + */ +function transformFile(srcPath, watchDir) { + const ext = path.extname(srcPath).toLowerCase(); + const relSrc = path.relative(watchDir, srcPath); + + if (MARKDOWN_EXT.has(ext)) { + const raw = fs.readFileSync(srcPath, 'utf8'); + const content = stampSourcePath(raw, relSrc); + const relOutputPath = relSrc; + return { relOutputPath, content }; + } + + // 非 md(docx/pptx/pdf...)— T4 待補 Markitdown 轉檔,這裡先誠實丟未實作,不假裝轉好。 + throw new NotImplementedError( + `${relSrc}:非 .md 格式轉檔待 T4(Markitdown adapter)補上,本骨架先跳過`, + ); +} + +/** md frontmatter 補 source_path(溯源用,design.md §4:「md frontmatter 記原檔路徑」)。 */ +function stampSourcePath(raw, relSrc) { + const stamp = `source_path: "${relSrc}"`; + if (raw.startsWith('---\n')) { + const end = raw.indexOf('\n---', 4); + if (end !== -1) { + const fm = raw.slice(4, end); + if (fm.includes('source_path:')) return raw; // 已有就不重複塞 + return `---\n${stamp}\n${fm}\n---${raw.slice(end + 4)}`; + } + } + return `---\n${stamp}\n---\n\n${raw}`; +} + +module.exports = { transformFile, NotImplementedError };