feat(daemon-beta t1): 多資料夾 config——watch_folders[]+單數相容、manifest 每根一份、多根掃描標 Root、托盤資料層相容;rag-wave1 closed→daemon-beta 實作卷 active
- direct.go: Folders() 正規化(去重保序)/manifestPathFor(單根沿用不丟狀態、多根 sha 尾碼)/RunDirectOnce 多根彙總 - 新測試 4 支全綠(單數相容/多根去重/缺欄驗證/多根 dry-run 標 Root);既有 7 支不動全綠 - tray: config 加 watch_folders 欄(防存檔洗掉)+addWatchFolder 資料層;勾選 UI=task 7
This commit is contained in:
@@ -21,6 +21,8 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -33,16 +35,17 @@ import (
|
||||
|
||||
// DirectConfig 是 direct 模式的設定檔(JSON)。設定只走檔案/環境,不落 code。
|
||||
type DirectConfig struct {
|
||||
WatchFolder string `json:"watch_folder"` // 監看的知識資料夾(必填)
|
||||
Manifest string `json:"manifest"` // manifest JSON 路徑(必填;不存在會建新)
|
||||
CypherURL string `json:"cypher_url"` // 實例 cypher base(必填),如 https://arcrun-cypher-executor.<acct>.workers.dev
|
||||
Namespace string `json:"namespace"` // 租戶 namespace(必填),如 demo
|
||||
APIKey string `json:"api_key"` // X-Arcrun-API-Key(空=沿用 namespace,demo 慣例)
|
||||
Library string `json:"library"` // 藏書地圖歸庫鍵(空=kb)
|
||||
IngestWF string `json:"ingest_workflow"` // 直送萃取 workflow 名(空=rag_ingest_direct)
|
||||
RemovedWF string `json:"removed_workflow"` // 下架 workflow 名(空=rag_takedown_direct;吃 {page_name,path})
|
||||
PollSec int `json:"poll_interval_sec"` // 輪詢間隔秒(空/0=5)
|
||||
MaxRemoved float64 `json:"max_removed_ratio"` // 大量刪除防呆門檻(空/0=0.4)
|
||||
WatchFolder string `json:"watch_folder,omitempty"` // 監看的知識資料夾(單數舊制;與 watch_folders 至少填一)
|
||||
WatchFolders []string `json:"watch_folders,omitempty"` // 監看的知識資料夾清單(daemon-beta task 1 多資料夾)
|
||||
Manifest string `json:"manifest"` // manifest JSON 路徑(必填;多資料夾時為基底名,每根一份帶尾碼)
|
||||
CypherURL string `json:"cypher_url"` // 實例 cypher base(必填),如 https://arcrun-cypher-executor.<acct>.workers.dev
|
||||
Namespace string `json:"namespace"` // 租戶 namespace(必填),如 demo
|
||||
APIKey string `json:"api_key"` // X-Arcrun-API-Key(空=沿用 namespace,demo 慣例)
|
||||
Library string `json:"library"` // 藏書地圖歸庫鍵(空=kb)
|
||||
IngestWF string `json:"ingest_workflow"` // 直送萃取 workflow 名(空=rag_ingest_direct)
|
||||
RemovedWF string `json:"removed_workflow"` // 下架 workflow 名(空=rag_takedown_direct;吃 {page_name,path})
|
||||
PollSec int `json:"poll_interval_sec"` // 輪詢間隔秒(空/0=5)
|
||||
MaxRemoved float64 `json:"max_removed_ratio"` // 大量刪除防呆門檻(空/0=0.4)
|
||||
}
|
||||
|
||||
// LoadDirectConfig 讀設定檔並補預設值 + 基本驗證。
|
||||
@@ -56,8 +59,8 @@ func LoadDirectConfig(path string) (*DirectConfig, error) {
|
||||
return nil, fmt.Errorf("config JSON 解析失敗:%w", err)
|
||||
}
|
||||
var missing []string
|
||||
if c.WatchFolder == "" {
|
||||
missing = append(missing, "watch_folder")
|
||||
if c.WatchFolder == "" && len(c.WatchFolders) == 0 {
|
||||
missing = append(missing, "watch_folder(或 watch_folders)")
|
||||
}
|
||||
if c.Manifest == "" {
|
||||
missing = append(missing, "manifest")
|
||||
@@ -93,6 +96,36 @@ func LoadDirectConfig(path string) (*DirectConfig, error) {
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// Folders 回傳監看根清單(正規化:單數舊制併入、去重、保序)。
|
||||
func (c *DirectConfig) Folders() []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
add := func(p string) {
|
||||
if p == "" || seen[p] {
|
||||
return
|
||||
}
|
||||
seen[p] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
add(c.WatchFolder)
|
||||
for _, p := range c.WatchFolders {
|
||||
add(p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// manifestPathFor 回傳某根的 manifest 路徑。單根=沿用 cfg.Manifest(升級不丟既有狀態);
|
||||
// 多根=每根一份,基底名加 root 絕對路徑的 sha256 前 8 碼尾碼(路徑穩定=尾碼穩定)。
|
||||
func (c *DirectConfig) manifestPathFor(absRoot string) string {
|
||||
folders := c.Folders()
|
||||
if len(folders) <= 1 {
|
||||
return c.Manifest
|
||||
}
|
||||
sum := sha256.Sum256([]byte(absRoot))
|
||||
ext := filepath.Ext(c.Manifest)
|
||||
return strings.TrimSuffix(c.Manifest, ext) + "-" + hex.EncodeToString(sum[:4]) + ext
|
||||
}
|
||||
|
||||
// directHTTP 是 direct 模式共用的 HTTP client(萃取 workflow 可能同步跑 LLM,放寬 timeout)。
|
||||
var directHTTP = &http.Client{Timeout: 300 * time.Second}
|
||||
|
||||
@@ -136,6 +169,7 @@ func pageNameOf(relPath string) string {
|
||||
|
||||
// DirectResult 是單一事件的直送結果(隨每輪 log 輸出)。
|
||||
type DirectResult struct {
|
||||
Root string `json:"root,omitempty"` // 多資料夾時標明事件屬於哪個根
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Status string `json:"status"` // ingested | removed | planned | failed | skipped
|
||||
@@ -143,17 +177,41 @@ type DirectResult struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// RunDirectOnce 掃一輪、直送 added/modified/renamed、下架 removed,並在 2xx 後回寫 manifest。
|
||||
// 回傳本輪結果清單與退出碼建議(有失敗=1)。
|
||||
// RunDirectOnce 對每個監看根掃一輪並彙總結果(daemon-beta task 1 多資料夾)。
|
||||
// 單根行為與舊制完全相同(含 manifest 路徑)。回傳彙總結果與退出碼建議(任一根失敗=1)。
|
||||
func RunDirectOnce(cfg *DirectConfig, dryRun bool) ([]DirectResult, int, *TriggerPayload) {
|
||||
results := []DirectResult{}
|
||||
exit := 0
|
||||
var lastPayload *TriggerPayload
|
||||
multi := len(cfg.Folders()) > 1
|
||||
for _, root := range cfg.Folders() {
|
||||
r, e, p := runDirectOnceRoot(cfg, root, dryRun)
|
||||
if multi {
|
||||
for i := range r {
|
||||
r[i].Root = root
|
||||
}
|
||||
}
|
||||
results = append(results, r...)
|
||||
if e != 0 {
|
||||
exit = e
|
||||
}
|
||||
if p != nil {
|
||||
lastPayload = p
|
||||
}
|
||||
}
|
||||
return results, exit, lastPayload
|
||||
}
|
||||
|
||||
absRoot, err := filepath.Abs(cfg.WatchFolder)
|
||||
// runDirectOnceRoot 對單一根掃一輪、直送 added/modified/renamed、下架 removed,2xx 後回寫該根 manifest。
|
||||
func runDirectOnceRoot(cfg *DirectConfig, root string, dryRun bool) ([]DirectResult, int, *TriggerPayload) {
|
||||
results := []DirectResult{}
|
||||
exit := 0
|
||||
|
||||
absRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return append(results, DirectResult{Status: "failed", Error: err.Error()}), 1, nil
|
||||
}
|
||||
absManifest, err := filepath.Abs(cfg.Manifest)
|
||||
absManifest, err := filepath.Abs(cfg.manifestPathFor(absRoot))
|
||||
if err != nil {
|
||||
return append(results, DirectResult{Status: "failed", Error: err.Error()}), 1, nil
|
||||
}
|
||||
@@ -267,9 +325,9 @@ func runDirect(args []string) int {
|
||||
results, exit, _ := RunDirectOnce(cfg, *dryRun)
|
||||
out, _ := json.MarshalIndent(struct {
|
||||
At string `json:"at"`
|
||||
Folder string `json:"folder"`
|
||||
Folders []string `json:"folders"`
|
||||
Results []DirectResult `json:"results"`
|
||||
}{time.Now().Format(time.RFC3339), cfg.WatchFolder, results}, "", " ")
|
||||
}{time.Now().Format(time.RFC3339), cfg.Folders(), results}, "", " ")
|
||||
fmt.Println(string(out))
|
||||
return exit
|
||||
}
|
||||
@@ -279,7 +337,7 @@ func runDirect(args []string) int {
|
||||
}
|
||||
// 常駐輪詢:純 stdlib ticker,跨平台。首輪立即跑。
|
||||
fmt.Fprintf(os.Stderr, "collector direct daemon 啟動:監看 %s → %s(每 %ds 掃一輪)\n",
|
||||
cfg.WatchFolder, cfg.triggerURL(cfg.IngestWF), cfg.PollSec)
|
||||
strings.Join(cfg.Folders(), "、"), cfg.triggerURL(cfg.IngestWF), cfg.PollSec)
|
||||
runOne()
|
||||
ticker := time.NewTicker(time.Duration(cfg.PollSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
Reference in New Issue
Block a user