From 058d19b743bb74c0248b830857ca57c0d886a798 Mon Sep 17 00:00:00 2001 From: richblack Date: Fri, 24 Jul 2026 12:23:52 +0800 Subject: [PATCH] =?UTF-8?q?feat(daemon-beta=20t5/t6):=20rag=5Fingest=5Fcar?= =?UTF-8?q?d=20=E6=94=B6=E5=8D=A1=20workflow=EF=BC=88=E9=9B=B6LLM=E9=9B=B6?= =?UTF-8?q?credential=EF=BC=89=EF=BC=8Bdirect=20=E8=BF=B4=E5=9C=88?= =?UTF-8?q?=E6=8E=A5=E8=90=83=E5=8F=96=E5=99=A8=E2=80=94=E2=80=94=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E8=90=83=E5=8D=A1=E5=8F=AA=E4=B8=8A=E5=8D=A1=E3=80=81?= =?UTF-8?q?=E5=8E=9F=E6=96=87=E4=B8=8D=E5=87=BA=E6=A9=9F=EF=BC=9Be2e=20?= =?UTF-8?q?=E6=B8=AC=E8=A9=A6=E9=A9=97=E3=80=8C=E5=8E=9F=E6=96=87=E4=B8=8D?= =?UTF-8?q?=E6=B4=A9=E4=B8=8A=E9=9B=B2=E3=80=8D=EF=BC=8B=E5=A4=B1=E6=95=97?= =?UTF-8?q?=E9=87=8D=E8=A9=A6=E8=AA=9E=E6=84=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- direct.go | 67 +++++++++++++++++++++++-- direct_extract_test.go | 110 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 direct_extract_test.go diff --git a/direct.go b/direct.go index 2e1b2f2..c619d16 100644 --- a/direct.go +++ b/direct.go @@ -42,10 +42,16 @@ type DirectConfig struct { Namespace string `json:"namespace"` // 租戶 namespace（必填），如 demo APIKey string `json:"api_key"` // X-Arcrun-API-Key（空＝沿用 namespace，demo 慣例） Library string `json:"library"` // 藏書地圖歸庫鍵（空＝kb） - IngestWF string `json:"ingest_workflow"` // 直送萃取 workflow 名（空＝rag_ingest_direct） + IngestWF string `json:"ingest_workflow"` // 直送萃取 workflow 名（空＝rag_ingest_direct；extractor 模式不用） RemovedWF string `json:"removed_workflow"` // 下架 workflow 名（空＝rag_takedown_direct；吃 {page_name,path}） - PollSec int `json:"poll_interval_sec"` // 輪詢間隔秒（空/0＝5） - MaxRemoved float64 `json:"max_removed_ratio"` // 大量刪除防呆門檻（空/0＝0.4） + // —— 四步定稿（daemon-beta t3/t4/t6）：本地萃卡模式 —— + Extractor string `json:"extractor,omitempty"` // "claude"｜"gemma"；空＝舊制（內容直送雲端萃） + ClaudeBin string `json:"claude_bin,omitempty"` // claude 執行檔（空＝PATH 找 claude） + GeminiAPIKey string `json:"gemini_api_key,omitempty"` // gemma 路的用戶 key + LLMModel string `json:"llm_model,omitempty"` // gemma 路模型（空＝gemma-4-31b-it） + CardIngestWF string `json:"card_ingest_workflow,omitempty"` // 收卡 workflow（空＝rag_ingest_card） + PollSec int `json:"poll_interval_sec"` // 輪詢間隔秒（空/0＝5） + MaxRemoved float64 `json:"max_removed_ratio"` // 大量刪除防呆門檻（空/0＝0.4） } // LoadDirectConfig 讀設定檔並補預設值 + 基本驗證。 @@ -86,6 +92,15 @@ func LoadDirectConfig(path string) (*DirectConfig, error) { if c.RemovedWF == "" { c.RemovedWF = "rag_takedown_direct" } + if c.CardIngestWF == "" { + c.CardIngestWF = "rag_ingest_card" + } + if c.LLMModel == "" { + c.LLMModel = defaultLLMModel + } + if c.Extractor != "" && c.Extractor != "claude" && c.Extractor != "gemma" { + return nil, fmt.Errorf("extractor 只能是 claude / gemma（或留空走舊制），got %q", c.Extractor) + } if c.PollSec <= 0 { c.PollSec = 5 } @@ -253,6 +268,52 @@ func runDirectOnceRoot(cfg *DirectConfig, root string, dryRun bool) ([]DirectRes results = append(results, res) continue } + if cfg.Extractor != "" { + // 四步定稿：本地萃卡 → 每張卡 POST rag_ingest_card（原文不出機） + var cards []string + var xerr error + switch cfg.Extractor { + case "claude": + cards, xerr = ExtractWithClaude(cfg.ClaudeBin, absRoot, ev.Path) + case "gemma": + cards, xerr = ExtractWithGemma(cfg.GeminiAPIKey, cfg.LLMModel, absRoot, ev.Path) + } + if xerr != nil { + res.Status, res.Error = "failed", "本地萃取失敗："+xerr.Error() + results = append(results, res) + exit = 1 + continue + } + ok := true + for _, cardRel := range cards { + cardData, cerr := os.ReadFile(filepath.Join(absRoot, filepath.FromSlash(cardRel))) + if cerr != nil { + res.Status, res.Error = "failed", "讀卡片失敗："+cerr.Error() + ok = false + break + } + status, _, perr := cfg.postJSON(cfg.triggerURL(cfg.CardIngestWF), map[string]any{ + "page_name": pageNameOf(cardRel), + "path": cardRel, + "card_content": string(cardData), + "library": cfg.Library, + }) + res.HTTPStatus = status + if perr != nil { + res.Status, res.Error = "failed", perr.Error() + ok = false + break + } + } + if ok { + res.Status = "ingested" + m.MarkIngested(ev.Path, ev.SourceHash, now) + } else { + exit = 1 + } + results = append(results, res) + continue + } status, _, perr := cfg.postJSON(cfg.triggerURL(cfg.IngestWF), map[string]any{ "page_name": pageNameOf(ev.Path), "path": ev.Path, diff --git a/direct_extract_test.go b/direct_extract_test.go new file mode 100644 index 0000000..6f4606a --- /dev/null +++ b/direct_extract_test.go @@ -0,0 +1,110 @@ +// direct_extract_test.go — task 6：extractor 模式端到端（本地萃卡→POST rag_ingest_card）。 +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// 完整鏈（claude stub 版）：丟原稿 → 萃卡落地本地 → 只有「卡片」被 POST 到 rag_ingest_card +// → 原文從未離開本機 → manifest 標 ingested（下一輪不重送）。 +func TestDirectExtractorModeE2E(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "報銷規則.md"), []byte("# 原稿機密內容 XYZZY"), 0o644); err != nil { + t.Fatal(err) + } + + // 假 cypher：收 rag_ingest_card、驗 payload、記帳 + var posted []map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/webhooks/named/demo/rag_ingest_card/trigger") { + t.Errorf("打錯端點：%s", r.URL.Path) + } + body, _ := io.ReadAll(r.Body) + var m map[string]any + _ = json.Unmarshal(body, &m) + posted = append(posted, m) + _ = json.NewEncoder(w).Encode(map[string]any{"success": true}) + })) + defer srv.Close() + + // stub claude：把原稿萃成卡（模擬 /wiki-capture 行為） + stubDir := t.TempDir() + stub := filepath.Join(stubDir, "claude") + script := "#!/bin/sh\nmkdir -p system-dev/wiki/cards\nprintf '# 報銷規則\\n## 一句話定義\\n測試卡\\n## 關聯\\n- 報銷規則 >> 屬於 >> 財務\\n' > 'system-dev/wiki/cards/報銷規則.md'\n" + if err := os.WriteFile(stub, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + + cfg := &DirectConfig{ + WatchFolders: []string{root}, + Manifest: filepath.Join(t.TempDir(), "m.json"), + CypherURL: srv.URL, Namespace: "demo", APIKey: "demo", + Library: "kb", Extractor: "claude", ClaudeBin: stub, + CardIngestWF: "rag_ingest_card", MaxRemoved: DefaultMaxRemovedRatio, + } + results, exit, _ := RunDirectOnce(cfg, false) + if exit != 0 { + t.Fatalf("exit=%d results=%+v", exit, results) + } + if len(results) != 1 || results[0].Status != "ingested" { + t.Fatalf("results=%+v", results) + } + // 卡片落地本地（用戶看得到自己的 wiki） + if _, err := os.Stat(filepath.Join(root, "system-dev", "wiki", "cards", "報銷規則.md")); err != nil { + t.Fatalf("卡片未落地：%v", err) + } + // 上雲的是卡片、不是原文 + if len(posted) != 1 { + t.Fatalf("應恰好 POST 一張卡，got %d", len(posted)) + } + cc, _ := posted[0]["card_content"].(string) + if !strings.Contains(cc, "## 一句話定義") { + t.Fatalf("card_content 不是卡片：%.80s", cc) + } + if strings.Contains(cc, "XYZZY") { + t.Fatal("原文內容洩上雲＝違反四步定稿邊界") + } + if p, _ := posted[0]["path"].(string); p != "system-dev/wiki/cards/報銷規則.md" { + t.Fatalf("path=%q", p) + } + // 第二輪：原稿沒變 → 不重萃不重送 + results2, exit2, _ := RunDirectOnce(cfg, false) + if exit2 != 0 || len(results2) != 0 || len(posted) != 1 { + t.Fatalf("第二輪應零事件：results=%+v posted=%d", results2, len(posted)) + } +} + +// 萃取失敗＝該檔標 failed、exit=1、manifest 不標（下輪重試），其他檔不受影響。 +func TestDirectExtractorFailKeepsRetry(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a.md"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + stubDir := t.TempDir() + stub := filepath.Join(stubDir, "claude") + if err := os.WriteFile(stub, []byte("#!/bin/sh\nexit 3\n"), 0o755); err != nil { + t.Fatal(err) + } + cfg := &DirectConfig{ + WatchFolders: []string{root}, + Manifest: filepath.Join(t.TempDir(), "m.json"), + CypherURL: "https://x.example", Namespace: "demo", APIKey: "demo", + Extractor: "claude", ClaudeBin: stub, MaxRemoved: DefaultMaxRemovedRatio, + } + results, exit, _ := RunDirectOnce(cfg, false) + if exit != 1 || len(results) != 1 || results[0].Status != "failed" { + t.Fatalf("exit=%d results=%+v", exit, results) + } + // 再跑一輪：仍是同一個事件（manifest 沒標 ingested＝會重試） + results2, _, _ := RunDirectOnce(cfg, false) + if len(results2) != 1 { + t.Fatalf("失敗檔應重試：%+v", results2) + } +}