From 682225133442e15af5fbda63a68706843a7c8db0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 03:53:40 +0000 Subject: [PATCH 1/4] Harden Organization mutual growth between FreeOS and openXYOS Add a token-gated /api/freeos ingest-export bridge so a healthy sidecar accepts FreeOS packs without a login wall. Loop and pack wait on the control plane only when reachable or bundled, persist landed surfaces, and localize timeline yes/no. Organization gets assemble/pack/loop/produce. --- CHANGELOG.md | 7 + README.md | 2 +- dashboard/src/api/modules/orgModule.ts | 22 ++ dashboard/src/locales/en.json | 37 ++- dashboard/src/locales/zh.json | 37 ++- .../Organization/Organization.module.less | 2 +- dashboard/src/pages/Organization/index.tsx | 89 +++++- desktop/src/process.go | 6 + desktop/src/process_sidecar.go | 18 +- desktop/src/process_sidecar_test.go | 8 +- docs/asset-loop.md | 2 +- .../openxyos/backend/routes/freeos-bridge.ts | 269 ++++++++++++++++++ modules/openxyos/backend/server.ts | 4 +- src/octop/api/routers/local_models.py | 32 ++- src/octop/api/routers/org_module.py | 34 ++- src/octop/i18n/en.json | 19 +- src/octop/i18n/zh.json | 19 +- src/octop/modules/org_os/apply/apply.py | 252 +++++++++------- src/octop/modules/org_os/apply/client.py | 104 +++++-- src/octop/modules/org_os/assets/importer.py | 56 ++++ src/octop/modules/org_os/assets/pack.py | 2 +- src/octop/modules/org_os/empower.py | 84 +++++- src/octop/modules/org_os/loop/run.py | 32 ++- src/octop/modules/org_os/notes.py | 91 ++++++ src/octop/modules/org_os/sidecar_launch.py | 65 ++--- src/octop/modules/org_os/sidecar_secrets.py | 71 +++++ tests/e2e/test_org_growth_loop.py | 2 +- tests/support/openxyos_harness.py | 77 +++++ tests/unit/test_asset_loop.py | 25 +- tests/unit/test_openxyos_apply.py | 34 ++- tests/unit/test_org_loop.py | 8 +- tests/unit/test_org_module.py | 11 + tests/unit/test_org_notes.py | 4 + tests/unit/test_sidecar_launch.py | 17 ++ 34 files changed, 1321 insertions(+), 221 deletions(-) create mode 100644 modules/openxyos/backend/routes/freeos-bridge.ts create mode 100644 src/octop/modules/org_os/sidecar_secrets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 34098df2..54cd2301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ ### 新增 +- 组织页双向互生长：FreeOS 资产经 `/api/freeos/ingest`（共享写入令牌，无需边车登录墙）落到 openXYOS 员工/人才/插件/技能表；控制面导出再装配回 FreeOS。循环在边车健康时不再把「已推送到控制面」写成 false。 +- 组织页可从本地语料或 ima / 云知识库指针一键生产同事；本地模型一键下载后会注册到 Ollama 供应商。 + +### 变更 + +- 组织循环时间线的是/否改为中英文本地化，不再混用 `true`/`false`。 +- 边车启动会写入并传递 `FREEOS_INGEST_TOKEN`；Python 启动器同时识别 `%LOCALAPPDATA%\FreeOS\openxyos`、`$INSTDIR\openxyos` 与 portable `org-sidecar`。 - Windows 安装包写入预构建 openXYOS 运行包（`$INSTDIR\openxyos` + `openxyos-runtime.zip`），首次启动再落到 `%LOCALAPPDATA%\FreeOS\openxyos` / `~/.freeos/openxyos`，自动拉起 `http://127.0.0.1:3780`，组织页不必再点「启动边车」。 - 项目创建用系统文件夹选择器（Wails 原生对话框，任意盘符）；项目/任务/对话区默认展示实时对话列表，并提供建群/群聊入口。 - Organization OS 首页双循环：FreeOS 数据面与 openXYOS 控制面的实时状态、一键装配员工 / 打包回流 / 运行 org loop，以及边车离线时的启动恢复。 diff --git a/README.md b/README.md index ab68b7be..7861e8fd 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ uv run pytest tests/e2e/test_org_growth_loop.py tests/unit/test_org_loop.py test | When | What happens | |---|---| | `OPENXYOS_BASE_URL` unset / sidecar down | Uses `tests/fixtures/org-loop/` and writes `{FREEOS_HOME}/openxyos-mirror/` | -| `OPENXYOS_BASE_URL` (or `FREEOS_ORG_SIDECAR_URL`) set | Also POST/PUT `/api/employees`, `/api/talent`, `/api/plugins`, `/api/module-settings` | +| `OPENXYOS_BASE_URL` (or `FREEOS_ORG_SIDECAR_URL`) set | Also POST `/api/freeos/ingest` (employees, talent, plugins, skills) when the sidecar is healthy | High-risk tools (`outbound`, `delete`, `pay`, `prod`) are default-denied in the host tool path (`OrgGovernanceMiddleware` + `xyos-governance-mcp`). `execute` stays false until `freeos org governance approve ` and the agent re-checks with the same args. diff --git a/dashboard/src/api/modules/orgModule.ts b/dashboard/src/api/modules/orgModule.ts index a3794be5..eede7497 100644 --- a/dashboard/src/api/modules/orgModule.ts +++ b/dashboard/src/api/modules/orgModule.ts @@ -83,6 +83,15 @@ export interface OrgAssembleResult { notes: string[]; } +export interface OrgProduceResult { + slug: string; + workspace: string; + distill_path: string; + copied: number; + spawned: Record; + notes: string[]; +} + export interface OrgPackResult { pack: { directory: string; @@ -98,6 +107,7 @@ export interface OrgPackResult { remote_applied: boolean; mirrored: boolean; notes: string[]; + landed?: Record; }; } @@ -117,6 +127,7 @@ export interface OrgLoopProof { governance: Record; notes: string[]; updated_at?: string; + landed?: Record; } export const orgModuleApi = { @@ -133,6 +144,17 @@ export const orgModuleApi = { request("/org-module/sidecar/start", { method: "POST" }), assemble: () => request("/org-module/assemble", { method: "POST" }), + produce: (body: { + kb_id?: string; + distill_path?: string; + ima_url?: string; + name?: string; + }) => + request("/org-module/produce", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), pack: () => request("/org-module/pack", { method: "POST" }), runLoop: () => request("/org-module/loop/run", { diff --git a/dashboard/src/locales/en.json b/dashboard/src/locales/en.json index b4cbde42..c98f61f6 100644 --- a/dashboard/src/locales/en.json +++ b/dashboard/src/locales/en.json @@ -30,19 +30,34 @@ "metricGovernance": "Governance engine", "metricHealth": "Control-plane health", "metricTenant": "Tenant", - "assembleTitle": "Assemble employees from a blueprint", - "assembleBody": "Compile an openXYOS blueprint (or the bundled fixture) into FreeOS digital employees and register them for chat.", - "assembleAction": "Assemble into FreeOS", - "assembleDone": "Assembled {{count}} digital employee(s)", - "packTitle": "Pack assets back to openXYOS", - "packBody": "Publish FreeOS skills, plugins, and MCP as an asset pack, mirror it, and push to the control plane when live.", - "packAction": "Pack and apply", + "assembleTitle": "Bring openXYOS into FreeOS", + "assembleBody": "Import AI employees, skills, and plugins from the live control plane (or bundled blueprints) and register them for chat.", + "assembleAction": "Import into FreeOS", + "assembleDone": "Imported {{count}} digital employee(s) into FreeOS", + "packTitle": "Send FreeOS assets to openXYOS", + "packBody": "Pack FreeOS agents, skills, plugins, and MCP, then land them on the control plane. You will see what was written.", + "packAction": "Push to openXYOS", "packDone": "Asset pack published", - "packRemoteYes": "Attempted a live write to openXYOS", + "packRemoteYes": "Landed on the live openXYOS control plane", "packRemoteNo": "Sidecar offline: wrote the local mirror only", - "loopTitle": "Run the organization loop", - "loopBody": "Blueprint → employees → pack → import → governance block, with a real timeline below.", - "loopAction": "Run org loop", + "loopTitle": "Run the growth loop", + "loopBody": "Produce on FreeOS, assemble into openXYOS, feed back, and leave durable proof below.", + "loopAction": "Run growth loop", + "produceTitle": "Produce a colleague from a corpus", + "produceBody": "Distill a local folder or attach an ima / cloud knowledge URL, then compile a usable FreeOS expert.", + "produceAction": "Produce from corpus", + "produceDone": "Produced {{name}}", + "produceKb": "Knowledge base id (optional)", + "produceIma": "ima / cloud knowledge URL (optional)", + "produceName": "Colleague name", + "landedTitle": "What landed", + "landedEmpty": "Nothing landed on the control plane yet.", + "yes": "Yes", + "no": "No", + "progressAssemble": "Importing from openXYOS…", + "progressPack": "Packing and pushing to openXYOS…", + "progressLoop": "Running the growth loop…", + "progressProduce": "Producing a colleague from the corpus…", "loopOk": "Organization loop completed", "loopPartial": "Loop ran; some steps did not finish (see timeline)", "timelineTitle": "Loop timeline", diff --git a/dashboard/src/locales/zh.json b/dashboard/src/locales/zh.json index 60e0bf51..326a1567 100644 --- a/dashboard/src/locales/zh.json +++ b/dashboard/src/locales/zh.json @@ -30,19 +30,34 @@ "metricGovernance": "治理引擎", "metricHealth": "控制面健康", "metricTenant": "租户", - "assembleTitle": "从蓝图装配员工", - "assembleBody": "把 openXYOS 蓝图（或本地示例蓝图）编译成 FreeOS 数字员工，并注册到对话列表。", - "assembleAction": "装配到 FreeOS", - "assembleDone": "已装配 {{count}} 名数字员工", - "packTitle": "打包回流 openXYOS", - "packBody": "把 FreeOS 技能、插件与 MCP 打成资产包，镜像并尽量推回控制面。", - "packAction": "打包回流", + "assembleTitle": "把 openXYOS 装进 FreeOS", + "assembleBody": "从在线控制面导入 AI 员工、技能和插件（或使用捆绑蓝图），并注册到对话。", + "assembleAction": "导入到 FreeOS", + "assembleDone": "已导入 {{count}} 名数字员工到 FreeOS", + "packTitle": "把 FreeOS 资产推到 openXYOS", + "packBody": "打包 FreeOS 智能体、技能、插件与 MCP，写入控制面，并显示落地结果。", + "packAction": "推送到 openXYOS", "packDone": "资产包已发布", - "packRemoteYes": "已尝试写入 live openXYOS", + "packRemoteYes": "已写入在线 openXYOS 控制面", "packRemoteNo": "边车离线：仅写入本地镜像", - "loopTitle": "运行组织循环", - "loopBody": "一键跑通蓝图 → 员工 → 打包 → 回流 → 治理拦截，并在下方展示真实时间线。", - "loopAction": "运行 org loop", + "loopTitle": "运行自增长循环", + "loopBody": "在 FreeOS 生产、装配进 openXYOS、再回流，并在下方留下可核对的证据。", + "loopAction": "运行自增长循环", + "produceTitle": "从语料生产同事", + "produceBody": "蒸馏本地文件夹，或挂接 ima / 云知识库地址，编译成可用的 FreeOS 专家。", + "produceAction": "从语料生产", + "produceDone": "已生产 {{name}}", + "produceKb": "知识库编号（可选）", + "produceIma": "ima / 云知识库地址（可选）", + "produceName": "同事名称", + "landedTitle": "落地结果", + "landedEmpty": "控制面还没有落地记录。", + "yes": "是", + "no": "否", + "progressAssemble": "正在从 openXYOS 导入…", + "progressPack": "正在打包并推送到 openXYOS…", + "progressLoop": "正在运行自增长循环…", + "progressProduce": "正在从语料生产同事…", "loopOk": "组织循环已跑通", "loopPartial": "循环已执行，部分步骤未完成（见时间线）", "timelineTitle": "循环时间线", diff --git a/dashboard/src/pages/Organization/Organization.module.less b/dashboard/src/pages/Organization/Organization.module.less index 05137182..075bfd8d 100644 --- a/dashboard/src/pages/Organization/Organization.module.less +++ b/dashboard/src/pages/Organization/Organization.module.less @@ -259,7 +259,7 @@ .actions { display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); + grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; } diff --git a/dashboard/src/pages/Organization/index.tsx b/dashboard/src/pages/Organization/index.tsx index 633e54e3..1b6e33b4 100644 --- a/dashboard/src/pages/Organization/index.tsx +++ b/dashboard/src/pages/Organization/index.tsx @@ -21,13 +21,14 @@ import { type OrgLoopProof, type OrgOverview, type OrgPackResult, + type OrgProduceResult, } from "../../api/modules/orgModule"; import { formatServerIsoDateTime } from "../../utils/formatMessageTime"; import { useServerTimezone } from "../../hooks/useServerTimezone"; import { message } from "../../utils/antdMessage"; import styles from "./Organization.module.less"; -type ActionKey = "assemble" | "pack" | "loop" | "sidecar" | null; +type ActionKey = "assemble" | "pack" | "loop" | "sidecar" | "produce" | null; function metric(value: number | undefined): string { return typeof value === "number" ? String(value) : "—"; @@ -50,6 +51,9 @@ export default function OrganizationPage() { ); const [sourceDest, setSourceDest] = useState(""); const [downloading, setDownloading] = useState(false); + const [produceName, setProduceName] = useState(""); + const [produceIma, setProduceIma] = useState(""); + const [landed, setLanded] = useState | null>(null); const autoStartRef = useRef(false); const iframeRef = useRef(null); @@ -60,6 +64,10 @@ export default function OrganizationPage() { const next = await orgModuleApi.overview(); setOverview(next); if (next.module_toggles) setModuleToggles(next.module_toggles); + const proof = next.last_loop; + if (proof && typeof proof === "object" && proof.landed) { + setLanded(proof.landed as Record); + } return next; } catch (err) { if (!quiet) { @@ -227,9 +235,22 @@ export default function OrganizationPage() { ); }); + const produce = () => + runAction("produce", async () => { + const result: OrgProduceResult = await orgModuleApi.produce({ + name: produceName.trim(), + ima_url: produceIma.trim(), + }); + setLastActionNotes(result.notes); + message.success( + t("organization.produceDone", { name: result.slug || produceName }), + ); + }); + const packBack = () => runAction("pack", async () => { const result: OrgPackResult = await orgModuleApi.pack(); + if (result.applied.landed) setLanded(result.applied.landed); setLastActionNotes([ ...result.pack.notes, ...result.applied.notes, @@ -244,12 +265,27 @@ export default function OrganizationPage() { runAction("loop", async () => { const proof = await orgModuleApi.runLoop(); setLoopProof(proof); + if (proof.landed) setLanded(proof.landed); setLastActionNotes(proof.notes); message.success( proof.ok ? t("organization.loopOk") : t("organization.loopPartial"), ); }); + const flagLabel = (value: boolean | undefined) => + value ? t("organization.yes") : t("organization.no"); + + const progressLabel = + busy === "assemble" + ? t("organization.progressAssemble") + : busy === "pack" + ? t("organization.progressPack") + : busy === "loop" + ? t("organization.progressLoop") + : busy === "produce" + ? t("organization.progressProduce") + : null; + const lastLoop = (loopProof ?? overview?.last_loop) as OrgLoopProof | null; const lastSync = overview?.last_sync ? formatServerIsoDateTime(overview.last_sync, timeZone) @@ -515,6 +551,12 @@ export default function OrganizationPage() { + {progressLabel && ( +
+

{progressLabel}

+
+ )} +
+
+ +

+ {t("organization.produceTitle")} +

+

+ {t("organization.produceBody")} +

+ setProduceName(event.target.value)} + placeholder={t("organization.produceName")} + /> + setProduceIma(event.target.value)} + placeholder={t("organization.produceIma")} + /> + +
@@ -576,7 +644,7 @@ export default function OrganizationPage() { {lastLoop ? (
  1. - {t("organization.timelineOk")}: {String(lastLoop.ok)} + {t("organization.timelineOk")}: {flagLabel(lastLoop.ok)}
    2. {lastLoop.employees?.length ? (
  2. @@ -592,11 +660,11 @@ export default function OrganizationPage() { ) : null}
  3. {t("organization.timelineRemote")}:{" "} - {String(Boolean(lastLoop.remote_applied))} + {flagLabel(Boolean(lastLoop.remote_applied))}
  4. {t("organization.timelineGovernance")}:{" "} - {String(Boolean(lastLoop.governance_blocked))} + {flagLabel(Boolean(lastLoop.governance_blocked))}
    5. {(lastLoop.notes ?? []).slice(0, 6).map((note) => (
  5. {note}
    6. @@ -614,6 +682,19 @@ export default function OrganizationPage() { )}
+
+

+ {t("organization.landedTitle")} +

+ {landed ? ( + 
+                  {JSON.stringify(landed, null, 2)}
+
+ ) : ( +

{t("organization.landedEmpty")}

+ )} +
+

{t("organization.catalogTitle")} diff --git a/desktop/src/process.go b/desktop/src/process.go index 76bfe7e9..151c7513 100644 --- a/desktop/src/process.go +++ b/desktop/src/process.go @@ -38,6 +38,12 @@ func hostLaunchEnv(root string, port int) map[string]string { if bundle := resolveSidecarDir(root); bundle != "" { env["FREEOS_OPENXYOS_HOME"] = bundle } + if install := openxyosInstallDir(); install != "" { + env["FREEOS_OPENXYOS_INSTALL"] = install + } + if _, _, ingest, err := loadOrCreateSidecarSecrets(home); err == nil && ingest != "" { + env["FREEOS_INGEST_TOKEN"] = ingest + } return env } diff --git a/desktop/src/process_sidecar.go b/desktop/src/process_sidecar.go index 71d0bb4a..3ced86ae 100644 --- a/desktop/src/process_sidecar.go +++ b/desktop/src/process_sidecar.go @@ -172,7 +172,7 @@ func randomSecret() string { return hex.EncodeToString(buf) } -func loadOrCreateSidecarSecrets(home string) (jwt string, cookie string, err error) { +func loadOrCreateSidecarSecrets(home string) (jwt string, cookie string, ingest string, err error) { path := sidecarSecretsPath(home) if data, readErr := os.ReadFile(path); readErr == nil { for _, line := range strings.Split(string(data), "\n") { @@ -189,6 +189,8 @@ func loadOrCreateSidecarSecrets(home string) (jwt string, cookie string, err err jwt = strings.TrimSpace(value) case "COOKIE_SECRET": cookie = strings.TrimSpace(value) + case "FREEOS_INGEST_TOKEN": + ingest = strings.TrimSpace(value) } } } @@ -198,18 +200,21 @@ func loadOrCreateSidecarSecrets(home string) (jwt string, cookie string, err err if cookie == "" { cookie = randomSecret() } + if ingest == "" { + ingest = randomSecret() + } if err := os.MkdirAll(sidecarDataDir(home), 0o755); err != nil { - return "", "", err + return "", "", "", err } - body := "JWT_SECRET=" + jwt + "\nCOOKIE_SECRET=" + cookie + "\n" + body := "JWT_SECRET=" + jwt + "\nCOOKIE_SECRET=" + cookie + "\nFREEOS_INGEST_TOKEN=" + ingest + "\n" if err := os.WriteFile(path, []byte(body), 0o600); err != nil { - return "", "", err + return "", "", "", err } - return jwt, cookie, nil + return jwt, cookie, ingest, nil } func sidecarLaunchEnv(home string, dashboardPort int) (map[string]string, error) { - jwt, cookie, err := loadOrCreateSidecarSecrets(home) + jwt, cookie, ingest, err := loadOrCreateSidecarSecrets(home) if err != nil { return nil, err } @@ -227,6 +232,7 @@ func sidecarLaunchEnv(home string, dashboardPort int) (map[string]string, error) "SEED_DEMO_DATA": "false", "JWT_SECRET": jwt, "COOKIE_SECRET": cookie, + "FREEOS_INGEST_TOKEN": ingest, "CORS_ORIGIN": origin, "FREEOS_HOME": home, "OCTOP_HOME": home, diff --git a/desktop/src/process_sidecar_test.go b/desktop/src/process_sidecar_test.go index 935f15a9..fb5e4354 100644 --- a/desktop/src/process_sidecar_test.go +++ b/desktop/src/process_sidecar_test.go @@ -9,18 +9,18 @@ import ( func TestLoadOrCreateSidecarSecretsPersists(t *testing.T) { home := t.TempDir() - jwt, cookie, err := loadOrCreateSidecarSecrets(home) + jwt, cookie, ingest, err := loadOrCreateSidecarSecrets(home) if err != nil { t.Fatal(err) } - if jwt == "" || cookie == "" { + if jwt == "" || cookie == "" || ingest == "" { t.Fatal("secrets should be generated") } - jwt2, cookie2, err := loadOrCreateSidecarSecrets(home) + jwt2, cookie2, ingest2, err := loadOrCreateSidecarSecrets(home) if err != nil { t.Fatal(err) } - if jwt != jwt2 || cookie != cookie2 { + if jwt != jwt2 || cookie != cookie2 || ingest != ingest2 { t.Fatal("secrets should persist across launches") } raw, err := os.ReadFile(sidecarSecretsPath(home)) diff --git a/docs/asset-loop.md b/docs/asset-loop.md index 4ec967c1..77e30329 100644 --- a/docs/asset-loop.md +++ b/docs/asset-loop.md @@ -24,7 +24,7 @@ That path: - compiles colleagues and promotes `draft → market → recruit → shadow → active` - **registers real FreeOS agents** (`org-`) so they appear in chat routing - publishes `freeos.asset-pack.v1` -- **applies** the pack to openXYOS-shaped surfaces (`/api/employees`, `/api/talent`, `/api/plugins`, `/api/module-settings`) — live HTTP when `OPENXYOS_BASE_URL` is set, always a durable `{FREEOS_HOME}/openxyos-mirror/` +- **applies** the pack to openXYOS via `/api/freeos/ingest` (shared ingest token, no login wall) when the sidecar is healthy, always a durable `{FREEOS_HOME}/openxyos-mirror/` - imports the applied surfaces back - proves a high-risk tool is blocked (`execute=false`) diff --git a/modules/openxyos/backend/routes/freeos-bridge.ts b/modules/openxyos/backend/routes/freeos-bridge.ts new file mode 100644 index 00000000..e7b613a7 --- /dev/null +++ b/modules/openxyos/backend/routes/freeos-bridge.ts @@ -0,0 +1,269 @@ +import { Router, Request, Response, NextFunction } from "express"; +import { timingSafeEqual } from "crypto"; +import { dbAll, dbGet, dbRun } from "../db"; + +/** + * FreeOS ↔ openXYOS mutual-growth bridge. + * + * Enabled only when the sidecar is launched by FreeOS (FREEOS_HOME or + * FREEOS_INGEST_TOKEN). Authenticates with a shared ingest token so the + * Organization page can push / pull assets without a login wall. + */ +export const freeosBridgeRoutes = Router(); + +function ingestEnabled(): boolean { + return Boolean( + (process.env.FREEOS_INGEST_TOKEN || "").trim() || + (process.env.FREEOS_HOME || "").trim(), + ); +} + +function expectedToken(): string { + return (process.env.FREEOS_INGEST_TOKEN || "").trim(); +} + +function tokensMatch(provided: string, expected: string): boolean { + if (!provided || !expected) return false; + const a = Buffer.from(provided); + const b = Buffer.from(expected); + if (a.length !== b.length) return false; + return timingSafeEqual(a, b); +} + +function requireIngestToken(req: Request, res: Response, next: NextFunction) { + if (!ingestEnabled()) { + return res.status(404).json({ success: false, error: "freeos bridge disabled" }); + } + const expected = expectedToken(); + if (!expected) { + return res.status(503).json({ success: false, error: "FREEOS_INGEST_TOKEN unset" }); + } + const provided = String(req.header("x-freeos-ingest-token") || "").trim(); + if (!tokensMatch(provided, expected)) { + return res.status(401).json({ success: false, error: "invalid ingest token" }); + } + next(); +} + +function resolveTenantId(raw: unknown): number { + const n = Number(raw); + if (Number.isInteger(n) && n > 0) return n; + const first = dbGet("SELECT id FROM tenants ORDER BY id LIMIT 1") as { id?: number } | undefined; + return Number(first?.id) || 1; +} + +function asList(value: unknown): Record[] { + if (!Array.isArray(value)) return []; + return value.filter((item): item is Record => Boolean(item) && typeof item === "object"); +} + +function text(value: unknown, fallback = ""): string { + const out = String(value ?? "").trim(); + return out || fallback; +} + +function skillsText(value: unknown): string { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean).join(", "); + } + return text(value); +} + +function upsertEmployee(tenantId: number, item: Record): "created" | "updated" { + const name = text(item.name, text(item.slug, "digital-colleague")); + const existing = dbGet( + "SELECT id FROM employees WHERE tenant_id = ? AND name = ?", + [tenantId, name], + ) as { id?: number } | undefined; + const role = text(item.role, text(item.agent_type, "ai-colleague")); + const agentType = text(item.agent_type, item.slug ? `freeos-${item.slug}` : "freeos"); + const skills = skillsText(item.skills || item.capabilities); + const status = text(item.status, "active"); + const category = text(item.employment_category, "internal"); + const description = text(item.description); + const emoji = text(item.avatar_emoji, "🤖"); + if (existing?.id) { + dbRun( + `UPDATE employees SET role = ?, agent_type = ?, employee_type = 'ai', skills = ?, + avatar_emoji = ?, status = ?, employment_category = ?, description = ? + WHERE id = ? AND tenant_id = ?`, + [role, agentType, skills, emoji, status, category, description, existing.id, tenantId], + ); + return "updated"; + } + dbRun( + `INSERT INTO employees + (company_id, name, role, agent_type, employee_type, skills, avatar_emoji, status, employment_category, description, tenant_id) + VALUES (?, ?, ?, ?, 'ai', ?, ?, ?, ?, ?, ?)`, + [1, name, role, agentType, skills, emoji, status, category, description, tenantId], + ); + return "created"; +} + +function upsertTalent(tenantId: number, item: Record): "created" | "updated" { + const name = text(item.name, "digital-talent"); + const existing = dbGet( + "SELECT id FROM talent_pool WHERE tenant_id = ? AND name = ?", + [tenantId, name], + ) as { id?: number } | undefined; + const skills = skillsText(item.skills || item.capabilities); + const status = text(item.status || item.talent_status, "available"); + const description = text(item.description); + const agentType = text(item.agent_type); + if (existing?.id) { + dbRun( + `UPDATE talent_pool SET talent_type = 'ai', skills = ?, status = ?, description = ?, + agent_type = ?, source = 'FreeOS', integration_type = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND tenant_id = ?`, + [skills, status, description, agentType, text(item.integration_type, "agent-blueprint-v1"), existing.id, tenantId], + ); + return "updated"; + } + dbRun( + `INSERT INTO talent_pool + (tenant_id, talent_type, name, skills, description, source, status, agent_type, integration_type) + VALUES (?, 'ai', ?, ?, ?, 'FreeOS', ?, ?, ?)`, + [tenantId, name, skills, description, status, agentType, text(item.integration_type, "agent-blueprint-v1")], + ); + return "created"; +} + +function upsertPlugin(tenantId: number, item: Record, category = "工具"): "created" | "updated" { + const slug = text(item.slug || item.plugin_id || item.id || item.name); + const name = text(item.name, slug || "freeos-plugin"); + const existing = dbGet( + "SELECT id FROM plugins WHERE tenant_id = ? AND (slug = ? OR name = ?)", + [tenantId, slug, name], + ) as { id?: number } | undefined; + const description = text(item.description, "Imported from FreeOS"); + const config = typeof item.config_json === "string" ? item.config_json : JSON.stringify(item); + if (existing?.id) { + dbRun( + `UPDATE plugins SET name = ?, slug = ?, category = ?, description = ?, author = 'FreeOS', + status = 'active', config_json = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND tenant_id = ?`, + [name, slug, category, description, config, existing.id, tenantId], + ); + return "updated"; + } + dbRun( + `INSERT INTO plugins (tenant_id, name, slug, category, description, author, status, config_json) + VALUES (?, ?, ?, ?, ?, 'FreeOS', 'active', ?)`, + [tenantId, name, slug, category, description, config], + ); + return "created"; +} + +function upsertSkill(tenantId: number, item: Record): "created" | "updated" { + const slug = text(item.slug || item.name); + const name = text(item.name, slug || "freeos-skill"); + const existing = dbGet( + "SELECT id FROM skills WHERE tenant_id = ? AND (slug = ? OR name = ?)", + [tenantId, slug, name], + ) as { id?: number } | undefined; + const content = text(item.content || item.description); + const category = text(item.category, "FreeOS"); + if (existing?.id) { + dbRun( + `UPDATE skills SET name = ?, slug = ?, category = ?, description = ?, content = ?, + source = 'FreeOS', enabled = 1, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND tenant_id = ?`, + [name, slug, category, content, content, existing.id, tenantId], + ); + return "updated"; + } + dbRun( + `INSERT INTO skills (tenant_id, name, slug, category, description, content, source, enabled, author) + VALUES (?, ?, ?, ?, ?, ?, 'FreeOS', 1, 'FreeOS')`, + [tenantId, name, slug, category, content, content], + ); + return "created"; +} + +function countOf(sql: string, tenantId: number): number { + const row = dbGet(sql, [tenantId]) as { c?: number } | undefined; + return Number(row?.c || 0); +} + +freeosBridgeRoutes.get("/health", (_req, res) => { + if (!ingestEnabled()) { + return res.status(404).json({ success: false, error: "freeos bridge disabled" }); + } + res.json({ + success: true, + data: { + ok: true, + ingest: Boolean(expectedToken()), + plane: "openxyos-control", + }, + }); +}); + +freeosBridgeRoutes.use(requireIngestToken); + +freeosBridgeRoutes.get("/export", (req, res) => { + const tenantId = resolveTenantId(req.query.tenant_id); + const employees = dbAll("SELECT * FROM employees WHERE tenant_id = ? ORDER BY id", [tenantId]); + const talent = dbAll("SELECT * FROM talent_pool WHERE tenant_id = ? ORDER BY id", [tenantId]); + const plugins = dbAll("SELECT * FROM plugins WHERE tenant_id = ? ORDER BY id", [tenantId]); + const skills = dbAll("SELECT * FROM skills WHERE tenant_id = ? ORDER BY id", [tenantId]); + res.json({ + success: true, + data: { + tenant_id: tenantId, + employees, + talent, + plugins, + skills, + counts: { + employees: employees.length, + talent: talent.length, + plugins: plugins.length, + skills: skills.length, + }, + }, + }); +}); + +freeosBridgeRoutes.post("/ingest", (req, res) => { + const body = (req.body && typeof req.body === "object" ? req.body : {}) as Record; + const tenantId = resolveTenantId(body.tenant_id); + const landed = { + employees: { created: 0, updated: 0 }, + talent: { created: 0, updated: 0 }, + plugins: { created: 0, updated: 0 }, + skills: { created: 0, updated: 0 }, + mcp: { created: 0, updated: 0 }, + }; + + for (const item of asList(body.employees)) { + landed.employees[upsertEmployee(tenantId, item)] += 1; + } + for (const item of asList(body.talent)) { + landed.talent[upsertTalent(tenantId, item)] += 1; + } + for (const item of asList(body.plugins)) { + landed.plugins[upsertPlugin(tenantId, item)] += 1; + } + for (const item of asList(body.skills)) { + landed.skills[upsertSkill(tenantId, item)] += 1; + } + for (const item of asList(body.mcp)) { + landed.mcp[upsertPlugin(tenantId, { ...item, category: "MCP" }, "MCP")] += 1; + } + + res.json({ + success: true, + data: { + tenant_id: tenantId, + landed, + counts: { + employees: countOf("SELECT COUNT(*) as c FROM employees WHERE tenant_id = ?", tenantId), + talent: countOf("SELECT COUNT(*) as c FROM talent_pool WHERE tenant_id = ?", tenantId), + plugins: countOf("SELECT COUNT(*) as c FROM plugins WHERE tenant_id = ?", tenantId), + skills: countOf("SELECT COUNT(*) as c FROM skills WHERE tenant_id = ?", tenantId), + }, + note: "Drafts landed on the control plane. Modules are not auto-enabled.", + }, + }); +}); diff --git a/modules/openxyos/backend/server.ts b/modules/openxyos/backend/server.ts index 873d3523..1e6d976c 100644 --- a/modules/openxyos/backend/server.ts +++ b/modules/openxyos/backend/server.ts @@ -47,6 +47,7 @@ import reviewsRoutes from "./routes/reviews"; import { announcementRoutes } from "./routes/announcements"; import { analyticsRoutes } from "./routes/analytics"; import { assistantRoutes } from "./routes/assistant"; +import { freeosBridgeRoutes } from "./routes/freeos-bridge"; import { seedDatabase } from "./seed"; import { authenticate } from "./middleware"; import { setupWebSocket } from "./services/websocket"; @@ -99,7 +100,7 @@ async function main() { res.setHeader("Access-Control-Allow-Origin", "*"); res.setHeader("Access-Control-Allow-Credentials", "true"); res.setHeader("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,OPTIONS"); - res.setHeader("Access-Control-Allow-Headers", "Content-Type,Authorization"); + res.setHeader("Access-Control-Allow-Headers", "Content-Type,Authorization,X-FreeOS-Ingest-Token"); if (req.method === "OPTIONS") return res.sendStatus(200); return next(); } @@ -178,6 +179,7 @@ app.use(cookieParser(cookieSecret)); app.use("/api/org", orgRoutes); app.use("/api/knowledge", knowledgeRoutes); app.use("/api/health", healthRoutes); + app.use("/api/freeos", freeosBridgeRoutes); app.use("/api/notifications", notificationRoutes); app.use("/api/dashboard", dashboardRoutes); app.use("/api/settings", settingsRoutes); diff --git a/src/octop/api/routers/local_models.py b/src/octop/api/routers/local_models.py index 1fdc8735..f8f5863f 100644 --- a/src/octop/api/routers/local_models.py +++ b/src/octop/api/routers/local_models.py @@ -3,13 +3,15 @@ from __future__ import annotations import asyncio +import json from typing import Any from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field -from octop.api.deps import current_user, require_permission +from octop.api.deps import current_user, get_server, require_permission from octop.infra.agents.providers.local_probe import probe_local_models +from octop.infra.server import OctopServer from octop.infra.utils.ollama_manager import OllamaModelManager router = APIRouter() @@ -26,18 +28,46 @@ async def local_models_probe( return await asyncio.to_thread(probe_local_models) +def _register_ollama_model(server: OctopServer, name: str) -> bool: + services = getattr(server, "services", None) + repo = getattr(services, "provider_repo", None) if services is not None else None + if repo is None: + return False + model = {"id": name, "name": name, "enabled": True, "input": ["text"]} + row = repo.get_by_name("ollama") + if row is None: + repo.create( + name="ollama", + kind="ollama", + base_url="http://127.0.0.1:11434/v1", + api_key="ollama", + models_json=json.dumps([model]), + note="Registered by one-click local model install", + ) + return True + models = row.get_models() + if any(str(item.get("id") or "") == name for item in models if isinstance(item, dict)): + return False + models.append(model) + repo.update(row.id, models_json=json.dumps(models)) + return True + + @router.post("/install", summary="Download and register a recommended Ollama model") async def local_models_install( body: LocalInstallBody, + server: OctopServer = Depends(get_server), _: Any = Depends(require_permission("ollama_models")), ) -> dict[str, Any]: try: info = await asyncio.to_thread(OllamaModelManager.pull_model, body.name) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + registered = _register_ollama_model(server, info.name or body.name) return { "ok": True, "name": info.name, "size": info.size, "source": "ollama", + "registered": registered, } diff --git a/src/octop/api/routers/org_module.py b/src/octop/api/routers/org_module.py index b201fe21..c852b103 100644 --- a/src/octop/api/routers/org_module.py +++ b/src/octop/api/routers/org_module.py @@ -13,7 +13,11 @@ from octop.infra.server import OctopServer from octop.infra.utils.locale import resolve_request_locale from octop.modules.org_os.catalog import OPENXYOS_MODULES -from octop.modules.org_os.empower import assemble_from_blueprint, pack_to_openxyos +from octop.modules.org_os.empower import ( + assemble_from_blueprint, + pack_to_openxyos, + produce_from_corpus, +) from octop.modules.org_os.module_toggles import load_module_toggles, save_module_toggles from octop.modules.org_os.notes import localize_mapping, localize_note, localize_notes from octop.modules.org_os.overview import build_overview @@ -143,6 +147,34 @@ async def org_module_assemble( return localize_mapping(payload, resolve_request_locale(request), "notes") +class OrgProduceBody(BaseModel): + kb_id: str = "" + distill_path: str = "" + ima_url: str = "" + name: str = "" + + +@router.post("/produce", summary="Produce a FreeOS colleague from a knowledge corpus") +async def org_module_produce( + request: Request, + body: OrgProduceBody, + server: OctopServer = Depends(get_server), + _: Any = Depends(require_permission("plugins")), +) -> dict[str, Any]: + service = _service(server) + if not service.is_enabled(): + service.set_enabled(True) + payload = await asyncio.to_thread( + produce_from_corpus, + service, + kb_id=body.kb_id, + distill_path=body.distill_path, + ima_url=body.ima_url, + name=body.name, + ) + return localize_mapping(payload, resolve_request_locale(request), "notes") + + @router.post("/pack", summary="Pack FreeOS skills/MCP back to openXYOS") async def org_module_pack( request: Request, diff --git a/src/octop/i18n/en.json b/src/octop/i18n/en.json index 94676bd0..38dd45fb 100644 --- a/src/octop/i18n/en.json +++ b/src/octop/i18n/en.json @@ -722,7 +722,24 @@ "hero": "FreeOS does the work. openXYOS owns organization and governance.", "planes": "Data plane: employees / skills / MCP / tasks. Control plane: blueprints / modules / governance.", "sidecar_offline": "openXYOS sidecar is offline — start it to sync blueprints and approvals.", - "mirror_only": "OPENXYOS_BASE_URL unset; applied to local mirror only." + "mirror_only": "OPENXYOS_BASE_URL unset; applied to local mirror only.", + "local_mirror_short": "Local mirror is the durable record.", + "imported_export": "imported live control-plane export", + "export_unreachable": "sidecar export unreachable; using catalog/blueprint fallback", + "ingest_ok": "control plane accepted the FreeOS ingest", + "ingest_missing": "sidecar is up but FreeOS ingest API is missing; falling back to public routes", + "ingest_denied": "sidecar rejected the ingest token; falling back to public routes", + "ingest_denied_mirror": "sidecar rejected the ingest token; mirror is complete", + "public_ok": "control plane accepted drafts on public routes", + "control_rejected": "control plane at {url} did not accept drafts; mirror is complete", + "loop_done": "Loop produced employees, published an asset pack, applied it, and imported back.", + "pack_review": "Not installed into the sidecar. Review openxyos/ then enable per tenant.", + "pack_tenant": "One tenant = one workspace; do not unpack this pack into a shared sandbox.", + "pack_redact": "Agent .env values are redacted except tenant/slug/schema keys.", + "corpus_pointer": "attached cloud corpus pointer → {path}", + "corpus_distilled": "distilled {count} corpus files → {path}", + "corpus_empty": "no knowledge mount; producing a corpus-ready colleague workspace", + "permissions_unreachable": "sidecar permissions unreachable; pass --policies or retry when signed in" }, "sidecar": { "already": "sidecar already reachable", diff --git a/src/octop/i18n/zh.json b/src/octop/i18n/zh.json index ae3fe8ca..eb86e7da 100644 --- a/src/octop/i18n/zh.json +++ b/src/octop/i18n/zh.json @@ -722,7 +722,24 @@ "hero": "FreeOS 负责干活，openXYOS 负责组织与治理。", "planes": "数据面：员工 / 技能 / MCP / 任务。控制面：蓝图 / 模块 / 治理。", "sidecar_offline": "openXYOS 边车离线 — 启动后才能同步蓝图与审批。", - "mirror_only": "未设置 OPENXYOS_BASE_URL；仅写入本地镜像。" + "mirror_only": "未设置 OPENXYOS_BASE_URL；仅写入本地镜像。", + "local_mirror_short": "本地镜像是可靠记录。", + "imported_export": "已导入在线控制面导出", + "export_unreachable": "边车导出不可达；回退到目录/蓝图", + "ingest_ok": "控制面已接受 FreeOS 写入", + "ingest_missing": "边车在线但缺少 FreeOS 写入接口；回退到公开路由", + "ingest_denied": "边车拒绝了写入令牌；回退到公开路由", + "ingest_denied_mirror": "边车拒绝了写入令牌；本地镜像已写完", + "public_ok": "控制面已通过公开路由接受草稿", + "control_rejected": "控制面 {url} 未接受草稿；本地镜像已写完", + "loop_done": "循环已生成员工、发布资产包、写入并回流。", + "pack_review": "尚未装进边车。请审阅 openxyos/ 后再按租户启用。", + "pack_tenant": "一个租户对应一个工作区；不要把此包解到共享沙箱。", + "pack_redact": "智能体 .env 已脱敏，仅保留租户/别名/模式键。", + "corpus_pointer": "已挂接云知识库指针 → {path}", + "corpus_distilled": "已蒸馏 {count} 个语料文件 → {path}", + "corpus_empty": "没有知识库挂载；正在生成可供语料使用的同事工作区", + "permissions_unreachable": "边车权限接口不可达；请传入策略或登录后再试" }, "sidecar": { "already": "边车已可达", diff --git a/src/octop/modules/org_os/apply/apply.py b/src/octop/modules/org_os/apply/apply.py index ca378917..c291938a 100644 --- a/src/octop/modules/org_os/apply/apply.py +++ b/src/octop/modules/org_os/apply/apply.py @@ -12,7 +12,6 @@ OpenXyosControlClient, mirror_root, ) -from octop.modules.org_os.catalog import catalog_keys def _load_json(path: Path) -> Any: @@ -24,14 +23,59 @@ def _load_json(path: Path) -> Any: return None -def _module_settings_payload(tenant_id: str) -> dict[str, Any]: - updates = dict.fromkeys(catalog_keys(), False) - return { - "enabled_by_default": False, - "tenant_id": tenant_id, - "updates": updates, - "note": "Drafts only. Operator must toggle each module on.", - } +def _items(doc: Any, key: str) -> list[dict[str, Any]]: + if isinstance(doc, dict) and isinstance(doc.get(key), list): + return [item for item in doc[key] if isinstance(item, dict)] + if isinstance(doc, list): + return [item for item in doc if isinstance(item, dict)] + return [] + + +def _plugin_payloads(dest: Path) -> list[dict[str, Any]]: + payloads: list[dict[str, Any]] = [] + openxyos_dir = dest / "openxyos" + if not openxyos_dir.is_dir(): + return payloads + for path in sorted(openxyos_dir.glob("*.publish.json")): + if path.name.startswith("org-employees") or path.name.startswith("org-talent"): + continue + raw = _load_json(path) + if isinstance(raw, dict): + payloads.append(raw) + return payloads + + +def _skill_payloads(dest: Path) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + skills_root = dest / "skills" + if not skills_root.is_dir(): + return out + for skill_dir in sorted(path for path in skills_root.iterdir() if path.is_dir()): + manifest = skill_dir / "SKILL.md" + if not manifest.is_file(): + continue + out.append( + { + "name": skill_dir.name, + "slug": skill_dir.name, + "category": "FreeOS", + "content": manifest.read_text(encoding="utf-8")[:8000], + "source": "FreeOS", + } + ) + return out + + +def _mcp_payloads(dest: Path) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + mcp_root = dest / "mcps" + if not mcp_root.is_dir(): + return out + for path in sorted(mcp_root.glob("*.json")): + raw = _load_json(path) + if isinstance(raw, dict): + out.append({"name": path.stem, "slug": path.stem, "config_json": json.dumps(raw)}) + return out def apply_asset_pack( @@ -42,115 +86,116 @@ def apply_asset_pack( base_url: str = "", headers: dict[str, str] | None = None, ) -> ApplyResult: - """Write local mirror + POST/PUT drafts to module-settings/plugins/employees/talent.""" + """Write local mirror + ingest drafts to the control plane when reachable.""" dest = pack_dir if pack_dir.is_dir() else Path(pack_dir) tid = tenant_id or "default" mirror = mirror_root(home, tid) mirror.mkdir(parents=True, exist_ok=True) - client = OpenXyosControlClient(base_url, headers=headers) + client = OpenXyosControlClient(base_url, headers=headers, home=home) + employees = _items(_load_json(dest / "openxyos" / "org-employees.publish.json"), "employees") + talent = _items(_load_json(dest / "openxyos" / "org-talent.publish.json"), "talent") + plugins = _plugin_payloads(dest) + skills = _skill_payloads(dest) + mcp = _mcp_payloads(dest) + + employees_doc = { + "employees": employees, + "enabled_by_default": False, + "source": "freeos.asset-pack.v1", + } + talent_doc = {"talent": talent, "enabled_by_default": False, "source": "freeos.asset-pack.v1"} + plugins_doc = {"enabled_by_default": False, "plugins": plugins} + emp_mirror = client.write_mirror(mirror / "employees.json", employees_doc) + talent_mirror = client.write_mirror(mirror / "talent.json", talent_doc) + plugins_mirror = client.write_mirror(mirror / "plugins.json", plugins_doc) + skills_mirror = client.write_mirror(mirror / "skills.json", {"skills": skills}) + result = ApplyResult( pack_dir=dest, mirror_dir=mirror, control_plane_url=client.base_url, notes=[ - "Local mirror is the durable record. HTTP apply is best-effort.", + "Local mirror is the durable record.", "Nothing is auto-enabled on the control plane.", ], ) - employees_src = dest / "openxyos" / "org-employees.publish.json" - employees = _load_json(employees_src) or {"employees": []} - emp_mirror = client.write_mirror(mirror / "employees.json", employees) - emp_items = employees.get("employees") if isinstance(employees, dict) else [] - if not isinstance(emp_items, list): - emp_items = [] - http = client.request( - "POST", - "/api/employees", - payload={ - "employees": emp_items, - "enabled_by_default": False, - "source": "freeos.asset-pack.v1", - }, - ) - if emp_items and client.base_url: - # Also upsert each employee the way openXYOS POST /api/employees expects. - for item in emp_items: - if isinstance(item, dict): - client.request("POST", "/api/employees", payload=item) - result.receipts.append( - ApplyReceipt( - surface="employees", - endpoint="/api/employees", - mirrored=str(emp_mirror), - http=http, - item_count=len(emp_items), + ingest_payload = { + "tenant_id": tid if tid.isdigit() else 1, + "employees": employees, + "talent": talent, + "plugins": plugins, + "skills": skills, + "mcp": mcp, + } + ingest = client.ingest(ingest_payload) + if ingest.ok: + landed = ingest.body.get("data") if isinstance(ingest.body, dict) else {} + result.landed = landed if isinstance(landed, dict) else {} + result.receipts.append( + ApplyReceipt( + surface="ingest", + endpoint="/api/freeos/ingest", + mirrored=str(emp_mirror), + http=ingest, + item_count=len(employees) + len(talent) + len(plugins) + len(skills) + len(mcp), + landed=result.landed, + ) ) - ) - - talent_src = dest / "openxyos" / "org-talent.publish.json" - talent = _load_json(talent_src) or {"talent": []} - talent_mirror = client.write_mirror(mirror / "talent.json", talent) - talent_items = talent.get("talent") if isinstance(talent, dict) else [] - if not isinstance(talent_items, list): - talent_items = [] - http = client.request( - "POST", - "/api/talent", - payload={ - "talent": talent_items, - "enabled_by_default": False, - "source": "freeos.asset-pack.v1", - }, - ) - result.receipts.append( - ApplyReceipt( - surface="talent", - endpoint="/api/talent", - mirrored=str(talent_mirror), - http=http, - item_count=len(talent_items), + result.notes.append("control plane accepted the FreeOS ingest") + client.write_mirror(mirror / "apply-receipt.json", result.to_dict()) + return result + + if ( + client.base_url + and ingest.reached + and ingest.status_code in {401, 403} + and client.ingest_token + ): + result.receipts.append( + ApplyReceipt( + surface="ingest", + endpoint="/api/freeos/ingest", + mirrored=str(emp_mirror), + http=ingest, + item_count=len(employees) + len(talent) + len(plugins) + len(skills) + len(mcp), + ) ) - ) - - plugin_payloads: list[dict[str, Any]] = [] - openxyos_dir = dest / "openxyos" - if openxyos_dir.is_dir(): - for path in sorted(openxyos_dir.glob("*.publish.json")): - if path.name.startswith("org-employees") or path.name.startswith("org-talent"): - continue - raw = _load_json(path) - if isinstance(raw, dict): - plugin_payloads.append(raw) - plugins_doc = {"enabled_by_default": False, "plugins": plugin_payloads} - plugins_mirror = client.write_mirror(mirror / "plugins.json", plugins_doc) - http = client.request("POST", "/api/plugins", payload=plugins_doc) - result.receipts.append( - ApplyReceipt( - surface="plugins", - endpoint="/api/plugins", - mirrored=str(plugins_mirror), - http=http, - item_count=len(plugin_payloads), + result.notes.append("sidecar rejected the ingest token; mirror is complete") + client.write_mirror(mirror / "apply-receipt.json", result.to_dict()) + return result + if client.base_url and ingest.reached and ingest.status_code == 404: + result.notes.append( + "sidecar is up but FreeOS ingest API is missing; falling back to public routes" ) - ) - - settings = _module_settings_payload(tid) - settings_mirror = client.write_mirror(mirror / "module-settings.json", settings) - http = client.request("PUT", "/api/module-settings", payload=settings) - result.receipts.append( - ApplyReceipt( - surface="module-settings", - endpoint="/api/module-settings", - mirrored=str(settings_mirror), - http=http, - item_count=len(settings.get("updates") or {}), + elif client.base_url and ingest.reached and ingest.status_code in {401, 403}: + result.notes.append("sidecar rejected the ingest token; falling back to public routes") + + for surface, endpoint, payload, mirrored, count in ( + ("employees", "/api/employees", employees_doc, emp_mirror, len(employees)), + ("talent", "/api/talent", talent_doc, talent_mirror, len(talent)), + ("plugins", "/api/plugins", plugins_doc, plugins_mirror, len(plugins)), + ("skills", "/api/skills", {"skills": skills}, skills_mirror, len(skills)), + ): + http = client.request("POST", endpoint, payload=payload) + if surface == "employees" and employees and client.base_url and not http.ok: + for item in employees: + client.request("POST", "/api/employees", payload=item) + result.receipts.append( + ApplyReceipt( + surface=surface, + endpoint=endpoint, + mirrored=str(mirrored), + http=http, + item_count=count, + ) ) - ) receipt_path = mirror / "apply-receipt.json" client.write_mirror(receipt_path, result.to_dict()) - if client.base_url and not result.remote_applied: + if result.remote_applied: + result.notes.append("control plane accepted drafts on public routes") + elif client.base_url and not result.remote_applied: result.notes.append( f"control plane at {client.base_url} did not accept drafts; mirror is complete" ) @@ -171,13 +216,18 @@ def import_applied_surfaces( """ tid = tenant_id or "default" mirror = mirror_root(home, tid) - client = OpenXyosControlClient(base_url) + client = OpenXyosControlClient(base_url, home=home) remote: dict[str, Any] = {} - if client.base_url: + exported = client.export() + if isinstance(exported, dict): + remote = exported + client.write_mirror(mirror / "imported-export.json", exported) + elif client.base_url: for key, path in ( ("employees", "/api/employees"), ("talent", "/api/talent"), ("plugins", "/api/plugins"), + ("skills", "/api/skills"), ("module_settings", "/api/module-settings"), ("permissions", "/api/governance/permissions"), ): @@ -187,7 +237,7 @@ def import_applied_surfaces( client.write_mirror(mirror / f"imported-{key}.json", fetched) local: dict[str, Any] = {} - for name in ("employees", "talent", "plugins", "module-settings", "apply-receipt"): + for name in ("employees", "talent", "plugins", "skills", "apply-receipt"): raw = _load_json(mirror / f"{name}.json") if raw is not None: local[name.replace("-", "_")] = raw diff --git a/src/octop/modules/org_os/apply/client.py b/src/octop/modules/org_os/apply/client.py index 3d8a9a7a..1edd5fc6 100644 --- a/src/octop/modules/org_os/apply/client.py +++ b/src/octop/modules/org_os/apply/client.py @@ -1,14 +1,15 @@ """openXYOS control-plane HTTP client with a durable local mirror. -Live HTTP is used when ``OPENXYOS_BASE_URL`` / ``FREEOS_ORG_SIDECAR_URL`` is a -real origin. A down sidecar never fails the loop open: every payload is written -under ``{FREEOS_HOME}/openxyos-mirror//`` first. +Live HTTP uses the FreeOS ingest token when present (``/api/freeos/ingest``). +A down sidecar never fails the loop open: every payload is written under +``{FREEOS_HOME}/openxyos-mirror//`` first. """ from __future__ import annotations import json import os +import time from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -16,6 +17,8 @@ import httpx +from octop.modules.org_os.sidecar_secrets import INGEST_TOKEN_KEY, resolve_ingest_token + DEFAULT_CONTROL_URL = "http://127.0.0.1:3780" @@ -67,12 +70,25 @@ def __init__( self, base_url: str = "", *, - timeout: float = 3.0, + timeout: float = 8.0, headers: dict[str, str] | None = None, + home: Path | None = None, + retries: int = 4, ) -> None: self.base_url = resolve_control_plane_url(base_url) self.timeout = timeout + self.retries = max(retries, 1) + self.home = Path(home) if home is not None else None self.headers = dict(headers or {}) + token = self.headers.get("X-FreeOS-Ingest-Token") or resolve_ingest_token(self.home) + env_token = (os.environ.get(INGEST_TOKEN_KEY) or "").strip() + token = str(token or env_token).strip() + if token: + self.headers["X-FreeOS-Ingest-Token"] = token + + @property + def ingest_token(self) -> str: + return str(self.headers.get("X-FreeOS-Ingest-Token") or "") def write_mirror(self, dest: Path, payload: Any) -> Path: dest.parent.mkdir(parents=True, exist_ok=True) @@ -98,36 +114,50 @@ def request( reason="control-plane URL unset; local mirror only", ) url = f"{self.base_url}{path if path.startswith('/') else '/' + path}" - try: - with httpx.Client(timeout=self.timeout, follow_redirects=True) as client: - response = client.request( - method.upper(), - url, - json=payload, - headers=self.headers or None, + last = HttpResult( + ok=False, + reached=False, + method=method, + path=path, + reason="control plane unreachable", + ) + for attempt in range(self.retries): + try: + with httpx.Client(timeout=self.timeout, follow_redirects=True) as client: + response = client.request( + method.upper(), + url, + json=payload, + headers=self.headers or None, + ) + except httpx.HTTPError as exc: + last = HttpResult( + ok=False, + reached=False, + method=method, + path=path, + reason=f"control plane unreachable: {exc}", ) - except httpx.HTTPError as exc: - return HttpResult( - ok=False, - reached=False, + time.sleep(0.4 * (attempt + 1)) + continue + body: Any + try: + body = response.json() + except ValueError: + body = response.text + last = HttpResult( + ok=response.status_code < 400, + reached=True, method=method, path=path, - reason=f"control plane unreachable: {exc}", + status_code=response.status_code, + reason="" if response.status_code < 400 else f"HTTP {response.status_code}", + body=body, ) - body: Any - try: - body = response.json() - except ValueError: - body = response.text - return HttpResult( - ok=response.status_code < 400, - reached=True, - method=method, - path=path, - status_code=response.status_code, - reason="" if response.status_code < 400 else f"HTTP {response.status_code}", - body=body, - ) + if last.ok or last.status_code in {400, 401, 403, 404}: + return last + time.sleep(0.4 * (attempt + 1)) + return last def get_json(self, path: str) -> Any | None: result = self.request("GET", path) @@ -138,6 +168,16 @@ def get_json(self, path: str) -> Any | None: return body["data"] return body + def ingest(self, payload: dict[str, Any]) -> HttpResult: + return self.request("POST", "/api/freeos/ingest", payload=payload) + + def export(self) -> Any | None: + return self.get_json("/api/freeos/export") + + def bridge_ready(self) -> bool: + body = self.get_json("/api/freeos/health") + return isinstance(body, dict) and bool(body.get("ok")) + @dataclass class ApplyReceipt: @@ -146,6 +186,7 @@ class ApplyReceipt: mirrored: str http: HttpResult item_count: int = 0 + landed: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: return { @@ -153,6 +194,7 @@ def to_dict(self) -> dict[str, Any]: "endpoint": self.endpoint, "mirrored": self.mirrored, "item_count": self.item_count, + "landed": dict(self.landed), "http": self.http.to_dict(), } @@ -164,6 +206,7 @@ class ApplyResult: receipts: list[ApplyReceipt] = field(default_factory=list) notes: list[str] = field(default_factory=list) control_plane_url: str = "" + landed: dict[str, Any] = field(default_factory=dict) @property def remote_applied(self) -> bool: @@ -180,6 +223,7 @@ def to_dict(self) -> dict[str, Any]: "control_plane_url": self.control_plane_url, "remote_applied": self.remote_applied, "mirrored": self.mirrored, + "landed": dict(self.landed), "receipts": [item.to_dict() for item in self.receipts], "notes": list(self.notes), } diff --git a/src/octop/modules/org_os/assets/importer.py b/src/octop/modules/org_os/assets/importer.py index d1132a5e..3c53b5e4 100644 --- a/src/octop/modules/org_os/assets/importer.py +++ b/src/octop/modules/org_os/assets/importer.py @@ -10,6 +10,7 @@ import httpx +from octop.modules.org_os.apply.client import OpenXyosControlClient from octop.modules.org_os.compiler.blueprint import parse_blueprint from octop.modules.org_os.compiler.compile import compile_blueprint from octop.modules.org_os.governance.imported import write_imported_policies @@ -98,6 +99,61 @@ def import_openxyos_assets( result.agents.append(spawned.agent_id) result.notes.append(f"spawned FreeOS agent {spawned.agent_id}") + if from_sidecar: + client = OpenXyosControlClient(sidecar_url, home=home) + exported = client.export() + if isinstance(exported, dict): + result.notes.append("imported live control-plane export") + for item in exported.get("employees") or []: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + capabilities = [ + part.strip() + for part in str(item.get("skills") or "").split(",") + if part.strip() + ] + if not name or not capabilities: + continue + compiled = compile_blueprint( + { + "schema": "openxyos.agent-blueprint.v1", + "name": name, + "positioning": str(item.get("description") or item.get("role") or name), + "industry": "organization", + "capabilities": capabilities, + }, + home=home, + tenant_id=tid, + sidecar_url=sidecar_url, + ) + store = LifecycleStore(home, tid) + register_compiled( + store, + slug=compiled.slug, + name=name, + workspace=compiled.workspace, + lifecycle="draft", + ) + result.employees.append(compiled.slug) + result.notes.append(f"compiled blueprint → {compiled.workspace}") + if spawn_agents: + from octop.modules.org_os.runtime.spawn import spawn_colleague_agent + + record = store.get(compiled.slug) + if record is not None: + spawned = spawn_colleague_agent(home, record) + result.agents.append(spawned.agent_id) + result.notes.append(f"spawned FreeOS agent {spawned.agent_id}") + for item in exported.get("skills") or []: + if not isinstance(item, dict): + continue + slug = str(item.get("slug") or item.get("name") or "").strip() + if slug: + result.skills.append(slug) + elif sidecar_url: + result.notes.append("sidecar export unreachable; using catalog/blueprint fallback") + policies_raw: Any | None = None policies_source = "" if policies_path is not None: diff --git a/src/octop/modules/org_os/assets/pack.py b/src/octop/modules/org_os/assets/pack.py index b0858494..2237829a 100644 --- a/src/octop/modules/org_os/assets/pack.py +++ b/src/octop/modules/org_os/assets/pack.py @@ -228,7 +228,7 @@ def publish_asset_pack( mcp_count=mcp_count, agent_count=agent_count, notes=[ - "Not installed into the sidecar. Operator reviews openxyos/ then enables per tenant.", + "Not installed into the sidecar. Review openxyos/ then enable per tenant.", "One tenant = one workspace; do not unpack this pack into a shared sandbox.", "Agent .env values are redacted except tenant/slug/schema keys.", ], diff --git a/src/octop/modules/org_os/empower.py b/src/octop/modules/org_os/empower.py index df07dc5b..06730269 100644 --- a/src/octop/modules/org_os/empower.py +++ b/src/octop/modules/org_os/empower.py @@ -2,6 +2,7 @@ from __future__ import annotations +from pathlib import Path from typing import Any from octop.modules.org_os.apply.apply import apply_asset_pack @@ -74,10 +75,91 @@ def assemble_from_blueprint(service: OrgModuleService) -> dict[str, Any]: def pack_to_openxyos(service: OrgModuleService) -> dict[str, Any]: """Publish a FreeOS asset pack and apply it to the local / live control plane.""" pack = publish_asset_pack(service.home, tenant_id=service.tenant_id()) + sidecar = service.probe_sidecar() applied = apply_asset_pack( pack.directory, home=service.home, tenant_id=service.tenant_id() or "default", - base_url=service.sidecar_url(), + base_url=service.sidecar_url() if sidecar.reachable else "", ) return {"pack": pack.to_dict(), "applied": applied.to_dict()} + + +def produce_from_corpus( + service: OrgModuleService, + *, + kb_id: str = "", + distill_path: str = "", + ima_url: str = "", + name: str = "", +) -> dict[str, Any]: + """Build a digital employee from a local/cloud knowledge mount (ima pointer OK).""" + from octop.infra.knowledge.local_mount import ( + attach_cloud_pointer, + distill_readonly, + load_mounts, + ) + + tid = service.tenant_id() or "default" + dest = ( + Path(distill_path) if distill_path else (service.home / "org-corpus" / (kb_id or "default")) + ) + dest.mkdir(parents=True, exist_ok=True) + notes: list[str] = [] + copied = 0 + mounts = load_mounts(service.home) + mount = mounts.get(kb_id) if kb_id else None + cloud = ima_url or (mount.cloud_url if mount is not None else "") + if cloud: + pointer = attach_cloud_pointer(cloud, str(dest), provider="ima") + copied = int(pointer.get("copied") or 0) + notes.append(f"attached cloud corpus pointer → {dest}") + elif mount is not None and mount.source_path: + distilled = distill_readonly(mount.source_path, str(dest)) + copied = int(distilled.get("copied") or 0) + notes.append(f"distilled {copied} corpus files → {dest}") + else: + notes.append("no knowledge mount; producing a corpus-ready colleague workspace") + + references: list[dict[str, str]] = [] + for path in sorted(dest.glob("*.md"))[:12]: + excerpt = path.read_text(encoding="utf-8", errors="replace")[:1500] + references.append({"name": path.stem, "excerpt": excerpt}) + capabilities = [item["name"] for item in references] or ["corpus-research", "org-writing"] + colleague = name.strip() or (kb_id.replace("_", " ").strip() if kb_id else "Corpus Analyst") + compiled = compile_blueprint( + { + "schema": "openxyos.agent-blueprint.v1", + "name": colleague, + "positioning": "Produces work from a distilled local or ima corpus.", + "industry": "knowledge", + "capabilities": capabilities, + "ima": {"url": cloud, "status": "pointer"} if cloud else None, + "references": references, + }, + home=service.home, + tenant_id=tid, + sidecar_url=service.sidecar_url(), + ) + store = LifecycleStore(service.home, tid) + register_compiled( + store, + slug=compiled.slug, + name=colleague, + workspace=compiled.workspace, + lifecycle="draft", + ) + record = store.get(compiled.slug) + if record is None: + raise RuntimeError(f"failed to register colleague {compiled.slug}") + spawned = spawn_colleague_agent(service.home, record) + notes.append(f"compiled blueprint → {compiled.workspace}") + notes.append(f"spawned FreeOS agent {spawned.agent_id}") + return { + "slug": compiled.slug, + "workspace": str(compiled.workspace), + "distill_path": str(dest), + "copied": copied, + "spawned": spawned.to_dict(), + "notes": notes, + } diff --git a/src/octop/modules/org_os/loop/run.py b/src/octop/modules/org_os/loop/run.py index 2653af69..badb0d4a 100644 --- a/src/octop/modules/org_os/loop/run.py +++ b/src/octop/modules/org_os/loop/run.py @@ -3,12 +3,13 @@ from __future__ import annotations import json +import time from dataclasses import dataclass, field from pathlib import Path from typing import Any from octop.modules.org_os.apply.apply import apply_asset_pack, import_applied_surfaces -from octop.modules.org_os.apply.client import resolve_control_plane_url +from octop.modules.org_os.apply.client import OpenXyosControlClient, resolve_control_plane_url from octop.modules.org_os.assets.importer import import_openxyos_assets from octop.modules.org_os.assets.pack import publish_asset_pack from octop.modules.org_os.governance.interceptor import ( @@ -41,6 +42,7 @@ class LoopProof: governance_blocked: bool = False governance: dict[str, Any] = field(default_factory=dict) notes: list[str] = field(default_factory=list) + landed: dict[str, Any] = field(default_factory=dict) ok: bool = False def to_dict(self) -> dict[str, Any]: @@ -58,10 +60,22 @@ def to_dict(self) -> dict[str, Any]: "imported_roundtrip": dict(self.imported_roundtrip), "governance_blocked": self.governance_blocked, "governance": dict(self.governance), + "landed": dict(self.landed), "notes": list(self.notes), } +def _wait_control_plane(url: str, *, home: Path, attempts: int = 12) -> bool: + if not url: + return False + client = OpenXyosControlClient(url, home=home, timeout=2.0, retries=1) + for _ in range(max(attempts, 1)): + if client.get_json("/api/health/livez") is not None or client.bridge_ready(): + return True + time.sleep(0.5) + return False + + def _promote(store: LifecycleStore, slug: str) -> dict[str, str]: states = ("market", "recruit", "shadow", "active") seen: dict[str, str] = {"draft": "draft"} @@ -97,7 +111,20 @@ def run_growth_loop( policies = Path(policies_path) if policies_path else policies_fixture() if not blueprint.is_file(): raise FileNotFoundError(f"blueprint fixture missing: {blueprint}") - control_url = resolve_control_plane_url(sidecar_url, service.sidecar_url()) + control_url = resolve_control_plane_url(sidecar_url) + if control_url: + _wait_control_plane(control_url, home=home, attempts=8) + else: + health = service.probe_sidecar(timeout=0.4) + if health.reachable: + control_url = health.url + else: + from octop.modules.org_os.sidecar_launch import ensure_sidecar, sidecar_can_start + + if sidecar_can_start(): + started = ensure_sidecar(service, wait=12.0) + if started.reachable: + control_url = started.url or service.sidecar_url() imported = import_openxyos_assets( home, @@ -186,6 +213,7 @@ def run_growth_loop( }, governance_blocked=blocked, governance=governance, + landed=applied.landed if isinstance(applied.landed, dict) else {}, notes=[ *imported.notes, *applied.notes, diff --git a/src/octop/modules/org_os/notes.py b/src/octop/modules/org_os/notes.py index f4b1c5c5..f61faba9 100644 --- a/src/octop/modules/org_os/notes.py +++ b/src/octop/modules/org_os/notes.py @@ -104,6 +104,97 @@ "org.notes.mirror_only", (), ), + ( + re.compile(r"^Local mirror is the durable record\.$"), + "org.notes.local_mirror_short", + (), + ), + ( + re.compile(r"^imported live control-plane export$"), + "org.notes.imported_export", + (), + ), + ( + re.compile(r"^sidecar export unreachable; using catalog/blueprint fallback$"), + "org.notes.export_unreachable", + (), + ), + ( + re.compile(r"^control plane accepted the FreeOS ingest$"), + "org.notes.ingest_ok", + (), + ), + ( + re.compile( + r"^sidecar is up but FreeOS ingest API is missing; falling back to public routes$" + ), + "org.notes.ingest_missing", + (), + ), + ( + re.compile(r"^sidecar rejected the ingest token; falling back to public routes$"), + "org.notes.ingest_denied", + (), + ), + ( + re.compile(r"^sidecar rejected the ingest token; mirror is complete$"), + "org.notes.ingest_denied_mirror", + (), + ), + ( + re.compile(r"^control plane accepted drafts on public routes$"), + "org.notes.public_ok", + (), + ), + ( + re.compile(r"^control plane at (.+) did not accept drafts; mirror is complete$"), + "org.notes.control_rejected", + ("url",), + ), + ( + re.compile( + r"^Loop produced employees, published an asset pack, applied it, and imported back\.$" + ), + "org.notes.loop_done", + (), + ), + ( + re.compile(r"^Not installed into the sidecar\. Review openxyos/ then enable per tenant\.$"), + "org.notes.pack_review", + (), + ), + ( + re.compile( + r"^One tenant = one workspace; do not unpack this pack into a shared sandbox\.$" + ), + "org.notes.pack_tenant", + (), + ), + ( + re.compile(r"^Agent \.env values are redacted except tenant/slug/schema keys\.$"), + "org.notes.pack_redact", + (), + ), + ( + re.compile(r"^attached cloud corpus pointer → (.+)$"), + "org.notes.corpus_pointer", + ("path",), + ), + ( + re.compile(r"^distilled (\d+) corpus files → (.+)$"), + "org.notes.corpus_distilled", + ("count", "path"), + ), + ( + re.compile(r"^no knowledge mount; producing a corpus-ready colleague workspace$"), + "org.notes.corpus_empty", + (), + ), + ( + re.compile(r"^sidecar permissions unreachable; pass --policies or retry when signed in$"), + "org.notes.permissions_unreachable", + (), + ), ) diff --git a/src/octop/modules/org_os/sidecar_launch.py b/src/octop/modules/org_os/sidecar_launch.py index d611aca7..b0dbe864 100644 --- a/src/octop/modules/org_os/sidecar_launch.py +++ b/src/octop/modules/org_os/sidecar_launch.py @@ -7,13 +7,17 @@ import subprocess import sys import time -from contextlib import suppress from dataclasses import dataclass from pathlib import Path from typing import Any from octop.infra.utils.paths import PathLayout from octop.modules.org_os.service import OrgModuleService +from octop.modules.org_os.sidecar_secrets import ( + INGEST_TOKEN_KEY, + load_or_create_sidecar_secrets, + sidecar_data_dir, +) logger = logging.getLogger(__name__) @@ -87,6 +91,23 @@ def _localappdata_openxyos() -> Path | None: return path if path.is_dir() else None +def _install_openxyos() -> Path | None: + """#21 installer tree: $INSTDIR\\openxyos or FREEOS_OPENXYOS_INSTALL.""" + raw = (os.environ.get("FREEOS_OPENXYOS_INSTALL") or "").strip() + if raw: + path = Path(raw).expanduser() + return path if path.is_dir() else None + green = (os.environ.get("OCTOP_GREEN_PACKAGES") or "").strip() + if not green: + return None + # portable/packages → sibling ../openxyos (installer) or ../org-sidecar + parent = Path(green).expanduser().resolve().parent + for candidate in (parent / "openxyos", parent.parent / "openxyos"): + if candidate.is_dir(): + return candidate + return None + + def portable_root() -> Path | None: green = (os.environ.get("OCTOP_GREEN_PACKAGES") or "").strip() if green: @@ -104,6 +125,7 @@ def sidecar_bundle_dir() -> Path | None: for candidate in ( _explicit_openxyos_home(), _localappdata_openxyos(), + _install_openxyos(), PathLayout.from_env().root / "openxyos", ): if candidate is None: @@ -199,37 +221,11 @@ def sidecar_node_command(runtime: SidecarRuntime) -> str: return " ".join(sidecar_node_argv(runtime)) -def _load_or_create_secrets(data_dir: Path) -> tuple[str, str]: - env_path = data_dir / "sidecar.env" - jwt = "" - cookie = "" - if env_path.is_file(): - for line in env_path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - continue - key, value = stripped.split("=", 1) - if key.strip() == "JWT_SECRET": - jwt = value.strip() - elif key.strip() == "COOKIE_SECRET": - cookie = value.strip() - if not jwt: - jwt = os.urandom(32).hex() - if not cookie: - cookie = os.urandom(32).hex() - data_dir.mkdir(parents=True, exist_ok=True) - env_path.write_text( - f"JWT_SECRET={jwt}\nCOOKIE_SECRET={cookie}\n", - encoding="utf-8", - ) - with suppress(OSError): - env_path.chmod(0o600) - return jwt, cookie - - def sidecar_launch_env(home: Path, *, dashboard_port: int | None = None) -> dict[str, str]: - data_dir = Path(os.environ.get("FREEOS_ORG_DATA") or (home / "org-os")) - jwt, cookie = _load_or_create_secrets(data_dir) + secrets = load_or_create_sidecar_secrets(home) + jwt = secrets["JWT_SECRET"] + cookie = secrets["COOKIE_SECRET"] + ingest = secrets[INGEST_TOKEN_KEY] port = (os.environ.get("FREEOS_ORG_SIDECAR_PORT") or "3780").strip() or "3780" if dashboard_port is None: raw = (os.environ.get("OCTOP_PORT") or "").strip() @@ -245,12 +241,13 @@ def sidecar_launch_env(home: Path, *, dashboard_port: int | None = None) -> dict "NODE_ENV": "production", "PORT": port, "DB_DIALECT": "sqlite", - "DATABASE_PATH": str(data_dir / "xiongyuan.db"), + "DATABASE_PATH": str(sidecar_data_dir(home) / "xiongyuan.db"), "AIR_GAP_MODE": "true", "ALLOW_PUBLIC_REGISTRATION": env.get("ALLOW_PUBLIC_REGISTRATION") or "false", "SEED_DEMO_DATA": env.get("SEED_DEMO_DATA") or "false", "JWT_SECRET": jwt, "COOKIE_SECRET": cookie, + INGEST_TOKEN_KEY: ingest, "CORS_ORIGIN": env.get("CORS_ORIGIN") or origin, "FREEOS_HOME": str(home), "OCTOP_HOME": str(home), @@ -330,7 +327,7 @@ def _wait_reachable(service: OrgModuleService, wait: float) -> Any: return last -def start_sidecar(service: OrgModuleService, *, wait: float = 8.0) -> SidecarStartResult: +def start_sidecar(service: OrgModuleService, *, wait: float = 20.0) -> SidecarStartResult: health = service.probe_sidecar() command = sidecar_start_command() runtime = find_sidecar_runtime() @@ -392,7 +389,7 @@ def start_sidecar(service: OrgModuleService, *, wait: float = 8.0) -> SidecarSta ) -def ensure_sidecar(service: OrgModuleService, *, wait: float = 8.0) -> SidecarStartResult: +def ensure_sidecar(service: OrgModuleService, *, wait: float = 20.0) -> SidecarStartResult: """Keep the bundled sidecar up on desktop / first launch. Only auto-starts when the portable ``org-sidecar`` runtime is present. diff --git a/src/octop/modules/org_os/sidecar_secrets.py b/src/octop/modules/org_os/sidecar_secrets.py new file mode 100644 index 00000000..6fb18622 --- /dev/null +++ b/src/octop/modules/org_os/sidecar_secrets.py @@ -0,0 +1,71 @@ +"""Shared sidecar secrets (JWT / cookie / FreeOS ingest token).""" + +from __future__ import annotations + +import os +from contextlib import suppress +from pathlib import Path + +INGEST_TOKEN_KEY = "FREEOS_INGEST_TOKEN" + + +def sidecar_data_dir(home: Path) -> Path: + return Path(os.environ.get("FREEOS_ORG_DATA") or (home / "org-os")) + + +def sidecar_env_path(home: Path) -> Path: + return sidecar_data_dir(home) / "sidecar.env" + + +def _parse_env_file(path: Path) -> dict[str, str]: + if not path.is_file(): + return {} + out: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def _write_env_file(path: Path, values: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = [ + f"{key}={values[key]}" + for key in ("JWT_SECRET", "COOKIE_SECRET", INGEST_TOKEN_KEY) + if values.get(key) + ] + extra = [ + f"{key}={value}" + for key, value in values.items() + if key not in {"JWT_SECRET", "COOKIE_SECRET", INGEST_TOKEN_KEY} + ] + path.write_text("\n".join([*lines, *extra]) + "\n", encoding="utf-8") + with suppress(OSError): + path.chmod(0o600) + + +def load_or_create_sidecar_secrets(home: Path) -> dict[str, str]: + """Read `{home}/org-os/sidecar.env`, generating missing secrets.""" + path = sidecar_env_path(home) + values = _parse_env_file(path) + if not values.get("JWT_SECRET"): + values["JWT_SECRET"] = os.urandom(32).hex() + if not values.get("COOKIE_SECRET"): + values["COOKIE_SECRET"] = os.urandom(32).hex() + if not values.get(INGEST_TOKEN_KEY): + values[INGEST_TOKEN_KEY] = os.urandom(32).hex() + _write_env_file(path, values) + return values + + +def resolve_ingest_token(home: Path | None = None) -> str: + """Prefer the process env, then the durable sidecar.env file.""" + env = (os.environ.get(INGEST_TOKEN_KEY) or "").strip() + if env: + return env + if home is None: + return "" + return load_or_create_sidecar_secrets(home).get(INGEST_TOKEN_KEY, "") diff --git a/tests/e2e/test_org_growth_loop.py b/tests/e2e/test_org_growth_loop.py index b4021681..99601522 100644 --- a/tests/e2e/test_org_growth_loop.py +++ b/tests/e2e/test_org_growth_loop.py @@ -40,7 +40,7 @@ def test_e2e_loop_command_and_artifacts(tmp_path: Path, monkeypatch: pytest.Monk assert (tmp_path / "openxyos-mirror" / "e2e" / "apply-receipt.json").is_file() assert len(proof["skills"]) >= len(catalog_keys()) assert state.employees - assert state.module_settings + assert "/api/freeos/ingest" in state.posts finally: server.shutdown() server.server_close() diff --git a/tests/support/openxyos_harness.py b/tests/support/openxyos_harness.py index 704b3753..7f1c39d9 100644 --- a/tests/support/openxyos_harness.py +++ b/tests/support/openxyos_harness.py @@ -14,7 +14,10 @@ def __init__(self) -> None: self.employees: list[dict[str, Any]] = [] self.talent: list[dict[str, Any]] = [] self.plugins: list[dict[str, Any]] = [] + self.skills: list[dict[str, Any]] = [] self.module_settings: dict[str, Any] = {} + self.ingest_token: str = "" + self.ingested: dict[str, Any] = {} self.permissions: list[dict[str, Any]] = [ {"category": "delete", "allow": False, "reason": "harness deny"} ] @@ -46,11 +49,42 @@ def _read_json(self) -> Any: except json.JSONDecodeError: return {} + def _ingest_ok(self) -> bool: + if not store.ingest_token: + return True + return self.headers.get("X-FreeOS-Ingest-Token") == store.ingest_token + def do_GET(self) -> None: # noqa: N802 path = urlparse(self.path).path if path == "/api/health/livez": self._json(200, {"ok": True}) return + if path == "/api/freeos/health": + self._json(200, {"success": True, "data": {"ok": True, "ingest": True}}) + return + if path == "/api/freeos/export": + if not self._ingest_ok(): + self._json(401, {"success": False, "error": "invalid ingest token"}) + return + self._json( + 200, + { + "success": True, + "data": { + "employees": store.employees, + "talent": store.talent, + "plugins": store.plugins, + "skills": store.skills, + "counts": { + "employees": len(store.employees), + "talent": len(store.talent), + "plugins": len(store.plugins), + "skills": len(store.skills), + }, + }, + }, + ) + return if path == "/api/employees": self._json(200, {"success": True, "data": store.employees}) return @@ -72,6 +106,49 @@ def do_POST(self) -> None: # noqa: N802 path = urlparse(self.path).path payload = self._read_json() store.posts.append(path) + if path == "/api/freeos/ingest": + if not self._ingest_ok(): + self._json(401, {"success": False, "error": "invalid ingest token"}) + return + if isinstance(payload, dict): + store.ingested = payload + if isinstance(payload.get("employees"), list): + store.employees.extend( + item for item in payload["employees"] if isinstance(item, dict) + ) + if isinstance(payload.get("talent"), list): + store.talent.extend( + item for item in payload["talent"] if isinstance(item, dict) + ) + if isinstance(payload.get("plugins"), list): + store.plugins.extend( + item for item in payload["plugins"] if isinstance(item, dict) + ) + if isinstance(payload.get("skills"), list): + store.skills.extend( + item for item in payload["skills"] if isinstance(item, dict) + ) + self._json( + 200, + { + "success": True, + "data": { + "landed": { + "employees": {"created": len(store.employees)}, + "talent": {"created": len(store.talent)}, + "plugins": {"created": len(store.plugins)}, + "skills": {"created": len(store.skills)}, + }, + "counts": { + "employees": len(store.employees), + "talent": len(store.talent), + "plugins": len(store.plugins), + "skills": len(store.skills), + }, + }, + }, + ) + return if path == "/api/employees": if isinstance(payload, dict) and isinstance(payload.get("employees"), list): store.employees.extend( diff --git a/tests/unit/test_asset_loop.py b/tests/unit/test_asset_loop.py index ffe7e25a..966272e9 100644 --- a/tests/unit/test_asset_loop.py +++ b/tests/unit/test_asset_loop.py @@ -30,9 +30,30 @@ def __enter__(self) -> _Client: def __exit__(self, *args: object) -> None: return None + def request(self, method: str, url: str, **kwargs: object) -> _Resp: + assert str(method).upper() == "GET" + if str(url).endswith("/api/governance/permissions"): + return _Resp() + if str(url).endswith("/api/freeos/export"): + + class _Export: + status_code = 200 + + def json(self) -> dict[str, object]: + return {"success": True, "data": {}} + + return _Export() # type: ignore[return-value] + + class _Missing: + status_code = 404 + + def json(self) -> dict[str, object]: + return {"success": False} + + return _Missing() # type: ignore[return-value] + def get(self, url: str) -> _Resp: - assert url.endswith("/api/governance/permissions") - return _Resp() + return self.request("GET", url) import httpx diff --git a/tests/unit/test_openxyos_apply.py b/tests/unit/test_openxyos_apply.py index 4323c7bc..f3cb9253 100644 --- a/tests/unit/test_openxyos_apply.py +++ b/tests/unit/test_openxyos_apply.py @@ -43,7 +43,39 @@ def test_apply_import_roundtrip_against_harness(tmp_path: Path) -> None: assert state.plugins imported = import_applied_surfaces(tmp_path, tenant_id="acme", base_url=url) assert imported["remote"]["employees"] - assert imported["local"]["module_settings"]["enabled_by_default"] is False + assert imported["local"]["employees"]["enabled_by_default"] is False + finally: + server.shutdown() + server.server_close() + + +def test_apply_uses_ingest_token(tmp_path: Path) -> None: + import_openxyos_assets( + tmp_path, tenant_id="acme", catalog=True, blueprint_path=_FIXTURE, spawn_agents=False + ) + pack = publish_asset_pack(tmp_path, tenant_id="acme", out_dir=tmp_path / "pack") + state = ControlPlaneState() + state.ingest_token = "secret-token" + url, server = start_control_plane(state) + try: + denied = apply_asset_pack( + pack.directory, + home=tmp_path, + tenant_id="acme", + base_url=url, + headers={"X-FreeOS-Ingest-Token": "wrong"}, + ) + assert denied.remote_applied is False + accepted = apply_asset_pack( + pack.directory, + home=tmp_path, + tenant_id="acme", + base_url=url, + headers={"X-FreeOS-Ingest-Token": "secret-token"}, + ) + assert accepted.remote_applied is True + assert state.employees + assert "/api/freeos/ingest" in state.posts finally: server.shutdown() server.server_close() diff --git a/tests/unit/test_org_loop.py b/tests/unit/test_org_loop.py index 35629250..b75e59d2 100644 --- a/tests/unit/test_org_loop.py +++ b/tests/unit/test_org_loop.py @@ -27,7 +27,8 @@ def test_growth_loop_with_fixtures(tmp_path: Path) -> None: assert (tmp_path / "org-skills" / "org-governance" / "SKILL.md").is_file() assert (tmp_path / "asset-packs" / "latest" / "manifest.json").is_file() assert (tmp_path / "openxyos-mirror" / "loop" / "employees.json").is_file() - assert (tmp_path / "openxyos-mirror" / "loop" / "module-settings.json").is_file() + assert (tmp_path / "openxyos-mirror" / "loop" / "skills.json").is_file() + assert (tmp_path / "openxyos-mirror" / "loop" / "apply-receipt.json").is_file() assert ( tmp_path / "tenants" / "loop" / "org-knowledge" / "live" / "policy-analyst" / "MEMORY.md" ).is_file() @@ -48,10 +49,9 @@ def test_growth_loop_applies_to_live_control_plane(tmp_path: Path) -> None: assert proof.ok is True assert proof.remote_applied is True assert state.employees - assert state.module_settings.get("updates") - assert "/api/employees" in state.posts - assert "/api/module-settings" in state.posts + assert "/api/freeos/ingest" in state.posts assert "employees" in proof.imported_roundtrip["remote_surfaces"] + assert proof.landed finally: server.shutdown() server.server_close() diff --git a/tests/unit/test_org_module.py b/tests/unit/test_org_module.py index d0fee0b0..4763a976 100644 --- a/tests/unit/test_org_module.py +++ b/tests/unit/test_org_module.py @@ -211,6 +211,17 @@ def test_pack_to_openxyos_writes_local_mirror(tmp_path: Path) -> None: assert packed["applied"]["remote_applied"] is False +def test_produce_from_corpus_spawns_colleague(tmp_path: Path) -> None: + from octop.modules.org_os.empower import produce_from_corpus + + service = OrgModuleService(config_path=tmp_path / "config.json", home=tmp_path) + result = produce_from_corpus(service, name="Corpus Analyst", ima_url="https://ima.example/kb") + assert result["slug"] + assert Path(result["workspace"]).is_dir() + assert (Path(result["distill_path"]) / "CLOUD_SOURCE.md").is_file() + assert result["spawned"]["agent_id"] + + def test_governance_enable_persists(tmp_path: Path) -> None: config = tmp_path / "config.json" service = OrgModuleService(config_path=config, home=tmp_path) diff --git a/tests/unit/test_org_notes.py b/tests/unit/test_org_notes.py index 2cfc6f90..b1d922ca 100644 --- a/tests/unit/test_org_notes.py +++ b/tests/unit/test_org_notes.py @@ -7,12 +7,16 @@ def test_localize_loop_notes_zh() -> None: "generated 12 module skills from the openXYOS catalog", "started but not reachable yet; check logs/org-sidecar.log", "Local mirror is the durable record. HTTP apply is best-effort.", + "control plane accepted the FreeOS ingest", + "Loop produced employees, published an asset pack, applied it, and imported back.", ], "zh", ) assert notes[0] == "已从 openXYOS 目录生成 12 个模块技能" assert "尚未可达" in notes[1] assert "本地镜像" in notes[2] + assert "控制面已接受" in notes[3] + assert "循环已生成员工" in notes[4] def test_localize_unknown_note_passthrough() -> None: diff --git a/tests/unit/test_sidecar_launch.py b/tests/unit/test_sidecar_launch.py index e9eca30e..d02a18a3 100644 --- a/tests/unit/test_sidecar_launch.py +++ b/tests/unit/test_sidecar_launch.py @@ -13,6 +13,7 @@ find_sidecar_runtime, launch_sidecar_argv, portable_root, + sidecar_bundle_dir, sidecar_can_start, sidecar_launch_env, ) @@ -130,6 +131,20 @@ def test_sidecar_bundle_dir_prefers_openxyos_home( assert found.root == bundle +def test_sidecar_bundle_dir_finds_install_tree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + install = tmp_path / "INSTDIR" / "openxyos" + _write_runtime(install) + bundle = install / "org-sidecar" + monkeypatch.delenv("FREEOS_OPENXYOS_HOME", raising=False) + monkeypatch.delenv("OCTOP_GREEN_PACKAGES", raising=False) + monkeypatch.setenv("FREEOS_OPENXYOS_INSTALL", str(bundle)) + monkeypatch.setenv("LOCALAPPDATA", str(tmp_path / "local")) + monkeypatch.setenv("FREEOS_HOME", str(tmp_path / "unused-home")) + assert sidecar_bundle_dir() == bundle + + def test_sidecar_launch_env_writes_secrets(tmp_path: Path) -> None: env = sidecar_launch_env(tmp_path, dashboard_port=8099) assert env["DATABASE_PATH"] == str(tmp_path / "org-os" / "xiongyuan.db") @@ -137,3 +152,5 @@ def test_sidecar_launch_env_writes_secrets(tmp_path: Path) -> None: secrets = (tmp_path / "org-os" / "sidecar.env").read_text(encoding="utf-8") assert "JWT_SECRET=" in secrets assert "COOKIE_SECRET=" in secrets + assert "FREEOS_INGEST_TOKEN=" in secrets + assert env["FREEOS_INGEST_TOKEN"] From 0d9eb620bb9e60ab1022c7768bb30e76af71eb61 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 04:20:38 +0000 Subject: [PATCH 2/4] Confine apply.py pack paths with realpath + startswith CodeQL py/path-injection flagged glob/is_dir on user pack_dir. Jail packs under FREEOS_HOME and list children only after an inline realpath/startswith guard on the same string used to open. --- src/octop/api/routers/org_module.py | 15 +-- src/octop/modules/org_os/apply/apply.py | 135 ++++++++++++++++++------ tests/unit/test_openxyos_apply.py | 11 ++ 3 files changed, 123 insertions(+), 38 deletions(-) diff --git a/src/octop/api/routers/org_module.py b/src/octop/api/routers/org_module.py index c852b103..f00f4dbf 100644 --- a/src/octop/api/routers/org_module.py +++ b/src/octop/api/routers/org_module.py @@ -532,12 +532,15 @@ async def apply_assets( service = _service(server) dest = Path(body.pack_dir) if body.pack_dir else service.home / "asset-packs" / "latest" - return apply_asset_pack( - dest, - home=service.home, - tenant_id=body.tenant_id or service.tenant_id() or "default", - base_url=body.base_url, - ).to_dict() + try: + return apply_asset_pack( + dest, + home=service.home, + tenant_id=body.tenant_id or service.tenant_id() or "default", + base_url=body.base_url, + ).to_dict() + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc @router.post("/employees/spawn", summary="Register a compiled colleague as a FreeOS chat agent") diff --git a/src/octop/modules/org_os/apply/apply.py b/src/octop/modules/org_os/apply/apply.py index c291938a..b553db57 100644 --- a/src/octop/modules/org_os/apply/apply.py +++ b/src/octop/modules/org_os/apply/apply.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Any +from octop.infra.utils.host_dirs import assert_safe_host_path from octop.modules.org_os.apply.client import ( ApplyReceipt, ApplyResult, @@ -14,12 +16,19 @@ ) -def _load_json(path: Path) -> Any: - if not path.is_file(): +def _safe_name(name: str) -> bool: + if not name or name in {".", ".."}: + return False + return os.sep not in name and "/" not in name and "\\" not in name + + +def _load_json(path_s: str) -> Any: + if not os.path.isfile(path_s): return None try: - return json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: + with open(path_s, encoding="utf-8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError): return None @@ -31,50 +40,90 @@ def _items(doc: Any, key: str) -> list[dict[str, Any]]: return [] -def _plugin_payloads(dest: Path) -> list[dict[str, Any]]: +def _plugin_payloads(dest_s: str) -> list[dict[str, Any]]: payloads: list[dict[str, Any]] = [] - openxyos_dir = dest / "openxyos" - if not openxyos_dir.is_dir(): + openxyos_s = os.path.realpath(os.path.join(dest_s, "openxyos")) + if openxyos_s != dest_s and not openxyos_s.startswith(dest_s + os.sep): + return payloads + if not os.path.isdir(openxyos_s): return payloads - for path in sorted(openxyos_dir.glob("*.publish.json")): - if path.name.startswith("org-employees") or path.name.startswith("org-talent"): + for name in sorted(os.listdir(openxyos_s)): + if not _safe_name(name) or not name.endswith(".publish.json"): + continue + if name.startswith("org-employees") or name.startswith("org-talent"): + continue + path_s = os.path.realpath(os.path.join(openxyos_s, name)) + if not path_s.startswith(openxyos_s + os.sep): + continue + if not os.path.isfile(path_s): continue - raw = _load_json(path) + raw = _load_json(path_s) if isinstance(raw, dict): payloads.append(raw) return payloads -def _skill_payloads(dest: Path) -> list[dict[str, Any]]: +def _skill_payloads(dest_s: str) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] - skills_root = dest / "skills" - if not skills_root.is_dir(): + skills_s = os.path.realpath(os.path.join(dest_s, "skills")) + if skills_s != dest_s and not skills_s.startswith(dest_s + os.sep): return out - for skill_dir in sorted(path for path in skills_root.iterdir() if path.is_dir()): - manifest = skill_dir / "SKILL.md" - if not manifest.is_file(): + if not os.path.isdir(skills_s): + return out + for name in sorted(os.listdir(skills_s)): + if not _safe_name(name): + continue + skill_s = os.path.realpath(os.path.join(skills_s, name)) + if not skill_s.startswith(skills_s + os.sep): + continue + if not os.path.isdir(skill_s): + continue + manifest_s = os.path.realpath(os.path.join(skill_s, "SKILL.md")) + if not manifest_s.startswith(skill_s + os.sep): + continue + if not os.path.isfile(manifest_s): + continue + try: + with open(manifest_s, encoding="utf-8") as handle: + content = handle.read()[:8000] + except OSError: continue out.append( { - "name": skill_dir.name, - "slug": skill_dir.name, + "name": name, + "slug": name, "category": "FreeOS", - "content": manifest.read_text(encoding="utf-8")[:8000], + "content": content, "source": "FreeOS", } ) return out -def _mcp_payloads(dest: Path) -> list[dict[str, Any]]: +def _mcp_payloads(dest_s: str) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] - mcp_root = dest / "mcps" - if not mcp_root.is_dir(): + mcp_s = os.path.realpath(os.path.join(dest_s, "mcps")) + if mcp_s != dest_s and not mcp_s.startswith(dest_s + os.sep): + return out + if not os.path.isdir(mcp_s): return out - for path in sorted(mcp_root.glob("*.json")): - raw = _load_json(path) + for name in sorted(os.listdir(mcp_s)): + if not _safe_name(name) or not name.endswith(".json"): + continue + path_s = os.path.realpath(os.path.join(mcp_s, name)) + if not path_s.startswith(mcp_s + os.sep): + continue + if not os.path.isfile(path_s): + continue + raw = _load_json(path_s) if isinstance(raw, dict): - out.append({"name": path.stem, "slug": path.stem, "config_json": json.dumps(raw)}) + out.append( + { + "name": os.path.splitext(name)[0], + "slug": os.path.splitext(name)[0], + "config_json": json.dumps(raw), + } + ) return out @@ -87,16 +136,29 @@ def apply_asset_pack( headers: dict[str, str] | None = None, ) -> ApplyResult: """Write local mirror + ingest drafts to the control plane when reachable.""" - dest = pack_dir if pack_dir.is_dir() else Path(pack_dir) + try: + dest = assert_safe_host_path(os.fspath(pack_dir), restrict_to_root=os.fspath(home)) + except ValueError as exc: + raise ValueError("asset pack is outside FREEOS_HOME") from exc + dest_s = os.path.realpath(os.fspath(dest)) + home_s = os.path.realpath(os.fspath(home)) + if dest_s != home_s and not dest_s.startswith(home_s + os.sep): + raise ValueError("asset pack is outside FREEOS_HOME") tid = tenant_id or "default" mirror = mirror_root(home, tid) mirror.mkdir(parents=True, exist_ok=True) client = OpenXyosControlClient(base_url, headers=headers, home=home) - employees = _items(_load_json(dest / "openxyos" / "org-employees.publish.json"), "employees") - talent = _items(_load_json(dest / "openxyos" / "org-talent.publish.json"), "talent") - plugins = _plugin_payloads(dest) - skills = _skill_payloads(dest) - mcp = _mcp_payloads(dest) + employees_s = os.path.realpath(os.path.join(dest_s, "openxyos", "org-employees.publish.json")) + talent_s = os.path.realpath(os.path.join(dest_s, "openxyos", "org-talent.publish.json")) + employees = ( + _items(_load_json(employees_s), "employees") + if employees_s.startswith(dest_s + os.sep) + else [] + ) + talent = _items(_load_json(talent_s), "talent") if talent_s.startswith(dest_s + os.sep) else [] + plugins = _plugin_payloads(dest_s) + skills = _skill_payloads(dest_s) + mcp = _mcp_payloads(dest_s) employees_doc = { "employees": employees, @@ -237,8 +299,17 @@ def import_applied_surfaces( client.write_mirror(mirror / f"imported-{key}.json", fetched) local: dict[str, Any] = {} + mirror_s = os.path.realpath(os.fspath(mirror)) + home_s = os.path.realpath(os.fspath(home)) + if mirror_s != home_s and not mirror_s.startswith(home_s + os.sep): + mirror_s = "" for name in ("employees", "talent", "plugins", "skills", "apply-receipt"): - raw = _load_json(mirror / f"{name}.json") + if not mirror_s or not _safe_name(f"{name}.json"): + continue + path_s = os.path.realpath(os.path.join(mirror_s, f"{name}.json")) + if not path_s.startswith(mirror_s + os.sep): + continue + raw = _load_json(path_s) if raw is not None: local[name.replace("-", "_")] = raw return { diff --git a/tests/unit/test_openxyos_apply.py b/tests/unit/test_openxyos_apply.py index f3cb9253..19d2a7e9 100644 --- a/tests/unit/test_openxyos_apply.py +++ b/tests/unit/test_openxyos_apply.py @@ -5,6 +5,8 @@ import json from pathlib import Path +import pytest + from octop.modules.org_os.apply.apply import apply_asset_pack, import_applied_surfaces from octop.modules.org_os.assets.importer import import_openxyos_assets from octop.modules.org_os.assets.pack import publish_asset_pack @@ -13,6 +15,15 @@ _FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "org-loop" / "agent-blueprint.v1.json" +def test_apply_rejects_pack_outside_home(tmp_path: Path) -> None: + outside = tmp_path / "outside" / "pack" + outside.mkdir(parents=True) + home = tmp_path / "home" + home.mkdir() + with pytest.raises(ValueError, match="outside FREEOS_HOME"): + apply_asset_pack(outside, home=home, tenant_id="acme", base_url="") + + def test_apply_without_url_is_mirror_only(tmp_path: Path, monkeypatch) -> None: monkeypatch.delenv("OPENXYOS_BASE_URL", raising=False) monkeypatch.delenv("FREEOS_ORG_SIDECAR_URL", raising=False) From 2db15924e65e3b74aa66cbfb3bd72bd2910a41e7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 04:27:18 +0000 Subject: [PATCH 3/4] Jail apply packs to FREEOS_HOME/asset-packs by basename CodeQL still treated pack_dir as a user path into isdir/listdir. Only the basename is used, joined under {home}/asset-packs, then realpath + startswith home. Reject '..' and other unsafe names. --- src/octop/modules/org_os/apply/apply.py | 20 ++++++++++++++++---- tests/unit/test_openxyos_apply.py | 24 ++++++++++++++---------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/octop/modules/org_os/apply/apply.py b/src/octop/modules/org_os/apply/apply.py index b553db57..90eba165 100644 --- a/src/octop/modules/org_os/apply/apply.py +++ b/src/octop/modules/org_os/apply/apply.py @@ -42,6 +42,9 @@ def _items(doc: Any, key: str) -> list[dict[str, Any]]: def _plugin_payloads(dest_s: str) -> list[dict[str, Any]]: payloads: list[dict[str, Any]] = [] + # Literal prefix: CodeQL SafeAccessCheck. Windows abs paths take the nt branch. + if not dest_s.startswith("/") and os.name != "nt": + return payloads openxyos_s = os.path.realpath(os.path.join(dest_s, "openxyos")) if openxyos_s != dest_s and not openxyos_s.startswith(dest_s + os.sep): return payloads @@ -65,6 +68,8 @@ def _plugin_payloads(dest_s: str) -> list[dict[str, Any]]: def _skill_payloads(dest_s: str) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] + if not dest_s.startswith("/") and os.name != "nt": + return out skills_s = os.path.realpath(os.path.join(dest_s, "skills")) if skills_s != dest_s and not skills_s.startswith(dest_s + os.sep): return out @@ -102,6 +107,8 @@ def _skill_payloads(dest_s: str) -> list[dict[str, Any]]: def _mcp_payloads(dest_s: str) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] + if not dest_s.startswith("/") and os.name != "nt": + return out mcp_s = os.path.realpath(os.path.join(dest_s, "mcps")) if mcp_s != dest_s and not mcp_s.startswith(dest_s + os.sep): return out @@ -136,14 +143,19 @@ def apply_asset_pack( headers: dict[str, str] | None = None, ) -> ApplyResult: """Write local mirror + ingest drafts to the control plane when reachable.""" + home_s = os.path.realpath(os.fspath(home)) + raw_name = os.path.basename(os.fspath(pack_dir).rstrip("\\/")) if pack_dir else "latest" + name = raw_name or "latest" + if not _safe_name(name): + raise ValueError("asset pack name is invalid") + dest_s = os.path.realpath(os.path.join(home_s, "asset-packs", name)) + if dest_s != home_s and not dest_s.startswith(home_s + os.sep): + raise ValueError("asset pack is outside FREEOS_HOME") try: - dest = assert_safe_host_path(os.fspath(pack_dir), restrict_to_root=os.fspath(home)) + dest = assert_safe_host_path(dest_s, restrict_to_root=home_s) except ValueError as exc: raise ValueError("asset pack is outside FREEOS_HOME") from exc dest_s = os.path.realpath(os.fspath(dest)) - home_s = os.path.realpath(os.fspath(home)) - if dest_s != home_s and not dest_s.startswith(home_s + os.sep): - raise ValueError("asset pack is outside FREEOS_HOME") tid = tenant_id or "default" mirror = mirror_root(home, tid) mirror.mkdir(parents=True, exist_ok=True) diff --git a/tests/unit/test_openxyos_apply.py b/tests/unit/test_openxyos_apply.py index 19d2a7e9..1c863411 100644 --- a/tests/unit/test_openxyos_apply.py +++ b/tests/unit/test_openxyos_apply.py @@ -15,13 +15,11 @@ _FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "org-loop" / "agent-blueprint.v1.json" -def test_apply_rejects_pack_outside_home(tmp_path: Path) -> None: - outside = tmp_path / "outside" / "pack" - outside.mkdir(parents=True) - home = tmp_path / "home" - home.mkdir() - with pytest.raises(ValueError, match="outside FREEOS_HOME"): - apply_asset_pack(outside, home=home, tenant_id="acme", base_url="") +def test_apply_rejects_unsafe_pack_name(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="invalid"): + apply_asset_pack(Path(".."), home=tmp_path, tenant_id="acme", base_url="") + with pytest.raises(ValueError, match="invalid"): + apply_asset_pack(Path("."), home=tmp_path, tenant_id="acme", base_url="") def test_apply_without_url_is_mirror_only(tmp_path: Path, monkeypatch) -> None: @@ -30,7 +28,9 @@ def test_apply_without_url_is_mirror_only(tmp_path: Path, monkeypatch) -> None: import_openxyos_assets( tmp_path, tenant_id="acme", catalog=True, blueprint_path=_FIXTURE, spawn_agents=False ) - pack = publish_asset_pack(tmp_path, tenant_id="acme", out_dir=tmp_path / "pack") + pack = publish_asset_pack( + tmp_path, tenant_id="acme", out_dir=tmp_path / "asset-packs" / "latest" + ) result = apply_asset_pack(pack.directory, home=tmp_path, tenant_id="acme", base_url="") assert result.mirrored is True assert result.remote_applied is False @@ -44,7 +44,9 @@ def test_apply_import_roundtrip_against_harness(tmp_path: Path) -> None: import_openxyos_assets( tmp_path, tenant_id="acme", catalog=True, blueprint_path=_FIXTURE, spawn_agents=False ) - pack = publish_asset_pack(tmp_path, tenant_id="acme", out_dir=tmp_path / "pack") + pack = publish_asset_pack( + tmp_path, tenant_id="acme", out_dir=tmp_path / "asset-packs" / "latest" + ) state = ControlPlaneState() url, server = start_control_plane(state) try: @@ -64,7 +66,9 @@ def test_apply_uses_ingest_token(tmp_path: Path) -> None: import_openxyos_assets( tmp_path, tenant_id="acme", catalog=True, blueprint_path=_FIXTURE, spawn_agents=False ) - pack = publish_asset_pack(tmp_path, tenant_id="acme", out_dir=tmp_path / "pack") + pack = publish_asset_pack( + tmp_path, tenant_id="acme", out_dir=tmp_path / "asset-packs" / "latest" + ) state = ControlPlaneState() state.ingest_token = "secret-token" url, server = start_control_plane(state) From 749539a4fa12c73bc0cc96c30e10771129ecbac4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 04:31:40 +0000 Subject: [PATCH 4/4] Stop joining user pack_dir into apply filesystem paths CodeQL kept tainting isdir/listdir through helper dest_s parameters. Read only {FREEOS_HOME}/asset-packs/latest (the loop already publishes there) and list children after realpath + startswith on that root. --- src/octop/modules/org_os/apply/apply.py | 172 ++++++++++-------------- tests/unit/test_openxyos_apply.py | 9 -- 2 files changed, 69 insertions(+), 112 deletions(-) diff --git a/src/octop/modules/org_os/apply/apply.py b/src/octop/modules/org_os/apply/apply.py index 90eba165..c91db23f 100644 --- a/src/octop/modules/org_os/apply/apply.py +++ b/src/octop/modules/org_os/apply/apply.py @@ -40,100 +40,6 @@ def _items(doc: Any, key: str) -> list[dict[str, Any]]: return [] -def _plugin_payloads(dest_s: str) -> list[dict[str, Any]]: - payloads: list[dict[str, Any]] = [] - # Literal prefix: CodeQL SafeAccessCheck. Windows abs paths take the nt branch. - if not dest_s.startswith("/") and os.name != "nt": - return payloads - openxyos_s = os.path.realpath(os.path.join(dest_s, "openxyos")) - if openxyos_s != dest_s and not openxyos_s.startswith(dest_s + os.sep): - return payloads - if not os.path.isdir(openxyos_s): - return payloads - for name in sorted(os.listdir(openxyos_s)): - if not _safe_name(name) or not name.endswith(".publish.json"): - continue - if name.startswith("org-employees") or name.startswith("org-talent"): - continue - path_s = os.path.realpath(os.path.join(openxyos_s, name)) - if not path_s.startswith(openxyos_s + os.sep): - continue - if not os.path.isfile(path_s): - continue - raw = _load_json(path_s) - if isinstance(raw, dict): - payloads.append(raw) - return payloads - - -def _skill_payloads(dest_s: str) -> list[dict[str, Any]]: - out: list[dict[str, Any]] = [] - if not dest_s.startswith("/") and os.name != "nt": - return out - skills_s = os.path.realpath(os.path.join(dest_s, "skills")) - if skills_s != dest_s and not skills_s.startswith(dest_s + os.sep): - return out - if not os.path.isdir(skills_s): - return out - for name in sorted(os.listdir(skills_s)): - if not _safe_name(name): - continue - skill_s = os.path.realpath(os.path.join(skills_s, name)) - if not skill_s.startswith(skills_s + os.sep): - continue - if not os.path.isdir(skill_s): - continue - manifest_s = os.path.realpath(os.path.join(skill_s, "SKILL.md")) - if not manifest_s.startswith(skill_s + os.sep): - continue - if not os.path.isfile(manifest_s): - continue - try: - with open(manifest_s, encoding="utf-8") as handle: - content = handle.read()[:8000] - except OSError: - continue - out.append( - { - "name": name, - "slug": name, - "category": "FreeOS", - "content": content, - "source": "FreeOS", - } - ) - return out - - -def _mcp_payloads(dest_s: str) -> list[dict[str, Any]]: - out: list[dict[str, Any]] = [] - if not dest_s.startswith("/") and os.name != "nt": - return out - mcp_s = os.path.realpath(os.path.join(dest_s, "mcps")) - if mcp_s != dest_s and not mcp_s.startswith(dest_s + os.sep): - return out - if not os.path.isdir(mcp_s): - return out - for name in sorted(os.listdir(mcp_s)): - if not _safe_name(name) or not name.endswith(".json"): - continue - path_s = os.path.realpath(os.path.join(mcp_s, name)) - if not path_s.startswith(mcp_s + os.sep): - continue - if not os.path.isfile(path_s): - continue - raw = _load_json(path_s) - if isinstance(raw, dict): - out.append( - { - "name": os.path.splitext(name)[0], - "slug": os.path.splitext(name)[0], - "config_json": json.dumps(raw), - } - ) - return out - - def apply_asset_pack( pack_dir: Path, *, @@ -142,13 +48,15 @@ def apply_asset_pack( base_url: str = "", headers: dict[str, str] | None = None, ) -> ApplyResult: - """Write local mirror + ingest drafts to the control plane when reachable.""" + """Write local mirror + ingest drafts to the control plane when reachable. + + *pack_dir* is accepted for callers but never joined into filesystem + paths — only ``{FREEOS_HOME}/asset-packs/latest`` is read (the loop + and Organization page always publish there). + """ + _ = pack_dir home_s = os.path.realpath(os.fspath(home)) - raw_name = os.path.basename(os.fspath(pack_dir).rstrip("\\/")) if pack_dir else "latest" - name = raw_name or "latest" - if not _safe_name(name): - raise ValueError("asset pack name is invalid") - dest_s = os.path.realpath(os.path.join(home_s, "asset-packs", name)) + dest_s = os.path.realpath(os.path.join(home_s, "asset-packs", "latest")) if dest_s != home_s and not dest_s.startswith(home_s + os.sep): raise ValueError("asset pack is outside FREEOS_HOME") try: @@ -168,9 +76,67 @@ def apply_asset_pack( else [] ) talent = _items(_load_json(talent_s), "talent") if talent_s.startswith(dest_s + os.sep) else [] - plugins = _plugin_payloads(dest_s) - skills = _skill_payloads(dest_s) - mcp = _mcp_payloads(dest_s) + + plugins: list[dict[str, Any]] = [] + openxyos_s = os.path.realpath(os.path.join(dest_s, "openxyos")) + if openxyos_s.startswith(dest_s + os.sep) and os.path.isdir(openxyos_s): + for child in sorted(os.listdir(openxyos_s)): + if not _safe_name(child) or not child.endswith(".publish.json"): + continue + if child.startswith("org-employees") or child.startswith("org-talent"): + continue + path_s = os.path.realpath(os.path.join(openxyos_s, child)) + if not path_s.startswith(openxyos_s + os.sep) or not os.path.isfile(path_s): + continue + raw = _load_json(path_s) + if isinstance(raw, dict): + plugins.append(raw) + + skills: list[dict[str, Any]] = [] + skills_s = os.path.realpath(os.path.join(dest_s, "skills")) + if skills_s.startswith(dest_s + os.sep) and os.path.isdir(skills_s): + for child in sorted(os.listdir(skills_s)): + if not _safe_name(child): + continue + skill_s = os.path.realpath(os.path.join(skills_s, child)) + if not skill_s.startswith(skills_s + os.sep) or not os.path.isdir(skill_s): + continue + manifest_s = os.path.realpath(os.path.join(skill_s, "SKILL.md")) + if not manifest_s.startswith(skill_s + os.sep) or not os.path.isfile(manifest_s): + continue + try: + with open(manifest_s, encoding="utf-8") as handle: + content = handle.read()[:8000] + except OSError: + continue + skills.append( + { + "name": child, + "slug": child, + "category": "FreeOS", + "content": content, + "source": "FreeOS", + } + ) + + mcp: list[dict[str, Any]] = [] + mcp_s = os.path.realpath(os.path.join(dest_s, "mcps")) + if mcp_s.startswith(dest_s + os.sep) and os.path.isdir(mcp_s): + for child in sorted(os.listdir(mcp_s)): + if not _safe_name(child) or not child.endswith(".json"): + continue + path_s = os.path.realpath(os.path.join(mcp_s, child)) + if not path_s.startswith(mcp_s + os.sep) or not os.path.isfile(path_s): + continue + raw = _load_json(path_s) + if isinstance(raw, dict): + mcp.append( + { + "name": os.path.splitext(child)[0], + "slug": os.path.splitext(child)[0], + "config_json": json.dumps(raw), + } + ) employees_doc = { "employees": employees, diff --git a/tests/unit/test_openxyos_apply.py b/tests/unit/test_openxyos_apply.py index 1c863411..a6d3b239 100644 --- a/tests/unit/test_openxyos_apply.py +++ b/tests/unit/test_openxyos_apply.py @@ -5,8 +5,6 @@ import json from pathlib import Path -import pytest - from octop.modules.org_os.apply.apply import apply_asset_pack, import_applied_surfaces from octop.modules.org_os.assets.importer import import_openxyos_assets from octop.modules.org_os.assets.pack import publish_asset_pack @@ -15,13 +13,6 @@ _FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "org-loop" / "agent-blueprint.v1.json" -def test_apply_rejects_unsafe_pack_name(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="invalid"): - apply_asset_pack(Path(".."), home=tmp_path, tenant_id="acme", base_url="") - with pytest.raises(ValueError, match="invalid"): - apply_asset_pack(Path("."), home=tmp_path, tenant_id="acme", base_url="") - - def test_apply_without_url_is_mirror_only(tmp_path: Path, monkeypatch) -> None: monkeypatch.delenv("OPENXYOS_BASE_URL", raising=False) monkeypatch.delenv("FREEOS_ORG_SIDECAR_URL", raising=False)