From 4058e52991210668f7f6d76d54ccaa2fca37a53d Mon Sep 17 00:00:00 2001 From: Jon Wiggins Date: Sun, 4 Oct 2026 01:17:02 -0600 Subject: [PATCH 1/3] feat(config): manifests, a mounted directory the API applies, export/apply in the CLI and UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Config as code (docs/config-as-code.md): every Job, scheduled Task, persistent agent, prompt, repo, MCP server, skill and connection can be a YAML manifest (apiVersion optio/v1, kind, metadata.name = identity, spec), with references by name and secrets by name or ${{SECRET_NAME}}, never values. - A cluster mounts a directory of them (OPTIO_CONFIG_DIR; Helm configAsCode.*, plus api.extraVolumes / extraVolumeMounts) that a worker reads every interval and applies to one workspace: create / adopt / update (a managed row edited in the UI is put back and reported as reverted) / replace / prune, each manifest its own error. Tables config_sources, config_objects. - One handler per kind (services/config/kinds), the engine in apply.ts, name resolution in context.ts, reading a directory in files.ts; dry runs plan without writing and register would-be rows so later manifests resolve. - Managed rows carry managedBy in every list and detail response; the web shows a Managed chip next to the Private one, a banner on detail and edit pages with the YAML and Detach, Download YAML, and Settings → Config as code (directory, last sync with per-file errors, Sync now, Preview, Export). - GET /api/config/schema.json (public JSON Schema), status, source/sync, apply, export[.yaml], objects/:id/detach. CLI: optio export [-o DIR], optio apply -f, optio diff -f, optio schema. - updateWork saves persistent agents too; createWork can skip a Job's first run. Tests: unit (schema, inlining, compare, files, when mapping, CLI reader), integration (apply: every kind, drift, replace, prune, adopt, errors, detach, export round trip), pipeline e2e (OPTIO_CONFIG_DIR at boot over HTTP), Playwright (Settings card, Managed chip). Swift/Kotlin types regenerated. --- CLAUDE.md | 2 + README.md | 2 +- .../dev/optio/core/model/SharedTypes.kt | 34 ++ apps/api/e2e/config-as-code.e2e.test.ts | 213 ++++++++ apps/api/package.json | 4 +- .../migrations/1791970000_config_as_code.sql | 49 ++ apps/api/src/db/migrations/meta/_journal.json | 7 + apps/api/src/db/schema.ts | 67 ++- apps/api/src/index.ts | 5 + apps/api/src/plugins/auth.ts | 1 + apps/api/src/routes/config.ts | 247 +++++++++ apps/api/src/routes/connections.ts | 7 +- apps/api/src/routes/installed-skills.ts | 3 +- apps/api/src/routes/mcp-servers.ts | 5 +- apps/api/src/routes/persistent-agents.ts | 5 +- apps/api/src/routes/prompt-templates.ts | 3 +- apps/api/src/routes/repos.ts | 5 +- apps/api/src/routes/skills.ts | 3 +- apps/api/src/routes/task-configs.ts | 3 +- apps/api/src/routes/workflows.ts | 3 +- apps/api/src/schemas/config.test.ts | 108 ++++ apps/api/src/schemas/config.ts | 511 ++++++++++++++++++ apps/api/src/schemas/work.ts | 2 + apps/api/src/server.ts | 2 + apps/api/src/services/config/apply.ts | 405 ++++++++++++++ apps/api/src/services/config/compare.test.ts | 46 ++ apps/api/src/services/config/compare.ts | 59 ++ .../services/config/config-apply.int.test.ts | 434 +++++++++++++++ apps/api/src/services/config/context.ts | 203 +++++++ apps/api/src/services/config/env.ts | 30 + apps/api/src/services/config/export.ts | 67 +++ apps/api/src/services/config/files.test.ts | 106 ++++ apps/api/src/services/config/files.ts | 141 +++++ .../src/services/config/kinds/connection.ts | 242 +++++++++ apps/api/src/services/config/kinds/index.ts | 65 +++ .../src/services/config/kinds/mcp-server.ts | 177 ++++++ apps/api/src/services/config/kinds/prompt.ts | 139 +++++ apps/api/src/services/config/kinds/repo.ts | 138 +++++ apps/api/src/services/config/kinds/skill.ts | 314 +++++++++++ .../services/config/kinds/work-when.test.ts | 60 ++ .../src/services/config/kinds/work-when.ts | 60 ++ apps/api/src/services/config/kinds/work.ts | 484 +++++++++++++++++ apps/api/src/services/config/managed.ts | 91 ++++ apps/api/src/services/config/source.ts | 213 ++++++++ .../src/services/persistent-agent-service.ts | 6 +- apps/api/src/services/work-service.ts | 5 +- apps/api/src/services/work-write-service.ts | 168 ++++-- apps/api/src/workers/config-sync-worker.ts | 49 ++ apps/cli/README.md | 16 + apps/cli/package.json | 3 +- apps/cli/src/__tests__/manifests-read.test.ts | 49 ++ apps/cli/src/commands/apply.ts | 105 ++++ apps/cli/src/commands/export.ts | 88 +++ apps/cli/src/commands/schema.ts | 16 + apps/cli/src/manifests/read.ts | 93 ++++ apps/cli/src/program.ts | 8 + apps/ios/Optio/Generated/SharedTypes.swift | 69 +++ apps/web/e2e/config-as-code.spec.ts | 59 ++ apps/web/e2e/launch-stack.ts | 24 +- apps/web/src/app/agents/[id]/page.tsx | 8 + apps/web/src/app/connections/page.tsx | 2 + apps/web/src/app/jobs/[id]/page.tsx | 12 + apps/web/src/app/repos/[id]/page.tsx | 20 +- apps/web/src/app/repos/page.tsx | 2 + apps/web/src/app/settings/page.tsx | 28 +- .../web/src/app/tasks/scheduled/[id]/page.tsx | 14 +- apps/web/src/app/templates/page.tsx | 4 + .../components/settings/config-as-code.tsx | 231 ++++++++ apps/web/src/components/ui/README.md | 2 + apps/web/src/components/ui/managed-banner.tsx | 114 ++++ apps/web/src/components/ui/managed-chip.tsx | 38 ++ apps/web/src/components/work-form/load.ts | 4 + .../src/components/work-form/work-form.tsx | 5 + apps/web/src/components/work-row.tsx | 2 + apps/web/src/lib/api-client.ts | 31 ++ docs/config-as-code.md | 183 +++++++ docs/plans/config-as-code.md | 268 +++++++++ helm/optio/templates/NOTES.txt | 17 + helm/optio/templates/api-deployment.yaml | 16 + .../templates/config-manifests-configmap.yaml | 17 + helm/optio/templates/secrets.yaml | 9 + helm/optio/values.yaml | 42 ++ packages/shared/src/config/inline.test.ts | 108 ++++ packages/shared/src/config/inline.ts | 102 ++++ packages/shared/src/config/manifest.ts | 313 +++++++++++ packages/shared/src/config/stable.ts | 17 + packages/shared/src/index.ts | 4 + packages/shared/src/types/config.ts | 16 + packages/shared/src/types/connection.ts | 3 + packages/shared/src/types/mcp.ts | 7 + packages/shared/src/types/persistent-agent.ts | 3 + packages/shared/src/work/feed.ts | 3 + pnpm-lock.yaml | 9 + 93 files changed, 7105 insertions(+), 72 deletions(-) create mode 100644 apps/api/e2e/config-as-code.e2e.test.ts create mode 100644 apps/api/src/db/migrations/1791970000_config_as_code.sql create mode 100644 apps/api/src/routes/config.ts create mode 100644 apps/api/src/schemas/config.test.ts create mode 100644 apps/api/src/schemas/config.ts create mode 100644 apps/api/src/services/config/apply.ts create mode 100644 apps/api/src/services/config/compare.test.ts create mode 100644 apps/api/src/services/config/compare.ts create mode 100644 apps/api/src/services/config/config-apply.int.test.ts create mode 100644 apps/api/src/services/config/context.ts create mode 100644 apps/api/src/services/config/env.ts create mode 100644 apps/api/src/services/config/export.ts create mode 100644 apps/api/src/services/config/files.test.ts create mode 100644 apps/api/src/services/config/files.ts create mode 100644 apps/api/src/services/config/kinds/connection.ts create mode 100644 apps/api/src/services/config/kinds/index.ts create mode 100644 apps/api/src/services/config/kinds/mcp-server.ts create mode 100644 apps/api/src/services/config/kinds/prompt.ts create mode 100644 apps/api/src/services/config/kinds/repo.ts create mode 100644 apps/api/src/services/config/kinds/skill.ts create mode 100644 apps/api/src/services/config/kinds/work-when.test.ts create mode 100644 apps/api/src/services/config/kinds/work-when.ts create mode 100644 apps/api/src/services/config/kinds/work.ts create mode 100644 apps/api/src/services/config/managed.ts create mode 100644 apps/api/src/services/config/source.ts create mode 100644 apps/api/src/workers/config-sync-worker.ts create mode 100644 apps/cli/src/__tests__/manifests-read.test.ts create mode 100644 apps/cli/src/commands/apply.ts create mode 100644 apps/cli/src/commands/export.ts create mode 100644 apps/cli/src/commands/schema.ts create mode 100644 apps/cli/src/manifests/read.ts create mode 100644 apps/web/e2e/config-as-code.spec.ts create mode 100644 apps/web/src/components/settings/config-as-code.tsx create mode 100644 apps/web/src/components/ui/managed-banner.tsx create mode 100644 apps/web/src/components/ui/managed-chip.tsx create mode 100644 docs/config-as-code.md create mode 100644 docs/plans/config-as-code.md create mode 100644 helm/optio/templates/config-manifests-configmap.yaml create mode 100644 packages/shared/src/config/inline.test.ts create mode 100644 packages/shared/src/config/inline.ts create mode 100644 packages/shared/src/config/manifest.ts create mode 100644 packages/shared/src/config/stable.ts create mode 100644 packages/shared/src/types/config.ts diff --git a/CLAUDE.md b/CLAUDE.md index fe75e29fb..70f4f6fe7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -167,6 +167,7 @@ These are well-documented in code; read the relevant service files for details: - **Reconciliation control plane** (`workers/reconcile-worker.ts`, `services/reconcile-{snapshot,executor,queue}.ts`, `packages/shared/src/reconcile/`): K8s-style reconciler with four `RunKind`s — `repo` (Repo Task runs in `tasks`), `standalone` (Job runs: `tasks` rows with `kind = 'standalone'`, through the `workflow_runs` view), `pr-review` (external PR reviews), and `persistent-agent` (Persistent Agents in `persistent_agents`). Pure decision functions consume a frozen `WorldSnapshot` and return a typed `Action`; the executor applies it under CAS so concurrent passes can't trample each other. Producers — `taskService.transitionTask`, `workflow-service`'s `transitionWorkflowRunCas`, `pr-watcher` poll cycle, `repo-cleanup` pod-health detection, `wakeAgent()` (PA inbox + trigger dispatch) — wake the reconciler via `enqueueReconcile`. Periodic resync (`OPTIO_RECONCILE_RESYNC_INTERVAL`, 5 min) catches anything missed. The reconciler owns: PR-driven transitions (auto-merge, complete-on-merge, fail-on-close), auto-resume on CI/conflict/review (capped by `OPTIO_MAX_AUTO_RESUMES`), review launch, stall + pod-death detection, control-intent (cancel/retry/resume/restart), and the Persistent Agent turn cycle. Schema: `control_intent`, `reconcile_backoff_until`, `reconcile_attempts` columns on `tasks` (every run) and `persistent_agents`. See `docs/reconciliation.md` - **Optio Local** (`local-host-service.ts`, `local-terminal-service.ts`, `local-blueprint-service.ts`, `local-relay.ts`, ws in `ws/local-daemon.ts` + `ws/local-terminal-stream.ts`, routes in `routes/local.ts`, daemon in `apps/cli/src/local/`): terminals on the **user's own machine**, managed from the web UI at `/local`. The CLI daemon (`optio local up`) makes one outbound WS to `/ws/local/daemon` (PAT auth), advertises a dir allowlist (kept in the machine's `local.json`; the Machines page and the New work form add / remove dirs through the daemon — a `dirs` frame, `local-dirs-service.ts` — unless it runs with `--no-remote-dirs`), and runs node-pty PTYs; the in-process relay (`local-relay.ts`, single API replica) routes frames to browser viewers on `/ws/local/terminals/:id/stream` (binary = terminal bytes, JSON = control); which viewer's screen the PTY is sized for is decided there too (`local-grid.ts`: the screen in use, from the viewers' `view` reports). Attention detection is daemon-side and layered: Claude Code hooks (Stop/Notification/UserPromptSubmit via a localhost hook server) → OSC-aware bell scanner → silence heuristic; persisted as `attention_state` (`working`/`needs_you`/`idle`) driving the cockpit's "needs you" queue. **Local Automations** (`work_definitions` rows with `kind = 'local-blueprint'`, "blueprints" in code) are agent / terminal specs spawned by `workflow_triggers` with `target_type="local_blueprint"`: any of the seven trigger types incl. the events `github` / `slack` / `linear` (signed ingress in `routes/event-ingress.ts` + the existing GitHub receiver; matching in `event-trigger-service.ts`, the same path that fires pod-side Jobs and scheduled Tasks). `sessionMode` picks `interactive` (agent halts at its prompt for chat) or `headless` (`claude -p` etc., exits when done); the daemon reports the agent's `session_id` so an exited run can be resumed (`POST /api/local/terminals/:id/resume`). Trigger params reach a command through `renderCommandTemplate` (shell variables, never spliced in) — payloads can never inject commands. Hosts are per-user (never workspace compute); local agents use the machine's own CLI auth (no server secrets ship to laptops). The daemon launches Claude Code with `--permission-mode auto` unless the spawn's `permissionMode` says otherwise (a headless `claude -p` would otherwise deny everything that needs approval), passes `effort`, and reports the models its Codex offers (`codex debug models`, an `agent-models` frame, merged into `GET /api/agents/openai/options`); see `cli/src/local/agent-command.ts` and `cli-probes.ts`. **Local runs** (`local-run-service.ts`): every Task, Job, and scheduled-Task blueprint has a run location (`run_target` = `cluster` | `local`, plus `local_host_id` / `local_dir` / `local_session_mode`) picked with the shared `components/run-location-picker.tsx`; the task / workflow workers dispatch `local` runs to the daemon as an agent terminal (`spawned_by = "job" | "task"`, `local_terminal_id` on the run) instead of a pod, and the terminal's frames drive the run's state (`syncLinkedRun`: running, usage → cost, PR link → `pr_opened`, exit → completed / failed). The reconciler skips capacity / stall / pod checks for local runs. Schema: `local_hosts`, `local_terminals`, `work_definitions`. See `docs/optio-local.md` - **Organization and private scope** (`services/ownership.ts`; the work-specific rules in `work-ownership.ts`): every scoped resource — secrets, connections, model providers, MCP servers, skills, prompts, and work of every kind — is the **organization's** (`owner_user_id` null; everyone in the workspace sees it) or one person's **private** one (visible to its owner alone, **read-only to workspace admins**, who see it named with its owner and may delete it; changing / running / using it is a 403 for everyone else, admins included). `visibleOwner` is the SQL term every list uses, `canSee` / `canChange` the row forms, `usableBy` what a piece of work may be given (the organization's rows plus its owner's own). A member fetching someone else's private row gets a 404. Private work is hidden from other members everywhere (Work list, per-kind lists and details, recent runs, the events socket); a private agent is addressable only by its owner's agents. Private secrets (`scope = "user"`) are stored without a workspace (a boot-time heal re-binds older rows). UI vocabulary is fixed: pickers **Organization / Private**, sections **Organization / Private / Other people's**, chip **Private** (`lib/owner.ts`, `hooks/use-current-user.ts`, `components/ui/{owner-segments,scoped-list,owner-picker,owner-chip}.tsx`; the Secrets page is the exemplar). See `docs/scope.md` +- **Config as code** (`services/config/`, `routes/config.ts`, `workers/config-sync-worker.ts`, schemas in `schemas/config.ts`, shared types in `packages/shared/src/config/`): every Job, scheduled Task, persistent agent, prompt, repo, MCP server, skill and connection can be a YAML **manifest** (`apiVersion: optio/v1`, `kind`, `metadata.name` = identity, `spec`; references by **name**, secrets by name or `${{SECRET_NAME}}`, never values). A cluster mounts a directory of them (`OPTIO_CONFIG_DIR`; Helm `configAsCode.*`) that the sync worker reads every `OPTIO_CONFIG_INTERVAL` and **applies** to one workspace (`OPTIO_CONFIG_WORKSPACE` slug, default the oldest): create / adopt / update (a managed row edited in the UI is **put back** and reported as reverted) / replace / prune (`OPTIO_CONFIG_PRUNE`, default on), each manifest its own error. One handler per kind in `services/config/kinds/*.ts` (desire / find / diff / create / update / remove / export); the engine is `apply.ts`. Managed rows carry `managedBy` (`withManagedBy` / `withManagedWork`, nothing is managed unless `OPTIO_CONFIG_DIR` is set) and show a **Managed** chip and banner; **Detach** stops it. `GET /api/config/schema.json` (public) is the JSON Schema; `GET /api/config/export[.yaml]` exports; `POST /api/config/apply` is what `optio apply -f` posts (a plain upsert: manages nothing, never prunes); `optio export -o DIR` writes one file per resource. Settings → Config as code shows the directory's last sync. See `docs/config-as-code.md` - **Model providers, owners, pod secrets** (`model-provider-service.ts`, `work-ownership.ts`, routes in `model-providers.ts`, `bedrockRuntime` in `@optio/shared`): Settings → Model providers saves Amazon Bedrock for Claude Code / Codex (region, per-agent model ids, a machine AWS profile, encrypted pod credentials or the pod's IAM role); work picks one with `agentOptions.modelProvider`. Providers, secrets, connections and work have an owner — the organization (`owner_user_id` null) or one person: private work runs with its owner's secrets / providers / connections and only its owner changes or runs it; organization work only uses organization resources. `podSecrets` is the list of secrets a piece of work gives its pod (`GET /api/secrets/pickable`). Workspaces can auto-join people by verified email domain. See `docs/model-providers.md` - **Task dependencies**: `task_dependencies` table for multi-step pipelines - **Cost tracking**: `GET /api/analytics/costs` with daily/repo/type breakdowns, UI at `/costs` @@ -321,6 +322,7 @@ Key `values.yaml` settings: 6. Enable ingress with TLS 7. Set `GITHUB_TOKEN` secret for PR watching, issue sync, repo detection 8. Install `metrics-server` in cluster +9. Optional: keep the workspace's Jobs, agents, prompts, repos and integrations in a repo as manifests and mount them (`configAsCode.*`; `docs/config-as-code.md`) ## Known Issues diff --git a/README.md b/README.md index 9905fcb03..8e8349ec4 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ API ......... http://localhost:30400 On Docker Desktop Kubernetes these are NodePorts reached via `kubectl port-forward` — the web UI listens on **30310** and the API on **30400**. -Open the web UI and the setup wizard will walk you through configuring sign-in (your organization's Google Workspace — the wizard asks for the one-time setup token the API prints in its log, and the first person to sign in becomes the deployment admin; or any OAuth provider by environment variables), GitHub access, agent credentials (API key, OAuth token, Vertex AI, or a Max/Pro subscription), and adding your first repository. Then hit **New session** and pick a preset — _Open a PR_, _Interactive chat_, _Scheduled run_, or _Persistent agent_ — or compose your own from the five attributes. +Open the web UI and the setup wizard will walk you through configuring sign-in (your organization's Google Workspace — the wizard asks for the one-time setup token the API prints in its log, and the first person to sign in becomes the deployment admin; or any OAuth provider by environment variables), GitHub access, agent credentials (API key, OAuth token, Vertex AI, or a Max/Pro subscription), and adding your first repository. Teams that keep infrastructure in a repo can keep Optio there too: every Job, agent, prompt, repo, MCP server, skill and connection is a YAML manifest a cluster reads from a mounted directory (`docs/config-as-code.md`; `optio export -o optio/` writes what you have). Then hit **New session** and pick a preset — _Open a PR_, _Interactive chat_, _Scheduled run_, or _Persistent agent_ — or compose your own from the five attributes. ### Pair your own machine (optional) diff --git a/apps/android/core/model/src/main/kotlin/dev/optio/core/model/SharedTypes.kt b/apps/android/core/model/src/main/kotlin/dev/optio/core/model/SharedTypes.kt index ffe615e64..93f040db9 100644 --- a/apps/android/core/model/src/main/kotlin/dev/optio/core/model/SharedTypes.kt +++ b/apps/android/core/model/src/main/kotlin/dev/optio/core/model/SharedTypes.kt @@ -257,6 +257,28 @@ data class AgentConfig( // endregion +// region config.ts + +/** + * Config as code: what a resource that a configuration directory manages + * carries in every list and detail response (`docs/config-as-code.md`). + * The file is the truth: edits made in the UI are put back at the next sync. + */ +@Serializable +data class ManagedBy( + /** The `config_objects` row — `POST /api/config/objects/:id/detach` takes it. */ + val objectId: String, + val sourceId: String, + /** The source's name as Settings shows it ("config directory"). */ + val sourceName: String, + /** The manifest's file, relative to the source's directory. */ + val path: String, + /** The manifest kind: Work, Prompt, Repo, McpServer, Skill, Connection. */ + val kind: String, +) + +// endregion + // region connection.ts @Serializable @@ -334,6 +356,8 @@ data class Connection( val ownerUserId: String? = null, /** Display name of `ownerUserId`, for a private connection (lists only). */ val ownerName: String? = null, + /** Set when a configuration directory manages it (the file is the truth). */ + val managedBy: ManagedBy? = null, val enabled: Boolean, val status: ConnectionStatus, val statusMessage: String? = null, @@ -433,6 +457,8 @@ data class RepoConnection( val ownerUserId: String? = null, /** Display name of `ownerUserId`, for a private connection (lists only). */ val ownerName: String? = null, + /** Set when a configuration directory manages it (the file is the truth). */ + val managedBy: ManagedBy? = null, val enabled: Boolean, val status: ConnectionStatus, val statusMessage: String? = null, @@ -2446,6 +2472,8 @@ data class McpServerConfig( */ val ownerUserId: String? = null, val ownerName: String? = null, + /** Set when a configuration directory manages it (the file is the truth). */ + val managedBy: ManagedBy? = null, val enabled: Boolean, val createdAt: Instant, val updatedAt: Instant, @@ -2514,6 +2542,8 @@ data class CustomSkillConfig( */ val ownerUserId: String? = null, val ownerName: String? = null, + /** Set when a configuration directory manages it (the file is the truth). */ + val managedBy: ManagedBy? = null, val layout: CustomSkillLayout, /** Extra files for skill-dir layout. Null/empty = none. */ val files: List? = null, @@ -2599,6 +2629,8 @@ data class InstalledSkillConfig( */ val ownerUserId: String? = null, val ownerName: String? = null, + /** Set when a configuration directory manages it (the file is the truth). */ + val managedBy: ManagedBy? = null, val agentTypes: List? = null, val enabled: Boolean, val lastSyncedAt: Instant? = null, @@ -3015,6 +3047,8 @@ data class PersistentAgent( * connections, and only they can change it. */ val ownerUserId: String? = null, + /** Set when a configuration directory manages it (the file is the truth). */ + val managedBy: ManagedBy? = null, /** * The secrets (by name) the agent gets in its pod. Null = the workspace's * legacy behavior (see `Workspace.restrictPodSecrets`). diff --git a/apps/api/e2e/config-as-code.e2e.test.ts b/apps/api/e2e/config-as-code.e2e.test.ts new file mode 100644 index 000000000..e7cb6094f --- /dev/null +++ b/apps/api/e2e/config-as-code.e2e.test.ts @@ -0,0 +1,213 @@ +/** + * E2E: config as code through the real API server with OPTIO_CONFIG_DIR set. + * + * Covers: boot applies the mounted directory → the rows show `managedBy` over + * HTTP (strict response schemas included) → Settings' status endpoint → a UI + * edit is put back by `POST /api/config/source/sync` → the public JSON Schema + * → `POST /api/config/apply` (what the CLI posts) → export → detach. + */ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { startApiServer, waitFor, type ApiServerHandle } from "../src/test-utils/e2e/api-server.js"; + +let server: ApiServerHandle; +let dir: string; + +const manifest = (kind: string, name: string, spec: Record) => + JSON.stringify({ apiVersion: "optio/v1", kind, metadata: { name }, spec }); + +beforeAll(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "optio-e2e-config-")); + await fs.mkdir(path.join(dir, "prompts"), { recursive: true }); + await fs.writeFile( + path.join(dir, "prompts", "e2e-managed.yaml"), + manifest("Prompt", "e2e-managed-prompt", { template: "From the directory {{x}}" }), + ); + await fs.writeFile( + path.join(dir, "e2e-job.yaml"), + manifest("Work", "e2e-managed-job", { + who: { runtime: "shell" }, + what: { prompt: "echo managed" }, + }), + ); + server = await startApiServer({ env: { OPTIO_CONFIG_DIR: dir, OPTIO_CONFIG_INTERVAL: "10000" } }); +}, 150_000); + +afterAll(async () => { + await server?.stop(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +async function api( + path: string, + init?: RequestInit, +): Promise<{ status: number; body: T; text: string }> { + // Only a body gets the JSON content type: Fastify answers 400 to an empty + // JSON body, and the sync and detach POSTs send none. + const res = await fetch(`${server.baseUrl}${path}`, { + ...init, + headers: init?.body ? { "content-type": "application/json" } : undefined, + }); + const text = await res.text(); + let body: T; + try { + body = JSON.parse(text) as T; + } catch { + body = undefined as T; + } + return { status: res.status, body, text }; +} + +interface Status { + enabled: boolean; + source: { + path: string; + lastSyncAt: string | null; + lastSync: { summary: Record } | null; + } | null; + schemaUrl: string; +} + +describe("config as code over HTTP", () => { + it("applies the directory at boot and reports it in the status", async () => { + const status = await waitFor( + async () => { + const { body } = await api("/api/config/status"); + return body.source?.lastSyncAt ? body : null; + }, + { timeoutMs: 60_000, label: "first sync of OPTIO_CONFIG_DIR" }, + ); + expect(status.enabled).toBe(true); + expect(status.source!.path).toBe(dir); + expect(status.source!.lastSync!.summary).toMatchObject({ created: 2, errors: 0 }); + expect(status.schemaUrl).toMatch(/\/api\/config\/schema\.json$/); + }); + + it("shows managedBy on the Work list row and the prompt, through the response schemas", async () => { + const work = await api<{ + rows: Array<{ name: string; managedBy?: { path: string; kind: string } }>; + }>("/api/work"); + const job = work.body.rows.find((r) => r.name === "e2e-managed-job"); + expect(job?.managedBy).toMatchObject({ kind: "Work", path: "e2e-job.yaml" }); + + const prompts = await api<{ templates: Array<{ name: string; managedBy?: { path: string } }> }>( + "/api/prompt-templates", + ); + const prompt = prompts.body.templates.find((t) => t.name === "e2e-managed-prompt"); + expect(prompt?.managedBy).toMatchObject({ path: "prompts/e2e-managed.yaml" }); + }); + + it("puts a UI edit back on the next sync and reports it as reverted", async () => { + const prompts = await api<{ templates: Array<{ id: string; name: string }> }>( + "/api/prompt-templates", + ); + const prompt = prompts.body.templates.find((t) => t.name === "e2e-managed-prompt")!; + const edit = await api(`/api/prompt-templates/${prompt.id}`, { + method: "PATCH", + body: JSON.stringify({ template: "Edited by hand" }), + }); + expect(edit.status).toBe(200); + + const sync = await api<{ + items: Array<{ name: string; action: string; reverted?: boolean }>; + summary: Record; + }>("/api/config/source/sync", { method: "POST" }); + expect(sync.status).toBe(200); + expect(sync.body.items.find((i) => i.name === "e2e-managed-prompt")).toMatchObject({ + action: "update", + reverted: true, + }); + expect(sync.body.summary.reverted).toBe(1); + + // There is no GET by id for named prompts: read it back from the list. + const after = await api<{ templates: Array<{ id: string; template: string }> }>( + "/api/prompt-templates", + ); + expect(after.body.templates.find((t) => t.id === prompt.id)?.template).toBe( + "From the directory {{x}}", + ); + }); + + it("serves the JSON Schema publicly and applies manifests the CLI posts", async () => { + const schema = await api<{ $schema: string }>("/api/config/schema.json"); + expect(schema.status).toBe(200); + expect(schema.body.$schema).toContain("draft-07"); + + const plan = await api<{ dryRun: boolean; items: Array<{ action: string; name: string }> }>( + "/api/config/apply", + { + method: "POST", + body: JSON.stringify({ + dryRun: true, + manifests: [ + { + path: "cli/prompt.yaml", + document: JSON.parse( + manifest("Prompt", "e2e-cli-prompt", { template: "From the CLI" }), + ), + }, + ], + }), + }, + ); + expect(plan.status).toBe(200); + expect(plan.body.dryRun).toBe(true); + expect(plan.body.items).toEqual([ + expect.objectContaining({ action: "create", name: "e2e-cli-prompt" }), + ]); + + const applied = await api<{ items: Array<{ action: string; resourceId?: string }> }>( + "/api/config/apply", + { + method: "POST", + body: JSON.stringify({ + manifests: [ + { + path: "cli/prompt.yaml", + document: JSON.parse( + manifest("Prompt", "e2e-cli-prompt", { template: "From the CLI" }), + ), + }, + ], + }), + }, + ); + expect(applied.body.items[0]).toMatchObject({ action: "create" }); + // A CLI apply manages nothing. + const prompts = await api<{ templates: Array<{ name: string; managedBy?: unknown }> }>( + "/api/prompt-templates", + ); + expect( + prompts.body.templates.find((t) => t.name === "e2e-cli-prompt")?.managedBy, + ).toBeUndefined(); + }); + + it("exports the workspace as manifests and as YAML", async () => { + const json = await api<{ manifests: Array<{ kind: string; name: string; path: string }> }>( + "/api/config/export?kind=Prompt,Work", + ); + expect(json.body.manifests.map((m) => m.name)).toEqual( + expect.arrayContaining(["e2e-managed-prompt", "e2e-cli-prompt", "e2e-managed-job"]), + ); + const yaml = await api("/api/config/export.yaml?kind=Work&download=1"); + expect(yaml.status).toBe(200); + expect(yaml.text).toContain("kind: Work"); + expect(yaml.text).toContain("name: e2e-managed-job"); + expect(yaml.text).toContain("yaml-language-server"); + }); + + it("detaches a managed resource", async () => { + const work = await api<{ rows: Array<{ name: string; managedBy?: { objectId: string } }> }>( + "/api/work", + ); + const job = work.body.rows.find((r) => r.name === "e2e-managed-job")!; + const detach = await api(`/api/config/objects/${job.managedBy!.objectId}/detach`, { + method: "POST", + }); + expect(detach.status).toBe(204); + const again = await api<{ rows: Array<{ name: string; managedBy?: unknown }> }>("/api/work"); + expect(again.body.rows.find((r) => r.name === "e2e-managed-job")!.managedBy).toBeUndefined(); + }); +}); diff --git a/apps/api/package.json b/apps/api/package.json index 65b93b75a..39003f9a3 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -59,7 +59,9 @@ "pino-pretty": "^13.0.0", "postgres": "^3.4.7", "web-push": "^3.6.7", - "zod": "^3.25.17" + "yaml": "2.9.0", + "zod": "^3.25.17", + "zod-to-json-schema": "3.25.2" }, "devDependencies": { "@redocly/cli": "^2.32.2", diff --git a/apps/api/src/db/migrations/1791970000_config_as_code.sql b/apps/api/src/db/migrations/1791970000_config_as_code.sql new file mode 100644 index 000000000..afd27d70c --- /dev/null +++ b/apps/api/src/db/migrations/1791970000_config_as_code.sql @@ -0,0 +1,49 @@ +-- Config as code (docs/plans/config-as-code.md): resources declared as YAML +-- manifests in a directory the API pod reads, applied by a worker. +-- +-- `config_sources`: where a workspace's manifests come from. One kind today +-- (`dir`, declared by OPTIO_CONFIG_DIR and mirrored here at boot so it has an +-- id and a status); the column is there for repositories and pushes later. +CREATE TABLE IF NOT EXISTS "config_sources" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "workspace_id" uuid REFERENCES "workspaces"("id") ON DELETE CASCADE, + "name" text NOT NULL, + "kind" text NOT NULL DEFAULT 'dir', + "path" text NOT NULL, + "prune" boolean NOT NULL DEFAULT true, + "enabled" boolean NOT NULL DEFAULT true, + "origin" text NOT NULL DEFAULT 'env', + "last_sync_at" timestamp with time zone, + "last_sync_hash" text, + "last_sync_error" text, + "last_sync_result" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "config_sources_workspace_name_key" + ON "config_sources" (COALESCE("workspace_id", '00000000-0000-0000-0000-000000000000'::uuid), "name"); +--> statement-breakpoint +-- `config_objects`: what each source manages — the manifest (kind, name, file) +-- and the row it became. One source per resource; one resource per +-- (source, kind, name). Pruning deletes what a source no longer declares; +-- detaching deletes the row here and leaves the resource. +CREATE TABLE IF NOT EXISTS "config_objects" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "source_id" uuid NOT NULL REFERENCES "config_sources"("id") ON DELETE CASCADE, + "workspace_id" uuid REFERENCES "workspaces"("id") ON DELETE CASCADE, + "kind" text NOT NULL, + "name" text NOT NULL, + "path" text NOT NULL, + "resource_table" text NOT NULL, + "resource_id" uuid NOT NULL, + "hash" text, + "applied_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "config_objects_source_kind_name_key" + ON "config_objects" ("source_id", "kind", "name"); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "config_objects_resource_key" + ON "config_objects" ("resource_table", "resource_id"); diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index c2e847543..226560ac4 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -806,6 +806,13 @@ "when": 1791960000000, "tag": "1791960000_sign_in", "breakpoints": true + }, + { + "idx": 115, + "version": "7", + "when": 1791970000000, + "tag": "1791970000_config_as_code", + "breakpoints": true } ] } diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 887f88713..f8fff3246 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -17,7 +17,7 @@ import { check, } from "drizzle-orm/pg-core"; import { sql } from "drizzle-orm"; -import type { WorkDefinitionKind, WorkSettings } from "@optio/shared"; +import type { ConfigApplyResult, WorkDefinitionKind, WorkSettings } from "@optio/shared"; // ── Workspace enums ───────────────────────────────────────────────────────── @@ -1037,6 +1037,71 @@ export const authProviderConfigs = pgTable("auth_provider_configs", { updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }); +// ── Config as code (docs/plans/config-as-code.md) ─────────────────────────── + +/** + * Where a workspace's manifests come from. One kind today — `dir`, the + * directory OPTIO_CONFIG_DIR names, mirrored here at boot (`origin = env`) so + * it has an id, a status and a place in Settings. `last_sync_result` is the + * apply result Settings shows (counts and per-file errors). + */ +export const configSources = pgTable( + "config_sources", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id").references(() => workspaces.id, { onDelete: "cascade" }), + name: text("name").notNull(), + kind: text("kind").notNull().default("dir"), + path: text("path").notNull(), + prune: boolean("prune").notNull().default(true), + enabled: boolean("enabled").notNull().default(true), + origin: text("origin").notNull().default("env"), // env | settings + lastSyncAt: timestamp("last_sync_at", { withTimezone: true }), + lastSyncHash: text("last_sync_hash"), + lastSyncError: text("last_sync_error"), + lastSyncResult: jsonb("last_sync_result").$type(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + uniqueIndex("config_sources_workspace_name_key").on( + sql`COALESCE(${table.workspaceId}, '00000000-0000-0000-0000-000000000000'::uuid)`, + table.name, + ), + ], +); + +/** + * What each source manages: the manifest (kind, name, file) and the row it + * became. One source per resource, one resource per (source, kind, name). + * Pruning deletes what a source no longer declares; detaching deletes the row + * here and leaves the resource. + */ +export const configObjects = pgTable( + "config_objects", + { + id: uuid("id").primaryKey().defaultRandom(), + sourceId: uuid("source_id") + .notNull() + .references(() => configSources.id, { onDelete: "cascade" }), + workspaceId: uuid("workspace_id").references(() => workspaces.id, { onDelete: "cascade" }), + kind: text("kind").notNull(), // Work | Prompt | Repo | McpServer | Skill | Connection + name: text("name").notNull(), + path: text("path").notNull(), + // work_definitions | persistent_agents | prompt_templates | repos | + // mcp_servers | custom_skills | installed_skills | connections + resourceTable: text("resource_table").notNull(), + resourceId: uuid("resource_id").notNull(), + hash: text("hash"), + appliedAt: timestamp("applied_at", { withTimezone: true }).notNull().defaultNow(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + uniqueIndex("config_objects_source_kind_name_key").on(table.sourceId, table.kind, table.name), + uniqueIndex("config_objects_resource_key").on(table.resourceTable, table.resourceId), + ], +); + // ── Connection Assignments (which repos get which connections) ────────────── export const connectionAssignments = pgTable( diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 30af812f7..a88f696ac 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -69,6 +69,7 @@ async function main() { const { startReconcileWorker, startReconcileResyncWorker } = await import("./workers/reconcile-worker.js"); const { startSkillSyncWorker } = await import("./workers/skill-sync-worker.js"); + const { startConfigSyncWorker } = await import("./workers/config-sync-worker.js"); const { getBullMQConnectionOptions } = await import("./services/redis-config.js"); const { logTlsStackInfo, initTlsObservability } = await import("./services/tls-observability.js"); @@ -259,6 +260,9 @@ async function main() { const localSweepWorker = startLocalSweepWorker(); logger.info("Local sweep worker started"); + // Config as code: the configuration directory, when the deployment has one. + const configSyncWorker = startConfigSyncWorker(); + // Check if metrics-server is available checkMetricsServer().catch(() => {}); @@ -286,6 +290,7 @@ async function main() { await reconcileResyncWorker.close(); await skillSyncWorker.close(); await localSweepWorker.close(); + await configSyncWorker?.close(); await app.close(); // Flush pending OTel spans/metrics with 5s timeout await shutdownTelemetry(); diff --git a/apps/api/src/plugins/auth.ts b/apps/api/src/plugins/auth.ts index 5ddeefb6d..886fa0983 100644 --- a/apps/api/src/plugins/auth.ts +++ b/apps/api/src/plugins/auth.ts @@ -73,6 +73,7 @@ function isSelfScopedMutation(path: string): boolean { /** Exact routes that are always public. */ const PUBLIC_ROUTES = new Set([ "/api/health", + "/api/config/schema.json", "/api/setup/status", "/api/notifications/vapid-public-key", "/api/internal/git-credentials", diff --git a/apps/api/src/routes/config.ts b/apps/api/src/routes/config.ts new file mode 100644 index 000000000..0ab362f7a --- /dev/null +++ b/apps/api/src/routes/config.ts @@ -0,0 +1,247 @@ +/** + * `/api/config` — config as code (docs/config-as-code.md): the manifest JSON + * Schema, the configuration directory's status and a sync of it, an apply of + * manifests the CLI read, an export of the workspace as manifests, and + * detaching a managed resource. + * + * Who may call what: the schema is public (it holds no data); status and + * export need a member (export writes the organization's resources only, and + * never a secret's value); sync, apply and detach need a workspace admin. + */ +import type { FastifyInstance } from "fastify"; +import type { ZodTypeProvider } from "fastify-type-provider-zod"; +import { z } from "zod"; +import { isManifestKind, type ManifestKind } from "@optio/shared"; +import { requireRole } from "../plugins/auth.js"; +import { ErrorResponseSchema, IdParamsSchema } from "../schemas/common.js"; +import { + ApplyBodySchema, + ConfigApplyResultSchema, + ConfigStatusSchema, + ExportQuerySchema, + ExportResponseSchema, + SyncQuerySchema, + manifestJsonSchema, +} from "../schemas/config.js"; +import { applyManifests, detachObject } from "../services/config/apply.js"; +import { exportManifests, manifestsToYaml } from "../services/config/export.js"; +import { configStatus, syncEnvSource } from "../services/config/source.js"; +import { logAction } from "../services/optio-action-service.js"; + +const SCHEMA_PATH = "/api/config/schema.json"; + +/** Where a browser (and the yaml-language-server) fetches the schema: through the web app's proxy. */ +function schemaUrl(): string { + const base = (process.env.PUBLIC_URL ?? "http://localhost:3000").replace(/\/$/, ""); + return `${base}${SCHEMA_PATH}`; +} + +function kindsOf(query: string | undefined): ManifestKind[] | undefined { + if (!query) return undefined; + const kinds = query + .split(",") + .map((k) => k.trim()) + .filter(Boolean); + const bad = kinds.find((k) => !isManifestKind(k)); + if (bad) throw new KindError(bad); + return kinds as ManifestKind[]; +} + +class KindError extends Error { + constructor(readonly kind: string) { + super(`Unknown manifest kind "${kind}"`); + } +} + +export async function configRoutes(rawApp: FastifyInstance) { + const app = rawApp.withTypeProvider(); + const admin = { preHandler: [requireRole("admin")] }; + const schema = manifestJsonSchema(); + + app.get( + SCHEMA_PATH, + { + schema: { + operationId: "getManifestSchema", + summary: "The JSON Schema of a manifest", + description: + "Every manifest kind (Work, Prompt, Repo, McpServer, Skill, Connection) as JSON Schema, " + + "for `# yaml-language-server: $schema=` in a file and for validation in CI. Public.", + tags: ["Config"], + response: { 200: z.record(z.unknown()) }, + }, + }, + async (_req, reply) => { + reply.header("cache-control", "public, max-age=300"); + reply.send(schema); + }, + ); + + app.get( + "/api/config/status", + { + schema: { + operationId: "getConfigStatus", + summary: "The configuration directory and its last sync", + description: + "Whether `OPTIO_CONFIG_DIR` feeds the current workspace, and how the last sync went " + + "(counts, per-file errors). `source` is null when config as code is off or points " + + "at another workspace.", + tags: ["Config"], + response: { 200: ConfigStatusSchema }, + }, + }, + async (req, reply) => { + reply.send(await configStatus(req.user?.workspaceId ?? null, schemaUrl())); + }, + ); + + app.post( + "/api/config/source/sync", + { + ...admin, + schema: { + operationId: "syncConfigSource", + summary: "Read the configuration directory now", + description: + "Applies the directory at once instead of waiting for the next interval; " + + "`?dryRun=true` only plans it. Requires `admin` role.", + tags: ["Config"], + querystring: SyncQuerySchema, + response: { 200: ConfigApplyResultSchema, 404: ErrorResponseSchema }, + }, + }, + async (req, reply) => { + const result = await syncEnvSource({ dryRun: req.query.dryRun === "true" }); + if (!result) { + return reply + .status(404) + .send({ error: "Config as code is off: OPTIO_CONFIG_DIR is not set" }); + } + reply.send(result); + }, + ); + + app.post( + "/api/config/apply", + { + ...admin, + schema: { + operationId: "applyConfig", + summary: "Apply manifests to the current workspace", + description: + "What `optio apply` posts: documents the CLI read (file fields inlined). Each manifest " + + "is created, updated or left alone by name; an existing resource with the name is " + + "taken over. A CLI apply manages nothing and never prunes. `dryRun` plans only. " + + "Requires `admin` role.", + tags: ["Config"], + body: ApplyBodySchema, + response: { 200: ConfigApplyResultSchema }, + }, + }, + async (req, reply) => { + const result = await applyManifests({ + workspaceId: req.user?.workspaceId ?? null, + manifests: req.body.manifests.map((m) => ({ path: m.path, document: m.document })), + dryRun: req.body.dryRun ?? false, + }); + if (!result.dryRun) { + logAction({ + workspaceId: req.user?.workspaceId ?? null, + userId: req.user?.id, + action: "config.apply", + params: { manifests: req.body.manifests.length }, + result: { ...result.summary }, + success: result.summary.errors === 0, + }).catch(() => {}); + } + reply.send(result); + }, + ); + + app.get( + "/api/config/export", + { + schema: { + operationId: "exportConfig", + summary: "The organization's resources as manifests", + description: + "Every Work definition, prompt, repo, MCP server, skill and connection of the current " + + "workspace as a manifest, with the file each would be saved as. `kind` narrows the kinds, " + + "`id` picks one resource. Never anyone's private resources, never a secret's value.", + tags: ["Config"], + querystring: ExportQuerySchema, + response: { 200: ExportResponseSchema, 400: ErrorResponseSchema }, + }, + }, + async (req, reply) => { + try { + const manifests = await exportManifests(req.user?.workspaceId ?? null, { + kinds: kindsOf(req.query.kind), + id: req.query.id, + }); + reply.send({ manifests }); + } catch (err) { + if (err instanceof KindError) return reply.status(400).send({ error: err.message }); + throw err; + } + }, + ); + + app.get( + "/api/config/export.yaml", + { + schema: { + operationId: "exportConfigYaml", + summary: "The export as one YAML file", + description: "`GET /api/config/export` as YAML text, manifests separated by `---`.", + tags: ["Config"], + querystring: ExportQuerySchema.extend({ + download: z.string().optional().describe("Any value: send as an attachment"), + }), + response: { 200: z.string(), 400: ErrorResponseSchema }, + }, + }, + async (req, reply) => { + try { + const manifests = await exportManifests(req.user?.workspaceId ?? null, { + kinds: kindsOf(req.query.kind), + id: req.query.id, + }); + reply.header("content-type", "application/yaml; charset=utf-8"); + if (req.query.download !== undefined) { + const name = + manifests.length === 1 ? manifests[0].path.split("/").pop()! : "optio-config.yaml"; + reply.header("content-disposition", `attachment; filename="${name}"`); + } + reply.send(manifestsToYaml(manifests, schemaUrl())); + } catch (err) { + if (err instanceof KindError) return reply.status(400).send({ error: err.message }); + throw err; + } + }, + ); + + app.post( + "/api/config/objects/:id/detach", + { + ...admin, + schema: { + operationId: "detachConfigObject", + summary: "Stop managing a resource", + description: + "Drops the bookkeeping that ties a resource to its manifest; the resource stays and is " + + "edited by hand from then on. If its manifest is still in the directory, the next sync " + + "takes it over again. Requires `admin` role.", + tags: ["Config"], + params: IdParamsSchema, + response: { 204: z.null(), 404: ErrorResponseSchema }, + }, + }, + async (req, reply) => { + const detached = await detachObject(req.params.id, req.user?.workspaceId ?? null); + if (!detached) return reply.status(404).send({ error: "Not a managed resource" }); + reply.status(204).send(null); + }, + ); +} diff --git a/apps/api/src/routes/connections.ts b/apps/api/src/routes/connections.ts index 39798ddf7..963127fc6 100644 --- a/apps/api/src/routes/connections.ts +++ b/apps/api/src/routes/connections.ts @@ -12,6 +12,7 @@ import { ConnectionSchema, ConnectionAssignmentSchema, } from "../schemas/integration.js"; +import { withManagedBy } from "../services/config/managed.js"; // ── Request schemas ─────────────────────────────────────────────────────── @@ -222,7 +223,7 @@ export async function connectionRoutes(rawApp: FastifyInstance) { (c) => !c.ownerUserId || c.ownerUserId === actor.userId || actor.isAdmin, ), ); - reply.send({ connections: conns }); + reply.send({ connections: await withManagedBy(conns, "connections") }); }, ); @@ -285,7 +286,7 @@ export async function connectionRoutes(rawApp: FastifyInstance) { if (conn.ownerUserId && conn.ownerUserId !== actor.userId && !actor.isAdmin) { return reply.status(404).send({ error: "Connection not found" }); } - reply.send({ connection: conn }); + reply.send({ connection: (await withManagedBy([conn], "connections"))[0] }); }, ); @@ -551,7 +552,7 @@ export async function connectionRoutes(rawApp: FastifyInstance) { (c) => !c.ownerUserId || c.ownerUserId === actor.userId || actor.isAdmin, ), ); - reply.send({ connections: conns }); + reply.send({ connections: await withManagedBy(conns, "connections") }); }, ); } diff --git a/apps/api/src/routes/installed-skills.ts b/apps/api/src/routes/installed-skills.ts index 18c7b04c9..e39c5a973 100644 --- a/apps/api/src/routes/installed-skills.ts +++ b/apps/api/src/routes/installed-skills.ts @@ -12,6 +12,7 @@ import { } from "../services/ownership.js"; import { ErrorResponseSchema, IdParamsSchema } from "../schemas/common.js"; import { requireRole } from "../plugins/auth.js"; +import { withManagedBy } from "../services/config/managed.js"; const InstalledSkillSchema = z.unknown().describe("Installed (marketplace-sourced) skill row"); @@ -104,7 +105,7 @@ export async function installedSkillRoutes(rawApp: FastifyInstance) { const skills = await withOwnerNames( await installedSkillService.listInstalledSkills(req.query.scope, workspaceId, actorOf(req)), ); - reply.send({ skills }); + reply.send({ skills: await withManagedBy(skills, "installed_skills") }); }, ); diff --git a/apps/api/src/routes/mcp-servers.ts b/apps/api/src/routes/mcp-servers.ts index b1e0ff2d3..2830f9744 100644 --- a/apps/api/src/routes/mcp-servers.ts +++ b/apps/api/src/routes/mcp-servers.ts @@ -14,6 +14,7 @@ import { logAction } from "../services/optio-action-service.js"; import { ErrorResponseSchema, IdParamsSchema } from "../schemas/common.js"; import { McpServerSchema } from "../schemas/integration.js"; import { requireRole } from "../plugins/auth.js"; +import { withManagedBy } from "../services/config/managed.js"; const scopeQuerySchema = z .object({ @@ -92,7 +93,7 @@ export async function mcpServerRoutes(rawApp: FastifyInstance) { const servers = await withOwnerNames( await mcpService.listMcpServers(req.query.scope, workspaceId, actorOf(req)), ); - reply.send({ servers }); + reply.send({ servers: await withManagedBy(servers, "mcp_servers") }); }, ); @@ -260,7 +261,7 @@ export async function mcpServerRoutes(rawApp: FastifyInstance) { workspaceId, req.user?.id ?? null, ); - reply.send({ servers }); + reply.send({ servers: await withManagedBy(servers, "mcp_servers") }); }, ); diff --git a/apps/api/src/routes/persistent-agents.ts b/apps/api/src/routes/persistent-agents.ts index 5390f3705..23d578f18 100644 --- a/apps/api/src/routes/persistent-agents.ts +++ b/apps/api/src/routes/persistent-agents.ts @@ -25,6 +25,7 @@ import * as triggerService from "../services/trigger-service.js"; import { CreateTriggerBodySchema, replyTriggerError } from "../schemas/trigger.js"; import { requireRole } from "../plugins/auth.js"; import { actorOf, withOwnerNames } from "../services/ownership.js"; +import { withManagedBy } from "../services/config/managed.js"; import { getRepo } from "../services/repo-service.js"; /** @@ -152,7 +153,7 @@ export async function persistentAgentRoutes(rawApp: FastifyInstance) { req.user?.workspaceId ?? null, actorOf(req), ); - reply.send({ agents }); + reply.send({ agents: await withManagedBy(agents, "persistent_agents") }); }, ); @@ -194,7 +195,7 @@ export async function persistentAgentRoutes(rawApp: FastifyInstance) { if (!agent) return; const inbox = await paService.listInboxSummary(id); // A private agent is named with its owner (what an admin's read-only view shows). - const [named] = await withOwnerNames([agent]); + const [named] = await withManagedBy(await withOwnerNames([agent]), "persistent_agents"); reply.send({ agent: named, inbox }); }, ); diff --git a/apps/api/src/routes/prompt-templates.ts b/apps/api/src/routes/prompt-templates.ts index 4b593aa75..264d3c1d9 100644 --- a/apps/api/src/routes/prompt-templates.ts +++ b/apps/api/src/routes/prompt-templates.ts @@ -28,6 +28,7 @@ import { DEFAULT_PROMPT_TEMPLATE } from "@optio/shared"; import { ErrorResponseSchema } from "../schemas/common.js"; import { PromptTemplateSchema } from "../schemas/integration.js"; import { requireRole } from "../plugins/auth.js"; +import { withManagedBy } from "../services/config/managed.js"; const repoUrlQuerySchema = z .object({ @@ -166,7 +167,7 @@ export async function promptTemplateRoutes(rawApp: FastifyInstance) { const templates = await withOwnerNames( await listPromptTemplates({ workspaceId, kind, viewer: actorOf(req) }), ); - reply.send({ templates }); + reply.send({ templates: await withManagedBy(templates, "prompt_templates") }); }, ); diff --git a/apps/api/src/routes/repos.ts b/apps/api/src/routes/repos.ts index 2eec56ba2..33d5b9356 100644 --- a/apps/api/src/routes/repos.ts +++ b/apps/api/src/routes/repos.ts @@ -20,6 +20,7 @@ import { ErrorResponseSchema, IdParamsSchema } from "../schemas/common.js"; import { RepoSchema } from "../schemas/integration.js"; import { resolveReviewConfig } from "../services/review-config.js"; import * as optioSettingsService from "../services/optio-settings-service.js"; +import { withManagedBy } from "../services/config/managed.js"; const createRepoSchema = z .object({ @@ -216,7 +217,7 @@ export async function repoRoutes(rawApp: FastifyInstance) { async (req, reply) => { const workspaceId = req.user?.workspaceId ?? null; const repos = await repoService.listRepos(workspaceId); - reply.send({ repos }); + reply.send({ repos: await withManagedBy(repos, "repos") }); }, ); @@ -417,7 +418,7 @@ export async function repoRoutes(rawApp: FastifyInstance) { result: { id }, success: true, }).catch(() => {}); - reply.send({ repo }); + reply.send({ repo: (await withManagedBy([repo], "repos"))[0] }); }, ); diff --git a/apps/api/src/routes/skills.ts b/apps/api/src/routes/skills.ts index faea0e9c1..2b4399605 100644 --- a/apps/api/src/routes/skills.ts +++ b/apps/api/src/routes/skills.ts @@ -13,6 +13,7 @@ import { import { ErrorResponseSchema, IdParamsSchema } from "../schemas/common.js"; import { SkillSchema } from "../schemas/integration.js"; import { requireRole } from "../plugins/auth.js"; +import { withManagedBy } from "../services/config/managed.js"; const scopeQuerySchema = z .object({ @@ -110,7 +111,7 @@ export async function skillRoutes(rawApp: FastifyInstance) { const skills = await withOwnerNames( await skillService.listSkills(req.query.scope, workspaceId, actorOf(req)), ); - reply.send({ skills }); + reply.send({ skills: await withManagedBy(skills, "custom_skills") }); }, ); diff --git a/apps/api/src/routes/task-configs.ts b/apps/api/src/routes/task-configs.ts index 68705f2ce..b1e2bf3ab 100644 --- a/apps/api/src/routes/task-configs.ts +++ b/apps/api/src/routes/task-configs.ts @@ -19,6 +19,7 @@ import { } from "../schemas/trigger.js"; import { requireRole } from "../plugins/auth.js"; import { actorOf, canSee, withOwnerNames } from "../services/ownership.js"; +import { withManagedBy } from "../services/config/managed.js"; const flexibleTimestamp = z.union([z.date(), z.string()]); @@ -272,7 +273,7 @@ export async function taskConfigRoutes(rawApp: FastifyInstance) { if (!canSee(taskConfig.ownerUserId, actorOf(req))) { return reply.status(404).send({ error: "Task config not found" }); } - const [named] = await withOwnerNames([taskConfig]); + const [named] = await withManagedBy(await withOwnerNames([taskConfig]), "work_definitions"); reply.send({ taskConfig: named }); }, ); diff --git a/apps/api/src/routes/workflows.ts b/apps/api/src/routes/workflows.ts index f03b84750..8ef0c445d 100644 --- a/apps/api/src/routes/workflows.ts +++ b/apps/api/src/routes/workflows.ts @@ -13,6 +13,7 @@ import { workChangeError, } from "../services/work-ownership.js"; import { actorOf, canSee, withOwnerNames } from "../services/ownership.js"; +import { withManagedBy } from "../services/config/managed.js"; import { ErrorResponseSchema, IdParamsSchema } from "../schemas/common.js"; import { WorkflowSchema, @@ -406,7 +407,7 @@ export async function workflowRoutes(rawApp: FastifyInstance) { if (!canSee(workflow.ownerUserId, actorOf(req))) { return reply.status(404).send({ error: "Workflow not found" }); } - const [named] = await withOwnerNames([workflow]); + const [named] = await withManagedBy(await withOwnerNames([workflow]), "work_definitions"); reply.send({ workflow: named }); }, ); diff --git a/apps/api/src/schemas/config.test.ts b/apps/api/src/schemas/config.test.ts new file mode 100644 index 000000000..6a22f37de --- /dev/null +++ b/apps/api/src/schemas/config.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import { ManifestSchema, manifestJsonSchema } from "./config.js"; + +const work = { + apiVersion: "optio/v1", + kind: "Work", + metadata: { name: "nightly-bump" }, + spec: { + when: { schedule: "0 3 * * 1-5" }, + where: { repo: "https://github.com/acme/api", branch: "main" }, + who: { runtime: "claude-code", options: { model: "claude-sonnet-4-5" } }, + what: { prompt: "Bump every dependency" }, + then: "until-merged", + secrets: ["GITHUB_TOKEN"], + environment: { connections: { add: ["Linear"] }, review: { enabled: true, trigger: "on_pr" } }, + }, +}; + +describe("ManifestSchema", () => { + it("accepts a Work manifest and each other kind", () => { + expect(ManifestSchema.safeParse(work).success).toBe(true); + const ok = (doc: unknown) => expect(ManifestSchema.safeParse(doc).success).toBe(true); + ok({ + apiVersion: "optio/v1", + kind: "Prompt", + metadata: { name: "pr" }, + spec: { template: "Hi" }, + }); + ok({ + apiVersion: "optio/v1", + kind: "Repo", + metadata: { name: "acme/api" }, + spec: { url: "https://github.com/acme/api", reviewEnabled: true, maxConcurrentTasks: 2 }, + }); + ok({ + apiVersion: "optio/v1", + kind: "McpServer", + metadata: { name: "docs" }, + spec: { command: "npx", args: ["-y", "docs-mcp"], env: { TOKEN: "${{DOCS_TOKEN}}" } }, + }); + ok({ + apiVersion: "optio/v1", + kind: "Skill", + metadata: { name: "notes" }, + spec: { source: { url: "https://github.com/acme/skills", path: "notes" } }, + }); + ok({ + apiVersion: "optio/v1", + kind: "Connection", + metadata: { name: "Linear" }, + spec: { provider: "linear", config: { LINEAR_API_KEY: "${{LINEAR_API_KEY}}" } }, + }); + }); + + it("takes a `shell` runtime and a promptFile in place of a prompt", () => { + const shell = { + ...work, + spec: { ...work.spec, who: { runtime: "shell" }, what: { promptFile: "./x.sh" } }, + }; + expect(ManifestSchema.safeParse(shell).success).toBe(true); + }); + + it("refuses an unknown kind, an unknown field, a bad `when`, and a missing prompt", () => { + const bad = (doc: unknown, message: RegExp) => { + const result = ManifestSchema.safeParse(doc); + expect(result.success).toBe(false); + if (!result.success) expect(JSON.stringify(result.error.issues)).toMatch(message); + }; + bad({ ...work, kind: "Job" }, /Invalid discriminator/); + bad({ ...work, spec: { ...work.spec, colour: "red" } }, /Unrecognized key/); + bad( + { ...work, spec: { ...work.spec, when: { schedule: "* * * * *", webhook: { path: "x" } } } }, + /Unrecognized key|Invalid input/, + ); + bad({ ...work, spec: { ...work.spec, what: {} } }, /what\.prompt or what\.promptFile/); + bad( + { + apiVersion: "optio/v1", + kind: "Skill", + metadata: { name: "s" }, + spec: { prompt: "a", source: { url: "u" } }, + }, + /either a prompt/, + ); + }); + + it("refuses a Repo setting the API doesn't take", () => { + const result = ManifestSchema.safeParse({ + apiVersion: "optio/v1", + kind: "Repo", + metadata: { name: "r" }, + spec: { url: "https://github.com/a/b", slackWebhookUrl: "https://hooks.slack.com/x" }, + }); + expect(result.success).toBe(false); + }); +}); + +describe("manifestJsonSchema", () => { + it("is a draft-07 schema that lists every kind", () => { + const schema = manifestJsonSchema(); + expect(schema.$schema).toContain("draft-07"); + const text = JSON.stringify(schema); + for (const kind of ["Work", "Prompt", "Repo", "McpServer", "Skill", "Connection"]) { + expect(text).toContain(`"${kind}"`); + } + expect(text).toContain("promptFile"); + }); +}); diff --git a/apps/api/src/schemas/config.ts b/apps/api/src/schemas/config.ts new file mode 100644 index 000000000..4660c01a2 --- /dev/null +++ b/apps/api/src/schemas/config.ts @@ -0,0 +1,511 @@ +/** + * Zod schemas for config as code (docs/config-as-code.md): the manifest + * format `POST /api/config/apply` and the configuration directory accept, and + * the apply result they answer with. `manifestJsonSchema()` serves the same + * shape as JSON Schema (`GET /api/config/schema.json`) for editors and CI. + */ +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; +import { + AGENT_TYPES, + CONFIG_ACTIONS, + MANIFEST_API_VERSION, + MANIFEST_KINDS, + SHELL_RUNTIME, + type ConfigApplyResult, + type ConnectionManifest, + type Manifest, + type McpServerManifest, + type PromptManifest, + type SkillManifest, + type WorkManifest, +} from "@optio/shared"; +import { ErrorResponseSchema } from "./common.js"; + +const name = z + .string() + .trim() + .min(1) + .max(200) + .describe("The identity: unique per workspace and kind"); + +const MetadataSchema = z + .object({ + name, + description: z.string().max(2000).nullable().optional(), + }) + .describe("Who the resource is"); + +const base = (kind: K) => ({ + apiVersion: z.literal(MANIFEST_API_VERSION), + kind: z.literal(kind), + metadata: MetadataSchema, +}); + +const secretName = z + .string() + .regex(/^[A-Za-z_][A-Za-z0-9_]{0,127}$/, "a secret name: letters, digits and underscores"); + +const repoUrl = z.string().url().describe("A repo URL, the way Repos lists it"); + +const NameOverridesSchema = z + .object({ + add: z.array(z.string().min(1)).max(200).optional(), + remove: z.array(z.string().min(1)).max(200).optional(), + }) + .strict() + .describe("Names added to the default set, and names taken out of it"); + +// ── Work ──────────────────────────────────────────────────────────────────── + +const WhenSchema = z + .union([ + z + .object({ + schedule: z.union([ + z.string().min(1).describe("A cron expression"), + z.object({ cron: z.string().min(1) }).strict(), + ]), + }) + .strict(), + z.object({ webhook: z.object({ path: z.string().min(1) }).strict() }).strict(), + z + .object({ + ticket: z.object({ source: z.string().min(1), labels: z.array(z.string()).optional() }), + }) + .strict(), + z.object({ github: z.record(z.unknown()) }).strict(), + z.object({ slack: z.record(z.unknown()) }).strict(), + z.object({ linear: z.record(z.unknown()) }).strict(), + ]) + .describe( + "What starts it: one key — the trigger type — holding that trigger's config. Absent = on demand.", + ); + +const WorkSpecSchema = z + .object({ + when: WhenSchema.optional(), + where: z + .object({ + repo: repoUrl.nullable().optional(), + branch: z + .string() + .regex(/^[a-zA-Z0-9._/-]+$/, "Invalid branch name") + .nullable() + .optional(), + }) + .strict() + .optional() + .describe("The repo it works in (pod work only)"), + who: z + .object({ + runtime: z + .enum([...AGENT_TYPES, SHELL_RUNTIME] as unknown as [string, ...string[]]) + .describe("Agent runtime, or `shell` for a Job that runs its prompt as a command"), + options: z.record(z.union([z.string(), z.boolean()])).optional(), + model: z.string().max(200).nullable().optional(), + }) + .strict(), + what: z + .object({ + prompt: z.string().max(100_000).optional(), + promptFile: z.string().min(1).optional().describe("A file next to the manifest"), + runTitle: z.string().max(200).nullable().optional(), + }) + .strict(), + then: z.enum(["exits", "until-merged", "waits-for-messages"]).optional(), + mergeWhenReady: z.boolean().optional(), + retries: z.number().int().min(0).max(10).optional(), + priority: z.number().int().min(1).max(1000).optional(), + secrets: z.array(secretName).max(100).optional().describe("Pod secrets, by name"), + environment: z + .object({ + connections: NameOverridesSchema.optional(), + mcpServers: NameOverridesSchema.optional(), + skills: NameOverridesSchema.optional(), + setupCommands: z.string().max(20_000).nullable().optional(), + review: z + .object({ enabled: z.boolean(), trigger: z.enum(["on_pr", "on_ci_pass"]).optional() }) + .strict() + .nullable() + .optional(), + cautiousMode: z.boolean().nullable().optional(), + maxAutoResumes: z.number().int().min(0).max(100).nullable().optional(), + }) + .strict() + .optional(), + agent: z + .object({ + slug: z + .string() + .regex(/^[a-z0-9][a-z0-9-]*$/, "lowercase letters, digits and hyphens only") + .optional(), + systemPrompt: z.string().nullable().optional(), + systemPromptFile: z.string().min(1).optional(), + agentsMd: z.string().nullable().optional(), + agentsMdFile: z.string().min(1).optional(), + podLifecycle: z.enum(["always-on", "sticky", "on-demand"]).optional(), + }) + .strict() + .optional(), + params: z.record(z.unknown()).nullable().optional(), + limits: z + .object({ + maxTurns: z.number().int().min(1).max(10_000).nullable().optional(), + budgetUsd: z.union([z.string(), z.number()]).nullable().optional(), + }) + .strict() + .optional(), + pods: z + .object({ + maxPodInstances: z.number().int().min(1).max(20).optional(), + maxAgentsPerPod: z.number().int().min(1).max(50).optional(), + }) + .strict() + .optional(), + enabled: z.boolean().optional(), + }) + .strict() + .refine((s) => s.what.prompt !== undefined || s.what.promptFile !== undefined, { + message: "what.prompt or what.promptFile is required", + path: ["what"], + }); + +export const WorkManifestSchema = z.object({ ...base("Work"), spec: WorkSpecSchema }).strict(); + +// ── Prompt ────────────────────────────────────────────────────────────────── + +const PromptSpecSchema = z + .object({ + kind: z.enum(["prompt", "review", "job", "task"]).optional(), + template: z.string().min(1).optional(), + templateFile: z.string().min(1).optional(), + params: z.record(z.unknown()).nullable().optional(), + defaultAgentType: z.string().nullable().optional(), + }) + .strict() + .refine((s) => s.template !== undefined || s.templateFile !== undefined, { + message: "template or templateFile is required", + }); + +export const PromptManifestSchema = z + .object({ ...base("Prompt"), spec: PromptSpecSchema }) + .strict(); + +// ── Repo ──────────────────────────────────────────────────────────────────── + +/** + * The settings a Repo manifest may name — what `PATCH /api/repos/:id` takes, + * minus the Slack webhook (a credential). Only the named ones are managed. + */ +export const RepoSettingsSchema = z + .object({ + defaultBranch: z.string().optional(), + imagePreset: z.string().optional(), + extraPackages: z.string().optional(), + setupCommands: z.string().optional(), + customDockerfile: z.string().nullable().optional(), + autoMerge: z.boolean().optional(), + cautiousMode: z.boolean().optional(), + defaultAgentType: z.enum([...AGENT_TYPES] as [string, ...string[]]).optional(), + promptTemplateOverride: z.string().nullable().optional(), + claudeModel: z.string().optional(), + claudeContextWindow: z.string().optional(), + claudeEffort: z.string().optional(), + copilotModel: z.string().optional(), + copilotEffort: z.string().optional(), + opencodeModel: z.string().optional(), + opencodeAgent: z.string().optional(), + opencodeProvider: z.string().optional(), + opencodeBaseUrl: z.string().url().nullable().optional(), + geminiModel: z.string().optional(), + geminiApprovalMode: z.string().optional(), + openclawModel: z.string().nullable().optional(), + openclawAgent: z.string().nullable().optional(), + cursorModel: z.string().nullable().optional(), + maxTurnsCoding: z.number().int().min(1).max(10000).optional(), + maxTurnsReview: z.number().int().min(1).max(10000).optional(), + autoResume: z.boolean().optional(), + planningModeEnabled: z.boolean().optional(), + maxConcurrentTasks: z.number().int().min(1).max(50).optional(), + maxPodInstances: z.number().int().min(1).max(20).optional(), + maxAgentsPerPod: z.number().int().min(1).max(50).optional(), + reviewEnabled: z.boolean().optional(), + reviewTrigger: z.enum(["manual", "on_pr", "on_ci_pass"]).optional(), + reviewPromptTemplate: z.string().nullable().optional(), + testCommand: z.string().optional(), + reviewAgentType: z + .enum([...AGENT_TYPES] as [string, ...string[]]) + .nullable() + .optional(), + reviewModel: z.string().nullable().optional(), + externalReviewMode: z.enum(["off", "on_request", "on_pr_hold", "on_pr_post"]).optional(), + externalReviewFilters: z + .object({ + skipDrafts: z.boolean().optional(), + skipOptioAuthored: z.boolean().optional(), + includeAuthors: z.array(z.string()).optional(), + excludeAuthors: z.array(z.string()).optional(), + includeLabels: z.array(z.string()).optional(), + excludeLabels: z.array(z.string()).optional(), + }) + .nullable() + .optional(), + externalReviewWaitForCi: z.boolean().optional(), + maxAutoResumes: z.number().int().min(1).max(100).nullable().optional(), + slackChannel: z.string().nullable().optional(), + slackNotifyOn: z + .array(z.enum(["completed", "failed", "needs_attention", "pr_opened"])) + .optional(), + slackEnabled: z.boolean().optional(), + networkPolicy: z.enum(["unrestricted", "restricted"]).optional(), + secretProxy: z.boolean().optional(), + offPeakOnly: z.boolean().optional(), + cpuRequest: z.string().nullable().optional(), + cpuLimit: z.string().nullable().optional(), + memoryRequest: z.string().nullable().optional(), + memoryLimit: z.string().nullable().optional(), + dockerInDocker: z.boolean().optional(), + }) + .strict(); + +export const REPO_SETTING_KEYS = Object.keys(RepoSettingsSchema.shape) as Array< + keyof z.infer +>; + +const RepoSpecSchema = RepoSettingsSchema.extend({ url: repoUrl }).strict(); + +export const RepoManifestSchema = z.object({ ...base("Repo"), spec: RepoSpecSchema }).strict(); + +// ── McpServer ─────────────────────────────────────────────────────────────── + +const McpServerSpecSchema = z + .object({ + command: z.string().min(1), + args: z.array(z.string()).optional(), + env: z.record(z.string()).optional().describe("Values may be `${{SECRET_NAME}}` references"), + installCommand: z.string().nullable().optional(), + repo: repoUrl.nullable().optional().describe("Scope it to one repo; absent = the workspace"), + enabled: z.boolean().optional(), + }) + .strict(); + +export const McpServerManifestSchema = z + .object({ ...base("McpServer"), spec: McpServerSpecSchema }) + .strict(); + +// ── Skill ─────────────────────────────────────────────────────────────────── + +const SkillSpecSchema = z + .object({ + prompt: z.string().min(1).optional(), + promptFile: z.string().min(1).optional(), + layout: z.enum(["commands", "skill-dir"]).optional(), + files: z + .record(z.string()) + .optional() + .describe("Extra files of a skill-dir skill, path → content"), + filesFrom: z.string().min(1).optional().describe("A directory next to the manifest"), + source: z + .object({ + url: z.string().min(1), + ref: z.string().optional(), + path: z.string().optional(), + }) + .strict() + .optional() + .describe("A marketplace skill, cloned from a git source"), + agentTypes: z.array(z.string()).optional(), + repo: repoUrl.nullable().optional(), + enabled: z.boolean().optional(), + }) + .strict() + .refine( + (s) => { + const custom = + s.prompt !== undefined || s.promptFile !== undefined || s.filesFrom !== undefined; + return custom !== (s.source !== undefined); + }, + { message: "a skill has either a prompt (prompt, promptFile or filesFrom) or a source" }, + ); + +export const SkillManifestSchema = z.object({ ...base("Skill"), spec: SkillSpecSchema }).strict(); + +// ── Connection ────────────────────────────────────────────────────────────── + +const ConnectionSpecSchema = z + .object({ + provider: z.string().min(1).describe("The provider's slug"), + config: z.record(z.unknown()).optional(), + repo: repoUrl.nullable().optional(), + enabled: z.boolean().optional(), + assignments: z + .array( + z + .object({ + repo: repoUrl.nullable().optional().describe("Absent = every repo"), + agentTypes: z.array(z.string()).optional(), + permission: z.enum(["read", "write", "full"]).optional(), + }) + .strict(), + ) + .max(100) + .optional(), + }) + .strict(); + +export const ConnectionManifestSchema = z + .object({ ...base("Connection"), spec: ConnectionSpecSchema }) + .strict(); + +// ── Any manifest ──────────────────────────────────────────────────────────── + +export const ManifestSchema = z + .discriminatedUnion("kind", [ + WorkManifestSchema, + PromptManifestSchema, + RepoManifestSchema, + McpServerManifestSchema, + SkillManifestSchema, + ConnectionManifestSchema, + ]) + .describe("One Optio resource as configuration"); + +export const ManifestKindSchema = z.enum(MANIFEST_KINDS); + +/** The JSON Schema of a manifest, for `# yaml-language-server: $schema=` and CI. */ +export function manifestJsonSchema(): Record { + return { + $schema: "http://json-schema.org/draft-07/schema#", + $id: "https://optio.dev/schemas/manifest.json", + title: "Optio manifest", + ...zodToJsonSchema(ManifestSchema, { $refStrategy: "none", target: "jsonSchema7" }), + }; +} + +// ── Applying ──────────────────────────────────────────────────────────────── + +export const ManifestInputSchema = z + .object({ + path: z.string().min(1).max(1000).describe("The file, relative to the directory applied"), + document: z.unknown().describe("The parsed document, `*File` fields already inlined"), + }) + .describe("One document as the CLI read it"); + +export const ApplyBodySchema = z + .object({ + manifests: z.array(ManifestInputSchema).max(1000), + dryRun: z.boolean().optional().describe("Plan only; write nothing"), + }) + .describe("Manifests to apply to the current workspace"); + +export const ConfigPlanItemSchema = z.object({ + kind: z.string(), + name: z.string(), + path: z.string(), + action: z.enum(CONFIG_ACTIONS), + resourceId: z.string().nullable().optional(), + changes: z.array(z.string()).optional(), + reverted: z.boolean().optional(), + message: z.string().optional(), +}); + +export const ConfigApplySummarySchema = z.object({ + created: z.number(), + updated: z.number(), + reverted: z.number(), + unchanged: z.number(), + adopted: z.number(), + replaced: z.number(), + pruned: z.number(), + errors: z.number(), +}); + +export const ConfigApplyResultSchema = z + .object({ + dryRun: z.boolean(), + source: z.object({ id: z.string(), name: z.string() }).nullable().optional(), + items: z.array(ConfigPlanItemSchema), + summary: ConfigApplySummarySchema, + at: z.string(), + }) + .describe("What an apply did, or (dry run) would do, per manifest"); + +export const ConfigSourceViewSchema = z.object({ + id: z.string(), + name: z.string(), + kind: z.enum(["dir"]), + path: z.string(), + workspaceId: z.string().nullable(), + prune: z.boolean(), + enabled: z.boolean(), + origin: z.enum(["env", "settings"]), + intervalMs: z.number(), + lastSyncAt: z.string().nullable(), + lastSyncHash: z.string().nullable(), + lastSyncError: z.string().nullable(), + lastSync: ConfigApplyResultSchema.nullable(), +}); + +export const ConfigStatusSchema = z + .object({ + enabled: z.boolean(), + source: ConfigSourceViewSchema.nullable(), + schemaUrl: z.string(), + }) + .describe("Whether a configuration directory feeds this workspace, and how its last sync went"); + +export const SyncQuerySchema = z.object({ + dryRun: z + .enum(["true", "false"]) + .optional() + .describe("`true` plans the directory without writing anything"), +}); + +export const ExportQuerySchema = z.object({ + kind: z + .string() + .optional() + .describe("Comma-separated manifest kinds (Work, Prompt, …); default every kind"), + id: z.string().optional().describe("One resource's id (with `kind`)"), +}); + +export const ExportedManifestSchema = z.object({ + kind: ManifestKindSchema, + name: z.string(), + path: z.string(), + document: z.unknown(), +}); + +export const ExportResponseSchema = z + .object({ manifests: z.array(ExportedManifestSchema) }) + .describe("The organization's resources as manifests"); + +export const ConfigErrorSchema = ErrorResponseSchema; + +/** `managedBy` on a row a configuration directory manages (shared `ManagedBy`). */ +export const ManagedBySchema = z + .object({ + objectId: z.string(), + sourceId: z.string(), + sourceName: z.string(), + path: z.string().describe("The manifest's file, relative to the directory"), + kind: z.string().describe("The manifest kind"), + }) + .describe("Set when a configuration directory manages the row: the file is the truth"); + +// The schemas accept the shared manifest types — fail the build if they drift. +const _work: WorkManifest = {} as z.infer; +const _prompt: PromptManifest = {} as z.infer; +const _mcp: McpServerManifest = {} as z.infer; +const _skill: SkillManifest = {} as z.infer; +const _connection: ConnectionManifest = {} as z.infer; +const _any: Manifest = {} as z.infer; +const _result: ConfigApplyResult = {} as z.infer; +void _work; +void _prompt; +void _mcp; +void _skill; +void _connection; +void _any; +void _result; diff --git a/apps/api/src/schemas/work.ts b/apps/api/src/schemas/work.ts index 4b9e3b8ff..bb23dd231 100644 --- a/apps/api/src/schemas/work.ts +++ b/apps/api/src/schemas/work.ts @@ -10,6 +10,7 @@ import { } from "@optio/shared"; import { AgentTypeSchema } from "./task.js"; import { WorkflowTriggerSchema } from "./workflow.js"; +import { ManagedBySchema } from "./config.js"; /** One entry of the Work list — see `WorkRow` in @optio/shared. */ export const WorkRowSchema = z @@ -63,6 +64,7 @@ export const WorkRowSchema = z .nullable() .optional() .describe("The owner's display name, for private work in an admin's list"), + managedBy: ManagedBySchema.nullable().optional(), }) .describe("A piece of work projected onto When / Where / Who / Then + a status"); diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index ee8c19785..6a515a8ae 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -25,6 +25,7 @@ import { ticketRoutes } from "./routes/tickets.js"; import { setupRoutes } from "./routes/setup.js"; import { authRoutes } from "./routes/auth.js"; import { signInRoutes } from "./routes/sign-in.js"; +import { configRoutes } from "./routes/config.js"; import { resumeRoutes } from "./routes/resume.js"; import { promptTemplateRoutes } from "./routes/prompt-templates.js"; import { repoRoutes } from "./routes/repos.js"; @@ -296,6 +297,7 @@ export async function buildServer() { await app.register(setupRoutes); await app.register(authRoutes); await app.register(signInRoutes); + await app.register(configRoutes); await app.register(resumeRoutes); await app.register(promptTemplateRoutes); await app.register(repoRoutes); diff --git a/apps/api/src/services/config/apply.ts b/apps/api/src/services/config/apply.ts new file mode 100644 index 000000000..88282b1a4 --- /dev/null +++ b/apps/api/src/services/config/apply.ts @@ -0,0 +1,405 @@ +/** + * The apply: make a workspace's resources match a set of manifests. Per + * manifest — validate, resolve, find the row it names, and create / update / + * leave it / recreate it; for a source, prune what it managed and no longer + * declares, and adopt existing rows with the manifest's name. Each manifest + * is its own unit: one bad file is one error item, the rest still apply. + * `dryRun` plans and writes nothing. docs/config-as-code.md. + */ +import { createHash } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { + MANIFEST_APPLY_ORDER, + isManifestKind, + stableStringify, + uninlinedFileFields, + type ConfigApplyResult, + type ConfigApplySummary, + type ConfigPlanItem, + type Manifest, + type ManifestInput, + type ManifestKind, +} from "@optio/shared"; +import { db } from "../../db/client.js"; +import { configObjects, configSources } from "../../db/schema.js"; +import { logger } from "../../logger.js"; +import { ManifestSchema } from "../../schemas/config.js"; +import { WorkError } from "../work-write-service.js"; +import { loadContext, ManifestError, type ResolveContext, type ResourceTable } from "./context.js"; +import { HANDLERS, type Identified } from "./kinds/index.js"; + +export type ConfigSourceRow = typeof configSources.$inferSelect; +type ObjectRow = typeof configObjects.$inferSelect; + +export interface ApplyOptions { + workspaceId: string | null; + manifests: ManifestInput[]; + dryRun: boolean; + /** The source the manifests come from; without one nothing becomes managed. */ + source?: ConfigSourceRow | null; + /** Delete what the source managed and no longer declares (sources only). */ + prune?: boolean; + /** Problems found before the apply (files that didn't parse), reported with the rest. */ + priorErrors?: ConfigPlanItem[]; +} + +interface Parsed { + path: string; + manifest: Manifest; + hash: string; +} + +const objectKey = (kind: string, name: string) => `${kind}\u0000${name}`; +const resourceKey = (table: string, id: string) => `${table}:${id}`; + +/** A manifest's content hash — what tells a changed file from a changed row. */ +export function manifestHash(manifest: unknown): string { + return createHash("sha256").update(stableStringify(manifest)).digest("hex").slice(0, 16); +} + +function emptySummary(): ConfigApplySummary { + return { + created: 0, + updated: 0, + reverted: 0, + unchanged: 0, + adopted: 0, + replaced: 0, + pruned: 0, + errors: 0, + }; +} + +function summarize(items: ConfigPlanItem[]): ConfigApplySummary { + const s = emptySummary(); + for (const item of items) { + switch (item.action) { + case "create": + s.created++; + break; + case "update": + s.updated++; + if (item.reverted) s.reverted++; + break; + case "unchanged": + s.unchanged++; + break; + case "adopt": + s.adopted++; + break; + case "replace": + s.replaced++; + break; + case "prune": + s.pruned++; + break; + case "error": + s.errors++; + break; + } + } + return s; +} + +/** What a document says it is, for an error about it. */ +function labelOf(document: unknown): { kind: string; name: string } { + const doc = (document ?? {}) as { kind?: unknown; metadata?: { name?: unknown } }; + return { + kind: typeof doc.kind === "string" ? doc.kind : "?", + name: typeof doc.metadata?.name === "string" ? doc.metadata.name : "?", + }; +} + +function errorItem(input: ManifestInput, message: string): ConfigPlanItem { + return { ...labelOf(input.document), path: input.path, action: "error", message }; +} + +/** Validate every input; invalid ones become error items. */ +function parse(inputs: ManifestInput[]): { parsed: Parsed[]; items: ConfigPlanItem[] } { + const parsed: Parsed[] = []; + const items: ConfigPlanItem[] = []; + const seen = new Set(); + for (const input of inputs) { + const left = uninlinedFileFields(input.document); + if (left.length) { + items.push( + errorItem(input, `${left.join(", ")}: file fields must be read in before an apply`), + ); + continue; + } + const result = ManifestSchema.safeParse(input.document); + if (!result.success) { + const problems = result.error.issues + .slice(0, 5) + .map((i) => `${i.path.join(".") || "document"}: ${i.message}`) + .join("; "); + items.push(errorItem(input, problems)); + continue; + } + const manifest = result.data as Manifest; + const key = objectKey(manifest.kind, manifest.metadata.name); + if (seen.has(key)) { + items.push( + errorItem(input, `${manifest.kind} "${manifest.metadata.name}" is declared more than once`), + ); + continue; + } + seen.add(key); + parsed.push({ path: input.path, manifest, hash: manifestHash(input.document) }); + } + return { parsed, items }; +} + +async function objectsOf(source: ConfigSourceRow | null | undefined): Promise { + if (!source) return []; + return db.select().from(configObjects).where(eq(configObjects.sourceId, source.id)); +} + +/** The source (any source) that manages a resource, if one does. */ +async function managerOf( + ident: Identified, +): Promise<{ object: ObjectRow; source: ConfigSourceRow } | null> { + const [found] = await db + .select({ object: configObjects, source: configSources }) + .from(configObjects) + .innerJoin(configSources, eq(configSources.id, configObjects.sourceId)) + .where( + and(eq(configObjects.resourceTable, ident.table), eq(configObjects.resourceId, ident.id)), + ); + return found ?? null; +} + +async function recordObject( + source: ConfigSourceRow, + kind: ManifestKind, + name: string, + path: string, + ident: Identified, + hash: string, +): Promise { + // The resource may have been managed under another name (a rename in the + // file): that bookkeeping goes, the new name's row points at the resource. + await db + .delete(configObjects) + .where( + and(eq(configObjects.resourceTable, ident.table), eq(configObjects.resourceId, ident.id)), + ); + await db + .insert(configObjects) + .values({ + sourceId: source.id, + workspaceId: source.workspaceId, + kind, + name, + path, + resourceTable: ident.table, + resourceId: ident.id, + hash, + appliedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [configObjects.sourceId, configObjects.kind, configObjects.name], + set: { + path, + resourceTable: ident.table, + resourceId: ident.id, + hash, + appliedAt: new Date(), + }, + }); +} + +function messageOf(err: unknown): string { + if (err instanceof ManifestError || err instanceof WorkError) return err.message; + const message = err instanceof Error ? err.message : String(err); + logger.warn({ err }, "config apply: unexpected error applying a manifest"); + return message; +} + +async function applyOne( + p: Parsed, + ctx: ResolveContext, + opts: ApplyOptions, + objects: Map, +): Promise<{ item: ConfigPlanItem; wrote: boolean }> { + const { kind } = p.manifest; + const name = p.manifest.metadata.name; + const handler = HANDLERS[kind]; + const base = { kind, name, path: p.path }; + const source = opts.source ?? null; + const object = objects.get(objectKey(kind, name)) ?? null; + const desired = await handler.desire(p.manifest, ctx); + + let row: unknown = object + ? await handler.get(object.resourceTable as ResourceTable, object.resourceId, ctx) + : null; + let adopted = false; + if (!row) { + row = await handler.find(desired, ctx); + if (row) { + const ident = handler.identify(row); + if (ident.ownerUserId) { + throw new ManifestError(`"${name}" exists, but it is someone's private ${kind}`); + } + const manager = await managerOf(ident); + if (manager && manager.source.id !== source?.id) { + throw new ManifestError( + `Already managed by ${manager.source.name} (${manager.object.path})`, + ); + } + adopted = !!source && !manager; + } + } + + const record = async (r: unknown) => { + if (source && !opts.dryRun) { + await recordObject(source, kind, name, p.path, handler.identify(r), p.hash); + } + }; + + if (!row) { + if (opts.dryRun) { + handler.stub?.(desired, ctx); + return { item: { ...base, action: "create" }, wrote: false }; + } + const created = await handler.create(desired, ctx); + await record(created); + return { + item: { ...base, action: "create", resourceId: handler.identify(created).id }, + wrote: true, + }; + } + + const ident = handler.identify(row); + const reason = handler.replaceReason(row, desired); + if (reason) { + if (opts.dryRun) { + handler.stub?.(desired, ctx); + return { + item: { ...base, action: "replace", resourceId: ident.id, message: reason }, + wrote: false, + }; + } + await handler.remove(ident.table, ident.id, ctx); + const created = await handler.create(desired, ctx); + await record(created); + return { + item: { + ...base, + action: "replace", + resourceId: handler.identify(created).id, + message: reason, + }, + wrote: true, + }; + } + + const changes = await handler.diff(row, desired, ctx); + if (changes.length === 0) { + await record(row); + return { + item: { ...base, action: adopted ? "adopt" : "unchanged", resourceId: ident.id }, + wrote: false, + }; + } + // The file is what it was when the row was last written, so the row moved: + // someone edited it in the UI, and the file wins. + const reverted = !!object && object.hash === p.hash; + const item: ConfigPlanItem = { + ...base, + action: "update", + resourceId: ident.id, + changes, + ...(reverted ? { reverted } : {}), + ...(adopted ? { message: "adopted an existing resource with this name" } : {}), + }; + if (opts.dryRun) return { item, wrote: false }; + const updated = await handler.update(row, desired, ctx); + await record(updated); + return { item, wrote: true }; +} + +/** Make the workspace match the manifests (or, `dryRun`, say what that would take). */ +export async function applyManifests(opts: ApplyOptions): Promise { + const { parsed, items } = parse(opts.manifests); + items.unshift(...(opts.priorErrors ?? [])); + const ctx = await loadContext(opts.workspaceId); + const objects = new Map( + (await objectsOf(opts.source)).map((o) => [objectKey(o.kind, o.name), o]), + ); + const declared = new Set(); + // A manifest that failed still "declares" its name: its resource is not pruned. + for (const input of opts.manifests) { + const { kind, name } = labelOf(input.document); + if (isManifestKind(kind)) declared.add(objectKey(kind, name)); + } + + for (const kind of MANIFEST_APPLY_ORDER) { + let wrote = false; + for (const p of parsed.filter((x) => x.manifest.kind === kind)) { + try { + const result = await applyOne(p, ctx, opts, objects); + items.push(result.item); + wrote ||= result.wrote; + } catch (err) { + items.push({ + kind, + name: p.manifest.metadata.name, + path: p.path, + action: "error", + message: messageOf(err), + }); + } + } + if (wrote) await ctx.reload(); + } + + // Prune (sources only): what the source managed that no manifest declares. + if (opts.source) { + const orphans = [...objects.values()].filter((o) => !declared.has(objectKey(o.kind, o.name))); + for (const o of orphans.sort( + (a, b) => a.kind.localeCompare(b.kind) || a.name.localeCompare(b.name), + )) { + const base = { kind: o.kind, name: o.name, path: o.path, resourceId: o.resourceId }; + if (!opts.prune) { + items.push({ ...base, action: "unchanged", message: "no longer declared; pruning is off" }); + continue; + } + if (opts.dryRun) { + items.push({ ...base, action: "prune" }); + continue; + } + try { + if (isManifestKind(o.kind)) { + await HANDLERS[o.kind].remove(o.resourceTable as ResourceTable, o.resourceId, ctx); + } + await db.delete(configObjects).where(eq(configObjects.id, o.id)); + items.push({ ...base, action: "prune" }); + } catch (err) { + items.push({ ...base, action: "error", message: `couldn't prune: ${messageOf(err)}` }); + } + } + } + + return { + dryRun: opts.dryRun, + source: opts.source ? { id: opts.source.id, name: opts.source.name } : null, + items, + summary: summarize(items), + at: new Date().toISOString(), + }; +} + +/** Stop managing one resource: its bookkeeping goes, the resource stays. */ +export async function detachObject(objectId: string, workspaceId: string | null): Promise { + const deleted = await db + .delete(configObjects) + .where( + and( + eq(configObjects.id, objectId), + workspaceId ? eq(configObjects.workspaceId, workspaceId) : undefined, + ), + ) + .returning({ id: configObjects.id }); + return deleted.length > 0; +} diff --git a/apps/api/src/services/config/compare.test.ts b/apps/api/src/services/config/compare.test.ts new file mode 100644 index 000000000..b76ef9e00 --- /dev/null +++ b/apps/api/src/services/config/compare.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { compact, fieldChanges, norm, sameSet } from "./compare.js"; + +describe("fieldChanges", () => { + it("names the desired fields that differ, treating null and undefined alike", () => { + const row = { name: "a", prompt: "p", settings: null, maxTurns: null, budgetUsd: "5.0" }; + expect(fieldChanges(row, { name: "a", prompt: "q" })).toEqual(["prompt"]); + expect(fieldChanges(row, { settings: undefined, maxTurns: null })).toEqual([]); + expect(fieldChanges(row, { budgetUsd: 5 })).toEqual([]); + expect(fieldChanges(row, { budgetUsd: "6" })).toEqual(["budgetUsd"]); + }); + + it("compares jsonb with keys in any order", () => { + const row = { settings: { connections: { add: ["x"] }, review: { enabled: true } } }; + expect( + fieldChanges(row, { settings: { review: { enabled: true }, connections: { add: ["x"] } } }), + ).toEqual([]); + expect(fieldChanges(row, { settings: { review: { enabled: false } } })).toEqual(["settings"]); + }); + + it("compares dates as instants", () => { + const at = new Date("2026-10-04T00:00:00Z"); + expect(norm(at)).toBe("2026-10-04T00:00:00.000Z"); + expect(fieldChanges({ at }, { at: new Date(at.getTime()) })).toEqual([]); + }); +}); + +describe("sameSet", () => { + it("ignores order and duplicates; null, undefined and [] are the same", () => { + expect(sameSet(["a", "b"], ["b", "a", "a"])).toBe(true); + expect(sameSet(null, [])).toBe(true); + expect(sameSet(undefined, null)).toBe(true); + expect(sameSet(["a"], ["b"])).toBe(false); + }); +}); + +describe("compact", () => { + it("drops null, undefined and empty objects but keeps false, 0 and arrays", () => { + expect(compact({ a: null, b: undefined, c: {}, d: false, e: 0, f: [], g: "x" })).toEqual({ + d: false, + e: 0, + f: [], + g: "x", + }); + }); +}); diff --git a/apps/api/src/services/config/compare.ts b/apps/api/src/services/config/compare.ts new file mode 100644 index 000000000..382b22f17 --- /dev/null +++ b/apps/api/src/services/config/compare.ts @@ -0,0 +1,59 @@ +/** + * How the apply decides a row already matches its manifest: each desired + * column, normalized, against the row's. Null and undefined are the same + * absence; numbers and numeric strings (`budgetUsd`) compare as numbers; + * jsonb compares with keys sorted; dates as ISO strings. + */ +import { stableStringify } from "@optio/shared"; + +export function norm(value: unknown): string { + if (value === undefined || value === null) return "null"; + if (value instanceof Date) return value.toISOString(); + if (typeof value === "number") return String(value); + if (typeof value === "string") { + return /^-?\d+(\.\d+)?$/.test(value.trim()) ? String(Number(value)) : value; + } + if (typeof value === "object") return stableStringify(value); + return String(value); +} + +/** The keys of `desired` whose value differs from the row's. */ +export function fieldChanges( + row: Record, + desired: Record, +): string[] { + const changed: string[] = []; + for (const [key, want] of Object.entries(desired)) { + if (want === undefined) continue; + if (norm(row[key]) !== norm(want)) changed.push(key); + } + return changed; +} + +/** Two lists as sets (order ignored); null, undefined and [] are the same. */ +export function sameSet( + a: readonly string[] | null | undefined, + b: readonly string[] | null | undefined, +): boolean { + const sa = [...new Set(a ?? [])].sort(); + const sb = [...new Set(b ?? [])].sort(); + return sa.length === sb.length && sa.every((v, i) => v === sb[i]); +} + +/** An object without its undefined, null and empty-object entries (what an export writes). */ +export function compact>(obj: T): Partial { + const out: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (value === undefined || value === null) continue; + if ( + typeof value === "object" && + !Array.isArray(value) && + !(value instanceof Date) && + Object.keys(value as Record).length === 0 + ) { + continue; + } + out[key] = value; + } + return out as Partial; +} diff --git a/apps/api/src/services/config/config-apply.int.test.ts b/apps/api/src/services/config/config-apply.int.test.ts new file mode 100644 index 000000000..e9cd4dc9f --- /dev/null +++ b/apps/api/src/services/config/config-apply.int.test.ts @@ -0,0 +1,434 @@ +/** + * The config apply against a real database: every kind created from a + * manifest, an apply that changes nothing, drift put back, a rename, a + * replace, pruning, adoption, per-manifest errors, and detach. The source is + * a row like the one OPTIO_CONFIG_DIR mirrors; the directory reader has its + * own unit test. + */ +import { and, eq } from "drizzle-orm"; +import { beforeAll, describe, expect, it } from "vitest"; +import type { ManifestInput } from "@optio/shared"; +import { db } from "../../db/client.js"; +import { configObjects, configSources, workDefinitions } from "../../db/schema.js"; +import { insertWorkspace } from "../../test-utils/integration/fixtures.js"; +import { storeSecret } from "../secret-service.js"; +import { seedBuiltInProviders, listConnections, getConnection } from "../connection-service.js"; +import { listMcpServers, updateMcpServer } from "../mcp-server-service.js"; +import { listPromptTemplates, updateNamedTemplate } from "../prompt-template-service.js"; +import { getRepoByUrl } from "../repo-service.js"; +import { listInstalledSkills } from "../installed-skill-service.js"; +import * as skillService from "../skill-service.js"; +import * as triggerService from "../trigger-service.js"; +import { listPersistentAgents } from "../persistent-agent-service.js"; +import { applyManifests, detachObject, type ConfigSourceRow } from "./apply.js"; +import { exportManifests } from "./export.js"; +import { managedByMap } from "./managed.js"; + +let ws: string; +let source: ConfigSourceRow; + +const REPO = "https://github.com/acme/api"; + +const m = ( + kind: string, + name: string, + spec: Record, + path = `${kind.toLowerCase()}s/${name}.yaml`, +): ManifestInput => ({ + path, + document: { apiVersion: "optio/v1", kind, metadata: { name }, spec }, +}); + +const baseline = (): ManifestInput[] => [ + m("Repo", "acme/api", { + url: REPO, + defaultBranch: "main", + reviewEnabled: true, + maxConcurrentTasks: 3, + }), + m("McpServer", "docs", { + command: "npx", + args: ["-y", "docs-mcp"], + env: { DOCS_TOKEN: "${{DOCS_TOKEN}}" }, + }), + m("Skill", "release-notes", { + prompt: "# Release notes", + files: { "scripts/gen.sh": "echo hi" }, + }), + m("Skill", "market", { source: { url: "https://github.com/acme/skills", path: "market" } }), + m("Connection", "Linear", { + provider: "linear", + config: { LINEAR_API_KEY: "${{LINEAR_API_KEY}}" }, + assignments: [{ repo: REPO, permission: "write" }], + }), + m("Prompt", "pr-description", { template: "Describe {{pr}}", kind: "prompt" }), + m("Work", "nightly-bump", { + when: { schedule: "0 3 * * 1-5" }, + where: { repo: REPO, branch: "main" }, + who: { runtime: "claude-code" }, + what: { prompt: "Bump dependencies" }, + then: "until-merged", + secrets: ["GITHUB_TOKEN"], + environment: { + connections: { add: ["Linear"] }, + mcpServers: { add: ["docs"] }, + skills: { add: ["release-notes"] }, + }, + params: { type: "object" }, + }), + m("Work", "hello-job", { who: { runtime: "shell" }, what: { prompt: "echo hello" } }), + m("Work", "helper", { + who: { runtime: "claude-code" }, + what: { prompt: "You help." }, + then: "waits-for-messages", + agent: { podLifecycle: "on-demand" }, + }), +]; + +beforeAll(async () => { + process.env.OPTIO_CONFIG_DIR = "/etc/optio-config"; // `managedByMap` only looks when the directory source is on + const workspace = await insertWorkspace({ slug: `cfg-${Date.now().toString(36)}` }); + ws = workspace.id; + await seedBuiltInProviders(); + // Instance-wide secrets (scope global, no workspace): pickable in every workspace. + await storeSecret("GITHUB_TOKEN", "ghp_x", "global", null, null); + await storeSecret("DOCS_TOKEN", "d", "global", null, null); + await storeSecret("LINEAR_API_KEY", "l", "global", null, null); + const [row] = await db + .insert(configSources) + .values({ + workspaceId: ws, + name: "config directory", + kind: "dir", + path: "/etc/optio-config", + prune: true, + }) + .returning(); + source = row; +}); + +const byKey = (result: Awaited>) => + new Map(result.items.map((i) => [`${i.kind}/${i.name}`, i])); + +async function apply(manifests: ManifestInput[], opts: { dryRun?: boolean; prune?: boolean } = {}) { + return applyManifests({ + workspaceId: ws, + manifests, + dryRun: opts.dryRun ?? false, + source, + prune: opts.prune ?? true, + }); +} + +describe("applyManifests", () => { + it("plans a fresh directory as creates and writes nothing in a dry run", async () => { + const plan = await apply(baseline(), { dryRun: true }); + expect(plan.dryRun).toBe(true); + expect(plan.summary).toMatchObject({ created: 9, errors: 0 }); + expect( + await db.select().from(configObjects).where(eq(configObjects.sourceId, source.id)), + ).toEqual([]); + }); + + it("creates every kind, in dependency order, with names resolved to ids", async () => { + const result = await apply(baseline()); + expect(result.summary).toMatchObject({ created: 9, errors: 0, updated: 0, pruned: 0 }); + + const repo = await getRepoByUrl(REPO, ws); + expect(repo).toMatchObject({ + reviewEnabled: true, + maxConcurrentTasks: 3, + defaultBranch: "main", + }); + + const [docs] = (await listMcpServers(undefined, ws)).filter((s) => s.name === "docs"); + expect(docs.env).toEqual({ DOCS_TOKEN: "${{DOCS_TOKEN}}" }); + + const custom = (await skillService.listSkills(undefined, ws)).find( + (s) => s.name === "release-notes", + ); + expect(custom).toMatchObject({ + layout: "skill-dir", + files: [{ relativePath: "scripts/gen.sh", content: "echo hi" }], + }); + expect( + (await listInstalledSkills(undefined, ws)).find((s) => s.name === "market"), + ).toMatchObject({ + sourceUrl: "https://github.com/acme/skills", + subpath: "market", + ref: "main", + }); + + const linear = (await listConnections(ws)).find((c) => c.name === "Linear")!; + const full = (await getConnection(linear.id))!; + expect(full.assignments).toHaveLength(1); + expect(full.assignments![0]).toMatchObject({ repoId: repo!.id, permission: "write" }); + + const [bump] = await db + .select() + .from(workDefinitions) + .where(and(eq(workDefinitions.workspaceId, ws), eq(workDefinitions.name, "nightly-bump"))); + expect(bump).toMatchObject({ + kind: "repo-blueprint", + repoUrl: REPO, + autoResume: true, + autoMerge: true, + podSecrets: ["GITHUB_TOKEN"], + ownerUserId: null, + paramsSchema: { type: "object" }, + }); + expect(bump.settings).toEqual({ + connections: { add: [linear.id] }, + mcpServers: { add: [docs.id] }, + skills: { add: [custom!.id] }, + }); + const triggers = await triggerService.listTriggers("task_config", bump.id); + expect(triggers.map((t) => [t.type, t.config])).toEqual([ + ["schedule", { cronExpression: "0 3 * * 1-5" }], + ]); + + const [job] = await db + .select() + .from(workDefinitions) + .where(and(eq(workDefinitions.workspaceId, ws), eq(workDefinitions.name, "hello-job"))); + expect(job).toMatchObject({ kind: "standalone", agentType: null, prompt: "echo hello" }); + + const agents = await listPersistentAgents(ws); + expect(agents.find((a) => a.slug === "helper")).toMatchObject({ + podLifecycle: "on-demand", + ownerUserId: null, + }); + + // Every row is managed by the source, under its manifest's path. + const managed = await managedByMap("work_definitions", [bump.id, job.id]); + expect(managed.get(bump.id)).toMatchObject({ + kind: "Work", + path: "works/nightly-bump.yaml", + sourceId: source.id, + }); + }); + + it("changes nothing on a second apply of the same files", async () => { + const result = await apply(baseline()); + expect(result.summary).toMatchObject({ + created: 0, + updated: 0, + unchanged: 9, + errors: 0, + pruned: 0, + }); + }); + + it("puts back a row edited by hand and reports it as reverted", async () => { + const prompt = (await listPromptTemplates({ workspaceId: ws })).find( + (p) => p.name === "pr-description", + )!; + await updateNamedTemplate(prompt.id, { template: "Edited in the UI" }); + const docs = (await listMcpServers(undefined, ws)).find((s) => s.name === "docs")!; + await updateMcpServer(docs.id, { command: "bunx" }); + + const result = await apply(baseline()); + const items = byKey(result); + expect(items.get("Prompt/pr-description")).toMatchObject({ + action: "update", + reverted: true, + changes: ["template"], + }); + expect(items.get("McpServer/docs")).toMatchObject({ + action: "update", + reverted: true, + changes: ["command"], + }); + expect(result.summary).toMatchObject({ updated: 2, reverted: 2 }); + expect( + (await listPromptTemplates({ workspaceId: ws })).find((p) => p.id === prompt.id)!.template, + ).toBe("Describe {{pr}}"); + }); + + it("applies a changed file as an update (not a revert), including the trigger", async () => { + const files = baseline().map((f) => + f.path === "works/nightly-bump.yaml" + ? m("Work", "nightly-bump", { + ...(f.document as { spec: Record }).spec, + when: { webhook: { path: "nightly" } }, + priority: 5, + }) + : f, + ); + const result = await apply(files); + const item = byKey(result).get("Work/nightly-bump")!; + expect(item.action).toBe("update"); + expect(item.reverted).toBeUndefined(); + expect(item.changes).toEqual(expect.arrayContaining(["priority", "when"])); + const [bump] = await db + .select() + .from(workDefinitions) + .where(and(eq(workDefinitions.workspaceId, ws), eq(workDefinitions.name, "nightly-bump"))); + expect(bump.priority).toBe(5); + const triggers = await triggerService.listTriggers("task_config", bump.id); + expect(triggers.map((t) => [t.type, t.config])).toEqual([["webhook", { path: "nightly" }]]); + // Back to the baseline for the tests below. + await apply(baseline()); + }); + + it("recreates a row whose kind changed, and prunes what the files no longer declare", async () => { + const files = baseline() + .filter((f) => f.path !== "works/helper.yaml") + .map((f) => + f.path === "works/hello-job.yaml" + ? m("Work", "hello-job", { + when: { schedule: "@daily" }, + where: { repo: REPO }, + who: { runtime: "claude-code" }, + what: { prompt: "now a scheduled Task" }, + }) + : f, + ); + const result = await apply(files); + const items = byKey(result); + expect(items.get("Work/hello-job")).toMatchObject({ action: "replace" }); + expect(items.get("Work/helper")).toMatchObject({ action: "prune" }); + expect(result.summary).toMatchObject({ replaced: 1, pruned: 1 }); + const [job] = await db + .select() + .from(workDefinitions) + .where(and(eq(workDefinitions.workspaceId, ws), eq(workDefinitions.name, "hello-job"))); + expect(job.kind).toBe("repo-blueprint"); + expect((await listPersistentAgents(ws)).find((a) => a.slug === "helper")).toBeUndefined(); + await apply(baseline()); + expect((await listPersistentAgents(ws)).find((a) => a.slug === "helper")).toBeDefined(); + }); + + it("only reports orphans when pruning is off", async () => { + const files = baseline().filter((f) => f.path !== "prompts/pr-description.yaml"); + const result = await apply(files, { prune: false }); + expect(byKey(result).get("Prompt/pr-description")).toMatchObject({ + action: "unchanged", + message: expect.stringMatching(/pruning is off/), + }); + expect( + (await listPromptTemplates({ workspaceId: ws })).some((p) => p.name === "pr-description"), + ).toBe(true); + }); + + it("fails one manifest at a time: unknown names, a credential in the clear, a one-off Task", async () => { + const files = [ + ...baseline(), + m("Work", "bad-env", { + who: { runtime: "shell" }, + what: { prompt: "x" }, + environment: { connections: { add: ["Nope"] } }, + }), + m("Work", "bad-secret", { + who: { runtime: "shell" }, + what: { prompt: "x" }, + secrets: ["MISSING"], + }), + m("Work", "one-off", { + where: { repo: REPO }, + who: { runtime: "claude-code" }, + what: { prompt: "x" }, + }), + m("Connection", "Leaky", { provider: "linear", config: { LINEAR_API_KEY: "lin_api_123" } }), + m("Prompt", "pr-description", { template: "dup" }, "prompts/dup.yaml"), + { path: "junk.yaml", document: { kind: "Work", metadata: { name: "junk" }, spec: {} } }, + ]; + const result = await apply(files); + const items = byKey(result); + expect(items.get("Work/bad-env")).toMatchObject({ + action: "error", + message: expect.stringMatching(/Unknown connection "Nope"/), + }); + expect(items.get("Work/bad-secret")).toMatchObject({ action: "error" }); + expect(items.get("Work/one-off")).toMatchObject({ + action: "error", + message: expect.stringMatching(/runs once/), + }); + expect(items.get("Connection/Leaky")).toMatchObject({ + action: "error", + message: expect.stringMatching(/credential/), + }); + expect(result.items.find((i) => i.path === "prompts/dup.yaml")).toMatchObject({ + action: "error", + message: expect.stringMatching(/more than once/), + }); + expect(result.items.find((i) => i.path === "junk.yaml")).toMatchObject({ action: "error" }); + expect(result.summary.errors).toBe(6); + // The good manifests still applied (nothing changed, nothing pruned). + expect(result.summary).toMatchObject({ unchanged: 9, pruned: 0 }); + }); + + it("adopts an existing unmanaged resource with the manifest's name", async () => { + const made = await skillService.createSkill({ name: "handmade", prompt: "by hand" }, ws); + const result = await apply([...baseline(), m("Skill", "handmade", { prompt: "by hand" })]); + expect(byKey(result).get("Skill/handmade")).toMatchObject({ + action: "adopt", + resourceId: made.id, + }); + expect((await managedByMap("custom_skills", [made.id])).get(made.id)).toMatchObject({ + kind: "Skill", + }); + // A later apply without it prunes it like any other managed row. + const pruned = await apply(baseline()); + expect(byKey(pruned).get("Skill/handmade")).toMatchObject({ action: "prune" }); + expect(await skillService.getSkill(made.id)).toBeNull(); + }); + + it("detaches: the bookkeeping goes, the row stays, the next apply adopts it again", async () => { + const docs = (await listMcpServers(undefined, ws)).find((s) => s.name === "docs")!; + const before = (await managedByMap("mcp_servers", [docs.id])).get(docs.id)!; + expect(await detachObject(before.objectId, ws)).toBe(true); + expect((await managedByMap("mcp_servers", [docs.id])).size).toBe(0); + expect(await detachObject(before.objectId, ws)).toBe(false); + const result = await apply(baseline()); + expect(byKey(result).get("McpServer/docs")).toMatchObject({ + action: "adopt", + resourceId: docs.id, + }); + }); + + it("exports what it applied, with names in place of ids and no secret values", async () => { + const exported = await exportManifests(ws); + const names = exported.map((e) => `${e.kind}/${e.name}`); + expect(names).toEqual( + expect.arrayContaining([ + "Repo/acme/api", + "McpServer/docs", + "Skill/release-notes", + "Skill/market", + "Connection/Linear", + "Prompt/pr-description", + "Work/nightly-bump", + "Work/hello-job", + "Work/helper", + ]), + ); + const bump = exported.find((e) => e.name === "nightly-bump")!.document as { + spec: Record; + }; + expect(bump.spec).toMatchObject({ + when: { schedule: "0 3 * * 1-5" }, + where: { repo: REPO, branch: "main" }, + who: { runtime: "claude-code" }, + then: "until-merged", + secrets: ["GITHUB_TOKEN"], + environment: { + connections: { add: ["Linear"] }, + mcpServers: { add: ["docs"] }, + skills: { add: ["release-notes"] }, + }, + }); + const linear = exported.find((e) => e.name === "Linear")!.document as { + spec: Record; + }; + expect(linear.spec).toMatchObject({ + provider: "linear", + config: { LINEAR_API_KEY: "${{LINEAR_API_KEY}}" }, + assignments: [{ repo: REPO, permission: "write" }], + }); + expect(exported.find((e) => e.name === "helper")!.path).toBe("work/helper.yaml"); + // Applying the export back changes nothing. + const again = await apply(exported.map((e) => ({ path: e.path, document: e.document }))); + expect(again.summary).toMatchObject({ errors: 0, created: 0, updated: 0, pruned: 0 }); + }); +}); diff --git a/apps/api/src/services/config/context.ts b/apps/api/src/services/config/context.ts new file mode 100644 index 000000000..e36a66e96 --- /dev/null +++ b/apps/api/src/services/config/context.ts @@ -0,0 +1,203 @@ +/** + * What an apply resolves manifest names against: the workspace's repos (by + * URL), connections, MCP servers and skills (by name), and the secrets a pod + * may be given (by name). Loaded once per apply and reloaded after each kind + * writes, so a Work manifest can use the connection declared next to it. + */ +import { + normalizeRepoUrl, + SECRET_REFERENCE, + type Connection, + type IdOverrides, + type McpServerConfig, + type NameOverrides, +} from "@optio/shared"; +import type { WorkActor } from "../work-ownership.js"; +import { listRepos, type RepoRecord } from "../repo-service.js"; +import { listConnections } from "../connection-service.js"; +import { listMcpServers } from "../mcp-server-service.js"; +import { listSkills } from "../skill-service.js"; +import { listInstalledSkills } from "../installed-skill-service.js"; +import { listPickableSecrets } from "../secret-service.js"; + +/** The tables a managed resource can live in (`config_objects.resource_table`). */ +export const RESOURCE_TABLES = [ + "work_definitions", + "persistent_agents", + "prompt_templates", + "repos", + "mcp_servers", + "custom_skills", + "installed_skills", + "connections", +] as const; +export type ResourceTable = (typeof RESOURCE_TABLES)[number]; + +/** The id a dry run gives a row it would create (handlers' `stub`); never written. */ +export const PLANNED_ID = "00000000-0000-4000-8000-000000000000"; + +/** A problem with one manifest: reported for it, the apply goes on. */ +export class ManifestError extends Error { + constructor(message: string) { + super(message); + this.name = "ManifestError"; + } +} + +export interface SkillRef { + table: "custom_skills" | "installed_skills"; + id: string; + name: string; +} + +export interface ResolveContext { + workspaceId: string | null; + /** The apply writes as the organization: no user, admin rights. */ + actor: WorkActor; + /** Repos by normalized URL. */ + repos: Map; + repoById: Map; + /** The organization's connections, MCP servers and skills, by name. */ + connections: Map; + connectionById: Map; + mcpServers: Map; + mcpById: Map; + skills: Map; + skillById: Map; + /** Secrets a pod in this workspace may be given, by name. */ + secretNames: Set; + /** Read everything again (after a kind created rows). */ + reload(): Promise; +} + +export async function loadContext(workspaceId: string | null): Promise { + const ctx: ResolveContext = { + workspaceId, + actor: { userId: null, workspaceId, isAdmin: true }, + repos: new Map(), + repoById: new Map(), + connections: new Map(), + connectionById: new Map(), + mcpServers: new Map(), + mcpById: new Map(), + skills: new Map(), + skillById: new Map(), + secretNames: new Set(), + reload: () => fill(ctx), + }; + await fill(ctx); + return ctx; +} + +async function fill(ctx: ResolveContext): Promise { + const ws = ctx.workspaceId; + const [repos, connections, mcpServers, customSkills, installedSkills, secrets] = + await Promise.all([ + listRepos(ws), + listConnections(ws), + listMcpServers(undefined, ws), + listSkills(undefined, ws), + listInstalledSkills(undefined, ws), + listPickableSecrets(ws, null), + ]); + ctx.repos = new Map(repos.map((r) => [normalizeRepoUrl(r.repoUrl), r])); + ctx.repoById = new Map(repos.map((r) => [r.id, r])); + const org = (rows: T[]) => + rows.filter((r) => !r.ownerUserId); + ctx.connections = new Map(org(connections).map((c) => [c.name, c])); + ctx.connectionById = new Map(org(connections).map((c) => [c.id, c])); + ctx.mcpServers = new Map(org(mcpServers).map((s) => [s.name, s])); + ctx.mcpById = new Map(org(mcpServers).map((s) => [s.id, s])); + const skills: SkillRef[] = [ + ...org(customSkills).map((s) => ({ table: "custom_skills" as const, id: s.id, name: s.name })), + ...org(installedSkills).map((s) => ({ + table: "installed_skills" as const, + id: s.id, + name: s.name, + })), + ]; + ctx.skills = new Map(skills.map((s) => [s.name, s])); + ctx.skillById = new Map(skills.map((s) => [s.id, s])); + ctx.secretNames = new Set(secrets.filter((s) => s.owner === "workspace").map((s) => s.name)); +} + +/** Names → ids for a `WorkSettings` override; an unknown name is the manifest's error. */ +export function resolveNames( + overrides: NameOverrides | undefined, + byName: Map, + what: string, +): IdOverrides | undefined { + if (!overrides) return undefined; + const ids = (names: string[] | undefined) => + names?.map((name) => { + const found = byName.get(name); + if (!found) throw new ManifestError(`Unknown ${what} "${name}"`); + return found.id; + }); + const add = ids(overrides.add); + const remove = ids(overrides.remove); + if (!add?.length && !remove?.length) return undefined; + return { ...(add?.length ? { add } : {}), ...(remove?.length ? { remove } : {}) }; +} + +/** Ids → names for an export; ids of things since deleted are dropped. */ +export function namesOf( + overrides: IdOverrides | undefined, + byId: Map, +): NameOverrides | undefined { + if (!overrides) return undefined; + const names = (ids: string[] | undefined) => + ids?.map((id) => byId.get(id)?.name).filter((n): n is string => !!n); + const add = names(overrides.add); + const remove = names(overrides.remove); + if (!add?.length && !remove?.length) return undefined; + return { ...(add?.length ? { add } : {}), ...(remove?.length ? { remove } : {}) }; +} + +/** The repo a manifest names, which must be registered in the workspace. */ +export function repoFor( + ctx: ResolveContext, + url: string | null | undefined, + field: string, +): RepoRecord | null { + if (!url) return null; + const repo = ctx.repos.get(normalizeRepoUrl(url)); + if (!repo) { + throw new ManifestError( + `${field}: ${url} isn't a repo of this workspace — add a Repo manifest for it, or register it under Repos`, + ); + } + return repo; +} + +/** The secret name a `${{NAME}}` value refers to, or null for a plain value. */ +export function secretReference(value: unknown): string | null { + if (typeof value !== "string") return null; + const match = SECRET_REFERENCE.exec(value); + return match ? match[1] : null; +} + +/** A secret the workspace's pods may be given — else the manifest's error. */ +export function requireSecret(ctx: ResolveContext, name: string, field: string): void { + if (!ctx.secretNames.has(name)) { + throw new ManifestError( + `${field}: no secret named "${name}" in this workspace — add it under Secrets first`, + ); + } +} + +/** Every `${{NAME}}` in a bag of strings must name an existing secret. */ +export function requireSecretReferences( + ctx: ResolveContext, + values: Record | string[] | null | undefined, + field: string, +): void { + if (!values) return; + const entries = Array.isArray(values) + ? values.map((v, i) => [String(i), v] as const) + : Object.entries(values); + for (const [key, value] of entries) { + const name = secretReference(value); + if (name) requireSecret(ctx, name, `${field}.${key}`); + } +} diff --git a/apps/api/src/services/config/env.ts b/apps/api/src/services/config/env.ts new file mode 100644 index 000000000..520d15fc8 --- /dev/null +++ b/apps/api/src/services/config/env.ts @@ -0,0 +1,30 @@ +/** + * What the deployment declares about config as code — `OPTIO_CONFIG_DIR` and + * its companions. Its own module, with no imports from the rest of the + * service, so the list decorators can ask "is this even on?" cheaply. + */ +import { parseIntEnv } from "@optio/shared"; + +export interface EnvConfigSource { + /** The directory in the API pod. */ + dir: string; + /** The workspace's slug, or null for the oldest workspace. */ + workspace: string | null; + /** Delete what the directory no longer declares (default true). */ + prune: boolean; + /** How often the directory is read (default 60s, at least 10s). */ + intervalMs: number; +} + +/** The directory source the env declares, or null when config as code is off. */ +export function envConfigSource(): EnvConfigSource | null { + const dir = process.env.OPTIO_CONFIG_DIR?.trim(); + if (!dir) return null; + const prune = (process.env.OPTIO_CONFIG_PRUNE ?? "true").trim().toLowerCase(); + return { + dir, + workspace: process.env.OPTIO_CONFIG_WORKSPACE?.trim() || null, + prune: !["false", "0", "no", "off"].includes(prune), + intervalMs: Math.max(10_000, parseIntEnv("OPTIO_CONFIG_INTERVAL", 60_000)), + }; +} diff --git a/apps/api/src/services/config/export.ts b/apps/api/src/services/config/export.ts new file mode 100644 index 000000000..fab61432f --- /dev/null +++ b/apps/api/src/services/config/export.ts @@ -0,0 +1,67 @@ +/** + * Export: the organization's resources as manifests — the way to start a + * configuration directory from what a workspace already has. Only the + * organization's rows (never anyone's private ones), never secret values. + */ +import YAML from "yaml"; +import { + MANIFEST_APPLY_ORDER, + MANIFEST_KIND_DIRS, + type ExportedManifest, + type Manifest, + type ManifestKind, +} from "@optio/shared"; +import { loadContext } from "./context.js"; +import { HANDLERS } from "./kinds/index.js"; + +/** Where an export files a manifest: `work/nightly-bump.yaml`. */ +export function manifestFilePath(kind: ManifestKind, name: string): string { + const file = + name + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80) || "unnamed"; + return `${MANIFEST_KIND_DIRS[kind]}/${file}.yaml`; +} + +export async function exportManifests( + workspaceId: string | null, + opts: { kinds?: ManifestKind[]; id?: string } = {}, +): Promise { + const ctx = await loadContext(workspaceId); + const kinds = opts.kinds?.length + ? MANIFEST_APPLY_ORDER.filter((k) => opts.kinds!.includes(k)) + : MANIFEST_APPLY_ORDER; + const out: ExportedManifest[] = []; + for (const kind of kinds) { + const handler = HANDLERS[kind]; + const rows = await handler.list(ctx); + for (const row of rows) { + const ident = handler.identify(row); + if (opts.id && ident.id !== opts.id) continue; + const document = (await handler.export(row, ctx)) as Manifest; + out.push({ + kind, + name: document.metadata.name, + path: manifestFilePath(kind, document.metadata.name), + document, + }); + } + } + return out; +} + +/** One YAML text: a header comment, then every manifest, `---` between them. */ +export function manifestsToYaml(items: ExportedManifest[], schemaUrl?: string): string { + const header = [ + schemaUrl ? `# yaml-language-server: $schema=${schemaUrl}` : null, + "# Exported from Optio. Secrets are referenced by name, never written here.", + ] + .filter(Boolean) + .join("\n"); + const docs = items.map((item) => + YAML.stringify(item.document, { lineWidth: 100, indent: 2, blockQuote: "literal" }).trimEnd(), + ); + return `${header}\n${docs.join("\n---\n")}\n`; +} diff --git a/apps/api/src/services/config/files.test.ts b/apps/api/src/services/config/files.test.ts new file mode 100644 index 000000000..8b55035b9 --- /dev/null +++ b/apps/api/src/services/config/files.test.ts @@ -0,0 +1,106 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { readManifestDirectory } from "./files.js"; + +let dir: string; + +beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "optio-config-")); +}); + +afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); +}); + +async function write(rel: string, text: string) { + const file = path.join(dir, rel); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, text); +} + +describe("readManifestDirectory", () => { + it("reads every yaml file recursively, every document in each, inlining file fields", async () => { + await write( + "work/nightly.yaml", + [ + "apiVersion: optio/v1", + "kind: Work", + "metadata: { name: nightly }", + "spec:", + " who: { runtime: claude-code }", + " what: { promptFile: ./nightly.md }", + ].join("\n"), + ); + await write("work/nightly.md", "Bump things\n"); + await write( + "prompts.yml", + [ + "apiVersion: optio/v1", + "kind: Prompt", + "metadata: { name: one }", + "spec: { template: one }", + "---", + "apiVersion: optio/v1", + "kind: Prompt", + "metadata: { name: two }", + "spec: { template: two }", + ].join("\n"), + ); + await write(".hidden/skip.yaml", "kind: Nope"); + await write("notes.txt", "not yaml"); + + const read = await readManifestDirectory(dir); + expect(read.errors).toEqual([]); + expect(read.files).toBe(2); + expect(read.manifests.map((m) => m.path)).toEqual([ + "prompts.yml#1", + "prompts.yml#2", + "work/nightly.yaml", + ]); + const work = read.manifests[2].document as { + spec: { what: { prompt: string; promptFile?: string } }; + }; + expect(work.spec.what).toEqual({ prompt: "Bump things\n" }); + expect(read.hash).toMatch(/^[0-9a-f]{12}$/); + }); + + it("reports a file that doesn't parse and a missing included file, and reads the rest", async () => { + await write("bad.yaml", "kind: [unterminated"); + await write( + "missing.yaml", + "kind: Prompt\nmetadata: { name: m }\nspec: { templateFile: nope.md }", + ); + await write("ok.yaml", "kind: Prompt\nmetadata: { name: ok }\nspec: { template: fine }"); + const read = await readManifestDirectory(dir); + expect(read.manifests.map((m) => m.path)).toEqual(["ok.yaml"]); + expect(read.errors.map((e) => [e.path, e.action])).toEqual([ + ["bad.yaml", "error"], + ["missing.yaml", "error"], + ]); + expect(read.errors[1].name).toBe("m"); + expect(read.errors[1].message).toMatch(/nope\.md/); + }); + + it("refuses an included file outside the directory", async () => { + await write( + "escape.yaml", + "kind: Prompt\nmetadata: { name: e }\nspec: { templateFile: ../../etc/passwd }", + ); + const read = await readManifestDirectory(dir); + expect(read.manifests).toEqual([]); + expect(read.errors[0].message).toMatch(/outside the configuration directory/); + }); + + it("changes its hash when a file changes", async () => { + await write("a.yaml", "kind: Prompt\nmetadata: { name: a }\nspec: { template: one }"); + const first = (await readManifestDirectory(dir)).hash; + await write("a.yaml", "kind: Prompt\nmetadata: { name: a }\nspec: { template: two }"); + expect((await readManifestDirectory(dir)).hash).not.toBe(first); + }); + + it("throws when the directory doesn't exist", async () => { + await expect(readManifestDirectory(path.join(dir, "nope"))).rejects.toThrow(); + }); +}); diff --git a/apps/api/src/services/config/files.ts b/apps/api/src/services/config/files.ts new file mode 100644 index 000000000..bea717763 --- /dev/null +++ b/apps/api/src/services/config/files.ts @@ -0,0 +1,141 @@ +/** + * Reading a configuration directory: every `*.yaml` / `*.yml` under it + * (recursively, dot-entries skipped), each file's documents parsed, their + * `*File` fields read from next to the file, and a hash of the whole tree. + * A file that doesn't parse is one error item; the rest still apply. + */ +import { createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import YAML from "yaml"; +import { + inlineManifestFiles, + type ConfigPlanItem, + type ManifestFileReader, + type ManifestInput, +} from "@optio/shared"; + +export interface DirectoryRead { + manifests: ManifestInput[]; + errors: ConfigPlanItem[]; + /** A short hash of every file's path and contents. */ + hash: string; + /** How many files were read. */ + files: number; +} + +const MAX_FILE_BYTES = 2 * 1024 * 1024; + +async function* walk(root: string, dir: string): AsyncGenerator { + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.name.startsWith(".") || entry.name === "node_modules") continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) yield* walk(root, full); + else if (entry.isFile() && /\.ya?ml$/i.test(entry.name)) yield full; + } +} + +/** Files a manifest names must stay inside the directory. */ +function inside(root: string, candidate: string): string { + const resolved = path.resolve(candidate); + const rootResolved = path.resolve(root); + if (resolved !== rootResolved && !resolved.startsWith(rootResolved + path.sep)) { + throw new Error(`${candidate} is outside the configuration directory`); + } + return resolved; +} + +/** A reader for the `*File` fields of a manifest at `file`. */ +export function readerFor(root: string, file: string): ManifestFileReader { + const dir = path.dirname(file); + return { + async readText(relativePath) { + const target = inside(root, path.resolve(dir, relativePath)); + const stat = await fs.stat(target); + if (stat.size > MAX_FILE_BYTES) throw new Error(`${relativePath} is larger than 2 MiB`); + return fs.readFile(target, "utf8"); + }, + async readDir(relativePath) { + const target = inside(root, path.resolve(dir, relativePath)); + const out: Record = {}; + const visit = async (current: string, rel: string) => { + for (const entry of await fs.readdir(current, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + const full = path.join(current, entry.name); + const relPath = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) await visit(full, relPath); + else if (entry.isFile()) { + const stat = await fs.stat(full); + if (stat.size > MAX_FILE_BYTES) throw new Error(`${relPath} is larger than 2 MiB`); + out[relPath] = await fs.readFile(full, "utf8"); + } + } + }; + await visit(target, ""); + return out; + }, + }; +} + +/** The documents of one YAML file, `*File` fields inlined; parse problems as error items. */ +export async function readManifestFile( + root: string, + file: string, +): Promise<{ manifests: ManifestInput[]; errors: ConfigPlanItem[]; text: string }> { + const relative = path.relative(root, file).split(path.sep).join("/"); + const text = await fs.readFile(file, "utf8"); + const manifests: ManifestInput[] = []; + const errors: ConfigPlanItem[] = []; + const documents = YAML.parseAllDocuments(text); + const many = documents.length > 1; + for (const [index, doc] of documents.entries()) { + const docPath = many ? `${relative}#${index + 1}` : relative; + if (doc.errors.length) { + errors.push({ + kind: "?", + name: "?", + path: docPath, + action: "error", + message: doc.errors.map((e) => e.message.split("\n")[0]).join("; "), + }); + continue; + } + const value = doc.toJS() as unknown; + if (value === null || value === undefined) continue; + try { + manifests.push({ + path: docPath, + document: await inlineManifestFiles(value, readerFor(root, file)), + }); + } catch (err) { + const label = (value ?? {}) as { kind?: unknown; metadata?: { name?: unknown } }; + errors.push({ + kind: typeof label.kind === "string" ? label.kind : "?", + name: typeof label.metadata?.name === "string" ? label.metadata.name : "?", + path: docPath, + action: "error", + message: err instanceof Error ? err.message : String(err), + }); + } + } + return { manifests, errors, text }; +} + +/** Every manifest under `root`. Throws when the directory can't be read at all. */ +export async function readManifestDirectory(root: string): Promise { + const stat = await fs.stat(root); + if (!stat.isDirectory()) throw new Error(`${root} is not a directory`); + const manifests: ManifestInput[] = []; + const errors: ConfigPlanItem[] = []; + const hash = createHash("sha256"); + let files = 0; + for await (const file of walk(root, root)) { + files++; + const read = await readManifestFile(root, file); + hash.update(path.relative(root, file)).update("\0").update(read.text).update("\0"); + manifests.push(...read.manifests); + errors.push(...read.errors); + } + return { manifests, errors, hash: hash.digest("hex").slice(0, 12), files }; +} diff --git a/apps/api/src/services/config/kinds/connection.ts b/apps/api/src/services/config/kinds/connection.ts new file mode 100644 index 000000000..96451984f --- /dev/null +++ b/apps/api/src/services/config/kinds/connection.ts @@ -0,0 +1,242 @@ +/** + * `kind: Connection` — an external service integration. Its provider's config + * schema marks the fields that hold credentials (`format: "secret"`); a + * manifest writes those as `${{SECRET_NAME}}` references and never values, + * and an export writes the reference back (never a stored literal). The + * provider and the scope can't change in place: either recreates the row. + */ +import { + MANIFEST_API_VERSION, + normalizeRepoUrl, + type Connection, + type ConnectionManifest, + type ConnectionProvider, +} from "@optio/shared"; +import { + createAssignment, + createConnection, + deleteAssignment, + deleteConnection, + getConnection, + getProviderBySlug, + listConnections, + listProviders, + updateConnection, +} from "../../connection-service.js"; +import { + ManifestError, + PLANNED_ID, + repoFor, + requireSecret, + secretReference, + type ResolveContext, + type ResourceTable, +} from "../context.js"; +import { compact, fieldChanges, norm } from "../compare.js"; +import type { Identified, KindHandler } from "./index.js"; + +interface DesiredAssignment { + repoId: string | null; + agentTypes: string[]; + permission: string; +} + +interface DesiredConnection { + name: string; + providerId: string; + providerSlug: string; + config: Record; + repoUrl: string | null; + enabled: boolean; + assignments: DesiredAssignment[]; +} + +/** The config keys a provider marks as credentials. */ +function secretFields( + provider: Pick | null | undefined, +): Set { + const props = ( + provider?.configSchema as { properties?: Record } | null + )?.properties; + return new Set( + Object.entries(props ?? {}) + .filter(([, def]) => def?.format === "secret") + .map(([key]) => key), + ); +} + +async function desire(m: ConnectionManifest, ctx: ResolveContext): Promise { + const s = m.spec; + const provider = await getProviderBySlug(s.provider, ctx.workspaceId); + if (!provider) throw new ManifestError(`provider: no connection provider "${s.provider}"`); + const secrets = secretFields(provider); + const config = { ...(s.config ?? {}) }; + for (const [key, value] of Object.entries(config)) { + const ref = secretReference(value); + if (ref) { + requireSecret(ctx, ref, `config.${key}`); + } else if (secrets.has(key) && value !== undefined && value !== null && value !== "") { + throw new ManifestError( + `config.${key} holds a credential — store it as a secret and write \${{SECRET_NAME}} here`, + ); + } + } + const repo = repoFor(ctx, s.repo, "repo"); + const assignments = (s.assignments ?? []).map((a, i) => ({ + repoId: repoFor(ctx, a.repo, `assignments[${i}].repo`)?.id ?? null, + agentTypes: [...(a.agentTypes ?? [])].sort(), + permission: a.permission ?? "read", + })); + return { + name: m.metadata.name, + providerId: provider.id, + providerSlug: provider.slug, + config, + repoUrl: repo ? normalizeRepoUrl(repo.repoUrl) : null, + enabled: s.enabled ?? true, + assignments, + }; +} + +async function workspaceRows(ctx: ResolveContext): Promise { + const rows = await listConnections(ctx.workspaceId); + return rows.filter((r) => (r.workspaceId ?? null) === ctx.workspaceId); +} + +async function find(d: DesiredConnection, ctx: ResolveContext): Promise { + const rows = (await workspaceRows(ctx)).filter((r) => r.name === d.name); + const org = rows.find((r) => !r.ownerUserId) ?? null; + if (!org && rows.length) { + throw new ManifestError( + `A connection named "${d.name}" exists, but it is someone's private connection`, + ); + } + return org ? getConnection(org.id) : null; +} + +async function get(_table: ResourceTable, id: string): Promise { + return getConnection(id); +} + +function identify(row: Connection): Identified { + return { table: "connections", id: row.id, name: row.name, ownerUserId: row.ownerUserId ?? null }; +} + +function replaceReason(row: Connection, d: DesiredConnection): string | null { + if (row.providerId !== d.providerId) return "its provider changed — recreated"; + const was = row.repoUrl ? normalizeRepoUrl(row.repoUrl) : null; + return norm(was) === norm(d.repoUrl) ? null : "its scope (repo / workspace) changed — recreated"; +} + +const assignmentKey = (a: { + repoId?: string | null; + agentTypes?: string[] | null; + permission?: string | null; + enabled?: boolean; +}) => + `${a.repoId ?? ""}|${[...(a.agentTypes ?? [])].sort().join(",")}|${a.permission ?? "read"}|${a.enabled === false ? "off" : "on"}`; + +function assignmentsDiffer(row: Connection, d: DesiredConnection): boolean { + const have = (row.assignments ?? []).map(assignmentKey).sort(); + const want = d.assignments.map(assignmentKey).sort(); + return have.length !== want.length || have.some((k, i) => k !== want[i]); +} + +async function diff(row: Connection, d: DesiredConnection): Promise { + const changes = fieldChanges(row as unknown as Record, { + name: d.name, + config: d.config, + enabled: d.enabled, + }); + if (assignmentsDiffer(row, d)) changes.push("assignments"); + if (row.ownerUserId) changes.push("owner"); + return changes; +} + +async function create(d: DesiredConnection, ctx: ResolveContext): Promise { + return createConnection( + { + name: d.name, + providerId: d.providerId, + config: d.config, + repoUrl: d.repoUrl ?? undefined, + scope: d.repoUrl ?? "global", + enabled: d.enabled, + ownerUserId: null, + assignments: d.assignments, + }, + ctx.workspaceId, + ); +} + +async function update(row: Connection, d: DesiredConnection): Promise { + await updateConnection(row.id, { name: d.name, config: d.config, enabled: d.enabled }); + const have = new Map((row.assignments ?? []).map((a) => [assignmentKey(a), a])); + const want = new Map(d.assignments.map((a) => [assignmentKey(a), a])); + for (const [key, a] of have) if (!want.has(key)) await deleteAssignment(a.id); + for (const [key, a] of want) if (!have.has(key)) await createAssignment(row.id, a); + return (await getConnection(row.id)) ?? row; +} + +async function remove(_table: ResourceTable, id: string): Promise { + await deleteConnection(id); +} + +async function list(ctx: ResolveContext): Promise { + const rows = (await workspaceRows(ctx)).filter((r) => !r.ownerUserId); + const full = await Promise.all(rows.map((r) => getConnection(r.id))); + return full.filter((r): r is Connection => !!r).sort((a, b) => a.name.localeCompare(b.name)); +} + +async function exportConnection(row: Connection, ctx: ResolveContext): Promise { + const provider = + row.provider ?? + (await listProviders(ctx.workspaceId)).find((p) => p.id === row.providerId) ?? + null; + const secrets = secretFields(provider); + const config: Record = {}; + for (const [key, value] of Object.entries(row.config ?? {})) { + if (value === undefined || value === null || value === "") continue; + // A stored credential never leaves the row: the reference the pod falls + // back to (the secret named like the key) stands in for it. + config[key] = secrets.has(key) && !secretReference(value) ? `\${{${key}}}` : value; + } + const assignments = (row.assignments ?? []).map((a) => + compact({ + repo: a.repoId ? ctx.repoById.get(a.repoId)?.repoUrl : undefined, + agentTypes: a.agentTypes?.length ? a.agentTypes : undefined, + permission: a.permission === "read" ? undefined : (a.permission as "write" | "full"), + }), + ); + return { + apiVersion: MANIFEST_API_VERSION, + kind: "Connection", + metadata: { name: row.name }, + spec: compact({ + provider: provider?.slug ?? row.providerId, + config: Object.keys(config).length ? config : undefined, + repo: row.repoUrl ?? undefined, + enabled: row.enabled ? undefined : false, + assignments: assignments.length ? assignments : undefined, + }) as ConnectionManifest["spec"], + }; +} + +export const connectionHandler: KindHandler = { + kind: "Connection", + stub: (d, ctx) => { + ctx.connections.set(d.name, { id: PLANNED_ID, name: d.name } as Connection); + }, + desire, + tableOf: () => "connections", + find, + get, + identify, + replaceReason, + diff, + create, + update, + remove, + list, + export: exportConnection, +}; diff --git a/apps/api/src/services/config/kinds/index.ts b/apps/api/src/services/config/kinds/index.ts new file mode 100644 index 000000000..18767f52c --- /dev/null +++ b/apps/api/src/services/config/kinds/index.ts @@ -0,0 +1,65 @@ +/** + * One handler per manifest kind: how a manifest becomes the rows it + * describes (`desire`), which row it names (`find`), whether that row already + * matches (`diff`), and how to create, update, remove and export one. The + * apply engine (services/config/apply.ts) is generic over these. + */ +import type { Manifest, ManifestKind } from "@optio/shared"; +import type { ResolveContext, ResourceTable } from "../context.js"; +import { workHandler } from "./work.js"; +import { promptHandler } from "./prompt.js"; +import { repoHandler } from "./repo.js"; +import { mcpServerHandler } from "./mcp-server.js"; +import { skillHandler } from "./skill.js"; +import { connectionHandler } from "./connection.js"; + +/** Which row a handler's value is. */ +export interface Identified { + table: ResourceTable; + id: string; + name: string; + /** Set when the row is someone's private one (a manifest can't manage those). */ + ownerUserId: string | null; +} + +export interface KindHandler { + kind: ManifestKind; + /** Resolve the manifest (names → ids, defaults) or throw `ManifestError`. */ + desire(manifest: M, ctx: ResolveContext): Promise; + /** The table the desired rows live in. */ + tableOf(desired: D): ResourceTable; + /** The existing row the manifest names in this workspace, if any (what an apply adopts). */ + find(desired: D, ctx: ResolveContext): Promise; + /** A managed row by its bookkeeping (null when it was deleted by hand). */ + get(table: ResourceTable, id: string, ctx: ResolveContext): Promise; + identify(row: R): Identified; + /** Why `row` can't be updated into `desired` and has to be recreated, or null. */ + replaceReason(row: R, desired: D): string | null; + /** + * A dry run writes nothing, so a manifest applied later that names this one + * (a Work's connection, a Connection's repo) would not resolve: register + * the would-be row in the context under its name, with a placeholder id. + */ + stub?(desired: D, ctx: ResolveContext): void; + /** The fields that differ ([] = the row already matches). */ + diff(row: R, desired: D, ctx: ResolveContext): Promise; + create(desired: D, ctx: ResolveContext): Promise; + update(row: R, desired: D, ctx: ResolveContext): Promise; + remove(table: ResourceTable, id: string, ctx: ResolveContext): Promise; + /** The organization's rows of this kind in the workspace (for an export). */ + list(ctx: ResolveContext): Promise; + export(row: R, ctx: ResolveContext): Promise; +} + +export type AnyHandler = KindHandler; + +const h = (handler: unknown) => handler as AnyHandler; + +export const HANDLERS: Record = { + Work: h(workHandler), + Prompt: h(promptHandler), + Repo: h(repoHandler), + McpServer: h(mcpServerHandler), + Skill: h(skillHandler), + Connection: h(connectionHandler), +}; diff --git a/apps/api/src/services/config/kinds/mcp-server.ts b/apps/api/src/services/config/kinds/mcp-server.ts new file mode 100644 index 000000000..32ee12f88 --- /dev/null +++ b/apps/api/src/services/config/kinds/mcp-server.ts @@ -0,0 +1,177 @@ +/** + * `kind: McpServer` — an MCP server the workspace's pods get. Env values may + * be `${{SECRET_NAME}}` references (resolved in the pod, never stored here as + * values); the secret has to exist. Its scope (a repo or the workspace) can't + * change in place, so a scope change recreates the row. + */ +import { + MANIFEST_API_VERSION, + normalizeRepoUrl, + type McpServerConfig, + type McpServerManifest, +} from "@optio/shared"; +import { + createMcpServer, + deleteMcpServer, + getMcpServer, + listMcpServers, + updateMcpServer, +} from "../../mcp-server-service.js"; +import { + ManifestError, + PLANNED_ID, + repoFor, + requireSecretReferences, + type ResolveContext, + type ResourceTable, +} from "../context.js"; +import { compact, fieldChanges, norm } from "../compare.js"; +import type { Identified, KindHandler } from "./index.js"; + +interface DesiredMcpServer { + name: string; + command: string; + args: string[]; + env: Record | null; + installCommand: string | null; + repoUrl: string | null; + enabled: boolean; +} + +const envOf = (env: Record | null | undefined) => + env && Object.keys(env).length ? env : null; + +async function desire(m: McpServerManifest, ctx: ResolveContext): Promise { + const s = m.spec; + requireSecretReferences(ctx, s.env, "env"); + requireSecretReferences(ctx, s.args, "args"); + const repo = repoFor(ctx, s.repo, "repo"); + return { + name: m.metadata.name, + command: s.command, + args: s.args ?? [], + env: envOf(s.env), + installCommand: s.installCommand ?? null, + repoUrl: repo ? normalizeRepoUrl(repo.repoUrl) : null, + enabled: s.enabled ?? true, + }; +} + +async function workspaceRows(ctx: ResolveContext): Promise { + const rows = await listMcpServers(undefined, ctx.workspaceId); + return rows.filter((r) => (r.workspaceId ?? null) === ctx.workspaceId); +} + +async function find(d: DesiredMcpServer, ctx: ResolveContext): Promise { + const rows = await workspaceRows(ctx); + const row = rows.find((r) => r.name === d.name && !r.ownerUserId) ?? null; + if (!row && rows.some((r) => r.name === d.name)) { + throw new ManifestError( + `An MCP server named "${d.name}" exists, but it is someone's private server`, + ); + } + return row; +} + +async function get(_table: ResourceTable, id: string): Promise { + return getMcpServer(id); +} + +function identify(row: McpServerConfig): Identified { + return { table: "mcp_servers", id: row.id, name: row.name, ownerUserId: row.ownerUserId ?? null }; +} + +function replaceReason(row: McpServerConfig, d: DesiredMcpServer): string | null { + const was = row.repoUrl ? normalizeRepoUrl(row.repoUrl) : null; + return norm(was) === norm(d.repoUrl) ? null : "its scope (repo / workspace) changed — recreated"; +} + +async function diff(row: McpServerConfig, d: DesiredMcpServer): Promise { + const changes = fieldChanges( + { ...row, env: envOf(row.env) }, + { + name: d.name, + command: d.command, + args: d.args, + env: d.env, + installCommand: d.installCommand, + enabled: d.enabled, + }, + ); + if (row.ownerUserId) changes.push("owner"); + return changes; +} + +async function create(d: DesiredMcpServer, ctx: ResolveContext): Promise { + return createMcpServer( + { + name: d.name, + command: d.command, + args: d.args, + env: d.env ?? undefined, + installCommand: d.installCommand ?? undefined, + repoUrl: d.repoUrl ?? undefined, + enabled: d.enabled, + ownerUserId: null, + }, + ctx.workspaceId, + ); +} + +async function update(row: McpServerConfig, d: DesiredMcpServer): Promise { + return ( + (await updateMcpServer(row.id, { + name: d.name, + command: d.command, + args: d.args, + env: d.env, + installCommand: d.installCommand, + enabled: d.enabled, + })) ?? row + ); +} + +async function remove(_table: ResourceTable, id: string): Promise { + await deleteMcpServer(id); +} + +async function list(ctx: ResolveContext): Promise { + return (await workspaceRows(ctx)) + .filter((r) => !r.ownerUserId) + .sort((a, b) => a.name.localeCompare(b.name)); +} + +async function exportServer(row: McpServerConfig): Promise { + return { + apiVersion: MANIFEST_API_VERSION, + kind: "McpServer", + metadata: { name: row.name }, + spec: compact({ + command: row.command, + args: row.args.length ? row.args : undefined, + env: envOf(row.env) ?? undefined, + installCommand: row.installCommand ?? undefined, + repo: row.repoUrl ?? undefined, + enabled: row.enabled ? undefined : false, + }) as McpServerManifest["spec"], + }; +} + +export const mcpServerHandler: KindHandler = { + kind: "McpServer", + stub: (d, ctx) => { + ctx.mcpServers.set(d.name, { id: PLANNED_ID, name: d.name } as McpServerConfig); + }, + desire, + tableOf: () => "mcp_servers", + find, + get, + identify, + replaceReason, + diff, + create, + update, + remove, + list, + export: exportServer, +}; diff --git a/apps/api/src/services/config/kinds/prompt.ts b/apps/api/src/services/config/kinds/prompt.ts new file mode 100644 index 000000000..f9b12d644 --- /dev/null +++ b/apps/api/src/services/config/kinds/prompt.ts @@ -0,0 +1,139 @@ +/** `kind: Prompt` — a named prompt template (`prompt_templates`). */ +import { MANIFEST_API_VERSION, type PromptManifest } from "@optio/shared"; +import { promptTemplates } from "../../../db/schema.js"; +import { + createNamedTemplate, + deleteNamedTemplate, + getPromptTemplateById, + listPromptTemplates, + updateNamedTemplate, +} from "../../prompt-template-service.js"; +import { ManifestError, type ResolveContext, type ResourceTable } from "../context.js"; +import { compact, fieldChanges } from "../compare.js"; +import type { Identified, KindHandler } from "./index.js"; + +type Row = typeof promptTemplates.$inferSelect; + +interface DesiredPrompt { + name: string; + template: string; + kind: string; + description: string | null; + paramsSchema: Record | null; + defaultAgentType: string | null; +} + +const NAMED_KINDS = new Set(["prompt", "review", "job", "task"]); + +async function desire(m: PromptManifest): Promise { + if (m.spec.template === undefined) { + throw new ManifestError("templateFile must be read into template before an apply"); + } + return { + name: m.metadata.name, + template: m.spec.template, + kind: m.spec.kind ?? "prompt", + description: m.metadata.description ?? null, + paramsSchema: m.spec.params ?? null, + defaultAgentType: m.spec.defaultAgentType ?? null, + }; +} + +/** The workspace's own named prompts (not the instance-wide system rows). */ +async function workspaceRows(ctx: ResolveContext): Promise { + const rows = await listPromptTemplates({ workspaceId: ctx.workspaceId ?? undefined }); + return rows.filter( + (r) => (r.workspaceId ?? null) === ctx.workspaceId && !r.isDefault && NAMED_KINDS.has(r.kind), + ); +} + +async function find(d: DesiredPrompt, ctx: ResolveContext): Promise { + const rows = await workspaceRows(ctx); + const row = rows.find((r) => r.name === d.name && !r.ownerUserId) ?? null; + if (!row && rows.some((r) => r.name === d.name)) { + throw new ManifestError( + `A prompt named "${d.name}" exists, but it is someone's private prompt`, + ); + } + return row; +} + +async function get(_table: ResourceTable, id: string): Promise { + return (await getPromptTemplateById(id)) ?? null; +} + +function identify(row: Row): Identified { + return { table: "prompt_templates", id: row.id, name: row.name, ownerUserId: row.ownerUserId }; +} + +function desiredColumns(d: DesiredPrompt) { + return { + name: d.name, + template: d.template, + kind: d.kind, + description: d.description, + paramsSchema: d.paramsSchema, + defaultAgentType: d.defaultAgentType, + }; +} + +async function diff(row: Row, d: DesiredPrompt): Promise { + const changes = fieldChanges(row as unknown as Record, desiredColumns(d)); + if (row.ownerUserId) changes.push("owner"); + return changes; +} + +async function create(d: DesiredPrompt, ctx: ResolveContext): Promise { + return createNamedTemplate({ + ...desiredColumns(d), + workspaceId: ctx.workspaceId, + ownerUserId: null, + }); +} + +async function update(row: Row, d: DesiredPrompt): Promise { + return (await updateNamedTemplate(row.id, desiredColumns(d))) ?? row; +} + +async function remove(_table: ResourceTable, id: string): Promise { + await deleteNamedTemplate(id); +} + +async function list(ctx: ResolveContext): Promise { + return (await workspaceRows(ctx)) + .filter((r) => !r.ownerUserId) + .sort((a, b) => a.name.localeCompare(b.name)); +} + +async function exportPrompt(row: Row): Promise { + return { + apiVersion: MANIFEST_API_VERSION, + kind: "Prompt", + metadata: compact({ + name: row.name, + description: row.description ?? undefined, + }) as PromptManifest["metadata"], + spec: compact({ + kind: row.kind === "prompt" ? undefined : (row.kind as PromptManifest["spec"]["kind"]), + template: row.template, + params: (row.paramsSchema as Record | null) ?? undefined, + defaultAgentType: row.defaultAgentType ?? undefined, + }) as PromptManifest["spec"], + }; +} + +export const promptHandler: KindHandler = { + kind: "Prompt", + desire, + tableOf: () => "prompt_templates", + find, + get, + identify, + replaceReason: () => null, + diff, + create, + update, + remove, + list, + export: exportPrompt, +}; diff --git a/apps/api/src/services/config/kinds/repo.ts b/apps/api/src/services/config/kinds/repo.ts new file mode 100644 index 000000000..bc07a04c8 --- /dev/null +++ b/apps/api/src/services/config/kinds/repo.ts @@ -0,0 +1,138 @@ +/** + * `kind: Repo` — a repository's registration and the settings the manifest + * names. Its identity is the URL. Only the settings a manifest spells out are + * managed: a repo has dozens, and an organization shouldn't have to declare + * every agent model to pin its review policy. + */ +import { + MANIFEST_API_VERSION, + normalizeRepoUrl, + parseRepoUrl, + type RepoManifest, +} from "@optio/shared"; +import { + createRepo, + deleteRepo, + getRepo, + getRepoByUrl, + listRepos, + updateRepo, + type RepoRecord, +} from "../../repo-service.js"; +import { REPO_SETTING_KEYS } from "../../../schemas/config.js"; +import { PLANNED_ID, type ResolveContext, type ResourceTable } from "../context.js"; +import { compact, fieldChanges } from "../compare.js"; +import type { Identified, KindHandler } from "./index.js"; + +interface DesiredRepo { + name: string; + url: string; + /** `owner/repo`, from the URL. */ + fullName: string; + /** Only the settings the manifest names. */ + settings: Record; +} + +type UpdateData = Parameters[1]; + +async function desire(m: RepoManifest): Promise { + const { url, ...rest } = m.spec; + const parsed = parseRepoUrl(url); + const settings: Record = {}; + for (const key of REPO_SETTING_KEYS) { + if (rest[key] !== undefined) settings[key] = rest[key]; + } + return { + name: m.metadata.name, + url: normalizeRepoUrl(url), + fullName: parsed ? `${parsed.owner}/${parsed.repo}` : m.metadata.name, + settings, + }; +} + +async function find(d: DesiredRepo, ctx: ResolveContext): Promise { + return getRepoByUrl(d.url, ctx.workspaceId); +} + +async function get(_table: ResourceTable, id: string): Promise { + return getRepo(id); +} + +function identify(row: RepoRecord): Identified { + return { table: "repos", id: row.id, name: row.fullName, ownerUserId: null }; +} + +async function diff(row: RepoRecord, d: DesiredRepo): Promise { + return fieldChanges(row as unknown as Record, d.settings); +} + +async function create(d: DesiredRepo, ctx: ResolveContext): Promise { + const made = await createRepo({ + repoUrl: d.url, + fullName: d.fullName, + defaultBranch: + typeof d.settings.defaultBranch === "string" ? d.settings.defaultBranch : undefined, + workspaceId: ctx.workspaceId, + }); + const rest = { ...d.settings }; + delete rest.defaultBranch; + if (Object.keys(rest).length === 0) return made; + return (await updateRepo(made.id, rest as UpdateData)) ?? made; +} + +async function update(row: RepoRecord, d: DesiredRepo): Promise { + return (await updateRepo(row.id, d.settings as UpdateData)) ?? row; +} + +async function remove(_table: ResourceTable, id: string): Promise { + await deleteRepo(id); +} + +async function list(ctx: ResolveContext): Promise { + return (await listRepos(ctx.workspaceId)).sort((a, b) => a.fullName.localeCompare(b.fullName)); +} + +/** The settings an export writes: every manageable one that is set. */ +async function exportRepo(row: RepoRecord): Promise { + const record = row as unknown as Record; + const settings: Record = {}; + for (const key of REPO_SETTING_KEYS) { + const value = record[key]; + if (value === undefined || value === null || value === "") continue; + if (Array.isArray(value) && value.length === 0) continue; + settings[key] = value; + } + return { + apiVersion: MANIFEST_API_VERSION, + kind: "Repo", + metadata: { name: row.fullName }, + spec: { url: row.repoUrl, ...compact(settings) }, + }; +} + +export const repoHandler: KindHandler = { + kind: "Repo", + stub: (d, ctx) => { + const planned = { + id: PLANNED_ID, + repoUrl: d.url, + fullName: d.fullName, + defaultBranch: + typeof d.settings.defaultBranch === "string" ? d.settings.defaultBranch : "main", + } as RepoRecord; + ctx.repos.set(d.url, planned); + ctx.repoById.set(PLANNED_ID, planned); + }, + desire, + tableOf: () => "repos", + find, + get, + identify, + replaceReason: () => null, + diff, + create, + update, + remove, + list, + export: exportRepo, +}; diff --git a/apps/api/src/services/config/kinds/skill.ts b/apps/api/src/services/config/kinds/skill.ts new file mode 100644 index 000000000..1c2fa8ed0 --- /dev/null +++ b/apps/api/src/services/config/kinds/skill.ts @@ -0,0 +1,314 @@ +/** + * `kind: Skill` — a custom skill (its files in the manifest, `custom_skills`) + * or a marketplace skill (cloned from a git source, `installed_skills`). The + * two share one namespace in a manifest; switching between them, or changing + * the scope, recreates the row. + */ +import { + MANIFEST_API_VERSION, + normalizeRepoUrl, + type CustomSkillConfig, + type CustomSkillFile, + type CustomSkillLayout, + type InstalledSkillConfig, + type SkillManifest, +} from "@optio/shared"; +import { + createSkill, + deleteSkill, + getSkill, + listSkills, + updateSkill, +} from "../../skill-service.js"; +import { + createInstalledSkill, + deleteInstalledSkill, + getInstalledSkill, + listInstalledSkills, + updateInstalledSkill, +} from "../../installed-skill-service.js"; +import { + ManifestError, + PLANNED_ID, + repoFor, + type ResolveContext, + type ResourceTable, +} from "../context.js"; +import { compact, fieldChanges, norm, sameSet } from "../compare.js"; +import type { Identified, KindHandler } from "./index.js"; + +export type SkillRow = + | { table: "custom_skills"; row: CustomSkillConfig } + | { table: "installed_skills"; row: InstalledSkillConfig }; + +interface DesiredBase { + name: string; + description: string | null; + repoUrl: string | null; + agentTypes: string[] | null; + enabled: boolean; +} + +export type DesiredSkill = DesiredBase & + ( + | { + table: "custom_skills"; + prompt: string; + layout: CustomSkillLayout; + files: CustomSkillFile[] | null; + } + | { table: "installed_skills"; sourceUrl: string; ref: string; subpath: string } + ); + +const DEFAULT_REF = "main"; + +/** What installed-skill-service stores for a subpath: no leading `./`, no trailing `/`, `.` for the root. */ +function normalizeSubpath(value: string | undefined): string { + const trimmed = (value ?? "").trim().replace(/^\.\//, "").replace(/\/+$/, ""); + return trimmed === "" ? "." : trimmed; +} + +function filesOf(files: Record | undefined): CustomSkillFile[] | null { + if (!files) return null; + const list = Object.entries(files) + .map(([relativePath, content]) => ({ relativePath, content })) + .sort((a, b) => a.relativePath.localeCompare(b.relativePath)); + return list.length ? list : null; +} + +async function desire(m: SkillManifest, ctx: ResolveContext): Promise { + const s = m.spec; + const repo = repoFor(ctx, s.repo, "repo"); + const base: DesiredBase = { + name: m.metadata.name, + description: m.metadata.description ?? null, + repoUrl: repo ? normalizeRepoUrl(repo.repoUrl) : null, + agentTypes: s.agentTypes?.length ? [...s.agentTypes].sort() : null, + enabled: s.enabled ?? true, + }; + if (s.source) { + return { + ...base, + table: "installed_skills", + sourceUrl: s.source.url, + ref: s.source.ref?.trim() || DEFAULT_REF, + subpath: normalizeSubpath(s.source.path), + }; + } + if (s.prompt === undefined) { + throw new ManifestError("promptFile / filesFrom must be read into prompt before an apply"); + } + const files = filesOf(s.files); + const layout: CustomSkillLayout = s.layout ?? (files ? "skill-dir" : "commands"); + return { + ...base, + table: "custom_skills", + prompt: s.prompt, + layout, + files: layout === "skill-dir" ? (files ?? []) : null, + }; +} + +async function workspaceRows(ctx: ResolveContext): Promise { + const [custom, installed] = await Promise.all([ + listSkills(undefined, ctx.workspaceId), + listInstalledSkills(undefined, ctx.workspaceId), + ]); + const ws = ctx.workspaceId; + return [ + ...custom + .filter((r) => (r.workspaceId ?? null) === ws) + .map((row) => ({ table: "custom_skills" as const, row })), + ...installed + .filter((r) => (r.workspaceId ?? null) === ws) + .map((row) => ({ table: "installed_skills" as const, row })), + ]; +} + +async function find(d: DesiredSkill, ctx: ResolveContext): Promise { + const rows = (await workspaceRows(ctx)).filter((r) => r.row.name === d.name); + const org = rows.find((r) => !r.row.ownerUserId) ?? null; + if (!org && rows.length) { + throw new ManifestError(`A skill named "${d.name}" exists, but it is someone's private skill`); + } + return org; +} + +async function get(table: ResourceTable, id: string): Promise { + if (table === "installed_skills") { + const row = await getInstalledSkill(id); + return row ? { table, row } : null; + } + const row = await getSkill(id); + return row ? { table: "custom_skills", row } : null; +} + +function identify(row: SkillRow): Identified { + return { + table: row.table, + id: row.row.id, + name: row.row.name, + ownerUserId: row.row.ownerUserId ?? null, + }; +} + +function replaceReason(row: SkillRow, d: DesiredSkill): string | null { + if (row.table !== d.table) { + return d.table === "installed_skills" + ? "was a custom skill, now a marketplace skill — recreated" + : "was a marketplace skill, now a custom skill — recreated"; + } + const was = row.row.repoUrl ? normalizeRepoUrl(row.row.repoUrl) : null; + return norm(was) === norm(d.repoUrl) ? null : "its scope (repo / workspace) changed — recreated"; +} + +async function diff(row: SkillRow, d: DesiredSkill): Promise { + const changes: string[] = []; + const stored = row.row as unknown as Record; + changes.push( + ...fieldChanges(stored, { name: d.name, description: d.description, enabled: d.enabled }), + ); + if (!sameSet(row.row.agentTypes, d.agentTypes)) changes.push("agentTypes"); + if (row.table === "custom_skills" && d.table === "custom_skills") { + changes.push( + ...fieldChanges(stored, { + prompt: d.prompt, + layout: d.layout, + files: d.layout === "skill-dir" ? (d.files ?? []) : null, + }), + ); + } else if (row.table === "installed_skills" && d.table === "installed_skills") { + changes.push( + ...fieldChanges(stored, { sourceUrl: d.sourceUrl, ref: d.ref, subpath: d.subpath }), + ); + } + if (row.row.ownerUserId) changes.push("owner"); + return changes; +} + +async function create(d: DesiredSkill, ctx: ResolveContext): Promise { + if (d.table === "installed_skills") { + const row = await createInstalledSkill( + { + name: d.name, + description: d.description ?? undefined, + sourceUrl: d.sourceUrl, + ref: d.ref, + subpath: d.subpath, + repoUrl: d.repoUrl ?? undefined, + agentTypes: d.agentTypes ?? undefined, + enabled: d.enabled, + ownerUserId: null, + }, + ctx.workspaceId, + ); + return { table: "installed_skills", row }; + } + const row = await createSkill( + { + name: d.name, + description: d.description ?? undefined, + prompt: d.prompt, + repoUrl: d.repoUrl ?? undefined, + layout: d.layout, + files: d.files ?? undefined, + agentTypes: d.agentTypes ?? undefined, + enabled: d.enabled, + ownerUserId: null, + }, + ctx.workspaceId, + ); + return { table: "custom_skills", row }; +} + +async function update(row: SkillRow, d: DesiredSkill): Promise { + if (row.table === "installed_skills" && d.table === "installed_skills") { + const updated = await updateInstalledSkill(row.row.id, { + name: d.name, + description: d.description, + ref: d.ref, + subpath: d.subpath, + agentTypes: d.agentTypes, + enabled: d.enabled, + }); + return { table: "installed_skills", row: updated }; + } + if (row.table === "custom_skills" && d.table === "custom_skills") { + const updated = await updateSkill(row.row.id, { + name: d.name, + description: d.description, + prompt: d.prompt, + layout: d.layout, + files: d.files, + agentTypes: d.agentTypes, + enabled: d.enabled, + }); + return { table: "custom_skills", row: updated }; + } + throw new ManifestError("a skill can't change between custom and marketplace in place"); +} + +async function remove(table: ResourceTable, id: string): Promise { + if (table === "installed_skills") await deleteInstalledSkill(id); + else await deleteSkill(id); +} + +async function list(ctx: ResolveContext): Promise { + return (await workspaceRows(ctx)) + .filter((r) => !r.row.ownerUserId) + .sort((a, b) => a.row.name.localeCompare(b.row.name)); +} + +async function exportSkill(row: SkillRow): Promise { + const common = { + agentTypes: row.row.agentTypes?.length ? row.row.agentTypes : undefined, + repo: row.row.repoUrl ?? undefined, + enabled: row.row.enabled ? undefined : false, + }; + const spec: SkillManifest["spec"] = + row.table === "installed_skills" + ? (compact({ + source: compact({ + url: row.row.sourceUrl, + ref: row.row.ref === DEFAULT_REF ? undefined : row.row.ref, + path: row.row.subpath === "." ? undefined : row.row.subpath, + }), + ...common, + }) as SkillManifest["spec"]) + : (compact({ + prompt: row.row.prompt, + layout: row.row.layout === "commands" ? undefined : row.row.layout, + files: row.row.files?.length + ? Object.fromEntries(row.row.files.map((f) => [f.relativePath, f.content])) + : undefined, + ...common, + }) as SkillManifest["spec"]); + return { + apiVersion: MANIFEST_API_VERSION, + kind: "Skill", + metadata: compact({ + name: row.row.name, + description: row.row.description ?? undefined, + }) as SkillManifest["metadata"], + spec, + }; +} + +export const skillHandler: KindHandler = { + kind: "Skill", + stub: (d, ctx) => { + ctx.skills.set(d.name, { table: d.table, id: PLANNED_ID, name: d.name }); + }, + desire, + tableOf: (d) => d.table, + find, + get, + identify, + replaceReason, + diff, + create, + update, + remove, + list, + export: exportSkill, +}; diff --git a/apps/api/src/services/config/kinds/work-when.test.ts b/apps/api/src/services/config/kinds/work-when.test.ts new file mode 100644 index 000000000..01b558b5d --- /dev/null +++ b/apps/api/src/services/config/kinds/work-when.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import type { WorkWhenManifest } from "@optio/shared"; +import { whenFromManifest, whenToManifest } from "./work-when.js"; + +describe("Work `when`", () => { + it("maps each manifest shape to the trigger the API stores, and back", () => { + expect(whenFromManifest(undefined)).toEqual({ type: "manual" }); + expect(whenFromManifest({ schedule: "0 3 * * *" })).toEqual({ + type: "schedule", + config: { cronExpression: "0 3 * * *" }, + }); + expect(whenFromManifest({ schedule: { cron: "@daily" } })).toEqual({ + type: "schedule", + config: { cronExpression: "@daily" }, + }); + expect(whenFromManifest({ webhook: { path: "nightly" } })).toEqual({ + type: "webhook", + config: { path: "nightly" }, + }); + expect(whenFromManifest({ ticket: { source: "github", labels: ["optio"] } })).toEqual({ + type: "ticket", + config: { source: "github", labels: ["optio"] }, + }); + expect(whenFromManifest({ github: { events: ["pr_opened"], repos: ["acme/api"] } })).toEqual({ + type: "github", + config: { events: ["pr_opened"], repos: ["acme/api"] }, + }); + }); + + it("writes a stored trigger as a manifest's `when`, never a webhook's secret", () => { + expect(whenToManifest(null)).toBeUndefined(); + expect(whenToManifest({ type: "schedule", config: { cronExpression: "0 3 * * *" } })).toEqual({ + schedule: "0 3 * * *", + }); + expect(whenToManifest({ type: "webhook", config: { path: "nightly", secret: "shh" } })).toEqual( + { + webhook: { path: "nightly" }, + }, + ); + expect(whenToManifest({ type: "ticket", config: { source: "linear", labels: [] } })).toEqual({ + ticket: { source: "linear" }, + }); + expect(whenToManifest({ type: "slack", config: { channelId: "C123" } })).toEqual({ + slack: { channelId: "C123" }, + }); + }); + + it("round-trips", () => { + for (const when of [ + { schedule: "0 3 * * 1-5" }, + { webhook: { path: "p" } }, + { ticket: { source: "github", labels: ["a"] } }, + { linear: { events: ["issue_created"], user: "me" } }, + ] as WorkWhenManifest[]) { + expect(whenToManifest(whenFromManifest(when) as { type: string; config: unknown })).toEqual( + when, + ); + } + }); +}); diff --git a/apps/api/src/services/config/kinds/work-when.ts b/apps/api/src/services/config/kinds/work-when.ts new file mode 100644 index 000000000..85d37147b --- /dev/null +++ b/apps/api/src/services/config/kinds/work-when.ts @@ -0,0 +1,60 @@ +/** + * A Work manifest's `when` ⇄ the trigger the API stores. Pure: the manifest + * says `schedule: ""`, `webhook: { path }`, `ticket: { source, labels }` + * or passes a GitHub / Slack / Linear config through; the row says + * `{ type, config }`. A webhook's signing secret lives only on the row. + */ +import type { WorkWhen, WorkWhenManifest } from "@optio/shared"; + +/** The manifest's `when` as the trigger `POST /api/work` takes. */ +export function whenFromManifest(when: WorkWhenManifest | undefined): WorkWhen { + if (!when) return { type: "manual" }; + if ("schedule" in when) { + const cron = typeof when.schedule === "string" ? when.schedule : when.schedule.cron; + return { type: "schedule", config: { cronExpression: cron } }; + } + if ("webhook" in when) return { type: "webhook", config: { path: when.webhook.path } }; + if ("ticket" in when) { + return { + type: "ticket", + config: { + source: when.ticket.source, + ...(when.ticket.labels?.length ? { labels: when.ticket.labels } : {}), + }, + }; + } + if ("github" in when) return { type: "github", config: when.github }; + if ("slack" in when) return { type: "slack", config: when.slack }; + return { type: "linear", config: when.linear }; +} + +/** A stored trigger as a manifest's `when` (a webhook's secret never leaves the row). */ +export function whenToManifest( + trigger: { type: string; config: unknown } | null, +): WorkWhenManifest | undefined { + if (!trigger) return undefined; + const config = { ...((trigger.config ?? {}) as Record) }; + switch (trigger.type) { + case "schedule": + return { schedule: String(config.cronExpression ?? "") }; + case "webhook": + return { webhook: { path: String(config.path ?? "") } }; + case "ticket": + return { + ticket: { + source: String(config.source ?? ""), + ...(Array.isArray(config.labels) && config.labels.length + ? { labels: config.labels as string[] } + : {}), + }, + }; + case "github": + return { github: config }; + case "slack": + return { slack: config }; + case "linear": + return { linear: config }; + default: + return undefined; + } +} diff --git a/apps/api/src/services/config/kinds/work.ts b/apps/api/src/services/config/kinds/work.ts new file mode 100644 index 000000000..ec097bc9f --- /dev/null +++ b/apps/api/src/services/config/kinds/work.ts @@ -0,0 +1,484 @@ +/** + * `kind: Work` — a Job, a scheduled Task, or a persistent agent, written + * through the same `WorkSpec` path as the New work form (`createWork` / + * `updateWork`), plus the columns the spec doesn't carry (`enabled`, + * `params`, `limits`, `pods`). A manifest names the organization's resources + * by name; this handler turns them into the ids the rows store, and back. + */ +import { and, eq, inArray, isNull } from "drizzle-orm"; +import { + kindOfSpec, + SHELL_RUNTIME, + slugify, + MANIFEST_API_VERSION, + type WorkDefinitionKind, + type WorkManifest, + type WorkManifestSpec, + type WorkSettings, + type WorkSpec, + type WorkWhen, +} from "@optio/shared"; +import { db } from "../../../db/client.js"; +import { persistentAgents, workDefinitions } from "../../../db/schema.js"; +import * as definitions from "../../work-definition-service.js"; +import * as workWrite from "../../work-write-service.js"; +import * as paService from "../../persistent-agent-service.js"; +import * as triggerService from "../../trigger-service.js"; +import { + ManifestError, + namesOf, + repoFor, + resolveNames, + type ResolveContext, + type ResourceTable, +} from "../context.js"; +import { compact, fieldChanges, sameSet } from "../compare.js"; +import { whenFromManifest, whenToManifest } from "./work-when.js"; + +export { whenFromManifest, whenToManifest }; +import type { Identified, KindHandler } from "./index.js"; + +type Definition = definitions.WorkDefinition; +type Agent = typeof persistentAgents.$inferSelect; + +export type WorkRow = + | { table: "work_definitions"; row: Definition } + | { table: "persistent_agents"; row: Agent }; + +type ManifestWorkKind = "standalone" | "repo-blueprint" | "persistent-agent"; + +/** The columns `WorkSpec` doesn't carry; undefined = not managed by the manifest. */ +interface WorkExtras { + enabled: boolean; + paramsSchema?: Record | null; + maxTurns?: number | null; + budgetUsd?: string | null; + maxPodInstances?: number; + maxAgentsPerPod?: number; +} + +export interface DesiredWork { + name: string; + kind: ManifestWorkKind; + spec: WorkSpec; + /** A persistent agent's slug (its identity). */ + slug: string | null; + extras: WorkExtras; +} + +const NOUN: Record = { + standalone: "Job", + "repo-blueprint": "scheduled Task", + "persistent-agent": "agent", +}; + +const DEFINITION_KINDS: WorkDefinitionKind[] = ["standalone", "repo-blueprint"]; + +/** The trigger `updateWork` edits — the same one an export reads. */ +async function currentTrigger(row: WorkRow) { + const targetType = + row.table === "persistent_agents" + ? "persistent_agent" + : definitions.TRIGGER_TARGET[row.row.kind]; + return workWrite.editedTrigger(await triggerService.listTriggers(targetType, row.row.id)); +} + +function sameTrigger(current: { type: string; config: unknown } | null, wanted: WorkWhen): boolean { + if (wanted.type === "manual") return current === null; + if (!current || current.type !== wanted.type) return false; + const { secret: _secret, ...stored } = (current.config ?? {}) as Record; + return ( + fieldChanges(stored, wanted.config).length === 0 && + fieldChanges(wanted.config, stored).length === 0 + ); +} + +// ── Desire ────────────────────────────────────────────────────────────────── + +function environmentSettings( + env: WorkManifestSpec["environment"], + ctx: ResolveContext, +): WorkSettings | null { + if (!env) return null; + return { + connections: resolveNames(env.connections, ctx.connections, "connection"), + mcpServers: resolveNames(env.mcpServers, ctx.mcpServers, "MCP server"), + skills: resolveNames(env.skills, ctx.skills, "skill"), + setupCommands: env.setupCommands ?? undefined, + review: env.review ?? undefined, + cautiousMode: env.cautiousMode ?? undefined, + maxAutoResumes: env.maxAutoResumes ?? undefined, + }; +} + +async function desire(m: WorkManifest, ctx: ResolveContext): Promise { + const s = m.spec; + if (s.what.prompt === undefined) { + throw new ManifestError("what.promptFile must be read into what.prompt before an apply"); + } + if (s.agent?.systemPromptFile !== undefined || s.agent?.agentsMdFile !== undefined) { + throw new ManifestError("agent.*File fields must be read into their fields before an apply"); + } + const then = s.then ?? "exits"; + const repo = repoFor(ctx, s.where?.repo, "where.repo"); + const spec: WorkSpec = { + name: m.metadata.name, + description: m.metadata.description ?? null, + when: whenFromManifest(s.when), + where: { + runTarget: "cluster", + repoUrl: repo?.repoUrl ?? null, + repoBranch: s.where?.branch ?? null, + }, + who: { + runtime: s.who.runtime === SHELL_RUNTIME ? null : s.who.runtime, + agentOptions: s.who.options ?? null, + model: s.who.model ?? null, + }, + what: { prompt: s.what.prompt, runTitle: s.what.runTitle ?? null }, + then, + ...(s.mergeWhenReady !== undefined ? { mergeWhenReady: s.mergeWhenReady } : {}), + ...(s.retries !== undefined ? { maxRetries: s.retries } : {}), + ...(s.priority !== undefined ? { priority: s.priority } : {}), + ...(s.agent + ? { + agent: { + slug: s.agent.slug, + systemPrompt: s.agent.systemPrompt ?? null, + agentsMd: s.agent.agentsMd ?? null, + podLifecycle: s.agent.podLifecycle, + }, + } + : {}), + owner: "workspace", + podSecrets: s.secrets ?? null, + settings: environmentSettings(s.environment, ctx), + }; + const kind = kindOfSpec(spec); + if (kind === "repo-task") { + throw new ManifestError( + "A Task that runs once isn't configuration — add `when` to make it a scheduled Task, or drop `where.repo` to make it a Job", + ); + } + if (kind !== "standalone" && kind !== "repo-blueprint" && kind !== "persistent-agent") { + throw new ManifestError(`This describes a ${kind}, which a manifest can't declare`); + } + if (then === "until-merged" && kind === "standalone") { + throw new ManifestError("`then: until-merged` needs `where.repo`"); + } + if (kind !== "persistent-agent" && s.agent) { + throw new ManifestError("`agent` is for `then: waits-for-messages`"); + } + const slug = + kind === "persistent-agent" ? s.agent?.slug?.trim() || slugify(m.metadata.name) : null; + if (kind === "persistent-agent" && !slug) { + throw new ManifestError("Give the agent a name with letters or digits"); + } + return { + name: m.metadata.name, + kind, + spec, + slug, + extras: { + enabled: s.enabled ?? true, + paramsSchema: s.params, + maxTurns: s.limits?.maxTurns, + budgetUsd: + s.limits?.budgetUsd === undefined + ? undefined + : s.limits.budgetUsd === null + ? null + : String(s.limits.budgetUsd), + maxPodInstances: s.pods?.maxPodInstances, + maxAgentsPerPod: s.pods?.maxAgentsPerPod, + }, + }; +} + +// ── Rows ──────────────────────────────────────────────────────────────────── + +const tableOf = (d: DesiredWork): ResourceTable => + d.kind === "persistent-agent" ? "persistent_agents" : "work_definitions"; + +function wsOf( + column: typeof workDefinitions.workspaceId | typeof persistentAgents.workspaceId, + ws: string | null, +) { + return ws ? eq(column, ws) : isNull(column); +} + +async function find(d: DesiredWork, ctx: ResolveContext): Promise { + if (d.kind === "persistent-agent") { + const row = await paService.getPersistentAgentBySlug(ctx.workspaceId, d.slug!); + if (!row) return null; + if (row.ownerUserId) { + throw new ManifestError( + `An agent with slug "${d.slug}" exists, but it is someone's private agent`, + ); + } + return { table: "persistent_agents", row }; + } + const [row] = await db + .select() + .from(workDefinitions) + .where( + and( + eq(workDefinitions.kind, d.kind), + wsOf(workDefinitions.workspaceId, ctx.workspaceId), + eq(workDefinitions.name, d.name), + ), + ); + if (!row) return null; + if (row.ownerUserId) { + throw new ManifestError( + `A ${NOUN[d.kind]} named "${d.name}" exists, but it is someone's private work`, + ); + } + return { table: "work_definitions", row }; +} + +async function get(table: ResourceTable, id: string): Promise { + if (table === "persistent_agents") { + const [row] = await db.select().from(persistentAgents).where(eq(persistentAgents.id, id)); + return row ? { table, row } : null; + } + const row = await definitions.getDefinition(id); + return row ? { table: "work_definitions", row } : null; +} + +function identify(row: WorkRow): Identified { + return { table: row.table, id: row.row.id, name: row.row.name, ownerUserId: row.row.ownerUserId }; +} + +function nounOf(row: WorkRow): string { + if (row.table === "persistent_agents") return "an agent"; + return row.row.kind === "local-blueprint" ? "an automation" : `a ${NOUN[row.row.kind]}`; +} + +function replaceReason(row: WorkRow, d: DesiredWork): string | null { + const was = row.table === "persistent_agents" ? "persistent-agent" : row.row.kind; + if (was === d.kind) return null; + return `was ${nounOf(row)}, now a ${NOUN[d.kind]} — recreated (its run history doesn't carry over)`; +} + +/** The extra columns as a patch — only the ones the manifest manages. */ +function extrasPatch(d: DesiredWork): Record { + const e = d.extras; + return { + enabled: e.enabled, + ...(e.paramsSchema !== undefined ? { paramsSchema: e.paramsSchema } : {}), + ...(e.maxTurns !== undefined ? { maxTurns: e.maxTurns } : {}), + ...(e.budgetUsd !== undefined ? { budgetUsd: e.budgetUsd } : {}), + ...(e.maxPodInstances !== undefined ? { maxPodInstances: e.maxPodInstances } : {}), + ...(e.maxAgentsPerPod !== undefined ? { maxAgentsPerPod: e.maxAgentsPerPod } : {}), + }; +} + +function viaWorkError(run: () => Promise): Promise { + return run().catch((err: unknown) => { + if (err instanceof workWrite.WorkError) throw new ManifestError(err.message); + throw err; + }); +} + +async function diff(row: WorkRow, d: DesiredWork, ctx: ResolveContext): Promise { + const changes: string[] = []; + const stored = row.row as unknown as Record; + if (row.table === "work_definitions") { + const columns = await viaWorkError(() => + workWrite.definitionColumns(d.kind as WorkDefinitionKind, d.spec, ctx.actor), + ); + changes.push(...fieldChanges(stored, columns as Record)); + } else { + const columns = await viaWorkError(() => workWrite.agentColumns(d.spec, ctx.actor)); + changes.push( + ...fieldChanges(stored, { ...columns, podLifecycle: columns.podLifecycle ?? "sticky" }), + ); + } + if (!sameSet(row.row.podSecrets, d.spec.podSecrets)) changes.push("secrets"); + if (row.row.ownerUserId) changes.push("owner"); + const extras = extrasPatch(d); + if (row.table === "persistent_agents") { + delete extras.paramsSchema; + delete extras.budgetUsd; + delete extras.maxPodInstances; + delete extras.maxAgentsPerPod; + } + changes.push(...fieldChanges(stored, extras)); + if (!sameTrigger(await currentTrigger(row), d.spec.when)) changes.push("when"); + return [...new Set(changes)]; +} + +async function applyExtras(row: WorkRow, d: DesiredWork, ctx: ResolveContext): Promise { + const patch = extrasPatch(d); + if (row.table === "persistent_agents") { + await paService.updatePersistentAgent( + row.row.id, + { + enabled: d.extras.enabled, + ...(d.extras.maxTurns != null ? { maxTurns: d.extras.maxTurns } : {}), + }, + ctx.workspaceId, + ); + } else { + await definitions.updateDefinition(row.row.id, row.row.kind, patch); + } +} + +async function create(d: DesiredWork, ctx: ResolveContext): Promise { + const made = await viaWorkError(() => workWrite.createWork(d.spec, ctx.actor, { start: false })); + const row = (await get(tableOf(d), made.id))!; + await applyExtras(row, d, ctx); + return (await get(tableOf(d), made.id))!; +} + +async function update(row: WorkRow, d: DesiredWork, ctx: ResolveContext): Promise { + await viaWorkError(() => workWrite.updateWork(row.row.id, d.spec, ctx.actor)); + await applyExtras(row, d, ctx); + return (await get(row.table, row.row.id))!; +} + +async function remove(table: ResourceTable, id: string, ctx: ResolveContext): Promise { + if (table === "persistent_agents") { + await paService.deletePersistentAgent(id, ctx.workspaceId); + return; + } + const row = await definitions.getDefinition(id); + if (row) await definitions.deleteDefinition(id, row.kind); +} + +async function list(ctx: ResolveContext): Promise { + const defs = await db + .select() + .from(workDefinitions) + .where( + and( + inArray(workDefinitions.kind, DEFINITION_KINDS), + wsOf(workDefinitions.workspaceId, ctx.workspaceId), + isNull(workDefinitions.ownerUserId), + ), + ) + .orderBy(workDefinitions.name); + const agents = await db + .select() + .from(persistentAgents) + .where( + and( + wsOf(persistentAgents.workspaceId, ctx.workspaceId), + isNull(persistentAgents.ownerUserId), + ), + ) + .orderBy(persistentAgents.name); + return [ + ...defs.map((row) => ({ table: "work_definitions" as const, row })), + ...agents.map((row) => ({ table: "persistent_agents" as const, row })), + ]; +} + +// ── Export ────────────────────────────────────────────────────────────────── + +function environmentOf( + settings: WorkSettings | null | undefined, + ctx: ResolveContext, +): WorkManifestSpec["environment"] { + if (!settings) return undefined; + const env = compact({ + connections: namesOf(settings.connections, ctx.connectionById), + mcpServers: namesOf(settings.mcpServers, ctx.mcpById), + skills: namesOf(settings.skills, ctx.skillById), + setupCommands: settings.setupCommands, + review: settings.review, + cautiousMode: settings.cautiousMode, + maxAutoResumes: settings.maxAutoResumes, + }); + return Object.keys(env).length ? env : undefined; +} + +async function exportWork(row: WorkRow, ctx: ResolveContext): Promise { + const when = whenToManifest(await currentTrigger(row)); + if (row.table === "persistent_agents") { + const a = row.row; + const repo = a.repoId ? ctx.repoById.get(a.repoId) : null; + const spec: WorkManifestSpec = compact({ + when, + where: repo ? compact({ repo: repo.repoUrl, branch: a.branch ?? undefined }) : undefined, + who: compact({ + runtime: a.agentRuntime, + options: a.agentOptions ?? undefined, + model: a.model ?? undefined, + }) as WorkManifestSpec["who"], + what: { prompt: a.initialPrompt }, + then: "waits-for-messages", + secrets: a.podSecrets ?? undefined, + environment: environmentOf(a.settings, ctx), + agent: compact({ + slug: a.slug === slugify(a.name) ? undefined : a.slug, + systemPrompt: a.systemPrompt ?? undefined, + agentsMd: a.agentsMd ?? undefined, + podLifecycle: a.podLifecycle === "sticky" ? undefined : a.podLifecycle, + }), + limits: a.maxTurns === 50 ? undefined : { maxTurns: a.maxTurns }, + enabled: a.enabled ? undefined : false, + }) as WorkManifestSpec; + return { + apiVersion: MANIFEST_API_VERSION, + kind: "Work", + metadata: compact({ + name: a.name, + description: a.description ?? undefined, + }) as WorkManifest["metadata"], + spec, + }; + } + const d = row.row; + const repoWork = d.kind === "repo-blueprint"; + const spec: WorkManifestSpec = compact({ + when, + where: repoWork + ? compact({ repo: d.repoUrl ?? undefined, branch: d.repoBranch ?? undefined }) + : undefined, + who: compact({ + runtime: d.agentType ?? SHELL_RUNTIME, + options: d.agentOptions ?? undefined, + model: d.agentType ? (d.model ?? undefined) : undefined, + }) as WorkManifestSpec["who"], + what: compact({ + prompt: d.prompt, + runTitle: d.runTitle ?? undefined, + }) as WorkManifestSpec["what"], + then: repoWork && d.autoResume ? "until-merged" : undefined, + mergeWhenReady: repoWork && d.autoResume && d.autoMerge === false ? false : undefined, + retries: d.maxRetries === (repoWork ? 3 : 1) ? undefined : d.maxRetries, + priority: repoWork && d.priority !== 100 ? d.priority : undefined, + secrets: d.podSecrets ?? undefined, + environment: environmentOf(d.settings, ctx), + params: d.paramsSchema ?? undefined, + limits: compact({ maxTurns: d.maxTurns ?? undefined, budgetUsd: d.budgetUsd ?? undefined }), + enabled: d.enabled ? undefined : false, + }) as WorkManifestSpec; + return { + apiVersion: MANIFEST_API_VERSION, + kind: "Work", + metadata: compact({ + name: d.name, + description: d.description ?? undefined, + }) as WorkManifest["metadata"], + spec, + }; +} + +export const workHandler: KindHandler = { + kind: "Work", + desire, + tableOf, + find, + get, + identify, + replaceReason, + diff, + create, + update, + remove, + list, + export: exportWork, +}; diff --git a/apps/api/src/services/config/managed.ts b/apps/api/src/services/config/managed.ts new file mode 100644 index 000000000..ae0148098 --- /dev/null +++ b/apps/api/src/services/config/managed.ts @@ -0,0 +1,91 @@ +/** + * `managedBy` on a row: which configuration source manages it and from which + * file. One query per list; rows nobody manages are returned as they are + * (like `withOwnerNames` leaves the organization's rows alone). + */ +import { and, eq, inArray } from "drizzle-orm"; +import type { ManagedBy, WorkSource } from "@optio/shared"; +import { db } from "../../db/client.js"; +import { configObjects, configSources } from "../../db/schema.js"; +import type { ResourceTable } from "./context.js"; +import { envConfigSource } from "./env.js"; + +/** Which table a Work list row's resource lives in (none for runs and sessions). */ +export const WORK_SOURCE_TABLE: Partial> = { + "repo-blueprint": "work_definitions", + standalone: "work_definitions", + "local-blueprint": "work_definitions", + "persistent-agent": "persistent_agents", +}; + +export async function managedByMap( + table: ResourceTable, + ids: string[], +): Promise> { + const unique = [...new Set(ids)].filter(Boolean); + // Only the configuration directory manages rows: with it off there is + // nothing to look up (and no query, which keeps mocked route tests honest). + if (unique.length === 0 || !envConfigSource()) return new Map(); + const rows = await db + .select({ + objectId: configObjects.id, + resourceId: configObjects.resourceId, + kind: configObjects.kind, + path: configObjects.path, + sourceId: configSources.id, + sourceName: configSources.name, + }) + .from(configObjects) + .innerJoin(configSources, eq(configSources.id, configObjects.sourceId)) + .where(and(eq(configObjects.resourceTable, table), inArray(configObjects.resourceId, unique))); + return new Map( + rows.map((r) => [ + r.resourceId, + { + objectId: r.objectId, + sourceId: r.sourceId, + sourceName: r.sourceName, + path: r.path, + kind: r.kind, + }, + ]), + ); +} + +/** Decorate the managed rows with `managedBy`; the rest are returned as they are. */ +export async function withManagedBy( + rows: T[], + table: ResourceTable, +): Promise> { + const managed = await managedByMap( + table, + rows.map((r) => r.id), + ); + if (managed.size === 0) return rows; + return rows.map((r) => { + const by = managed.get(r.id); + return by ? { ...r, managedBy: by } : r; + }); +} + +/** `withManagedBy` for Work list rows, whose resources live in different tables. */ +export async function withManagedWork( + rows: T[], +): Promise> { + const byTable = new Map(); + for (const r of rows) { + const table = WORK_SOURCE_TABLE[r.source]; + if (table) byTable.set(table, [...(byTable.get(table) ?? []), r.id]); + } + const maps = await Promise.all( + [...byTable.entries()].map( + async ([table, ids]) => [table, await managedByMap(table, ids)] as const, + ), + ); + const lookup = new Map(maps); + return rows.map((r) => { + const table = WORK_SOURCE_TABLE[r.source]; + const by = table ? lookup.get(table)?.get(r.id) : undefined; + return by ? { ...r, managedBy: by } : r; + }); +} diff --git a/apps/api/src/services/config/source.ts b/apps/api/src/services/config/source.ts new file mode 100644 index 000000000..f6ab35eee --- /dev/null +++ b/apps/api/src/services/config/source.ts @@ -0,0 +1,213 @@ +/** + * The configuration directory as a source: declared by `OPTIO_CONFIG_DIR` + * (+ `OPTIO_CONFIG_WORKSPACE`, `OPTIO_CONFIG_PRUNE`, `OPTIO_CONFIG_INTERVAL`), + * mirrored into `config_sources` so it has an id, a status and a place in + * Settings, read every interval by the sync worker, and applied. + */ +import { and, asc, eq, isNull } from "drizzle-orm"; +import type { ConfigApplyResult, ConfigSourceView, ConfigStatus } from "@optio/shared"; +import { db } from "../../db/client.js"; +import { configSources, workspaces } from "../../db/schema.js"; +import { logger } from "../../logger.js"; +import { applyManifests, type ConfigSourceRow } from "./apply.js"; +import { readManifestDirectory } from "./files.js"; +import { envConfigSource, type EnvConfigSource } from "./env.js"; +import { isAuthDisabled } from "../oauth/index.js"; + +export { envConfigSource, type EnvConfigSource } from "./env.js"; + +/** The one directory source's name, as Settings shows it. */ +export const CONFIG_SOURCE_NAME = "config directory"; + +/** The workspace a slug names, else the oldest one (the organization's, as the sign-in bootstrap picks). */ +export async function resolveSourceWorkspaceId(slug: string | null): Promise { + if (slug) { + const [ws] = await db + .select({ id: workspaces.id }) + .from(workspaces) + .where(eq(workspaces.slug, slug)); + return ws?.id ?? null; + } + const [oldest] = await db + .select({ id: workspaces.id }) + .from(workspaces) + .orderBy(asc(workspaces.createdAt)) + .limit(1); + return oldest?.id ?? null; +} + +async function sourceRow(workspaceId: string | null): Promise { + const [row] = await db + .select() + .from(configSources) + .where( + and( + workspaceId + ? eq(configSources.workspaceId, workspaceId) + : isNull(configSources.workspaceId), + eq(configSources.name, CONFIG_SOURCE_NAME), + ), + ); + return row ?? null; +} + +/** + * The directory source's row, created or brought up to date from the env. + * Null when config as code is off, or the workspace it names doesn't exist + * (yet — a fresh install has none until someone signs in). + */ +export async function ensureEnvSource(): Promise { + const env = envConfigSource(); + if (!env) return null; + // With auth disabled requests carry no workspace and rows are stored under + // none (whatever workspaces the migrations made): the directory feeds that + // one tenant, unless a slug says otherwise. + if (!env.workspace && isAuthDisabled()) return ensureSourceRow(null, env); + const workspaceId = await resolveSourceWorkspaceId(env.workspace); + if (!workspaceId) { + logger.warn( + { workspace: env.workspace }, + env.workspace + ? "OPTIO_CONFIG_DIR: no workspace with that slug yet; waiting" + : "OPTIO_CONFIG_DIR: no workspace yet; waiting for the first sign-in", + ); + return null; + } + return ensureSourceRow(workspaceId, env); +} + +async function ensureSourceRow( + workspaceId: string | null, + env: EnvConfigSource, +): Promise { + const existing = await sourceRow(workspaceId); + if (existing) { + if (existing.path === env.dir && existing.prune === env.prune && existing.enabled) + return existing; + const [updated] = await db + .update(configSources) + .set({ path: env.dir, prune: env.prune, enabled: true, updatedAt: new Date() }) + .where(eq(configSources.id, existing.id)) + .returning(); + return updated; + } + const [created] = await db + .insert(configSources) + .values({ + workspaceId, + name: CONFIG_SOURCE_NAME, + kind: "dir", + path: env.dir, + prune: env.prune, + enabled: true, + origin: "env", + }) + .onConflictDoNothing() + .returning(); + return created ?? (await sourceRow(workspaceId)); +} + +// One apply at a time in this process: a tick and a "Sync now" that overlap +// would race each other's creates. +let running: Promise = Promise.resolve(); + +function serialized(run: () => Promise): Promise { + const next = running.then(run, run); + running = next.catch(() => {}); + return next; +} + +/** Read the directory and apply it (or plan it). Null when config as code is off. */ +export async function syncEnvSource( + opts: { dryRun?: boolean } = {}, +): Promise { + const source = await ensureEnvSource(); + if (!source) return null; + return serialized(async () => { + const dryRun = opts.dryRun ?? false; + let result: ConfigApplyResult; + let hash: string | null = null; + let error: string | null = null; + try { + const read = await readManifestDirectory(source.path); + hash = read.hash; + result = await applyManifests({ + workspaceId: source.workspaceId, + manifests: read.manifests, + dryRun, + source, + prune: source.prune, + priorErrors: read.errors, + }); + } catch (err) { + error = err instanceof Error ? err.message : String(err); + logger.error({ err, dir: source.path }, "config directory: sync failed"); + result = { + dryRun, + source: { id: source.id, name: source.name }, + items: [], + summary: { + created: 0, + updated: 0, + reverted: 0, + unchanged: 0, + adopted: 0, + replaced: 0, + pruned: 0, + errors: 1, + }, + at: new Date().toISOString(), + }; + } + if (!dryRun) { + await db + .update(configSources) + .set({ + lastSyncAt: new Date(), + lastSyncHash: hash, + lastSyncError: error, + lastSyncResult: result, + updatedAt: new Date(), + }) + .where(eq(configSources.id, source.id)); + const s = result.summary; + if (s.created || s.updated || s.replaced || s.pruned || s.errors || error) { + logger.info({ dir: source.path, ...s, error }, "config directory applied"); + } + } + return result; + }); +} + +export function toSourceView(row: ConfigSourceRow, intervalMs: number): ConfigSourceView { + return { + id: row.id, + name: row.name, + kind: "dir", + path: row.path, + workspaceId: row.workspaceId, + prune: row.prune, + enabled: row.enabled, + origin: row.origin === "settings" ? "settings" : "env", + intervalMs, + lastSyncAt: row.lastSyncAt?.toISOString() ?? null, + lastSyncHash: row.lastSyncHash, + lastSyncError: row.lastSyncError, + lastSync: row.lastSyncResult ?? null, + }; +} + +/** `GET /api/config/status` for a workspace. */ +export async function configStatus( + workspaceId: string | null, + schemaUrl: string, +): Promise { + const env = envConfigSource(); + if (!env) return { enabled: false, source: null, schemaUrl }; + const row = await sourceRow(workspaceId); + if (!row) { + // Configured, but for another workspace (or none exists yet). + return { enabled: false, source: null, schemaUrl }; + } + return { enabled: row.enabled, source: toSourceView(row, env.intervalMs), schemaUrl }; +} diff --git a/apps/api/src/services/persistent-agent-service.ts b/apps/api/src/services/persistent-agent-service.ts index 0e277df50..31d05cac8 100644 --- a/apps/api/src/services/persistent-agent-service.ts +++ b/apps/api/src/services/persistent-agent-service.ts @@ -181,6 +181,8 @@ export async function createPersistentAgent( } export interface UpdatePersistentAgentInput { + /** Its identity in URLs and inter-agent messages; unique per workspace. */ + slug?: string; name?: string; description?: string | null; agentRuntime?: string; @@ -207,8 +209,10 @@ export async function updatePersistentAgent( id: string, input: UpdatePersistentAgentInput, workspaceId: string | null, + /** A transaction, so the row and its trigger can change together. */ + tx: Pick = db, ) { - const [row] = await db + const [row] = await tx .update(persistentAgents) .set({ ...input, updatedAt: new Date() }) .where(and(eq(persistentAgents.id, id), wsPredicate(workspaceId))) diff --git a/apps/api/src/services/work-service.ts b/apps/api/src/services/work-service.ts index e71d8e8bb..ae82da37b 100644 --- a/apps/api/src/services/work-service.ts +++ b/apps/api/src/services/work-service.ts @@ -39,6 +39,7 @@ import * as sessionService from "./interactive-session-service.js"; import * as paService from "./persistent-agent-service.js"; import { canAccessBlueprint, ownedBy } from "./local-blueprint-service.js"; import { canSee, ownerNameFor, ownerNames, visibleOwner, type Actor } from "./ownership.js"; +import { withManagedWork } from "./config/managed.js"; import type { LocalTerminalRow } from "./local-terminal-service.js"; import type { WorkDefinition } from "./work-definition-service.js"; @@ -139,8 +140,8 @@ export async function listWork(scope: WorkScope): Promise { /** Private rows carry their owner's name (what an admin's list shows). */ async function nameOwners(rows: WorkRow[]): Promise { const names = await ownerNames(rows.map((r) => r.ownerUserId)); - return rows.map((r) => - r.ownerUserId ? { ...r, ownerName: ownerNameFor(r.ownerUserId, names) } : r, + return withManagedWork( + rows.map((r) => (r.ownerUserId ? { ...r, ownerName: ownerNameFor(r.ownerUserId, names) } : r)), ); } diff --git a/apps/api/src/services/work-write-service.ts b/apps/api/src/services/work-write-service.ts index f827548d1..6d4cf528e 100644 --- a/apps/api/src/services/work-write-service.ts +++ b/apps/api/src/services/work-write-service.ts @@ -17,6 +17,8 @@ import { toLocalAgentKind, type PersistentAgentPodLifecycle, type LocalTerminalSpec, + type TriggerTargetType, + type WorkWhen, type WorkCreated, type WorkDefinitionKind, type WorkKind, @@ -198,9 +200,10 @@ function options(spec: WorkSpec): Record | null { /** * The definition columns a spec sets — one mapping for create and edit, per * kind. Ownership (workspace, owner, creator) and `enabled` are the - * caller's. + * caller's. Exported for the config apply, which compares a managed row + * against what its manifest would write (services/config/kinds/work.ts). */ -async function definitionColumns( +export async function definitionColumns( kind: WorkDefinitionKind, spec: WorkSpec, actor: Actor, @@ -263,6 +266,35 @@ async function definitionColumns( } } +/** + * A persistent agent's columns from a spec — identity (slug), runtime, prompts, + * pod, repo checkout and environment. Ownership and `enabled` are the caller's. + */ +export async function agentColumns(spec: WorkSpec, actor: Actor) { + const slug = spec.agent?.slug?.trim() || slugify(spec.name); + if (!slug) throw new WorkError(400, "Give the agent a name with letters or digits"); + // An agent with a repo works in a checkout of it, turn after turn. + const repo = spec.where.repoUrl + ? await getRepoByUrl(spec.where.repoUrl, actor.workspaceId) + : null; + if (spec.where.repoUrl && !repo) throw new WorkError(400, "Pick one of your repos"); + return { + slug, + name: spec.name.trim(), + description: spec.description?.trim() || null, + agentRuntime: spec.who.runtime!, + model: spec.who.model ?? null, + agentOptions: options(spec), + systemPrompt: spec.agent?.systemPrompt || null, + agentsMd: spec.agent?.agentsMd || null, + initialPrompt: spec.what.prompt.trim(), + podLifecycle: spec.agent?.podLifecycle as PersistentAgentPodLifecycle | undefined, + repoId: repo?.id ?? null, + branch: repo ? spec.where.repoBranch || repo.defaultBranch : null, + settings: settingsOf(spec, "persistent-agent"), + }; +} + function conflict(err: unknown, kind: WorkKind, spec: WorkSpec): never { if (err instanceof Error && err.message === "duplicate_webhook_path") { const path = spec.when.type === "manual" ? "" : String(spec.when.config.path ?? ""); @@ -283,8 +315,16 @@ function conflict(err: unknown, kind: WorkKind, spec: WorkSpec): never { // ── Create ────────────────────────────────────────────────────────────────── -/** Create a piece of work from its attributes; a Job started now starts its first run. */ -export async function createWork(spec: WorkSpec, actor: Actor): Promise { +/** + * Create a piece of work from its attributes; a Job started now starts its + * first run (unless `start: false` — the config apply declares Jobs, it + * doesn't run them). + */ +export async function createWork( + spec: WorkSpec, + actor: Actor, + opts: { start?: boolean } = {}, +): Promise { const kind = kindOfSpec(spec); check(spec, kind); const trigger = triggerOf(spec, kind); @@ -351,7 +391,7 @@ export async function createWork(spec: WorkSpec, actor: Actor): Promise>; try { agent = await db.transaction(async (tx) => { const row = await paService.createPersistentAgent( { - slug, - name: spec.name.trim(), - description: spec.description?.trim() || undefined, - agentRuntime: spec.who.runtime!, - model: spec.who.model ?? null, - agentOptions: options(spec), - systemPrompt: spec.agent?.systemPrompt || null, - agentsMd: spec.agent?.agentsMd || null, - initialPrompt: spec.what.prompt.trim(), - podLifecycle: spec.agent?.podLifecycle as PersistentAgentPodLifecycle | undefined, - repoId: repo?.id ?? null, - branch: repo ? spec.where.repoBranch || repo.defaultBranch : null, - settings: settingsOf(spec, kind), + ...columns, workspaceId: actor.workspaceId, createdBy: actor.userId, ...owned, @@ -498,15 +520,51 @@ export function editedTrigger(triggers: T[]): T } /** - * Save a definition from its attributes. Its kind is fixed: the answers are - * read for the saved kind, and ones it can't take are refused. The one trigger the form edits - * follows the answer — patched in place when the type is the same (a webhook - * keeps its path, a schedule its id), replaced when it changes, removed for - * "now" — and other triggers are left alone. Row and trigger change together. + * Save the one trigger an edit replaces (`editedTrigger`): patched in place + * when the type is the same (a webhook keeps its path and secret, a schedule + * its id), replaced when it changes, removed for "now" (`wanted` null). Other + * triggers are left alone. + */ +async function saveEditedTrigger( + targetType: TriggerTargetType, + targetId: string, + wanted: { type: WorkWhen["type"]; config: Record } | null, + tx: Parameters[2], +): Promise { + const current = editedTrigger(await triggerService.listTriggers(targetType, targetId, tx)); + if (!wanted) { + if (current) await triggerService.deleteTrigger(current.id, tx); + } else if (current && current.type === wanted.type) { + const config = { ...kept(current), ...wanted.config }; + await triggerService.updateTrigger(current.id, { config }, tx); + } else { + await triggerService.createTrigger({ targetType, targetId, ...wanted }, tx); + if (current) await triggerService.deleteTrigger(current.id, tx); + } +} + +/** A persistent agent in the actor's workspace that they may see. */ +async function getOwnAgent(id: string, actor: Actor) { + return paService.getPersistentAgentScoped(id, actor.workspaceId, { + userId: actor.userId, + workspaceId: actor.workspaceId, + isAdmin: actor.isAdmin, + }); +} + +/** + * Save a definition — or a persistent agent — from its attributes. Its kind + * is fixed: the answers are read for the saved kind, and ones it can't take + * are refused. The one trigger the form edits follows the answer + * (`saveEditedTrigger`). Row and trigger change together. */ export async function updateWork(id: string, spec: WorkSpec, actor: Actor): Promise { const existing = await getOwnDefinition(id, actor); - if (!existing) throw new WorkError(404, "Work not found"); + if (!existing) { + const agent = await getOwnAgent(id, actor); + if (!agent) throw new WorkError(404, "Work not found"); + return updateAgentWork(agent, spec, actor); + } // The saved kind stays, and the answers are read for it. The form keeps an // edit from moving it (`kindLock`), but a row can load at a point that // derives elsewhere — a scheduled Task with no trigger reads as "now", a @@ -536,16 +594,7 @@ export async function updateWork(id: string, spec: WorkSpec, actor: Actor): Prom try { await db.transaction(async (tx) => { await definitions.updateDefinition(id, kind, columns, tx); - const current = editedTrigger(await triggerService.listTriggers(targetType, id, tx)); - if (!wanted) { - if (current) await triggerService.deleteTrigger(current.id, tx); - } else if (current && current.type === wanted.type) { - const config = { ...kept(current), ...wanted.config }; - await triggerService.updateTrigger(current.id, { config }, tx); - } else { - await triggerService.createTrigger({ targetType, targetId: id, ...wanted }, tx); - if (current) await triggerService.deleteTrigger(current.id, tx); - } + await saveEditedTrigger(targetType, id, wanted, tx); }); } catch (err) { conflict(err, kind, spec); @@ -553,6 +602,43 @@ export async function updateWork(id: string, spec: WorkSpec, actor: Actor): Prom return { kind, id, href: workHref(kind, id) }; } +/** Save a persistent agent from its attributes (Then stays "waits for messages"). */ +async function updateAgentWork( + agent: NonNullable>>, + spec: WorkSpec, + actor: Actor, +): Promise { + const kind: WorkKind = "persistent-agent"; + check(spec, kind); + const wanted = triggerOf(spec, kind); + const columns = await agentColumns(spec, actor); + const planned = await planWorkUpdate( + agent, + { owner: spec.owner, podSecrets: spec.podSecrets, agentOptions: spec.who.agentOptions }, + actor, + { agentType: spec.who.runtime ?? "claude-code", runsOn: "pod", touchesRuntime: true }, + ); + if (!planned.ok) throw new WorkError(planned.status, planned.error); + try { + await db.transaction(async (tx) => { + await paService.updatePersistentAgent( + agent.id, + { + ...columns, + ownerUserId: planned.ownerUserId, + ...(planned.podSecrets !== undefined ? { podSecrets: planned.podSecrets } : {}), + }, + actor.workspaceId, + tx, + ); + await saveEditedTrigger("persistent_agent", agent.id, wanted, tx); + }); + } catch (err) { + conflict(err, kind, spec); + } + return { kind, id: agent.id, href: workHref(kind, agent.id) }; +} + /** * Delete a definition and its triggers (a Job's runs go with it; spawned * tasks and terminals stay). Personal work is its owner's, or an admin's, diff --git a/apps/api/src/workers/config-sync-worker.ts b/apps/api/src/workers/config-sync-worker.ts new file mode 100644 index 000000000..63af07440 --- /dev/null +++ b/apps/api/src/workers/config-sync-worker.ts @@ -0,0 +1,49 @@ +/** + * Reads the configuration directory (`OPTIO_CONFIG_DIR`) every interval and + * applies it — services/config/source.ts. The apply is idempotent (rows that + * already match aren't written), so a tick with no changes costs a few + * queries; a tick after someone edited a managed row in the UI puts the + * file's version back. Only started when the directory is configured; the + * first pass runs at once so a fresh deployment comes up with its config. + */ +import { Queue, Worker } from "bullmq"; +import { logger } from "../logger.js"; +import { getBullMQConnectionOptions } from "../services/redis-config.js"; +import { envConfigSource, syncEnvSource } from "../services/config/source.js"; + +const QUEUE = "config-sync"; + +export function startConfigSyncWorker(): Worker | null { + const env = envConfigSource(); + if (!env) return null; + const connection = getBullMQConnectionOptions(); + const queue = new Queue(QUEUE, { connection }); + queue + .add( + "sync", + {}, + { + repeat: { every: env.intervalMs }, + removeOnComplete: { count: 20 }, + removeOnFail: { count: 20 }, + }, + ) + .catch((err) => logger.error({ err }, "config sync: could not schedule")); + + const worker = new Worker( + QUEUE, + async () => { + await syncEnvSource(); + }, + { connection, concurrency: 1 }, + ); + worker.on("failed", (_job, err) => logger.error({ err }, "config sync failed")); + + // Don't wait a whole interval for the first pass. + syncEnvSource().catch((err) => logger.error({ err }, "config sync: first pass failed")); + logger.info( + { dir: env.dir, everyMs: env.intervalMs, prune: env.prune }, + "Config sync worker started", + ); + return worker; +} diff --git a/apps/cli/README.md b/apps/cli/README.md index 5620ad122..59e5082ec 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -136,6 +136,22 @@ Every command supports: `--server `, `--api-key `, | `optio workspace list` | List workspaces | | `optio workspace switch ` | Switch workspace | +### Config as code + +Resources as YAML manifests (`docs/config-as-code.md`): export what a +workspace has, apply a file or a directory, see what an apply would change. + +| Command | Description | +| --------------------------------------------- | -------------------------------------------------------------- | +| `optio export [--kind k] [--name n] [-o DIR]` | Export the organization's resources; `-o` writes one file each | +| `optio apply -f FILE\|DIR... [--dry-run]` | Create or update the resources the manifests describe | +| `optio diff -f FILE\|DIR...` | What `apply` would change (a dry run) | +| `optio schema` | The JSON Schema manifests validate against | + +A CLI apply is a plain upsert: it manages nothing and never prunes. A cluster +that should keep a directory applied mounts it as `OPTIO_CONFIG_DIR` (Helm +`configAsCode.*`). Exit code 1 when any manifest failed. + ### Other | Command | Description | diff --git a/apps/cli/package.json b/apps/cli/package.json index 8c4f21cd6..aaad2d209 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -29,7 +29,8 @@ "commander": "^12.1.0", "node-pty": "^1.0.0", "open": "^10.1.0", - "ws": "^8.21.0" + "ws": "^8.21.0", + "yaml": "2.9.0" }, "devDependencies": { "@types/node": "^22.15.0", diff --git a/apps/cli/src/__tests__/manifests-read.test.ts b/apps/cli/src/__tests__/manifests-read.test.ts new file mode 100644 index 000000000..86d3c5425 --- /dev/null +++ b/apps/cli/src/__tests__/manifests-read.test.ts @@ -0,0 +1,49 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { readManifests } from "../manifests/read.js"; + +let dir: string; + +beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), "optio-cli-manifests-")); +}); + +afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); +}); + +async function write(rel: string, text: string) { + const file = path.join(dir, rel); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, text); +} + +describe("readManifests", () => { + it("reads files and directories, inlines file fields, and reports what doesn't parse", async () => { + await write( + "optio/work/a.yaml", + "kind: Work\nmetadata: { name: a }\nspec:\n who: { runtime: shell }\n what: { promptFile: a.sh }\n", + ); + await write("optio/work/a.sh", "echo hi\n"); + await write("optio/bad.yaml", "kind: [oops"); + await write("single.yaml", "kind: Prompt\nmetadata: { name: p }\nspec: { template: t }\n"); + + const read = await readManifests(["optio", "single.yaml"], dir); + expect(read.manifests.map((m) => m.path)).toEqual(["optio/work/a.yaml", "single.yaml"]); + const work = read.manifests[0].document as { spec: { what: { prompt: string } } }; + expect(work.spec.what).toEqual({ prompt: "echo hi\n" }); + expect(read.problems).toHaveLength(1); + expect(read.problems[0].path).toBe("optio/bad.yaml"); + }); + + it("numbers the documents of a multi-document file", async () => { + await write( + "many.yaml", + "kind: Prompt\nmetadata: { name: a }\nspec: { template: a }\n---\nkind: Prompt\nmetadata: { name: b }\nspec: { template: b }\n", + ); + const read = await readManifests(["many.yaml"], dir); + expect(read.manifests.map((m) => m.path)).toEqual(["many.yaml#1", "many.yaml#2"]); + }); +}); diff --git a/apps/cli/src/commands/apply.ts b/apps/cli/src/commands/apply.ts new file mode 100644 index 000000000..cbe246b4e --- /dev/null +++ b/apps/cli/src/commands/apply.ts @@ -0,0 +1,105 @@ +/** + * `optio apply -f FILE|DIR...` — make the workspace match the manifests + * (docs/config-as-code.md); `optio diff` is the same with `--dry-run`. A CLI + * apply is a plain upsert: it manages nothing and never prunes — the + * configuration directory (OPTIO_CONFIG_DIR) does that. + */ +import { Command } from "commander"; +import type { ConfigApplyResult, ConfigPlanItem } from "@optio/shared"; +import { buildClient } from "../api/client.js"; +import { bold, cyan, dim, green, red, yellow } from "../output/colors.js"; +import { isJsonMode, outputJson } from "../output/formatter.js"; +import { friendlyError } from "../utils/errors.js"; +import { readManifests } from "../manifests/read.js"; + +const ACTION_COLOR: Record string> = { + create: green, + update: cyan, + unchanged: dim, + adopt: cyan, + replace: yellow, + prune: red, + error: red, +}; + +function detailOf(item: ConfigPlanItem): string { + if (item.action === "error") return item.message ?? ""; + const parts: string[] = []; + if (item.changes?.length) parts.push(item.changes.join(", ")); + if (item.reverted) parts.push("(a UI edit, put back)"); + if (item.message) parts.push(item.message); + return parts.join(" "); +} + +/** The result as `kubectl apply` would print it, one line per manifest. */ +export function printApplyResult( + result: ConfigApplyResult, + write = (s: string) => process.stdout.write(s), +): void { + const items = [...result.items].sort( + (a, b) => a.kind.localeCompare(b.kind) || a.name.localeCompare(b.name), + ); + const width = Math.max(9, ...items.map((i) => i.action.length)); + for (const item of items) { + const color = ACTION_COLOR[item.action] ?? ((s: string) => s); + const detail = detailOf(item); + write( + `${color(item.action.padEnd(width))} ${bold(item.kind)}/${item.name}` + + `${dim(` ${item.path}`)}${detail ? ` ${dim(detail)}` : ""}\n`, + ); + } + const s = result.summary; + const counts = [ + s.created && `${s.created} created`, + s.updated && `${s.updated} updated${s.reverted ? ` (${s.reverted} UI edits put back)` : ""}`, + s.adopted && `${s.adopted} adopted`, + s.replaced && `${s.replaced} replaced`, + s.pruned && `${s.pruned} pruned`, + s.unchanged && `${s.unchanged} unchanged`, + s.errors && red(`${s.errors} errors`), + ].filter(Boolean); + write(`\n${result.dryRun ? "Plan" : "Applied"}: ${counts.join(", ") || "nothing"}\n`); +} + +async function run(files: string[], opts: { dryRun?: boolean }, cmd: Command): Promise { + try { + if (!files.length) { + process.stderr.write("Pass the manifests to apply: optio apply -f optio/\n"); + process.exitCode = 2; + return; + } + const read = await readManifests(files); + const client = buildClient(cmd.optsWithGlobals()); + const result = await client.post("/api/config/apply", { + manifests: read.manifests, + dryRun: opts.dryRun ?? false, + }); + // Files that didn't parse never reached the server; they count as errors here. + for (const p of read.problems) { + result.items.push({ + kind: "?", + name: "?", + path: p.path, + action: "error", + message: p.message, + }); + result.summary.errors++; + } + if (isJsonMode()) outputJson(result); + else printApplyResult(result); + if (result.summary.errors > 0) process.exitCode = 1; + } catch (err) { + friendlyError(err); + } +} + +export const applyCommand = new Command("apply") + .description("Apply manifests (YAML files or directories) to the current workspace") + .requiredOption("-f, --file ", "A manifest file or a directory of them") + .option("--dry-run", "Show what would change without changing anything") + .action((opts: { file: string[]; dryRun?: boolean }, cmd) => run(opts.file, opts, cmd)); + +export const diffCommand = new Command("diff") + .description("Show what `optio apply` would change (a dry run)") + .requiredOption("-f, --file ", "A manifest file or a directory of them") + .action((opts: { file: string[] }, cmd) => run(opts.file, { dryRun: true }, cmd)); diff --git a/apps/cli/src/commands/export.ts b/apps/cli/src/commands/export.ts new file mode 100644 index 000000000..16dadc5fd --- /dev/null +++ b/apps/cli/src/commands/export.ts @@ -0,0 +1,88 @@ +/** + * `optio export` — the organization's resources as manifests: to stdout as + * one YAML stream, or with `-o DIR` one file per resource + * (`work/.yaml`, `prompts/.yaml`, …), the way to start a + * configuration directory from what a workspace already has. Never anyone's + * private resources, never a secret's value. + */ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { Command } from "commander"; +import YAML from "yaml"; +import { MANIFEST_KINDS, type ExportedManifest, type ManifestKind } from "@optio/shared"; +import { buildClient } from "../api/client.js"; +import { dim, green } from "../output/colors.js"; +import { isJsonMode, outputJson } from "../output/formatter.js"; +import { friendlyError } from "../utils/errors.js"; + +/** `work`, `mcp-server`, `McpServer` … → the manifest kind. */ +export function parseKinds(value: string | undefined): ManifestKind[] | undefined { + if (!value) return undefined; + const byLower = new Map(MANIFEST_KINDS.map((k) => [k.toLowerCase().replace(/-/g, ""), k])); + return value.split(",").map((raw) => { + const kind = byLower.get( + raw + .trim() + .toLowerCase() + .replace(/[-_\s]/g, ""), + ); + if (!kind) + throw new Error(`Unknown kind "${raw.trim()}". One of: ${MANIFEST_KINDS.join(", ")}`); + return kind; + }); +} + +function toYaml(doc: unknown): string { + return YAML.stringify(doc, { lineWidth: 100, indent: 2, blockQuote: "literal" }).trimEnd(); +} + +export const exportCommand = new Command("export") + .description("Export the organization's resources as YAML manifests") + .option( + "--kind ", + "Comma-separated kinds: work, prompt, repo, mcp-server, skill, connection", + ) + .option("--name ", "Only the resource with this name") + .option("-o, --out ", "Write one file per resource under this directory") + .action(async (opts: { kind?: string; name?: string; out?: string }, cmd) => { + try { + const kinds = parseKinds(opts.kind); + const client = buildClient(cmd.optsWithGlobals()); + const query = kinds ? `?kind=${encodeURIComponent(kinds.join(","))}` : ""; + const { manifests } = await client.get<{ manifests: ExportedManifest[] }>( + `/api/config/export${query}`, + ); + const picked = opts.name ? manifests.filter((m) => m.name === opts.name) : manifests; + if (opts.name && picked.length === 0) { + process.stderr.write( + `Nothing named "${opts.name}"${kinds ? ` of kind ${kinds.join(", ")}` : ""}.\n`, + ); + process.exitCode = 1; + return; + } + const schema = `# yaml-language-server: $schema=${client.serverUrl}/api/config/schema.json`; + + if (opts.out) { + for (const m of picked) { + const file = path.resolve(opts.out, m.path); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, `${schema}\n${toYaml(m.document)}\n`); + if (!isJsonMode()) + process.stdout.write(`${green("wrote")} ${path.relative(process.cwd(), file)}\n`); + } + if (isJsonMode()) + outputJson(picked.map((m) => ({ kind: m.kind, name: m.name, path: m.path }))); + else + process.stdout.write(dim(`${picked.length} manifest${picked.length === 1 ? "" : "s"}\n`)); + return; + } + + if (isJsonMode()) { + outputJson(picked); + return; + } + process.stdout.write(`${schema}\n${picked.map((m) => toYaml(m.document)).join("\n---\n")}\n`); + } catch (err) { + friendlyError(err); + } + }); diff --git a/apps/cli/src/commands/schema.ts b/apps/cli/src/commands/schema.ts new file mode 100644 index 000000000..8a051e558 --- /dev/null +++ b/apps/cli/src/commands/schema.ts @@ -0,0 +1,16 @@ +/** `optio schema` — the JSON Schema of a manifest, for editors and CI. */ +import { Command } from "commander"; +import { buildClient } from "../api/client.js"; +import { friendlyError } from "../utils/errors.js"; + +export const schemaCommand = new Command("schema") + .description("Print the JSON Schema that manifests validate against") + .action(async (_opts, cmd) => { + try { + const client = buildClient(cmd.optsWithGlobals()); + const schema = await client.get>("/api/config/schema.json"); + process.stdout.write(JSON.stringify(schema, null, 2) + "\n"); + } catch (err) { + friendlyError(err); + } + }); diff --git a/apps/cli/src/manifests/read.ts b/apps/cli/src/manifests/read.ts new file mode 100644 index 000000000..eefd43d56 --- /dev/null +++ b/apps/cli/src/manifests/read.ts @@ -0,0 +1,93 @@ +/** + * Reading manifests for `optio apply` / `optio diff`: files and directories + * (`*.yaml` / `*.yml`, recursively, dot-entries skipped), every document in + * each, with `*File` fields read from next to the file — the same inlining + * the API does for a configuration directory (`inlineManifestFiles` in + * @optio/shared), so a directory applies the same either way. + */ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import YAML from "yaml"; +import { inlineManifestFiles, type ManifestFileReader, type ManifestInput } from "@optio/shared"; + +export interface ReadProblem { + path: string; + message: string; +} + +export interface ReadResult { + manifests: ManifestInput[]; + problems: ReadProblem[]; +} + +async function* walk(dir: string): AsyncGenerator { + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.name.startsWith(".") || entry.name === "node_modules") continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) yield* walk(full); + else if (entry.isFile() && /\.ya?ml$/i.test(entry.name)) yield full; + } +} + +function readerFor(file: string): ManifestFileReader { + const dir = path.dirname(file); + return { + readText: (rel) => fs.readFile(path.resolve(dir, rel), "utf8"), + async readDir(rel) { + const root = path.resolve(dir, rel); + const out: Record = {}; + const visit = async (current: string, prefix: string) => { + for (const entry of await fs.readdir(current, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + const full = path.join(current, entry.name); + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) await visit(full, relPath); + else if (entry.isFile()) out[relPath] = await fs.readFile(full, "utf8"); + } + }; + await visit(root, ""); + return out; + }, + }; +} + +/** Every manifest in the given files and directories, paths shown relative to the cwd. */ +export async function readManifests(targets: string[], cwd = process.cwd()): Promise { + const files: string[] = []; + for (const target of targets) { + const full = path.resolve(cwd, target); + const stat = await fs.stat(full); + if (stat.isDirectory()) for await (const f of walk(full)) files.push(f); + else files.push(full); + } + const manifests: ManifestInput[] = []; + const problems: ReadProblem[] = []; + for (const file of files) { + const shown = path.relative(cwd, file).split(path.sep).join("/") || path.basename(file); + const text = await fs.readFile(file, "utf8"); + const documents = YAML.parseAllDocuments(text); + const many = documents.length > 1; + for (const [index, doc] of documents.entries()) { + const docPath = many ? `${shown}#${index + 1}` : shown; + if (doc.errors.length) { + problems.push({ + path: docPath, + message: doc.errors.map((e) => e.message.split("\n")[0]).join("; "), + }); + continue; + } + const value = doc.toJS() as unknown; + if (value === null || value === undefined) continue; + try { + manifests.push({ + path: docPath, + document: await inlineManifestFiles(value, readerFor(file)), + }); + } catch (err) { + problems.push({ path: docPath, message: err instanceof Error ? err.message : String(err) }); + } + } + } + return { manifests, problems }; +} diff --git a/apps/cli/src/program.ts b/apps/cli/src/program.ts index 5259be491..4fd088b10 100644 --- a/apps/cli/src/program.ts +++ b/apps/cli/src/program.ts @@ -14,6 +14,9 @@ import { sessionCommand } from "./commands/session/index.js"; import { secretCommand } from "./commands/secret/index.js"; import { workspaceCommand } from "./commands/workspace/index.js"; import { localCommand } from "./commands/local/index.js"; +import { applyCommand, diffCommand } from "./commands/apply.js"; +import { exportCommand } from "./commands/export.js"; +import { schemaCommand } from "./commands/schema.js"; export function createProgram(): Command { const program = new Command("optio") @@ -42,6 +45,11 @@ export function createProgram(): Command { program.addCommand(secretCommand); program.addCommand(workspaceCommand); program.addCommand(localCommand); + // Config as code: manifests in, manifests out (docs/config-as-code.md). + program.addCommand(applyCommand); + program.addCommand(diffCommand); + program.addCommand(exportCommand); + program.addCommand(schemaCommand); return program; } diff --git a/apps/ios/Optio/Generated/SharedTypes.swift b/apps/ios/Optio/Generated/SharedTypes.swift index e538410d5..96790f3c4 100644 --- a/apps/ios/Optio/Generated/SharedTypes.swift +++ b/apps/ios/Optio/Generated/SharedTypes.swift @@ -569,6 +569,45 @@ public struct AgentConfig: Codable, Hashable, Sendable { } } +// MARK: - config.ts + +/// Config as code: what a resource that a configuration directory manages +/// carries in every list and detail response (`docs/config-as-code.md`). +/// The file is the truth: edits made in the UI are put back at the next sync. +public struct ManagedBy: Codable, Hashable, Sendable { + /// The `config_objects` row — `POST /api/config/objects/:id/detach` takes it. + public let objectId: String + public let sourceId: String + /// The source's name as Settings shows it ("config directory"). + public let sourceName: String + /// The manifest's file, relative to the source's directory. + public let path: String + /// The manifest kind: Work, Prompt, Repo, McpServer, Skill, Connection. + public let kind: String + + private enum CodingKeys: String, CodingKey { + case objectId = "objectId" + case sourceId = "sourceId" + case sourceName = "sourceName" + case path = "path" + case kind = "kind" + } + + public init( + objectId: String, + sourceId: String, + sourceName: String, + path: String, + kind: String + ) { + self.objectId = objectId + self.sourceId = sourceId + self.sourceName = sourceName + self.path = path + self.kind = kind + } +} + // MARK: - connection.ts public struct ConnectionProviderMcpConfig: Codable, Hashable, Sendable { @@ -758,6 +797,8 @@ public struct Connection: Codable, Hashable, Sendable { public let ownerUserId: String? /// Display name of `ownerUserId`, for a private connection (lists only). public let ownerName: String? + /// Set when a configuration directory manages it (the file is the truth). + public let managedBy: ManagedBy? public let enabled: Bool public let status: ConnectionStatus public let statusMessage: String? @@ -777,6 +818,7 @@ public struct Connection: Codable, Hashable, Sendable { case workspaceId = "workspaceId" case ownerUserId = "ownerUserId" case ownerName = "ownerName" + case managedBy = "managedBy" case enabled = "enabled" case status = "status" case statusMessage = "statusMessage" @@ -797,6 +839,7 @@ public struct Connection: Codable, Hashable, Sendable { workspaceId: String? = nil, ownerUserId: String? = nil, ownerName: String? = nil, + managedBy: ManagedBy? = nil, enabled: Bool, status: ConnectionStatus, statusMessage: String? = nil, @@ -815,6 +858,7 @@ public struct Connection: Codable, Hashable, Sendable { self.workspaceId = workspaceId self.ownerUserId = ownerUserId self.ownerName = ownerName + self.managedBy = managedBy self.enabled = enabled self.status = status self.statusMessage = statusMessage @@ -1018,6 +1062,8 @@ public struct RepoConnection: Codable, Hashable, Sendable { public let ownerUserId: String? /// Display name of `ownerUserId`, for a private connection (lists only). public let ownerName: String? + /// Set when a configuration directory manages it (the file is the truth). + public let managedBy: ManagedBy? public let enabled: Bool public let status: ConnectionStatus public let statusMessage: String? @@ -1040,6 +1086,7 @@ public struct RepoConnection: Codable, Hashable, Sendable { case workspaceId = "workspaceId" case ownerUserId = "ownerUserId" case ownerName = "ownerName" + case managedBy = "managedBy" case enabled = "enabled" case status = "status" case statusMessage = "statusMessage" @@ -1061,6 +1108,7 @@ public struct RepoConnection: Codable, Hashable, Sendable { workspaceId: String? = nil, ownerUserId: String? = nil, ownerName: String? = nil, + managedBy: ManagedBy? = nil, enabled: Bool, status: ConnectionStatus, statusMessage: String? = nil, @@ -1080,6 +1128,7 @@ public struct RepoConnection: Codable, Hashable, Sendable { self.workspaceId = workspaceId self.ownerUserId = ownerUserId self.ownerName = ownerName + self.managedBy = managedBy self.enabled = enabled self.status = status self.statusMessage = statusMessage @@ -5172,6 +5221,8 @@ public struct McpServerConfig: Codable, Hashable, Sendable { /// Lists carry `ownerName` for private rows. public let ownerUserId: String? public let ownerName: String? + /// Set when a configuration directory manages it (the file is the truth). + public let managedBy: ManagedBy? public let enabled: Bool public let createdAt: Date public let updatedAt: Date @@ -5188,6 +5239,7 @@ public struct McpServerConfig: Codable, Hashable, Sendable { case workspaceId = "workspaceId" case ownerUserId = "ownerUserId" case ownerName = "ownerName" + case managedBy = "managedBy" case enabled = "enabled" case createdAt = "createdAt" case updatedAt = "updatedAt" @@ -5205,6 +5257,7 @@ public struct McpServerConfig: Codable, Hashable, Sendable { workspaceId: String? = nil, ownerUserId: String? = nil, ownerName: String? = nil, + managedBy: ManagedBy? = nil, enabled: Bool, createdAt: Date, updatedAt: Date @@ -5220,6 +5273,7 @@ public struct McpServerConfig: Codable, Hashable, Sendable { self.workspaceId = workspaceId self.ownerUserId = ownerUserId self.ownerName = ownerName + self.managedBy = managedBy self.enabled = enabled self.createdAt = createdAt self.updatedAt = updatedAt @@ -5351,6 +5405,8 @@ public struct CustomSkillConfig: Codable, Hashable, Sendable { /// Lists carry `ownerName` for private rows. public let ownerUserId: String? public let ownerName: String? + /// Set when a configuration directory manages it (the file is the truth). + public let managedBy: ManagedBy? public let layout: CustomSkillLayout /// Extra files for skill-dir layout. Null/empty = none. public let files: [CustomSkillFile]? @@ -5370,6 +5426,7 @@ public struct CustomSkillConfig: Codable, Hashable, Sendable { case workspaceId = "workspaceId" case ownerUserId = "ownerUserId" case ownerName = "ownerName" + case managedBy = "managedBy" case layout = "layout" case files = "files" case agentTypes = "agentTypes" @@ -5388,6 +5445,7 @@ public struct CustomSkillConfig: Codable, Hashable, Sendable { workspaceId: String? = nil, ownerUserId: String? = nil, ownerName: String? = nil, + managedBy: ManagedBy? = nil, layout: CustomSkillLayout, files: [CustomSkillFile]? = nil, agentTypes: [String]? = nil, @@ -5404,6 +5462,7 @@ public struct CustomSkillConfig: Codable, Hashable, Sendable { self.workspaceId = workspaceId self.ownerUserId = ownerUserId self.ownerName = ownerName + self.managedBy = managedBy self.layout = layout self.files = files self.agentTypes = agentTypes @@ -5567,6 +5626,8 @@ public struct InstalledSkillConfig: Codable, Hashable, Sendable { /// Lists carry `ownerName` for private rows. public let ownerUserId: String? public let ownerName: String? + /// Set when a configuration directory manages it (the file is the truth). + public let managedBy: ManagedBy? public let agentTypes: [String]? public let enabled: Bool public let lastSyncedAt: Date? @@ -5591,6 +5652,7 @@ public struct InstalledSkillConfig: Codable, Hashable, Sendable { case workspaceId = "workspaceId" case ownerUserId = "ownerUserId" case ownerName = "ownerName" + case managedBy = "managedBy" case agentTypes = "agentTypes" case enabled = "enabled" case lastSyncedAt = "lastSyncedAt" @@ -5616,6 +5678,7 @@ public struct InstalledSkillConfig: Codable, Hashable, Sendable { workspaceId: String? = nil, ownerUserId: String? = nil, ownerName: String? = nil, + managedBy: ManagedBy? = nil, agentTypes: [String]? = nil, enabled: Bool, lastSyncedAt: Date? = nil, @@ -5639,6 +5702,7 @@ public struct InstalledSkillConfig: Codable, Hashable, Sendable { self.workspaceId = workspaceId self.ownerUserId = ownerUserId self.ownerName = ownerName + self.managedBy = managedBy self.agentTypes = agentTypes self.enabled = enabled self.lastSyncedAt = lastSyncedAt @@ -6452,6 +6516,8 @@ public struct PersistentAgent: Codable, Hashable, Sendable { /// Personal work runs with that person's secrets, model providers and /// connections, and only they can change it. public let ownerUserId: String? + /// Set when a configuration directory manages it (the file is the truth). + public let managedBy: ManagedBy? /// The secrets (by name) the agent gets in its pod. Null = the workspace's /// legacy behavior (see `Workspace.restrictPodSecrets`). public let podSecrets: [String]? @@ -6492,6 +6558,7 @@ public struct PersistentAgent: Codable, Hashable, Sendable { case reconcileAttempts = "reconcileAttempts" case createdBy = "createdBy" case ownerUserId = "ownerUserId" + case managedBy = "managedBy" case podSecrets = "podSecrets" case createdAt = "createdAt" case updatedAt = "updatedAt" @@ -6531,6 +6598,7 @@ public struct PersistentAgent: Codable, Hashable, Sendable { reconcileAttempts: Double, createdBy: String? = nil, ownerUserId: String? = nil, + managedBy: ManagedBy? = nil, podSecrets: [String]? = nil, createdAt: Date, updatedAt: Date @@ -6568,6 +6636,7 @@ public struct PersistentAgent: Codable, Hashable, Sendable { self.reconcileAttempts = reconcileAttempts self.createdBy = createdBy self.ownerUserId = ownerUserId + self.managedBy = managedBy self.podSecrets = podSecrets self.createdAt = createdAt self.updatedAt = updatedAt diff --git a/apps/web/e2e/config-as-code.spec.ts b/apps/web/e2e/config-as-code.spec.ts new file mode 100644 index 000000000..aa1f3fc61 --- /dev/null +++ b/apps/web/e2e/config-as-code.spec.ts @@ -0,0 +1,59 @@ +/** + * Config as code in the UI: the e2e stack mounts a directory with one Prompt + * manifest (launch-stack.ts), so Settings → Config as code shows the source + * and its last sync, Sync now works, and the Prompts page marks the managed + * row. The apply itself is covered by the API's integration and e2e tests. + */ +import { expect, test } from "@playwright/test"; + +const API = "http://127.0.0.1:4931"; + +interface Status { + enabled: boolean; + source: { path: string; lastSyncAt: string | null } | null; +} + +async function syncedStatus(): Promise { + const deadline = Date.now() + 60_000; + for (;;) { + const res = await fetch(`${API}/api/config/status`); + const status = (await res.json()) as Status; + if (status.source?.lastSyncAt) return status; + if (Date.now() > deadline) throw new Error("the configuration directory never synced"); + await new Promise((r) => setTimeout(r, 500)); + } +} + +test.describe("Config as code", () => { + test("Settings shows the directory, its last sync, and syncs on demand", async ({ page }) => { + const status = await syncedStatus(); + expect(status.enabled).toBe(true); + + await page.goto("/settings"); + const card = page.locator("section", { hasText: "Config as code" }).first(); + await expect(page.getByText("Config as code", { exact: true })).toBeVisible({ + timeout: 30_000, + }); + await expect(card.getByText(status.source!.path, { exact: true })).toBeVisible(); + await expect(card.getByText(/^\d+ unchanged$|^\d+ created$/)).toBeVisible(); + + await card.getByRole("button", { name: "Sync now" }).click(); + await expect(page.getByText(/^Synced/)).toBeVisible({ timeout: 30_000 }); + }); + + test("Prompts marks the managed row", async ({ page }) => { + await syncedStatus(); + await page.goto("/templates"); + await expect(page.getByText("E2E managed prompt", { exact: true })).toBeVisible({ + timeout: 30_000, + }); + const row = page + .locator("li, tr, div", { hasText: "E2E managed prompt" }) + .filter({ hasText: "Managed" }) + .first(); + await expect(row).toBeVisible(); + // The seeded, hand-made prompt carries no Managed chip. + const seeded = page.getByText("E2E seed prompt", { exact: true }); + await expect(seeded).toBeVisible(); + }); +}); diff --git a/apps/web/e2e/launch-stack.ts b/apps/web/e2e/launch-stack.ts index 2af897e81..190dcc169 100644 --- a/apps/web/e2e/launch-stack.ts +++ b/apps/web/e2e/launch-stack.ts @@ -21,6 +21,8 @@ * Fixed ports (chosen to avoid dev defaults): API 4931, web 3131. */ import { execFileSync, spawn, type ChildProcess } from "node:child_process"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import buildTestInfra from "../../api/src/test-utils/integration/global-setup.js"; @@ -195,7 +197,27 @@ async function main(): Promise { console.warn("[stack] starting API server..."); // Assigned before anything can throw, so shutdown() and the exit handler // always find the API server. - apiServer = await startApiServer({ port: API_PORT, logLevel: "warn" }); + // Config as code: a directory with one manifest, so Settings → Config as + // code has a source and the Prompts page a Managed row (config-as-code.spec.ts). + const configDir = mkdtempSync(join(tmpdir(), "optio-e2e-config-")); + mkdirSync(join(configDir, "prompts")); + writeFileSync( + join(configDir, "prompts", "e2e-managed-prompt.yaml"), + [ + "apiVersion: optio/v1", + "kind: Prompt", + "metadata:", + " name: E2E managed prompt", + "spec:", + " template: From the configuration directory {{thing}}", + "", + ].join("\n"), + ); + apiServer = await startApiServer({ + port: API_PORT, + logLevel: "warn", + env: { OPTIO_CONFIG_DIR: configDir, OPTIO_CONFIG_INTERVAL: "15000" }, + }); console.warn(`[stack] API ready at ${apiServer.baseUrl}`); console.warn("[stack] seeding data..."); diff --git a/apps/web/src/app/agents/[id]/page.tsx b/apps/web/src/app/agents/[id]/page.tsx index b005f4b3d..de43ffb58 100644 --- a/apps/web/src/app/agents/[id]/page.tsx +++ b/apps/web/src/app/agents/[id]/page.tsx @@ -30,8 +30,12 @@ import { RunsAsBadge } from "@/components/runs-as-badge"; import { DetailHeader } from "@/components/detail-header"; import { Segmented } from "@/components/ui/segmented"; import { AgentIcon } from "@/components/brand-icon"; +import { ManagedBanner } from "@/components/ui/managed-banner"; +import { ManagedChip } from "@/components/ui/managed-chip"; +import type { ManagedBy } from "@optio/shared"; interface Agent { + managedBy?: ManagedBy | null; /** Private work: whose it is (null = the organization's); `ownerName` when the API names them. */ ownerUserId?: string | null; ownerName?: string | null; @@ -238,6 +242,7 @@ export default function AgentDetailPage() { <> @{agent.slug} + } metaItems={[ @@ -295,6 +300,9 @@ export default function AgentDetailPage() { } /> + {agent.managedBy && ( + + )}
{agent.description ? ( diff --git a/apps/web/src/app/connections/page.tsx b/apps/web/src/app/connections/page.tsx index 5de19b8c0..db9a1772e 100644 --- a/apps/web/src/app/connections/page.tsx +++ b/apps/web/src/app/connections/page.tsx @@ -34,6 +34,7 @@ import { OwnerPicker } from "@/components/ui/owner-picker"; import { OwnerSegments, useOwnerFilter } from "@/components/ui/owner-segments"; import { ScopedList } from "@/components/ui/scoped-list"; import { OwnerChip } from "@/components/ui/owner-chip"; +import { ManagedChip } from "@/components/ui/managed-chip"; import { brandFor, brandIconComponent } from "@/components/brand-icon"; import { countByOwner, ownerOf, ownerScope, privateHint, scopeOf } from "@/lib/owner"; @@ -733,6 +734,7 @@ function ConnectionsBody() { {provider && {provider.name}} {/* Sections already say the scope; the chip is for a flat (filtered) list. */} {scope === null && } + {scope === "others" && ( {conn.ownerName ?? "someone"} diff --git a/apps/web/src/app/jobs/[id]/page.tsx b/apps/web/src/app/jobs/[id]/page.tsx index 3976e86a6..ae79b56cf 100644 --- a/apps/web/src/app/jobs/[id]/page.tsx +++ b/apps/web/src/app/jobs/[id]/page.tsx @@ -39,10 +39,14 @@ import { RunsAsBadge } from "@/components/runs-as-badge"; import { EmptyState } from "@/components/empty-state"; import { Panel, PanelEmpty } from "@/components/ui/panel"; import { Segmented } from "@/components/ui/segmented"; +import { ManagedBanner } from "@/components/ui/managed-banner"; +import { ManagedChip } from "@/components/ui/managed-chip"; +import type { ManagedBy } from "@optio/shared"; // ── Types ────────────────────────────────────────────────────────────────────── interface WorkflowDetail { + managedBy?: ManagedBy | null; /** Private work: whose it is (null = the organization's); `ownerName` when the API names them. */ ownerUserId?: string | null; ownerName?: string | null; @@ -264,6 +268,7 @@ export default function WorkflowDetailPage({ params }: { params: Promise<{ id: s } state={workflow.enabled ? "enabled" : "disabled"} + extraBadges={} metaItems={ workflow.description || workflow.ownerUserId ? [ @@ -350,6 +355,13 @@ export default function WorkflowDetailPage({ params }: { params: Promise<{ id: s } /> + {workflow.managedBy && ( + + )}
{/* Stats bar */} diff --git a/apps/web/src/app/repos/[id]/page.tsx b/apps/web/src/app/repos/[id]/page.tsx index 8797fe2df..f4b5f188d 100644 --- a/apps/web/src/app/repos/[id]/page.tsx +++ b/apps/web/src/app/repos/[id]/page.tsx @@ -41,6 +41,8 @@ import { repoAgentValues, runtimeLabel, } from "@/components/agent-choice-model"; +import { ManagedBanner } from "@/components/ui/managed-banner"; +import { ManagedChip } from "@/components/ui/managed-chip"; const INPUT = "w-full px-3 py-2 rounded-lg bg-bg border border-border text-sm focus:outline-none focus:border-primary focus:ring-1 focus:ring-primary/20"; @@ -373,10 +375,17 @@ export default function RepoDetailPage({ params }: { params: Promise<{ id: strin } extraBadges={ - - {repo.isPrivate ? : } - {repo.isPrivate ? "Private" : "Public"} - + <> + + {repo.isPrivate ? ( + + ) : ( + + )} + {repo.isPrivate ? "Private" : "Public"} + + + } metaItems={[ <> @@ -403,6 +412,9 @@ export default function RepoDetailPage({ params }: { params: Promise<{ id: strin } /> + {repo.managedBy && ( + + )}
{sessions.length > 0 && ( )} +
diff --git a/apps/web/src/app/settings/page.tsx b/apps/web/src/app/settings/page.tsx index f406f433e..050873904 100644 --- a/apps/web/src/app/settings/page.tsx +++ b/apps/web/src/app/settings/page.tsx @@ -42,6 +42,9 @@ import { Segmented } from "@/components/ui/segmented"; import { Disclosure } from "@/components/ui/disclosure"; import { OwnerPicker } from "@/components/ui/owner-picker"; import { OwnerChip } from "@/components/ui/owner-chip"; +import { ManagedChip } from "@/components/ui/managed-chip"; +import { ConfigAsCodeSettings } from "@/components/settings/config-as-code"; +import type { ManagedBy } from "@optio/shared"; import { useCurrentUser } from "@/hooks/use-current-user"; import { OWNER_SCOPE_LABEL, ownerOf, ownerScope, type Owned, type OwnerScope } from "@/lib/owner"; import { @@ -398,13 +401,26 @@ function ScopeTags({ scope, viewerId, }: { - row: Owned; + row: Owned & { managedBy?: ManagedBy | null }; scope: OwnerScope | null; viewerId: string | null; }) { - if (scope === null) return ; - if (scope === "others") return {row.ownerName ?? "someone"}; - return null; + const managed = ; + if (scope === null) + return ( + <> + + {managed} + + ); + if (scope === "others") + return ( + <> + {row.ownerName ?? "someone"} + {managed} + + ); + return managed; } function GlobalMcpServers() { @@ -2017,6 +2033,10 @@ export default function SettingsPage() { + + + + diff --git a/apps/web/src/app/tasks/scheduled/[id]/page.tsx b/apps/web/src/app/tasks/scheduled/[id]/page.tsx index f88c97c87..7608cfc1e 100644 --- a/apps/web/src/app/tasks/scheduled/[id]/page.tsx +++ b/apps/web/src/app/tasks/scheduled/[id]/page.tsx @@ -31,6 +31,9 @@ import { EmptyState } from "@/components/empty-state"; import { Panel, PanelEmpty } from "@/components/ui/panel"; import { Segmented } from "@/components/ui/segmented"; import { cn } from "@/lib/utils"; +import { ManagedBanner } from "@/components/ui/managed-banner"; +import { ManagedChip } from "@/components/ui/managed-chip"; +import type { ManagedBy } from "@optio/shared"; /** * A scheduled Task's runs and actions. Its five answers (when / where / who / @@ -41,6 +44,7 @@ import { cn } from "@/lib/utils"; type Tab = "runs" | "triggers"; interface TaskConfig { + managedBy?: ManagedBy | null; /** Private work: whose it is (null = the organization's); `ownerName` when the API names them. */ ownerUserId?: string | null; ownerName?: string | null; @@ -258,7 +262,12 @@ function ScheduledTaskDetailInner({ id }: { id: string }) { } state={config.enabled ? "enabled" : "disabled"} - extraBadges={} + extraBadges={ + <> + + + + } metaItems={[ <> @@ -322,6 +331,9 @@ function ScheduledTaskDetailInner({ id }: { id: string }) { } /> + {config.managedBy && ( + + )}
{scheduleTrigger && scheduleTrigger.enabled && config.enabled && ( diff --git a/apps/web/src/app/templates/page.tsx b/apps/web/src/app/templates/page.tsx index c97474633..d339cbe2d 100644 --- a/apps/web/src/app/templates/page.tsx +++ b/apps/web/src/app/templates/page.tsx @@ -13,6 +13,7 @@ import { OwnerPicker } from "@/components/ui/owner-picker"; import { OwnerSegments, useOwnerFilter } from "@/components/ui/owner-segments"; import { ScopedList } from "@/components/ui/scoped-list"; import { OwnerChip } from "@/components/ui/owner-chip"; +import { ManagedChip } from "@/components/ui/managed-chip"; import { countByOwner, inOwnerFilter, @@ -21,11 +22,13 @@ import { privateHint, type OwnerScope, } from "@/lib/owner"; +import type { ManagedBy } from "@optio/shared"; type TemplateKind = "prompt" | "review" | "job" | "task"; type PickedScope = "organization" | "private"; interface Template { + managedBy?: ManagedBy | null; id: string; name: string; template: string; @@ -242,6 +245,7 @@ function PromptsList() { {/* Sections already say the scope; the chip is for a flat list. */} {scope === null && } + {scope === "others" && ( {t.ownerName ?? "someone"} diff --git a/apps/web/src/components/settings/config-as-code.tsx b/apps/web/src/components/settings/config-as-code.tsx new file mode 100644 index 000000000..bd0ae10fd --- /dev/null +++ b/apps/web/src/components/settings/config-as-code.tsx @@ -0,0 +1,231 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { Download, Eye, FileCode2, RefreshCw } from "lucide-react"; +import { toast } from "sonner"; +import type { ConfigApplyResult, ConfigPlanItem, ConfigStatus } from "@optio/shared"; +import { api } from "@/lib/api-client"; +import { useCurrentUser } from "@/hooks/use-current-user"; +import { SectionCard } from "@/components/ui/section-card"; +import { BTN_HEADER, SkeletonCard, Tag } from "@/components/settings/settings-ui"; + +/** + * Settings → Config as code: the configuration directory a cluster reads + * (`OPTIO_CONFIG_DIR`), its workspace, how the last sync went — counts and + * every error with its file — and, for admins, **Sync now** and **Preview** (a + * dry run). Without a directory the card says how to turn it on, and the + * export and the CLI are there either way. docs/config-as-code.md. + */ +export function ConfigAsCodeSettings() { + const { isAdmin, loaded } = useCurrentUser(); + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState<"sync" | "preview" | null>(null); + const [preview, setPreview] = useState(null); + + const refresh = useCallback(() => { + api + .getConfigStatus() + .then(setStatus) + .catch(() => setStatus(null)) + .finally(() => setLoading(false)); + }, []); + + useEffect(() => { + refresh(); + }, [refresh]); + + const run = async (dryRun: boolean) => { + setBusy(dryRun ? "preview" : "sync"); + try { + const result = await api.syncConfigSource(dryRun); + if (dryRun) { + setPreview(result); + } else { + setPreview(null); + const s = result.summary; + toast.success( + s.errors + ? `Synced with ${s.errors} error${s.errors === 1 ? "" : "s"}` + : `Synced: ${s.created} created, ${s.updated} updated, ${s.pruned} pruned`, + ); + refresh(); + } + } catch (err) { + toast.error(err instanceof Error ? err.message : "Sync failed"); + } finally { + setBusy(null); + } + }; + + const label = "Config as code"; + const hint = + "YAML manifests a cluster reads · Jobs, agents, prompts, repos, MCP servers, skills, connections"; + if (loading || !loaded) return ; + + const source = status?.source ?? null; + const last = source?.lastSync ?? null; + + return ( + + + + Export YAML + + {source && isAdmin && ( + <> + + + + )} +
+ } + bodyClassName="p-4 space-y-3" + > + {source ? ( + <> +
+
Directory
+
{source.path}
+
Reads
+
+ every {Math.round(source.intervalMs / 1000)}s · prune {source.prune ? "on" : "off"} + {source.origin === "env" && ( + · set by the deployment + )} +
+
Last sync
+
+ {source.lastSyncAt ? new Date(source.lastSyncAt).toLocaleString() : "never"} + {source.lastSyncHash && ( + · {source.lastSyncHash} + )} +
+
+ {source.lastSyncError && ( +

+ {source.lastSyncError} +

+ )} + {last && } + {preview && ( +
+
+ + Preview — what the next sync would do + + +
+ +
+ )} + + ) : ( +

+ Mount a directory of manifests into the API and set{" "} + OPTIO_CONFIG_DIR (Helm:{" "} + configAsCode.enabled) to have this workspace follow + it. Start from what you have: Export YAML here, or{" "} + optio export -o optio/, then commit the files. +

+ )} +

+ + Schema for editors and CI:{" "} + + {status?.schemaUrl ?? "/api/config/schema.json"} + + {" · "} + optio apply -f optio/ applies files by hand (no pruning). +

+ + ); +} + +function summaryLine(result: ConfigApplyResult): string { + const s = result.summary; + const parts = [ + s.created && `${s.created} created`, + s.updated && `${s.updated} updated`, + s.reverted && `${s.reverted} reverted`, + s.pruned && `${s.pruned} pruned`, + s.errors && `${s.errors} error${s.errors === 1 ? "" : "s"}`, + ].filter(Boolean); + return parts.length ? parts.join(" · ") : `${s.unchanged} unchanged`; +} + +const TONE: Record = { + create: "primary", + update: "primary", + adopt: "primary", + replace: "warning", + prune: "error", + error: "error", + unchanged: undefined, +}; + +/** Counts, then the items worth reading: errors always, everything with `all`. */ +function ResultSummary({ result, all = false }: { result: ConfigApplyResult; all?: boolean }) { + const s = result.summary; + const items = result.items.filter((i) => all || i.action === "error" || i.reverted); + return ( +
+
+ {s.unchanged} unchanged + {s.created > 0 && {s.created} created} + {s.updated > 0 && {s.updated} updated} + {s.reverted > 0 && {s.reverted} UI edits put back} + {s.adopted > 0 && {s.adopted} adopted} + {s.replaced > 0 && {s.replaced} replaced} + {s.pruned > 0 && {s.pruned} pruned} + {s.errors > 0 && {s.errors} errors} +
+ {items.length > 0 && ( +
    + {items.map((item, i) => ( +
  • + {item.reverted ? "reverted" : item.action} + + {item.kind}/{item.name} + + {item.path} + {(item.message || item.changes?.length) && ( + + {item.message ?? item.changes?.join(", ")} + + )} +
    • + ))} +
+ )} +
+ ); +} diff --git a/apps/web/src/components/ui/README.md b/apps/web/src/components/ui/README.md index f29d22d21..7d082814f 100644 --- a/apps/web/src/components/ui/README.md +++ b/apps/web/src/components/ui/README.md @@ -18,6 +18,8 @@ there is no `danger`). | `ScopedList` (`scoped-list.tsx`) | Rows grouped by scope. `All` renders one `Panel` per scope (Organization / Private / Other people's) so every scope shows at a glance; any other filter renders the matching rows flat. `render(rows, scope)` draws the rows; `privateEmpty` is the Private section's one line. | | `OwnerPicker` (`owner-picker.tsx`) | The **Owner** row in every create / edit form: Organization or Private, one helper sentence per resource (`what`), the Organization pill disabled with its reason when the viewer can't make one (`canOrg`). | | `OwnerChip` (`owner-chip.tsx`) | The chip for a private row where scopes mix without sections (search, the Work list, pickers, detail headers): **Private** for the viewer's own, **Private · Name** for someone else's. Organization rows carry none. | +| `ManagedChip` (`managed-chip.tsx`) | The chip for a row a configuration directory manages (config as code): **Managed**, the file on hover. Sits next to `OwnerChip` in every list. Rows nobody manages carry none. | +| `ManagedBanner`, `DownloadYamlLink` (`managed-banner.tsx`) | The notice under a managed resource's header: the file, "edits here are put back at the next sync", **YAML** (the export) and, for admins, **Detach**. `DownloadYamlLink` alone is the YAML action for any exportable page. | | `StatTile` (`stat-tile.tsx`) | A headline number: uppercase 11px `label` with `icon`, `text-2xl tabular-nums` `value`, `tone` (e.g. `text-warning`) only when it needs attention, optional `href`. Lay out in a `grid gap-3`. | ## Elsewhere in `components/` diff --git a/apps/web/src/components/ui/managed-banner.tsx b/apps/web/src/components/ui/managed-banner.tsx new file mode 100644 index 000000000..d60732362 --- /dev/null +++ b/apps/web/src/components/ui/managed-banner.tsx @@ -0,0 +1,114 @@ +"use client"; + +import { useState } from "react"; +import { Download, FileCode2, Unlink } from "lucide-react"; +import { toast } from "sonner"; +import type { ManagedBy } from "@optio/shared"; +import { api } from "@/lib/api-client"; +import { useCurrentUser } from "@/hooks/use-current-user"; +import { cn } from "@/lib/utils"; + +/** + * The notice on the pages of a resource a configuration directory manages + * (config as code): the file is the truth, so an edit made here is put back at + * the next sync. Offers the YAML as the API would export it today, and — for + * admins — **Detach**, which makes the resource ordinary again. + */ +export function ManagedBanner({ + managedBy, + resourceId, + onDetached, + className, +}: { + managedBy?: ManagedBy | null; + /** The resource's id (what `GET /api/config/export?id=` takes). */ + resourceId: string; + /** Called after a detach succeeded (the page should reload its row). */ + onDetached?: () => void; + className?: string; +}) { + const { isAdmin } = useCurrentUser(); + const [busy, setBusy] = useState(false); + if (!managedBy) return null; + + const detach = async () => { + if ( + !confirm( + `Stop managing this from ${managedBy.path}? It stays as it is and is edited by hand from now on. If the file stays in the directory, the next sync takes it over again.`, + ) + ) { + return; + } + setBusy(true); + try { + await api.detachConfigObject(managedBy.objectId); + toast.success("Detached — this is now edited by hand"); + onDetached?.(); + } catch (err) { + toast.error(err instanceof Error ? err.message : "Couldn't detach"); + } finally { + setBusy(false); + } + }; + + return ( +
+ + + Managed by {managedBy.sourceName} + {" · "} + {managedBy.path} + {" — edits made here are put back at the next sync; change the file instead."} + + + + YAML + + {isAdmin && ( + + )} +
+ ); +} + +/** A "Download YAML" action for the page of any exportable resource. */ +export function DownloadYamlLink({ + kind, + resourceId, + className, +}: { + kind: string; + resourceId: string; + className?: string; +}) { + return ( + + + YAML + + ); +} diff --git a/apps/web/src/components/ui/managed-chip.tsx b/apps/web/src/components/ui/managed-chip.tsx new file mode 100644 index 000000000..2cb43ccc7 --- /dev/null +++ b/apps/web/src/components/ui/managed-chip.tsx @@ -0,0 +1,38 @@ +import { FileCode2 } from "lucide-react"; +import type { ManagedBy } from "@optio/shared"; +import { cn } from "@/lib/utils"; + +/** What a managed row's chip and banner say about the file. */ +export function managedTitle(managedBy: ManagedBy): string { + return `Managed by ${managedBy.sourceName} · ${managedBy.path} — edits made here are put back at the next sync; change the file instead`; +} + +/** + * The one chip for a row a configuration directory manages (config as code): + * **Managed**, with the file on hover. Rows nobody manages carry no chip. + * Shown next to the Private chip wherever rows are listed. + */ +export function ManagedChip({ + managedBy, + size = "xs", + className, +}: { + managedBy?: ManagedBy | null; + size?: "xs" | "sm"; + className?: string; +}) { + if (!managedBy) return null; + return ( + + + Managed + + ); +} diff --git a/apps/web/src/components/work-form/load.ts b/apps/web/src/components/work-form/load.ts index d26b2fc64..8d451d045 100644 --- a/apps/web/src/components/work-form/load.ts +++ b/apps/web/src/components/work-form/load.ts @@ -1,5 +1,6 @@ import { getProviderCatalog, providerForAgentType } from "@optio/shared"; import { api } from "@/lib/api-client"; +import type { ManagedBy } from "@optio/shared"; import { runLocationFromRow } from "@/components/run-location-picker"; import type { TriggerConfig } from "@/components/trigger-selector"; import { @@ -37,6 +38,8 @@ export interface EditTarget { */ foreignOwnerId: string | null; foreignOwnerName: string | null; + /** Set when a configuration directory manages the row (edits are put back at the next sync). */ + managedBy: ManagedBy | null; } /** @@ -221,5 +224,6 @@ export async function loadEditTarget(id: string): Promise { draft: draftFromRow(work, trigger, meId), foreignOwnerId, foreignOwnerName: foreignOwnerId ? (row?.ownerName ?? null) : null, + managedBy: row?.managedBy ?? null, }; } diff --git a/apps/web/src/components/work-form/work-form.tsx b/apps/web/src/components/work-form/work-form.tsx index b4e5f20be..f7adb4de8 100644 --- a/apps/web/src/components/work-form/work-form.tsx +++ b/apps/web/src/components/work-form/work-form.tsx @@ -40,6 +40,7 @@ import { SectionCard as Section } from "@/components/ui/section-card"; import { Segmented } from "@/components/ui/segmented"; import { Disclosure } from "@/components/ui/disclosure"; import { OwnerChip } from "@/components/ui/owner-chip"; +import { ManagedBanner } from "@/components/ui/managed-banner"; import { AgentChoice, DefaultsHint } from "@/components/agent-choice"; import { RunLocationPicker } from "@/components/run-location-picker"; import { AgentIcon, PrIcon, TriggerIcon } from "@/components/brand-icon"; @@ -685,6 +686,10 @@ export function WorkForm({ edit }: { edit?: EditTarget } = {}) { ))}
+ {edit?.managedBy && ( + + )} + {readOnly && (
{row.name} +
{row.statusLabel} diff --git a/apps/web/src/lib/api-client.ts b/apps/web/src/lib/api-client.ts index 8bc64c3fc..603164efa 100644 --- a/apps/web/src/lib/api-client.ts +++ b/apps/web/src/lib/api-client.ts @@ -20,6 +20,7 @@ import type { WorkSpec, WorkEnvironmentOptions, } from "@optio/shared"; +import type { ConfigApplyResult, ConfigStatus, ExportedManifest } from "@optio/shared"; /** Read the current workspace ID from localStorage (set by workspace switcher). */ function getWorkspaceId(): string | null { @@ -2189,4 +2190,34 @@ export const api = { deleteLocalBlueprintTrigger: (id: string, triggerId: string) => request<{}>(`/api/local/blueprints/${id}/triggers/${triggerId}`, { method: "DELETE" }), + + // Config as code (docs/config-as-code.md) + getConfigStatus: () => request("/api/config/status"), + syncConfigSource: (dryRun = false) => + request(`/api/config/source/sync${dryRun ? "?dryRun=true" : ""}`, { + method: "POST", + }), + applyConfig: (manifests: Array<{ path: string; document: unknown }>, dryRun = false) => + request("/api/config/apply", { + method: "POST", + body: JSON.stringify({ manifests, dryRun }), + }), + exportConfig: (kind?: string, id?: string) => { + const qs = new URLSearchParams(); + if (kind) qs.set("kind", kind); + if (id) qs.set("id", id); + const query = qs.toString(); + return request<{ manifests: ExportedManifest[] }>( + `/api/config/export${query ? `?${query}` : ""}`, + ); + }, + /** The browser URL that downloads the export (one resource, or everything) as YAML. */ + configExportUrl: (kind?: string, id?: string) => { + const qs = new URLSearchParams({ download: "1" }); + if (kind) qs.set("kind", kind); + if (id) qs.set("id", id); + return `/api/config/export.yaml?${qs.toString()}`; + }, + detachConfigObject: (objectId: string) => + request(`/api/config/objects/${objectId}/detach`, { method: "POST" }), }; diff --git a/docs/config-as-code.md b/docs/config-as-code.md new file mode 100644 index 000000000..2731a7e76 --- /dev/null +++ b/docs/config-as-code.md @@ -0,0 +1,183 @@ +# Config as code + +Optio's resources — Jobs, scheduled Tasks, persistent agents, prompts, repos, +MCP servers, skills and connections — can live as YAML **manifests** in a +directory a cluster reads. The files are the truth: a manifest that changes is +applied, one that disappears is pruned, and a managed resource someone edits in +the UI is put back at the next sync. `optio export` writes what a workspace has +as manifests; `optio apply` applies a file or a directory by hand. Design and +decisions: [plans/config-as-code.md](plans/config-as-code.md). + +## A manifest + +One YAML document per resource; a file may hold several, separated by `---`. + +```yaml +# yaml-language-server: $schema=https://optio.example.com/api/config/schema.json +apiVersion: optio/v1 +kind: Work +metadata: + name: nightly-dependency-bump # the identity: unique per workspace and kind + description: Bumps dependencies every weeknight and opens a PR +spec: + when: + schedule: "0 3 * * 1-5" + where: + repo: https://github.com/acme/api + branch: main + who: + runtime: claude-code + options: { model: claude-sonnet-4-5, effort: high } + what: + promptFile: ./prompts/dependency-bump.md + runTitle: "Dependency bump" + then: until-merged + secrets: [GITHUB_TOKEN, NPM_TOKEN] + environment: + connections: { add: [Linear] } + review: { enabled: true, trigger: on_pr } +``` + +The JSON Schema every manifest validates against is served at +`GET /api/config/schema.json` (public) — point your editor's YAML language +server at it, or validate in CI with any JSON Schema tool (`optio schema` +prints it). + +### Kinds + +| Kind | Identity | `spec` | +| ------------ | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Work` | `metadata.name` | `when`, `where`, `who`, `what`, `then` (the New work form's attributes), `mergeWhenReady`, `retries`, `priority`, `secrets` (pod secrets by name), `environment` (connections / MCP servers / skills by **name**, setup commands, review, cautious mode, resume cap), `agent` (a persistent agent's slug, prompts, pod), `params`, `limits`, `pods`, `enabled` | +| `Prompt` | `metadata.name` | `kind` (prompt / review / job / task), `template` or `templateFile`, `params`, `defaultAgentType` | +| `Repo` | `spec.url` | `defaultBranch` and any of the settings `PATCH /api/repos/:id` takes (image, setup commands, review, PR behavior, concurrency, pod resources). **Only the settings a manifest names are managed**; the rest keep their values. The Slack webhook (a credential) isn't one of them. | +| `McpServer` | `metadata.name` | `command`, `args`, `env` (values may be `${{SECRET_NAME}}`), `installCommand`, `repo` (scope), `enabled` | +| `Skill` | `metadata.name` | a custom skill — `prompt` / `promptFile` / `filesFrom` (a directory: `SKILL.md` is the prompt, the rest the files), `layout`, `files` — or a marketplace one — `source: { url, ref, path }`; plus `agentTypes`, `repo`, `enabled` | +| `Connection` | `metadata.name` | `provider` (slug), `config` (secret fields as `${{SECRET_NAME}}` references), `repo` (scope), `enabled`, `assignments: [{ repo?, agentTypes?, permission? }]` | + +### Rules + +- **`when`** has one key, the trigger type, holding that trigger's config: + `schedule: ""`, `webhook: { path }`, `ticket: { source, labels? }`, + `github: {…}`, `slack: {…}`, `linear: {…}`. Absent = on demand. A webhook's + signing secret is never in a file; set it once through the trigger API and + it is kept across applies. +- **`then`** is `exits` (default), `until-merged` (needs `where.repo`), or + `waits-for-messages` (a persistent agent, with `agent: { slug, systemPrompt | +systemPromptFile, agentsMd | agentsMdFile, podLifecycle }`). A Task that runs + once (`exits`, a repo, no `when`) is a run, not configuration, and is refused; + so is work on a machine. +- **`who.runtime`** is an agent runtime id, or `shell` for a Job that runs its + prompt as a command. +- **References are names.** Repos by URL, connections / MCP servers / skills by + name, secrets by name, agents by runtime id. An unknown name fails that + manifest alone. A repo a manifest names must be registered (a `Repo` + manifest next to it does that; kinds apply in dependency order). +- **Secrets are never in files.** `secrets:` lists pod secrets by name; a + connection's or MCP server's credential fields are `${{SECRET_NAME}}` + references, and a literal value in a credential field is refused. An export + writes the reference (`${{}}`, the name the pod falls back to) + in place of any stored literal. +- **`*File` fields** (`promptFile`, `templateFile`, `systemPromptFile`, + `agentsMdFile`, `filesFrom`) name files next to the manifest. Whoever reads + the directory inlines them — the API for the configuration directory, the CLI + for `apply` — and a field given both ways is an error. +- Everything a manifest declares is **the organization's**. Private work, + prompts and the like are not expressible, and a manifest whose name collides + with someone's private resource fails. + +## The configuration directory + +One directory per cluster, bound to one workspace, declared by the deployment: + +| Env | Helm | Meaning | +| ------------------------ | ------------------------- | ------------------------------------------------------------------------- | +| `OPTIO_CONFIG_DIR` | `configAsCode.mountPath` | The directory in the API pod. Set = config as code is on. | +| `OPTIO_CONFIG_WORKSPACE` | `configAsCode.workspace` | The workspace's slug. Default: the oldest workspace (the organization's). | +| `OPTIO_CONFIG_PRUNE` | `configAsCode.prune` | Delete resources whose manifest is gone (default `true`). | +| `OPTIO_CONFIG_INTERVAL` | `configAsCode.intervalMs` | How often the directory is read (default 60000 ms, at least 10000). | + +The chart mounts the directory from inline manifests (`configAsCode.files`, +rendered into a ConfigMap), from a ConfigMap you maintain +(`configAsCode.configMap`), or from anything you mount at `mountPath` with +`api.extraVolumes` / `api.extraVolumeMounts` (a PVC, a sync sidecar's volume). +A changed ConfigMap needs no pod roll: the mount updates in place and the next +tick reads it. + +Every interval (and once at boot) the sync worker reads every `*.yaml` / +`*.yml` under the directory and **applies** it: + +1. Each document is validated and resolved; a file that doesn't parse or a + manifest that names something unknown is one error item, the rest apply. +2. Per manifest: **create** when nothing has its name; **adopt** an existing + unmanaged resource with its name (so `optio export`, commit, mount takes a + workspace over without conflicts); **update** when the row differs from + what the manifest would write — including **drift**, a row someone edited in + the UI, which is put back and reported as reverted; **unchanged** otherwise; + **replace** (delete and recreate, run history lost) when the row can't be + changed in place — a Job that gained a repo, an MCP server that moved scope, + a connection that changed provider. +3. **Prune**: what the directory managed and no longer declares is deleted + (off with `OPTIO_CONFIG_PRUNE=false`: orphans are then only reported). + +Kinds apply in dependency order (Repo, McpServer, Skill, Connection, Prompt, +Work), each manifest on its own, so a Work manifest can name the connection +declared next to it. The apply is idempotent: rows that already match aren't +written, so a quiet tick costs a few queries. + +**Settings → Config as code** shows the directory, its workspace, the last sync +(time, counts, every error with its file) and has **Sync now** and **Preview** +(a dry run) for admins. `POST /api/config/source/sync[?dryRun=true]` is the +same over HTTP. The source is mirrored into `config_sources`; what it manages +is in `config_objects` (manifest kind + name → the row). + +### Managed resources + +A resource the directory manages carries `managedBy` (source, file, kind) in +every list and detail response, shows a **Managed** chip next to its name, and +a banner on its pages saying edits are put back at the next sync. It stays +editable — operating it (run now, cancel, message an agent) and even editing +it is allowed; the file just wins. **Detach** (admin, on the banner or +`POST /api/config/objects/:id/detach`) stops that for one resource: it becomes +ordinary. If its manifest is still in the directory, the next sync adopts it +again — remove the file too. + +## The CLI + +``` +optio export [--kind work,prompt,…] [--name X] [-o DIR] # the organization's resources as manifests +optio apply -f FILE|DIR... [--dry-run] # create / update what the manifests describe +optio diff -f FILE|DIR... # apply --dry-run +optio schema # the JSON Schema +``` + +`export -o optio/` writes one file per resource (`work/.yaml`, +`prompts/.yaml`, `repos/…`, `mcp-servers/…`, `skills/…`, +`connections/…`), each with the schema comment; without `-o` it prints one YAML +stream. `apply` reads files and directories, inlines `*File` fields, posts the +documents to `POST /api/config/apply` and prints one line per manifest +(`create` / `update` / `unchanged` / `adopt` / `replace` / `error`) and a +summary; exit code 1 when any manifest failed. **A CLI apply is a plain +upsert**: it manages nothing and never prunes — that is the directory's job. + +Every page about a work definition, prompt, repo, MCP server, skill or +connection also offers **Download YAML**, and Settings → Config as code +**Export workspace as YAML** (`GET /api/config/export.yaml`). + +## Permissions + +| Call | Who | +| ----------------------------------------------------------------- | ---------------------------------------------- | +| `GET /api/config/schema.json` | public | +| `GET /api/config/status`, `GET /api/config/export[.yaml]` | any member (the organization's resources only) | +| `POST /api/config/apply`, `…/source/sync`, `…/objects/:id/detach` | workspace admin | + +## Code + +`apps/api/src/services/config/`: `apply.ts` (the engine), `kinds/*.ts` (one +handler per kind: desire / find / diff / create / update / remove / export), +`context.ts` (name resolution), `files.ts` (reading a directory), `source.ts` +(the env-declared directory, its row and sync), `managed.ts` (`withManagedBy`), +`export.ts`. Schemas in `schemas/config.ts`; routes in `routes/config.ts`; the +worker in `workers/config-sync-worker.ts`. Shared types and the `*File` +inlining in `packages/shared/src/config/`. CLI commands in +`apps/cli/src/commands/{apply,export,schema}.ts`. diff --git a/docs/plans/config-as-code.md b/docs/plans/config-as-code.md new file mode 100644 index 000000000..bb72924f5 --- /dev/null +++ b/docs/plans/config-as-code.md @@ -0,0 +1,268 @@ +# Config as code: manifests, sources, and the CLI + +Teams that run their infrastructure from a repository want Optio to work the +same way: the Jobs, scheduled Tasks, agents, prompts, repos, MCP servers and +skills a workspace runs are files in a directory, reviewed in pull requests, +and a cluster loads them by itself. Today every one of those lives only in the +database, is created by hand in the UI, and has no export. + +This plan adds **manifests** (one YAML document per resource), **config +sources** (where a cluster reads them from), an **apply** that makes the +workspace match the files, and the CLI and UI around them. + +**Status (2026-10-04):** implemented — see [config-as-code.md](../config-as-code.md) +for the shipped behavior; this document keeps the reasoning and the decisions. + +## Vocabulary + +| Word | Meaning | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Manifest** | One YAML document describing one resource: `apiVersion: optio/v1`, `kind`, `metadata.name`, `spec`. A file may hold several, separated by `---`. | +| **Config source** | Where a workspace's manifests come from: in this version a **directory** in the API pod, mounted by the chart. | +| **Managed** | A resource a source created or adopted. It shows **Managed by _source_ · _path_**; the file is the truth and the UI is read-only for it. | +| **Apply** | Make the resources match a set of manifests: create, update, leave unchanged, prune what the source no longer declares, and report errors per manifest. | +| **Detach** | Stop managing a resource (admin). It stays, as an ordinary one you edit in the UI. | + +## Starting point (verified in code) + +- `WorkSpec` (`packages/shared/src/work/spec.ts`), the body of `POST /api/work`, + already describes every kind of work by its five attributes, and the server + derives the kind. It is the right skeleton for a `Work` manifest. +- Identity is uneven. Work definitions are unique on (kind, workspace, name), + persistent agents on (workspace, slug), prompts on (workspace, owner, name), + repos on (url, workspace). **MCP servers, skills and connections have no + unique name**; the apply has to enforce one for what it manages. +- `WorkSettings` names connections, MCP servers and skills by **UUID**, which + means nothing in another cluster. Manifests name them by **name**; the apply + resolves names in the workspace. +- `PATCH /api/work/:id` only saves `work_definitions`: a persistent agent can't + be saved from a spec today (`getOwnDefinition`). The apply needs that path. +- Some definition columns aren't in `WorkSpec` (`enabled`, `maxTurns`, + `budgetUsd`, `maxPodInstances`, `maxAgentsPerPod`, `paramsSchema`). An export + that dropped them would lose configuration, so the manifest carries them. +- The API container's root filesystem is read-only: `/tmp` and the skills + cache are the writable paths, so a mounted configuration directory is read, + never written. +- The chart has no `extraVolumes` / ConfigMap hooks; the only pattern to copy + is the installed-skills cache PVC. +- The CLI is commander-based with a JSON `ApiClient` (`get/post/patch/delete`), + stores its PAT and workspace id in `~/.config/optio/`, and has no `export` / + `apply`. No package depends on a YAML library yet (`yaml` 2.9 is a pnpm override). +- No resource has a "managed by" notion. `agent_pods.managed_by` exists and + means something else; the name is avoided. + +## The manifest format + +```yaml +# yaml-language-server: $schema=https://optio.example.com/api/config/schema.json +apiVersion: optio/v1 +kind: Work +metadata: + name: nightly-dependency-bump # the identity: unique per workspace and kind + description: Bumps dependencies every weeknight and opens a PR +spec: + when: + schedule: "0 3 * * 1-5" # or webhook / ticket / github / slack / linear; absent = on demand + where: + repo: https://github.com/acme/api + branch: main + who: + runtime: claude-code + options: { model: claude-sonnet-4-5, effort: high } + what: + promptFile: ./prompts/dependency-bump.md # or prompt: | ... + runTitle: "Dependency bump {{date}}" + then: until-merged # exits | until-merged | waits-for-messages + mergeWhenReady: true + retries: 3 + priority: 100 + secrets: [GITHUB_TOKEN, NPM_TOKEN] # pod secrets, by name — never values + environment: # WorkSettings, by name + connections: { add: [Linear] } + mcpServers: { add: [internal-docs] } + skills: { add: [release-notes] } + setupCommands: pnpm install + review: { enabled: true, trigger: on_pr } + cautiousMode: true + maxAutoResumes: 3 + params: { ... } # paramsSchema for triggered work + limits: { maxTurns: 40, budgetUsd: "5" } + pods: { maxPodInstances: 1, maxAgentsPerPod: 2 } + enabled: true +``` + +Rules: + +- **`when`** has one key, the trigger type, holding that trigger's config in the + shape the API stores (`schedule` takes the cron string as a shorthand; `webhook` + takes `{ path }`; `github` / `slack` / `linear` / `ticket` take their config + objects). A webhook's signing secret is not in the file; it is set once through + the trigger API and kept across applies (as `updateWork` keeps it today). +- **`then: waits-for-messages`** makes a persistent agent, with + `agent: { slug, systemPrompt | systemPromptFile, agentsMd | agentsMdFile, podLifecycle }`. +- **Kinds a manifest can't be**: a one-off Task (`then: exits` with a repo and + no trigger — a run, not configuration; the apply says "give it a trigger"), + a pod session, and work on a machine (bound to one person's machine). +- **References are names**: repos by URL, connections / MCP servers / skills by + name, secrets by name, agents by runtime id. An unknown name fails that + manifest, not the apply. +- **`*File` fields** (`promptFile`, `templateFile`, `systemPromptFile`, + `agentsMdFile`, `filesFrom`) are paths relative to the manifest; whoever reads + the directory inlines them (the API for a dir or git source, the CLI for a push). +- **Owner**: everything a source manages is the organization's. A manifest + cannot make private work. +- Every manifest validates against a zod schema in the API; the same schema is + served as JSON Schema at `GET /api/config/schema.json` (public, no secrets) + for editor completion and CI validation. + +The other kinds: + +| Kind | Identity | `spec` | +| ----------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `Prompt` | name | `kind` (prompt / review / job / task), `template` or `templateFile`, `params` (paramsSchema), `defaultAgentType` | +| `Repo` | `url` | `defaultBranch`, `imagePreset`, `extraPackages`, `setupCommands`, `defaultAgentType`, review (`reviewEnabled`, `reviewTrigger`, `reviewAgentType`, `reviewModel`, `testCommand`), PR behavior (`autoMerge`, `cautiousMode`, `autoResume`, `maxAutoResumes`), concurrency (`maxConcurrentTasks`, `maxPodInstances`, `maxAgentsPerPod`), pod (`cpu*`, `memory*`, `networkPolicy`, `stallThresholdMs`, `offPeakOnly`) — the fields `PATCH /api/repos/:id` accepts, minus the Slack webhook (a credential) | +| `McpServer` | name | `command`, `args`, `env` (values may be `${{SECRET_NAME}}`), `installCommand`, `repo` (scope; absent = workspace-wide), `enabled` | +| `Skill` | name | either `source: { url, ref, path }` (a marketplace skill) or `files: { "SKILL.md": ... }` / `filesFrom: ./dir` (a custom skill), plus `agentTypes`, `repo` (scope), `enabled` | + +| `Connection` | name | `provider` (slug), `config` (secret fields as `${{SECRET_NAME}}`), `repo` (scope), `enabled`, `assignments: [{ repo?, agentTypes?, permission? }]` | + +Not in this plan: **Secrets** (never in files), **Model providers** +(credentials), **Workspaces**. + +## Sources + +One source per cluster in this version: a **directory** in the API pod, bound +to one workspace. + +- `OPTIO_CONFIG_DIR=/etc/optio/config` turns it on; `OPTIO_CONFIG_WORKSPACE=` + picks the workspace (default: the oldest, as the sign-in bootstrap picks); + `OPTIO_CONFIG_PRUNE` (default true) and `OPTIO_CONFIG_INTERVAL` (default 60s). +- The chart grows `configAsCode.enabled`, `configAsCode.configMap` (an existing + ConfigMap to mount) or `configAsCode.files` (inline, rendered into one), plus + generic `api.extraVolumes` / `api.extraVolumeMounts` so a sync sidecar or a + PVC can feed the directory. +- A worker reads the tree every interval and applies it; the apply is + idempotent (unchanged rows are not written) so a tick with no changes costs a + few queries. **Sync now** in Settings and `POST /api/config/source/sync` run + it at once; `?dryRun=true` previews. +- The source is mirrored into `config_sources` at boot (one row, `kind = dir`, + `origin = env`) so it has an id, a status (last sync at, hash, error, + summary) and a place in Settings, read-only. + +Tables: `config_sources` (workspace, name, kind, path, prune, enabled, origin, +last sync at / hash / error / summary) and `config_objects` (source, kind, name +→ resource kind + id, path, hash, applied_at; unique per (source, kind, name) +and per resource). + +## Apply + +`config-apply-service.ts`, one function for every source kind and the CLI: + +1. Parse every `*.yaml` / `*.yml` under the path (recursively); a parse or + schema error marks that manifest and continues. +2. Resolve references in the workspace: repo URLs → repos, names → ids, secret + names → pickable secrets, runtimes → the agent catalog. +3. Plan, per manifest: `create` / `update` / `unchanged` / `error`; `update` + also covers **drift** — a row someone edited in the UI differs from its + manifest's desired columns and is put back (reported as reverted); for a + source, `prune` for each object it manages that no manifest declares; + `adopt` for an existing unmanaged resource with the manifest's name (so + exporting a workspace, committing, and syncing takes everything over + without conflicts). +4. Execute in dependency order — Repo, McpServer, Skill, Prompt, Work — each + object in its own transaction through the existing write services + (`createWork` / `updateWork`, extended to persistent agents; the repo, prompt, + MCP and skill services). A manifest whose derived kind changed (a Job gained + a repo) is a `replace`: delete and recreate, reported as such. +5. Record the summary on the source (counts + per-manifest errors) and the + objects in `config_objects`. + +`dryRun: true` stops after step 3 and returns the plan; that is `optio config +diff` and the "Preview" button. + +## What managed means + +- Every list and detail response for the five kinds gains + `managedBy: { sourceId, sourceName, path } | null` (one decorator, + `withManagedBy`, like `withOwnerNames`; added to the strict response schemas). +- A managed resource stays editable everywhere (decision 2). The UI says, on + the row and on its pages, that the file is the truth and edits are reverted + at the next sync; the sync reports what it reverted. +- **Detach** (admin, `POST /api/config/objects/:id/detach`) drops the + bookkeeping; the resource becomes ordinary. If its manifest is still in the + source, the next sync re-adopts it (the file is the truth; remove the file too). + +## UI / UX + +- **Settings → Config as code** (new card): the directory source — path, + workspace, last sync (time, counts: created / updated / reverted / pruned / + unchanged) and the error list per file; **Sync now** and **Preview** (dry + run) for admins; **Export workspace as YAML**; the schema URL and the CLI + commands. Without `OPTIO_CONFIG_DIR` the card explains how to turn it on. +- A **Managed** chip (like the Private chip) on Work rows, prompts, repos, MCP + servers and skills, with the source and path on hover. +- Detail and edit pages of a managed resource: a banner "Managed by config · + _path_. Edits here are reverted at the next sync — change the file instead." + with **View YAML** and (admin) **Detach**. +- **Download YAML** / **Copy as YAML** on every work, prompt, repo, MCP server + and skill detail page (`GET /api/config/export?kind=&id=`), the way to start a + config directory from what exists. + +## CLI + +`optio config` is taken (the CLI's own settings), so these are top-level, the +way kubectl's are: + +``` +optio export [--kind work|prompt|repo|mcp-server|skill|connection] [--name X] [-o DIR] +optio apply -f FILE|DIR... [--dry-run] +optio diff -f FILE|DIR... # apply --dry-run +optio schema # the JSON Schema +``` + +`export` writes one file per object (`work/.yaml`, `prompts/.yaml`, +…) with `*File` fields split out for long prompts; `apply` inlines them, posts +`POST /api/config/apply { manifests: [{ path, document }], dryRun }` and prints +the plan as a table. A CLI apply is a plain upsert: it manages nothing and never +prunes. Both honor `--json`. + +## Permissions + +Apply, sync and detach: workspace **admin**. Export: member (only the +organization's resources — never anyone's private ones, never secret values, +never webhook secrets). The schema is public. + +## Testing + +Unit: parse / normalize / export round-trip for every kind, `when` mapping, +name resolution. Integration: plan + execute against Postgres — create, update, +unchanged, drift reverted, prune, adopt, replace, per-manifest errors, detach. Pipeline e2e: `POST /api/config/apply` over HTTP then +`GET /api/work` shows `managedBy`; boot with `OPTIO_CONFIG_DIR` and the objects +appear. Playwright: the Settings card, the chip, the read-only detail page. +Live: a ConfigMap-mounted directory via Helm values on the local cluster; the Job +it declares appears and runs. + +## Decisions (confirmed 2026-10-04) + +1. **Kinds**: `Work`, `Prompt`, `Repo`, `McpServer`, `Skill` **and `Connection`**. + A connection's secret fields are written as `${{SECRET_NAME}}` references; + a literal value in a secret field is refused at apply, and an export writes + the reference (`${{}}`, the name the pod falls back to) in place + of any stored literal. +2. **Managed resources stay editable.** No 409: the UI shows the Managed chip + and a banner saying edits are reverted at the next sync. The sync compares + the manifest's desired columns with the row and puts the file's version back, + reporting what it reverted. **Detach** (admin) stops that for one resource. +3. **One source kind: a directory** in the API pod (`OPTIO_CONFIG_DIR`, mounted + by the chart from a ConfigMap, a PVC, or a git-sync sidecar through + `extraVolumes`). No repository polling from the API and no push source; the + CLI's `apply` is a plain upsert that manages nothing. The tables keep a + `kind` column so other source kinds can come later. +4. **Prune by default** (`OPTIO_CONFIG_PRUNE=false` turns it off). + +## Out of scope + +Secrets as manifests; repositories polled by the API and CI push sources (the +tables are ready for them); templating / overlays (Kustomize-style); +multi-workspace manifests (a source is one workspace's); importing from other +tools. diff --git a/helm/optio/templates/NOTES.txt b/helm/optio/templates/NOTES.txt index 37ae1ac2c..071bb1437 100644 --- a/helm/optio/templates/NOTES.txt +++ b/helm/optio/templates/NOTES.txt @@ -60,6 +60,23 @@ It asks for the one-time setup token, which the API prints in its log: (or set auth.setupToken to choose it). The first person to sign in afterwards becomes the deployment admin. {{- end }} +{{- if .Values.configAsCode.enabled }} + +--- + +CONFIG AS CODE + +The API reads the manifests mounted at {{ .Values.configAsCode.mountPath }} +every {{ .Values.configAsCode.intervalMs }} ms and applies them to the +{{- if .Values.configAsCode.workspace }} "{{ .Values.configAsCode.workspace }}"{{ else }} oldest{{ end }} workspace +(prune: {{ .Values.configAsCode.prune }}). Settings → Config as code shows the +last sync; `optio export -o optio/` writes what a workspace has as manifests. +{{- if not (or .Values.configAsCode.configMap .Values.configAsCode.files) }} + +Nothing is mounted there yet: set configAsCode.files or configAsCode.configMap, +or mount a volume at that path with api.extraVolumes / api.extraVolumeMounts. +{{- end }} +{{- end }} --- diff --git a/helm/optio/templates/api-deployment.yaml b/helm/optio/templates/api-deployment.yaml index 3d9ca216c..d96d3339d 100644 --- a/helm/optio/templates/api-deployment.yaml +++ b/helm/optio/templates/api-deployment.yaml @@ -75,6 +75,14 @@ spec: mountPath: /tmp - name: installed-skills-cache mountPath: /opt/optio/skills-cache + {{- if and .Values.configAsCode.enabled (or .Values.configAsCode.configMap .Values.configAsCode.files) }} + - name: config-manifests + mountPath: {{ .Values.configAsCode.mountPath }} + readOnly: true + {{- end }} + {{- with .Values.api.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- if and .Values.postgresql.enabled .Values.postgresql.tls.enabled }} - name: pg-ca mountPath: /etc/optio @@ -206,6 +214,14 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if and .Values.configAsCode.enabled (or .Values.configAsCode.configMap .Values.configAsCode.files) }} + - name: config-manifests + configMap: + name: {{ .Values.configAsCode.configMap | default (printf "%s-config-manifests" .Release.Name) }} + {{- end }} + {{- with .Values.api.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- if and .Values.postgresql.enabled .Values.postgresql.tls.enabled }} - name: pg-ca secret: diff --git a/helm/optio/templates/config-manifests-configmap.yaml b/helm/optio/templates/config-manifests-configmap.yaml new file mode 100644 index 000000000..3baf97b1a --- /dev/null +++ b/helm/optio/templates/config-manifests-configmap.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.configAsCode.enabled .Values.configAsCode.files (not .Values.configAsCode.configMap) }} +# Config as code: the inline manifests (configAsCode.files) as the directory +# the API reads (docs/config-as-code.md). One key per file. The API re-reads +# the mount every interval, so a changed ConfigMap needs no pod roll. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-config-manifests + namespace: {{ .Values.namespace }} + labels: + {{- include "optio.labels" . | nindent 4 }} +data: + {{- range $name, $content := .Values.configAsCode.files }} + {{ $name }}: | + {{- $content | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/optio/templates/secrets.yaml b/helm/optio/templates/secrets.yaml index a1522e925..52f946a21 100644 --- a/helm/optio/templates/secrets.yaml +++ b/helm/optio/templates/secrets.yaml @@ -33,6 +33,15 @@ stringData: {{- if .Values.auth.deploymentAdmins }} OPTIO_DEPLOYMENT_ADMINS: {{ join "," .Values.auth.deploymentAdmins | quote }} {{- end }} + {{- if .Values.configAsCode.enabled }} + # Config as code: the mounted manifest directory (docs/config-as-code.md). + OPTIO_CONFIG_DIR: {{ .Values.configAsCode.mountPath | quote }} + {{- if .Values.configAsCode.workspace }} + OPTIO_CONFIG_WORKSPACE: {{ .Values.configAsCode.workspace | quote }} + {{- end }} + OPTIO_CONFIG_PRUNE: {{ .Values.configAsCode.prune | quote }} + OPTIO_CONFIG_INTERVAL: {{ .Values.configAsCode.intervalMs | quote }} + {{- end }} {{- if .Values.auth.github.clientId }} GITHUB_OAUTH_CLIENT_ID: {{ .Values.auth.github.clientId | quote }} GITHUB_OAUTH_CLIENT_SECRET: {{ .Values.auth.github.clientSecret | quote }} diff --git a/helm/optio/values.yaml b/helm/optio/values.yaml index 0a640aac2..9ab18e841 100644 --- a/helm/optio/values.yaml +++ b/helm/optio/values.yaml @@ -32,6 +32,10 @@ api: memory: 1Gi env: LOG_LEVEL: info + # Extra volumes and mounts for the API container, verbatim (a PVC or a sync + # sidecar's volume for configAsCode, a CA bundle, …). + extraVolumes: [] + extraVolumeMounts: [] # Rollout strategy for the API Deployment ({} = Kubernetes' RollingUpdate). # Upgrades whose migrations move data between tables (see CHANGELOG) are # safest with `{ type: Recreate }`: no old replica keeps serving against @@ -214,6 +218,44 @@ installedSkillsCache: accessModes: - ReadWriteOnce +# ────────────────────────────────────────────────────────────────────────────── +# Config as code (docs/config-as-code.md) +# +# A directory of YAML manifests — Jobs, scheduled Tasks, agents, prompts, +# repos, MCP servers, skills, connections — that the API reads every +# `interval` and applies to one workspace. The files are the truth: a manifest +# that changes is applied, one that disappears is pruned (unless prune=false), +# and a managed resource edited in the UI is put back at the next sync. +# +# Feed the directory one of three ways: +# files: inline manifests, rendered into a ConfigMap by this chart +# configMap: the name of a ConfigMap you maintain (each key is a file) +# api.extraVolumes / api.extraVolumeMounts: anything else (a PVC, a sync +# sidecar's volume) mounted at `mountPath` +# ────────────────────────────────────────────────────────────────────────────── +configAsCode: + enabled: false + # Where the directory is mounted in the API container (OPTIO_CONFIG_DIR). + mountPath: /etc/optio-config + # The workspace the manifests belong to, by slug. Empty = the oldest + # workspace (the organization's, as the sign-in bootstrap makes it). + workspace: "" + # Delete resources whose manifest is gone (OPTIO_CONFIG_PRUNE). + prune: true + # How often the directory is read, in milliseconds (OPTIO_CONFIG_INTERVAL). + intervalMs: 60000 + # An existing ConfigMap to mount as the directory (one key per file, e.g. + # "work-nightly.yaml"). Takes precedence over `files`. + configMap: "" + # Inline manifests, one key per file name; rendered into a ConfigMap. + # files: + # nightly-bump.yaml: | + # apiVersion: optio/v1 + # kind: Work + # metadata: { name: nightly-dependency-bump } + # spec: { ... } + files: {} + # ────────────────────────────────────────────────────────────────────────────── # PostgreSQL (built-in) # Set enabled=false and configure externalDatabase.url for managed Postgres. diff --git a/packages/shared/src/config/inline.test.ts b/packages/shared/src/config/inline.test.ts new file mode 100644 index 000000000..2971b0a9c --- /dev/null +++ b/packages/shared/src/config/inline.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import { inlineManifestFiles, uninlinedFileFields, type ManifestFileReader } from "./inline.js"; + +function reader( + files: Record, + dirs: Record> = {}, +): ManifestFileReader { + return { + async readText(path) { + if (!(path in files)) throw new Error(`no such file: ${path}`); + return files[path]; + }, + async readDir(path) { + if (!(path in dirs)) throw new Error(`no such directory: ${path}`); + return dirs[path]; + }, + }; +} + +describe("inlineManifestFiles", () => { + it("reads a Work prompt and an agent's prompts from files", async () => { + const doc = { + kind: "Work", + spec: { + what: { promptFile: "./prompt.md" }, + agent: { systemPromptFile: "sys.md", agentsMdFile: "AGENTS.md" }, + }, + }; + const out = (await inlineManifestFiles( + doc, + reader({ "./prompt.md": "Do the thing", "sys.md": "You are…", "AGENTS.md": "# Agents" }), + )) as typeof doc & { + spec: { what: { prompt: string }; agent: { systemPrompt: string; agentsMd: string } }; + }; + expect(out.spec.what).toEqual({ prompt: "Do the thing" }); + expect(out.spec.agent).toEqual({ systemPrompt: "You are…", agentsMd: "# Agents" }); + // The input is left alone. + expect(doc.spec.what).toEqual({ promptFile: "./prompt.md" }); + }); + + it("reads a Prompt's template and a Skill's prompt", async () => { + const prompt = await inlineManifestFiles( + { kind: "Prompt", spec: { templateFile: "t.md" } }, + reader({ "t.md": "Review {{pr}}" }), + ); + expect((prompt as { spec: { template: string } }).spec).toEqual({ template: "Review {{pr}}" }); + const skill = await inlineManifestFiles( + { kind: "Skill", spec: { promptFile: "SKILL.md" } }, + reader({ "SKILL.md": "# Skill" }), + ); + expect((skill as { spec: { prompt: string } }).spec).toEqual({ prompt: "# Skill" }); + }); + + it("does not read spec.promptFile for a kind that isn't Skill", async () => { + const out = await inlineManifestFiles({ kind: "Work", spec: { promptFile: "x" } }, reader({})); + expect((out as { spec: { promptFile: string } }).spec.promptFile).toBe("x"); + }); + + it("turns a Skill's filesFrom directory into its prompt and files", async () => { + const out = (await inlineManifestFiles( + { kind: "Skill", spec: { filesFrom: "./release-notes" } }, + reader( + {}, + { "./release-notes": { "SKILL.md": "# Release notes", "scripts/gen.sh": "echo hi" } }, + ), + )) as { spec: Record }; + expect(out.spec).toEqual({ + prompt: "# Release notes", + files: { "scripts/gen.sh": "echo hi" }, + layout: "skill-dir", + }); + }); + + it("refuses a field given both ways, and a directory with no SKILL.md and no prompt", async () => { + await expect( + inlineManifestFiles( + { kind: "Work", spec: { what: { prompt: "a", promptFile: "b" } } }, + reader({ b: "x" }), + ), + ).rejects.toThrow("spec.what.promptFile and spec.what.prompt are both set"); + await expect( + inlineManifestFiles( + { kind: "Skill", spec: { filesFrom: "d" } }, + reader({}, { d: { "a.txt": "" } }), + ), + ).rejects.toThrow("no SKILL.md"); + }); + + it("leaves non-objects alone", async () => { + expect(await inlineManifestFiles("nope", reader({}))).toBe("nope"); + expect(await inlineManifestFiles(null, reader({}))).toBeNull(); + }); +}); + +describe("uninlinedFileFields", () => { + it("names the file fields a document still carries", () => { + expect( + uninlinedFileFields({ + kind: "Work", + spec: { what: { promptFile: "p" }, agent: { agentsMdFile: "a" } }, + }), + ).toEqual(["spec.what.promptFile", "spec.agent.agentsMdFile"]); + expect(uninlinedFileFields({ kind: "Skill", spec: { filesFrom: "d" } })).toEqual([ + "spec.filesFrom", + ]); + expect(uninlinedFileFields({ kind: "Work", spec: { what: { prompt: "p" } } })).toEqual([]); + }); +}); diff --git a/packages/shared/src/config/inline.ts b/packages/shared/src/config/inline.ts new file mode 100644 index 000000000..0670cfee0 --- /dev/null +++ b/packages/shared/src/config/inline.ts @@ -0,0 +1,102 @@ +/** + * `*File` fields of a manifest — `promptFile`, `templateFile`, + * `systemPromptFile`, `agentsMdFile`, `filesFrom` — name files next to the + * manifest so long prompts and skill directories don't have to live inside + * YAML. Whoever reads the directory inlines them before the document is + * validated: the API for the configuration directory, the CLI for `apply`. + * Pure: the reader is passed in. + */ + +export interface ManifestFileReader { + /** A file's text, by its path relative to the manifest's directory. */ + readText(relativePath: string): Promise; + /** Every file under a directory (relative to the manifest), path → text. */ + readDir(relativePath: string): Promise>; +} + +const isRecord = (v: unknown): v is Record => + typeof v === "object" && v !== null && !Array.isArray(v); + +/** The field a `File` key inlines into. */ +export const INLINE_FIELDS: ReadonlyArray<{ + /** `spec` path to the object holding the pair. */ + at: string[]; + file: string; + into: string; +}> = [ + { at: ["spec", "what"], file: "promptFile", into: "prompt" }, + { at: ["spec", "agent"], file: "systemPromptFile", into: "systemPrompt" }, + { at: ["spec", "agent"], file: "agentsMdFile", into: "agentsMd" }, + { at: ["spec"], file: "templateFile", into: "template" }, + { at: ["spec"], file: "promptFile", into: "prompt" }, +]; + +function dig(doc: Record, path: string[]): Record | null { + let node: unknown = doc; + for (const key of path) { + if (!isRecord(node)) return null; + node = node[key]; + } + return isRecord(node) ? node : null; +} + +/** + * The document with every `*File` field read and replaced by its target + * field (`promptFile` → `prompt`, …) and a Skill's `filesFrom` directory + * read into `files` (its `SKILL.md` becomes the prompt when none is given). + * A field given both ways is an error, like a missing file. Returns a copy; + * anything that isn't an object is returned as it is. + */ +export async function inlineManifestFiles( + document: unknown, + read: ManifestFileReader, +): Promise { + if (!isRecord(document)) return document; + const doc = structuredClone(document); + const kind = doc.kind; + for (const { at, file, into } of INLINE_FIELDS) { + // `spec.promptFile` is a Skill's; `spec.what.promptFile` is Work's. + if (at.length === 1 && file === "promptFile" && kind !== "Skill") continue; + const holder = dig(doc, at); + if (!holder || typeof holder[file] !== "string") continue; + if (holder[into] !== undefined) { + throw new Error(`${[...at, file].join(".")} and ${[...at, into].join(".")} are both set`); + } + holder[into] = await read.readText(holder[file] as string); + delete holder[file]; + } + if (kind === "Skill") { + const spec = dig(doc, ["spec"]); + if (spec && typeof spec.filesFrom === "string") { + if (spec.files !== undefined) throw new Error("spec.filesFrom and spec.files are both set"); + const files = await read.readDir(spec.filesFrom); + const skillMd = Object.keys(files).find((p) => p.toLowerCase() === "skill.md"); + if (skillMd) { + if (spec.prompt === undefined) spec.prompt = files[skillMd]; + delete files[skillMd]; + } + if (spec.prompt === undefined) { + throw new Error(`spec.filesFrom: ${spec.filesFrom} has no SKILL.md and no prompt is set`); + } + spec.files = files; + if (spec.layout === undefined) spec.layout = "skill-dir"; + delete spec.filesFrom; + } + } + return doc; +} + +/** The `*File` fields a document still carries (ones the reader didn't inline). */ +export function uninlinedFileFields(document: unknown): string[] { + if (!isRecord(document)) return []; + const left: string[] = []; + for (const { at, file } of INLINE_FIELDS) { + if (at.length === 1 && file === "promptFile" && document.kind !== "Skill") continue; + const holder = dig(document, at); + if (holder && holder[file] !== undefined) left.push([...at, file].join(".")); + } + const spec = dig(document, ["spec"]); + if (document.kind === "Skill" && spec && spec.filesFrom !== undefined) + left.push("spec.filesFrom"); + return left; +} diff --git a/packages/shared/src/config/manifest.ts b/packages/shared/src/config/manifest.ts new file mode 100644 index 000000000..44a4b82bc --- /dev/null +++ b/packages/shared/src/config/manifest.ts @@ -0,0 +1,313 @@ +/** + * Config as code — the manifest format (`docs/config-as-code.md`): one YAML + * document per resource, `apiVersion: optio/v1`, a `kind`, a `metadata.name` + * that is its identity, and a `spec` in the kind's shape. The API validates + * them (schemas/config.ts, the same shape served as JSON Schema), the CLI + * reads and writes them, and the apply result below is what both report. + * Like `work/`, this lives outside `types/`: it is not a mobile model. + */ +import type { WorkReviewTrigger } from "../work/settings.js"; + +export const MANIFEST_API_VERSION = "optio/v1"; + +/** A secret reference as a connection's or an MCP server's config carries it: `${{NAME}}`. */ +export const SECRET_REFERENCE = /^\$\{\{\s*([A-Za-z_][A-Za-z0-9_]{0,127})\s*\}\}$/; + +export const MANIFEST_KINDS = [ + "Work", + "Prompt", + "Repo", + "McpServer", + "Skill", + "Connection", +] as const; +export type ManifestKind = (typeof MANIFEST_KINDS)[number]; + +export const isManifestKind = (kind: string): kind is ManifestKind => + (MANIFEST_KINDS as readonly string[]).includes(kind); + +/** The directory an export files each kind under, and the order kinds apply in. */ +export const MANIFEST_KIND_DIRS: Record = { + Repo: "repos", + McpServer: "mcp-servers", + Skill: "skills", + Connection: "connections", + Prompt: "prompts", + Work: "work", +}; + +/** Kinds in the order an apply writes them: what Work refers to comes first. */ +export const MANIFEST_APPLY_ORDER: readonly ManifestKind[] = [ + "Repo", + "McpServer", + "Skill", + "Connection", + "Prompt", + "Work", +]; + +export interface ManifestMetadata { + /** The identity: unique per workspace and kind. */ + name: string; + description?: string | null; +} + +interface ManifestOf { + apiVersion: typeof MANIFEST_API_VERSION; + kind: K; + metadata: ManifestMetadata; + spec: S; +} + +// ── Work ──────────────────────────────────────────────────────────────────── + +/** Names added to a default set, and names taken out of it. */ +export interface NameOverrides { + add?: string[]; + remove?: string[]; +} + +/** + * What starts the work: one key, the trigger type, holding that trigger's + * config in the shape the API stores. Absent = on demand. + */ +export type WorkWhenManifest = + | { schedule: string | { cron: string } } + | { webhook: { path: string } } + | { ticket: { source: string; labels?: string[] } } + | { github: Record } + | { slack: Record } + | { linear: Record }; + +export interface WorkManifestSpec { + when?: WorkWhenManifest; + /** The repo it works in (pod work only; a manifest can't describe work on a machine). */ + where?: { repo?: string | null; branch?: string | null }; + who: { + /** Agent runtime, or `shell` for a Job that runs its prompt as a command. */ + runtime: string; + options?: Record; + model?: string | null; + }; + what: { + prompt?: string; + /** A file next to the manifest holding the prompt; inlined before apply. */ + promptFile?: string; + runTitle?: string | null; + }; + /** Default `exits`. `waits-for-messages` makes a persistent agent. */ + then?: "exits" | "until-merged" | "waits-for-messages"; + mergeWhenReady?: boolean; + retries?: number; + priority?: number; + /** Pod secrets, by name — never values. */ + secrets?: string[]; + /** `WorkSettings`, with connections, MCP servers and skills by name. */ + environment?: { + connections?: NameOverrides; + mcpServers?: NameOverrides; + skills?: NameOverrides; + setupCommands?: string | null; + review?: { enabled: boolean; trigger?: WorkReviewTrigger } | null; + cautiousMode?: boolean | null; + maxAutoResumes?: number | null; + }; + /** A persistent agent's identity and pod. */ + agent?: { + slug?: string; + systemPrompt?: string | null; + systemPromptFile?: string; + agentsMd?: string | null; + agentsMdFile?: string; + podLifecycle?: "always-on" | "sticky" | "on-demand"; + }; + /** JSON Schema of the `{{param}}`s triggered work takes. */ + params?: Record | null; + limits?: { maxTurns?: number | null; budgetUsd?: string | number | null }; + pods?: { maxPodInstances?: number; maxAgentsPerPod?: number }; + enabled?: boolean; +} + +// ── Prompt ────────────────────────────────────────────────────────────────── + +export interface PromptManifestSpec { + kind?: "prompt" | "review" | "job" | "task"; + template?: string; + templateFile?: string; + params?: Record | null; + defaultAgentType?: string | null; +} + +// ── Repo ──────────────────────────────────────────────────────────────────── + +/** + * The repo's URL (its identity) plus any of the settings `PATCH /api/repos/:id` + * takes (the Slack webhook, a credential, excepted). Only the settings the + * manifest names are managed; the rest keep their values. + */ +export interface RepoManifestSpec { + url: string; + defaultBranch?: string; + [setting: string]: unknown; +} + +// ── McpServer ─────────────────────────────────────────────────────────────── + +export interface McpServerManifestSpec { + command: string; + args?: string[]; + /** Values may be `${{SECRET_NAME}}` references. */ + env?: Record; + installCommand?: string | null; + /** A repo URL scopes the server to that repo; absent = the whole workspace. */ + repo?: string | null; + enabled?: boolean; +} + +// ── Skill ─────────────────────────────────────────────────────────────────── + +export interface SkillManifestSpec { + /** A custom skill: the SKILL.md body (or the command file, layout `commands`). */ + prompt?: string; + promptFile?: string; + layout?: "commands" | "skill-dir"; + /** Extra files of a `skill-dir` skill, path → content. */ + files?: Record; + /** A directory next to the manifest: SKILL.md is the prompt, the rest the files. */ + filesFrom?: string; + /** A marketplace skill instead: cloned from a git source. */ + source?: { url: string; ref?: string; path?: string }; + agentTypes?: string[]; + repo?: string | null; + enabled?: boolean; +} + +// ── Connection ────────────────────────────────────────────────────────────── + +export interface ConnectionManifestSpec { + /** The provider's slug (`github`, `slack`, `notion`, …). */ + provider: string; + /** Provider config; secret fields must be `${{SECRET_NAME}}` references. */ + config?: Record; + repo?: string | null; + enabled?: boolean; + assignments?: Array<{ + /** A repo URL, or absent for every repo. */ + repo?: string | null; + agentTypes?: string[]; + permission?: "read" | "write" | "full"; + }>; +} + +export type WorkManifest = ManifestOf<"Work", WorkManifestSpec>; +export type PromptManifest = ManifestOf<"Prompt", PromptManifestSpec>; +export type RepoManifest = ManifestOf<"Repo", RepoManifestSpec>; +export type McpServerManifest = ManifestOf<"McpServer", McpServerManifestSpec>; +export type SkillManifest = ManifestOf<"Skill", SkillManifestSpec>; +export type ConnectionManifest = ManifestOf<"Connection", ConnectionManifestSpec>; + +export type Manifest = + | WorkManifest + | PromptManifest + | RepoManifest + | McpServerManifest + | SkillManifest + | ConnectionManifest; + +// ── Applying ──────────────────────────────────────────────────────────────── + +/** One document as it reached the apply: where it came from and what it said. */ +export interface ManifestInput { + /** The file, relative to the directory applied (what errors and `managedBy` name). */ + path: string; + /** The parsed document, `*File` fields already inlined. */ + document: unknown; +} + +export const CONFIG_ACTIONS = [ + "create", + "update", + "unchanged", + "adopt", + "replace", + "prune", + "error", +] as const; +export type ConfigAction = (typeof CONFIG_ACTIONS)[number]; + +export interface ConfigPlanItem { + kind: string; + name: string; + path: string; + action: ConfigAction; + /** The resource it is or would be (absent for a new one in a dry run, and for errors). */ + resourceId?: string | null; + /** `update`: the fields that differ; `reverted` says they were changed in the UI, not the file. */ + changes?: string[]; + reverted?: boolean; + /** `error`: what is wrong; `replace`: why the row had to be recreated. */ + message?: string; +} + +export interface ConfigApplySummary { + created: number; + updated: number; + /** Updates that put back a UI edit (counted in `updated` too). */ + reverted: number; + unchanged: number; + adopted: number; + replaced: number; + pruned: number; + errors: number; +} + +export interface ConfigApplyResult { + dryRun: boolean; + /** The source applied, when one was (a CLI apply has none). */ + source?: { id: string; name: string } | null; + items: ConfigPlanItem[]; + summary: ConfigApplySummary; + /** When the apply ran (ISO-8601). */ + at: string; +} + +export const CONFIG_SOURCE_KINDS = ["dir"] as const; +export type ConfigSourceKind = (typeof CONFIG_SOURCE_KINDS)[number]; + +/** The configuration directory as Settings shows it. */ +export interface ConfigSourceView { + id: string; + name: string; + kind: ConfigSourceKind; + /** The directory in the API pod. */ + path: string; + workspaceId: string | null; + prune: boolean; + enabled: boolean; + /** `env`: declared by the deployment (read-only in Settings). */ + origin: "env" | "settings"; + intervalMs: number; + lastSyncAt: string | null; + /** A short hash of the directory's contents at the last sync. */ + lastSyncHash: string | null; + lastSyncError: string | null; + lastSync: ConfigApplyResult | null; +} + +/** `GET /api/config/status`. */ +export interface ConfigStatus { + /** `OPTIO_CONFIG_DIR` is set and points at this workspace. */ + enabled: boolean; + source: ConfigSourceView | null; + /** Where the JSON Schema of a manifest is served. */ + schemaUrl: string; +} + +/** `GET /api/config/export`: one exported resource. */ +export interface ExportedManifest { + kind: ManifestKind; + name: string; + /** Where an export files it: `work/nightly-bump.yaml`. */ + path: string; + document: Manifest; +} diff --git a/packages/shared/src/config/stable.ts b/packages/shared/src/config/stable.ts new file mode 100644 index 000000000..b8ab40e87 --- /dev/null +++ b/packages/shared/src/config/stable.ts @@ -0,0 +1,17 @@ +/** JSON with object keys sorted at every level, so equal values hash equal. */ +export function stableStringify(value: unknown): string { + return JSON.stringify(sortKeys(value)); +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys); + if (value && typeof value === "object" && !(value instanceof Date)) { + const out: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + const v = (value as Record)[key]; + if (v !== undefined) out[key] = sortKeys(v); + } + return out; + } + return value; +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index c57d3ba5b..3eafe5c5f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -48,3 +48,7 @@ export * from "./work/settings.js"; export * from "./types/model-provider.js"; export * from "./utils/pr-tool-calls.js"; export * from "./utils/terminal.js"; +export * from "./types/config.js"; +export * from "./config/manifest.js"; +export * from "./config/inline.js"; +export * from "./config/stable.js"; diff --git a/packages/shared/src/types/config.ts b/packages/shared/src/types/config.ts new file mode 100644 index 000000000..e07df5761 --- /dev/null +++ b/packages/shared/src/types/config.ts @@ -0,0 +1,16 @@ +/** + * Config as code: what a resource that a configuration directory manages + * carries in every list and detail response (`docs/config-as-code.md`). + * The file is the truth: edits made in the UI are put back at the next sync. + */ +export interface ManagedBy { + /** The `config_objects` row — `POST /api/config/objects/:id/detach` takes it. */ + objectId: string; + sourceId: string; + /** The source's name as Settings shows it ("config directory"). */ + sourceName: string; + /** The manifest's file, relative to the source's directory. */ + path: string; + /** The manifest kind: Work, Prompt, Repo, McpServer, Skill, Connection. */ + kind: string; +} diff --git a/packages/shared/src/types/connection.ts b/packages/shared/src/types/connection.ts index 85fa36673..7fc44944c 100644 --- a/packages/shared/src/types/connection.ts +++ b/packages/shared/src/types/connection.ts @@ -1,5 +1,6 @@ // ── Connection Provider (catalog entry) ──────────────────────────────────── +import type { ManagedBy } from "./config.js"; export interface ConnectionProviderMcpConfig { command: string; args: string[]; @@ -59,6 +60,8 @@ export interface Connection { ownerUserId?: string | null; /** Display name of `ownerUserId`, for a private connection (lists only). */ ownerName?: string | null; + /** Set when a configuration directory manages it (the file is the truth). */ + managedBy?: ManagedBy | null; enabled: boolean; status: ConnectionStatus; statusMessage?: string | null; diff --git a/packages/shared/src/types/mcp.ts b/packages/shared/src/types/mcp.ts index 89056703f..d1dd94b5a 100644 --- a/packages/shared/src/types/mcp.ts +++ b/packages/shared/src/types/mcp.ts @@ -1,4 +1,5 @@ import type { ResourceOwner } from "./model-provider.js"; +import type { ManagedBy } from "./config.js"; export interface McpServerConfig { id: string; @@ -17,6 +18,8 @@ export interface McpServerConfig { */ ownerUserId?: string | null; ownerName?: string | null; + /** Set when a configuration directory manages it (the file is the truth). */ + managedBy?: ManagedBy | null; enabled: boolean; createdAt: Date; updatedAt: Date; @@ -72,6 +75,8 @@ export interface CustomSkillConfig { */ ownerUserId?: string | null; ownerName?: string | null; + /** Set when a configuration directory manages it (the file is the truth). */ + managedBy?: ManagedBy | null; layout: CustomSkillLayout; /** Extra files for skill-dir layout. Null/empty = none. */ files?: CustomSkillFile[] | null; @@ -139,6 +144,8 @@ export interface InstalledSkillConfig { */ ownerUserId?: string | null; ownerName?: string | null; + /** Set when a configuration directory manages it (the file is the truth). */ + managedBy?: ManagedBy | null; agentTypes?: string[] | null; enabled: boolean; lastSyncedAt?: Date | null; diff --git a/packages/shared/src/types/persistent-agent.ts b/packages/shared/src/types/persistent-agent.ts index 201d38dfd..d42664f03 100644 --- a/packages/shared/src/types/persistent-agent.ts +++ b/packages/shared/src/types/persistent-agent.ts @@ -2,6 +2,7 @@ // // Long-lived, named, message-driven agent processes. See docs/persistent-agents.md. +import type { ManagedBy } from "./config.js"; export enum PersistentAgentState { IDLE = "idle", QUEUED = "queued", @@ -81,6 +82,8 @@ export interface PersistentAgent { * connections, and only they can change it. */ ownerUserId?: string | null; + /** Set when a configuration directory manages it (the file is the truth). */ + managedBy?: ManagedBy | null; /** * The secrets (by name) the agent gets in its pod. Null = the workspace's * legacy behavior (see `Workspace.restrictPodSecrets`). diff --git a/packages/shared/src/work/feed.ts b/packages/shared/src/work/feed.ts index d51a49a47..157bc0f51 100644 --- a/packages/shared/src/work/feed.ts +++ b/packages/shared/src/work/feed.ts @@ -10,6 +10,7 @@ */ /** What happens when a turn ends — the form's **Then**. */ +import type { ManagedBy } from "../types/config.js"; export const WORK_THENS = ["exits", "until-merged", "waits-for-me", "waits-for-messages"] as const; export type WorkThen = (typeof WORK_THENS)[number]; @@ -110,6 +111,8 @@ export interface WorkRow { */ ownerUserId?: string | null; ownerName?: string | null; + /** Set when a configuration directory manages it (the file is the truth). */ + managedBy?: ManagedBy | null; } export interface WorkCounts { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 507eb6019..a68601ab8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -173,9 +173,15 @@ importers: web-push: specifier: ^3.6.7 version: 3.6.7 + yaml: + specifier: 2.9.0 + version: 2.9.0 zod: specifier: ^3.25.17 version: 3.25.76 + zod-to-json-schema: + specifier: 3.25.2 + version: 3.25.2(zod@3.25.76) devDependencies: '@redocly/cli': specifier: ^2.32.2 @@ -234,6 +240,9 @@ importers: ws: specifier: 8.21.0 version: 8.21.0 + yaml: + specifier: 2.9.0 + version: 2.9.0 devDependencies: '@types/node': specifier: ^22.15.0 From 6a8d506265e257c4acd7728440f55002f6f02f94 Mon Sep 17 00:00:00 2001 From: Jon Wiggins Date: Sun, 4 Oct 2026 01:27:21 -0600 Subject: [PATCH 2/3] fix(config): follow symlinks in the mounted directory, default chart values, one spec locator - The directory walker follows symlinks: kubelet lays a ConfigMap out as links into a ..data snapshot, so Dirent.isFile() alone saw an empty directory (the live check synced nothing). Covered in files.test.ts. - configAsCode.mountPath and intervalMs fall back to the chart defaults in the templates: an upgrade with --reuse-values doesn't pick up new defaults and rendered an empty mountPath. - The Playwright Settings assertion picks the first of the two count labels. --- apps/api/src/services/config/files.test.ts | 13 ++++++++++ apps/api/src/services/config/files.ts | 29 +++++++++++++++++++--- apps/cli/src/manifests/read.ts | 7 ++++-- apps/web/e2e/config-as-code.spec.ts | 3 ++- helm/optio/templates/api-deployment.yaml | 2 +- helm/optio/templates/secrets.yaml | 5 ++-- 6 files changed, 49 insertions(+), 10 deletions(-) diff --git a/apps/api/src/services/config/files.test.ts b/apps/api/src/services/config/files.test.ts index 8b55035b9..c9d2efc15 100644 --- a/apps/api/src/services/config/files.test.ts +++ b/apps/api/src/services/config/files.test.ts @@ -93,6 +93,19 @@ describe("readManifestDirectory", () => { expect(read.errors[0].message).toMatch(/outside the configuration directory/); }); + it("follows symlinks, the way a ConfigMap mount lays files out", async () => { + // kubelet: / -> ..data/, ..data -> ..2026_10_04_…/ + await write( + "..2026_10_04/prompts/linked.yaml", + "kind: Prompt\nmetadata: { name: linked }\nspec: { template: t }", + ); + await fs.symlink(path.join(dir, "..2026_10_04"), path.join(dir, "..data")); + await fs.symlink(path.join(dir, "..data", "prompts"), path.join(dir, "prompts")); + const read = await readManifestDirectory(dir); + expect(read.errors).toEqual([]); + expect(read.manifests.map((m) => m.path)).toEqual(["prompts/linked.yaml"]); + }); + it("changes its hash when a file changes", async () => { await write("a.yaml", "kind: Prompt\nmetadata: { name: a }\nspec: { template: one }"); const first = (await readManifestDirectory(dir)).hash; diff --git a/apps/api/src/services/config/files.ts b/apps/api/src/services/config/files.ts index bea717763..e04d2b863 100644 --- a/apps/api/src/services/config/files.ts +++ b/apps/api/src/services/config/files.ts @@ -26,13 +26,33 @@ export interface DirectoryRead { const MAX_FILE_BYTES = 2 * 1024 * 1024; +/** + * A directory entry's kind, following symlinks: a ConfigMap mount is a + * directory of symlinks into a `..data` snapshot, so `Dirent.isFile()` alone + * would see nothing there. + */ +async function kindOf( + full: string, + entry: { isDirectory(): boolean; isFile(): boolean; isSymbolicLink(): boolean }, +) { + if (!entry.isSymbolicLink()) + return entry.isDirectory() ? "dir" : entry.isFile() ? "file" : "other"; + try { + const stat = await fs.stat(full); + return stat.isDirectory() ? "dir" : stat.isFile() ? "file" : "other"; + } catch { + return "other"; // a dangling link + } +} + async function* walk(root: string, dir: string): AsyncGenerator { const entries = await fs.readdir(dir, { withFileTypes: true }); for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { if (entry.name.startsWith(".") || entry.name === "node_modules") continue; const full = path.join(dir, entry.name); - if (entry.isDirectory()) yield* walk(root, full); - else if (entry.isFile() && /\.ya?ml$/i.test(entry.name)) yield full; + const kind = await kindOf(full, entry); + if (kind === "dir") yield* walk(root, full); + else if (kind === "file" && /\.ya?ml$/i.test(entry.name)) yield full; } } @@ -64,8 +84,9 @@ export function readerFor(root: string, file: string): ManifestFileReader { if (entry.name.startsWith(".")) continue; const full = path.join(current, entry.name); const relPath = rel ? `${rel}/${entry.name}` : entry.name; - if (entry.isDirectory()) await visit(full, relPath); - else if (entry.isFile()) { + const kind = await kindOf(full, entry); + if (kind === "dir") await visit(full, relPath); + else if (kind === "file") { const stat = await fs.stat(full); if (stat.size > MAX_FILE_BYTES) throw new Error(`${relPath} is larger than 2 MiB`); out[relPath] = await fs.readFile(full, "utf8"); diff --git a/apps/cli/src/manifests/read.ts b/apps/cli/src/manifests/read.ts index eefd43d56..d40bb6e7f 100644 --- a/apps/cli/src/manifests/read.ts +++ b/apps/cli/src/manifests/read.ts @@ -25,8 +25,11 @@ async function* walk(dir: string): AsyncGenerator { for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { if (entry.name.startsWith(".") || entry.name === "node_modules") continue; const full = path.join(dir, entry.name); - if (entry.isDirectory()) yield* walk(full); - else if (entry.isFile() && /\.ya?ml$/i.test(entry.name)) yield full; + // Symlinks count as what they point at (a checked-out directory may hold them). + const stat = entry.isSymbolicLink() ? await fs.stat(full).catch(() => null) : entry; + if (!stat) continue; + if (stat.isDirectory()) yield* walk(full); + else if (stat.isFile() && /\.ya?ml$/i.test(entry.name)) yield full; } } diff --git a/apps/web/e2e/config-as-code.spec.ts b/apps/web/e2e/config-as-code.spec.ts index aa1f3fc61..dabab42c7 100644 --- a/apps/web/e2e/config-as-code.spec.ts +++ b/apps/web/e2e/config-as-code.spec.ts @@ -35,7 +35,8 @@ test.describe("Config as code", () => { timeout: 30_000, }); await expect(card.getByText(status.source!.path, { exact: true })).toBeVisible(); - await expect(card.getByText(/^\d+ unchanged$|^\d+ created$/)).toBeVisible(); + // The summary line and the result tags both say it; one is enough. + await expect(card.getByText(/^\d+ unchanged$|^\d+ created$/).first()).toBeVisible(); await card.getByRole("button", { name: "Sync now" }).click(); await expect(page.getByText(/^Synced/)).toBeVisible({ timeout: 30_000 }); diff --git a/helm/optio/templates/api-deployment.yaml b/helm/optio/templates/api-deployment.yaml index d96d3339d..951551dfe 100644 --- a/helm/optio/templates/api-deployment.yaml +++ b/helm/optio/templates/api-deployment.yaml @@ -77,7 +77,7 @@ spec: mountPath: /opt/optio/skills-cache {{- if and .Values.configAsCode.enabled (or .Values.configAsCode.configMap .Values.configAsCode.files) }} - name: config-manifests - mountPath: {{ .Values.configAsCode.mountPath }} + mountPath: {{ .Values.configAsCode.mountPath | default "/etc/optio-config" }} readOnly: true {{- end }} {{- with .Values.api.extraVolumeMounts }} diff --git a/helm/optio/templates/secrets.yaml b/helm/optio/templates/secrets.yaml index 52f946a21..6a73a7ca1 100644 --- a/helm/optio/templates/secrets.yaml +++ b/helm/optio/templates/secrets.yaml @@ -35,12 +35,13 @@ stringData: {{- end }} {{- if .Values.configAsCode.enabled }} # Config as code: the mounted manifest directory (docs/config-as-code.md). - OPTIO_CONFIG_DIR: {{ .Values.configAsCode.mountPath | quote }} + # (`default`s: an upgrade with --reuse-values doesn't pick up new chart defaults.) + OPTIO_CONFIG_DIR: {{ .Values.configAsCode.mountPath | default "/etc/optio-config" | quote }} {{- if .Values.configAsCode.workspace }} OPTIO_CONFIG_WORKSPACE: {{ .Values.configAsCode.workspace | quote }} {{- end }} OPTIO_CONFIG_PRUNE: {{ .Values.configAsCode.prune | quote }} - OPTIO_CONFIG_INTERVAL: {{ .Values.configAsCode.intervalMs | quote }} + OPTIO_CONFIG_INTERVAL: {{ .Values.configAsCode.intervalMs | default 60000 | quote }} {{- end }} {{- if .Values.auth.github.clientId }} GITHUB_OAUTH_CLIENT_ID: {{ .Values.auth.github.clientId | quote }} From f65a3b56c772a22886c1ae56678a003b425053d0 Mon Sep 17 00:00:00 2001 From: Jon Wiggins Date: Sun, 4 Oct 2026 01:40:36 -0600 Subject: [PATCH 3/3] fix(helm): render an empty OPTIO_CONFIG_DIR when config as code is off A key dropped from a Secret's stringData lingers in its data, so disabling configAsCode left the pod with the old directory and the sync running (seen on the local cluster). An explicit empty value turns it off. --- helm/optio/templates/secrets.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/optio/templates/secrets.yaml b/helm/optio/templates/secrets.yaml index 6a73a7ca1..de0544867 100644 --- a/helm/optio/templates/secrets.yaml +++ b/helm/optio/templates/secrets.yaml @@ -42,6 +42,10 @@ stringData: {{- end }} OPTIO_CONFIG_PRUNE: {{ .Values.configAsCode.prune | quote }} OPTIO_CONFIG_INTERVAL: {{ .Values.configAsCode.intervalMs | default 60000 | quote }} + {{- else }} + # Off. An explicit empty value: a key dropped from stringData lingers in the + # Secret's data, which would keep a once-enabled directory on. + OPTIO_CONFIG_DIR: "" {{- end }} {{- if .Values.auth.github.clientId }} GITHUB_OAUTH_CLIENT_ID: {{ .Values.auth.github.clientId | quote }}