diff --git a/.agents/skills/openclaw-pixellab-avatar/SKILL.md b/.agents/skills/openclaw-pixellab-avatar/SKILL.md index 132a71e..9e8baa0 100644 --- a/.agents/skills/openclaw-pixellab-avatar/SKILL.md +++ b/.agents/skills/openclaw-pixellab-avatar/SKILL.md @@ -35,6 +35,11 @@ Ask the user: At minimum include `idle` and `thinking` (the watch auto-plays `thinking` while waiting for replies). - **Agent id** this will be wired into (default: `agent`). +- **Voice** — which ElevenLabs voice the agent should speak with. Offer + to run `--list-voices` (see Step 5b) so the user can pick; if they have + no preference, suggest `--voice auto` for a smoke-test voice they can + change later. A voice id drops an `elevenlabs` voice block into the + wired config so TTS just works the moment the gateway restarts. ### 2. Create the character (step 1 of 4) @@ -94,16 +99,44 @@ Expect ~5–10 min per emotion. The script prints `✓ complete` as each job finishes. If any emotion fails, the script exits non-zero with a summary — rerun with just the failed emotions to fill in the gaps. -### 5. Export into SpriteCore (step 4 of 4) +### 5a. (Optional) Discover ElevenLabs voices + +Before exporting, if the user wants to pin a specific voice, list what's +in their ElevenLabs library: + +```bash +node scripts/pixellab-export.mjs --list-voices +``` + +No `--uid` required — this is a read-only lookup. Each line prints +` [category] `. Copy the voice id the user +picks and pass it to the export step via `--voice-id`. + +Auth: `ELEVENLABS_API_KEY` env, `--elevenlabs-api-key-command `, +or `pass show elevenlabs/api-key`. + +### 5b. Export into SpriteCore (step 4 of 4) Once animations exist on the character, the exporter pulls the ZIP bundle, calls `/characters//animations` for canonical emotion names (`happy`, `sad`, etc. — not the verbose pixellab slugs), packs frames into a WebP -atlas, writes the manifest, and prints the config snippet ready to paste -into `openclaw.json`. +atlas, writes the manifest, generates the config block (including the +voice if a voice id was supplied), and with `--apply` patches +`openclaw.json` directly. ```bash -# Writes atlas + manifest to ~/.openclaw/assets/avatars// +# Normal path: write atlas + manifest AND patch openclaw.json in one shot. +# The exporter backs up the config before writing; restart the gateway +# afterward so it picks up the new agent entry. +node scripts/pixellab-export.mjs \ + --uid \ + --agent-id \ + --overwrite \ + --apply + +# Dry-style path: write atlas + manifest only; print the openclaw.json +# snippet for manual review/paste. Use this when you want to eyeball the +# block before wiring it in. node scripts/pixellab-export.mjs \ --uid \ --agent-id \ @@ -180,11 +213,16 @@ exporter, so any existing call site keeps working. ### 6. Wire into `openclaw.json` -Copy the config snippet the exporter prints into `openclaw.json` under -`plugins.entries["sprite-core"].config.agents.`, then restart the -gateway. Default state from the snippet is `idle` when present; otherwise -the first animation. Review before saving — sometimes you want a more -specific default. +If you ran the exporter with `--apply`, this step is already done — the +exporter wrote the agent block under +`plugins.entries["sprite-core"].config.agents.` and printed the +backup path. Skip to the restart in Step 7. + +If you ran without `--apply`, copy the config snippet the exporter printed +into `openclaw.json` under +`plugins.entries["sprite-core"].config.agents.`. Default state +from the snippet is `idle` when present; otherwise the first animation. +Review before saving — sometimes you want a more specific default. ### 7. Verify @@ -208,15 +246,22 @@ specific default. ## Open TODOs -- `--apply` flag for the exporter: patch the snippet directly into - `openclaw.json` under `plugins.entries["sprite-core"].config.agents.` - and optionally restart the gateway, so the whole flow is one command. -- Animate-then-tag pipeline: today the operator passes `--rename` so the - exporter can collapse pixellab's verbose slugs back to canonical emotion - names. Better would be for `pixellab-animate.mjs` to tag each animation - upstream with the emotion key it was invoked with, making `--rename` - unnecessary. -- `/characters//animations` currently 404s in our environment so the - exporter falls back to slug names. Confirm the pixellab API contract - (endpoint path / required auth / response shape) and update the exporter, - or remove the fetch entirely if it's permanently gone. +- Animate-then-tag pipeline: today the exporter's `DEFAULT_CANONICAL_RENAMES` + map collapses pixellab's verbose slugs back to canonical emotion names + (`idle`, `happy`, `sad`, …) when `/characters//animations` 404s. + Better would be for `pixellab-animate.mjs` to tag each animation upstream + with the emotion key it was invoked with (via a name field in the + animate-character request or a follow-up PATCH), so the mapping is + upstream and the exporter never has to guess. +- Auto-restart the gateway after `--apply`: currently the exporter prints a + restart command but intentionally does not run it (visible side effect + the operator should own). Add `--restart-gateway` as opt-in sugar for + unattended runs. +- `/characters//animations` currently 404s in our environment; the + exporter falls back to slug names plus the canonical rename map. + Confirm the pixellab API contract (endpoint path / required auth / + response shape) and update the exporter, or remove the fetch entirely + if it's permanently gone. +- Pixellab 3-concurrent-job cap: account-wide limit seen during batch + runs. Future: add a simple semaphore to the animate or batch scripts so + parallel pipelines block-and-retry instead of 429'ing out on create. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9a0d5d2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,100 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + # Fast "does the metadata parse" job that gates everything else. + metadata: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Validate root package.json + run: node -e "require('./package.json')" + - name: Validate plugin package.json + openclaw.plugin.json + run: | + node -e "require('./packages/plugin/package.json')" + node -e "require('./packages/plugin/openclaw.plugin.json')" + - name: Validate client-js package.json + run: node -e "require('./packages/client-js/package.json')" + - name: Validate schema package.json + run: node -e "require('./schema/package.json')" + - name: Validate fixture JSON + run: | + set -e + find fixtures -name '*.json' | while read f; do + node -e "JSON.parse(require('fs').readFileSync('$f', 'utf8'))" + done + - name: Verify plugin name matches install.npmSpec + working-directory: packages/plugin + run: | + node -e " + const pkg = require('./package.json'); + const spec = pkg.openclaw?.install?.npmSpec; + if (spec !== pkg.name) { + console.error('Mismatch: name=' + pkg.name + ' vs install.npmSpec=' + spec); + process.exit(1); + } + " + - name: Verify version alignment across packages + run: node scripts/check-versions.mjs + + typescript: + runs-on: ubuntu-latest + needs: metadata + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: pnpm/action-setup@v4 + with: + version: 9 + - name: pnpm install + run: pnpm install --frozen-lockfile + - name: Build schema + client-js + run: pnpm --filter ./schema --filter ./packages/client-js run --if-present build + - name: Test client-js + run: pnpm --filter ./packages/client-js test + - name: Typecheck client-js + run: pnpm --filter ./packages/client-js typecheck + # Plugin typecheck has pre-existing drift against the pinned openclaw + # version (see PR description). Skipped until that's sorted in a + # follow-up; uncomment once the plugin is re-aligned with the openclaw + # plugin-sdk it targets. + # - name: Typecheck plugin + # run: pnpm --filter ./packages/plugin typecheck + + kotlin: + runs-on: ubuntu-latest + needs: metadata + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - uses: gradle/actions/setup-gradle@v4 + - name: Gradle build + test (:core, :android) + working-directory: packages/client-kotlin + run: gradle :core:build :core:test :android:build :android:test --no-daemon + + swift: + runs-on: macos-latest + needs: metadata + steps: + - uses: actions/checkout@v4 + - uses: swift-actions/setup-swift@v2 + with: + swift-version: "5.10" + - name: swift build + working-directory: packages/client-swift + run: swift build + - name: swift test + working-directory: packages/client-swift + run: swift test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..066d9e5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,120 @@ +name: Release + +on: + push: + tags: + - 'v*' + +# A single v tag publishes all four artifacts from one commit: +# - @tyler-rng/sprite-core (npm — GitHub Packages) +# - @tyler-rng/sprite-core-client (npm — GitHub Packages) +# - ai.openclaw.spritecore:sprite-core-client(-android) (Maven — GitHub Packages) +# - SpriteCoreClient (SwiftPM consumes the git tag directly) +# +# All publish jobs depend on `verify-versions`, which fails the whole release +# if any package.json / Gradle version disagrees with the tag. + +jobs: + verify-versions: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - id: resolve + name: Resolve tag version + run: | + TAG_VERSION="${GITHUB_REF_NAME#v}" + echo "version=$TAG_VERSION" >> "$GITHUB_OUTPUT" + echo "TAG_VERSION=$TAG_VERSION" >> "$GITHUB_ENV" + - name: Enforce all packages match tag + run: node scripts/check-versions.mjs "$TAG_VERSION" + + publish-plugin: + needs: verify-versions + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + defaults: + run: + working-directory: packages/plugin + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + registry-url: 'https://npm.pkg.github.com' + scope: '@tyler-rng' + - name: Publish plugin to GitHub Packages + run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-client-js: + needs: verify-versions + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + registry-url: 'https://npm.pkg.github.com' + scope: '@tyler-rng' + - uses: pnpm/action-setup@v4 + with: + version: 9 + - name: pnpm install + run: pnpm install --frozen-lockfile + - name: Build schema + client-js + run: pnpm --filter ./schema --filter ./packages/client-js run --if-present build + - name: Publish client-js to GitHub Packages + working-directory: packages/client-js + run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-client-kotlin: + needs: verify-versions + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - uses: gradle/actions/setup-gradle@v4 + - name: Publish :core + :android to GitHub Packages + working-directory: packages/client-kotlin + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gradle \ + :core:publishMavenPublicationToGitHubPackagesRepository \ + :android:publishReleasePublicationToGitHubPackagesRepository \ + -Pversion="${{ needs.verify-versions.outputs.version }}" \ + --no-daemon + + validate-client-swift: + needs: verify-versions + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: swift-actions/setup-swift@v2 + with: + swift-version: "5.10" + - name: swift test + working-directory: packages/client-swift + run: swift test + # SwiftPM consumers pull by git tag — no publish step. This job exists + # purely to gate the tag: if swift test fails, the tag shouldn't stand. diff --git a/.gitignore b/.gitignore index cfa3bac..c9ecb51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,21 @@ node_modules/ dist/ +packages/plugin/ui-dist/ .DS_Store *.log .env .env.local coverage/ .turbo/ +*.tsbuildinfo + +# Kotlin / Gradle +packages/client-kotlin/**/build/ +packages/client-kotlin/**/.gradle/ +packages/client-kotlin/.gradle/ +packages/client-kotlin/local.properties + +# Swift / SwiftPM +packages/client-swift/.build/ +packages/client-swift/.swiftpm/ +packages/client-swift/Package.resolved diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d07e134 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,48 @@ +# Changelog + +All four packages in this repo (`@tyler-rng/sprite-core`, +`@tyler-rng/sprite-core-client`, `ai.openclaw.spritecore:sprite-core-client` +and `-android`, `SpriteCoreClient`) release together at one version. Tag +format: `v` (e.g. `v1.0.0`). + +## Unreleased + +### Added + +- Workspace layout: `packages/plugin`, `packages/client-js`, + `packages/client-kotlin`, `packages/client-swift`, plus shared `schema/` + and `fixtures/`. +- Cross-language client SDKs (TypeScript, Kotlin, Swift) implementing the + `CharacterManifest` wire protocol: `AnimationGraph` projection, coroutine / + async-iterator / `actor` sprite player, `FrameSource` adapter seam, + streaming `<<>>` / `<<>>` marker parser, asset cache. +- `schema/` package publishing TypeBox definitions as the single source of + truth for wire types (downstream Kotlin and Swift types kept in lockstep + via the `fixtures/` conformance suite). +- Release workflow publishing all four artifacts from a single `v*` tag: + - `@tyler-rng/sprite-core` → npm (GitHub Packages) + - `@tyler-rng/sprite-core-client` → npm (GitHub Packages) + - `ai.openclaw.spritecore:sprite-core-client` + `-android` → Maven + (GitHub Packages) + - `SpriteCoreClient` — consumed via git tag by SwiftPM +- `scripts/check-versions.mjs` — pre-flight gate that fails the release if + any package's declared version disagrees with the tag. + +### Changed + +- Plugin moved from repo root into `packages/plugin/`. Package name and + `install.npmSpec` unchanged — operators continue to install + `@tyler-rng/sprite-core` via the existing instructions. + +### Fixed + +- TypeScript marker parser upgraded to match Kotlin semantics (`<<>>` + play-count suffix). All three language ports now produce identical parse + results for the same inputs. +- `emotions` field added to the Kotlin `CharacterManifest` data class + (previously drifted from the TypeScript wire schema). + +## 1.0.0 — 2026-04-23 + +Initial release of the plugin extracted from `openclaw-src/extensions/sprite-core/`. +See `packages/plugin/README.md` for plugin-specific documentation. diff --git a/README.md b/README.md index 7c257c1..9b0c466 100644 --- a/README.md +++ b/README.md @@ -1,390 +1,119 @@ # SpriteCore -OpenClaw plugin that owns the data plane for multi-state sprite avatars and -voice/TTS. Once enabled, SpriteCore is the single source of truth for: +Plugin + cross-language client SDKs for multi-state sprite avatars, streaming +TTS, and streaming STT on [OpenClaw](https://github.com/openclaw/openclaw). -- per-agent avatar config (atlas image + manifest) -- per-agent voice descriptor (provider + voiceId for the watch / phone) -- the prompt block that teaches the model which avatar states exist (so it - knows when to emit `<<>>`, `<<>>`, etc., optionally with a - `-N` play-count suffix like `<<>>` or `<<>>`) -- HTTP asset serving (`/openclaw-assets/*`) -- streaming TTS proxy (`/stream/tts`) -- streaming STT proxy (`/stream/stt`) -- the gateway RPC `node.getCharacterManifest` that ships the watch a - ready-to-render manifest +This repo holds four publishable artifacts, sharing one version and one +wire-schema source of truth so the server plugin and every client language +stay in lockstep: -The agent's `identity.avatar` field in `openclaw.json` stays narrow: a -workspace-relative image path, an http(s) URL, a data URI, or a short string / -emoji. Anything richer (atlas, multiple states, prompting vocabulary, voice -selection) lives in this plugin's config block. +| Package | Language | Artifact | Purpose | +|---|---|---|---| +| [`packages/plugin`](./packages/plugin) | TypeScript (Node) | `@tyler-rng/sprite-core` (npm) | OpenClaw gateway plugin — asset serving, TTS/STT proxy, prompt block, `node.getCharacterManifest` RPC | +| [`packages/client-js`](./packages/client-js) | TypeScript | `@tyler-rng/sprite-core-client` (npm) | Browser / Node reference implementation of the render engine | +| [`packages/client-kotlin`](./packages/client-kotlin) | Kotlin (JVM + Android) | `ai.openclaw.spritecore:sprite-core-client` (Maven) | Kotlin kit for wearables, phones, desktop | +| [`packages/client-swift`](./packages/client-swift) | Swift | `SpriteCoreClient` (SwiftPM) | iOS / macOS / tvOS / watchOS kit | -## Enable +The canonical wire schema lives in [`schema/`](./schema) (TypeBox). Kotlin and +Swift type files are generated from it so they cannot drift. Runtime behaviour +is locked down by the shared [`fixtures/`](./fixtures) suite — every language's +test harness replays the same JSON cases and must produce byte-identical +outputs. -```jsonc -{ - "plugins": { - "entries": { - "sprite-core": { - "enabled": true, - "config": { - "assets": { - "enabled": true, - "assetsDir": "./assets", - "publicAssets": false, - "maxAssetSizeBytes": 10485760, - "publicBaseUrl": "https://..ts.net", - }, - "streamTts": { - "enabled": true, - "provider": "elevenlabs", - "apiKey": { "source": "env", "id": "ELEVENLABS_API_KEY" }, - "defaultModel": "eleven_turbo_v2", - }, - "agents": { - "agent": { - "avatar": { - "kind": "atlas", - "default": "idle", - "manifest": "avatars/agent/agent.atlas.json", - }, - "voice": { - "provider": "elevenlabs", - "voiceId": "", - "label": "default", - }, - "prompting": { - "descriptions": { - "idle": "calm / listening", - "thinking": "processing the user's request", - "happy": "warm / pleased", - "sad": "sympathy / disappointment", - }, - }, - }, - }, - }, - }, - }, - }, -} -``` - -## Default `agent` template - -Ships under `template/agent/` in this repo. It declares four states -(`idle`, `thinking`, `happy`, `sad`) and includes a placeholder atlas image -(four solid-colored squares) so the runtime works the moment you enable the -plugin — no art required. - -To use the template: - -1. Copy `template/agent/` from this repo into - `~/.openclaw/assets/avatars/agent/` (or wherever your `assetsDir` resolves - to under the `avatars//` convention). -2. Paste the config block from `template/agent/README.md` into your - `openclaw.json` under `plugins.entries["sprite-core"].config.agents.agent`. -3. Restart the gateway. The watch will fetch the manifest, render the four - placeholder colors, and auto-swap to `thinking` on every send. - -Replace the placeholder image with real art whenever you have it; the manifest -schema does not need to change. See `template/agent/README.md` for the swap -procedure. - -## Config reference - -### `assets` - -Static asset serving for atlas images, frame trees, audio clips. - -| Field | Type | Notes | -| ------------------- | --------- | ------------------------------------------------------------------------------------------------- | -| `enabled` | `boolean` | Required to be `true` for the route to register. | -| `assetsDir` | `string` | Path the route serves from. Relative paths resolve under `~/.openclaw/state`. Default `./assets`. | -| `publicAssets` | `boolean` | When `true`, `/openclaw-assets/*` skips gateway auth. Use only when intentional. | -| `maxAssetSizeBytes` | `number` | Hard cap on per-file size. Default 10 MiB. | -| `publicBaseUrl` | `string` | URL the plugin advertises to clients in `/sprite-core/agents`. Useful for Tailscale endpoints. | - -Path traversal (`..`), symlinks pointing outside `assetsDir`, and dotfiles are -rejected. ETag + 24 h `Cache-Control` are set automatically. - -### `streamTts` +## Layout -Streaming TTS proxy. Today only ElevenLabs is wired. - -| Field | Type | Notes | -| -------------- | -------------- | -------------------------------------------------------------------------------------------------- | -| `enabled` | `boolean` | Required to be `true` for the route to register. | -| `provider` | `"elevenlabs"` | Only value supported today. | -| `apiKey` | `SecretInput` | Use `{ "source": "env", "id": "ELEVENLABS_API_KEY" }`. Plain strings are accepted but discouraged. | -| `defaultModel` | `string` | ElevenLabs model id. Default `eleven_turbo_v2`. Override per request via `?model=` query param. | - -> **The plugin ships without an ElevenLabs key.** You provide your own. -> Without `streamTts.enabled = true` and a valid `apiKey`, `/stream/tts` -> returns 503 and the watch falls back silently — agents still work, the -> avatar still animates, just no spoken audio. See [ElevenLabs setup](#elevenlabs-setup). -> -> For the full wire protocol of `/stream/tts` (query params, streaming MP3 -> response, how emotion directives map to ElevenLabs `voice_settings`, client -> composition examples) see [`docs/tts-integration.md`](docs/tts-integration.md). - -### `streamStt` - -Streaming STT proxy. Parallel to `streamTts` — same provider, same key, same -auth model. Clients POST raw audio; the plugin wraps it in multipart and -forwards to ElevenLabs's `/v1/speech-to-text`. - -| Field | Type | Notes | -| -------------- | -------------- | ----------------------------------------------------------------------------------------------- | -| `enabled` | `boolean` | Required to be `true` for the route to register. | -| `provider` | `"elevenlabs"` | Only value supported today. | -| `apiKey` | `SecretInput` | Same key as TTS — ElevenLabs uses one key for both. Reuse `{ "source": "env", "id": "ELEVENLABS_API_KEY" }`. | -| `defaultModel` | `string` | ElevenLabs model id. Default `scribe_v1`. Override per request via `?model=`. | -| `maxBodyBytes` | `number` | Optional plugin-level cap on inbound body size (checked against `Content-Length`). No default. | - -> For the full wire protocol of `/stream/stt` (accepted audio formats, query -> params → multipart field mapping, response JSON shape, error codes, curl -> example, phone-side press-and-hold flow) see -> [`docs/stt-integration.md`](docs/stt-integration.md). - -### `agents.` +``` +sprite-core/ +├── pnpm-workspace.yaml +├── package.json ← workspace root +├── schema/ ← TypeBox wire schema (source of truth) +├── fixtures/ ← language-agnostic conformance JSON +├── scripts/ ← (TODO) codegen TS → Kotlin / Swift +├── docs/ ← shared avatar/TTS/STT protocol docs +└── packages/ + ├── plugin/ ← OpenClaw plugin (was the root of this repo) + ├── client-js/ ← TypeScript reference renderer + ├── client-kotlin/ ← Kotlin kit (core + android modules) + └── client-swift/ ← Swift kit (SwiftPM) +``` -Per-agent rich descriptor that supersedes the legacy -`agents.list[].identity.avatar` object form and `agents.list[].voice` block. +## Quickstart -| Field | Type | Notes | -| ----------- | ----------------- | ------------------------------------------------------------------------------------------------- | -| `avatar` | `AvatarConfig` | Atlas descriptor — see below. | -| `voice` | `VoiceConfig` | `{ provider, voiceId, label, … }` — extra keys passed through to the watch. | -| `prompting` | `PromptingConfig` | Per-state descriptions used to build the model-side instruction. Optional `instruction` override. | +```bash +pnpm install +pnpm test # runs schema + TS client + plugin tests +./gradlew -p packages/client-kotlin test +swift test --package-path packages/client-swift +``` -#### `AvatarConfig` — `kind: "atlas"` (only kind currently supported) +Each package has its own README with language-specific install + usage. The +**`<<>>` / `<<>>` marker grammar** and the +**`CharacterManifest` wire shape** are defined once in `schema/` and mirrored +everywhere else. -| Field | Type | Notes | -| ---------- | --------- | ------------------------------------------------------------ | -| `kind` | `"atlas"` | Discriminator. | -| `default` | `string` | State the agent holds when idle. Conventionally `idle`. | -| `manifest` | `string` | Path to the atlas JSON manifest, resolved under `assetsDir`. | +## Versioning -The manifest itself owns frame rects, animations, and transitions — see -`docs/avatars/formats.md` for the full atlas schema. +All four packages release together at one version. A bump to `schema/` +invalidates conformance and requires a release across all four. See +[`CHANGELOG.md`](./CHANGELOG.md) *(to be added)*. -#### `VoiceConfig` +## Consuming from OpenClaw -Pass-through descriptor surfaced to the watch / phone via -`/sprite-core/agents`. Extra keys are allowed. +OpenClaw installs the plugin like any other npm-served extension: ```jsonc -"voice": { - "provider": "elevenlabs", - "voiceId": "21m00Tcm4TlvDq8ikWAM", - "label": "default" +// openclaw.json +{ + "plugins": { + "entries": { + "sprite-core": { "enabled": true, "config": { /* ... */ } } + } + } } ``` -#### `PromptingConfig` - -Drives the system-prompt block that teaches the model the avatar's emotion -vocabulary. - -| Field | Type | Notes | -| -------------- | ----------------------- | ------------------------------------------------------------------------------------------------- | -| `descriptions` | `Record` | One entry per state. Used to render `- : ` lines in the injected instruction. | -| `instruction` | `string` (optional) | Explicit override. When set, replaces the auto-generated text entirely. | +Full config schema and the dashboard UI are documented in +[`packages/plugin/README.md`](./packages/plugin/README.md). -The state names you list here must match keys in the atlas manifest's -`animations` table — that's how the watch maps a model-emitted -`<<>>` marker to the right animation. +And the apps pull in the language-appropriate client SDK: -The keyword vocabulary (state names) lives in the gateway plugin; the parsing -of `<<>>` markers from the model output stays on the gateway side -(`src/gateway/avatar-marker-parser.ts`) and the playback code stays on the -edge devices (Wear OS DisplayKit). So edge devices stay generic — any state -name in the manifest just works. +- Web / Electron / React Native → `@tyler-rng/sprite-core-client` (npm) +- Android phone + Wear OS watch → `ai.openclaw.spritecore:sprite-core-client` (Maven) +- iOS / macOS → `SpriteCoreClient` (SwiftPM) -## Routes +For live-development against OpenClaw without publishing, both Gradle +(`includeBuild`) and SwiftPM (`path:`) support local path links. -| Path | Auth | Purpose | -| ----------------------------- | --------- | ---------------------------------------------------------------------- | -| `GET /openclaw-assets/` | gateway\* | Static asset serving. \*`auth: "plugin"` when `publicAssets: true`. | -| `GET /stream/tts` | gateway | Streaming TTS proxy (ElevenLabs). | -| `POST /stream/stt` | gateway | Streaming STT proxy (ElevenLabs). | -| `GET /sprite-core/agents` | gateway | `{ agents: { : { avatar, voice } }, publicBaseUrl? }` for clients. | +## Installing a dev build into your local OpenClaw -## Gateway RPC - -| Method | Purpose | -| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `node.getCharacterManifest` | Returns `{ manifest, revision }` — a ready-to-render manifest assembled from the plugin's per-agent atlas config + on-disk atlas JSON. The watch calls this through the phone relay. | - -`node.getCharacterManifest` is registered by this plugin via -`api.registerGatewayMethod` from `index.ts`. When the -plugin is disabled, the method is unregistered and returns "method not found" -naturally — operators get a graceful degradation rather than a stale handler. - -## How `thinking` auto-plays - -The Wear OS phone-relay (`apps/android/app/src/main/java/ai/openclaw/app/wear/WearRelayService.kt`) -publishes a `state: "thinking"` cue on the `/openclaw/avatars//state` -DataClient path the moment the user sends a message. If your manifest declares -a `thinking` animation, DisplayKit swaps to it. If it doesn't, the watch -no-ops and stays on the previous state. - -Model-emitted `<<>>` markers (parsed by -`src/gateway/avatar-marker-parser.ts`) override this state mid-reply — last -write wins. - -## ElevenLabs setup - -The plugin **does not** ship with a key. Steps for an operator: - -1. Create an ElevenLabs account at . -2. Get your API key from the profile menu. -3. Export it in your shell environment (the gateway must inherit it): - ```bash - export ELEVENLABS_API_KEY="sk_..." - ``` -4. Pick a voice id from your ElevenLabs voice library. -5. Wire both into `openclaw.json` under `plugins.entries["sprite-core"].config`: - - `streamTts.apiKey = { "source": "env", "id": "ELEVENLABS_API_KEY" }` - - `agents..voice = { "provider": "elevenlabs", "voiceId": "" }` -6. Restart the gateway. - -If you don't enable `streamTts`, agents still work normally — the watch's -TTS playback path falls back silently. - -## Security - -- Asset serving rejects path traversal (`..`), symlinks pointing outside - `assetsDir`, and dotfile access. -- File size capped by `maxAssetSizeBytes`. -- `publicAssets: true` skips gateway auth — only set this when you intentionally - serve operator-chosen files to anonymous clients (e.g. avatars on a public web page). -- The ElevenLabs API key should be a `SecretRef` (env, file, keychain), never - inlined as a plain string in committed config. - -## Plugin self-containment - -Everything avatar / character-manifest now lives in this plugin: - -- `src/prompting.ts` owns `buildPromptingInstruction` + `isAtlasAvatarConfig`. -- `src/character-manifest.ts` owns `buildCharacterManifest` and the wire-shape - inlined `CharacterManifest` type. -- `index.ts` registers `node.getCharacterManifest` via - `api.registerGatewayMethod` and reads fresh plugin config per call. - -Core has no atlas-shaped types: `IdentityConfig.avatar` is narrowed back to -`string` (path / URL / data URI / emoji), `AgentAvatarAtlasConfig` and -friends are deleted, the gateway agent row no longer carries an `avatarAtlas` -block. Disable the plugin and the only thing that stops working is the -multi-state sprite avatar (the simple string avatar still resolves through -core's `resolveAgentAvatar`). - -### Open follow-ups - -- None of substance. The prompt instruction is live (wired via - `api.registerSystemPromptContribution` from `index.ts`), and per-agent - `voice` has been removed from core — the plugin is the sole owner. - -## Pixellab.ai pipeline - -The plugin ships two Node scripts. Together they cover the create → animate -→ package flow end to end (once the animate step has its own script). - -### Create a character - -```bash -node scripts/pixellab-create.mjs \ - --name "elf" \ - --description "a magical elf with pointed ears" -``` - -Queues a 4-direction character on pixellab, polls the background job, and -prints the new `character_id` plus the four rotation URLs so you can eyeball -the look before adding animations. `--json` emits just the id + rotations -for scripting. - -### Add animations - -Not yet ported. Use the pixellab.ai web UI or the animate-character script -(operator-supplied). - -### Export into SpriteCore - -The plugin ships a Node exporter that downloads a finished pixellab.ai -character bundle by UUID and writes a SpriteCore-compatible atlas + manifest -directly into `/avatars//`: +If you're testing plugin changes against a real gateway without publishing to +npm, use the helper script — it builds the UI, packs the plugin, drops it into +your OpenClaw install's `node_modules`, and restarts the daemon: ```bash -# Quick path — assumes pixellab key is in `pass` or exported as PIXELLAB_API_KEY -node scripts/pixellab-export.mjs \ - --uid +# Defaults to ~/.openclaw/app (the global `npm i -g openclaw` location) +scripts/install-into-openclaw.sh -# Explicit key command + custom output root -PIXELLAB_API_KEY="$(op read op://vault/pixellab/api-key)" \ - node scripts/pixellab-export.mjs \ - --uid \ - --assets-root ~/.openclaw/state/assets/avatars \ - --overwrite +# Or point at a different install: +scripts/install-into-openclaw.sh --install-dir /path/to/openclaw -# Dry-run the plan without touching pixellab or disk -node scripts/pixellab-export.mjs --uid --dry-run +# Faster iteration: reuse an existing ui-dist/ build +scripts/install-into-openclaw.sh --skip-build ``` -Auth resolution order: `PIXELLAB_API_KEY` env → `--api-key-command ""` -→ `pass show pixellab/api-key`. Pick whichever matches your secret store. - -Output: - -- `/avatars//.atlas.webp` — packed atlas image. -- `/avatars//.atlas.json` — manifest. -- `/avatars//frames//NN.webp` — per-state frame - tree (useful for re-packing via `pnpm avatar:pack`). - -The exporter pairs zip-folder hashes with the pixellab API's -`animation_type` field (via `GET /characters//animations`) to emit clean -canonical SpriteCore state names — `happy`, `sad`, `thinking`, `idle` — and -generates descriptions from the animation's `display_name` (or the original -emotion prompt when no display name is set). Duplicate canonical names (e.g. -two `idle` animations of different lengths) get `_2`/`_3` suffixes. If the -metadata fetch fails, it falls back to verbose slug names. - -For the end-to-end create → approve → animate → export flow, see the -`openclaw-pixellab-avatar` skill at -`.agents/skills/openclaw-pixellab-avatar/SKILL.md`. - -The `pixellab.ai` online pixel-sprite generator is a candidate art pipeline -for the template. The intent is: +The script is idempotent — every run does an atomic swap and keeps the +previous copy at `node_modules/@tyler-rng/sprite-core.prev` for rollback. -1. Operator runs a Claude Code skill (`.agents/skills/openclaw-pixellab-avatar/SKILL.md`). -2. Skill walks them through pixellab signup + API key extraction. -3. Skill prompts pixellab to generate a character + the emotions/states the - operator wants. -4. A packaging script (`scripts/avatars/pixellab-import.mjs`) downloads the - results and wires them into the SpriteCore template layout - (`avatars//.atlas.{webp,json}`). +After it finishes, enable the plugin in your `openclaw.json` (see +[packages/plugin/README.md → Enable](./packages/plugin/README.md#enable)) +and browse to `http://localhost:18789/sprite-core/ui/` — the dashboard's +HTML shell is served publicly; its API calls are same-origin and ride the +session you already have for the OpenClaw Control UI. -The skill exists as a stub. The packaging script is not yet implemented (the -upstream pixellab.ai API contract needs to be confirmed first); see -`scripts/avatars/pixellab-import.mjs` for the placeholder. +`scripts/sync-to-openclaw.sh` serves a narrower use case: it mirrors this +repo's plugin sources into a sibling `openclaw-src/extensions/sprite-core/` +checkout. Only relevant if you're developing OpenClaw core at the same time. -## Open follow-ups +## License -- **Pixellab exporter transition cleanup.** `scripts/pixellab-export.mjs` - unconditionally writes `*->thinking` / `thinking->*` transitions into - every atlas manifest, even when the `thinking` animation has no phased - `.intro` / `.outro` sub-sequences (the common case for v3-mode outputs). - Lint noise in the generated manifest; the runtime silently no-ops on the - missing phases. Only emit those transitions when the thinking animation - actually has intro/outro phases. ~10-line fix. -- **Pixellab `animate` template-mode investigation.** `scripts/pixellab-animate.mjs` - uses `mode: "v3"`, which produces `animation_type: "custom-"` names - instead of canonical `happy` / `sad` / `thinking` names. The exporter - currently papers over this with a `--rename` mapping. Pixellab's API may - expose a `template_animation_id` path (or a PATCH for `display_name`) - that would eliminate the workaround — confirm against - `https://api.pixellab.ai/v2/openapi.json` and migrate if available. -- **Authenticated end-to-end smoke against ElevenLabs.** Unit tests cover - the handler logic exhaustively, but nothing has sent real audio through - `POST /stream/stt` + real text through `GET /stream/tts` on a paired - device end-to-end recently. Worth one credit-burning pass periodically. +MIT. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..fed05eb --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,146 @@ +# Releasing + +This repo ships four artifacts from one git tag. Everything publishes to +GitHub Packages (npm + Maven) under the `Tyler-RNG/sprite-core` repository +scope, plus SwiftPM consumes the git tag directly. + +## One-time setup (done once per repo) + +No additional secrets are required. GitHub Actions automatically provides a +`GITHUB_TOKEN` with `packages: write` to the release workflow — that's the +only credential needed to publish to GitHub Packages from CI. + +For SwiftPM, there is nothing to configure — consumers pull by git tag. + +## Cutting a release + +1. Ensure `main` is green on CI. +2. Bump every `version` field to the new version (must match exactly): + - `packages/plugin/package.json` + - `packages/client-js/package.json` + - `schema/package.json` + - `packages/client-kotlin/core/build.gradle.kts` (the literal in the + `version = findProperty("version")?.toString() ?: "X.Y.Z"` fallback) + - `packages/client-kotlin/android/build.gradle.kts` (same) + - Swift has no declared version — the git tag IS the Swift version. +3. Verify locally: + ``` + node scripts/check-versions.mjs + ``` +4. Update `CHANGELOG.md`: move items from **Unreleased** into a new + `## X.Y.Z — YYYY-MM-DD` heading. +5. Commit: + ``` + git commit -am "Release vX.Y.Z" + ``` +6. Tag: + ``` + git tag vX.Y.Z + ``` +7. Push both: + ``` + git push origin main --follow-tags + ``` +8. Watch the Release workflow. The order is: + 1. `verify-versions` — fails the release if `check-versions.mjs` disagrees + with the tag. + 2. In parallel: `publish-plugin`, `publish-client-js`, + `publish-client-kotlin`, `validate-client-swift`. + + A single failure in any of those jobs stops that artifact from publishing + (the others may already have shipped; this is an intentional tradeoff — + each artifact publish is idempotent per-version, so a retry after fix is + safe). + +## Consuming the releases + +### npm packages (plugin + client-js) + +Both are scoped to `@tyler-rng` and live on GitHub Packages. A consumer +needs to point the scope at GitHub Packages in their `.npmrc`: + +``` +@tyler-rng:registry=https://npm.pkg.github.com +//npm.pkg.github.com/:_authToken=${GITHUB_READ_PACKAGES_TOKEN} +``` + +The token only needs the `read:packages` scope. See +`packages/plugin/README.md` for the private-beta install recipe (this is +already documented for the plugin). + +### Maven (Kotlin core + android) + +From a consuming Gradle project: + +```kotlin +// settings.gradle.kts +dependencyResolutionManagement { + repositories { + mavenCentral() + google() + maven { + url = uri("https://maven.pkg.github.com/Tyler-RNG/sprite-core") + credentials { + username = providers.gradleProperty("gpr.user").orNull + ?: System.getenv("GITHUB_ACTOR") + password = providers.gradleProperty("gpr.key").orNull + ?: System.getenv("GITHUB_TOKEN") + } + } + } +} + +// app/build.gradle.kts +dependencies { + implementation("ai.openclaw.spritecore:sprite-core-client:1.0.0") + implementation("ai.openclaw.spritecore:sprite-core-client-android:1.0.0") +} +``` + +Put `gpr.user` + `gpr.key` in `~/.gradle/gradle.properties` (with +`gpr.key` = a PAT with `read:packages`) so local builds can resolve +without env vars. + +### SwiftPM (Swift client) + +```swift +.package(url: "https://github.com/Tyler-RNG/sprite-core.git", from: "1.0.0") +``` + +Then depend on the `"SpriteCoreClient"` product. SwiftPM resolves by git +tag, so no auth is needed beyond the repo clone — if your consuming Xcode +project is on a machine that can already git-clone the repo, it'll work. + +## Publishing locally (rarely needed) + +Only useful for ad-hoc testing. Don't cut real releases this way — use the +workflow so every artifact goes out at the same version. + +- **npm**: `cd packages/plugin && npm publish` (requires + `NODE_AUTH_TOKEN` in env, `write:packages` scope PAT). +- **npm (client-js)**: `cd packages/client-js && pnpm build && npm publish`. +- **Gradle**: `cd packages/client-kotlin && GITHUB_ACTOR=you GITHUB_TOKEN=... gradle :core:publish :android:publish -Pversion=X.Y.Z`. +- **SwiftPM**: no local publish — just tag + push. + +## Rollback / yank + +- **npm (GitHub Packages)**: `npm unpublish @tyler-rng/sprite-core@X.Y.Z` is + allowed for 72h after publish. After that, publish a `X.Y.(Z+1)` that + supersedes it. +- **Maven (GitHub Packages)**: versions are immutable once published. Bump + and ship a new one. +- **SwiftPM (git tag)**: `git tag -d vX.Y.Z && git push --delete origin vX.Y.Z` + plus any SPM cache buster. Don't rewrite history — create a superseding + tag instead. + +## Secrets checklist + +| Secret | Where | Why | +|---|---|---| +| `GITHUB_TOKEN` | auto-provided by GitHub Actions | npm + Maven publish to GitHub Packages | +| `read:packages` PAT | operator local `~/.npmrc` | pull `@tyler-rng/*` from GitHub Packages | +| `read:packages` PAT | operator local `~/.gradle/gradle.properties` | pull `ai.openclaw.spritecore:*` from Maven GitHub Packages | +| `write:packages` PAT | *only* if publishing locally | not needed for normal CI releases | + +No extra org/team secrets need to be configured in the repo's Settings → +Secrets and variables. diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..6f1b39a --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,118 @@ +# Conformance Fixtures + +Language-agnostic test oracles. All three client SDKs (`client-js`, +`client-kotlin`, `client-swift`) load fixtures from this directory and +replay the declared inputs; byte-identical behaviour is the contract. + +When a fixture passes on all three, the runtimes are conforming. + +## Directory layout + +``` +fixtures/ +├── README.md ← this file +├── animation-graph/ ← `AnimationGraph` behaviour +│ └── wildcard-transitions.json +├── sprite-player/ ← `SpriteAnimationPlayer` behaviour +│ ├── phased-intro.json +│ ├── play-count.json +│ └── ping-pong.json +├── marker/ ← `AvatarMarkerParser` behaviour +│ ├── bare-markers.json +│ ├── play-count-markers.json +│ ├── split-across-chunks.json +│ └── invalid-shapes.json +└── manifest/ ← decoder shape checks + └── minimal-headshot.json +``` + +## Fixture kinds + +### `animation-graph/*.json` + +```jsonc +{ + "kind": "animation-graph", + "description": "wildcard precedence resolves most-specific → least-specific", + "manifest": { /* CharacterManifest */ }, + "mode": "headshot", + "cases": [ + { + "name": "exact match beats wildcard", + "resolveTransition": { "from": "idle", "to": "thinking" }, + "expected": "thinking.intro" + } + ] +} +``` + +Runners: +1. Deserialize `manifest`. +2. Build the graph via `AnimationGraph.fromManifest(manifest, mode)`. +3. For each case, run `resolveTransition(from, to)` and compare. + - Expected string → phase reference (TransitionRef.Phase) + - Expected object `{ blend, ms }` → crossfade + +### `sprite-player/*.json` + +```jsonc +{ + "kind": "sprite-player", + "description": "playing a phased state once runs intro then loop", + "manifest": { /* CharacterManifest */ }, + "mode": "headshot", + "requests": [ + { "target": "thinking", "playCount": null, "advanceMs": 500 } + ], + "expectedRefSequence": [ + "thinking.intro.00", "thinking.intro.01", "thinking.loop.00" + ] +} +``` + +Runners use a fake `Ticker` that advances virtual time and record every +`currentRef` emission. Expected sequence is compared after all requests +and their `advanceMs` intervals have been processed. + +### `marker/*.json` + +```jsonc +{ + "kind": "marker", + "description": "bare <<>> stripped and surfaced", + "cases": [ + { + "name": "single marker", + "chunks": ["hello <<>> world"], + "expectedCleanedText": "hello world", + "expectedMarkers": [{ "state": "happy", "count": null }] + }, + { + "name": "split across chunks", + "chunks": ["start <<>> end"], + "expectedCleanedText": "start end", + "expectedMarkers": [{ "state": "happy", "count": null }] + } + ] +} +``` + +Runners create a fresh parser per case, push chunks in order, call flush, +and compare concatenated cleaned text + total markers list. + +### `manifest/*.json` + +Simplest form — just a JSON object to decode + assertions about the result. +Used to pin decoder behaviour (required fields, optional defaults, union +handling for `TransitionRef`). Expected output is implementation-defined +per runner — usually "it decodes without error and fields match." + +## Adding a new fixture + +1. Drop the JSON file in the right sub-directory. +2. Add a runner assertion in each language's test suite (or, for pure + marker/manifest fixtures, rely on the shared loader). +3. `pnpm test` / `./gradlew test` / `swift test` should all pass. + +If a fixture is meant to exercise a **future** feature, mark it with +`"skip": true` at the top and a note in `description` saying why. diff --git a/fixtures/animation-graph/wildcard-transitions.json b/fixtures/animation-graph/wildcard-transitions.json new file mode 100644 index 0000000..91629b4 --- /dev/null +++ b/fixtures/animation-graph/wildcard-transitions.json @@ -0,0 +1,65 @@ +{ + "kind": "animation-graph", + "description": "Wildcard transitions resolve by specificity: exact → from-wildcard → to-wildcard → *->*.", + "manifest": { + "version": 1, + "agentId": "agent", + "modes": ["headshot"], + "stateMap": { "idle": "idle", "thinking": "thinking", "happy": "happy" }, + "content": { + "headshot": { + "animations": { + "idle": { + "sequence": { "frames": [{ "ref": "idle.00" }], "fps": 12, "loop": "infinite" } + }, + "thinking": { + "intro": { "frames": [{ "ref": "thinking.intro.00" }], "fps": 24, "loop": "once" }, + "loop": { "frames": [{ "ref": "thinking.loop.00" }], "fps": 12, "loop": "infinite" }, + "outro": { "frames": [{ "ref": "thinking.outro.00" }], "fps": 24, "loop": "once" } + }, + "happy": { + "sequence": { "frames": [{ "ref": "happy.00" }], "fps": 24, "loop": "once" } + } + }, + "transitions": { + "idle->happy": "happy", + "thinking->*": "thinking.outro", + "*->thinking": "thinking.intro", + "*->*": "idle" + } + } + }, + "assets": { + "refs": { + "idle.00": "p/idle_00", + "thinking.intro.00": "p/thinking_intro_00", + "thinking.loop.00": "p/thinking_loop_00", + "thinking.outro.00": "p/thinking_outro_00", + "happy.00": "p/happy_00" + } + } + }, + "mode": "headshot", + "cases": [ + { + "name": "exact pair match wins over all wildcards", + "resolveTransition": { "from": "idle", "to": "happy" }, + "expected": "happy" + }, + { + "name": "from-wildcard wins when no exact pair", + "resolveTransition": { "from": "thinking", "to": "happy" }, + "expected": "thinking.outro" + }, + { + "name": "to-wildcard wins when no exact or from-wildcard", + "resolveTransition": { "from": "happy", "to": "thinking" }, + "expected": "thinking.intro" + }, + { + "name": "star-star catches what nothing else does", + "resolveTransition": { "from": "happy", "to": "idle" }, + "expected": "idle" + } + ] +} diff --git a/fixtures/manifest/minimal-headshot.json b/fixtures/manifest/minimal-headshot.json new file mode 100644 index 0000000..d5fd332 --- /dev/null +++ b/fixtures/manifest/minimal-headshot.json @@ -0,0 +1,26 @@ +{ + "kind": "manifest", + "description": "Smallest valid manifest: one mode, one flat animation, one asset ref", + "manifest": { + "version": 1, + "agentId": "ginger", + "modes": ["headshot"], + "stateMap": { "idle": "idle" }, + "content": { + "headshot": { + "animations": { + "idle": { + "sequence": { + "frames": [{ "ref": "idle.00" }], + "fps": 12, + "loop": "infinite" + } + } + } + } + }, + "assets": { + "refs": { "idle.00": "atlas/idle_00.webp" } + } + } +} diff --git a/fixtures/marker/bare-markers.json b/fixtures/marker/bare-markers.json new file mode 100644 index 0000000..581d7cc --- /dev/null +++ b/fixtures/marker/bare-markers.json @@ -0,0 +1,33 @@ +{ + "kind": "marker", + "description": "Bare <<>> markers without play-count.", + "cases": [ + { + "name": "single marker in middle of text", + "chunks": ["hello <<>> world"], + "expectedCleanedText": "hello world", + "expectedMarkers": [{ "state": "happy", "count": null }] + }, + { + "name": "two markers in same chunk", + "chunks": ["<<>> a <<>> b"], + "expectedCleanedText": " a b", + "expectedMarkers": [ + { "state": "idle", "count": null }, + { "state": "thinking", "count": null } + ] + }, + { + "name": "no markers in text", + "chunks": ["just some words"], + "expectedCleanedText": "just some words", + "expectedMarkers": [] + }, + { + "name": "empty input", + "chunks": [""], + "expectedCleanedText": "", + "expectedMarkers": [] + } + ] +} diff --git a/fixtures/marker/invalid-shapes.json b/fixtures/marker/invalid-shapes.json new file mode 100644 index 0000000..0f080ad --- /dev/null +++ b/fixtures/marker/invalid-shapes.json @@ -0,0 +1,24 @@ +{ + "kind": "marker", + "description": "Invalid marker bodies emit as literal text; nothing silently vanishes.", + "cases": [ + { + "name": "body contains whitespace", + "chunks": ["keep <<>> me"], + "expectedCleanedText": "keep <<>> me", + "expectedMarkers": [] + }, + { + "name": "empty body", + "chunks": ["keep <<<>>> me"], + "expectedCleanedText": "keep <<<>>> me", + "expectedMarkers": [] + }, + { + "name": "body contains disallowed punctuation", + "chunks": ["keep <<>> me"], + "expectedCleanedText": "keep <<>> me", + "expectedMarkers": [] + } + ] +} diff --git a/fixtures/marker/play-count-markers.json b/fixtures/marker/play-count-markers.json new file mode 100644 index 0000000..9823b90 --- /dev/null +++ b/fixtures/marker/play-count-markers.json @@ -0,0 +1,36 @@ +{ + "kind": "marker", + "description": "<<>> markers carry a numeric play-count suffix.", + "cases": [ + { + "name": "explicit play-once marker", + "chunks": ["<<>> goodbye"], + "expectedCleanedText": " goodbye", + "expectedMarkers": [{ "state": "wink", "count": 1 }] + }, + { + "name": "multi-play count", + "chunks": ["hi <<>>"], + "expectedCleanedText": "hi ", + "expectedMarkers": [{ "state": "happy", "count": 3 }] + }, + { + "name": "count zero == explicit loop forever", + "chunks": ["<<>>"], + "expectedCleanedText": "", + "expectedMarkers": [{ "state": "idle", "count": 0 }] + }, + { + "name": "hyphenated state name without numeric suffix", + "chunks": ["<<>>"], + "expectedCleanedText": "", + "expectedMarkers": [{ "state": "head-cocked", "count": null }] + }, + { + "name": "hyphenated state name with numeric suffix — last dash wins", + "chunks": ["<<>>"], + "expectedCleanedText": "", + "expectedMarkers": [{ "state": "head_cocked", "count": 2 }] + } + ] +} diff --git a/fixtures/marker/split-across-chunks.json b/fixtures/marker/split-across-chunks.json new file mode 100644 index 0000000..039d494 --- /dev/null +++ b/fixtures/marker/split-across-chunks.json @@ -0,0 +1,24 @@ +{ + "kind": "marker", + "description": "The streaming parser recognises markers whose open/body/close is split across push() calls.", + "cases": [ + { + "name": "marker split between chunks", + "chunks": ["start <<>> end"], + "expectedCleanedText": "start end", + "expectedMarkers": [{ "state": "happy", "count": null }] + }, + { + "name": "partial <<< at tail of chunk buffered for next", + "chunks": ["prefix <<", ">> tail"], + "expectedCleanedText": "prefix tail", + "expectedMarkers": [{ "state": "happy", "count": null }] + }, + { + "name": "unterminated marker flushed as literal", + "chunks": ["keep me <<=2026.4.15-beta.1" - }, - "peerDependenciesMeta": { - "openclaw": { - "optional": true - } - }, - "openclaw": { - "extensions": [ - "./index.ts" - ], - "install": { - "npmSpec": "@openclaw/sprite-core", - "defaultChoice": "npm", - "minHostVersion": ">=2026.4.10" - }, - "compat": { - "pluginApi": ">=2026.4.15-beta.1" - }, - "build": { - "openclawVersion": "2026.4.15-beta.1" - } - } + "packageManager": "pnpm@9.0.0" } diff --git a/packages/client-js/README.md b/packages/client-js/README.md new file mode 100644 index 0000000..52b3ee5 --- /dev/null +++ b/packages/client-js/README.md @@ -0,0 +1,66 @@ +# @tyler-rng/sprite-core-client + +TypeScript client SDK for the SpriteCore plugin. Consumes the `CharacterManifest` +wire shape emitted by `node.getCharacterManifest` and drives a portable +animation graph + sprite player that any JS runtime (browser, Node, Electron, +React Native) can render. + +This is the reference implementation — the Kotlin and Swift kits in sibling +packages are functional mirrors of this code, validated against the shared +`fixtures/` suite at the repo root. + +## Install + +``` +npm install @tyler-rng/sprite-core-client +``` + +Published to GitHub Packages under the `@tyler-rng` scope. + +## Minimal usage + +```ts +import { + AnimationGraph, + SpriteAnimationPlayer, + InMemorySpriteSource, + createAvatarMarkerParser, +} from "@tyler-rng/sprite-core-client"; + +const envelope = await fetchCharacterManifest("my-agent"); +const graph = AnimationGraph.fromManifest(envelope.manifest, "headshot"); + +const frameSource = new InMemorySpriteSource((bytes) => + createImageBitmap(new Blob([bytes])), +); +for (const [refKey, bytes] of Object.entries(assetBytes)) { + frameSource.put(refKey, bytes); +} + +const player = new SpriteAnimationPlayer(graph); +player.currentRef.subscribe((ref) => { + if (!ref) return; + const bitmap = frameSource.frame(ref); + drawToCanvas(bitmap); +}); + +// When the model emits `<<>>`: +const parser = createAvatarMarkerParser(); +const { markers, cleanedText } = parser.push(streamedChunk); +for (const m of markers) { + player.requestState(m.state, m.count); +} +``` + +## Surface + +- `CharacterManifest` types and TypeBox schemas (re-exported from + `@tyler-rng/sprite-core-schema`) +- `AnimationGraph.fromManifest(manifest, mode)` — projection + wildcard + transition resolver +- `SpriteAnimationPlayer` — state machine, phases, play-count, ping-pong +- `FrameSource` interface + `InMemorySpriteSource` +- `AssetSource` — manifest + asset cache with revision checks +- `createAvatarMarkerParser()` / `splitByMarkers()` — streaming marker parser +- `MutableObservable` — minimal StateFlow equivalent for `currentRef` / + `currentState` diff --git a/packages/client-js/package.json b/packages/client-js/package.json new file mode 100644 index 0000000..084e926 --- /dev/null +++ b/packages/client-js/package.json @@ -0,0 +1,44 @@ +{ + "name": "@tyler-rng/sprite-core-client", + "version": "1.0.0", + "description": "TypeScript client SDK for the SpriteCore plugin — portable animation graph + sprite player for browsers, Node, and any JS runtime", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./schema": { + "types": "./dist/schema.d.ts", + "import": "./dist/schema.js" + } + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/Tyler-RNG/sprite-core.git", + "directory": "packages/client-js" + }, + "license": "MIT", + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@tyler-rng/sprite-core-schema": "workspace:*" + }, + "devDependencies": { + "typescript": "^5.6.0", + "vitest": "^2.0.0" + }, + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} diff --git a/packages/client-js/src/animation-graph.test.ts b/packages/client-js/src/animation-graph.test.ts new file mode 100644 index 0000000..be9f545 --- /dev/null +++ b/packages/client-js/src/animation-graph.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { AnimationGraph, resolveTransition } from "./animation-graph.js"; +import type { CharacterManifest } from "./schema.js"; + +const baseManifest: CharacterManifest = { + version: 1, + agentId: "agent", + modes: ["headshot"], + stateMap: { idle: "idle", thinking: "thinking" }, + content: { + headshot: { + animations: { + idle: { + sequence: { + frames: [{ ref: "idle.00" }], + fps: 12, + loop: "infinite", + }, + }, + thinking: { + intro: { + frames: [{ ref: "thinking.intro.00" }], + fps: 24, + loop: "once", + }, + loop: { + frames: [{ ref: "thinking.loop.00" }], + fps: 12, + loop: "infinite", + }, + outro: { + frames: [{ ref: "thinking.outro.00" }], + fps: 24, + loop: "once", + }, + }, + }, + transitions: { + "*->thinking": "thinking.intro", + "thinking->*": "thinking.outro", + }, + }, + }, + assets: { + refs: { + "idle.00": "atlas/idle_00.webp", + "thinking.intro.00": "atlas/thinking_intro_00.webp", + "thinking.loop.00": "atlas/thinking_loop_00.webp", + "thinking.outro.00": "atlas/thinking_outro_00.webp", + }, + }, +}; + +describe("AnimationGraph.fromManifest", () => { + it("projects a mode's content into a graph", () => { + const g = AnimationGraph.fromManifest(baseManifest, "headshot"); + expect(g.defaultState).toBe("idle"); + expect(Object.keys(g.animations).sort()).toEqual(["idle", "thinking"]); + expect(g.transitions["*->thinking"]).toBe("thinking.intro"); + }); + + it("throws when the mode is absent", () => { + expect(() => AnimationGraph.fromManifest(baseManifest, "fullbody")).toThrow( + /no content for mode/, + ); + }); + + it("resolves transitions with wildcard precedence", () => { + const g = AnimationGraph.fromManifest(baseManifest, "headshot"); + expect(g.resolveTransition("idle", "thinking")).toBe("thinking.intro"); + expect(g.resolveTransition("thinking", "idle")).toBe("thinking.outro"); + expect(g.resolveTransition("unknown", "also-unknown")).toBeNull(); + }); +}); + +describe("resolveTransition", () => { + it("parses phase refs", () => { + expect(resolveTransition("thinking.intro")).toEqual({ + animation: "thinking", + phase: "intro", + }); + expect(resolveTransition("thinking")).toEqual({ + animation: "thinking", + phase: "loop", + }); + }); +}); diff --git a/packages/client-js/src/animation-graph.ts b/packages/client-js/src/animation-graph.ts new file mode 100644 index 0000000..c33576a --- /dev/null +++ b/packages/client-js/src/animation-graph.ts @@ -0,0 +1,101 @@ +import type { + Animation, + CharacterManifest, + TransitionRef, +} from "./schema.js"; + +/** + * Resolved animation table + transition graph for a single mode of a single + * character. Both sprite and atlas manifests project into this shape so the + * player stays format-agnostic. + * + * Build via [fromManifest] to pull a mode's content out of a server-synthesized + * [CharacterManifest], or construct directly for tests. + */ +export class AnimationGraph { + constructor( + readonly defaultState: string, + readonly animations: Readonly>, + readonly transitions: Readonly>, + ) {} + + /** + * Resolve a state→state transition against the transitions table using + * wildcard pattern matching. Specificity order (most→least specific): + * + * "->" → "->*" → "*->" → "*->*" + * + * Returns null when nothing matches; the caller then swaps instantly. + */ + resolveTransition(from: string, to: string): TransitionRef | null { + const keys = [`${from}->${to}`, `${from}->*`, `*->${to}`, `*->*`]; + for (const k of keys) { + const t = this.transitions[k]; + if (t !== undefined) return t; + } + return null; + } + + /** + * Extract a single mode's animation graph from a character manifest. The + * default state is taken from [stateMap] — the first key that maps to an + * animation present in [mode]'s content — or fails if no animation is + * present. + */ + static fromManifest(manifest: CharacterManifest, mode: string): AnimationGraph { + const content = manifest.content[mode]; + if (!content) { + throw new Error( + `manifest has no content for mode '${mode}'. Available: [${Object.keys(manifest.content).join(", ")}]`, + ); + } + const defaultState = resolveDefaultState(manifest.stateMap, content.animations); + return new AnimationGraph(defaultState, content.animations, content.transitions ?? {}); + } +} + +function resolveDefaultState( + stateMap: Record, + animations: Record, +): string { + for (const [, animName] of Object.entries(stateMap)) { + if (animName in animations) return animName; + } + const first = Object.keys(animations)[0]; + if (first === undefined) { + throw new Error("manifest mode has no animations"); + } + return first; +} + +/** The three phases of a phased animation; flat animations use `loop`. */ +export type Phase = "intro" | "loop" | "outro"; + +/** + * A transition target resolved for playback: which animation + phase to play + * once before entering the target state's own loop. Used by the player when a + * phase-string `TransitionRef` fires on state change. + */ +export type ResolvedTransition = { + animation: string; + phase: Phase; +}; + +/** Parse `"thinking.intro"` into `{ animation: "thinking", phase: "intro" }`. Unqualified → loop. */ +export function resolveTransition(ref: string): ResolvedTransition { + const dot = ref.indexOf("."); + if (dot < 0) return { animation: ref, phase: "loop" }; + const phase = ref.slice(dot + 1) as Phase; + if (phase !== "intro" && phase !== "loop" && phase !== "outro") { + throw new Error(`unknown phase: ${phase}`); + } + return { animation: ref.slice(0, dot), phase }; +} + +/** + * Treat a flat animation as the `loop` phase so the player can always look up + * phases by name without special-casing flat vs phased at every site. + */ +export function effectiveLoop(anim: Animation) { + return anim.loop ?? anim.sequence; +} diff --git a/packages/client-js/src/asset-source.ts b/packages/client-js/src/asset-source.ts new file mode 100644 index 0000000..30dfb72 --- /dev/null +++ b/packages/client-js/src/asset-source.ts @@ -0,0 +1,192 @@ +import type { CharacterManifest, NodeGetCharacterManifestResult } from "./schema.js"; +import { MutableObservable, type Observable } from "./observable.js"; + +/** + * Versioned per-agent animation signal. `version` bumps on every + * `setAgentState` call so UI consumers keyed on the signal re-trigger even + * when the state name is unchanged. `count` is forwarded from the parsed + * `<<>>` marker and governs playback cadence. + */ +export type AvatarMarkerSignal = { + state: string; + count: number | null; + version: number; +}; + +export type CachedAgent = { + agentId: string; + envelope: NodeGetCharacterManifestResult; + assetBytes: Readonly>; +}; + +export type AssetSourceHooks = { + fetchManifest: (agentId: string) => Promise; + fetchAsset: (relativePath: string) => Promise; +}; + +/** + * Client-side unified fetcher + cache for per-agent CharacterManifest + * envelopes and their asset bytes. Ports the Kotlin `AgentAvatarSource`: + * + * - `characterManifests` — latest envelope per agent + * - `characterAssets` — latest asset bytes map per agent + * - `agentMarkerSignals` — monotonic versioned state signal per agent + * + * Fetch policy is explicit: callers invoke `refresh(agentIds)`. An agent + * already present at the same revision is left alone; revision bumps trigger + * a re-fetch of changed asset refs. + */ +export class AssetSource { + private readonly _characterManifests = new MutableObservable< + Readonly> + >({}); + private readonly _characterAssets = new MutableObservable< + Readonly>>> + >({}); + private readonly _agentStates = new MutableObservable>>({}); + private readonly _agentMarkerSignals = new MutableObservable< + Readonly> + >({}); + private signalVersionSeq = 0; + private inflight: Promise | null = null; + + readonly characterManifests: Observable>> = + this._characterManifests; + readonly characterAssets: Observable< + Readonly>>> + > = this._characterAssets; + readonly agentStates: Observable>> = this._agentStates; + readonly agentMarkerSignals: Observable>> = + this._agentMarkerSignals; + + constructor(private readonly hooks: AssetSourceHooks) {} + + /** + * Kick off a refresh for each agent. Returns when all fetches settle. + * No-ops for agents whose manifest is already cached at the current + * revision. + */ + async refresh(agentIds: readonly string[]): Promise { + if (agentIds.length === 0) return; + // Serialize refreshes to match the Kotlin Mutex behavior. + const prev = this.inflight ?? Promise.resolve(); + let release!: () => void; + this.inflight = new Promise((r) => { + release = r; + }); + try { + await prev; + for (const agentId of agentIds) { + await this.refreshOne(agentId); + } + } finally { + release(); + } + } + + /** + * Update the current state for an agent. Called when an `<<>>` or + * `<<>>` marker fires. + */ + setAgentState(agentId: string, stateName: string, count: number | null = null): void { + this._agentStates.set({ ...this._agentStates.value, [agentId]: stateName }); + this.signalVersionSeq += 1; + const signal: AvatarMarkerSignal = { + state: stateName, + count, + version: this.signalVersionSeq, + }; + this._agentMarkerSignals.set({ + ...this._agentMarkerSignals.value, + [agentId]: signal, + }); + } + + /** + * Snapshot of the current cache. Values are consistent per call; + * concurrent cache updates between calls are expected and safe. + */ + snapshot(): readonly CachedAgent[] { + const manifests = this._characterManifests.value; + const assets = this._characterAssets.value; + return Object.entries(manifests).map(([agentId, envelope]) => ({ + agentId, + envelope, + assetBytes: assets[agentId] ?? {}, + })); + } + + /** Drop any cached entries for agents no longer in [keepIds]. */ + retainOnly(keepIds: Iterable): void { + const keep = new Set(keepIds); + this._characterManifests.set(filterKeys(this._characterManifests.value, keep)); + this._characterAssets.set(filterKeys(this._characterAssets.value, keep)); + this._agentStates.set(filterKeys(this._agentStates.value, keep)); + } + + clear(): void { + this._characterManifests.set({}); + this._characterAssets.set({}); + this._agentStates.set({}); + } + + /** + * Resolve the default state name for [agentId] from its cached manifest. + * Mirrors `AnimationGraph.fromManifest` default-state logic so the two + * never drift. + */ + defaultStateFor(agentId: string): string | null { + const envelope = this._characterManifests.value[agentId]; + if (!envelope) return null; + return resolveDefaultStateName(envelope.manifest); + } + + // --- internals --- + + private async refreshOne(agentId: string): Promise { + const envelope = await this.hooks.fetchManifest(agentId); + if (!envelope) return; + const existing = this._characterManifests.value[agentId]; + if (existing && existing.revision === envelope.revision) return; + this._characterManifests.set({ + ...this._characterManifests.value, + [agentId]: envelope, + }); + + const refs = envelope.manifest.assets.refs; + const bytesByRef: Record = {}; + for (const [refKey, relPath] of Object.entries(refs)) { + const bytes = await this.hooks.fetchAsset(relPath); + if (bytes !== null) { + bytesByRef[refKey] = bytes; + } + } + this._characterAssets.set({ + ...this._characterAssets.value, + [agentId]: bytesByRef, + }); + } +} + +function filterKeys( + source: Readonly>, + keep: Set, +): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(source)) { + if (keep.has(k)) out[k] = v; + } + return out; +} + +function resolveDefaultStateName(manifest: CharacterManifest): string | null { + const mode = manifest.modes.find((m) => m in manifest.content); + if (!mode) return null; + const animations = manifest.content[mode]?.animations; + if (!animations) return null; + for (const [, animName] of Object.entries(manifest.stateMap)) { + if (animName in animations) return animName; + } + const first = Object.keys(animations)[0]; + return first ?? null; +} diff --git a/packages/client-js/src/fixture-runner.test.ts b/packages/client-js/src/fixture-runner.test.ts new file mode 100644 index 0000000..f725b35 --- /dev/null +++ b/packages/client-js/src/fixture-runner.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join, dirname, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + AnimationGraph, + InMemorySpriteSource, + SpriteAnimationPlayer, + createAvatarMarkerParser, +} from "./index.js"; +import type { CharacterManifest, FrameRef } from "./schema.js"; +import type { Ticker } from "./ticker.js"; + +// Walk up to the repo root (two levels above packages/client-js). +const fixturesRoot = join( + dirname(fileURLToPath(import.meta.url)), + "..", + "..", + "..", + "fixtures", +); + +type AnyFixture = + | ManifestFixture + | MarkerFixture + | AnimationGraphFixture + | SpritePlayerFixture; + +type ManifestFixture = { + kind: "manifest"; + description: string; + manifest: CharacterManifest; +}; + +type MarkerFixture = { + kind: "marker"; + description: string; + cases: { + name: string; + chunks: string[]; + expectedCleanedText: string; + expectedMarkers: { state: string; count: number | null }[]; + }[]; +}; + +type TransitionExpectation = string | { blend: string; ms: number } | null; + +type AnimationGraphFixture = { + kind: "animation-graph"; + description: string; + manifest: CharacterManifest; + mode: string; + cases: { + name: string; + resolveTransition: { from: string; to: string }; + expected: TransitionExpectation; + }[]; +}; + +type SpritePlayerFixture = { + kind: "sprite-player"; + description: string; + manifest: CharacterManifest; + mode: string; + requests: { target: string; playCount: number | null; advanceMs: number }[]; + expectedRefSequencePrefix: string[]; + expectedHoldRef?: string; +}; + +function listFixtureFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) out.push(...listFixtureFiles(full)); + else if (entry.endsWith(".json")) out.push(full); + } + return out; +} + +function loadFixture(path: string): AnyFixture { + return JSON.parse(readFileSync(path, "utf8")) as AnyFixture; +} + +/** Fake ticker that resolves each delay on the next microtask. */ +class MicrotaskTicker implements Ticker { + async delay(_ms: number): Promise { + await Promise.resolve(); + } +} + +async function flushMicrotasks(n: number): Promise { + for (let i = 0; i < n; i++) await Promise.resolve(); +} + +async function collectPlayerRefs( + fixture: SpritePlayerFixture, +): Promise<{ emitted: FrameRef[]; holdRef: FrameRef | null }> { + const graph = AnimationGraph.fromManifest(fixture.manifest, fixture.mode); + const player = new SpriteAnimationPlayer(graph, new MicrotaskTicker()); + const emitted: FrameRef[] = []; + let lastRef: FrameRef | null = null; + const unsub = player.currentRef.subscribe((ref) => { + if (ref && ref !== lastRef) { + emitted.push(ref); + lastRef = ref; + } + }); + // Let the default-state start-up run. + await flushMicrotasks(40); + for (const req of fixture.requests) { + await player.requestState(req.target, req.playCount); + await flushMicrotasks(Math.max(20, Math.floor(req.advanceMs / 8))); + } + const holdRef = player.currentRef.value; + unsub(); + await player.dispose(); + return { emitted, holdRef }; +} + +function runMarkerCase(c: MarkerFixture["cases"][number]): void { + const parser = createAvatarMarkerParser(); + let cleaned = ""; + const markers: { state: string; count: number | null }[] = []; + for (const chunk of c.chunks) { + const r = parser.push(chunk); + cleaned += r.cleanedText; + markers.push(...r.markers); + } + const tail = parser.flush(); + cleaned += tail.cleanedText; + markers.push(...tail.markers); + expect(cleaned).toBe(c.expectedCleanedText); + expect(markers).toEqual(c.expectedMarkers); +} + +function runAnimationGraphCase( + fixture: AnimationGraphFixture, + c: AnimationGraphFixture["cases"][number], +): void { + const graph = AnimationGraph.fromManifest(fixture.manifest, fixture.mode); + const actual = graph.resolveTransition(c.resolveTransition.from, c.resolveTransition.to); + if (c.expected === null) { + expect(actual).toBeNull(); + } else if (typeof c.expected === "string") { + expect(actual).toBe(c.expected); + } else { + expect(actual).toEqual(c.expected); + } +} + +describe("fixtures", () => { + const files = listFixtureFiles(fixturesRoot); + for (const file of files) { + const rel = relative(fixturesRoot, file); + const fixture = loadFixture(file); + describe(rel, () => { + if (fixture.kind === "manifest") { + it("decodes", () => { + // Round-trip through JSON.parse → schema types. The TS types are + // structural, so just asserting the fields exist is enough; the + // AJV validators live in the schema package and aren't needed + // here. + expect(fixture.manifest.version).toBeTypeOf("number"); + expect(fixture.manifest.agentId.length).toBeGreaterThan(0); + expect(fixture.manifest.modes.length).toBeGreaterThan(0); + }); + } else if (fixture.kind === "marker") { + for (const c of fixture.cases) { + it(c.name, () => { + runMarkerCase(c); + }); + } + } else if (fixture.kind === "animation-graph") { + for (const c of fixture.cases) { + it(c.name, () => { + runAnimationGraphCase(fixture, c); + }); + } + } else if (fixture.kind === "sprite-player") { + it("emits the expected prefix", async () => { + const { emitted, holdRef } = await collectPlayerRefs(fixture); + const emittedRefs = emitted.map((r) => r.ref); + expect(emittedRefs.slice(0, fixture.expectedRefSequencePrefix.length)).toEqual( + fixture.expectedRefSequencePrefix, + ); + if (fixture.expectedHoldRef !== undefined) { + expect(holdRef?.ref).toBe(fixture.expectedHoldRef); + } + }); + } else { + const anyFixture = fixture as { kind: string }; + it("unknown kind", () => { + throw new Error(`unknown fixture kind: ${anyFixture.kind}`); + }); + } + }); + } +}); diff --git a/packages/client-js/src/frame-source.ts b/packages/client-js/src/frame-source.ts new file mode 100644 index 0000000..cfbc0ea --- /dev/null +++ b/packages/client-js/src/frame-source.ts @@ -0,0 +1,47 @@ +import type { FrameRef } from "./schema.js"; + +/** + * Platform-specific resolver from a [FrameRef] to a concrete renderable + * (e.g. `HTMLImageElement`, `ImageBitmap`, an `` URL, whatever the + * caller chooses). The kit itself never constructs frames — callers own the + * pixel pipeline and only feed the player's emitted `FrameRef` into their + * own `FrameSource` when rendering. + * + * Atlas sources honor the optional `x/y/w/h` fields on `FrameRef`; sprite + * sources ignore them and treat `ref` as the whole-image key. + */ +export interface FrameSource { + frame(ref: FrameRef): F | null; +} + +/** + * Simple in-memory sprite source: callers prime a map of whole-image bytes + * keyed by the ref name, and decode happens lazily through [decode]. Useful + * for unit tests and thin clients that don't need per-platform image types. + */ +export class InMemorySpriteSource implements FrameSource { + private readonly bytesByRef = new Map(); + private readonly cache = new Map(); + + constructor(private readonly decode: (bytes: Uint8Array) => F | null) {} + + put(refKey: string, bytes: Uint8Array): void { + this.bytesByRef.set(refKey, bytes); + this.cache.delete(refKey); + } + + keys(): ReadonlySet { + return new Set(this.bytesByRef.keys()); + } + + frame(ref: FrameRef): F | null { + const cached = this.cache.get(ref.ref); + if (cached !== undefined) return cached; + const bytes = this.bytesByRef.get(ref.ref); + if (bytes === undefined) return null; + const decoded = this.decode(bytes); + if (decoded === null) return null; + this.cache.set(ref.ref, decoded); + return decoded; + } +} diff --git a/packages/client-js/src/index.ts b/packages/client-js/src/index.ts new file mode 100644 index 0000000..7954dd0 --- /dev/null +++ b/packages/client-js/src/index.ts @@ -0,0 +1,7 @@ +export * from "./schema.js"; +export * from "./frame-source.js"; +export * from "./ticker.js"; +export * from "./animation-graph.js"; +export * from "./sprite-player.js"; +export * from "./observable.js"; +export * from "./asset-source.js"; diff --git a/packages/client-js/src/marker.test.ts b/packages/client-js/src/marker.test.ts new file mode 100644 index 0000000..9b5f5dd --- /dev/null +++ b/packages/client-js/src/marker.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect } from "vitest"; +import { + createAvatarMarkerParser, + parseAvatarMarkers, + resolveStateAndCount, + splitByMarkers, +} from "./schema.js"; + +describe("resolveStateAndCount", () => { + it("parses bare state with no dash", () => { + expect(resolveStateAndCount("happy")).toEqual({ state: "happy", count: null }); + }); + it("parses state-N with numeric suffix", () => { + expect(resolveStateAndCount("happy-3")).toEqual({ state: "happy", count: 3 }); + expect(resolveStateAndCount("wink-1")).toEqual({ state: "wink", count: 1 }); + expect(resolveStateAndCount("happy-0")).toEqual({ state: "happy", count: 0 }); + }); + it("leaves non-numeric dash suffixes alone", () => { + expect(resolveStateAndCount("head-cocked")).toEqual({ + state: "head-cocked", + count: null, + }); + }); + it("triggers on the last dash only", () => { + expect(resolveStateAndCount("head_cocked-1")).toEqual({ + state: "head_cocked", + count: 1, + }); + }); +}); + +describe("createAvatarMarkerParser", () => { + it("strips a single marker and surfaces it", () => { + const p = createAvatarMarkerParser(); + const { cleanedText, markers } = p.push("hello <<>> world"); + expect(cleanedText).toBe("hello world"); + expect(markers).toEqual([{ state: "happy", count: null }]); + }); + + it("preserves invalid marker shapes as literal text", () => { + const { cleanedText, markers } = parseAvatarMarkers("bad <<>> marker"); + expect(cleanedText).toBe("bad <<>> marker"); + expect(markers).toEqual([]); + }); + + it("recognizes a marker split across two chunks", () => { + const p = createAvatarMarkerParser(); + const a = p.push("start <<>> end"); + expect(a.cleanedText + b.cleanedText).toBe("start end"); + expect([...a.markers, ...b.markers]).toEqual([{ state: "happy", count: null }]); + }); + + it("surfaces play-count markers", () => { + const { markers } = parseAvatarMarkers("say <<>> it"); + expect(markers).toEqual([{ state: "wink", count: 1 }]); + }); + + it("flushes unterminated markers as literal text", () => { + const p = createAvatarMarkerParser(); + const a = p.push("tail << { + const p = createAvatarMarkerParser(); + const a = p.push("ok <<"); + expect(a.cleanedText).toBe("ok "); + const b = p.push(">> done"); + expect(a.cleanedText + b.cleanedText).toBe("ok done"); + expect([...a.markers, ...b.markers]).toEqual([{ state: "happy", count: null }]); + }); +}); + +describe("splitByMarkers", () => { + it("splits with preceding emotion attached to each segment", () => { + const segs = splitByMarkers("hi <<>> world <<>> end"); + expect(segs).toEqual([ + { text: "hi ", emotion: null, emotionCount: null }, + { text: " world ", emotion: "happy", emotionCount: null }, + { text: " end", emotion: "sad", emotionCount: null }, + ]); + }); + + it("forwards count onto the segment", () => { + const segs = splitByMarkers("<<>> hello"); + expect(segs).toEqual([{ text: " hello", emotion: "wink", emotionCount: 2 }]); + }); +}); diff --git a/packages/client-js/src/observable.ts b/packages/client-js/src/observable.ts new file mode 100644 index 0000000..c3a8559 --- /dev/null +++ b/packages/client-js/src/observable.ts @@ -0,0 +1,40 @@ +/** + * Minimal observable value. Held as a single writable cell with a listener + * set — matches `MutableStateFlow` on the Kotlin side closely enough that + * subscribers see the current value + every subsequent update. Not a + * reactive system; exists only so the player can expose `currentRef` / + * `currentState` without pulling in rxjs. + */ +export interface Observable { + readonly value: T; + subscribe(listener: (value: T) => void): () => void; +} + +export class MutableObservable implements Observable { + private current: T; + private readonly listeners = new Set<(value: T) => void>(); + + constructor(initial: T) { + this.current = initial; + } + + get value(): T { + return this.current; + } + + set(next: T): void { + if (Object.is(this.current, next)) return; + this.current = next; + for (const listener of this.listeners) { + listener(next); + } + } + + subscribe(listener: (value: T) => void): () => void { + this.listeners.add(listener); + listener(this.current); + return () => { + this.listeners.delete(listener); + }; + } +} diff --git a/packages/client-js/src/schema.ts b/packages/client-js/src/schema.ts new file mode 100644 index 0000000..c117443 --- /dev/null +++ b/packages/client-js/src/schema.ts @@ -0,0 +1,3 @@ +// Re-export the wire schema + marker grammar from the source-of-truth package +// so downstream consumers can import everything through this client SDK. +export * from "@tyler-rng/sprite-core-schema"; diff --git a/packages/client-js/src/sprite-player.test.ts b/packages/client-js/src/sprite-player.test.ts new file mode 100644 index 0000000..949e595 --- /dev/null +++ b/packages/client-js/src/sprite-player.test.ts @@ -0,0 +1,91 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { AnimationGraph } from "./animation-graph.js"; +import { SpriteAnimationPlayer } from "./sprite-player.js"; +import type { CharacterManifest } from "./schema.js"; +import type { Ticker } from "./ticker.js"; + +const manifest: CharacterManifest = { + version: 1, + agentId: "agent", + modes: ["headshot"], + stateMap: { idle: "idle", wink: "wink" }, + content: { + headshot: { + animations: { + idle: { + sequence: { + frames: [{ ref: "idle.00" }, { ref: "idle.01" }], + fps: 60, + loop: "infinite", + }, + }, + wink: { + sequence: { + frames: [{ ref: "wink.00" }, { ref: "wink.01" }], + fps: 60, + loop: "once", + holdLastFrame: true, + }, + }, + }, + }, + }, + assets: { + refs: { + "idle.00": "p/idle_00", + "idle.01": "p/idle_01", + "wink.00": "p/wink_00", + "wink.01": "p/wink_01", + }, + }, +}; + +/** Ticker whose `delay` resolves immediately — tests just walk state. */ +class ImmediateTicker implements Ticker { + async delay(_ms: number): Promise { + // microtask yield so the player's loop advances + await Promise.resolve(); + } +} + +async function flushMicrotasks(n = 8): Promise { + for (let i = 0; i < n; i++) await Promise.resolve(); +} + +describe("SpriteAnimationPlayer", () => { + let player: SpriteAnimationPlayer | null = null; + afterEach(async () => { + if (player) await player.dispose(); + player = null; + }); + + it("starts on the default state and emits the first frame", async () => { + const graph = AnimationGraph.fromManifest(manifest, "headshot"); + player = new SpriteAnimationPlayer(graph, new ImmediateTicker()); + await flushMicrotasks(); + expect(player.currentState.value).toBe("idle"); + expect(player.currentRef.value?.ref).toMatch(/^idle\./); + }); + + it("requestState switches state", async () => { + const graph = AnimationGraph.fromManifest(manifest, "headshot"); + player = new SpriteAnimationPlayer(graph, new ImmediateTicker()); + await flushMicrotasks(); + await player.requestState("wink"); + await flushMicrotasks(); + expect(player.currentState.value).toBe("wink"); + }); + + it("requestState with playCount replays even when already in state", async () => { + const graph = AnimationGraph.fromManifest(manifest, "headshot"); + player = new SpriteAnimationPlayer(graph, new ImmediateTicker()); + await flushMicrotasks(); + await player.requestState("wink", 1); + await flushMicrotasks(); + expect(player.currentState.value).toBe("wink"); + // Second call with the same state+count should not error or hang. + await player.requestState("wink", 1); + await flushMicrotasks(); + expect(player.currentState.value).toBe("wink"); + }); +}); diff --git a/packages/client-js/src/sprite-player.ts b/packages/client-js/src/sprite-player.ts new file mode 100644 index 0000000..68efd1f --- /dev/null +++ b/packages/client-js/src/sprite-player.ts @@ -0,0 +1,260 @@ +import type { + Animation, + FrameRef, + FrameSequence, + LoopMode, +} from "./schema.js"; +import { + AnimationGraph, + effectiveLoop, + resolveTransition, + type Phase, +} from "./animation-graph.js"; +import { type Observable, MutableObservable } from "./observable.js"; +import { SystemTicker, type Ticker } from "./ticker.js"; + +const MIN_FRAME_DELAY_MS = 16; + +class CancelledError extends Error { + constructor() { + super("cancelled"); + this.name = "CancelledError"; + } +} + +/** + * Platform-independent playback engine. One instance per character per mode. + * Drives `currentRef` forward over time according to the `AnimationGraph`'s + * animations and transitions; callers materialize frames via their own + * `FrameSource`. + * + * Mirrors the Kotlin `SpriteAnimationPlayer`. Thread/task safety: `requestState` + * is safe to call from any context; internal transitions cancel the previous + * playback via an `AbortController` before the new one starts. + */ +export class SpriteAnimationPlayer { + private readonly ticker: Ticker; + private readonly _currentRef = new MutableObservable(null); + private readonly _currentState: MutableObservable; + private abortController: AbortController | null = null; + private runningTask: Promise | null = null; + + readonly currentRef: Observable = this._currentRef; + readonly currentState: Observable; + + constructor(private readonly graph: AnimationGraph, ticker: Ticker = new SystemTicker()) { + this.ticker = ticker; + this._currentState = new MutableObservable(graph.defaultState); + this.currentState = this._currentState; + // Start playing the default state (entering=true so intros fire). + this.runningTask = this.spawn((signal) => this.playState(graph.defaultState, true, null, signal)); + } + + /** + * Request a state change. If the graph's transitions table has a match for + * `currentState → target`, that transition plays once before the target + * state's own loop starts. + * + * `playCount` semantics (from `<<>>`): + * - null or 0 — loop indefinitely until the next `requestState` + * - N >= 1 — play the loop phase exactly N times, then hold the last + * frame indefinitely. Intro (if any) still plays once. + * + * When `playCount` is non-null we always replay — even when `target` is + * already the current state — so a model emitting the same marker twice in + * a row visibly replays the animation instead of being swallowed as a no-op. + */ + async requestState(target: string, playCount: number | null = null): Promise { + const previousState = this._currentState.value; + const sameState = target === previousState; + const effectiveCount = playCount !== null && playCount > 0 ? playCount : null; + + // Cancel whatever was running. + await this.cancelRunning(); + + if (sameState && effectiveCount === null) { + return; + } + + this.runningTask = this.spawn(async (signal) => { + if (!sameState) { + const transition = this.graph.resolveTransition(previousState, target); + if (typeof transition === "string") { + const { animation, phase } = resolveTransition(transition); + await this.playPhase(animation, phase, "once", signal); + } + // Crossfade transitions are a rendering-side concern the consumer + // applies when the ref changes; the player just snaps through. + } + await this.playState(target, !sameState, effectiveCount, signal); + }); + } + + /** Cancel playback and release internal resources. */ + async dispose(): Promise { + await this.cancelRunning(); + } + + // --- internals --- + + private spawn(body: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + this.abortController = controller; + const task = (async () => { + try { + await body(controller.signal); + } catch (err) { + if (!(err instanceof CancelledError)) { + throw err; + } + } + })(); + return task; + } + + private async cancelRunning(): Promise { + const prev = this.runningTask; + const prevController = this.abortController; + this.runningTask = null; + this.abortController = null; + if (prevController) prevController.abort(); + if (prev) { + try { + await prev; + } catch { + /* swallow */ + } + } + } + + private async playState( + state: string, + entering: boolean, + playCountOverride: number | null, + signal: AbortSignal, + ): Promise { + this._currentState.set(state); + const anim = this.graph.animations[state]; + if (!anim) return; + if (entering && anim.intro) { + await this.playPhase(state, "intro", null, signal); + } + if (playCountOverride !== null && playCountOverride >= 1) { + await this.playPhaseFinite(state, "loop", playCountOverride, signal); + return; + } + // Flat states fall through to `effectiveLoop`; phased states play `loop`. + // `outro` fires only via requestState() transitions. + await this.playPhase(state, "loop", null, signal); + } + + private async playPhaseFinite( + animName: string, + phase: Phase, + times: number, + signal: AbortSignal, + ): Promise { + const anim = this.graph.animations[animName]; + if (!anim) return; + const seq = pickPhase(anim, phase); + if (!seq || seq.frames.length === 0) return; + const frameDelayMs = Math.max(Math.floor(1000 / seq.fps), MIN_FRAME_DELAY_MS); + for (let round = 0; round < times; round++) { + for (const ref of seq.frames) { + this.throwIfCancelled(signal); + this._currentRef.set(ref); + await this.delayCancellable(frameDelayMs, signal); + } + } + this._currentRef.set(seq.frames[seq.frames.length - 1] ?? null); + await awaitCancellation(signal); + } + + private async playPhase( + animName: string, + phase: Phase, + loopOverride: LoopMode | null, + signal: AbortSignal, + ): Promise { + const anim = this.graph.animations[animName]; + if (!anim) return; + const seq = pickPhase(anim, phase); + if (!seq || seq.frames.length === 0) return; + const frameDelayMs = Math.max(Math.floor(1000 / seq.fps), MIN_FRAME_DELAY_MS); + const loop: LoopMode = loopOverride ?? seq.loop; + + if (loop === "once") { + for (const ref of seq.frames) { + this.throwIfCancelled(signal); + this._currentRef.set(ref); + await this.delayCancellable(frameDelayMs, signal); + } + if (!seq.holdLastFrame) { + this._currentRef.set(null); + } + return; + } + + if (loop === "ping-pong") { + const cap = seq.iterations ?? Number.MAX_SAFE_INTEGER; + let rounds = 0; + while (rounds < cap) { + for (const ref of seq.frames) { + this.throwIfCancelled(signal); + this._currentRef.set(ref); + await this.delayCancellable(frameDelayMs, signal); + } + for (let i = seq.frames.length - 2; i >= 1; i--) { + this.throwIfCancelled(signal); + const ref = seq.frames[i]; + if (ref !== undefined) this._currentRef.set(ref); + await this.delayCancellable(frameDelayMs, signal); + } + rounds++; + } + return; + } + + // infinite + for (;;) { + for (const ref of seq.frames) { + this.throwIfCancelled(signal); + this._currentRef.set(ref); + await this.delayCancellable(frameDelayMs, signal); + } + } + } + + private throwIfCancelled(signal: AbortSignal): void { + if (signal.aborted) throw new CancelledError(); + } + + private async delayCancellable(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) throw new CancelledError(); + await Promise.race([ + this.ticker.delay(ms), + new Promise((_, reject) => { + const onAbort = () => reject(new CancelledError()); + if (signal.aborted) onAbort(); + else signal.addEventListener("abort", onAbort, { once: true }); + }), + ]); + if (signal.aborted) throw new CancelledError(); + } +} + +function pickPhase(anim: Animation, phase: Phase): FrameSequence | undefined { + if (phase === "intro") return anim.intro; + if (phase === "outro") return anim.outro; + return effectiveLoop(anim); +} + +function awaitCancellation(signal: AbortSignal): Promise { + return new Promise((_, reject) => { + if (signal.aborted) { + reject(new CancelledError()); + return; + } + signal.addEventListener("abort", () => reject(new CancelledError()), { once: true }); + }); +} diff --git a/packages/client-js/src/ticker.ts b/packages/client-js/src/ticker.ts new file mode 100644 index 0000000..08a19eb --- /dev/null +++ b/packages/client-js/src/ticker.ts @@ -0,0 +1,14 @@ +/** + * Timing abstraction for frame advancement. The default implementation uses + * `setTimeout`; tests inject a fake ticker that advances virtual time. The + * Kotlin + Swift ports expose the same seam. + */ +export interface Ticker { + delay(ms: number): Promise; +} + +export class SystemTicker implements Ticker { + delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/packages/client-js/tsconfig.json b/packages/client-js/tsconfig.json new file mode 100644 index 0000000..7dde006 --- /dev/null +++ b/packages/client-js/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "isolatedModules": true, + "declaration": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["./src/**/*.ts"], + "exclude": ["./dist/**", "./node_modules/**", "./src/**/*.test.ts"] +} diff --git a/packages/client-js/vitest.config.ts b/packages/client-js/vitest.config.ts new file mode 100644 index 0000000..ce36a74 --- /dev/null +++ b/packages/client-js/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + environment: "node", + }, +}); diff --git a/packages/client-kotlin/README.md b/packages/client-kotlin/README.md new file mode 100644 index 0000000..f5bd58d --- /dev/null +++ b/packages/client-kotlin/README.md @@ -0,0 +1,80 @@ +# sprite-core-client (Kotlin) + +Kotlin client kit for the SpriteCore plugin. Two modules: + +- **`:core`** — pure JVM. `CharacterManifest` data classes, `AnimationGraph`, + `SpriteAnimationPlayer`, `FrameSource`, marker parser, `AgentAvatarSource`. + Works on any JVM target including Android, Wear OS, desktop, or a + JVM server. +- **`:android`** — Android library. `BitmapFrameSource` bridges the core kit + to Android `Bitmap`, decoded via `BitmapFactory`. + +Artifact coordinates (after publish): + +``` +ai.openclaw.spritecore:sprite-core-client:1.0.0 +ai.openclaw.spritecore:sprite-core-client-android:1.0.0 +``` + +## Consuming locally (Gradle composite build) + +From a consuming Gradle project (e.g. an Android app): + +```kotlin +// settings.gradle.kts +includeBuild("../sprite-core/packages/client-kotlin") + +// app/build.gradle.kts +dependencies { + implementation("ai.openclaw.spritecore:sprite-core-client") + implementation("ai.openclaw.spritecore:sprite-core-client-android") +} +``` + +This avoids publishing during active development — Gradle resolves the +modules directly from the checked-out path. + +## Publishing + +To publish snapshots / releases to a Maven registry (GitHub Packages or +Maven Central), configure the registry URL + credentials in +`~/.gradle/gradle.properties` or via CI environment: + +``` +./gradlew :core:publish :android:publish +``` + +Registry coordinates are configurable via `-Pregistry=` in the Gradle +invocation or through CI secrets. See the repo-root `README.md` for the +build/publish conversation in progress. + +## Layout + +``` +packages/client-kotlin/ +├── settings.gradle.kts +├── build.gradle.kts ← plugin version pins +├── gradle.properties +├── core/ +│ ├── build.gradle.kts +│ └── src/main/kotlin/ai/openclaw/spritecore/client/ +│ ├── CharacterManifest.kt ← wire types + JSON parser + ready check +│ ├── AnimationGraph.kt ← projection + transition resolver +│ ├── SpriteAnimationPlayer.kt ← coroutine-driven state machine +│ ├── FrameSource.kt ← platform adapter interface +│ ├── Ticker.kt ← timing abstraction +│ ├── AvatarMarkerParser.kt ← `<<>>` / `<<>>` parser +│ └── AgentAvatarSource.kt ← manifest + asset cache +└── android/ + └── src/main/kotlin/ai/openclaw/spritecore/client/android/ + └── BitmapFrameSource.kt ← `FrameSource` via BitmapFactory +``` + +## Conformance + +Both modules' logic is validated against the fixtures at `../../fixtures/`. +When the wire schema at `../../schema/` changes, regenerate + rerun: + +``` +./gradlew :core:test :android:test +``` diff --git a/packages/client-kotlin/android/build.gradle.kts b/packages/client-kotlin/android/build.gradle.kts new file mode 100644 index 0000000..617f593 --- /dev/null +++ b/packages/client-kotlin/android/build.gradle.kts @@ -0,0 +1,84 @@ +plugins { + id("com.android.library") + kotlin("android") + kotlin("plugin.serialization") + `maven-publish` +} + +group = "ai.openclaw.spritecore" +version = findProperty("version")?.toString() ?: "1.0.0" + +android { + namespace = "ai.openclaw.spritecore.client.android" + compileSdk = 36 + + defaultConfig { + // Lowest common denominator across phone (31) and wear (30) consumers. + minSdk = 30 + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + sourceSets { + getByName("main") { + java.setSrcDirs(listOf("src/main/kotlin")) + } + } + + publishing { + singleVariant("release") { + withSourcesJar() + } + } +} + +kotlin { + jvmToolchain(17) + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + api(project(":core")) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0") +} + +afterEvaluate { + publishing { + publications { + create("release") { + from(components["release"]) + artifactId = "sprite-core-client-android" + pom { + name.set("SpriteCore Client (Android)") + description.set("Bitmap-backed FrameSource for the SpriteCore client kit on Android / Wear OS") + url.set("https://github.com/Tyler-RNG/sprite-core") + licenses { + license { + name.set("MIT License") + url.set("https://opensource.org/licenses/MIT") + } + } + scm { + connection.set("scm:git:git://github.com/Tyler-RNG/sprite-core.git") + url.set("https://github.com/Tyler-RNG/sprite-core") + } + } + } + } + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/Tyler-RNG/sprite-core") + credentials { + username = System.getenv("GITHUB_ACTOR") ?: findProperty("gpr.user")?.toString() + password = System.getenv("GITHUB_TOKEN") ?: findProperty("gpr.key")?.toString() + } + } + } + } +} diff --git a/packages/client-kotlin/android/src/main/AndroidManifest.xml b/packages/client-kotlin/android/src/main/AndroidManifest.xml new file mode 100644 index 0000000..8072ee0 --- /dev/null +++ b/packages/client-kotlin/android/src/main/AndroidManifest.xml @@ -0,0 +1,2 @@ + + diff --git a/packages/client-kotlin/android/src/main/kotlin/ai/openclaw/spritecore/client/android/BitmapFrameSource.kt b/packages/client-kotlin/android/src/main/kotlin/ai/openclaw/spritecore/client/android/BitmapFrameSource.kt new file mode 100644 index 0000000..6b09db1 --- /dev/null +++ b/packages/client-kotlin/android/src/main/kotlin/ai/openclaw/spritecore/client/android/BitmapFrameSource.kt @@ -0,0 +1,76 @@ +package ai.openclaw.spritecore.client.android + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.util.Log +import ai.openclaw.spritecore.client.FrameRef +import ai.openclaw.spritecore.client.FrameSource + +/** + * Bridges the pure-JVM client kit to the Android `Bitmap` world. Given a + * [CharacterManifest] and the raw bytes for each `assets.refs` entry, + * [BitmapFrameSource] resolves any [FrameRef] the player emits to a + * concrete [Bitmap]: + * + * - **Sprite-style** frames reference a whole-image asset by key; the full + * decoded bitmap is returned. + * - **Atlas-style** frames reference the atlas image and carry an + * `x/y/w/h` crop rect; the returned bitmap is a + * `createBitmap(src, x, y, w, h)` slice of the decoded atlas, cached per + * `(ref, rect)` pair. + * + * Parsing + ready-check helpers live on the pure-JVM core + * ([ai.openclaw.spritecore.client.CharacterManifestJson], + * [ai.openclaw.spritecore.client.characterManifestBytesReady]). + */ +class BitmapFrameSource( + private val bytesByRef: Map, +) : FrameSource { + private val decoded = mutableMapOf() + private val sliceCache = mutableMapOf() + + override fun frame(ref: FrameRef): Bitmap? { + val whole = decodedFor(ref.ref) ?: return null + if (ref.x == null && ref.y == null && ref.w == null && ref.h == null) { + return whole + } + val key = "${ref.ref}@${ref.x},${ref.y},${ref.w},${ref.h}" + sliceCache[key]?.let { return it } + val x = ref.x ?: 0 + val y = ref.y ?: 0 + val w = ref.w ?: (whole.width - x) + val h = ref.h ?: (whole.height - y) + if (w <= 0 || h <= 0 || x < 0 || y < 0 || x + w > whole.width || y + h > whole.height) { + Log.w( + TAG, + "slice out of bounds ref=${ref.ref} rect=($x,$y,$w,$h) size=(${whole.width},${whole.height})", + ) + return null + } + return try { + val slice = Bitmap.createBitmap(whole, x, y, w, h) + sliceCache[key] = slice + slice + } catch (e: Throwable) { + Log.w(TAG, "slice failed for $key", e) + null + } + } + + private fun decodedFor(refKey: String): Bitmap? { + decoded[refKey]?.let { return it } + val bytes = bytesByRef[refKey] ?: return null + return try { + val bm = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) + if (bm != null) decoded[refKey] = bm + bm + } catch (e: Throwable) { + Log.w(TAG, "decode failed for $refKey", e) + null + } + } + + companion object { + private const val TAG = "BitmapFrameSource" + } +} diff --git a/packages/client-kotlin/build.gradle.kts b/packages/client-kotlin/build.gradle.kts new file mode 100644 index 0000000..8e220ce --- /dev/null +++ b/packages/client-kotlin/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + kotlin("jvm") version "2.0.20" apply false + kotlin("plugin.serialization") version "2.0.20" apply false + id("com.android.library") version "8.6.0" apply false +} + +// Per-module config lives in core/build.gradle.kts and android/build.gradle.kts. +// This root exists only to pin plugin versions. diff --git a/packages/client-kotlin/core/build.gradle.kts b/packages/client-kotlin/core/build.gradle.kts new file mode 100644 index 0000000..2cf42b8 --- /dev/null +++ b/packages/client-kotlin/core/build.gradle.kts @@ -0,0 +1,66 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") + `maven-publish` +} + +group = "ai.openclaw.spritecore" +version = findProperty("version")?.toString() ?: "1.0.0" + +kotlin { + jvmToolchain(17) + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0") + + testImplementation(kotlin("test")) + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") +} + +tasks.test { + useJUnitPlatform() +} + +java { + withSourcesJar() + withJavadocJar() +} + +publishing { + publications { + create("maven") { + from(components["java"]) + artifactId = "sprite-core-client" + pom { + name.set("SpriteCore Client (Kotlin core)") + description.set("Pure-JVM client kit for the SpriteCore plugin — animation graph + sprite player") + url.set("https://github.com/Tyler-RNG/sprite-core") + licenses { + license { + name.set("MIT License") + url.set("https://opensource.org/licenses/MIT") + } + } + scm { + connection.set("scm:git:git://github.com/Tyler-RNG/sprite-core.git") + url.set("https://github.com/Tyler-RNG/sprite-core") + } + } + } + } + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/Tyler-RNG/sprite-core") + credentials { + username = System.getenv("GITHUB_ACTOR") ?: findProperty("gpr.user")?.toString() + password = System.getenv("GITHUB_TOKEN") ?: findProperty("gpr.key")?.toString() + } + } + } +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AgentAvatarSource.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AgentAvatarSource.kt new file mode 100644 index 0000000..c7c5215 --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AgentAvatarSource.kt @@ -0,0 +1,172 @@ +package ai.openclaw.spritecore.client + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.util.concurrent.atomic.AtomicLong + +/** + * Client-side unified fetcher + cache for per-agent CharacterManifest + * envelopes and their asset bytes. Pure JVM — no Android deps — so any + * Kotlin client (wearable, phone, desktop, JVM server) can use it. + * + * Fetch policy is explicit: callers invoke [refresh] with an agent-id list. + * An agent already present at the same manifest revision is left alone; + * revision bumps trigger a re-fetch of the asset refs. + * + * This is the Kotlin mirror of `@tyler-rng/sprite-core-client`'s + * `AssetSource`. The two stay behaviourally identical — the conformance + * suite in `fixtures/` at the repo root enforces it. + */ +class AgentAvatarSource( + private val scope: CoroutineScope, + private val fetchManifest: suspend (agentId: String) -> CharacterManifestEnvelope?, + private val fetchAsset: suspend (relativePath: String) -> ByteArray?, + private val logger: (level: LogLevel, tag: String, msg: String) -> Unit = { _, _, _ -> }, +) { + enum class LogLevel { DEBUG, WARN } + + private val _characterManifests = + MutableStateFlow>(emptyMap()) + val characterManifests: StateFlow> = + _characterManifests.asStateFlow() + + private val _characterAssets = + MutableStateFlow>>(emptyMap()) + val characterAssets: StateFlow>> = + _characterAssets.asStateFlow() + + private val _agentStates = MutableStateFlow>(emptyMap()) + val agentStates: StateFlow> = _agentStates.asStateFlow() + + private val _agentMarkerSignals = MutableStateFlow>(emptyMap()) + val agentMarkerSignals: StateFlow> = + _agentMarkerSignals.asStateFlow() + + private val signalVersionSeq = AtomicLong(0L) + private val fetchMutex = Mutex() + + /** + * Kick off a background fetch for each agent. No-ops for agents whose + * manifest is already cached at the current revision. Returns immediately; + * the flows update as results land. + */ + fun refresh(agentIds: List) { + if (agentIds.isEmpty()) return + scope.launch { + fetchMutex.withLock { + for (agentId in agentIds) { + refreshOne(agentId) + } + } + } + } + + /** + * Update the current state for an agent. Called by the chat-reply path + * when an `<<>>` or `<<>>` marker fires. + */ + fun setAgentState(agentId: String, stateName: String, count: Int? = null) { + _agentStates.update { it + (agentId to stateName) } + val signal = AvatarMarkerSignal( + state = stateName, + count = count, + version = signalVersionSeq.incrementAndGet(), + ) + _agentMarkerSignals.update { it + (agentId to signal) } + } + + /** Snapshot of the current cache for callers to iterate. */ + fun snapshot(): List { + val manifests = _characterManifests.value + val assets = _characterAssets.value + return manifests.map { (agentId, envelope) -> + CachedAgent(agentId = agentId, envelope = envelope, assetBytes = assets[agentId].orEmpty()) + } + } + + /** Drop any cached entries for agents no longer in [keepIds]. */ + fun retainOnly(keepIds: Collection) { + val keep = keepIds.toSet() + _characterManifests.update { it.filterKeys { id -> id in keep } } + _characterAssets.update { it.filterKeys { id -> id in keep } } + _agentStates.update { it.filterKeys { id -> id in keep } } + } + + fun clear() { + _characterManifests.update { emptyMap() } + _characterAssets.update { emptyMap() } + _agentStates.update { emptyMap() } + } + + /** + * Resolve the default state name for [agentId] from its cached manifest. + * Mirrors [AnimationGraph.fromManifest] default-state logic so the two + * never drift. + */ + fun defaultStateFor(agentId: String): String? { + val envelope = _characterManifests.value[agentId] ?: return null + val manifest = envelope.manifest + val mode = manifest.modes.firstOrNull { manifest.content.containsKey(it) } ?: return null + val animations = manifest.content[mode]?.animations ?: return null + val firstFromMap = manifest.stateMap.entries.firstOrNull { animations.containsKey(it.value) } + if (firstFromMap != null) return firstFromMap.value + return animations.keys.firstOrNull() + } + + // --- internals --- + + private suspend fun refreshOne(agentId: String) { + val envelope = fetchManifest(agentId) ?: run { + logger(LogLevel.DEBUG, TAG, "manifest skip $agentId (no structured avatar or RPC failed)") + return + } + val existing = _characterManifests.value[agentId] + if (existing != null && existing.revision == envelope.revision) { + return + } + _characterManifests.update { it + (agentId to envelope) } + + val bytesByRef = mutableMapOf() + for ((refKey, relPath) in envelope.manifest.assets.refs) { + val bytes = fetchAsset(relPath) + if (bytes != null) { + bytesByRef[refKey] = bytes + } else { + logger(LogLevel.WARN, TAG, "asset fetch failed $agentId $refKey") + } + } + _characterAssets.update { it + (agentId to bytesByRef) } + logger( + LogLevel.DEBUG, + TAG, + "cached $agentId rev=${envelope.revision} (${bytesByRef.size}/${envelope.manifest.assets.refs.size} assets)", + ) + } + + data class CachedAgent( + val agentId: String, + val envelope: CharacterManifestEnvelope, + val assetBytes: Map, + ) + + /** + * Versioned per-agent animation signal. [version] bumps on every + * [setAgentState] call so UI consumers keyed on the signal re-trigger + * their effects even when the state name is unchanged. + */ + data class AvatarMarkerSignal( + val state: String, + val count: Int?, + val version: Long, + ) + + companion object { + private const val TAG = "AgentAvatarSource" + } +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AnimationGraph.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AnimationGraph.kt new file mode 100644 index 0000000..6730229 --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AnimationGraph.kt @@ -0,0 +1,100 @@ +package ai.openclaw.spritecore.client + +/** + * Resolved animation table + transition graph for a single mode of a single + * character. Both sprite and atlas manifests project into this shape so the + * player stays format-agnostic. + * + * Build via [fromManifest] to pull a mode's content out of a server-synthesized + * [CharacterManifest], or construct directly for tests. + */ +data class AnimationGraph( + val defaultState: String, + val animations: Map, + val transitions: Map, +) { + /** + * Resolve a state→state transition against the transitions table using + * wildcard pattern matching. Specificity order (most→least specific): + * + * "->" → "->*" → "*->" → "*->*" + * + * Returns null when nothing matches; the caller then swaps instantly. + */ + fun resolveTransition(from: String, to: String): TransitionRef? { + val keys = listOf("$from->$to", "$from->*", "*->$to", "*->*") + for (k in keys) { + transitions[k]?.let { return it } + } + return null + } + + companion object { + /** + * Extract a single mode's animation graph from a character manifest. + * The default state is taken from [stateMap] — the first key that maps + * to an animation present in [mode]'s content — or fails if no + * animation is present. + */ + fun fromManifest(manifest: CharacterManifest, mode: String): AnimationGraph { + val content = manifest.content[mode] + ?: throw IllegalArgumentException( + "manifest has no content for mode '$mode'. Available: ${manifest.content.keys}", + ) + val default = resolveDefaultState(manifest.stateMap, content.animations) + return AnimationGraph( + defaultState = default, + animations = content.animations, + transitions = content.transitions ?: emptyMap(), + ) + } + + private fun resolveDefaultState( + stateMap: Map, + animations: Map, + ): String { + // stateMap maps agent-state → animation name. Prefer the first + // agent-state whose animation exists in this mode; otherwise fall + // back to any animation name we have. + val firstFromMap = stateMap.entries.firstOrNull { animations.containsKey(it.value) } + if (firstFromMap != null) { + return firstFromMap.value + } + return animations.keys.firstOrNull() + ?: throw IllegalArgumentException("manifest mode has no animations") + } + } +} + +/** + * A transition target resolved for playback: which animation + phase to play + * once before entering the target state's own loop. Used by the player when a + * [TransitionRef.Phase] fires on state change. + */ +data class ResolvedTransition(val animation: String, val phase: Phase) { + companion object { + /** Parse `"thinking.intro"` into `(thinking, intro)`. Unqualified → loop. */ + fun parse(ref: String): ResolvedTransition { + val dot = ref.indexOf('.') + return if (dot < 0) { + ResolvedTransition(ref, Phase.LOOP) + } else { + ResolvedTransition(ref.substring(0, dot), Phase.fromWire(ref.substring(dot + 1))) + } + } + } +} + +/** The three phases of a phased animation; flat animations use [LOOP]. */ +enum class Phase(val wire: String) { + INTRO("intro"), + LOOP("loop"), + OUTRO("outro"), + ; + + companion object { + fun fromWire(value: String): Phase = + entries.firstOrNull { it.wire == value } + ?: throw IllegalArgumentException("unknown phase: $value") + } +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AvatarMarkerParser.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AvatarMarkerParser.kt new file mode 100644 index 0000000..e7e7585 --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/AvatarMarkerParser.kt @@ -0,0 +1,227 @@ +package ai.openclaw.spritecore.client + +/** + * Kotlin port of `src/gateway/avatar-marker-parser.ts`. Recognizes the + * inline escape `<<>>` anywhere in assistant text — not restricted to + * its own line. Matching markers are stripped from the visible text and + * surfaced separately; invalid marker shapes (empty or disallowed state + * names) are emitted verbatim so nothing is silently lost. + * + * The triple-angle-bracket escape is deliberately unusual so the model is + * unlikely to emit it by accident. Prior syntax was `[avatar:state]` on its + * own line; the new syntax permits inline emotion changes mid-utterance so + * TTS and avatar state can switch at each marker boundary. + * + * The parser is stateful across pushes: a marker split mid-token across two + * chunks is still recognized. Non-marker content is emitted immediately when + * possible so streaming UX isn't delayed. + * + * Lives at `ai.openclaw.spritecore.client` — both the phone voice path and the wear + * relay path consume this. Keep in sync with the TS reference. + */ + +const val AVATAR_MARKER_OPEN = "<<<" +const val AVATAR_MARKER_CLOSE = ">>>" + +/** + * One parsed `<<>>` / `<<>>` marker. + * + * [count] semantics forwarded from the wire format: + * null — bare `<<>>`, defaults to "loop until next marker" client-side. + * 0 — explicit loop (same as bare). + * N >= 1 — play animation N times and hold on the last frame. + */ +data class AvatarMarker(val state: String, val count: Int? = null) + +data class AvatarParseResult( + val cleanedText: String, + val markers: List, +) + +/** + * Text segment produced by [splitByMarkers]. `emotion` is the state name of + * the marker immediately preceding this segment (with any `-N` count suffix + * stripped off into [emotionCount]), or `null` for the leading segment + * (before any marker) and for segments introduced by an invalid marker + * shape (which is emitted as literal text). + */ +data class TextSegmentWithEmotion( + val text: String, + val emotion: String?, + val emotionCount: Int? = null, +) + +private val STATE_NAME_RE = Regex("^[a-zA-Z0-9_-]+$") + +private fun isValidStateName(name: String): Boolean = + name.isNotEmpty() && STATE_NAME_RE.matches(name) + +/** + * Splits a raw marker body into (state, count). Triggers on the *last* dash + * when the suffix is a non-negative integer — `head_cocked_1` (N=1) becomes + * `head_cocked` + 1, but `head-cocked` (no digits after dash) stays as + * `head-cocked` + null. Returns null count when the body is all-state. + */ +internal fun resolveStateAndCount(body: String): Pair { + val dashIdx = body.lastIndexOf('-') + if (dashIdx <= 0 || dashIdx == body.length - 1) return body to null + val countPart = body.substring(dashIdx + 1) + val count = countPart.toIntOrNull() + if (count == null || count < 0) return body to null + val state = body.substring(0, dashIdx) + if (state.isEmpty()) return body to null + return state to count +} + +class AvatarMarkerParser { + private var buffer: String = "" + + fun push(chunk: String): AvatarParseResult { + if (chunk.isEmpty()) return AvatarParseResult("", emptyList()) + val combined = buffer + chunk + val (cleaned, markers, remainder) = processSafePrefix(combined) + buffer = remainder + return AvatarParseResult(cleaned, markers) + } + + fun flush(): AvatarParseResult { + if (buffer.isEmpty()) return AvatarParseResult("", emptyList()) + // End of stream: any still-buffered bytes can no longer become a + // marker. Emit them as literal text. + val leftover = buffer + buffer = "" + return AvatarParseResult(leftover, emptyList()) + } + + fun reset() { + buffer = "" + } +} + +/** + * Convenience: parse a complete (non-streamed) string in one shot. + */ +fun parseAvatarMarkers(text: String): AvatarParseResult { + val parser = AvatarMarkerParser() + val a = parser.push(text) + val b = parser.flush() + if (b.cleanedText.isEmpty() && b.markers.isEmpty()) return a + return AvatarParseResult( + cleanedText = a.cleanedText + b.cleanedText, + markers = a.markers + b.markers, + ) +} + +/** + * Split [text] into segments delimited by `<<>>` markers. Each + * segment carries the preceding marker's state as its [TextSegmentWithEmotion.emotion] + * (null for the leading segment before any marker). + * + * Invalid marker shapes — empty state names or disallowed characters — are + * treated as literal text and merged into the enclosing segment. + * + * Empty-text segments are dropped; a reply of pure markers returns an empty + * list. Callers that need the state of an all-markers reply should read the + * markers through [parseAvatarMarkers] directly. + */ +fun splitByMarkers(text: String): List { + if (text.isEmpty()) return emptyList() + val segments = mutableListOf() + val currentText = StringBuilder() + var currentEmotion: String? = null + var currentEmotionCount: Int? = null + var i = 0 + while (i < text.length) { + val openAt = text.indexOf(AVATAR_MARKER_OPEN, i) + if (openAt == -1) { + currentText.append(text, i, text.length) + break + } + // Accumulate literal text up to the opener. + currentText.append(text, i, openAt) + val closeAt = text.indexOf(AVATAR_MARKER_CLOSE, openAt + AVATAR_MARKER_OPEN.length) + if (closeAt == -1) { + // Unterminated marker — rest of string is literal. + currentText.append(text, openAt, text.length) + break + } + val rawBody = text.substring(openAt + AVATAR_MARKER_OPEN.length, closeAt) + if (isValidStateName(rawBody)) { + val (stateName, stateCount) = resolveStateAndCount(rawBody) + // Close the current segment and start a new one tagged with the + // marker's state. + if (currentText.isNotEmpty()) { + segments.add( + TextSegmentWithEmotion( + currentText.toString(), + currentEmotion, + currentEmotionCount, + ), + ) + currentText.setLength(0) + } + currentEmotion = stateName + currentEmotionCount = stateCount + } else { + // Invalid marker shape — emit verbatim as literal text within + // the current segment. + currentText.append(text, openAt, closeAt + AVATAR_MARKER_CLOSE.length) + } + i = closeAt + AVATAR_MARKER_CLOSE.length + } + if (currentText.isNotEmpty()) { + segments.add( + TextSegmentWithEmotion(currentText.toString(), currentEmotion, currentEmotionCount), + ) + } + return segments +} + +private data class ProcessResult( + val cleanedText: String, + val markers: List, + val remainder: String, +) + +/** + * Process as much of [combined] as possible without consuming a potential + * partial marker at the tail. Returns the clean-output prefix, extracted + * markers, and the byte suffix that might still become a marker when more + * input arrives — callers buffer the remainder until the next push. + */ +private fun processSafePrefix(combined: String): ProcessResult { + val markers = mutableListOf() + val out = StringBuilder() + var i = 0 + while (i < combined.length) { + val openAt = combined.indexOf(AVATAR_MARKER_OPEN, i) + if (openAt == -1) { + // No complete `<<<` left. But the tail might be a partial start + // (`<` or `<<`) that could extend into a marker with more input; + // buffer those trailing `<` characters so the next chunk can + // complete them. + var j = combined.length + while (j > i && combined[j - 1] == '<') { + j -= 1 + } + out.append(combined, i, j) + return ProcessResult(out.toString(), markers, combined.substring(j)) + } + out.append(combined, i, openAt) + val closeAt = combined.indexOf(AVATAR_MARKER_CLOSE, openAt + AVATAR_MARKER_OPEN.length) + if (closeAt == -1) { + // Unterminated marker — buffer everything from the opener onward. + return ProcessResult(out.toString(), markers, combined.substring(openAt)) + } + val rawState = combined.substring(openAt + AVATAR_MARKER_OPEN.length, closeAt) + if (isValidStateName(rawState)) { + val (stateName, stateCount) = resolveStateAndCount(rawState) + markers.add(AvatarMarker(stateName, stateCount)) + } else { + // Invalid marker shape — emit verbatim so nothing is silently lost. + out.append(combined, openAt, closeAt + AVATAR_MARKER_CLOSE.length) + } + i = closeAt + AVATAR_MARKER_CLOSE.length + } + return ProcessResult(out.toString(), markers, "") +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/CharacterManifest.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/CharacterManifest.kt new file mode 100644 index 0000000..bb5e1d8 --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/CharacterManifest.kt @@ -0,0 +1,217 @@ +package ai.openclaw.spritecore.client + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonContentPolymorphicSerializer +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive + +/** + * Pure-Kotlin mirror of the `CharacterManifest` wire schema served by the + * gateway at `node.getCharacterManifest`. These data classes carry zero + * platform deps — Android, iOS (via Kotlin/Native if ever), tests, and thin + * JVM clients all parse the same bytes. + * + * Source of truth lives in `schema/src/display.ts` at the repo root. This + * Kotlin mirror must stay byte-compatible — the conformance suite in + * `fixtures/` at the repo root proves it. + */ +@Serializable +data class CharacterManifest( + val version: Int, + val agentId: String, + val name: String? = null, + val modes: List, + val stateMap: Map, + val content: Map, + val assets: AssetBundle, + val emotions: Map? = null, +) + +@Serializable +data class ModeContent( + val atlas: AtlasRef? = null, + val animations: Map, + val transitions: Map? = null, +) + +@Serializable +data class AtlasRef( + val image: String, + val size: Size, + val frameSize: Size? = null, +) + +@Serializable +data class Size(val w: Int, val h: Int) + +@Serializable +data class FrameRef( + val ref: String, + val x: Int? = null, + val y: Int? = null, + val w: Int? = null, + val h: Int? = null, +) + +@Serializable +data class FrameSequence( + val frames: List, + val fps: Int, + val loop: LoopMode, + val holdLastFrame: Boolean = false, + val iterations: Int? = null, +) + +@Serializable +data class Animation( + val description: String? = null, + val sequence: FrameSequence? = null, + val intro: FrameSequence? = null, + val loop: FrameSequence? = null, + val outro: FrameSequence? = null, +) { + /** + * Treat a flat sequence as the `loop` phase so the player can always look + * up phases by name without special-casing flat vs phased at every site. + */ + val effectiveLoop: FrameSequence? get() = loop ?: sequence +} + +@Serializable(with = LoopModeSerializer::class) +enum class LoopMode(val wire: String) { + INFINITE("infinite"), + ONCE("once"), + PING_PONG("ping-pong"), + ; + + companion object { + fun fromWire(value: String): LoopMode = + entries.firstOrNull { it.wire == value } + ?: throw IllegalArgumentException("unknown loop mode: $value") + } +} + +private object LoopModeSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("LoopMode", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: LoopMode) { + encoder.encodeString(value.wire) + } + + override fun deserialize(decoder: Decoder): LoopMode = LoopMode.fromWire(decoder.decodeString()) +} + +/** + * A transition is either a named phase reference (e.g. `"thinking.intro"`) + * the runtime plays once on state swap, or an inline blend directive the + * runtime applies as a visual effect during the swap. + */ +@Serializable(with = TransitionRefSerializer::class) +sealed class TransitionRef { + @Serializable + data class Phase(val value: String) : TransitionRef() + + @Serializable + data class Crossfade(val blend: String = "crossfade", val ms: Int) : TransitionRef() +} + +private object TransitionRefSerializer : + JsonContentPolymorphicSerializer(TransitionRef::class) { + override fun selectDeserializer(element: JsonElement) = when (element) { + is JsonPrimitive -> PhaseStringSerializer + else -> TransitionRef.Crossfade.serializer() + } +} + +private object PhaseStringSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("TransitionRef.Phase", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: TransitionRef.Phase) { + encoder.encodeString(value.value) + } + + override fun deserialize(decoder: Decoder): TransitionRef.Phase = + TransitionRef.Phase(decoder.decodeString()) +} + +@Serializable +data class AssetBundle(val refs: Map) { + /** Look up the asset path for a frame ref. Returns null if the ref is unknown. */ + fun pathFor(ref: FrameRef): String? = refs[ref.ref] +} + +/** + * Per-state emotion entry: only the wire-visible `directive` ships to clients. + * (Prompt-visible descriptions are server-only — see `schema/src/display.ts`.) + */ +@Serializable +data class EmotionEntry( + val directive: EmotionDirective? = null, +) + +/** + * Per-emotion TTS voice-directive override. Applied by clients after they + * parse `<<>>` markers out of assistant text — the text segment that + * follows a marker inherits the base TalkDirective merged field-by-field with + * this override. + */ +@Serializable +data class EmotionDirective( + val voiceId: String? = null, + val stability: Double? = null, + val similarity: Double? = null, + val style: Double? = null, + val speakerBoost: Boolean? = null, + val speed: Double? = null, + /** Optional inline audio-tag prefix (e.g. `[happy]`). */ + val audioTag: String? = null, +) + +@Serializable +data class CharacterManifestEnvelope( + val manifest: CharacterManifest, + val revision: Int, +) + +/** + * JSON parser for the envelope published by `node.getCharacterManifest`. + * Lives in core so any JVM client can use it without pulling Android-specific + * JSON helpers. + */ +object CharacterManifestJson { + private val json = Json { ignoreUnknownKeys = true } + + fun parse(text: String): CharacterManifestEnvelope? = try { + json.decodeFromString(CharacterManifestEnvelope.serializer(), text) + } catch (_: Throwable) { + null + } + + /** Pick the first mode in `manifest.modes` whose content is present. */ + fun pickMode(manifest: CharacterManifest): String? = + manifest.modes.firstOrNull { manifest.content.containsKey(it) } +} + +/** + * Returns true when every asset ref declared by `envelope.manifest.assets.refs` + * has bytes in `assetBytes`. Consumers use this to decide whether to render + * (all bytes present, player will find frames) or fall back until bytes + * arrive. Empty `refs` returns true. + */ +fun characterManifestBytesReady( + envelope: CharacterManifestEnvelope, + assetBytes: Map, +): Boolean { + val refs = envelope.manifest.assets.refs.keys + if (refs.isEmpty()) return true + return refs.all { assetBytes.containsKey(it) } +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/FrameSource.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/FrameSource.kt new file mode 100644 index 0000000..e1f959c --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/FrameSource.kt @@ -0,0 +1,43 @@ +package ai.openclaw.spritecore.client + +/** + * Platform-specific resolver from a [FrameRef] to a concrete renderable + * (e.g. Android `Bitmap`, iOS `UIImage`, a byte array, whatever the caller + * chooses). The kit itself never constructs frames — callers own the pixel + * pipeline and only feed [SpriteAnimationPlayer.currentRef] into their own + * [FrameSource] when rendering. + * + * Atlas sources honor the optional `x/y/w/h` fields on [FrameRef]; sprite + * sources ignore them and treat `ref` as the whole-image key. + */ +fun interface FrameSource { + /** Return the frame for [ref], or null if the ref is unknown. */ + fun frame(ref: FrameRef): FrameT? +} + +/** + * Simple in-memory sprite source: callers prime a byte-array map, decode + * happens lazily through [decode]. Useful for unit tests and thin clients + * that don't need the platform-specific image types. + */ +class InMemorySpriteSource( + private val decode: (ByteArray) -> FrameT?, +) : FrameSource { + private val bytesByRef = mutableMapOf() + private val cache = mutableMapOf() + + fun put(refKey: String, bytes: ByteArray) { + bytesByRef[refKey] = bytes + cache.remove(refKey) + } + + fun keys(): Set = bytesByRef.keys + + override fun frame(ref: FrameRef): FrameT? { + cache[ref.ref]?.let { return it } + val bytes = bytesByRef[ref.ref] ?: return null + val decoded = decode(bytes) ?: return null + cache[ref.ref] = decoded + return decoded + } +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/SpriteAnimationPlayer.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/SpriteAnimationPlayer.kt new file mode 100644 index 0000000..ced8e58 --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/SpriteAnimationPlayer.kt @@ -0,0 +1,220 @@ +package ai.openclaw.spritecore.client + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch + +/** + * Platform-independent playback engine. One instance per character per mode. + * Drives [currentRef] forward over time according to the [AnimationGraph]'s + * animations and transitions; callers materialize frames via their own + * [FrameSource]. + * + * Thread safety: [requestState] is safe to call from any thread. Internal + * state mutations happen on the supplied coroutine scope's dispatcher. + */ +class SpriteAnimationPlayer( + private val graph: AnimationGraph, + private val ticker: Ticker = SystemTicker(), + scope: CoroutineScope? = null, +) { + private val owned = scope == null + private val scope: CoroutineScope = scope ?: CoroutineScope(SupervisorJob() + Dispatchers.Default) + + private val _currentRef = MutableStateFlow(null) + /** The frame the caller should be rendering right now. Null = blank. */ + val currentRef: StateFlow = _currentRef.asStateFlow() + + private val _currentState = MutableStateFlow(graph.defaultState) + /** The agent state the player is currently in (post-transition). */ + val currentState: StateFlow = _currentState.asStateFlow() + + private var activeJob: Job? = null + + init { + activeJob = this.scope.launch { + playState(graph.defaultState, entering = true) + } + } + + /** + * Request a state change. If the [graph]'s transitions table has a match + * for `currentState → target`, that transition plays once before the + * target state's own loop starts. + * + * [playCount] semantics (from the client-parsed `<<>>` marker): + * null or 0 → default: the state's configured loop plays indefinitely + * until the next [requestState] cancels it. + * N >= 1 → play the loop phase exactly N times, then hold the last + * frame indefinitely. Intro (if any) still plays once. + * + * When [playCount] is non-null we always replay — even when [target] is + * already the current state — so a model emitting the same marker twice + * in a row ("<<>> then <<>>") visibly replays the + * animation instead of being swallowed as a no-op. + */ + fun requestState(target: String, playCount: Int? = null): Job { + // Capture the job reference before reassignment. Reading activeJob + // from inside the launched block is a race — by the time the block + // runs, activeJob has been overwritten to `job` itself, so the old + // job (often the init-spawned infinite loop) would never be cancelled. + val previousJob = activeJob + val job = scope.launch { + val sameState = target == _currentState.value + if (sameState && (playCount == null || playCount <= 0)) { + previousJob?.cancelAndJoin() + return@launch + } + val previousState = _currentState.value + previousJob?.cancelAndJoin() + if (!sameState) { + val transition = graph.resolveTransition(previousState, target) + if (transition is TransitionRef.Phase) { + val resolved = ResolvedTransition.parse(transition.value) + playPhase( + animName = resolved.animation, + phase = resolved.phase, + loopOverride = LoopMode.ONCE, + ) + } + // Crossfade transitions are currently played as an instant + // swap; the visual blend is a rendering-side concern the + // consumer applies when the ref changes. + } + playState(target, entering = !sameState, playCountOverride = playCount) + } + activeJob = job + return job + } + + /** Cancel playback and, if we own it, the internal scope. */ + fun dispose() { + activeJob?.cancel() + if (owned) { + scope.cancel() + } + } + + // --- internals --- + + private suspend fun playState( + state: String, + entering: Boolean, + playCountOverride: Int? = null, + ) { + _currentState.value = state + val anim = graph.animations[state] ?: return + if (entering && anim.intro != null) { + playPhase(state, Phase.INTRO) + } + if (playCountOverride != null && playCountOverride >= 1) { + playPhaseFinite(state, Phase.LOOP, playCountOverride) + return + } + // Flat states fall through to `effectiveLoop`; phased states play + // `loop` here. `outro` fires only on requestState() via transitions. + playPhase(state, Phase.LOOP) + } + + /** + * Play the [phase] of [animName] exactly [times] times, then hold the + * last frame indefinitely (until this coroutine is cancelled by the + * next [requestState]). Implements the `<<>>` N>=1 contract: + * play N times and pause on the last frame. + */ + private suspend fun playPhaseFinite( + animName: String, + phase: Phase, + times: Int, + ) { + val anim = graph.animations[animName] ?: return + val seq = when (phase) { + Phase.INTRO -> anim.intro + Phase.LOOP -> anim.effectiveLoop + Phase.OUTRO -> anim.outro + } ?: return + if (seq.frames.isEmpty()) return + val frameDelayMs = (1000L / seq.fps).coerceAtLeast(MIN_FRAME_DELAY_MS) + repeat(times) { + for (ref in seq.frames) { + _currentRef.value = ref + ticker.delay(frameDelayMs) + } + } + _currentRef.value = seq.frames.last() + awaitCancellation() + } + + private suspend fun playPhase( + animName: String, + phase: Phase, + loopOverride: LoopMode? = null, + ) { + val anim = graph.animations[animName] ?: return + val seq = when (phase) { + Phase.INTRO -> anim.intro + Phase.LOOP -> anim.effectiveLoop + Phase.OUTRO -> anim.outro + } ?: return + if (seq.frames.isEmpty()) { + return + } + val frameDelayMs = (1000L / seq.fps).coerceAtLeast(MIN_FRAME_DELAY_MS) + val loop = loopOverride ?: seq.loop + + when (loop) { + LoopMode.ONCE -> { + for (ref in seq.frames) { + _currentRef.value = ref + ticker.delay(frameDelayMs) + } + if (!seq.holdLastFrame) { + _currentRef.value = null + } + } + LoopMode.PING_PONG -> { + val cap = seq.iterations ?: Int.MAX_VALUE + var rounds = 0 + while (rounds < cap) { + for (ref in seq.frames) { + _currentRef.value = ref + ticker.delay(frameDelayMs) + } + for (i in seq.frames.size - 2 downTo 1) { + _currentRef.value = seq.frames[i] + ticker.delay(frameDelayMs) + } + rounds++ + } + } + LoopMode.INFINITE -> { + while (true) { + for (ref in seq.frames) { + _currentRef.value = ref + ticker.delay(frameDelayMs) + } + } + } + } + } + + private suspend fun Job.cancelAndJoin() { + cancel() + try { + join() + } catch (_: Throwable) { + // Cancellation unwinds through here; swallow so caller's flow continues. + } + } + + companion object { + private const val MIN_FRAME_DELAY_MS = 16L + } +} diff --git a/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/Ticker.kt b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/Ticker.kt new file mode 100644 index 0000000..8f54de4 --- /dev/null +++ b/packages/client-kotlin/core/src/main/kotlin/ai/openclaw/spritecore/client/Ticker.kt @@ -0,0 +1,29 @@ +package ai.openclaw.spritecore.client + +import kotlinx.coroutines.delay as coroutineDelay + +/** + * Wall clock + scheduler injected into the player so tests can drive playback + * deterministically without real delays. Production code uses [SystemTicker]. + */ +interface Ticker { + /** Current monotonic time in milliseconds. */ + fun nowMs(): Long + + /** Suspend for [ms]; clamped to >= 0 at the implementation. */ + suspend fun delay(ms: Long) +} + +/** Default production ticker: backed by [System.currentTimeMillis] + coroutine delay. */ +class SystemTicker : Ticker { + override fun nowMs(): Long = System.currentTimeMillis() + override suspend fun delay(ms: Long) { + if (ms > 0L) { + // Aliased import — an unqualified `delay(ms)` resolves to this + // enclosing member and self-recurses until StackOverflowError. + // Tests couldn't catch it because they inject a TestTicker; only + // the production SystemTicker path ever exercises this call. + coroutineDelay(ms) + } + } +} diff --git a/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/AnimationGraphTest.kt b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/AnimationGraphTest.kt new file mode 100644 index 0000000..339dc4a --- /dev/null +++ b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/AnimationGraphTest.kt @@ -0,0 +1,118 @@ +package ai.openclaw.spritecore.client + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertIs +import kotlin.test.assertNull + +class AnimationGraphTest { + private fun graph( + transitions: Map = emptyMap(), + ) = AnimationGraph( + defaultState = "neutral", + animations = mapOf("neutral" to flatAnim(), "thinking" to flatAnim(), "happy" to flatAnim()), + transitions = transitions, + ) + + private fun flatAnim(): Animation = Animation( + sequence = FrameSequence(listOf(FrameRef("x")), fps = 12, loop = LoopMode.INFINITE), + ) + + @Test + fun returnsNullWhenNoTransitionMatches() { + assertNull(graph().resolveTransition("neutral", "happy")) + } + + @Test + fun prefersConcreteOverWildcardMatches() { + val g = graph( + transitions = mapOf( + "*->*" to TransitionRef.Phase("wild.loop"), + "*->happy" to TransitionRef.Phase("wildhappy.intro"), + "neutral->*" to TransitionRef.Phase("neutralout.outro"), + "neutral->happy" to TransitionRef.Phase("direct.intro"), + ), + ) + val t = g.resolveTransition("neutral", "happy") + assertIs(t) + assertEquals("direct.intro", t.value) + } + + @Test + fun fromToWildcardBeatsToFromWildcard() { + // Rule: `->*` wins over `*->` when both are set. + val g = graph( + transitions = mapOf( + "*->happy" to TransitionRef.Phase("a.intro"), + "neutral->*" to TransitionRef.Phase("b.outro"), + ), + ) + val t = g.resolveTransition("neutral", "happy") + assertIs(t) + assertEquals("b.outro", t.value) + } + + @Test + fun resolvesBlendTransitionsAsBlendRefs() { + val g = graph( + transitions = mapOf( + "*->happy" to TransitionRef.Crossfade(ms = 150), + ), + ) + val t = g.resolveTransition("neutral", "happy") + assertIs(t) + assertEquals(150, t.ms) + } + + @Test + fun parsesTransitionPhaseRefs() { + assertEquals( + ResolvedTransition("thinking", Phase.INTRO), + ResolvedTransition.parse("thinking.intro"), + ) + assertEquals( + ResolvedTransition("thinking", Phase.LOOP), + ResolvedTransition.parse("thinking"), + ) + } + + @Test + fun fromManifestPullsOutRequestedMode() { + val manifest = CharacterManifest( + version = 1, + agentId = "ginger", + modes = listOf("headshot"), + stateMap = mapOf("neutral" to "neutral"), + content = mapOf( + "headshot" to ModeContent( + animations = mapOf( + "neutral" to flatAnim(), + "happy" to flatAnim(), + ), + transitions = mapOf("*->happy" to TransitionRef.Phase("happy.intro")), + ), + ), + assets = AssetBundle(mapOf("x" to "a/x.png")), + ) + val g = AnimationGraph.fromManifest(manifest, "headshot") + assertEquals("neutral", g.defaultState) + assertEquals(setOf("neutral", "happy"), g.animations.keys) + assertEquals("happy.intro", (g.transitions["*->happy"] as TransitionRef.Phase).value) + } + + @Test + fun fromManifestFailsWhenModeMissing() { + val manifest = CharacterManifest( + version = 1, + agentId = "ginger", + modes = listOf("headshot"), + stateMap = mapOf("neutral" to "neutral"), + content = emptyMap(), + assets = AssetBundle(emptyMap()), + ) + assertFailsWith { + AnimationGraph.fromManifest(manifest, "headshot") + } + } +} diff --git a/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/AvatarMarkerParserTest.kt b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/AvatarMarkerParserTest.kt new file mode 100644 index 0000000..1bc81a8 --- /dev/null +++ b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/AvatarMarkerParserTest.kt @@ -0,0 +1,195 @@ +package ai.openclaw.spritecore.client + +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.test.Test + +class AvatarMarkerParserTest { + + @Test + fun oneShotStripsInlineMarker() { + val r = parseAvatarMarkers("Hello <<>> world") + assertEquals("Hello world", r.cleanedText) + assertEquals(listOf(AvatarMarker("happy")), r.markers) + } + + @Test + fun oneShotHandlesMarkerOnOwnLineToo() { + val r = parseAvatarMarkers("Hello\n<<>>\nworld\n") + assertEquals("Hello\n\nworld\n", r.cleanedText) + assertEquals(listOf(AvatarMarker("happy")), r.markers) + } + + @Test + fun passesThroughWithNoMarkers() { + val r = parseAvatarMarkers("no marker here.\nsecond line\n") + assertEquals("no marker here.\nsecond line\n", r.cleanedText) + assertTrue(r.markers.isEmpty()) + } + + @Test + fun handlesMultipleMarkersInSequence() { + val r = parseAvatarMarkers("<<>>A<<>>B<<>>") + assertEquals("AB", r.cleanedText) + assertEquals( + listOf(AvatarMarker("happy"), AvatarMarker("sad"), AvatarMarker("neutral")), + r.markers, + ) + } + + @Test + fun letsMarkerMidSentenceSegmentText() { + val r = parseAvatarMarkers("I feel <<>>about this but <<>>about that.") + assertEquals("I feel about this but about that.", r.cleanedText) + assertEquals(listOf(AvatarMarker("happy"), AvatarMarker("sad")), r.markers) + } + + @Test + fun emitsMarkerAtStreamEndWithoutTrailingText() { + val r = parseAvatarMarkers("Hi <<>>") + assertEquals("Hi ", r.cleanedText) + assertEquals(listOf(AvatarMarker("happy")), r.markers) + } + + @Test + fun preservesPartialTrailingNonMarker() { + val r = parseAvatarMarkers("alpha\nbeta") + assertEquals("alpha\nbeta", r.cleanedText) + assertTrue(r.markers.isEmpty()) + } + + @Test + fun treatsInvalidStateNamesAsLiteral() { + val r = parseAvatarMarkers("<<>> then <<<>>>") + assertEquals("<<>> then <<<>>>", r.cleanedText) + assertTrue(r.markers.isEmpty()) + } + + @Test + fun acceptsDashesAndUnderscoresInStateNames() { + val r = parseAvatarMarkers("<<>>") + assertEquals("", r.cleanedText) + assertEquals(listOf(AvatarMarker("head-cocked_1")), r.markers) + } + + @Test + fun doesNotStripUnterminatedOpenerAtEndOfStream() { + val r = parseAvatarMarkers("hi <<", ">", ">") + val outBuilder = StringBuilder() + val markers = mutableListOf() + for (c in chunks) { + val r = parser.push(c) + outBuilder.append(r.cleanedText) + markers.addAll(r.markers) + } + val end = parser.flush() + outBuilder.append(end.cleanedText) + markers.addAll(end.markers) + assertEquals("", outBuilder.toString()) + assertEquals(listOf(AvatarMarker("happy")), markers) + } + + @Test + fun streamingEmitsNonMarkerTailImmediately() { + val parser = AvatarMarkerParser() + val r = parser.push("hello world") + assertEquals("hello world", r.cleanedText) + assertTrue(r.markers.isEmpty()) + val f = parser.flush() + assertEquals("", f.cleanedText) + assertTrue(f.markers.isEmpty()) + } + + @Test + fun streamingBuffersSingleAngleBracketInCaseItExtendsToMarker() { + val parser = AvatarMarkerParser() + val r1 = parser.push("text <") + assertEquals("text ", r1.cleanedText) + assertTrue(r1.markers.isEmpty()) + val r2 = parser.push("<>>") + assertEquals("", r2.cleanedText) + assertEquals(listOf(AvatarMarker("happy")), r2.markers) + } + + @Test + fun streamingBuffersDoubleAngleBracketInCaseItExtendsToMarker() { + val parser = AvatarMarkerParser() + val r1 = parser.push("text <<") + assertEquals("text ", r1.cleanedText) + assertTrue(r1.markers.isEmpty()) + val r2 = parser.push(">>") + assertEquals("", r2.cleanedText) + assertEquals(listOf(AvatarMarker("happy")), r2.markers) + } + + @Test + fun streamingFlushesUnterminatedOpenerAsLiteral() { + val parser = AvatarMarkerParser() + parser.push("text <<>>") + // After reset, the `<<` buffer was dropped — `>>` alone isn't + // a valid opener so it emits as literal. + assertEquals(">>", r.cleanedText) + assertTrue(r.markers.isEmpty()) + } + + @Test + fun splitByMarkersTagsEachSegmentWithPrecedingMarker() { + val segments = splitByMarkers("Hi <<>>I'm glad, <<>>but also down.") + assertEquals( + listOf( + TextSegmentWithEmotion("Hi ", null), + TextSegmentWithEmotion("I'm glad, ", "happy"), + TextSegmentWithEmotion("but also down.", "sad"), + ), + segments, + ) + } + + @Test + fun splitByMarkersReturnsSingleSegmentWhenNoMarkers() { + val segments = splitByMarkers("plain text") + assertEquals(listOf(TextSegmentWithEmotion("plain text", null)), segments) + } + + @Test + fun splitByMarkersReturnsEmptyListForEmptyInput() { + assertTrue(splitByMarkers("").isEmpty()) + } + + @Test + fun splitByMarkersSkipsEmptySegmentsBetweenAdjacentMarkers() { + val segments = splitByMarkers("<<>><<>>real text") + assertEquals( + listOf(TextSegmentWithEmotion("real text", "sad")), + segments, + ) + } + + @Test + fun splitByMarkersTreatsInvalidMarkersAsLiteralWithinEnclosingSegment() { + val segments = splitByMarkers("prefix <<>> suffix") + assertEquals( + listOf(TextSegmentWithEmotion("prefix <<>> suffix", null)), + segments, + ) + } +} diff --git a/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/ManifestParseTest.kt b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/ManifestParseTest.kt new file mode 100644 index 0000000..5bc10bd --- /dev/null +++ b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/ManifestParseTest.kt @@ -0,0 +1,143 @@ +package ai.openclaw.spritecore.client + +import kotlinx.serialization.json.Json +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class ManifestParseTest { + private val json = Json { ignoreUnknownKeys = false } + + @Test + fun parsesMinimalHeadshotManifest() { + val manifest = json.decodeFromString( + """ + { + "version": 1, + "agentId": "ginger", + "modes": ["headshot"], + "stateMap": { "neutral": "neutral" }, + "content": { + "headshot": { + "animations": { + "neutral": { + "sequence": { + "frames": [{ "ref": "neutral" }], + "fps": 12, + "loop": "infinite" + } + } + } + } + }, + "assets": { "refs": { "neutral": "avatars/ginger/neutral.gif" } } + } + """.trimIndent(), + ) + assertEquals("ginger", manifest.agentId) + assertEquals(listOf("headshot"), manifest.modes) + val anim = manifest.content.getValue("headshot").animations.getValue("neutral") + assertEquals(LoopMode.INFINITE, anim.sequence?.loop) + assertEquals("avatars/ginger/neutral.gif", manifest.assets.pathFor(FrameRef("neutral"))) + } + + @Test + fun parsesPhasedAnimationsAndTransitions() { + val manifest = json.decodeFromString( + """ + { + "version": 1, + "agentId": "ginger", + "modes": ["headshot"], + "stateMap": { "thinking": "thinking" }, + "content": { + "headshot": { + "animations": { + "thinking": { + "intro": { "frames": [{ "ref": "a" }], "fps": 24, "loop": "once" }, + "loop": { "frames": [{ "ref": "b" }], "fps": 12, "loop": "infinite" }, + "outro": { "frames": [{ "ref": "c" }], "fps": 24, "loop": "once", "holdLastFrame": true } + } + }, + "transitions": { + "*->thinking": "thinking.intro", + "*->happy": { "blend": "crossfade", "ms": 150 } + } + } + }, + "assets": { "refs": { "a": "p/a", "b": "p/b", "c": "p/c" } } + } + """.trimIndent(), + ) + val thinking = manifest.content.getValue("headshot").animations.getValue("thinking") + assertNotNull(thinking.intro) + assertEquals(24, thinking.intro.fps) + assertEquals(LoopMode.ONCE, thinking.intro.loop) + assertTrue(thinking.outro?.holdLastFrame == true) + + val transitions = manifest.content.getValue("headshot").transitions.orEmpty() + val toThinking = transitions.getValue("*->thinking") + assertIs(toThinking) + assertEquals("thinking.intro", toThinking.value) + + val toHappy = transitions.getValue("*->happy") + assertIs(toHappy) + assertEquals(150, toHappy.ms) + } + + @Test + fun parsesAtlasContent() { + val manifest = json.decodeFromString( + """ + { + "version": 1, + "agentId": "ginger", + "modes": ["headshot"], + "stateMap": { "neutral": "neutral" }, + "content": { + "headshot": { + "atlas": { "image": "atlas.webp", "size": { "w": 1024, "h": 1024 }, "frameSize": { "w": 256, "h": 256 } }, + "animations": { + "neutral": { + "sequence": { + "frames": [ + { "ref": "atlas.webp", "x": 0, "y": 0, "w": 256, "h": 256 } + ], + "fps": 12, + "loop": "infinite" + } + } + } + } + }, + "assets": { "refs": { "atlas.webp": "avatars/ginger/atlas.webp" } } + } + """.trimIndent(), + ) + val headshot = manifest.content.getValue("headshot") + assertNotNull(headshot.atlas) + assertEquals(1024, headshot.atlas.size.w) + assertEquals(256, headshot.atlas.frameSize?.w) + val frame = headshot.animations.getValue("neutral").sequence!!.frames[0] + assertEquals("atlas.webp", frame.ref) + assertEquals(0, frame.x) + assertEquals(256, frame.w) + assertNull(headshot.transitions) + } + + @Test + fun animationEffectiveLoopFallsBackToSequence() { + val flat = Animation( + sequence = FrameSequence(listOf(FrameRef("a")), fps = 12, loop = LoopMode.INFINITE), + ) + assertEquals(flat.sequence, flat.effectiveLoop) + + val phased = Animation( + loop = FrameSequence(listOf(FrameRef("a")), fps = 12, loop = LoopMode.INFINITE), + ) + assertEquals(phased.loop, phased.effectiveLoop) + } +} diff --git a/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/SpriteAnimationPlayerTest.kt b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/SpriteAnimationPlayerTest.kt new file mode 100644 index 0000000..965ee4d --- /dev/null +++ b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/SpriteAnimationPlayerTest.kt @@ -0,0 +1,235 @@ +package ai.openclaw.spritecore.client + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +@OptIn(ExperimentalCoroutinesApi::class) +class SpriteAnimationPlayerTest { + // ---- helpers ---- + + private fun seq( + count: Int, + loop: LoopMode = LoopMode.INFINITE, + fps: Int = 10, // 100ms / frame → easy to reason about under advanceTimeBy + holdLastFrame: Boolean = false, + iterations: Int? = null, + ) = FrameSequence( + frames = (0 until count).map { FrameRef("f$it") }, + fps = fps, + loop = loop, + holdLastFrame = holdLastFrame, + iterations = iterations, + ) + + private fun anim( + sequence: FrameSequence? = null, + intro: FrameSequence? = null, + loop: FrameSequence? = null, + outro: FrameSequence? = null, + ) = Animation(sequence = sequence, intro = intro, loop = loop, outro = outro) + + private class TestTicker(private val scope: TestScope) : Ticker { + override fun nowMs(): Long = scope.testScheduler.currentTime + override suspend fun delay(ms: Long) { + if (ms > 0L) kotlinx.coroutines.delay(ms) + } + } + + private fun TestScope.newPlayer(graph: AnimationGraph): SpriteAnimationPlayer { + val scopeForPlayer = CoroutineScope(coroutineContext) + return SpriteAnimationPlayer(graph, TestTicker(this), scopeForPlayer) + } + + // ---- tests ---- + + @Test + fun playsInfiniteLoopAdvancingOneFramePerTick() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "neutral", + animations = mapOf("neutral" to anim(sequence = seq(count = 3, loop = LoopMode.INFINITE))), + transitions = emptyMap(), + ) + val player = newPlayer(g) + + advanceTimeBy(10) // kick coroutine past init + assertEquals("f0", player.currentRef.value?.ref) + + advanceTimeBy(100) + assertEquals("f1", player.currentRef.value?.ref) + + advanceTimeBy(100) + assertEquals("f2", player.currentRef.value?.ref) + + advanceTimeBy(100) + assertEquals("f0", player.currentRef.value?.ref) // wrapped + + player.dispose() + } + + @Test + fun onceWithHoldLastFrameFreezesOnFinalFrame() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "state", + animations = mapOf( + "state" to anim(sequence = seq(count = 3, loop = LoopMode.ONCE, holdLastFrame = true)), + ), + transitions = emptyMap(), + ) + val player = newPlayer(g) + + // Advance way past playback. + advanceTimeBy(10_000) + assertEquals("f2", player.currentRef.value?.ref) + + player.dispose() + } + + @Test + fun onceWithoutHoldClearsAfterLastFrame() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "state", + animations = mapOf( + "state" to anim(sequence = seq(count = 2, loop = LoopMode.ONCE, holdLastFrame = false)), + ), + transitions = emptyMap(), + ) + val player = newPlayer(g) + + advanceTimeBy(10_000) + assertNull(player.currentRef.value) + + player.dispose() + } + + @Test + fun pingPongBouncesAndCapsAtIterations() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "state", + animations = mapOf( + "state" to anim( + sequence = seq(count = 3, loop = LoopMode.PING_PONG, iterations = 1), + ), + ), + transitions = emptyMap(), + ) + val player = newPlayer(g) + + val seen = mutableListOf() + advanceTimeBy(10) + fun capture() { + val ref = player.currentRef.value?.ref + if (ref != null && (seen.isEmpty() || seen.last() != ref)) { + seen.add(ref) + } + } + + // Walk through: f0, f1, f2, f1 (bounce), then stops (iterations=1 → one round-trip). + capture() + repeat(30) { + advanceTimeBy(100) + capture() + } + + // After the bounce finishes the loop exits; the last ref stays visible. + // Minimum expected ordering within the round trip. + val indexF0 = seen.indexOf("f0") + val indexF2 = seen.indexOf("f2") + val indexF1Bounce = seen.lastIndexOf("f1") + assertEquals(0, indexF0) + assert(indexF2 > indexF0) + assert(indexF1Bounce > indexF2) { "expected ping-pong to return to f1 after f2, got $seen" } + + player.dispose() + } + + @Test + fun enteringAStateWithIntroPlaysIntroThenLoop() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "thinking", + animations = mapOf( + "thinking" to anim( + intro = seq(count = 2, loop = LoopMode.ONCE), + loop = seq(count = 2, loop = LoopMode.INFINITE), + ), + ), + transitions = emptyMap(), + ) + val player = newPlayer(g) + + val order = mutableListOf() + advanceTimeBy(10) + order.add(player.currentRef.value!!.ref) // f0 intro + advanceTimeBy(100) + order.add(player.currentRef.value!!.ref) // f1 intro + advanceTimeBy(100) + order.add(player.currentRef.value!!.ref) // f0 loop + advanceTimeBy(100) + order.add(player.currentRef.value!!.ref) // f1 loop + advanceTimeBy(100) + order.add(player.currentRef.value!!.ref) // f0 loop (cycled) + + assertEquals(listOf("f0", "f1", "f0", "f1", "f0"), order) + + player.dispose() + } + + @Test + fun requestStatePlaysTransitionPhaseBeforeEnteringTargetState() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "neutral", + animations = mapOf( + "neutral" to anim(sequence = seq(count = 1, loop = LoopMode.INFINITE)), + "thinking" to anim( + intro = seq(count = 1, loop = LoopMode.ONCE), + loop = seq(count = 1, loop = LoopMode.INFINITE), + ), + ), + transitions = mapOf("*->thinking" to TransitionRef.Phase("thinking.intro")), + ) + val player = newPlayer(g) + // Kick past init so the neutral loop has rendered its first frame. + // advanceUntilIdle() would hang here — default state is an infinite loop. + advanceTimeBy(10) + val before = player.currentRef.value?.ref + assertEquals("f0", before) + + player.requestState("thinking") + advanceTimeBy(10) + // First thing played on transition is thinking.intro f0 (via transition ref). + assertEquals("f0", player.currentRef.value?.ref) + // Advance past the transition's ONCE playback (100ms @ 10fps) into the + // target state's own loop; target state is now "thinking". + advanceTimeBy(300) + assertEquals("thinking", player.currentState.value) + + player.dispose() + } + + @Test + fun requestSameStateIsNoop() = runTest(StandardTestDispatcher()) { + val g = AnimationGraph( + defaultState = "neutral", + animations = mapOf("neutral" to anim(sequence = seq(count = 1, loop = LoopMode.INFINITE))), + transitions = mapOf("*->*" to TransitionRef.Phase("neutral.intro")), + ) + val player = newPlayer(g) + // Default state is infinite; advanceTimeBy is the only safe way to + // kick past init without hanging on the perpetual frame loop. + advanceTimeBy(10) + val before = player.currentState.value + + player.requestState(before) + advanceTimeBy(10) + + assertEquals(before, player.currentState.value) + player.dispose() + } +} diff --git a/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/SystemTickerTest.kt b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/SystemTickerTest.kt new file mode 100644 index 0000000..f61e6eb --- /dev/null +++ b/packages/client-kotlin/core/src/test/kotlin/ai/openclaw/spritecore/client/SystemTickerTest.kt @@ -0,0 +1,39 @@ +package ai.openclaw.spritecore.client + +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertTrue + +/** + * Regression test for the member-shadowing bug in SystemTicker.delay(): an + * unqualified call to `delay(ms)` inside the overridden method resolves to + * the same member and self-recurses until StackOverflowError. The bug is + * production-only because the rest of the suite injects TestTicker, so + * this file exists explicitly to exercise the real class with real time. + * + * Keep short delays here (single-digit ms) so the suite stays fast. + */ +class SystemTickerTest { + @Test + fun systemTickerDelayDoesNotSelfRecurse() = runBlocking { + val ticker = SystemTicker() + val start = ticker.nowMs() + ticker.delay(5L) + val elapsed = ticker.nowMs() - start + // We only care that it returns without StackOverflowError. Lower bound + // is loose because the scheduler isn't guaranteed-punctual; upper + // bound guards against a delay that accidentally dropped into a busy + // loop one day. + assertTrue(elapsed in 0..500, "unexpected elapsed=$elapsed ms for a 5ms delay") + } + + @Test + fun systemTickerDelayZeroOrNegativeIsNoop() = runBlocking { + val ticker = SystemTicker() + val start = ticker.nowMs() + ticker.delay(0L) + ticker.delay(-1L) + val elapsed = ticker.nowMs() - start + assertTrue(elapsed < 100, "noop delays shouldn't actually sleep, got $elapsed ms") + } +} diff --git a/packages/client-kotlin/gradle.properties b/packages/client-kotlin/gradle.properties new file mode 100644 index 0000000..07df4f5 --- /dev/null +++ b/packages/client-kotlin/gradle.properties @@ -0,0 +1,5 @@ +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +org.gradle.parallel=true +org.gradle.caching=true +kotlin.code.style=official +android.useAndroidX=true diff --git a/packages/client-kotlin/settings.gradle.kts b/packages/client-kotlin/settings.gradle.kts new file mode 100644 index 0000000..48fd8c0 --- /dev/null +++ b/packages/client-kotlin/settings.gradle.kts @@ -0,0 +1,20 @@ +pluginManagement { + repositories { + gradlePluginPortal() + mavenCentral() + google() + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + mavenCentral() + google() + } +} + +rootProject.name = "sprite-core-client" + +include(":core") +include(":android") diff --git a/packages/client-swift/Package.swift b/packages/client-swift/Package.swift new file mode 100644 index 0000000..8378876 --- /dev/null +++ b/packages/client-swift/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: "SpriteCoreClient", + platforms: [ + .iOS(.v15), + .macOS(.v12), + .tvOS(.v15), + .watchOS(.v8), + ], + products: [ + .library( + name: "SpriteCoreClient", + targets: ["SpriteCoreClient"] + ), + ], + dependencies: [], + targets: [ + .target( + name: "SpriteCoreClient", + path: "Sources/SpriteCoreClient" + ), + .testTarget( + name: "SpriteCoreClientTests", + dependencies: ["SpriteCoreClient"], + path: "Tests/SpriteCoreClientTests" + ), + ] +) diff --git a/packages/client-swift/README.md b/packages/client-swift/README.md new file mode 100644 index 0000000..100bcb1 --- /dev/null +++ b/packages/client-swift/README.md @@ -0,0 +1,65 @@ +# SpriteCoreClient (Swift) + +Swift client SDK for the SpriteCore plugin. SwiftPM library, zero deps +beyond Foundation, supports iOS 15+, macOS 12+, tvOS 15+, watchOS 8+. + +## Install + +From a consuming Xcode project: + +```swift +// In Package.swift, or via Xcode: File → Add Package Dependencies +.package(path: "../sprite-core/packages/client-swift") // local dev +// or +.package(url: "https://github.com/Tyler-RNG/sprite-core.git", from: "1.0.0") +``` + +Then add `"SpriteCoreClient"` to your target dependencies. + +## Minimal usage + +```swift +import SpriteCoreClient +import UIKit + +let envelope = CharacterManifestJson.parse(jsonData)! +let graph = try AnimationGraph.fromManifest(envelope.manifest, mode: "headshot") + +let frameSource = InMemorySpriteSource { data in UIImage(data: data) } +for (refKey, bytes) in assetBytes { frameSource.put(refKey, bytes: bytes) } + +let player = SpriteAnimationPlayer(graph: graph) + +Task { + for await ref in await player.refStream() { + guard let ref, let img = frameSource.frame(for: ref) else { continue } + await MainActor.run { imageView.image = img } + } +} + +// When the model emits `<<>>`: +let parser = AvatarMarkerParser() +let parsed = parser.push(streamChunk) +for m in parsed.markers { + await player.requestState(m.state, playCount: m.count) +} +``` + +## Surface + +- `CharacterManifest` / `CharacterManifestEnvelope` — `Codable` wire types +- `CharacterManifestJson.parse(_:)` — envelope parser +- `AnimationGraph.fromManifest(_:mode:)` — projection + wildcard resolver +- `SpriteAnimationPlayer` — `actor`-isolated state machine, async-streams refs +- `FrameSource` protocol + `InMemorySpriteSource` +- `AvatarMarkerParser` / `parseAvatarMarkers(_:)` / `splitByMarkers(_:)` + +## Conformance + +This Swift implementation mirrors the Kotlin and TypeScript ports in the +sibling packages. The shared fixture suite at `../../fixtures/` is the +oracle. Run: + +``` +swift test +``` diff --git a/packages/client-swift/Sources/SpriteCoreClient/AnimationGraph.swift b/packages/client-swift/Sources/SpriteCoreClient/AnimationGraph.swift new file mode 100644 index 0000000..8e76407 --- /dev/null +++ b/packages/client-swift/Sources/SpriteCoreClient/AnimationGraph.swift @@ -0,0 +1,88 @@ +import Foundation + +/// Resolved animation table + transition graph for a single mode of a single +/// character. Both sprite and atlas manifests project into this shape so the +/// player stays format-agnostic. +public struct AnimationGraph: Sendable { + public let defaultState: String + public let animations: [String: Animation] + public let transitions: [String: TransitionRef] + + public init(defaultState: String, animations: [String: Animation], transitions: [String: TransitionRef]) { + self.defaultState = defaultState + self.animations = animations + self.transitions = transitions + } + + /// Resolve a state→state transition against the transitions table using + /// wildcard pattern matching. Specificity order (most→least specific): + /// + /// `"->"` → `"->*"` → `"*->"` → `"*->*"` + /// + /// Returns nil when nothing matches. + public func resolveTransition(from: String, to: String) -> TransitionRef? { + let keys = ["\(from)->\(to)", "\(from)->*", "*->\(to)", "*->*"] + for k in keys { + if let t = transitions[k] { return t } + } + return nil + } + + /// Extract a single mode's animation graph from a character manifest. + public static func fromManifest(_ manifest: CharacterManifest, mode: String) throws -> AnimationGraph { + guard let content = manifest.content[mode] else { + throw GraphError.modeNotFound(mode: mode, available: Array(manifest.content.keys)) + } + let defaultState = try resolveDefaultState(stateMap: manifest.stateMap, animations: content.animations) + return AnimationGraph( + defaultState: defaultState, + animations: content.animations, + transitions: content.transitions ?? [:] + ) + } +} + +public enum GraphError: Error, CustomStringConvertible { + case modeNotFound(mode: String, available: [String]) + case noAnimations + + public var description: String { + switch self { + case .modeNotFound(let mode, let available): + return "manifest has no content for mode '\(mode)'. Available: \(available)" + case .noAnimations: + return "manifest mode has no animations" + } + } +} + +private func resolveDefaultState(stateMap: [String: String], animations: [String: Animation]) throws -> String { + for (_, animName) in stateMap { + if animations[animName] != nil { return animName } + } + if let first = animations.keys.first { return first } + throw GraphError.noAnimations +} + +/// The three phases of a phased animation; flat animations use `.loop`. +public enum Phase: String, Sendable { + case intro + case loop + case outro +} + +/// A transition target resolved for playback: which animation + phase to +/// play once before entering the target state's own loop. +public struct ResolvedTransition: Sendable { + public let animation: String + public let phase: Phase + + /// Parse `"thinking.intro"` → `(thinking, .intro)`. Unqualified → `.loop`. + public static func parse(_ ref: String) -> ResolvedTransition { + if let dot = ref.firstIndex(of: ".") { + let phase = Phase(rawValue: String(ref[ref.index(after: dot)...])) ?? .loop + return ResolvedTransition(animation: String(ref[..>>` / `<<>>` avatar-state markers +/// embedded in assistant text. Matching markers are stripped from the visible +/// text and surfaced separately; invalid marker shapes are treated as literal +/// text. +/// +/// Mirrors the TS `createAvatarMarkerParser()` — the fixtures at the repo +/// root enforce byte-equivalent behaviour. + +public let AVATAR_MARKER_OPEN = "<<<" +public let AVATAR_MARKER_CLOSE = ">>>" + +public struct AvatarMarker: Equatable, Sendable { + public let state: String + /// `nil` for bare markers / zero-count; `N >= 1` for play-N-times markers. + public let count: Int? + public init(state: String, count: Int? = nil) { + self.state = state + self.count = count + } +} + +public struct AvatarMarkerParseResult: Equatable, Sendable { + public let cleanedText: String + public let markers: [AvatarMarker] + public init(cleanedText: String, markers: [AvatarMarker]) { + self.cleanedText = cleanedText + self.markers = markers + } +} + +public struct TextSegmentWithEmotion: Equatable, Sendable { + public let text: String + public let emotion: String? + public let emotionCount: Int? + public init(text: String, emotion: String?, emotionCount: Int?) { + self.text = text + self.emotion = emotion + self.emotionCount = emotionCount + } +} + +private let stateNameRegex = try! NSRegularExpression(pattern: "^[A-Za-z0-9_-]+$") + +private func isValidStateName(_ name: String) -> Bool { + if name.isEmpty { return false } + let range = NSRange(location: 0, length: name.utf16.count) + return stateNameRegex.firstMatch(in: name, options: [], range: range) != nil +} + +/// Split a raw marker body into (state, count). Triggers on the *last* dash +/// when the suffix is a non-negative integer. Exported for test coverage. +public func resolveStateAndCount(_ body: String) -> (state: String, count: Int?) { + guard let dashIdx = body.lastIndex(of: "-"), dashIdx != body.startIndex else { + return (body, nil) + } + let after = body.index(after: dashIdx) + guard after != body.endIndex else { return (body, nil) } + let countPart = String(body[after...]) + guard Int(countPart) != nil, countPart.allSatisfy(\.isNumber) else { + return (body, nil) + } + guard let count = Int(countPart), count >= 0 else { return (body, nil) } + let state = String(body[.. AvatarMarkerParseResult { + if chunk.isEmpty { return AvatarMarkerParseResult(cleanedText: "", markers: []) } + let combined = buffer + chunk + let (cleaned, markers, remainder) = processSafePrefix(combined) + buffer = remainder + return AvatarMarkerParseResult(cleanedText: cleaned, markers: markers) + } + + public func flush() -> AvatarMarkerParseResult { + if buffer.isEmpty { return AvatarMarkerParseResult(cleanedText: "", markers: []) } + let leftover = buffer + buffer = "" + return AvatarMarkerParseResult(cleanedText: leftover, markers: []) + } + + public func reset() { + buffer = "" + } +} + +/// Convenience: parse a complete (non-streamed) string in one shot. +public func parseAvatarMarkers(_ text: String) -> AvatarMarkerParseResult { + let parser = AvatarMarkerParser() + let a = parser.push(text) + let b = parser.flush() + return AvatarMarkerParseResult( + cleanedText: a.cleanedText + b.cleanedText, + markers: a.markers + b.markers + ) +} + +/// Split `text` into segments delimited by `<<>>` markers. Each segment +/// carries the preceding marker's state as its `emotion`. +public func splitByMarkers(_ text: String) -> [TextSegmentWithEmotion] { + if text.isEmpty { return [] } + var segments: [TextSegmentWithEmotion] = [] + var currentText = "" + var currentEmotion: String? = nil + var currentEmotionCount: Int? = nil + var i = text.startIndex + while i < text.endIndex { + guard let openAt = text.range(of: AVATAR_MARKER_OPEN, range: i.. (cleanedText: String, markers: [AvatarMarker], remainder: String) { + var markers: [AvatarMarker] = [] + var out = "" + var i = combined.startIndex + + while i < combined.endIndex { + guard let openRange = combined.range(of: AVATAR_MARKER_OPEN, range: i.. i, combined[combined.index(before: j)] == "<" { + j = combined.index(before: j) + } + out += combined[i.. String? { refs[ref.ref] } +} + +public struct EmotionEntry: Codable, Sendable, Equatable { + public let directive: EmotionDirective? + public init(directive: EmotionDirective? = nil) { self.directive = directive } +} + +public struct EmotionDirective: Codable, Sendable, Equatable { + public let voiceId: String? + public let stability: Double? + public let similarity: Double? + public let style: Double? + public let speakerBoost: Bool? + public let speed: Double? + public let audioTag: String? + + public init( + voiceId: String? = nil, + stability: Double? = nil, + similarity: Double? = nil, + style: Double? = nil, + speakerBoost: Bool? = nil, + speed: Double? = nil, + audioTag: String? = nil + ) { + self.voiceId = voiceId + self.stability = stability + self.similarity = similarity + self.style = style + self.speakerBoost = speakerBoost + self.speed = speed + self.audioTag = audioTag + } +} + +public struct CharacterManifestEnvelope: Codable, Sendable, Equatable { + public let manifest: CharacterManifest + public let revision: Int + public init(manifest: CharacterManifest, revision: Int) { + self.manifest = manifest + self.revision = revision + } +} + +/// JSON parser for the envelope published by `node.getCharacterManifest`. +public enum CharacterManifestJson { + public static func parse(_ data: Data) -> CharacterManifestEnvelope? { + try? JSONDecoder().decode(CharacterManifestEnvelope.self, from: data) + } + + public static func parse(_ text: String) -> CharacterManifestEnvelope? { + guard let data = text.data(using: .utf8) else { return nil } + return parse(data) + } + + /// Pick the first mode in `manifest.modes` whose content is present. + public static func pickMode(_ manifest: CharacterManifest) -> String? { + manifest.modes.first { manifest.content[$0] != nil } + } +} + +/// Returns true when every asset ref declared by `envelope.manifest.assets.refs` +/// has bytes in `assetBytes`. Empty refs returns true. +public func characterManifestBytesReady( + envelope: CharacterManifestEnvelope, + assetBytes: [String: Data] +) -> Bool { + let refs = envelope.manifest.assets.refs.keys + if refs.isEmpty { return true } + return refs.allSatisfy { assetBytes[$0] != nil } +} diff --git a/packages/client-swift/Sources/SpriteCoreClient/FrameSource.swift b/packages/client-swift/Sources/SpriteCoreClient/FrameSource.swift new file mode 100644 index 0000000..4c4fd08 --- /dev/null +++ b/packages/client-swift/Sources/SpriteCoreClient/FrameSource.swift @@ -0,0 +1,50 @@ +import Foundation + +/// Platform-specific resolver from a `FrameRef` to a concrete renderable +/// (e.g. `UIImage`, `CGImage`, or raw bytes). The kit itself never constructs +/// frames — callers own the pixel pipeline and feed the player's emitted +/// `FrameRef` into their own `FrameSource` when rendering. +/// +/// Atlas sources honor the optional `x/y/w/h` fields on `FrameRef`; sprite +/// sources ignore them and treat `ref` as the whole-image key. +public protocol FrameSource { + associatedtype Frame + func frame(for ref: FrameRef) -> Frame? +} + +/// Simple in-memory sprite source: callers prime bytes per ref key, decode +/// happens lazily through the closure. Useful for tests + thin clients that +/// don't need a platform-specific image type. +public final class InMemorySpriteSource: FrameSource, @unchecked Sendable { + private let decode: (Data) -> Frame? + private var bytesByRef: [String: Data] = [:] + private var cache: [String: Frame] = [:] + private let lock = NSLock() + + public init(decode: @escaping (Data) -> Frame?) { + self.decode = decode + } + + public func put(_ refKey: String, bytes: Data) { + lock.lock() + defer { lock.unlock() } + bytesByRef[refKey] = bytes + cache.removeValue(forKey: refKey) + } + + public func keys() -> Set { + lock.lock() + defer { lock.unlock() } + return Set(bytesByRef.keys) + } + + public func frame(for ref: FrameRef) -> Frame? { + lock.lock() + defer { lock.unlock() } + if let cached = cache[ref.ref] { return cached } + guard let bytes = bytesByRef[ref.ref] else { return nil } + guard let decoded = decode(bytes) else { return nil } + cache[ref.ref] = decoded + return decoded + } +} diff --git a/packages/client-swift/Sources/SpriteCoreClient/SpriteAnimationPlayer.swift b/packages/client-swift/Sources/SpriteCoreClient/SpriteAnimationPlayer.swift new file mode 100644 index 0000000..38a4139 --- /dev/null +++ b/packages/client-swift/Sources/SpriteCoreClient/SpriteAnimationPlayer.swift @@ -0,0 +1,218 @@ +import Foundation + +/// Platform-independent playback engine. One instance per character per mode. +/// Drives `currentRef` forward over time according to the `AnimationGraph`'s +/// animations and transitions; callers materialize frames via their own +/// `FrameSource`. +/// +/// Mirrors the Kotlin `SpriteAnimationPlayer` and the TS `SpriteAnimationPlayer`. +/// Observability is via `AsyncStream` and `AsyncStream` — a +/// SwiftUI consumer can also read the latest synchronous values through +/// `currentRef` / `currentState`. +public actor SpriteAnimationPlayer { + private let graph: AnimationGraph + private let ticker: Ticker + private let minFrameDelayMs = 16 + + private var _currentRef: FrameRef? + private var _currentState: String + private var refContinuations: [UUID: AsyncStream.Continuation] = [:] + private var stateContinuations: [UUID: AsyncStream.Continuation] = [:] + + private var runningTask: Task? + + public init(graph: AnimationGraph, ticker: Ticker = SystemTicker()) { + self.graph = graph + self.ticker = ticker + self._currentState = graph.defaultState + // Kick off default-state playback on actor init. Use a detached Task + // because the actor isn't fully initialized until this init returns. + Task { [weak self] in + await self?.startDefaultState() + } + } + + // MARK: - Public surface + + public var currentRef: FrameRef? { _currentRef } + public var currentState: String { _currentState } + + public func refStream() -> AsyncStream { + AsyncStream { continuation in + let id = UUID() + continuation.yield(_currentRef) + refContinuations[id] = continuation + continuation.onTermination = { [weak self] _ in + Task { await self?.removeRefContinuation(id) } + } + } + } + + public func stateStream() -> AsyncStream { + AsyncStream { continuation in + let id = UUID() + continuation.yield(_currentState) + stateContinuations[id] = continuation + continuation.onTermination = { [weak self] _ in + Task { await self?.removeStateContinuation(id) } + } + } + } + + /// Request a state change. If the graph's transitions table has a match + /// for `currentState → target`, that transition plays once before the + /// target state's own loop starts. + /// + /// `playCount` semantics (from `<<>>`): + /// - nil or 0 — loop indefinitely + /// - N >= 1 — play N times and hold last frame + public func requestState(_ target: String, playCount: Int? = nil) async { + let previousState = _currentState + let sameState = target == previousState + let effectiveCount: Int? = (playCount ?? 0) >= 1 ? playCount : nil + + await cancelRunning() + + if sameState && effectiveCount == nil { + return + } + + let task = Task { [weak self] in + guard let self else { return } + if !sameState { + if case let .phase(ref) = await self.graph.resolveTransition(from: previousState, to: target) ?? .phase("") { + if !ref.isEmpty { + let resolved = ResolvedTransition.parse(ref) + await self.playPhase(animName: resolved.animation, phase: resolved.phase, loopOverride: .once) + } + } + } + await self.playState(target, entering: !sameState, playCountOverride: effectiveCount) + } + runningTask = task + } + + public func dispose() async { + await cancelRunning() + for (_, c) in refContinuations { c.finish() } + for (_, c) in stateContinuations { c.finish() } + refContinuations.removeAll() + stateContinuations.removeAll() + } + + // MARK: - Internals + + private func removeRefContinuation(_ id: UUID) { + refContinuations.removeValue(forKey: id) + } + + private func removeStateContinuation(_ id: UUID) { + stateContinuations.removeValue(forKey: id) + } + + private func startDefaultState() async { + let task = Task { [weak self] in + guard let self else { return } + await self.playState(await self.graph.defaultState, entering: true, playCountOverride: nil) + } + runningTask = task + } + + private func cancelRunning() async { + runningTask?.cancel() + runningTask = nil + } + + private func setRef(_ ref: FrameRef?) { + _currentRef = ref + for (_, c) in refContinuations { c.yield(ref) } + } + + private func setState(_ state: String) { + if _currentState != state { + _currentState = state + for (_, c) in stateContinuations { c.yield(state) } + } + } + + private func playState(_ state: String, entering: Bool, playCountOverride: Int?) async { + setState(state) + guard let anim = graph.animations[state] else { return } + if entering, anim.intro != nil { + await playPhase(animName: state, phase: .intro, loopOverride: nil) + if Task.isCancelled { return } + } + if let count = playCountOverride, count >= 1 { + await playPhaseFinite(animName: state, phase: .loop, times: count) + return + } + await playPhase(animName: state, phase: .loop, loopOverride: nil) + } + + private func playPhaseFinite(animName: String, phase: Phase, times: Int) async { + guard let anim = graph.animations[animName], let seq = pickPhase(anim, phase), !seq.frames.isEmpty else { return } + let frameDelayMs = max(1000 / seq.fps, minFrameDelayMs) + for _ in 0.. 2 { + for i in stride(from: seq.frames.count - 2, through: 1, by: -1) { + if Task.isCancelled { return } + setRef(seq.frames[i]) + do { try await ticker.delay(ms: frameDelayMs) } catch { return } + } + } + rounds += 1 + } + case .infinite: + while !Task.isCancelled { + for ref in seq.frames { + if Task.isCancelled { return } + setRef(ref) + do { try await ticker.delay(ms: frameDelayMs) } catch { return } + } + } + } + } + + private func pickPhase(_ anim: Animation, _ phase: Phase) -> FrameSequence? { + switch phase { + case .intro: return anim.intro + case .outro: return anim.outro + case .loop: return anim.effectiveLoop + } + } +} diff --git a/packages/client-swift/Sources/SpriteCoreClient/Ticker.swift b/packages/client-swift/Sources/SpriteCoreClient/Ticker.swift new file mode 100644 index 0000000..c9a6509 --- /dev/null +++ b/packages/client-swift/Sources/SpriteCoreClient/Ticker.swift @@ -0,0 +1,14 @@ +import Foundation + +/// Timing abstraction for frame advancement. The default implementation uses +/// `Task.sleep`; tests inject a fake ticker that advances virtual time. +public protocol Ticker: Sendable { + func delay(ms: Int) async throws +} + +public struct SystemTicker: Ticker { + public init() {} + public func delay(ms: Int) async throws { + try await Task.sleep(nanoseconds: UInt64(ms) * 1_000_000) + } +} diff --git a/packages/client-swift/Tests/SpriteCoreClientTests/AvatarMarkerParserTests.swift b/packages/client-swift/Tests/SpriteCoreClientTests/AvatarMarkerParserTests.swift new file mode 100644 index 0000000..b005f6e --- /dev/null +++ b/packages/client-swift/Tests/SpriteCoreClientTests/AvatarMarkerParserTests.swift @@ -0,0 +1,53 @@ +import XCTest +@testable import SpriteCoreClient + +final class AvatarMarkerParserTests: XCTestCase { + func testResolveStateAndCount_bare() { + let r = resolveStateAndCount("happy") + XCTAssertEqual(r.state, "happy") + XCTAssertNil(r.count) + } + + func testResolveStateAndCount_withNumericSuffix() { + let r = resolveStateAndCount("happy-3") + XCTAssertEqual(r.state, "happy") + XCTAssertEqual(r.count, 3) + } + + func testResolveStateAndCount_dashHyphenatedName() { + let r = resolveStateAndCount("head-cocked") + XCTAssertEqual(r.state, "head-cocked") + XCTAssertNil(r.count) + } + + func testStripsSingleMarker() { + let p = AvatarMarkerParser() + let r = p.push("hello <<>> world") + XCTAssertEqual(r.cleanedText, "hello world") + XCTAssertEqual(r.markers, [AvatarMarker(state: "happy")]) + } + + func testRecognisesMarkerSplitAcrossChunks() { + let p = AvatarMarkerParser() + let a = p.push("start <<>> end") + XCTAssertEqual(a.cleanedText + b.cleanedText, "start end") + XCTAssertEqual(a.markers + b.markers, [AvatarMarker(state: "happy")]) + } + + func testPlayCountMarker() { + let r = parseAvatarMarkers("say <<>> it") + XCTAssertEqual(r.markers, [AvatarMarker(state: "wink", count: 1)]) + } + + func testInvalidMarkerStaysLiteral() { + let r = parseAvatarMarkers("bad <<>> marker") + XCTAssertEqual(r.cleanedText, "bad <<>> marker") + XCTAssertTrue(r.markers.isEmpty) + } + + func testSplitByMarkers_countForwarded() { + let segs = splitByMarkers("<<>> hello") + XCTAssertEqual(segs, [TextSegmentWithEmotion(text: " hello", emotion: "wink", emotionCount: 2)]) + } +} diff --git a/packages/client-swift/Tests/SpriteCoreClientTests/ManifestParseTests.swift b/packages/client-swift/Tests/SpriteCoreClientTests/ManifestParseTests.swift new file mode 100644 index 0000000..585a253 --- /dev/null +++ b/packages/client-swift/Tests/SpriteCoreClientTests/ManifestParseTests.swift @@ -0,0 +1,65 @@ +import XCTest +@testable import SpriteCoreClient + +final class ManifestParseTests: XCTestCase { + func testDecodesMinimalHeadshotManifest() throws { + let json = """ + { + "version": 1, + "agentId": "ginger", + "modes": ["headshot"], + "stateMap": { "idle": "idle" }, + "content": { + "headshot": { + "animations": { + "idle": { + "sequence": { + "frames": [{ "ref": "idle.00" }], + "fps": 12, + "loop": "infinite" + } + } + } + } + }, + "assets": { + "refs": { "idle.00": "atlas/idle_00.webp" } + } + } + """ + let manifest = try JSONDecoder().decode(CharacterManifest.self, from: Data(json.utf8)) + XCTAssertEqual(manifest.agentId, "ginger") + XCTAssertEqual(manifest.modes, ["headshot"]) + let content = try XCTUnwrap(manifest.content["headshot"]) + let idle = try XCTUnwrap(content.animations["idle"]) + XCTAssertEqual(idle.sequence?.frames.count, 1) + } + + func testDecodesTransitionRefStringAndCrossfade() throws { + let json = """ + { + "version": 1, + "agentId": "a", + "modes": ["m"], + "stateMap": {}, + "content": { + "m": { + "animations": { "x": { "sequence": { "frames": [{ "ref": "r" }], "fps": 12, "loop": "once" } } }, + "transitions": { + "*->*": "x.intro", + "a->b": { "blend": "crossfade", "ms": 150 } + } + } + }, + "assets": { "refs": { "r": "path" } } + } + """ + let m = try JSONDecoder().decode(CharacterManifest.self, from: Data(json.utf8)) + let content = try XCTUnwrap(m.content["m"]) + let t = try XCTUnwrap(content.transitions) + guard case .phase(let s) = t["*->*"]! else { return XCTFail("expected phase ref") } + XCTAssertEqual(s, "x.intro") + guard case .crossfade(let ms) = t["a->b"]! else { return XCTFail("expected crossfade") } + XCTAssertEqual(ms, 150) + } +} diff --git a/packages/plugin/README.md b/packages/plugin/README.md new file mode 100644 index 0000000..92bf1bc --- /dev/null +++ b/packages/plugin/README.md @@ -0,0 +1,485 @@ +# SpriteCore + +OpenClaw plugin that owns the data plane for multi-state sprite avatars and +voice/TTS. Once enabled, SpriteCore is the single source of truth for: + +- per-agent avatar config (atlas image + manifest) +- per-agent voice descriptor (provider + voiceId for the watch / phone) +- the prompt block that teaches the model which avatar states exist (so it + knows when to emit `<<>>`, `<<>>`, etc., optionally with a + `-N` play-count suffix like `<<>>` or `<<>>`) +- HTTP asset serving (`/openclaw-assets/*`) +- streaming TTS proxy (`/stream/tts`) +- streaming STT proxy (`/stream/stt`) +- the gateway RPC `node.getCharacterManifest` that ships the watch a + ready-to-render manifest + +The agent's `identity.avatar` field in `openclaw.json` stays narrow: a +workspace-relative image path, an http(s) URL, a data URI, or a short string / +emoji. Anything richer (atlas, multiple states, prompting vocabulary, voice +selection) lives in this plugin's config block. + +## Install (private beta) + +This plugin is currently private. Installing it requires a GitHub Personal +Access Token and a one-time npm config. You must be a collaborator on the +`Tyler-RNG/sprite-core` GitHub repo. + +**1. Create a GitHub Personal Access Token.** Go to + and generate a classic token with +the `read:packages` scope (that's the only scope you need). Copy the token +(it starts with `ghp_`). + +**2. Point the `@tyler-rng` npm scope at GitHub Packages.** Add these two +lines to your `~/.npmrc` (create it if it doesn't exist): + +``` +@tyler-rng:registry=https://npm.pkg.github.com +//npm.pkg.github.com/:_authToken=ghp_YOUR_TOKEN_HERE +``` + +Replace `ghp_YOUR_TOKEN_HERE` with the token from step 1. Then +`chmod 600 ~/.npmrc` so other users on the machine can't read your token. + +**3. Install with the normal openclaw command.** + +```bash +openclaw plugin install @tyler-rng/sprite-core +``` + +OpenClaw resolves the `@tyler-rng` scope against GitHub Packages using your +token, downloads the tarball, and extracts it into your plugin directory. +Updates later: `openclaw plugin update @tyler-rng/sprite-core`. + +**4. Enable and configure it.** See [Enable](#enable) below for the +`openclaw.json` config block to paste in, then restart your gateway. + +**Troubleshooting:** + +- `401 Unauthorized` — your token is wrong, expired, or missing the + `read:packages` scope. Regenerate it. +- `404 Not Found` — either you're not a collaborator on + `Tyler-RNG/sprite-core`, or your `~/.npmrc` doesn't have the + `@tyler-rng:registry=...` line pointing at `npm.pkg.github.com`. +- Plugin installs but doesn't load — confirm + `plugins.entries["sprite-core"].enabled: true` is in your `openclaw.json` + and restart the gateway. + +## Enable + +```jsonc +{ + "plugins": { + "entries": { + "sprite-core": { + "enabled": true, + "config": { + "assets": { + "enabled": true, + "assetsDir": "./assets", + "publicAssets": false, + "maxAssetSizeBytes": 10485760, + "publicBaseUrl": "https://..ts.net", + }, + "streamTts": { + "enabled": true, + "provider": "elevenlabs", + "apiKey": { "source": "env", "id": "ELEVENLABS_API_KEY" }, + "defaultModel": "eleven_turbo_v2", + }, + "agents": { + "agent": { + "avatar": { + "kind": "atlas", + "default": "idle", + "manifest": "avatars/agent/agent.atlas.json", + }, + "voice": { + "provider": "elevenlabs", + "voiceId": "", + "label": "default", + }, + "prompting": { + "descriptions": { + "idle": "calm / listening", + "thinking": "processing the user's request", + "happy": "warm / pleased", + "sad": "sympathy / disappointment", + }, + }, + }, + }, + }, + }, + }, + }, +} +``` + +## Default `agent` template + +Ships under `template/agent/` in this repo. It declares four states +(`idle`, `thinking`, `happy`, `sad`) and includes a placeholder atlas image +(four solid-colored squares) so the runtime works the moment you enable the +plugin — no art required. + +To use the template: + +1. Copy `template/agent/` from this repo into + `~/.openclaw/assets/avatars/agent/` (or wherever your `assetsDir` resolves + to under the `avatars//` convention). +2. Paste the config block from `template/agent/README.md` into your + `openclaw.json` under `plugins.entries["sprite-core"].config.agents.agent`. +3. Restart the gateway. The watch will fetch the manifest, render the four + placeholder colors, and auto-swap to `thinking` on every send. + +Replace the placeholder image with real art whenever you have it; the manifest +schema does not need to change. See `template/agent/README.md` for the swap +procedure. + +## Config reference + +### `assets` + +Static asset serving for atlas images, frame trees, audio clips. + +| Field | Type | Notes | +| ------------------- | --------- | ------------------------------------------------------------------------------------------------- | +| `enabled` | `boolean` | Required to be `true` for the route to register. | +| `assetsDir` | `string` | Path the route serves from. Relative paths resolve under `~/.openclaw/state`. Default `./assets`. | +| `publicAssets` | `boolean` | When `true`, `/openclaw-assets/*` skips gateway auth. Use only when intentional. | +| `maxAssetSizeBytes` | `number` | Hard cap on per-file size. Default 10 MiB. | +| `publicBaseUrl` | `string` | URL the plugin advertises to clients in `/sprite-core/agents`. Useful for Tailscale endpoints. | + +Path traversal (`..`), symlinks pointing outside `assetsDir`, and dotfiles are +rejected. ETag + 24 h `Cache-Control` are set automatically. + +### `streamTts` + +Streaming TTS proxy. Today only ElevenLabs is wired. + +| Field | Type | Notes | +| -------------- | -------------- | -------------------------------------------------------------------------------------------------- | +| `enabled` | `boolean` | Required to be `true` for the route to register. | +| `provider` | `"elevenlabs"` | Only value supported today. | +| `apiKey` | `SecretInput` | Use `{ "source": "env", "id": "ELEVENLABS_API_KEY" }`. Plain strings are accepted but discouraged. | +| `defaultModel` | `string` | ElevenLabs model id. Default `eleven_turbo_v2`. Override per request via `?model=` query param. | + +> **The plugin ships without an ElevenLabs key.** You provide your own. +> Without `streamTts.enabled = true` and a valid `apiKey`, `/stream/tts` +> returns 503 and the watch falls back silently — agents still work, the +> avatar still animates, just no spoken audio. See [ElevenLabs setup](#elevenlabs-setup). +> +> For the full wire protocol of `/stream/tts` (query params, streaming MP3 +> response, how emotion directives map to ElevenLabs `voice_settings`, client +> composition examples) see [`docs/tts-integration.md`](docs/tts-integration.md). + +### `streamStt` + +Streaming STT proxy. Parallel to `streamTts` — same provider, same key, same +auth model. Clients POST raw audio; the plugin wraps it in multipart and +forwards to ElevenLabs's `/v1/speech-to-text`. + +| Field | Type | Notes | +| -------------- | -------------- | ----------------------------------------------------------------------------------------------- | +| `enabled` | `boolean` | Required to be `true` for the route to register. | +| `provider` | `"elevenlabs"` | Only value supported today. | +| `apiKey` | `SecretInput` | Same key as TTS — ElevenLabs uses one key for both. Reuse `{ "source": "env", "id": "ELEVENLABS_API_KEY" }`. | +| `defaultModel` | `string` | ElevenLabs model id. Default `scribe_v1`. Override per request via `?model=`. | +| `maxBodyBytes` | `number` | Optional plugin-level cap on inbound body size (checked against `Content-Length`). No default. | + +> For the full wire protocol of `/stream/stt` (accepted audio formats, query +> params → multipart field mapping, response JSON shape, error codes, curl +> example, phone-side press-and-hold flow) see +> [`docs/stt-integration.md`](docs/stt-integration.md). + +### `agents.` + +Per-agent rich descriptor that supersedes the legacy +`agents.list[].identity.avatar` object form and `agents.list[].voice` block. + +| Field | Type | Notes | +| ----------- | ----------------- | ------------------------------------------------------------------------------------------------- | +| `avatar` | `AvatarConfig` | Atlas descriptor — see below. | +| `voice` | `VoiceConfig` | `{ provider, voiceId, label, … }` — extra keys passed through to the watch. | +| `prompting` | `PromptingConfig` | Per-state descriptions used to build the model-side instruction. Optional `instruction` override. | + +#### `AvatarConfig` — `kind: "atlas"` (only kind currently supported) + +| Field | Type | Notes | +| ---------- | --------- | ------------------------------------------------------------ | +| `kind` | `"atlas"` | Discriminator. | +| `default` | `string` | State the agent holds when idle. Conventionally `idle`. | +| `manifest` | `string` | Path to the atlas JSON manifest, resolved under `assetsDir`. | + +The manifest itself owns frame rects, animations, and transitions — see +`docs/avatars/formats.md` for the full atlas schema. + +#### `VoiceConfig` + +Pass-through descriptor surfaced to the watch / phone via +`/sprite-core/agents`. Extra keys are allowed. + +```jsonc +"voice": { + "provider": "elevenlabs", + "voiceId": "21m00Tcm4TlvDq8ikWAM", + "label": "default" +} +``` + +#### `PromptingConfig` + +Drives the system-prompt block that teaches the model the avatar's emotion +vocabulary. + +| Field | Type | Notes | +| -------------- | ----------------------- | ------------------------------------------------------------------------------------------------- | +| `descriptions` | `Record` | One entry per state. Used to render `- : ` lines in the injected instruction. | +| `instruction` | `string` (optional) | Explicit override. When set, replaces the auto-generated text entirely. | + +The state names you list here must match keys in the atlas manifest's +`animations` table — that's how the watch maps a model-emitted +`<<>>` marker to the right animation. + +The keyword vocabulary (state names) lives in the gateway plugin; the parsing +of `<<>>` markers from the model output stays on the gateway side +(`src/gateway/avatar-marker-parser.ts`) and the playback code stays on the +edge devices (Wear OS DisplayKit). So edge devices stay generic — any state +name in the manifest just works. + +## Routes + +| Path | Auth | Purpose | +| ------------------------------------------------------------ | --------- | ----------------------------------------------------------------------------------------------- | +| `GET /openclaw-assets/` | gateway\* | Static asset serving. \*`auth: "plugin"` when `publicAssets: true`. | +| `GET /stream/tts` | gateway | Streaming TTS proxy (ElevenLabs). | +| `POST /stream/stt` | gateway | Streaming STT proxy (ElevenLabs). | +| `GET /sprite-core/agents` | gateway | `{ agents: { : { avatar, voice } }, publicBaseUrl? }` for clients. | +| `PUT /sprite-core/agents/:id` | gateway | Replace a single agent entry. Body: `AgentEntry`. Dashboard UI writes here. | +| `PUT /sprite-core/agents/:id/emotions/:state` | gateway | Replace a single emotion entry. Body: `{ description, directive? }`. Dashboard UI writes here. | +| `GET /sprite-core/character-manifest?agentId=[&mode=...]` | gateway | HTTP sibling of `node.getCharacterManifest` — used by the dashboard UI preview. | +| `GET /sprite-core/ui[/path]` | plugin | Dashboard UI bundle (static HTML + JS, no secrets). See [Dashboard UI](#dashboard-ui). | + +## Dashboard UI + +SpriteCore ships with a browser dashboard for editing per-agent avatar, +voice, and emotion config. It's served by the plugin itself — no changes to +the OpenClaw Control UI are required. + +**URL:** `https:///sprite-core/ui` + +The HTML shell is served publicly so the SPA can bootstrap. Every API call +attaches an `Authorization: Bearer ` header read same-origin from +browser storage. The dashboard tries (in order) the Control UI's +`openclaw.device.auth.v1` store and the legacy +`openclaw.control.token.v1[:]` key. If neither is present (e.g. +the Control UI keeps its session token in memory only), the dashboard +renders a one-time "paste your token" panel. Copy the value from +`~/.openclaw/openclaw.json` → `gateway.auth.token`, paste it once, and +the dashboard remembers it under +`sprite-core.dashboard.gatewayToken.v1`. + +The dashboard uses the same TypeScript client SDK (`@tyler-rng/sprite-core-client`) +that the phone and watch use to render avatars. Previews in the editor drive +through the real playback engine, so what you see in the dashboard is exactly +what users see on-device. + +Writes go through the OpenClaw SDK's config-file write path +(`readConfigFileSnapshotForWrite` + `writeConfigFile`). Saving in the +dashboard is equivalent to hand-editing `openclaw.json`'s +`plugins.entries["sprite-core"].config` branch — and no other branches are +ever touched. + +### Building the UI bundle + +The dashboard is prebuilt into `packages/plugin/ui-dist/` before publish, so +npm-installed copies of the plugin serve the UI out of the box. For +in-workspace development: + +```sh +# from repo root +pnpm --filter @tyler-rng/sprite-core-ui build # one-shot build +pnpm --filter @tyler-rng/sprite-core-ui dev # Vite dev server (HMR) +``` + +In dev mode, Vite proxies `/sprite-core/*` to the gateway URL in +`SPRITE_CORE_GATEWAY_URL` (default `http://localhost:8080`). + +## Gateway RPC + +| Method | Purpose | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `node.getCharacterManifest` | Returns `{ manifest, revision }` — a ready-to-render manifest assembled from the plugin's per-agent atlas config + on-disk atlas JSON. The watch calls this through the phone relay. | + +`node.getCharacterManifest` is registered by this plugin via +`api.registerGatewayMethod` from `index.ts`. When the +plugin is disabled, the method is unregistered and returns "method not found" +naturally — operators get a graceful degradation rather than a stale handler. + +## How `thinking` auto-plays + +The Wear OS phone-relay (`apps/android/app/src/main/java/ai/openclaw/app/wear/WearRelayService.kt`) +publishes a `state: "thinking"` cue on the `/openclaw/avatars//state` +DataClient path the moment the user sends a message. If your manifest declares +a `thinking` animation, DisplayKit swaps to it. If it doesn't, the watch +no-ops and stays on the previous state. + +Model-emitted `<<>>` markers (parsed by +`src/gateway/avatar-marker-parser.ts`) override this state mid-reply — last +write wins. + +## ElevenLabs setup + +The plugin **does not** ship with a key. Steps for an operator: + +1. Create an ElevenLabs account at . +2. Get your API key from the profile menu. +3. Export it in your shell environment (the gateway must inherit it): + ```bash + export ELEVENLABS_API_KEY="sk_..." + ``` +4. Pick a voice id from your ElevenLabs voice library. +5. Wire both into `openclaw.json` under `plugins.entries["sprite-core"].config`: + - `streamTts.apiKey = { "source": "env", "id": "ELEVENLABS_API_KEY" }` + - `agents..voice = { "provider": "elevenlabs", "voiceId": "" }` +6. Restart the gateway. + +If you don't enable `streamTts`, agents still work normally — the watch's +TTS playback path falls back silently. + +## Security + +- Asset serving rejects path traversal (`..`), symlinks pointing outside + `assetsDir`, and dotfile access. +- File size capped by `maxAssetSizeBytes`. +- `publicAssets: true` skips gateway auth — only set this when you intentionally + serve operator-chosen files to anonymous clients (e.g. avatars on a public web page). +- The ElevenLabs API key should be a `SecretRef` (env, file, keychain), never + inlined as a plain string in committed config. + +## Plugin self-containment + +Everything avatar / character-manifest now lives in this plugin: + +- `src/prompting.ts` owns `buildPromptingInstruction` + `isAtlasAvatarConfig`. +- `src/character-manifest.ts` owns `buildCharacterManifest` and the wire-shape + inlined `CharacterManifest` type. +- `index.ts` registers `node.getCharacterManifest` via + `api.registerGatewayMethod` and reads fresh plugin config per call. + +Core has no atlas-shaped types: `IdentityConfig.avatar` is narrowed back to +`string` (path / URL / data URI / emoji), `AgentAvatarAtlasConfig` and +friends are deleted, the gateway agent row no longer carries an `avatarAtlas` +block. Disable the plugin and the only thing that stops working is the +multi-state sprite avatar (the simple string avatar still resolves through +core's `resolveAgentAvatar`). + +### Open follow-ups + +- None of substance. The prompt instruction is live (wired via + `api.registerSystemPromptContribution` from `index.ts`), and per-agent + `voice` has been removed from core — the plugin is the sole owner. + +## Pixellab.ai pipeline + +The plugin ships two Node scripts. Together they cover the create → animate +→ package flow end to end (once the animate step has its own script). + +### Create a character + +```bash +node scripts/pixellab-create.mjs \ + --name "elf" \ + --description "a magical elf with pointed ears" +``` + +Queues a 4-direction character on pixellab, polls the background job, and +prints the new `character_id` plus the four rotation URLs so you can eyeball +the look before adding animations. `--json` emits just the id + rotations +for scripting. + +### Add animations + +Not yet ported. Use the pixellab.ai web UI or the animate-character script +(operator-supplied). + +### Export into SpriteCore + +The plugin ships a Node exporter that downloads a finished pixellab.ai +character bundle by UUID and writes a SpriteCore-compatible atlas + manifest +directly into `/avatars//`: + +```bash +# Quick path — assumes pixellab key is in `pass` or exported as PIXELLAB_API_KEY +node scripts/pixellab-export.mjs \ + --uid + +# Explicit key command + custom output root +PIXELLAB_API_KEY="$(op read op://vault/pixellab/api-key)" \ + node scripts/pixellab-export.mjs \ + --uid \ + --assets-root ~/.openclaw/state/assets/avatars \ + --overwrite + +# Dry-run the plan without touching pixellab or disk +node scripts/pixellab-export.mjs --uid --dry-run +``` + +Auth resolution order: `PIXELLAB_API_KEY` env → `--api-key-command ""` +→ `pass show pixellab/api-key`. Pick whichever matches your secret store. + +Output: + +- `/avatars//.atlas.webp` — packed atlas image. +- `/avatars//.atlas.json` — manifest. +- `/avatars//frames//NN.webp` — per-state frame + tree (useful for re-packing via `pnpm avatar:pack`). + +The exporter pairs zip-folder hashes with the pixellab API's +`animation_type` field (via `GET /characters//animations`) to emit clean +canonical SpriteCore state names — `happy`, `sad`, `thinking`, `idle` — and +generates descriptions from the animation's `display_name` (or the original +emotion prompt when no display name is set). Duplicate canonical names (e.g. +two `idle` animations of different lengths) get `_2`/`_3` suffixes. If the +metadata fetch fails, it falls back to verbose slug names. + +For the end-to-end create → approve → animate → export flow, see the +`openclaw-pixellab-avatar` skill at +`.agents/skills/openclaw-pixellab-avatar/SKILL.md`. + +The `pixellab.ai` online pixel-sprite generator is a candidate art pipeline +for the template. The intent is: + +1. Operator runs a Claude Code skill (`.agents/skills/openclaw-pixellab-avatar/SKILL.md`). +2. Skill walks them through pixellab signup + API key extraction. +3. Skill prompts pixellab to generate a character + the emotions/states the + operator wants. +4. A packaging script (`scripts/avatars/pixellab-import.mjs`) downloads the + results and wires them into the SpriteCore template layout + (`avatars//.atlas.{webp,json}`). + +The skill exists as a stub. The packaging script is not yet implemented (the +upstream pixellab.ai API contract needs to be confirmed first); see +`scripts/avatars/pixellab-import.mjs` for the placeholder. + +## Open follow-ups + +- **Pixellab exporter transition cleanup.** `scripts/pixellab-export.mjs` + unconditionally writes `*->thinking` / `thinking->*` transitions into + every atlas manifest, even when the `thinking` animation has no phased + `.intro` / `.outro` sub-sequences (the common case for v3-mode outputs). + Lint noise in the generated manifest; the runtime silently no-ops on the + missing phases. Only emit those transitions when the thinking animation + actually has intro/outro phases. ~10-line fix. +- **Pixellab `animate` template-mode investigation.** `scripts/pixellab-animate.mjs` + uses `mode: "v3"`, which produces `animation_type: "custom-"` names + instead of canonical `happy` / `sad` / `thinking` names. The exporter + currently papers over this with a `--rename` mapping. Pixellab's API may + expose a `template_animation_id` path (or a PATCH for `display_name`) + that would eliminate the workaround — confirm against + `https://api.pixellab.ai/v2/openapi.json` and migrate if available. +- **Authenticated end-to-end smoke against ElevenLabs.** Unit tests cover + the handler logic exhaustively, but nothing has sent real audio through + `POST /stream/stt` + real text through `GET /stream/tts` on a paired + device end-to-end recently. Worth one credit-burning pass periodically. diff --git a/index.ts b/packages/plugin/index.ts similarity index 78% rename from index.ts rename to packages/plugin/index.ts index d7812ec..c35b2ee 100644 --- a/index.ts +++ b/packages/plugin/index.ts @@ -1,8 +1,16 @@ import { loadConfig } from "openclaw/plugin-sdk/config-runtime"; import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { AGENTS_ROUTE_PATH, handleAgentsRequest } from "./src/agents-route.js"; +import { + AGENTS_WRITE_ROUTE_PREFIX, + handleAgentsWriteRequest, +} from "./src/agents-write-route.js"; import { ASSETS_ROUTE_PATH, handleAssetsRequest } from "./src/assets-route.js"; import { buildCharacterManifest } from "./src/character-manifest.js"; +import { + CHARACTER_MANIFEST_ROUTE_PATH, + handleCharacterManifestRequest, +} from "./src/character-manifest-route.js"; import { buildPromptingInstruction, hasSpriteDisplayCapability, @@ -10,6 +18,7 @@ import { } from "./src/prompting.js"; import { handleSttRequest, STT_ROUTE_PATH } from "./src/stt-route.js"; import { handleTtsRequest, TTS_ROUTE_PATH } from "./src/tts-route.js"; +import { handleUiRequest, UI_ROUTE_PATH } from "./src/ui-route.js"; import type { SpriteCoreConfig } from "./src/types.js"; const SPRITE_CORE_PLUGIN_ID = "sprite-core"; @@ -91,6 +100,45 @@ export default definePluginEntry({ }, }); + // Dashboard UI (browser SPA). Serves the built bundle from + // packages/plugin/ui-dist/ shipped inside the plugin package. Prefix + // match so nested asset paths (/sprite-core/ui/assets/app.abc123.js) + // resolve through the same handler. + // + // auth: "plugin" — the static HTML + JS bundle has no secrets and must + // be servable to a fresh browser so the SPA can bootstrap. The SPA then + // uses `credentials: "same-origin"` for its API calls to `/sprite-core/*` + // which remain gateway-gated. This mirrors how openclaw's Control UI + // serves its HTML shell at `/` unauthenticated and authenticates its + // API calls after the bundle has loaded. + api.registerHttpRoute({ + path: UI_ROUTE_PATH, + match: "prefix", + auth: "plugin", + handler: (req, res) => handleUiRequest(req, res), + }); + + // HTTP sibling of node.getCharacterManifest — the dashboard UI consumes + // this to drive the client SDK's AssetSource without speaking the + // gateway WebSocket. Keeps everything the UI needs on the HTTP plane. + api.registerHttpRoute({ + path: CHARACTER_MANIFEST_ROUTE_PATH, + match: "exact", + auth: "gateway", + handler: (req, res) => + handleCharacterManifestRequest(req, res, { readPluginConfig }), + }); + + // Write endpoints: PUT /sprite-core/agents/:id and + // PUT /sprite-core/agents/:id/emotions/:state. The handler performs its + // own path parsing, so register as a prefix match. + api.registerHttpRoute({ + path: AGENTS_WRITE_ROUTE_PREFIX, + match: "prefix", + auth: "gateway", + handler: (req, res) => handleAgentsWriteRequest(req, res), + }); + // System-prompt contribution: teach the model the `<<>>` marker // vocabulary, but only for sessions whose connected client can render a // sprite. Dashboard / Telegram / headless chat never see this block even diff --git a/openclaw.plugin.json b/packages/plugin/openclaw.plugin.json similarity index 100% rename from openclaw.plugin.json rename to packages/plugin/openclaw.plugin.json diff --git a/packages/plugin/package.json b/packages/plugin/package.json new file mode 100644 index 0000000..3adac02 --- /dev/null +++ b/packages/plugin/package.json @@ -0,0 +1,60 @@ +{ + "name": "@tyler-rng/sprite-core", + "version": "1.0.0", + "description": "OpenClaw SpriteCore plugin — in-gateway asset + TTS + STT data plane for multi-state avatars", + "type": "module", + "repository": { + "type": "git", + "url": "git+https://github.com/Tyler-RNG/sprite-core.git", + "directory": "packages/plugin" + }, + "license": "MIT", + "files": [ + "index.ts", + "src", + "template", + "ui-dist", + "openclaw.plugin.json", + "tsconfig.json", + "LICENSE", + "README.md" + ], + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vitest run", + "build:ui": "pnpm --filter @tyler-rng/sprite-core-ui build", + "dev:ui": "pnpm --filter @tyler-rng/sprite-core-ui dev" + }, + "devDependencies": { + "openclaw": "2026.4.15-beta.1", + "typescript": "^5.6.0", + "vitest": "^2.0.0" + }, + "peerDependencies": { + "openclaw": ">=2026.4.10" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ], + "install": { + "npmSpec": "@tyler-rng/sprite-core", + "defaultChoice": "npm", + "minHostVersion": ">=2026.4.10" + }, + "compat": { + "pluginApi": ">=2026.4.15-beta.1" + }, + "build": { + "openclawVersion": "2026.4.15-beta.1" + } + }, + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} diff --git a/scripts/pixellab-animate.mjs b/packages/plugin/scripts/pixellab-animate.mjs similarity index 89% rename from scripts/pixellab-animate.mjs rename to packages/plugin/scripts/pixellab-animate.mjs index 57a421b..bb9bc2b 100755 --- a/scripts/pixellab-animate.mjs +++ b/packages/plugin/scripts/pixellab-animate.mjs @@ -43,21 +43,23 @@ const DEFAULT_FRAME_COUNT = 8; // Pixellab reads `action_description` as a natural-language prompt. The // clean `animation_type` we want back comes from pixellab's own classifier -// on the prompt, so the wording matters. These default prompts were tuned -// against test characters to produce the corresponding animation_type. +// on the prompt, so the wording matters. Motion-oriented verbs + explicit +// body language land on-target far more often than bare emotion tokens — +// "happy" alone often produces a neutral stance; "big smile, eyes bright, +// slight excited bounce" reliably produces a recognizable happy loop. const DEFAULT_PROMPTS = { - idle: "standing still breathing gently", - thinking: "hand on chin looking up hmm expression pondering", - happy: "warm smile bright eyes joyful expression", - sad: "downturned mouth droopy eyes sorrowful expression", - angry: "furrowed brow scowl frustrated expression", - surprised: "wide eyes open mouth shocked expression", - smile: "subtle smirk to genuine smile", - frown: "neutral to disappointed frown", - love: "blushing heart eyes shy smile adoring expression", - wink: "playful wink with smirk flirty expression", - sleepy: "alert to drowsy half-closed eyes", - annoyed: "neutral to frustrated eye roll", + idle: "standing still, gentle breathing, subtle weight shift side to side", + thinking: "hand on chin, eyes glancing upward, head tilting thoughtfully, pondering", + happy: "big open-mouth smile, eyes bright and crinkled in joy, slight excited bounce", + sad: "head lowered, eyes downcast, shoulders slumped, downturned mouth, sorrowful", + angry: "furrowed brow, clenched teeth, fists tightened, body leaning forward, frustrated", + surprised: "eyes wide open, mouth agape, slight step back, arms raised, shocked", + smile: "gentle mouth curving from neutral to a warm genuine smile, eyes softening", + frown: "neutral expression turning to a disappointed frown, brow slightly furrowed", + love: "eyes becoming hearts, cheeks blushing pink, hands clasped near chest, shy adoring smile", + wink: "one eye closed in a playful wink, confident smirk, slight head tilt, finger-gun gesture", + sleepy: "eyes transitioning from alert to drowsy half-closed, head gently nodding, yawning", + annoyed: "neutral turning to frustrated eye roll, arms crossing, exasperated sigh", }; function parseArgs(argv) { diff --git a/scripts/pixellab-create.mjs b/packages/plugin/scripts/pixellab-create.mjs similarity index 100% rename from scripts/pixellab-create.mjs rename to packages/plugin/scripts/pixellab-create.mjs diff --git a/scripts/pixellab-export.mjs b/packages/plugin/scripts/pixellab-export.mjs similarity index 81% rename from scripts/pixellab-export.mjs rename to packages/plugin/scripts/pixellab-export.mjs index 280d6bd..850e063 100755 --- a/scripts/pixellab-export.mjs +++ b/packages/plugin/scripts/pixellab-export.mjs @@ -47,6 +47,52 @@ const PIXELLAB_API_BASE = "https://api.pixellab.ai/v2"; const ATLAS_COLS = 7; // matches the 1024×1024 / 136-px-frame layout used upstream const DEFAULT_ATLAS_SIDE = 1024; +// Fallback slug → canonical-emotion rename map applied when the pixellab +// `/characters//animations` metadata endpoint returns 404 (often) and the +// operator did not pass `--rename`. These substrings mirror the default +// prompts in `pixellab-animate.mjs`. Without this, the manifest ends up with +// verbose state keys like `big_open-mouth_smile_eyes_bright_and_crinkled_in_j` +// that break the SpriteCore state contract. User-supplied `--rename` overrides +// these (so a stoic character with a custom "warm smile" prompt can still map +// cleanly). Match is case-insensitive substring after normalizing `_`/`-` to +// spaces. +const DEFAULT_CANONICAL_RENAMES = { + idle: "gentle breathing", + thinking: "hand on chin", + happy: "big open mouth smile", + sad: "shoulders slumped", + angry: "clenched teeth", + surprised: "mouth agape", + love: "eyes becoming hearts", + wink: "playful wink", + smile: "warm genuine smile", + frown: "disappointed frown", + sleepy: "drowsy half closed", + annoyed: "frustrated eye roll", +}; + +// Full human-readable descriptions for each canonical emotion. Used to +// override pixellab's 50-char-truncated slug description when the rename +// step matched a canonical emotion. Keys match `DEFAULT_CANONICAL_RENAMES` +// and `pixellab-animate.mjs#DEFAULT_PROMPTS` (the prompt text we fed +// pixellab to generate the animation in the first place). Without this, +// snippets look like `"description": "big open-mouth smile eyes bright and +// crinkled in j"` (chopped mid-word by the pixellab slug limit). +const DEFAULT_CANONICAL_DESCRIPTIONS = { + idle: "standing still, gentle breathing, subtle weight shift side to side", + thinking: "hand on chin, eyes glancing upward, head tilting thoughtfully, pondering", + happy: "big open-mouth smile, eyes bright and crinkled in joy, slight excited bounce", + sad: "head lowered, eyes downcast, shoulders slumped, downturned mouth, sorrowful", + angry: "furrowed brow, clenched teeth, fists tightened, body leaning forward, frustrated", + surprised: "eyes wide open, mouth agape, slight step back, arms raised, shocked", + love: "eyes becoming hearts, cheeks blushing pink, hands clasped near chest, shy adoring smile", + wink: "one eye closed in a playful wink, confident smirk, slight head tilt, finger-gun gesture", + smile: "gentle mouth curving from neutral to a warm genuine smile, eyes softening", + frown: "neutral expression turning to a disappointed frown, brow slightly furrowed", + sleepy: "eyes transitioning from alert to drowsy half-closed, head gently nodding, yawning", + annoyed: "neutral turning to frustrated eye roll, arms crossing, exasperated sigh", +}; + function parseArgs(argv) { const opts = { uid: null, @@ -64,6 +110,8 @@ function parseArgs(argv) { voiceAuto: false, listVoices: false, elevenApiKeyCommand: null, + apply: false, + configPath: null, }; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; @@ -122,8 +170,11 @@ function parseArgs(argv) { // ElevenLabs library. Any other value is treated as a voice id. { const v = argv[++i]; - if (v === "auto") opts.voiceAuto = true; - else opts.voiceId = v; + if (v === "auto") { + opts.voiceAuto = true; + } else { + opts.voiceId = v; + } } break; case "--list-voices": @@ -132,6 +183,15 @@ function parseArgs(argv) { case "--elevenlabs-api-key-command": opts.elevenApiKeyCommand = argv[++i]; break; + case "--apply": + // Patch the generated snippet directly into openclaw.json under + // `plugins.entries["sprite-core"].config.agents.`. Backs + // up the existing config with a timestamp suffix before writing. + opts.apply = true; + break; + case "--config-path": + opts.configPath = argv[++i]; + break; case "-h": case "--help": printUsage(); @@ -181,6 +241,68 @@ function printUsage() { console.error(" --list-voices List available ElevenLabs voices and exit"); console.error(" --elevenlabs-api-key-command Stdout is the ElevenLabs API key;"); console.error(" fallback when ELEVENLABS_API_KEY env is unset"); + console.error(""); + console.error(" Config apply:"); + console.error(" --apply Patch openclaw.json directly (backed up first)"); + console.error(" --config-path

Override openclaw.json location (default: ~/.openclaw/openclaw.json)"); +} + +/** + * Patch `plugins.entries["sprite-core"].config.agents.` in + * openclaw.json with the freshly-generated block. Writes a timestamped + * backup next to the config before overwriting. Creates any missing + * intermediate objects so fresh configs still work. + * + * The gateway reads config at startup, so the caller still needs to + * restart the gateway for the new block to become visible — we print a + * reminder but deliberately do NOT auto-restart (visible side effect + * the operator should own). + */ +async function applyToConfig({ agentId, agentBlock, configPath }) { + const resolved = path.resolve( + configPath ?? path.join(os.homedir(), ".openclaw", "openclaw.json"), + ); + let raw; + try { + raw = await readFile(resolved, "utf8"); + } catch (err) { + throw new Error(`--apply: cannot read ${resolved}: ${err.message}`, { cause: err }); + } + let cfg; + try { + cfg = JSON.parse(raw); + } catch (err) { + throw new Error(`--apply: ${resolved} is not valid JSON: ${err.message}`, { cause: err }); + } + + const timestamp = new Date() + .toISOString() + .replace(/[-:]/g, "") + .replace(/\..+/, "") + .replace("T", "-"); + const backup = `${resolved}.pre-apply-${timestamp}`; + await writeFile(backup, raw); + + cfg.plugins ??= {}; + cfg.plugins.entries ??= {}; + cfg.plugins.entries["sprite-core"] ??= { enabled: true, config: {} }; + cfg.plugins.entries["sprite-core"].config ??= {}; + cfg.plugins.entries["sprite-core"].config.agents ??= {}; + const existed = + agentId in cfg.plugins.entries["sprite-core"].config.agents; + cfg.plugins.entries["sprite-core"].config.agents[agentId] = agentBlock; + + // Preserve trailing newline behavior from the source so diffs stay clean. + const trailingNewline = raw.endsWith("\n") ? "\n" : ""; + await writeFile(resolved, `${JSON.stringify(cfg, null, 2)}${trailingNewline}`); + + console.log(""); + console.log(`✓ applied to ${resolved}`); + console.log(` ${existed ? "replaced" : "added"} plugins.entries["sprite-core"].config.agents.${agentId}`); + console.log(` backup: ${backup}`); + console.log(""); + console.log("↻ restart the gateway to pick up the new entry:"); + console.log(" pkill -9 -f openclaw-gateway; nohup openclaw gateway run --bind loopback --port 18789 --force > /tmp/openclaw-gateway.log 2>&1 &"); } function resolveApiKey(apiKeyCommand) { @@ -225,14 +347,18 @@ function resolveApiKey(apiKeyCommand) { function resolveElevenLabsKey(apiKeyCommand) { const fromEnv = process.env.ELEVENLABS_API_KEY?.trim(); - if (fromEnv) return fromEnv; + if (fromEnv) { + return fromEnv; + } if (apiKeyCommand) { try { const out = execFileSync("sh", ["-c", apiKeyCommand], { encoding: "utf8", stdio: ["ignore", "pipe", "inherit"], }).trim(); - if (out) return out; + if (out) { + return out; + } } catch (err) { console.error(`--elevenlabs-api-key-command failed: ${err.message}`); return null; @@ -243,7 +369,9 @@ function resolveElevenLabsKey(apiKeyCommand) { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], }).trim(); - if (out) return out; + if (out) { + return out; + } } catch { // pass not available or entry missing — silent; caller decides fallback. } @@ -470,7 +598,10 @@ async function detectAnimationDirs(extractDir, apiMetadata, renameMap) { const { readdir } = await import("node:fs/promises"); const dirents = await readdir(animationsRoot, { withFileTypes: true }); - const renameEntries = renameMap ? Object.entries(renameMap) : []; + // User-supplied renames override defaults so custom per-character prompts + // still land on canonical emotion keys. + const mergedRenames = { ...DEFAULT_CANONICAL_RENAMES, ...renameMap }; + const renameEntries = Object.entries(mergedRenames); // Normalize `_` and `-` to spaces so natural-language needles // (e.g. "standing still") match underscored pixellab slugs // (e.g. "standing_still_breathing_gently"). @@ -509,7 +640,15 @@ async function detectAnimationDirs(extractDir, apiMetadata, renameMap) { const apiType = apiEntry?.animationType?.trim(); const renamed = tryRename(slug, apiType); const cleanName = renamed ?? apiType ?? slug; - const description = apiEntry?.displayName || prettifySlug(slug); + // Prefer the canonical full-prompt description over pixellab's truncated + // slug when we renamed into a known canonical emotion. User-supplied + // display_name (from the API metadata endpoint) still wins when present. + const canonicalDescription = + renamed && DEFAULT_CANONICAL_DESCRIPTIONS[renamed] + ? DEFAULT_CANONICAL_DESCRIPTIONS[renamed] + : null; + const description = + apiEntry?.displayName || canonicalDescription || prettifySlug(slug); raw.push({ dir: facingDir, slug, @@ -709,7 +848,7 @@ async function main() { } for (const v of voices) { const labelBits = Object.entries(v.labels || {}) - .map(([k, val]) => `${k}=${val}`) + .map(([k, val]) => `${k}=${String(val)}`) .join(" "); const cat = v.category ? ` [${v.category}]` : ""; console.log(`${v.voiceId} ${v.name}${cat}${labelBits ? ` ${labelBits}` : ""}`); @@ -874,8 +1013,14 @@ async function main() { ), }; - console.log('paste into openclaw.json under plugins.entries["sprite-core"].config.agents:'); - console.log(JSON.stringify({ [agentId]: agentBlock }, null, 2)); + if (opts.apply) { + await applyToConfig({ agentId, agentBlock, configPath: opts.configPath }); + } else { + console.log('paste into openclaw.json under plugins.entries["sprite-core"].config.agents:'); + console.log(JSON.stringify({ [agentId]: agentBlock }, null, 2)); + console.log(""); + console.log("tip: pass --apply to patch openclaw.json directly (backs up first)."); + } if (!voiceBlock) { console.log(""); console.log( diff --git a/src/agents-route.test.ts b/packages/plugin/src/agents-route.test.ts similarity index 100% rename from src/agents-route.test.ts rename to packages/plugin/src/agents-route.test.ts diff --git a/src/agents-route.ts b/packages/plugin/src/agents-route.ts similarity index 100% rename from src/agents-route.ts rename to packages/plugin/src/agents-route.ts diff --git a/packages/plugin/src/agents-write-route.ts b/packages/plugin/src/agents-write-route.ts new file mode 100644 index 0000000..0bc0bb6 --- /dev/null +++ b/packages/plugin/src/agents-write-route.ts @@ -0,0 +1,220 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { sendJson, sendMethodNotAllowed } from "./http-helpers.js"; +import { updateSpriteCoreConfig } from "./config-writes.js"; +import { validateAgentEntry, validateEmotionEntry } from "./validation.js"; + +export const AGENTS_WRITE_ROUTE_PREFIX = "/sprite-core/agents/"; + +const MAX_BODY_BYTES = 64 * 1024; + +// Parse `/sprite-core/agents/:id` and `/sprite-core/agents/:id/emotions/:state`. +type ParsedTarget = + | { kind: "agent"; agentId: string } + | { kind: "emotion"; agentId: string; state: string } + | null; + +function parseTarget(pathname: string): ParsedTarget { + if (!pathname.startsWith(AGENTS_WRITE_ROUTE_PREFIX)) { + return null; + } + const tail = pathname.slice(AGENTS_WRITE_ROUTE_PREFIX.length); + if (!tail) { + return null; + } + + const parts = tail.split("/"); + if (parts.length === 1) { + const agentId = safeDecode(parts[0]); + if (!agentId) { + return null; + } + return { kind: "agent", agentId }; + } + if (parts.length === 3 && parts[1] === "emotions") { + const agentId = safeDecode(parts[0]); + const state = safeDecode(parts[2]); + if (!agentId || !state) { + return null; + } + return { kind: "emotion", agentId, state }; + } + return null; +} + +function safeDecode(seg: string | undefined): string | null { + if (!seg) { + return null; + } + try { + const d = decodeURIComponent(seg); + if (!d || d.includes("/") || d.includes("\0")) { + return null; + } + return d; + } catch { + return null; + } +} + +async function readJsonBody(req: IncomingMessage): Promise { + return await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + req.on("data", (chunk: Buffer) => { + total += chunk.length; + if (total > MAX_BODY_BYTES) { + reject(new BodyTooLargeError()); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + if (!raw) { + resolve(undefined); + return; + } + try { + resolve(JSON.parse(raw)); + } catch { + reject(new JsonParseError()); + } + }); + req.on("error", (err) => reject(err)); + }); +} + +class BodyTooLargeError extends Error { + constructor() { + super("body too large"); + this.name = "BodyTooLargeError"; + } +} +class JsonParseError extends Error { + constructor() { + super("invalid JSON"); + this.name = "JsonParseError"; + } +} + +/** + * Handles `PUT /sprite-core/agents/:id` and + * `PUT /sprite-core/agents/:id/emotions/:state`. Returns `true` if the + * request was handled (including error responses), `false` if the path + * didn't match so the dispatcher can try the next handler. + * + * Registered with `auth: "gateway"` so the plugin HTTP dispatcher enforces + * gateway auth before this handler runs. + */ +export async function handleAgentsWriteRequest( + req: IncomingMessage, + res: ServerResponse, +): Promise { + const urlRaw = req.url; + if (!urlRaw) { + return false; + } + let url: URL; + try { + url = new URL(urlRaw, "http://localhost"); + } catch { + return false; + } + const target = parseTarget(url.pathname); + if (!target) { + return false; + } + + if (req.method !== "PUT") { + sendMethodNotAllowed(res, "PUT"); + return true; + } + + let body: unknown; + try { + body = await readJsonBody(req); + } catch (err) { + if (err instanceof BodyTooLargeError) { + sendJson(res, 413, { + error: { message: `body exceeds ${MAX_BODY_BYTES} bytes`, type: "invalid_request_error" }, + }); + return true; + } + sendJson(res, 400, { + error: { message: "invalid JSON body", type: "invalid_request_error" }, + }); + return true; + } + + if (target.kind === "emotion") { + const parsed = validateEmotionEntry(body); + if (!parsed.ok) { + sendJson(res, 400, { + error: { message: parsed.errors.join("; "), type: "invalid_request_error" }, + }); + return true; + } + try { + await updateSpriteCoreConfig((cfg) => { + const agents = { ...cfg.agents }; + const agent = agents[target.agentId]; + if (!agent) { + throw new AgentNotFoundError(target.agentId); + } + agents[target.agentId] = { + ...agent, + emotions: { + ...agent.emotions, + [target.state]: parsed.value, + }, + }; + return { ...cfg, agents }; + }); + sendJson(res, 200, { ok: true }); + } catch (err) { + respondMutationError(res, err); + } + return true; + } + + // target.kind === "agent" + const parsed = validateAgentEntry(body); + if (!parsed.ok) { + sendJson(res, 400, { + error: { message: parsed.errors.join("; "), type: "invalid_request_error" }, + }); + return true; + } + try { + await updateSpriteCoreConfig((cfg) => { + const agents = { ...cfg.agents }; + agents[target.agentId] = parsed.value; + return { ...cfg, agents }; + }); + sendJson(res, 200, { ok: true }); + } catch (err) { + respondMutationError(res, err); + } + return true; +} + +class AgentNotFoundError extends Error { + constructor(agentId: string) { + super(`unknown agent: ${agentId}`); + this.name = "AgentNotFoundError"; + } +} + +function respondMutationError(res: ServerResponse, err: unknown): void { + if (err instanceof AgentNotFoundError) { + sendJson(res, 404, { + error: { message: err.message, type: "invalid_request_error" }, + }); + return; + } + const message = err instanceof Error ? err.message : String(err); + sendJson(res, 500, { + error: { message: `config write failed: ${message}`, type: "invalid_request_error" }, + }); +} diff --git a/src/assets-route.test.ts b/packages/plugin/src/assets-route.test.ts similarity index 100% rename from src/assets-route.test.ts rename to packages/plugin/src/assets-route.test.ts diff --git a/src/assets-route.ts b/packages/plugin/src/assets-route.ts similarity index 100% rename from src/assets-route.ts rename to packages/plugin/src/assets-route.ts diff --git a/packages/plugin/src/character-manifest-route.ts b/packages/plugin/src/character-manifest-route.ts new file mode 100644 index 0000000..ffacd34 --- /dev/null +++ b/packages/plugin/src/character-manifest-route.ts @@ -0,0 +1,74 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { buildCharacterManifest } from "./character-manifest.js"; +import { sendJson, sendMethodNotAllowed } from "./http-helpers.js"; +import type { SpriteCoreConfig } from "./types.js"; + +export const CHARACTER_MANIFEST_ROUTE_PATH = "/sprite-core/character-manifest"; + +export type CharacterManifestRouteOptions = { + readPluginConfig: () => SpriteCoreConfig | undefined; +}; + +/** + * `GET /sprite-core/character-manifest?agentId=[&mode=]` — HTTP + * sibling of the `node.getCharacterManifest` gateway RPC. Exposes the same + * manifest the phone/watch clients already fetch over WebSocket, but on the + * HTTP plane so the plugin's browser UI can drive the SDK's `AssetSource` + * without speaking the gateway WebSocket protocol. + * + * Response matches `NodeGetCharacterManifestResult`: `{ manifest, revision }`. + * Registered with `auth: "gateway"` by the caller. + */ +export async function handleCharacterManifestRequest( + req: IncomingMessage, + res: ServerResponse, + opts: CharacterManifestRouteOptions, +): Promise { + const urlRaw = req.url; + if (!urlRaw) { + return false; + } + let url: URL; + try { + url = new URL(urlRaw, "http://localhost"); + } catch { + return false; + } + if (url.pathname !== CHARACTER_MANIFEST_ROUTE_PATH) { + return false; + } + + if (req.method !== "GET") { + sendMethodNotAllowed(res, "GET"); + return true; + } + + const agentId = url.searchParams.get("agentId")?.trim(); + if (!agentId) { + sendJson(res, 400, { + error: { message: "agentId required", type: "invalid_request_error" }, + }); + return true; + } + + const modeParam = url.searchParams.get("mode"); + const modes = modeParam ? [modeParam] : undefined; + + const result = await buildCharacterManifest({ + pluginConfig: opts.readPluginConfig(), + agentId, + modes, + }); + if (!result.ok) { + const status = result.code === "unknown-agent" ? 404 : 503; + sendJson(res, status, { + error: { message: result.message, type: "invalid_request_error" }, + }); + return true; + } + sendJson(res, 200, { + manifest: result.manifest, + revision: result.revision, + }); + return true; +} diff --git a/src/character-manifest.test.ts b/packages/plugin/src/character-manifest.test.ts similarity index 100% rename from src/character-manifest.test.ts rename to packages/plugin/src/character-manifest.test.ts diff --git a/src/character-manifest.ts b/packages/plugin/src/character-manifest.ts similarity index 100% rename from src/character-manifest.ts rename to packages/plugin/src/character-manifest.ts diff --git a/packages/plugin/src/config-writes.ts b/packages/plugin/src/config-writes.ts new file mode 100644 index 0000000..1e851c2 --- /dev/null +++ b/packages/plugin/src/config-writes.ts @@ -0,0 +1,68 @@ +import { + readConfigFileSnapshotForWrite, + writeConfigFile, +} from "openclaw/plugin-sdk/config-runtime"; +import type { SpriteCoreConfig } from "./types.js"; + +const SPRITE_CORE_PLUGIN_ID = "sprite-core"; + +// Serial write lock. Two concurrent PUTs to different agent fields can still +// clobber each other because each reads the full snapshot, mutates its slice, +// then writes. Serializing through this queue makes the read-modify-write +// atomic at the plugin layer. (Openclaw itself doesn't guarantee cross-write +// ordering.) +let writeChain: Promise = Promise.resolve(); + +export type SpriteCoreConfigMutator = (current: SpriteCoreConfig) => SpriteCoreConfig; + +/** + * Read the live config, project our plugin slice through [mutate], and write + * back using the SDK's read-for-write snapshot. Pairing the read snapshot's + * `writeOptions` with the write is essential — it carries the env-snapshot + * needed to re-preserve `${VAR}` interpolations. Reading with `loadConfig()` + * and writing with `writeConfigFile(...)` loses that and re-persists secrets + * as plaintext. + * + * Stays strictly inside `plugins.entries["sprite-core"].config`. Other plugin + * slices, channel config, provider config, etc. are never touched. + */ +export async function updateSpriteCoreConfig(mutate: SpriteCoreConfigMutator): Promise { + const run = async () => { + const { snapshot, writeOptions } = await readConfigFileSnapshotForWrite(); + // Mutate `resolved`, not `config`. `resolved` is the post-include, + // post-${VAR} form without runtime defaults baked in — that's what the + // write path expects so it can re-wrap env substitutions. Using the + // runtime form would freeze defaults into the persisted file. + const cfg = snapshot.resolved; + const plugins = cfg.plugins ?? {}; + const entries = plugins.entries ?? {}; + const ourEntry = entries[SPRITE_CORE_PLUGIN_ID] ?? {}; + const currentPluginCfg = (ourEntry.config ?? {}) as SpriteCoreConfig; + + const nextPluginCfg = mutate(currentPluginCfg); + + const nextCfg = { + ...cfg, + plugins: { + ...plugins, + entries: { + ...entries, + [SPRITE_CORE_PLUGIN_ID]: { + ...ourEntry, + // SpriteCoreConfig has named optional fields; the slot type wants + // Record. They're structurally compatible but + // TS rejects the direct assignment; cast through unknown. + config: nextPluginCfg as unknown as Record, + }, + }, + }, + }; + + await writeConfigFile(nextCfg, writeOptions); + }; + const next = writeChain.then(run, run); + writeChain = next.catch(() => { + /* swallow so the chain doesn't get wedged by a single failure */ + }); + await next; +} diff --git a/src/http-helpers.ts b/packages/plugin/src/http-helpers.ts similarity index 100% rename from src/http-helpers.ts rename to packages/plugin/src/http-helpers.ts diff --git a/src/prompting.test.ts b/packages/plugin/src/prompting.test.ts similarity index 100% rename from src/prompting.test.ts rename to packages/plugin/src/prompting.test.ts diff --git a/src/prompting.ts b/packages/plugin/src/prompting.ts similarity index 100% rename from src/prompting.ts rename to packages/plugin/src/prompting.ts diff --git a/src/provider-auth.test.ts b/packages/plugin/src/provider-auth.test.ts similarity index 100% rename from src/provider-auth.test.ts rename to packages/plugin/src/provider-auth.test.ts diff --git a/src/provider-auth.ts b/packages/plugin/src/provider-auth.ts similarity index 100% rename from src/provider-auth.ts rename to packages/plugin/src/provider-auth.ts diff --git a/src/stt-route.test.ts b/packages/plugin/src/stt-route.test.ts similarity index 100% rename from src/stt-route.test.ts rename to packages/plugin/src/stt-route.test.ts diff --git a/src/stt-route.ts b/packages/plugin/src/stt-route.ts similarity index 100% rename from src/stt-route.ts rename to packages/plugin/src/stt-route.ts diff --git a/src/tts-route.test.ts b/packages/plugin/src/tts-route.test.ts similarity index 100% rename from src/tts-route.test.ts rename to packages/plugin/src/tts-route.test.ts diff --git a/src/tts-route.ts b/packages/plugin/src/tts-route.ts similarity index 100% rename from src/tts-route.ts rename to packages/plugin/src/tts-route.ts diff --git a/src/types.ts b/packages/plugin/src/types.ts similarity index 100% rename from src/types.ts rename to packages/plugin/src/types.ts diff --git a/packages/plugin/src/ui-route.ts b/packages/plugin/src/ui-route.ts new file mode 100644 index 0000000..f0a8bcf --- /dev/null +++ b/packages/plugin/src/ui-route.ts @@ -0,0 +1,186 @@ +import { createReadStream } from "node:fs"; +import fs from "node:fs/promises"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { sendJson, sendMethodNotAllowed } from "./http-helpers.js"; + +export const UI_ROUTE_PATH = "/sprite-core/ui"; + +const MIME_BY_EXT: Record = { + ".html": "text/html; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".mjs": "application/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".map": "application/json; charset=utf-8", + ".svg": "image/svg+xml", + ".png": "image/png", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".woff": "font/woff", + ".woff2": "font/woff2", +}; + +// UI bundle ships at `/ui-dist/`. The plugin is consumed in two +// layouts: +// 1. Source / workspace: src/ui-route.ts sits next to ../ui-dist/ +// 2. Bundled (tsdown): index.js is the whole plugin, with ui-dist/ +// as a sibling at the same level +// We accept either — probe both candidates and pick the one that exists. +import fsSync from "node:fs"; + +function resolveUiDistDir(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + const candidates = [path.resolve(here, "..", "ui-dist"), path.resolve(here, "ui-dist")]; + for (const c of candidates) { + if (fsSync.existsSync(path.join(c, "index.html"))) { + return c; + } + } + // Fall back to the source-layout path; the route handler will emit a 503 + // pointing at the build step if the bundle really isn't there. + return candidates[0] ?? path.resolve(here, "..", "ui-dist"); +} + +const UI_DIST_DIR = resolveUiDistDir(); + +type ValidatedPath = + | { ok: true; absPath: string; stat: { size: number; mtimeMs: number } } + | { ok: false; status: number; message: string }; + +async function resolveRequestedFile(relPath: string): Promise { + const safe = relPath.replace(/^\/+/, ""); + if (safe.includes("\0")) { + return { ok: false, status: 400, message: "invalid path" }; + } + const absPath = path.resolve(UI_DIST_DIR, safe); + const rel = path.relative(UI_DIST_DIR, absPath); + if (rel.startsWith("..") || path.isAbsolute(rel)) { + return { ok: false, status: 403, message: "path traversal rejected" }; + } + try { + const stat = await fs.stat(absPath); + if (!stat.isFile()) { + return { ok: false, status: 404, message: "not a file" }; + } + return { ok: true, absPath, stat: { size: stat.size, mtimeMs: stat.mtimeMs } }; + } catch { + return { ok: false, status: 404, message: "not found" }; + } +} + +function buildEtag(stat: { size: number; mtimeMs: number }): string { + return `"${stat.mtimeMs.toString(36)}-${stat.size.toString(36)}"`; +} + +function isHashedAsset(relPath: string): boolean { + // Vite emits hashed filenames under /assets/ — cache those for a year. + // index.html and anything unhashed gets a no-cache policy. + return relPath.startsWith("assets/") || /[-.][0-9a-f]{8,}\.[a-z0-9]+$/i.test(relPath); +} + +/** + * `GET /sprite-core/ui[/path]` — serves the built UI bundle from + * `packages/plugin/ui-dist/`. Unknown paths fall back to `index.html` so + * client-side routing (if we add it later) works. Returns 404 when the + * bundle hasn't been built yet — the error message points at the build step. + * + * Registered as a prefix route with `auth: "plugin"` — the static HTML + + * JS bundle must be servable to a fresh browser so the SPA can bootstrap. + * The SPA then makes same-origin API calls to `/sprite-core/*` endpoints + * that stay gateway-gated. Registering this route with `auth: "gateway"` + * would 401 the HTML shell itself and block the bundle from loading. + */ +export async function handleUiRequest( + req: IncomingMessage, + res: ServerResponse, +): Promise { + const urlRaw = req.url; + if (!urlRaw) { + return false; + } + let url: URL; + try { + url = new URL(urlRaw, "http://localhost"); + } catch { + return false; + } + if (url.pathname !== UI_ROUTE_PATH && !url.pathname.startsWith(`${UI_ROUTE_PATH}/`)) { + return false; + } + + if (req.method !== "GET" && req.method !== "HEAD") { + sendMethodNotAllowed(res, "GET, HEAD"); + return true; + } + + // Strip the route prefix; treat the rest as a path into ui-dist. + const afterPrefix = url.pathname.slice(UI_ROUTE_PATH.length).replace(/^\//, ""); + const relPath = afterPrefix || "index.html"; + + let validated = await resolveRequestedFile(relPath); + // SPA fallback: any unknown path that doesn't have a file extension gets + // index.html so client-side routing can handle it. Paths with extensions + // get a real 404 — a missing `app.123.js` is not an SPA route. + if (!validated.ok && validated.status === 404 && !path.extname(relPath)) { + validated = await resolveRequestedFile("index.html"); + } + + if (!validated.ok) { + // If index.html itself is missing, the bundle hasn't been built. + if (relPath === "index.html" || (!path.extname(relPath) && validated.status === 404)) { + sendJson(res, 503, { + error: { + message: + "SpriteCore UI bundle not built. Run `pnpm --filter @tyler-rng/sprite-core-ui build`.", + type: "invalid_request_error", + }, + }); + return true; + } + sendJson(res, validated.status, { + error: { message: validated.message, type: "invalid_request_error" }, + }); + return true; + } + + const ext = path.extname(validated.absPath).toLowerCase(); + const contentType = MIME_BY_EXT[ext] ?? "application/octet-stream"; + const etag = buildEtag(validated.stat); + + if (isHashedAsset(relPath)) { + res.setHeader("Cache-Control", "public, max-age=31536000, immutable"); + } else { + res.setHeader("Cache-Control", "no-cache"); + } + res.setHeader("ETag", etag); + + if (req.headers["if-none-match"] === etag) { + res.statusCode = 304; + res.end(); + return true; + } + + res.statusCode = 200; + res.setHeader("Content-Type", contentType); + res.setHeader("Content-Length", String(validated.stat.size)); + + if (req.method === "HEAD") { + res.end(); + return true; + } + + const stream = createReadStream(validated.absPath); + stream.once("error", () => { + if (!res.headersSent) { + sendJson(res, 500, { + error: { message: "read error", type: "invalid_request_error" }, + }); + return; + } + res.destroy(); + }); + stream.pipe(res); + return true; +} diff --git a/packages/plugin/src/validation.ts b/packages/plugin/src/validation.ts new file mode 100644 index 0000000..db2200f --- /dev/null +++ b/packages/plugin/src/validation.ts @@ -0,0 +1,272 @@ +import type { + SpriteCoreAgentEntry, + SpriteCoreAvatarConfig, + SpriteCoreEmotionDirective, + SpriteCoreEmotionEntry, + SpriteCoreVoiceConfig, + SpriteCorePromptingConfig, +} from "./types.js"; + +// Narrow, hand-rolled validators for the write-side payloads. Mirrors the +// plugin's openclaw.plugin.json configSchema shapes but only the subset we +// accept over HTTP: full AgentEntry and single EmotionEntry. Kept dep-free +// so the plugin bundle doesn't grow an ajv/zod runtime. +// +// Returns the canonicalized value on success (unknown keys stripped) and a +// list of path-qualified error messages on failure. + +export type ValidationResult = + | { ok: true; value: T } + | { ok: false; errors: string[] }; + +export function validateEmotionEntry(input: unknown): ValidationResult { + const errors: string[] = []; + const obj = asObject(input, "emotion", errors); + if (!obj) { + return { ok: false, errors }; + } + + const description = asString(obj["description"], "emotion.description", errors, { required: true }); + let directive: SpriteCoreEmotionDirective | undefined; + if (obj["directive"] !== undefined) { + directive = validateDirective(obj["directive"], "emotion.directive", errors); + } + + if (errors.length > 0) { + return { ok: false, errors }; + } + return { + ok: true, + value: { + description: description ?? "", + ...(directive ? { directive } : {}), + }, + }; +} + +export function validateAgentEntry(input: unknown): ValidationResult { + const errors: string[] = []; + const obj = asObject(input, "agent", errors); + if (!obj) { + return { ok: false, errors }; + } + + const out: SpriteCoreAgentEntry = {}; + if (obj["avatar"] !== undefined) { + const avatar = validateAvatar(obj["avatar"], "agent.avatar", errors); + if (avatar) { + out.avatar = avatar; + } + } + if (obj["voice"] !== undefined) { + const voice = validateVoice(obj["voice"], "agent.voice", errors); + if (voice) { + out.voice = voice; + } + } + if (obj["prompting"] !== undefined) { + const prompting = validatePrompting(obj["prompting"], "agent.prompting", errors); + if (prompting) { + out.prompting = prompting; + } + } + if (obj["emotions"] !== undefined) { + const emotions = validateEmotionsMap(obj["emotions"], "agent.emotions", errors); + if (emotions) { + out.emotions = emotions; + } + } + + if (errors.length > 0) { + return { ok: false, errors }; + } + return { ok: true, value: out }; +} + +// --- helpers --- + +function asObject(v: unknown, path: string, errors: string[]): Record | null { + if (v === null || typeof v !== "object" || Array.isArray(v)) { + errors.push(`${path}: expected object`); + return null; + } + return v as Record; +} + +function asString( + v: unknown, + path: string, + errors: string[], + opts: { required?: boolean } = {}, +): string | undefined { + if (v === undefined) { + if (opts.required) { + errors.push(`${path}: required`); + } + return undefined; + } + if (typeof v !== "string") { + errors.push(`${path}: expected string`); + return undefined; + } + return v; +} + +function asNumberInRange( + v: unknown, + path: string, + errors: string[], + min: number, + max: number, +): number | undefined { + if (v === undefined) { + return undefined; + } + if (typeof v !== "number" || !Number.isFinite(v)) { + errors.push(`${path}: expected finite number`); + return undefined; + } + if (v < min || v > max) { + errors.push(`${path}: expected ${min}..${max}, got ${v}`); + return undefined; + } + return v; +} + +function asBool(v: unknown, path: string, errors: string[]): boolean | undefined { + if (v === undefined) { + return undefined; + } + if (typeof v !== "boolean") { + errors.push(`${path}: expected boolean`); + return undefined; + } + return v; +} + +function validateDirective( + input: unknown, + path: string, + errors: string[], +): SpriteCoreEmotionDirective | undefined { + const obj = asObject(input, path, errors); + if (!obj) { + return undefined; + } + const out: SpriteCoreEmotionDirective = {}; + const voiceId = asString(obj["voiceId"], `${path}.voiceId`, errors); + if (voiceId !== undefined) { + out.voiceId = voiceId; + } + const stability = asNumberInRange(obj["stability"], `${path}.stability`, errors, 0, 1); + if (stability !== undefined) { + out.stability = stability; + } + const similarity = asNumberInRange(obj["similarity"], `${path}.similarity`, errors, 0, 1); + if (similarity !== undefined) { + out.similarity = similarity; + } + const style = asNumberInRange(obj["style"], `${path}.style`, errors, 0, 1); + if (style !== undefined) { + out.style = style; + } + const speakerBoost = asBool(obj["speakerBoost"], `${path}.speakerBoost`, errors); + if (speakerBoost !== undefined) { + out.speakerBoost = speakerBoost; + } + const speed = asNumberInRange(obj["speed"], `${path}.speed`, errors, 0.25, 4); + if (speed !== undefined) { + out.speed = speed; + } + const audioTag = asString(obj["audioTag"], `${path}.audioTag`, errors); + if (audioTag !== undefined) { + out.audioTag = audioTag; + } + return out; +} + +function validateAvatar( + input: unknown, + path: string, + errors: string[], +): SpriteCoreAvatarConfig | undefined { + // Avatar shapes are operator territory today — accept the raw object if it + // has a known `kind`, otherwise reject. We don't deep-validate the nested + // sprite/atlas shapes here; the next `loadConfig()` call runs openclaw's + // full JSON-schema validator and rejects malformed branches at load. + const obj = asObject(input, path, errors); + if (!obj) { + return undefined; + } + const kind = obj["kind"]; + if (kind !== "states" && kind !== "sprites" && kind !== "atlas") { + errors.push(`${path}.kind: expected "states" | "sprites" | "atlas"`); + return undefined; + } + return obj as unknown as SpriteCoreAvatarConfig; +} + +function validateVoice( + input: unknown, + path: string, + errors: string[], +): SpriteCoreVoiceConfig | undefined { + const obj = asObject(input, path, errors); + if (!obj) { + return undefined; + } + return obj as SpriteCoreVoiceConfig; +} + +function validatePrompting( + input: unknown, + path: string, + errors: string[], +): SpriteCorePromptingConfig | undefined { + const obj = asObject(input, path, errors); + if (!obj) { + return undefined; + } + const out: SpriteCorePromptingConfig = {}; + if (obj["descriptions"] !== undefined) { + const descObj = asObject(obj["descriptions"], `${path}.descriptions`, errors); + if (descObj) { + const descriptions: Record = {}; + for (const [k, v] of Object.entries(descObj)) { + const s = asString(v, `${path}.descriptions.${k}`, errors); + if (s !== undefined) { + descriptions[k] = s; + } + } + out.descriptions = descriptions; + } + } + const instruction = asString(obj["instruction"], `${path}.instruction`, errors); + if (instruction !== undefined) { + out.instruction = instruction; + } + return out; +} + +function validateEmotionsMap( + input: unknown, + path: string, + errors: string[], +): Record | undefined { + const obj = asObject(input, path, errors); + if (!obj) { + return undefined; + } + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + const r = validateEmotionEntry(v); + if (!r.ok) { + for (const e of r.errors) { + errors.push(e.replace(/^emotion/, `${path}.${k}`)); + } + continue; + } + out[k] = r.value; + } + return out; +} diff --git a/template/agent/README.md b/packages/plugin/template/agent/README.md similarity index 100% rename from template/agent/README.md rename to packages/plugin/template/agent/README.md diff --git a/template/agent/agent.atlas.json b/packages/plugin/template/agent/agent.atlas.json similarity index 100% rename from template/agent/agent.atlas.json rename to packages/plugin/template/agent/agent.atlas.json diff --git a/template/agent/agent.atlas.webp b/packages/plugin/template/agent/agent.atlas.webp similarity index 100% rename from template/agent/agent.atlas.webp rename to packages/plugin/template/agent/agent.atlas.webp diff --git a/template/agent/regenerate-placeholder-atlas.mjs b/packages/plugin/template/agent/regenerate-placeholder-atlas.mjs similarity index 100% rename from template/agent/regenerate-placeholder-atlas.mjs rename to packages/plugin/template/agent/regenerate-placeholder-atlas.mjs diff --git a/packages/plugin/tsconfig.json b/packages/plugin/tsconfig.json new file mode 100644 index 0000000..09f33b5 --- /dev/null +++ b/packages/plugin/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "allowImportingTsExtensions": false, + "resolveJsonModule": true, + "strict": true, + "noImplicitOverride": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "isolatedModules": true, + "declaration": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "." + }, + "include": ["./*.ts", "./src/**/*.ts"], + "exclude": [ + "./**/*.test.ts", + "./dist/**", + "./node_modules/**" + ] +} diff --git a/packages/plugin/ui/index.html b/packages/plugin/ui/index.html new file mode 100644 index 0000000..16c1161 --- /dev/null +++ b/packages/plugin/ui/index.html @@ -0,0 +1,13 @@ + + + + + + + SpriteCore + + +

+ + + diff --git a/packages/plugin/ui/package.json b/packages/plugin/ui/package.json new file mode 100644 index 0000000..e0033db --- /dev/null +++ b/packages/plugin/ui/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tyler-rng/sprite-core-ui", + "version": "1.0.0", + "description": "Dashboard UI for the SpriteCore plugin — browser SPA served by the plugin at /sprite-core/ui", + "type": "module", + "private": true, + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "typecheck": "tsc -b --noEmit", + "preview": "vite preview" + }, + "dependencies": { + "@tanstack/react-query": "^5.51.0", + "@tyler-rng/sprite-core-client": "workspace:*", + "@tyler-rng/sprite-core-schema": "workspace:*", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^4.3.1", + "typescript": "^5.6.0", + "vite": "^5.4.0" + } +} diff --git a/packages/plugin/ui/src/App.tsx b/packages/plugin/ui/src/App.tsx new file mode 100644 index 0000000..3b1e279 --- /dev/null +++ b/packages/plugin/ui/src/App.tsx @@ -0,0 +1,174 @@ +import { useMemo, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { getAgents } from "./api/client.js"; +import { + clearOverrideToken, + MissingAuthTokenError, + readOverrideToken, + setOverrideToken, +} from "./api/auth.js"; +import { EmotionEditor } from "./components/EmotionEditor.js"; +import { AvatarPreview } from "./preview/AvatarPreview.js"; +import type { AgentEntry } from "./api/types.js"; + +export function App(): JSX.Element { + const agentsQuery = useQuery({ + queryKey: ["agents"], + queryFn: ({ signal }) => getAgents(signal), + retry: false, + }); + const [selectedAgentId, setSelectedAgentId] = useState(null); + + const agents = agentsQuery.data?.agents ?? {}; + const agentIds = useMemo(() => Object.keys(agents).sort(), [agents]); + const activeId = selectedAgentId && agents[selectedAgentId] ? selectedAgentId : agentIds[0] ?? null; + const active: AgentEntry | null = activeId ? agents[activeId] ?? null : null; + + const error = agentsQuery.error; + const needsToken = + error instanceof MissingAuthTokenError || + (error instanceof Error && /401|unauthorized/i.test(error.message)); + + return ( +
+
+

SpriteCore

+ dashboard · plugin UI +
+ +
+ {needsToken ? ( + + ) : active && activeId ? ( + + ) : ( +
Select an agent to begin.
+ )} +
+
+ ); +} + +function TokenPrompt({ error }: { error: Error }): JSX.Element { + const qc = useQueryClient(); + const [draft, setDraft] = useState(() => readOverrideToken() ?? ""); + const existing = readOverrideToken(); + return ( +
+

Sign-in needed

+

+ The dashboard couldn't find a Control UI auth token in your browser + storage on this origin, so every API call returns 401. Paste your + gateway token below — you'll find it in your{" "} + ~/.openclaw/openclaw.json at gateway.auth.token, + or in the OpenClaw Control UI under Settings → Gateway. The token is + kept in localStorage on this device only. +

+
+ setDraft(e.target.value)} + placeholder="gateway token" + spellCheck={false} + autoComplete="off" + style={{ fontFamily: "monospace" }} + /> +
+ + {existing && ( + + )} +
+
+ {error.message} +
+
+
+ ); +} + +function AgentPane({ agentId, agent }: { agentId: string; agent: AgentEntry }): JSX.Element { + const emotionKeys = useMemo(() => { + const fromEmotions = Object.keys(agent.emotions ?? {}); + const fromAvatar = avatarStates(agent); + // Union: everything that has a description OR is an avatar state, sorted. + return Array.from(new Set([...fromEmotions, ...fromAvatar])).sort(); + }, [agent]); + + return ( + <> +
+

{agentId}

+ {emotionKeys.length === 0 && ( +
+
+ No avatar states or emotions configured for this agent. +
+
+ )} + {emotionKeys.map((state) => ( + + ))} +
+ + + ); +} + +function avatarStates(agent: AgentEntry): string[] { + const avatar = agent.avatar; + if (!avatar) return []; + if (avatar.kind === "states") return Object.keys(avatar.states); + if (avatar.kind === "sprites") return Object.keys(avatar.states); + if (avatar.kind === "atlas") return Object.keys(avatar.descriptions ?? {}); + return []; +} diff --git a/packages/plugin/ui/src/api/auth.ts b/packages/plugin/ui/src/api/auth.ts new file mode 100644 index 0000000..6426391 --- /dev/null +++ b/packages/plugin/ui/src/api/auth.ts @@ -0,0 +1,136 @@ +// The OpenClaw gateway authenticates plugin HTTP routes via +// `Authorization: Bearer `. The Control UI persists its device-bound +// auth on this origin under one of two known shapes; we read whichever is +// present and attach it to our same-origin fetches. +// +// Shape 1 — device auth (current): localStorage["openclaw.device.auth.v1"] +// JSON: { version: 1, deviceId: string, tokens: { [role]: string } } +// Roles seen in the wild: "user", "assistant", "system", "unknown". +// Any non-empty token in `tokens` is a valid Bearer for /sprite-core/*. +// +// Shape 2 — legacy per-tenant token: localStorage["openclaw.control.token.v1:"] +// value is the raw token string. +// +// If neither is present, the user has not signed in to the Control UI on +// this origin; the SPA cannot bootstrap auth on its own. + +const DEVICE_AUTH_KEY = "openclaw.device.auth.v1"; +const V1_PREFIX = "openclaw.control.token.v1:"; +const V1_LEGACY = "openclaw.control.token.v1"; +const ROLE_PREFERENCE = ["user", "assistant", "system", "unknown"]; +// Local override — set by the dashboard's "paste your token" UI when none of +// the Control UI's storage shapes yields a usable token. +const OVERRIDE_KEY = "sprite-core.dashboard.gatewayToken.v1"; + +export function setOverrideToken(token: string): void { + if (typeof localStorage === "undefined") return; + const t = token.trim(); + if (!t) { + localStorage.removeItem(OVERRIDE_KEY); + return; + } + localStorage.setItem(OVERRIDE_KEY, t); +} + +export function clearOverrideToken(): void { + if (typeof localStorage === "undefined") return; + localStorage.removeItem(OVERRIDE_KEY); +} + +export function readOverrideToken(): string | null { + if (typeof localStorage === "undefined") return null; + const v = (localStorage.getItem(OVERRIDE_KEY) ?? "").trim(); + return v || null; +} + +type DeviceAuthStore = { + version: number; + deviceId: string; + tokens: Record; +}; + +function readDeviceAuth(store: Storage): string | null { + const raw = store.getItem(DEVICE_AUTH_KEY); + if (!raw) return null; + let parsed: DeviceAuthStore; + try { + parsed = JSON.parse(raw) as DeviceAuthStore; + } catch { + return null; + } + if (!parsed || parsed.version !== 1 || !parsed.tokens) return null; + for (const role of ROLE_PREFERENCE) { + const t = parsed.tokens[role]; + if (typeof t === "string" && t.trim().length > 0) return t.trim(); + } + // Any other role we didn't anticipate. + for (const t of Object.values(parsed.tokens)) { + if (typeof t === "string" && t.trim().length > 0) return t.trim(); + } + return null; +} + +function readPerTenantToken(store: Storage): string | null { + for (let i = 0; i < store.length; i++) { + const key = store.key(i); + if (!key) continue; + if (key !== V1_LEGACY && !key.startsWith(V1_PREFIX)) continue; + const value = (store.getItem(key) ?? "").trim(); + if (value) return value; + } + return null; +} + +function listOpenclawKeys(store: Storage, name: string): string[] { + const out: string[] = []; + for (let i = 0; i < store.length; i++) { + const key = store.key(i); + if (key && key.startsWith("openclaw.")) out.push(`${name}:${key}`); + } + return out; +} + +type TokenScan = { token: string | null; openclawKeys: string[] }; + +function scan(): TokenScan { + // 1. Manual override (set via the dashboard's paste-token UI) wins. + const override = readOverrideToken(); + if (override) return { token: override, openclawKeys: [] }; + + // 2. Try anything the Control UI may have persisted. + const stores: Array<[Storage, string]> = []; + if (typeof localStorage !== "undefined") stores.push([localStorage, "local"]); + if (typeof sessionStorage !== "undefined") stores.push([sessionStorage, "session"]); + + for (const [store] of stores) { + const t = readDeviceAuth(store) ?? readPerTenantToken(store); + if (t) return { token: t, openclawKeys: [] }; + } + const openclawKeys: string[] = []; + for (const [store, name] of stores) openclawKeys.push(...listOpenclawKeys(store, name)); + return { token: null, openclawKeys }; +} + +export class MissingAuthTokenError extends Error { + readonly openclawKeys: readonly string[]; + constructor(openclawKeys: readonly string[]) { + const hint = + openclawKeys.length === 0 + ? "No openclaw.* keys were found in localStorage on this origin. " + + "Open the OpenClaw Control UI in another tab (same origin), sign in, then reload." + : `Found openclaw.* keys but no usable token: ${openclawKeys.join(", ")}.`; + super(`Sprite dashboard could not find the Control UI auth token. ${hint}`); + this.name = "MissingAuthTokenError"; + this.openclawKeys = openclawKeys; + } +} + +export function readAuthToken(): string | null { + return scan().token; +} + +export function authHeader(): Record { + const s = scan(); + if (!s.token) throw new MissingAuthTokenError(s.openclawKeys); + return { Authorization: `Bearer ${s.token}` }; +} diff --git a/packages/plugin/ui/src/api/client.ts b/packages/plugin/ui/src/api/client.ts new file mode 100644 index 0000000..ed50160 --- /dev/null +++ b/packages/plugin/ui/src/api/client.ts @@ -0,0 +1,62 @@ +import { authHeader } from "./auth.js"; +import type { AgentEntry, AgentsResponse, EmotionEntry } from "./types.js"; + +// The UI is served from the same origin as the plugin routes, so relative +// paths "just work" under `/sprite-core/ui/`. In Vite dev the config proxies +// `/sprite-core/*` to a gateway URL, also matching this base. +const BASE = "/sprite-core"; + +async function parseJson(res: Response): Promise { + const text = await res.text(); + if (!res.ok) { + let message = `${res.status} ${res.statusText}`; + try { + const body = JSON.parse(text); + if (body?.error?.message) { + message = body.error.message; + } + } catch { + if (text) { + message = text; + } + } + throw new Error(message); + } + return text ? (JSON.parse(text) as T) : (undefined as T); +} + +export async function getAgents(signal?: AbortSignal): Promise { + const res = await fetch(`${BASE}/agents`, { + signal, + credentials: "same-origin", + headers: { ...authHeader() }, + }); + return parseJson(res); +} + +export async function putAgent(agentId: string, entry: AgentEntry): Promise { + const res = await fetch(`${BASE}/agents/${encodeURIComponent(agentId)}`, { + method: "PUT", + credentials: "same-origin", + headers: { "Content-Type": "application/json", ...authHeader() }, + body: JSON.stringify(entry), + }); + await parseJson<{ ok: true }>(res); +} + +export async function putEmotion( + agentId: string, + state: string, + entry: EmotionEntry, +): Promise { + const res = await fetch( + `${BASE}/agents/${encodeURIComponent(agentId)}/emotions/${encodeURIComponent(state)}`, + { + method: "PUT", + credentials: "same-origin", + headers: { "Content-Type": "application/json", ...authHeader() }, + body: JSON.stringify(entry), + }, + ); + await parseJson<{ ok: true }>(res); +} diff --git a/packages/plugin/ui/src/api/types.ts b/packages/plugin/ui/src/api/types.ts new file mode 100644 index 0000000..c84b9c1 --- /dev/null +++ b/packages/plugin/ui/src/api/types.ts @@ -0,0 +1,98 @@ +// Mirrors packages/plugin/src/types.ts. Duplicated on purpose: the plugin is +// a server-side package with openclaw imports that don't belong in a browser +// bundle, so we copy only the wire types the UI needs. Keep in sync with the +// source; a lint or test-level sync check is a reasonable future addition. +// +// See: packages/plugin/src/types.ts + +export type AvatarStateEntry = { file: string; description?: string }; +export type AvatarStatesConfig = { + kind: "states"; + default: string; + states: Record; + instruction?: string; +}; + +export type LoopMode = "infinite" | "once" | "ping-pong"; +export type SpriteSequence = { + count: number; + fps?: number; + loop?: LoopMode; + holdLastFrame?: boolean; + iterations?: number; +}; + +export type SpriteStatePhased = { + intro?: SpriteSequence; + loop: SpriteSequence; + outro?: SpriteSequence; + description?: string; +}; + +export type SpriteState = + | (SpriteSequence & { description?: string }) + | SpriteStatePhased; + +export type AvatarTransition = string | { blend: "crossfade"; ms: number }; + +export type AvatarSpritesConfig = { + kind: "sprites"; + default: string; + basePath: string; + format?: "webp" | "png" | "jpg"; + states: Record; + transitions?: Record; + instruction?: string; +}; + +export type AvatarAtlasConfig = { + kind: "atlas"; + default: string; + manifest: string; + descriptions?: Record; + instruction?: string; +}; + +export type AvatarConfig = + | AvatarStatesConfig + | AvatarSpritesConfig + | AvatarAtlasConfig; + +export type VoiceConfig = { + provider?: string; + voiceId?: string; + label?: string; + [key: string]: unknown; +}; + +export type PromptingConfig = { + descriptions?: Record; + instruction?: string; +}; + +export type EmotionDirective = { + voiceId?: string; + stability?: number; + similarity?: number; + style?: number; + speakerBoost?: boolean; + speed?: number; + audioTag?: string; +}; + +export type EmotionEntry = { + description: string; + directive?: EmotionDirective; +}; + +export type AgentEntry = { + avatar?: AvatarConfig; + voice?: VoiceConfig; + prompting?: PromptingConfig; + emotions?: Record; +}; + +export type AgentsResponse = { + agents: Record; + publicBaseUrl?: string; +}; diff --git a/packages/plugin/ui/src/components/EmotionEditor.tsx b/packages/plugin/ui/src/components/EmotionEditor.tsx new file mode 100644 index 0000000..160d40a --- /dev/null +++ b/packages/plugin/ui/src/components/EmotionEditor.tsx @@ -0,0 +1,182 @@ +import { useEffect, useState } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { putEmotion } from "../api/client.js"; +import type { EmotionDirective, EmotionEntry } from "../api/types.js"; + +type Props = { + agentId: string; + stateName: string; + initial: EmotionEntry; +}; + +/** + * Edit one per-state emotion entry. Writes back to: + * PUT /sprite-core/agents/:agentId/emotions/:state + * + * Kept intentionally narrow — one state at a time — so the PUT is tiny and + * the plugin's read-modify-write stays scoped to a single config branch. + */ +export function EmotionEditor({ agentId, stateName, initial }: Props): JSX.Element { + const qc = useQueryClient(); + const [description, setDescription] = useState(initial.description); + const [directive, setDirective] = useState(initial.directive ?? {}); + + useEffect(() => { + setDescription(initial.description); + setDirective(initial.directive ?? {}); + }, [agentId, stateName, initial]); + + const save = useMutation({ + mutationFn: async () => { + const entry: EmotionEntry = { + description, + ...(hasAny(directive) ? { directive } : {}), + }; + await putEmotion(agentId, stateName, entry); + }, + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["agents"] }); + }, + }); + + const dirty = + description !== initial.description || + JSON.stringify(directive) !== JSON.stringify(initial.directive ?? {}); + + return ( +
+

+ emotions.{stateName} +

+
+ +