diff --git a/.well-known/agent-skills/didww-connect.tar.gz b/.well-known/agent-skills/didww-connect.tar.gz new file mode 100644 index 0000000..6ce99f4 Binary files /dev/null and b/.well-known/agent-skills/didww-connect.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-elevenlabs.tar.gz b/.well-known/agent-skills/didww-to-elevenlabs.tar.gz new file mode 100644 index 0000000..e466a4a Binary files /dev/null and b/.well-known/agent-skills/didww-to-elevenlabs.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-livekit.tar.gz b/.well-known/agent-skills/didww-to-livekit.tar.gz new file mode 100644 index 0000000..3f7bfbc Binary files /dev/null and b/.well-known/agent-skills/didww-to-livekit.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-openai-realtime.tar.gz b/.well-known/agent-skills/didww-to-openai-realtime.tar.gz new file mode 100644 index 0000000..029eef0 Binary files /dev/null and b/.well-known/agent-skills/didww-to-openai-realtime.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-retell.tar.gz b/.well-known/agent-skills/didww-to-retell.tar.gz new file mode 100644 index 0000000..6f2adf1 Binary files /dev/null and b/.well-known/agent-skills/didww-to-retell.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-ultravox.tar.gz b/.well-known/agent-skills/didww-to-ultravox.tar.gz new file mode 100644 index 0000000..a2d6029 Binary files /dev/null and b/.well-known/agent-skills/didww-to-ultravox.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-vapi.tar.gz b/.well-known/agent-skills/didww-to-vapi.tar.gz new file mode 100644 index 0000000..df46726 Binary files /dev/null and b/.well-known/agent-skills/didww-to-vapi.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-vogent.tar.gz b/.well-known/agent-skills/didww-to-vogent.tar.gz new file mode 100644 index 0000000..790f557 Binary files /dev/null and b/.well-known/agent-skills/didww-to-vogent.tar.gz differ diff --git a/.well-known/agent-skills/didww-to-xai.tar.gz b/.well-known/agent-skills/didww-to-xai.tar.gz new file mode 100644 index 0000000..48594fd Binary files /dev/null and b/.well-known/agent-skills/didww-to-xai.tar.gz differ diff --git a/.well-known/agent-skills/index.json b/.well-known/agent-skills/index.json new file mode 100644 index 0000000..31b44f1 --- /dev/null +++ b/.well-known/agent-skills/index.json @@ -0,0 +1,99 @@ +{ + "$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json", + "name": "DIDWW Agent Skills", + "description": "Official DIDWW agent skills: put an AI voice agent on a real DIDWW phone number with any major voice AI platform, through the DIDWW MCP (https://api.didww.com/mcp).", + "repository": "https://github.com/didww/ai", + "homepage": "https://www.didww.com/", + "skills": [ + { + "name": "didww-connect", + "type": "archive", + "description": "Connect a DIDWW phone number to ANY AI voice platform: helps choose the platform, then wires the SIP trunk pair and number. Use whenever the user wants a voice AI agent on a DIDWW number and has not committed to a platform, asks which AI platform to use with DIDWW, or names a platform that has no dedicated didww-to-X skill installed. Covers ElevenLabs, Vapi, Retell, Ultravox, Vogent, OpenAI Realtime, xAI, LiveKit and the enterprise-gated platforms.", + "url": "didww-connect.tar.gz", + "digest": "sha256:50a913c53d738eeb409f565143d176998a04edd8008dcbdbf0777a056a52a9f5", + "version": "1.0.2", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-connect.md", + "install": "npx didww-connect" + }, + { + "name": "didww-to-elevenlabs", + "type": "archive", + "description": "Connect a DIDWW phone number to an ElevenLabs voice agent over SIP, as automatically as both platforms allow. Use whenever the user wants an ElevenLabs agent (Agents Platform, Conversational AI) on a real phone number via DIDWW, mentions pairing DIDWW with ElevenLabs, importing a SIP trunk into ElevenLabs, sip.rtc.elevenlabs.io, or debugging calls between the two. Covers inbound, outbound, transfers and verification.", + "url": "didww-to-elevenlabs.tar.gz", + "digest": "sha256:522f41eada5f5e6269f9ea71de7452d13e57d077d11643abb6b222fc50bc1e9c", + "version": "1.1.0", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-elevenlabs.md", + "install": "npx didww-to-elevenlabs" + }, + { + "name": "didww-to-livekit", + "type": "archive", + "description": "Connect a DIDWW phone number to a LiveKit Agents worker over SIP, on LiveKit Cloud or self-hosted: inbound trunk, dispatch rule to a named agent, optional outbound trunk. Use whenever the user wants a LiveKit agent (any STT, LLM and TTS combination, including Claude as the brain) answering a real number via DIDWW, mentions sip.livekit.cloud, SIP inbound trunks or dispatch rules, lk sip commands, agent_name dispatch, or debugging calls between DIDWW and LiveKit. This is the bring-your-own-brain pairing: the agent is code the user runs.", + "url": "didww-to-livekit.tar.gz", + "digest": "sha256:b080e937be6335de5dab427e74584f15d8621445e6bc7969a8b5b24607b2aa62", + "version": "1.1.1", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-livekit.md", + "install": "npx didww-to-livekit" + }, + { + "name": "didww-to-openai-realtime", + "type": "archive", + "description": "Point a DIDWW phone number directly at OpenAI's Realtime API over SIP, with a generated webhook backend for call control. Use whenever the user wants gpt-realtime answering a real phone number via DIDWW, mentions sip.api.openai.com, realtime.call.incoming webhooks, Realtime SIP, or debugging calls between DIDWW and OpenAI. This is the developer-grade pairing: the agent behavior is code the user owns.", + "url": "didww-to-openai-realtime.tar.gz", + "digest": "sha256:78f33de78d5aa29d849480e4df51c43802d738350a1d0403be725c808a190c25", + "version": "1.1.0", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-openai-realtime.md", + "install": "npx didww-to-openai-realtime" + }, + { + "name": "didww-to-retell", + "type": "archive", + "description": "Connect a DIDWW phone number to a Retell AI agent over custom telephony, with full MCP automation: Retell's official MCP can create agents AND import numbers. Use whenever the user wants a Retell agent on a real phone number via DIDWW, mentions pairing DIDWW with Retell, custom telephony, sip.retellai.com, import-phone-number, or debugging calls between the two.", + "url": "didww-to-retell.tar.gz", + "digest": "sha256:fe40bae473df088d22c28343df1fab04b86d1b96a250a922cf7e5aa93e82c1b0", + "version": "1.0.2", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-retell.md", + "install": "npx didww-to-retell" + }, + { + "name": "didww-to-ultravox", + "type": "archive", + "description": "Connect a DIDWW phone number to an Ultravox Realtime voice agent over SIP. Use whenever the user wants an Ultravox agent on a real phone number via DIDWW, mentions pairing DIDWW with Ultravox, an account SIP domain, agent_ addresses, allowedCidrRanges, or debugging calls between the two. Covers the address-based routing model that makes Ultravox different from every number-import platform.", + "url": "didww-to-ultravox.tar.gz", + "digest": "sha256:fbf193be6f46d53965c42f9f086c8b6752885e7cb9497b3708df5bca6ed412e5", + "version": "1.1.0", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-ultravox.md", + "install": "npx didww-to-ultravox" + }, + { + "name": "didww-to-vapi", + "type": "archive", + "description": "Connect a DIDWW phone number to a Vapi assistant over BYO SIP trunking, as automatically as Vapi allows. Use whenever the user wants a Vapi assistant on a real phone number via DIDWW, mentions pairing DIDWW with Vapi, byo-sip-trunk credentials, sip.vapi.ai, or debugging calls between the two. Covers inbound, outbound, transfers and agent binding, following Vapi's and DIDWW's own DIDWW guides.", + "url": "didww-to-vapi.tar.gz", + "digest": "sha256:d624bbb9ffa4f3ad69488a779c42ed85a06964c03539a41763556746581cc582", + "version": "1.1.0", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-vapi.md", + "install": "npx didww-to-vapi" + }, + { + "name": "didww-to-vogent", + "type": "archive", + "description": "Connect a DIDWW phone number to a Vogent voice agent over SIP import. Use whenever the user wants a Vogent agent on a real phone number via DIDWW, mentions pairing DIDWW with Vogent, sip_import numbers, sip.vogent.ai, or debugging calls between the two.", + "url": "didww-to-vogent.tar.gz", + "digest": "sha256:1c2227a36a7909cc0fd6c7ebe496c275070b466086a9e0c18614bd05b4df31bd", + "version": "1.0.2", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-vogent.md", + "install": "npx didww-to-vogent" + }, + { + "name": "didww-to-xai", + "type": "archive", + "description": "Point a DIDWW phone number at xAI's Grok voice over SIP. Use whenever the user wants Grok answering a real phone number via DIDWW, mentions sip.voice.x.ai, xAI realtime calls, registering a phone number with xAI, or debugging calls between DIDWW and Grok. Developer-grade: agent behavior is configured per call over a websocket the user's code owns.", + "url": "didww-to-xai.tar.gz", + "digest": "sha256:7d9fbc775ed9718758b43a9afc93e3b8e020f77780fd9226b4c4dee65f0e6c79", + "version": "1.0.2", + "documentation": "https://github.com/didww/ai/blob/main/docs/didww-to-xai.md", + "install": "npx didww-to-xai" + } + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 59be5f2..bb10552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to the skills in this repository. Each skill is versioned independently in its `package.json`. +## Agent Skills discovery index (2026-10-30) + +- New `.well-known/agent-skills/index.json` in the [Agent Skills discovery](https://github.com/cloudflare/agent-skills-discovery-rfc) format v0.2.0, the same format Telnyx serves at `telnyx.com/.well-known/agent-skills/index.json`. Every skill is a `type: "archive"` entry: a `.tar.gz` with `SKILL.md`, `scripts/`, `references/` and `LICENSE` at the root, and its sha256 `digest`. Extra fields (`version`, `documentation`, `install`) are ignored by clients that do not know them. +- `scripts/build_index.py` generates the folder from `skills/` (descriptions from the SKILL.md frontmatter, versions from `package.json`); archives are rewritten only when their contents change, so digests stay stable. `--check` runs in `scripts/test.sh` and CI. +- `scripts/mirror_index.py` lets didww.com keep its copy current from GitHub on a schedule: it verifies every digest and archive before swapping the new set in, and leaves the live folder untouched on any failure. Handoff for the web team: [`docs/agent-skills-index.md`](docs/agent-skills-index.md). + ## Security and correctness review (2026-10-13) `didww-to-openai-realtime` 1.1.0, `didww-to-vapi` 1.1.0, `didww-to-ultravox` 1.1.0, `didww-to-elevenlabs` 1.1.0, `didww-to-livekit` 1.1.1, and 1.0.2 of `didww-connect`, `didww-to-retell`, `didww-to-vogent` and `didww-to-xai`. Started from an automated review; every finding was reproduced in code and checked against the vendor's own documentation or OpenAPI schema before it was fixed, and the request shapes of all scripts were rechecked against those schemas. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a3f263..2b4c59a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,8 @@ skills// one Claude Code skill = one npm package tests/ request-shape tests against stubbed HTTP; no keys, no network package.json npm name = skill name; files whitelist docs/ pages for doc.didww.com, one per skill -scripts/ family-wide test and publish runners +scripts/ family-wide test and publish runners, the index builder and the website mirror +.well-known/agent-skills/ generated: discovery index.json + .tar.gz per skill (never edit by hand) ``` ## Before you open a pull request @@ -24,6 +25,7 @@ scripts/ family-wide test and publish runners 3. Never commit credentials, account identifiers or real phone numbers. Tests use stubbed HTTP and fake keys. 4. Verify platform facts against the platform's current documentation and update the "verified" date at the end of the skill's `SKILL.md`. 5. If you change a skill's behaviour, update its page in `docs/` in the same change. +6. Run `python3 scripts/build_index.py` and commit `.well-known/` with the change. CI fails while the index or an archive is out of date with `skills/`. ## Adding a platform diff --git a/README.md b/README.md index 1c22a7f..6aaf958 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Open-source tools from [DIDWW](https://www.didww.com/) for building AI voice age | [`skills/`](skills/) | Nine Claude Code agent skills that connect a DIDWW number to an AI voice platform. Each is its own npm package. | | [`docs/`](docs/) | Documentation pages for the skills, written for [doc.didww.com](https://doc.didww.com/). | | [`scripts/`](scripts/) | Family-wide test and publish runners used by maintainers and CI. | +| [`.well-known/agent-skills/`](.well-known/agent-skills/) | The [Agent Skills discovery](https://github.com/cloudflare/agent-skills-discovery-rfc) index plus one archive per skill, generated from `skills/`, for didww.com to serve at `/.well-known/agent-skills/index.json`. | ## Skills @@ -34,6 +35,8 @@ npx didww-connect npx didww-to-livekit ``` +Any agent that reads the Agent Skills discovery format can also find all of them from one URL, [`.well-known/agent-skills/index.json`](.well-known/agent-skills/index.json), which lists every skill with a sha256-verified `.tar.gz`. + Straight from this repository, without the npm registry: `npx skills add didww/ai --skill `, or `npx skills add didww/ai --list` to see all nine. Restart Claude Code after installing. ### Requirements @@ -51,6 +54,7 @@ The installers copy files and nothing else. The skills store no credentials; pla bash scripts/test.sh # every skill's checks: stdlib Python plus node, no network, no keys bash scripts/publish.sh # publish all nine to npm (after npm login) bash scripts/publish.sh didww-to-vapi # publish one +python3 scripts/build_index.py # regenerate .well-known/agent-skills/ after changing a skill ``` GitHub Actions runs the same tests on every push and pull request. See [CONTRIBUTING.md](CONTRIBUTING.md) for the skill layout, versioning and how to add a platform, and [CHANGELOG.md](CHANGELOG.md) for releases. diff --git a/docs/agent-skills-index.md b/docs/agent-skills-index.md new file mode 100644 index 0000000..58b3392 --- /dev/null +++ b/docs/agent-skills-index.md @@ -0,0 +1,53 @@ +# Serving the agent skills index on didww.com + +For the DIDWW web team. The goal is one URL that lets AI agents and skill directories find every DIDWW agent skill: + +``` +https://www.didww.com/.well-known/agent-skills/index.json +``` + +It uses the [Agent Skills discovery](https://github.com/cloudflare/agent-skills-discovery-rfc) format (v0.2.0), the same one Telnyx serves at `https://telnyx.com/.well-known/agent-skills/index.json`. Nothing on the site is written by hand: the files are generated in the [didww/ai](https://github.com/didww/ai) repository and the site keeps a copy. + +## What gets served + +The folder [`.well-known/agent-skills/`](../.well-known/agent-skills/) in the repository, as is: + +| File | Content-Type | +|---|---| +| `index.json` | `application/json` | +| `.tar.gz` (one per skill) | `application/gzip` | + +The index refers to each archive by a relative URL (`didww-connect.tar.gz`), so the files must sit side by side under `/.well-known/agent-skills/`. + +## Keeping it current + +Run the mirror script on a schedule, for example hourly from cron. It needs Python 3 and nothing else: + +```bash +python3 mirror_index.py --dest /path/to/webroot/.well-known/agent-skills +``` + +Download [`scripts/mirror_index.py`](../scripts/mirror_index.py) once, or run it from a clone of the repository. It reads the index from GitHub (`main` branch), downloads every archive, checks each sha256 digest and that every archive is safe to unpack, then replaces the folder in a single step. If anything fails (network, a digest mismatch, a malformed archive) it exits with code 1 and leaves the live folder exactly as it was. It only fetches files that sit next to the index on GitHub. + +Example crontab line: + +``` +17 * * * * python3 /opt/didww-skills/mirror_index.py --dest /var/www/didww.com/.well-known/agent-skills >> /var/log/didww-skills-mirror.log 2>&1 +``` + +## Web server settings + +- The Content-Types above. With nginx, `location ~ ^/\.well-known/agent-skills/.+\.tar\.gz$ { types { } default_type application/gzip; }` covers the archives; `.json` is mapped to `application/json` by the default `mime.types`. +- `GET` and `HEAD`, and `404` for files that do not exist. +- `Cache-Control: public, max-age=3600` matches an hourly mirror. +- `Access-Control-Allow-Origin: *`, so browser-based agents can read the index. +- No redirect from `didww.com` to `www.didww.com` is required, but agents follow redirects if there is one. + +## Checking it + +```bash +curl -sI https://www.didww.com/.well-known/agent-skills/index.json # 200, application/json +curl -s https://www.didww.com/.well-known/agent-skills/index.json | python3 -m json.tool | head +``` + +Every `digest` in the index must equal `sha256:` plus the SHA-256 of the archive the site serves. The mirror guarantees this as long as nothing else writes to the folder. diff --git a/scripts/build_index.py b/scripts/build_index.py new file mode 100644 index 0000000..e2ad51c --- /dev/null +++ b/scripts/build_index.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Builds .well-known/agent-skills/: the Agent Skills discovery index (v0.2.0) +plus one .tar.gz per skill, so any site can serve the family from one folder. + + python3 scripts/build_index.py write index.json and archives + python3 scripts/build_index.py --check fail if they are out of date (CI) + +Each archive holds what bin/install.js installs (SKILL.md, scripts/, +references/) plus LICENSE, at the archive root as the spec requires. +Archives are rewritten only when their contents change, so digests stay +put between builds. Spec: github.com/cloudflare/agent-skills-discovery-rfc +""" +import glob, gzip, hashlib, io, json, os, re, sys, tarfile + +ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +OUT = os.path.join(ROOT, ".well-known", "agent-skills") +SCHEMA = "https://schemas.agentskills.io/discovery/0.2.0/schema.json" +REPO = "https://github.com/didww/ai" +PARTS = ["SKILL.md", "LICENSE", "scripts", "references"] +SKIP = re.compile(r"(^|/)(__pycache__|\.DS_Store|.*\.pyc)(/|$)") +NAME = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") + +def frontmatter(path): + """name and description from SKILL.md; handles plain and folded (>) scalars.""" + text = open(path, encoding="utf-8").read() + m = re.match(r"^---\n(.*?)\n---\n", text, re.S) + if not m: + sys.exit(f"{path}: no YAML frontmatter") + fields, key, lines = {}, None, [] + for line in m.group(1).split("\n") + ["end:"]: + top = re.match(r"^([A-Za-z_-]+):\s*(.*)$", line) + if top: + if key: + fields[key] = re.sub(r"\s+", " ", " ".join(lines)).strip() + key, value = top.groups() + lines = [] if value in (">", ">-", "|", "|-") else [value.strip().strip('"\'')] + elif key: + lines.append(line.strip()) + return fields + +def members(skill_dir): + """(archive path, mode, bytes) for every file the archive should hold, sorted.""" + files = [] + for part in PARTS: + src = os.path.join(skill_dir, part) + paths = [src] if os.path.isfile(src) else sorted( + os.path.join(p, f) for p, _, fs in os.walk(src) for f in fs) + for p in paths: + rel = os.path.relpath(p, skill_dir).replace(os.sep, "/") + if SKIP.search(rel) or os.path.islink(p): + continue + mode = 0o755 if os.stat(p).st_mode & 0o111 else 0o644 + files.append((rel, mode, open(p, "rb").read())) + return sorted(files) + +def pack(files): + """Deterministic tar.gz: fixed owner and times, no wrapper directory.""" + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as tar: + for rel, mode, data in files: + info = tarfile.TarInfo(rel) + info.size, info.mode, info.mtime = len(data), mode, 0 + info.uid = info.gid = 0; info.uname = info.gname = "" + tar.addfile(info, io.BytesIO(data)) + out = io.BytesIO() + with gzip.GzipFile(fileobj=out, mode="wb", mtime=0, filename="") as gz: + gz.write(raw.getvalue()) + return out.getvalue() + +def unpack(blob): + """(path, mode, bytes) of a built archive, to compare with the sources.""" + with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar: + return sorted((m.name, m.mode, tar.extractfile(m).read()) for m in tar.getmembers() if m.isfile()) + +def build(check): + stale, entries = [], [] + for skill_md in sorted(glob.glob(os.path.join(ROOT, "skills", "*", "SKILL.md"))): + skill_dir = os.path.dirname(skill_md) + folder = os.path.basename(skill_dir) + fm = frontmatter(skill_md) + name, desc = fm.get("name", ""), fm.get("description", "") + if name != folder or not NAME.match(name) or len(name) > 64: + sys.exit(f"{skill_md}: name '{name}' must match the folder and the Agent Skills naming rules") + if not desc or len(desc) > 1024: + sys.exit(f"{skill_md}: description must be 1 to 1024 characters (has {len(desc)})") + version = json.load(open(os.path.join(skill_dir, "package.json")))["version"] + files = members(skill_dir) + archive = os.path.join(OUT, f"{name}.tar.gz") + blob = open(archive, "rb").read() if os.path.isfile(archive) else None + if blob is None or unpack(blob) != files: + stale.append(os.path.relpath(archive, ROOT)) + blob = pack(files) + if not check: + os.makedirs(OUT, exist_ok=True) + open(archive, "wb").write(blob) + entries.append({ + "name": name, + "type": "archive", + "description": desc, + "url": f"{name}.tar.gz", + "digest": "sha256:" + hashlib.sha256(blob).hexdigest(), + "version": version, + "documentation": f"{REPO}/blob/main/docs/{name}.md" + if os.path.isfile(os.path.join(ROOT, "docs", f"{name}.md")) + else f"{REPO}/tree/main/skills/{name}", + "install": f"npx {name}", + }) + index = { + "$schema": SCHEMA, + "name": "DIDWW Agent Skills", + "description": "Official DIDWW agent skills: put an AI voice agent on a real DIDWW phone " + "number with any major voice AI platform, through the DIDWW MCP " + "(https://api.didww.com/mcp).", + "repository": REPO, + "homepage": "https://www.didww.com/", + "skills": entries, + } + text = json.dumps(index, indent=2, ensure_ascii=False) + "\n" + index_path = os.path.join(OUT, "index.json") + current = open(index_path, encoding="utf-8").read() if os.path.isfile(index_path) else None + if current != text: + stale.append(os.path.relpath(index_path, ROOT)) + if not check: + os.makedirs(OUT, exist_ok=True) + open(index_path, "w", encoding="utf-8").write(text) + # Archives of skills that no longer exist. + known = {f"{e['name']}.tar.gz" for e in entries} | {"index.json"} + for extra in sorted(set(os.listdir(OUT)) - known if os.path.isdir(OUT) else []): + if extra.endswith(".tar.gz"): + stale.append(os.path.relpath(os.path.join(OUT, extra), ROOT)) + if not check: + os.remove(os.path.join(OUT, extra)) + return entries, stale + +if __name__ == "__main__": + check = "--check" in sys.argv[1:] + entries, stale = build(check) + if check: + if stale: + print("Out of date: " + ", ".join(stale)) + print("Run: python3 scripts/build_index.py, then commit .well-known/") + sys.exit(1) + print(f"PASS .well-known/agent-skills is current ({len(entries)} skills)") + else: + print(f"{len(entries)} skills; " + (f"updated {', '.join(stale)}" if stale else "nothing changed")) diff --git a/scripts/mirror_index.py b/scripts/mirror_index.py new file mode 100644 index 0000000..bd62ceb --- /dev/null +++ b/scripts/mirror_index.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Mirrors the DIDWW agent skills index onto a web server, for cron. + + python3 scripts/mirror_index.py --dest /var/www/didww.com/.well-known/agent-skills + +Downloads index.json and every archive it lists from GitHub (or --source), +checks each sha256 digest and that every archive is safe to unpack (SKILL.md +at the root, no absolute paths, no "..", no links), then swaps the new folder +in at once. On any failure the live folder is left untouched and the exit +code is 1, so a half-updated or tampered set is never served. Stdlib only. +""" +import argparse, hashlib, io, json, os, shutil, sys, tarfile, tempfile, urllib.parse, urllib.request + +DEFAULT_SOURCE = "https://raw.githubusercontent.com/didww/ai/main/.well-known/agent-skills/index.json" +SCHEMA = "https://schemas.agentskills.io/discovery/0.2.0/schema.json" +MAX_BYTES = 20 * 1024 * 1024 + +def fetch(url): + if "://" not in url: # a local path, for tests and manual runs + return open(url, "rb").read() + if urllib.parse.urlparse(url).scheme != "https": + raise ValueError(f"{url}: only https sources are allowed") + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "didww-skills-mirror"}), timeout=60) as r: + data = r.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + raise ValueError(f"{url}: larger than {MAX_BYTES} bytes") + return data + +def check_archive(name, blob): + with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar: + names = [] + for m in tar.getmembers(): + if m.name.startswith("/") or ".." in m.name.split("/") or not (m.isfile() or m.isdir()): + raise ValueError(f"{name}: unsafe archive member {m.name!r}") + names.append(m.name) + if "SKILL.md" not in names: + raise ValueError(f"{name}: no SKILL.md at the archive root") + +def mirror(source, dest): + index_bytes = fetch(source) + index = json.loads(index_bytes) + if index.get("$schema") != SCHEMA: + raise ValueError(f"unexpected $schema {index.get('$schema')!r}") + files = {"index.json": index_bytes} + remote = "://" in source + base = source.rsplit("/", 1)[0] if remote else os.path.dirname(os.path.abspath(source)) + for skill in index["skills"]: + url = skill["url"] + # Only plain file names next to the index: the mirror never fetches other hosts. + if "/" in url or "\\" in url or url.startswith(".") or not url.endswith(".tar.gz"): + raise ValueError(f"{skill['name']}: url {url!r} is not a file next to the index") + blob = fetch(f"{base}/{urllib.parse.quote(url)}" if remote else os.path.join(base, url)) + digest = "sha256:" + hashlib.sha256(blob).hexdigest() + if digest != skill["digest"]: + raise ValueError(f"{skill['name']}: digest mismatch ({digest} != {skill['digest']})") + check_archive(skill["name"], blob) + files[url] = blob + dest = os.path.abspath(dest) + parent = os.path.dirname(dest) + os.makedirs(parent, exist_ok=True) + staging = tempfile.mkdtemp(prefix=".agent-skills-new-", dir=parent) + try: + for fname, data in files.items(): + # Names come from the downloaded index: keep every write inside staging. + target = os.path.realpath(os.path.join(staging, os.path.basename(fname))) + if os.path.commonpath([target, os.path.realpath(staging)]) != os.path.realpath(staging): + raise ValueError(f"{fname}: would be written outside {dest}") + with open(target, "wb") as f: + f.write(data) + os.chmod(target, 0o644) + os.chmod(staging, 0o755) + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + old = None + if os.path.exists(dest): + old = tempfile.mkdtemp(prefix=".agent-skills-old-", dir=parent) + os.rmdir(old) + os.rename(dest, old) + os.rename(staging, dest) + if old: + shutil.rmtree(old) + return len(index["skills"]) + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--dest", required=True, help="folder served as /.well-known/agent-skills/") + ap.add_argument("--source", default=DEFAULT_SOURCE, help="index.json URL or local path") + args = ap.parse_args() + try: + n = mirror(args.source, args.dest) + except Exception as e: + print(f"Mirror failed, live folder unchanged: {e}", file=sys.stderr) + sys.exit(1) + print(f"Mirrored {n} skills to {args.dest}") diff --git a/scripts/test.sh b/scripts/test.sh index adc4ff2..d5a73f9 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,3 +4,4 @@ set -euo pipefail cd "$(dirname "$0")/.." for d in skills/*/; do echo "== ${d%/}"; bash "${d}test.sh"; echo; done echo "== installers"; python3 -B scripts/test_installers.py +echo; echo "== discovery index"; python3 -B scripts/test_index.py diff --git a/scripts/test_index.py b/scripts/test_index.py new file mode 100644 index 0000000..91e32a6 --- /dev/null +++ b/scripts/test_index.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Tests for the agent skills discovery index and its website mirror. + +Checks that .well-known/agent-skills/ is current, that every entry follows the +discovery spec v0.2.0 (names, descriptions, sha256 digests of the archives, +SKILL.md at the archive root), that the build is deterministic, and that +mirror_index.py publishes a good set but refuses a tampered one while leaving +the live folder untouched. Offline; nothing outside a temporary directory. +""" +import glob, hashlib, io, json, os, re, shutil, subprocess, sys, tarfile, tempfile + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +OUT = os.path.join(ROOT, ".well-known", "agent-skills") +sys.path.insert(0, os.path.join(ROOT, "scripts")) +import build_index, mirror_index + +results = [] +def check(name, ok, detail=""): + results.append(bool(ok)); print(f"{'PASS' if ok else 'FAIL'} {name} {detail}") + +r = subprocess.run([sys.executable, "-B", os.path.join(ROOT, "scripts", "build_index.py"), "--check"], + capture_output=True, text=True) +check("index and archives are current", r.returncode == 0, (r.stdout + r.stderr).strip()[-300:]) + +index = json.load(open(os.path.join(OUT, "index.json"))) +skills = sorted(os.path.basename(os.path.dirname(p)) for p in + glob.glob(os.path.join(ROOT, "skills", "*", "SKILL.md"))) +check("$schema is discovery v0.2.0", index.get("$schema") == build_index.SCHEMA) +check("one entry per skill folder", [e["name"] for e in index["skills"]] == skills, + [e["name"] for e in index["skills"]]) +for e in index["skills"]: + blob = open(os.path.join(OUT, e["url"]), "rb").read() + with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar: + names = tar.getnames() + skill_md = tar.extractfile("SKILL.md").read() if "SKILL.md" in names else b"" + src_md = open(os.path.join(ROOT, "skills", e["name"], "SKILL.md"), "rb").read() + check(f"{e['name']}: spec fields and naming", + e["type"] == "archive" and re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", e["name"]) + and 0 < len(e["description"]) <= 1024 and re.match(r"^sha256:[0-9a-f]{64}$", e["digest"])) + check(f"{e['name']}: digest is the archive's sha256", + e["digest"] == "sha256:" + hashlib.sha256(blob).hexdigest()) + check(f"{e['name']}: SKILL.md at the root, same bytes as the source, scripts included, no caches", + skill_md == src_md and any(n.startswith("scripts/") for n in names) + and not any(n.startswith("/") or ".." in n.split("/") or "__pycache__" in n for n in names)) + check(f"{e['name']}: description matches SKILL.md frontmatter", + e["description"] == build_index.frontmatter(os.path.join(ROOT, "skills", e["name"], "SKILL.md"))["description"]) + files = build_index.members(os.path.join(ROOT, "skills", e["name"])) + check(f"{e['name']}: build is deterministic", build_index.pack(files) == build_index.pack(files)) + +with tempfile.TemporaryDirectory() as tmp: + live = os.path.join(tmp, "www", "agent-skills") + code = mirror_index.mirror(os.path.join(OUT, "index.json"), live) + check("mirror publishes every skill with verified digests", + code == len(skills) and sorted(os.listdir(live)) == sorted([e["url"] for e in index["skills"]] + ["index.json"])) + before = {f: open(os.path.join(live, f), "rb").read() for f in os.listdir(live)} + + bad = os.path.join(tmp, "bad"); shutil.copytree(OUT, bad) + first = index["skills"][0]["url"] + with open(os.path.join(bad, first), "ab") as f: + f.write(b"tampered") + try: + mirror_index.mirror(os.path.join(bad, "index.json"), live); refused = False + except ValueError as err: + refused = "digest mismatch" in str(err) + after = {f: open(os.path.join(live, f), "rb").read() for f in os.listdir(live)} + check("mirror refuses a tampered archive and leaves the live folder as it was", refused and after == before) + + evil = os.path.join(tmp, "evil"); shutil.copytree(OUT, evil) + idx = json.load(open(os.path.join(evil, "index.json"))) + idx["skills"][0]["url"] = "https://example.com/x.tar.gz" + json.dump(idx, open(os.path.join(evil, "index.json"), "w")) + try: + mirror_index.mirror(os.path.join(evil, "index.json"), live); refused = False + except ValueError: + refused = True + check("mirror refuses URLs pointing at other hosts", refused) + + try: + mirror_index.fetch("file:///etc/passwd"); refused = False + except ValueError: + refused = True + check("mirror refuses non-https sources", refused) + + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w:gz") as tar: + for n in ("SKILL.md", "../escape"): + info = tarfile.TarInfo(n); info.size = 1; tar.addfile(info, io.BytesIO(b"x")) + try: + mirror_index.check_archive("t", raw.getvalue()); refused = False + except ValueError: + refused = True + check("mirror refuses an archive with path traversal", refused) + +print(f"\n{sum(results)}/{len(results)} passed") +sys.exit(0 if all(results) else 1)