diff --git a/.claude/skills/data-connector/SKILL.md b/.claude/skills/data-connector/SKILL.md index 79f9abd..4112e79 100644 --- a/.claude/skills/data-connector/SKILL.md +++ b/.claude/skills/data-connector/SKILL.md @@ -1,7 +1,7 @@ --- name: data-connector description: > - Build new Vana DataConnect data connectors that export user data from web platforms. + Build new data connectors that export user data from web platforms. Use when asked to "create a connector", "add a connector", "build a connector", "new data connector", "write a connector for X", or when working on the data-connectors repository and the task involves adding or modifying a Playwright-based connector. @@ -9,7 +9,7 @@ description: > # Data Connector Builder -Build Playwright-based data connectors for the Vana DataConnect ecosystem. Connectors export +Build Playwright-based data connectors for the data-connectors ecosystem. Connectors export a user's personal data from web platforms (LinkedIn, ChatGPT, Spotify, etc.) using browser automation. Credentials never leave the device. diff --git a/.github/workflows/consumer-update-prs.yml b/.github/workflows/consumer-update-prs.yml deleted file mode 100644 index 04cdb57..0000000 --- a/.github/workflows/consumer-update-prs.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Consumer Update PRs - -on: - workflow_dispatch: - push: - branches: [main] - paths: - - "connector-index.json" - - "artifacts/**" - - "packages/connector-installer-core/**" - - "packages/connector-installer-cli/**" - - "schemas/connector-index.schema.json" - - "package.json" - -permissions: - contents: read - -jobs: - preflight: - runs-on: ubuntu-latest - outputs: - has_token: ${{ steps.check.outputs.has_token }} - steps: - - id: check - env: - CONSUMER_PR_TOKEN: ${{ secrets.CONSUMER_PR_TOKEN }} - run: | - if [ -n "$CONSUMER_PR_TOKEN" ]; then - echo "has_token=true" >> "$GITHUB_OUTPUT" - else - echo "has_token=false" >> "$GITHUB_OUTPUT" - fi - - update-context-gateway: - needs: preflight - if: needs.preflight.outputs.has_token == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - path: data-connectors - - - uses: actions/checkout@v4 - with: - repository: vana-com/context-gateway - token: ${{ secrets.CONSUMER_PR_TOKEN }} - ref: staging - path: context-gateway - - - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Install context-gateway deps - working-directory: context-gateway - run: npm install - - - name: Install local shared installer package - working-directory: context-gateway - run: npm install --no-save ../data-connectors - - - name: Resolve connectors and regenerate catalog - working-directory: context-gateway - run: | - node scripts/resolve-connectors.mjs --from-local ../data-connectors - node scripts/generate-connector-catalog.mjs - node scripts/validate-connector-overlay.mjs - - - name: Create pull request - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ secrets.CONSUMER_PR_TOKEN }} - path: context-gateway - branch: automation/update-data-connectors - delete-branch: true - commit-message: "chore: refresh connector snapshot" - title: "chore: refresh connector snapshot" - body: | - Automated update from `vana-com/data-connectors`. - - Changes: - - refreshed `config/data-connectors-snapshot/` - - refreshed generated connector catalog - - update-data-connect: - needs: preflight - if: needs.preflight.outputs.has_token == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - path: data-connectors - - - uses: actions/checkout@v4 - with: - repository: vana-com/data-connect - token: ${{ secrets.CONSUMER_PR_TOKEN }} - ref: main - path: data-connect - - - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Install data-connect deps - working-directory: data-connect - run: npm install - - - name: Install local shared installer package - working-directory: data-connect - run: npm install --no-save ../data-connectors - - - name: Resolve connectors and regenerate platform registry - working-directory: data-connect - run: | - node scripts/resolve-connectors.js --from-local ../data-connectors - node scripts/generate-platform-registry.js - - - name: Run tests - working-directory: data-connect - run: npm test - - - name: Create pull request - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ secrets.CONSUMER_PR_TOKEN }} - path: data-connect - branch: automation/update-data-connectors - delete-branch: true - commit-message: "chore: refresh bundled connectors" - title: "chore: refresh bundled connectors" - body: | - Automated update from `vana-com/data-connectors`. - - Changes: - - refreshed bundled connectors under `connectors/` - - refreshed generated platform registry diff --git a/.gitignore b/.gitignore index 4ca591d..2fb7318 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ test-results/ # Non-connector projects (stray checkouts) clearcut/ proxy-test/ + +# Vendored playwright-runner's own install +playwright-runner/node_modules/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..3ef460a --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +Copyright © 2026 The PDP-Connect Contributors +Licensed under the Apache License, Version 2.0. diff --git a/README.md b/README.md index 4ff9929..668eb29 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Data Connectors -Playwright-based data connectors for Vana Desktop in [Unity Surfaces](https://github.com/vana-com/unity-surfaces). Each connector exports your data from a web platform using browser automation. Credentials never leave your device. +Playwright-based data connectors for local desktop collection. Each connector exports your data from a web platform using browser automation. Credentials never leave your device. + +This repo bundles its own standalone runner (`playwright-runner/`) -- no other repo is required to run a connector. See [Running a connector](#running-a-connector) below. ## Connector status @@ -14,7 +16,7 @@ Each connector has a status indicating its maturity level: This table is an overview, not the public contract. [`scope-catalog.json`](scope-catalog.json) is the authoritative machine-readable contract for sources and scopes. [`SCOPES.md`](SCOPES.md) is its generated human-readable view. -The catalog separates two collection paths. **Vana Desktop (local)** runs registered Playwright connectors on your device. **Vana Web (hosted)** requests data from Open Data Labs' hosted collection service, called Data Pipe. A website cannot run the local Node.js/Playwright connector or read your signed-in session on another site. Vana Web could ask an installed Vana Desktop app to collect the data, but that is still Desktop collection. +The catalog describes the local desktop collection path: registered Playwright connectors run on your device and use your browser session. @@ -58,12 +60,15 @@ See the [Context Gateway docs](https://dev.opendatalabs.com/docs/data-sources) f ## Running a connector ```bash +# One-time setup: install the bundled runner's dependencies + Chromium +bash skills/pdp-connect/scripts/setup.sh + node run-connector.cjs ./connectors/github/github-playwright.js # JSON output (for agents) node run-connector.cjs ./connectors/github/github-playwright.js --pretty # colored output (for humans) node run-connector.cjs ./connectors/github/github-playwright.js --inputs '{"username":"x","password":"y"}' ``` -See [`skills/vana-connect/`](skills/vana-connect/) for the agent skill: setup, running, creating new connectors, and data recipes. +`run-connector.cjs` resolves the bundled `playwright-runner/` in this repo automatically -- no other repo needs to be cloned. See [`skills/pdp-connect/SETUP.md`](skills/pdp-connect/SETUP.md) for details, and [`skills/pdp-connect/`](skills/pdp-connect/) for the agent skill: setup, running, creating new connectors, and data recipes. ## Repository structure @@ -71,7 +76,7 @@ See [`skills/vana-connect/`](skills/vana-connect/) for the agent skill: setup, r |--------|--------------|----------| | **`connectors/`** | All platform connectors (`/-playwright.js` + `.json` + connector-local `schemas/` + local assets like `icons/`) | Everyone | | **`scripts/`** | Developer tooling: scaffold, test, validate, session capture | Human developers | -| **`skills/`** | AI agent skill for creating/running connectors (`vana-connect/`) | AI agents (Claude, etc.) | +| **`skills/`** | AI agent skill for creating/running connectors (`pdp-connect/`) | AI agents (Claude, etc.) | | **`schemas/`** | Shared meta-schemas such as `manifest.schema.json` | Validation | | **`types/`** | TypeScript type definitions (`connector.d.ts`) | TypeScript consumers | | **`fixture-index.json`** | Public index of schema-valid connector fixtures with main-branch raw GitHub URLs and checksums | Builders and AI agents | @@ -94,7 +99,7 @@ scripts/ # Developer tooling (human-driven) ├── validate-connector.cjs # Structure + output validator └── reference/ # Templates and API docs │ -skills/vana-connect/ # Agent skill (AI-agent-driven) +skills/pdp-connect/ # Agent skill (AI-agent-driven) ├── SKILL.md # Skill entry point (setup, connect, recipes) ├── CREATE.md # Full walkthrough for building connectors └── scripts/ # Agent-facing scripts (runner, validator, etc.) @@ -105,24 +110,24 @@ schemas/ # Shared meta-schemas registry.json # Central registry (checksums, versions, OTA) fixture-index.json # Public fixture catalog for builders and agents scope-catalog.json # Generated public source/scope contract -scopes/web-capabilities.json # Hand-authored Vana Web hosted-collection truth and limits -run-connector.cjs # Symlink → skills/vana-connect/scripts/run-connector.cjs +run-connector.cjs # Symlink → skills/pdp-connect/scripts/run-connector.cjs test-connector.cjs # Standalone test runner create-connector.sh # Quick autonomous scaffold script ``` ## Distribution contract -`data-connectors` is the canonical connector distribution source for both -`unity-surfaces` and `context-gateway`. +`data-connectors` is an open collection of data connectors that any consumer +can pin and resolve as a versioned dependency. Vana's Context Gateway and +Unity Surfaces are two example consumers; nothing here is specific to them. - `connector-index.json` is the authoritative release index. - `scope-catalog.json` is the generated, schema-validated public contract for - source IDs, scope IDs, payload schema paths, Vana Desktop/Vana Web fulfillment, material + source IDs, scope IDs, payload schema paths, desktop fulfillment, material limits, and maturity. `SCOPES.md` is generated from the same model. - Pin `scope-catalog.json` from an immutable GitHub release named `connectors-`, for example - `https://github.com/vana-com/data-connectors/releases/download/connectors-/scope-catalog.json`. + `https://github.com/PDP-Connect/data-connectors/releases/download/connectors-/scope-catalog.json`. Release catalogs embed the full source commit and an immutable `schema.url` for every payload schema. Checked-in catalogs retain `schema.path` for local repository use. A raw `main` URL is not a version pin. @@ -196,11 +201,11 @@ npm run fixture-index:check The repo has two interfaces for building and running connectors. They serve different audiences but share the same connector format and output: -| | `scripts/` | `skills/vana-connect/` | +| | `scripts/` | `skills/pdp-connect/` | |---|---|---| | **Audience** | Human developers at a terminal | AI agents (Claude, etc.) | -| **Entry point** | `scripts/create-connector.sh` | `skills/vana-connect/SKILL.md` | -| **Login** | Manual browser login via `capture-session.cjs` | CLI-driven (`vana connect`) | +| **Entry point** | `scripts/create-connector.sh` | `skills/pdp-connect/SKILL.md` | +| **Login** | Manual browser login via `capture-session.cjs` | Runner-driven (`run-connector.cjs`) | | **Testing** | `scripts/test-connector.cjs` | `run-connector.cjs` | | **When to use** | Local development, debugging, manual QA | Automated connector creation and data export | @@ -210,7 +215,7 @@ Both produce the same connector files (`connectors//-playwright.j ## How connectors work -Connectors run in a sandboxed Playwright browser managed by Vana Desktop in Unity Surfaces. The runner provides a `page` API object (not raw Playwright). The browser starts headless; connectors call `page.showBrowser()` when login is needed and `page.goHeadless()` after. +Connectors run in a sandboxed Playwright browser managed by the bundled runner (`playwright-runner/`) or, in the desktop app, an equivalent driver. The runner provides a `page` API object (not raw Playwright). The browser starts headless; connectors call `page.showBrowser()` when login is needed and `page.goHeadless()` after. ### Two-phase architecture @@ -354,20 +359,20 @@ The runner classifies each run into one of: ## Building a new connector -See [`skills/vana-connect/CREATE.md`](skills/vana-connect/CREATE.md) for the full walkthrough. Summary: +See [`skills/pdp-connect/CREATE.md`](skills/pdp-connect/CREATE.md) for the full walkthrough. Summary: -1. **Scaffold:** `node skills/vana-connect/scripts/scaffold.cjs [company]` -- generates script, metadata, and connector-local stub schema +1. **Scaffold:** `node skills/pdp-connect/scripts/scaffold.cjs [company]` -- generates script, metadata, and connector-local stub schema 2. **Implement:** Write login + data collection logic (see CREATE.md for auth patterns, extraction strategies, and reference connectors) 3. **Validate structure:** `node scripts/validate-connector.cjs connectors//-playwright.js` 4. **Test:** `node run-connector.cjs connectors//-playwright.js --inputs '{"username":"x","password":"y"}'` -5. **Validate output:** `node scripts/validate-connector.cjs connectors//-playwright.js --check-result ~/.vana/desktop/last-result.json` -6. **Register:** `node skills/vana-connect/scripts/register.cjs connectors//-playwright.js` -- adds entry + checksums to `registry.json` +5. **Validate output:** `node scripts/validate-connector.cjs connectors//-playwright.js --check-result ~/.pdp-connect/desktop/last-result.json` +6. **Register:** `node skills/pdp-connect/scripts/register.cjs connectors//-playwright.js` -- adds entry + checksums to `registry.json` --- ## Page API reference -The `page` object is available as a global in connector scripts. The runner implementation lives in [Unity Surfaces](https://github.com/vana-com/unity-surfaces/tree/main/apps/desktop/playwright-runner). +The `page` object is available as a global in connector scripts. The runner implementation is bundled in this repo at [`playwright-runner/`](playwright-runner/). | Method | Description | |--------|-------------| @@ -415,7 +420,7 @@ const { email, password } = await page.requestInput({ }); ``` -The runner relays the request to the driver (Tauri app, agent, CLI) and resolves with the response. The `schema` field uses JSON Schema — the same format used by OpenAI, Anthropic, and Google for LLM tool definitions. See the [headless-first runner spec](https://github.com/vana-com/unity-surfaces/blob/main/apps/desktop/docs/260310-headless-first-runner-spec.md) for the full protocol design. +The runner relays the request to the driver (Tauri app, agent, CLI) and resolves with the response. The `schema` field uses JSON Schema — the same format used by OpenAI, Anthropic, and Google for LLM tool definitions. ### Progress reporting @@ -436,17 +441,11 @@ await page.setProgress({ ## Testing locally -Vana Desktop development belongs to [Unity Surfaces](https://github.com/vana-com/unity-surfaces/tree/main/apps/desktop). Follow that repo's Desktop instructions for app setup, connector syncing, and runtime debugging. - ### Standalone test runner -Test connectors without the full Vana Desktop app. The runner spawns `playwright-runner` as a child process and outputs JSON protocol messages. - -**Prerequisites:** Clone [Unity Surfaces](https://github.com/vana-com/unity-surfaces) alongside this repo, then point `PLAYWRIGHT_RUNNER_DIR` at its Desktop runner. +Test connectors without any other app or repo. `run-connector.cjs` spawns the bundled `playwright-runner/` (see [Running a connector](#running-a-connector) for one-time setup) as a child process and outputs JSON protocol messages. ```bash -export PLAYWRIGHT_RUNNER_DIR=../unity-surfaces/apps/desktop/playwright-runner - # Run a connector (headed by default, browser visible) node run-connector.cjs ./connectors/linkedin/linkedin-playwright.js @@ -517,19 +516,21 @@ shasum -a 256 connectors//-playwright.json | awk '{print "sha256: ## How consumers get connectors -Both [Context Gateway](https://github.com/vana-com/context-gateway) and [Unity Surfaces](https://github.com/vana-com/unity-surfaces) consume connectors from this repo as pinned dependencies. - -Each consumer declares version constraints in a `connector-dependencies.json` file and resolves them with: +Any project can consume connectors from this repo as pinned dependencies. A +consumer declares version constraints in a `connector-dependencies.json` file +and resolves them with a `connectors:resolve` script (naming and invocation +are up to the consumer's own tooling), for example: ```bash -# Context Gateway +# Example: a consumer's package.json script npm run connectors:resolve - -# Unity Surfaces -pnpm --dir apps/desktop connectors:resolve ``` This fetches matching versions from the signed `connectors-latest` `connector-index.json`, verifies the detached Sigstore bundles plus the artifact/manifest/script checksums, and writes the scripts + manifests to a local snapshot directory. It's analogous to `npm install`. + +Vana's [Context Gateway](https://github.com/vana-com/context-gateway) and +[Unity Surfaces](https://github.com/vana-com/unity-surfaces) are two examples +of consumers that resolve connectors this way. diff --git a/SCOPES.md b/SCOPES.md index 4f65ae5..0655207 100644 --- a/SCOPES.md +++ b/SCOPES.md @@ -3,12 +3,7 @@ The public human-readable view of [`scope-catalog.json`](scope-catalog.json). -This catalog shows two collection paths. When both paths support a scope, they use the same scope ID and payload schema: - -- **Vana Desktop (local):** A registered Playwright connector runs on your device and uses your browser session. Connector manifests plus `registry.json` own this column. -- **Vana Web (hosted):** Vana Web requests data from Open Data Labs' hosted collection service, called Data Pipe. [`scopes/web-capabilities.json`](scopes/web-capabilities.json) owns this column and its limits. - -A website cannot run the local Node.js/Playwright connector or read your signed-in session on another site. Vana Web could ask an installed Vana Desktop app to collect the data, but that is still Desktop collection. The table uses **Yes**, **No**, or **Blocked** for each path. **Blocked** names the evidence required before making a claim. +This catalog describes the local connector collection path: a registered Playwright connector runs on your device and uses your browser session. Connector manifests plus `registry.json` own this data. ## Versioned artifact contract @@ -16,68 +11,59 @@ Every immutable `connectors-` GitHub release includes `scope-catalog.j ## Coverage -| Source | Scope | Description | Schema | Vana Web (hosted) | Vana Desktop (local) | Material limits | Desktop connector(s) / maturity | -|---|---|---|---|:--:|:--:|---|---| -| amazon | `amazon.orders` | Order history including items, prices, dates, and delivery status | [JSON Schema](connectors/amazon/schemas/amazon.orders.json) | No | Yes | None | amazon-playwright (beta) | -| amazon | `amazon.profile` | Account name and Prime membership status | [JSON Schema](connectors/amazon/schemas/amazon.profile.json) | No | Yes | None | amazon-playwright (beta) | -| chatgpt | `chatgpt.conversations` | All conversation history including messages with ChatGPT | [JSON Schema](connectors/openai/schemas/chatgpt.conversations.json) | No | Yes | None | chatgpt-playwright (stable) | -| chatgpt | `chatgpt.memories` | Saved memories that ChatGPT uses to personalize responses | [JSON Schema](connectors/openai/schemas/chatgpt.memories.json) | No | Yes | None | chatgpt-playwright (stable) | -| claude | `claude.conversations` | All Claude conversations with full message threads, from Anthropic's official data export. | [JSON Schema](connectors/anthropic/schemas/claude.conversations.json) | No | Yes | None | claude-export-playwright (experimental) | -| claude | `claude.projects` | Claude projects and their detail, from Anthropic's official data export. | [JSON Schema](connectors/anthropic/schemas/claude.projects.json) | No | Yes | None | claude-export-playwright (experimental) | -| doordash | `doordash.orders` | Complete order history with restaurant names, items ordered, dates, totals, and delivery status | [JSON Schema](connectors/doordash/schemas/doordash.orders.json) | No | Yes | None | doordash-playwright (beta) | -| github | `github.contributions` | Daily contribution counts for the last 4 years (per-year scrape, not just rolling 12 months), monthly totals, per-year totals, and your top day | [JSON Schema](connectors/github/schemas/github.contributions.json) | No | Yes | Vana Desktop (local; github-playwright): The Desktop connector collects the four most recent calendar years. | github-playwright (stable) | -| github | `github.events` | Recent public activity (pushes, pull requests, issues, comments, branches, releases) across every repository you have touched, including organization-owned repos | [JSON Schema](connectors/github/schemas/github.events.json) | No | Yes | Vana Desktop (local; github-playwright): The Desktop connector returns at most 300 public events.; Vana Desktop (local; github-playwright): The GitHub public-events API covers at most the previous 90 days. | github-playwright (stable) | -| github | `github.history` | Pull requests and issues you have authored across public repositories via the GitHub Search API, including title, body, dates, repository, labels, and state. | [JSON Schema](connectors/github/schemas/github.history.json) | No | Yes | Vana Desktop (local; github-playwright): The GitHub Search API returns at most 1,000 pull requests and 1,000 issues per query. | github-playwright (stable) | -| github | `github.profile` | Account profile information such as name, bio, follower counts, pinned repositories, organization memberships, and achievement badges | [JSON Schema](connectors/github/schemas/github.profile.json) | Yes | Yes | None | github-playwright (stable) | -| github | `github.repositories` | Your repository list with metadata like language, stars, and update time | [JSON Schema](connectors/github/schemas/github.repositories.json) | No | Yes | None | github-playwright (stable) | -| github | `github.starred` | Repositories you have starred on GitHub | [JSON Schema](connectors/github/schemas/github.starred.json) | No | Yes | None | github-playwright (stable) | -| heb | `heb.nutrition` | Nutrition facts (calories, macros, sodium, fiber, vitamins) for each product ordered. Aligned to Schema.org NutritionInformation. | [JSON Schema](connectors/heb/schemas/heb.nutrition.json) | No | Yes | None | heb-playwright (experimental) | -| heb | `heb.orders` | Curbside and delivery orders with item names, quantities, prices, and order dates | [JSON Schema](connectors/heb/schemas/heb.orders.json) | No | Yes | None | heb-playwright (experimental) | -| heb | `heb.profile` | Your name, email, phone number, and saved delivery addresses | [JSON Schema](connectors/heb/schemas/heb.profile.json) | No | Yes | None | heb-playwright (experimental) | -| icloud_notes | `icloud_notes.folders` | Folder structure for organizing notes | [JSON Schema](connectors/apple/schemas/icloud_notes.folders.json) | No | Yes | None | icloud-notes-playwright (experimental) | -| icloud_notes | `icloud_notes.notes` | Note content, titles, created/modified timestamps, and folder assignment | [JSON Schema](connectors/apple/schemas/icloud_notes.notes.json) | No | Yes | None | icloud-notes-playwright (experimental) | -| instagram | `instagram.ads` | Advertisers the user has seen ads from and ad topic interests based on their Instagram activity | [JSON Schema](connectors/meta/schemas/instagram.ads.json) | No | Yes | None | instagram-playwright (stable); instagram-ads-playwright (beta) | -| instagram | `instagram.following` | The list of Instagram accounts you follow | [JSON Schema](connectors/meta/schemas/instagram.following.json) | No | Yes | None | instagram-playwright (stable) | -| instagram | `instagram.posts` | Published posts from your feed | [JSON Schema](connectors/meta/schemas/instagram.posts.json) | Yes | Yes | Vana Web (hosted): The Web path returns at most the 150 most recent posts. | instagram-playwright (stable) | -| instagram | `instagram.profile` | Profile information including bio and follower counts | [JSON Schema](connectors/meta/schemas/instagram.profile.json) | Yes | Yes | None | instagram-playwright (stable) | -| linkedin | `linkedin.connections` | Your LinkedIn connections with names, headlines, profile URLs, and date connected | [JSON Schema](connectors/linkedin/schemas/linkedin.connections.json) | Blocked (linkedin-live-output) | Yes | None | linkedin-playwright (stable) | -| linkedin | `linkedin.education` | Your education history including schools, degrees, years, and grades | [JSON Schema](connectors/linkedin/schemas/linkedin.education.json) | Blocked (linkedin-live-output) | Yes | None | linkedin-playwright (stable) | -| linkedin | `linkedin.experience` | Your work experience including job titles, companies, dates, locations, and descriptions | [JSON Schema](connectors/linkedin/schemas/linkedin.experience.json) | Blocked (linkedin-live-output) | Yes | None | linkedin-playwright (stable) | -| linkedin | `linkedin.languages` | Your languages and proficiency levels | [JSON Schema](connectors/linkedin/schemas/linkedin.languages.json) | Blocked (linkedin-live-output) | Yes | None | linkedin-playwright (stable) | -| linkedin | `linkedin.profile` | Your LinkedIn profile including name, headline, location, about, and profile picture | [JSON Schema](connectors/linkedin/schemas/linkedin.profile.json) | Blocked (linkedin-live-output) | Yes | None | linkedin-playwright (stable) | -| linkedin | `linkedin.skills` | Your skills and endorsement counts | [JSON Schema](connectors/linkedin/schemas/linkedin.skills.json) | Blocked (linkedin-live-output) | Yes | None | linkedin-playwright (stable) | -| oura | `oura.activity` | Daily activity scores, step counts, calorie burn, and activity level breakdowns | [JSON Schema](connectors/oura/schemas/oura.activity.json) | No | Yes | None | oura-playwright (stable) | -| oura | `oura.readiness` | Daily readiness scores and contributing factors like HRV balance, resting heart rate, recovery, and body temperature | [JSON Schema](connectors/oura/schemas/oura.readiness.json) | No | Yes | None | oura-playwright (stable) | -| oura | `oura.sleep` | Daily sleep scores with contributor breakdowns and detailed sleep period data including duration, stages, heart rate, and HRV | [JSON Schema](connectors/oura/schemas/oura.sleep.json) | No | Yes | None | oura-playwright (stable) | -| shop | `shop.orders` | Complete order history from the Shop app including merchants, items, totals, and delivery status | [JSON Schema](connectors/shopify/schemas/shop.orders.json) | No | Yes | None | shop-playwright (beta) | -| spotify | `spotify.playlists` | Playlists and their tracks | [JSON Schema](connectors/spotify/schemas/spotify.playlists.json) | No | Yes | None | spotify-playwright (stable) | -| spotify | `spotify.profile` | Profile information | [JSON Schema](connectors/spotify/schemas/spotify.profile.json) | Yes | Yes | None | spotify-playwright (stable) | -| spotify | `spotify.savedTracks` | Saved/liked tracks in your library | [JSON Schema](connectors/spotify/schemas/spotify.savedTracks.json) | No | Yes | None | spotify-playwright (stable) | -| steam | `steam.friends` | Friend list with persona names, avatars, and friendship dates | [JSON Schema](connectors/valve/schemas/steam.friends.json) | No | Yes | None | steam-playwright (experimental) | -| steam | `steam.games` | Owned games with playtime statistics, plus recently played games | [JSON Schema](connectors/valve/schemas/steam.games.json) | No | Yes | None | steam-playwright (experimental) | -| steam | `steam.profile` | Steam profile including persona name, avatar, level, country, and account age | [JSON Schema](connectors/valve/schemas/steam.profile.json) | No | Yes | None | steam-playwright (experimental) | -| uber | `uber.receipts` | Fare breakdowns and receipt details for each trip | [JSON Schema](connectors/uber/schemas/uber.receipts.json) | No | Yes | None | uber-playwright (beta) | -| uber | `uber.trips` | Complete history of your Uber rides including pickup/dropoff locations, dates, and driver info | [JSON Schema](connectors/uber/schemas/uber.trips.json) | No | Yes | None | uber-playwright (beta) | -| wholefoods | `wholefoods.nutrition` | Nutrition facts (calories, macros, sodium, fiber, vitamins) for each product ordered. Aligned to Schema.org NutritionInformation. | [JSON Schema](connectors/wholefoods/schemas/wholefoods.nutrition.json) | No | Yes | None | wholefoods-playwright (experimental) | -| wholefoods | `wholefoods.orders` | Whole Foods delivery and pickup orders with item names, quantities, prices, and order dates | [JSON Schema](connectors/wholefoods/schemas/wholefoods.orders.json) | No | Yes | None | wholefoods-playwright (experimental) | -| wholefoods | `wholefoods.profile` | Your Amazon account name and email associated with Whole Foods orders | [JSON Schema](connectors/wholefoods/schemas/wholefoods.profile.json) | No | Yes | None | wholefoods-playwright (experimental) | -| youtube | `youtube.history` | Raw watch history entries for the top 50 most recent items | [JSON Schema](connectors/google/schemas/youtube.history.json) | No | Yes | Vana Desktop (local; youtube-playwright): The Desktop connector returns at most the 50 most recent watch history items. | youtube-playwright (beta) | -| youtube | `youtube.likes` | Videos you have liked | [JSON Schema](connectors/google/schemas/youtube.likes.json) | No | Yes | None | youtube-playwright (beta) | -| youtube | `youtube.playlistItems` | Videos inside your playlists, including titles and URLs | [JSON Schema](connectors/google/schemas/youtube.playlistItems.json) | No | Yes | None | youtube-playwright (beta) | -| youtube | `youtube.playlists` | Playlists visible in your YouTube Library | [JSON Schema](connectors/google/schemas/youtube.playlists.json) | No | Yes | None | youtube-playwright (beta) | -| youtube | `youtube.profile` | Email, channel URL, handle, and joined date (plus optional channel fields like title, avatar, and stats) | [JSON Schema](connectors/google/schemas/youtube.profile.json) | Yes | Yes | None | youtube-playwright (beta) | -| youtube | `youtube.subscriptions` | Channels you are subscribed to | [JSON Schema](connectors/google/schemas/youtube.subscriptions.json) | No | Yes | None | youtube-playwright (beta) | -| youtube | `youtube.watchLater` | Videos saved to Watch Later | [JSON Schema](connectors/google/schemas/youtube.watchLater.json) | No | Yes | None | youtube-playwright (beta) | - -## Blocked evidence - -- `linkedin.connections`: LinkedIn Web scope capability is not classified without live end-to-end evidence. Required: A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set. The resulting Personal Server scope inventory for the same run and user after sync. -- `linkedin.education`: LinkedIn Web scope capability is not classified without live end-to-end evidence. Required: A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set. The resulting Personal Server scope inventory for the same run and user after sync. -- `linkedin.experience`: LinkedIn Web scope capability is not classified without live end-to-end evidence. Required: A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set. The resulting Personal Server scope inventory for the same run and user after sync. -- `linkedin.languages`: LinkedIn Web scope capability is not classified without live end-to-end evidence. Required: A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set. The resulting Personal Server scope inventory for the same run and user after sync. -- `linkedin.profile`: LinkedIn Web scope capability is not classified without live end-to-end evidence. Required: A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set. The resulting Personal Server scope inventory for the same run and user after sync. -- `linkedin.skills`: LinkedIn Web scope capability is not classified without live end-to-end evidence. Required: A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set. The resulting Personal Server scope inventory for the same run and user after sync. +| Source | Scope | Description | Schema | Material limits | Desktop connector(s) / maturity | +|---|---|---|---|---|---| +| amazon | `amazon.orders` | Order history including items, prices, dates, and delivery status | [JSON Schema](connectors/amazon/schemas/amazon.orders.json) | None | amazon-playwright (beta) | +| amazon | `amazon.profile` | Account name and Prime membership status | [JSON Schema](connectors/amazon/schemas/amazon.profile.json) | None | amazon-playwright (beta) | +| chatgpt | `chatgpt.conversations` | All conversation history including messages with ChatGPT | [JSON Schema](connectors/openai/schemas/chatgpt.conversations.json) | None | chatgpt-playwright (stable) | +| chatgpt | `chatgpt.memories` | Saved memories that ChatGPT uses to personalize responses | [JSON Schema](connectors/openai/schemas/chatgpt.memories.json) | None | chatgpt-playwright (stable) | +| claude | `claude.conversations` | All Claude conversations with full message threads, from Anthropic's official data export. | [JSON Schema](connectors/anthropic/schemas/claude.conversations.json) | None | claude-export-playwright (experimental) | +| claude | `claude.projects` | Claude projects and their detail, from Anthropic's official data export. | [JSON Schema](connectors/anthropic/schemas/claude.projects.json) | None | claude-export-playwright (experimental) | +| doordash | `doordash.orders` | Complete order history with restaurant names, items ordered, dates, totals, and delivery status | [JSON Schema](connectors/doordash/schemas/doordash.orders.json) | None | doordash-playwright (beta) | +| github | `github.contributions` | Daily contribution counts for the last 4 years (per-year scrape, not just rolling 12 months), monthly totals, per-year totals, and your top day | [JSON Schema](connectors/github/schemas/github.contributions.json) | Desktop (local; github-playwright): The Desktop connector collects the four most recent calendar years. | github-playwright (stable) | +| github | `github.events` | Recent public activity (pushes, pull requests, issues, comments, branches, releases) across every repository you have touched, including organization-owned repos | [JSON Schema](connectors/github/schemas/github.events.json) | Desktop (local; github-playwright): The Desktop connector returns at most 300 public events.; Desktop (local; github-playwright): The GitHub public-events API covers at most the previous 90 days. | github-playwright (stable) | +| github | `github.history` | Pull requests and issues you have authored across public repositories via the GitHub Search API, including title, body, dates, repository, labels, and state. | [JSON Schema](connectors/github/schemas/github.history.json) | Desktop (local; github-playwright): The GitHub Search API returns at most 1,000 pull requests and 1,000 issues per query. | github-playwright (stable) | +| github | `github.profile` | Account profile information such as name, bio, follower counts, pinned repositories, organization memberships, and achievement badges | [JSON Schema](connectors/github/schemas/github.profile.json) | None | github-playwright (stable) | +| github | `github.repositories` | Your repository list with metadata like language, stars, and update time | [JSON Schema](connectors/github/schemas/github.repositories.json) | None | github-playwright (stable) | +| github | `github.starred` | Repositories you have starred on GitHub | [JSON Schema](connectors/github/schemas/github.starred.json) | None | github-playwright (stable) | +| heb | `heb.nutrition` | Nutrition facts (calories, macros, sodium, fiber, vitamins) for each product ordered. Aligned to Schema.org NutritionInformation. | [JSON Schema](connectors/heb/schemas/heb.nutrition.json) | None | heb-playwright (experimental) | +| heb | `heb.orders` | Curbside and delivery orders with item names, quantities, prices, and order dates | [JSON Schema](connectors/heb/schemas/heb.orders.json) | None | heb-playwright (experimental) | +| heb | `heb.profile` | Your name, email, phone number, and saved delivery addresses | [JSON Schema](connectors/heb/schemas/heb.profile.json) | None | heb-playwright (experimental) | +| icloud_notes | `icloud_notes.folders` | Folder structure for organizing notes | [JSON Schema](connectors/apple/schemas/icloud_notes.folders.json) | None | icloud-notes-playwright (experimental) | +| icloud_notes | `icloud_notes.notes` | Note content, titles, created/modified timestamps, and folder assignment | [JSON Schema](connectors/apple/schemas/icloud_notes.notes.json) | None | icloud-notes-playwright (experimental) | +| instagram | `instagram.ads` | Advertisers the user has seen ads from and ad topic interests based on their Instagram activity | [JSON Schema](connectors/meta/schemas/instagram.ads.json) | None | instagram-playwright (stable); instagram-ads-playwright (beta) | +| instagram | `instagram.following` | The list of Instagram accounts you follow | [JSON Schema](connectors/meta/schemas/instagram.following.json) | None | instagram-playwright (stable) | +| instagram | `instagram.posts` | Published posts from your feed | [JSON Schema](connectors/meta/schemas/instagram.posts.json) | None | instagram-playwright (stable) | +| instagram | `instagram.profile` | Profile information including bio and follower counts | [JSON Schema](connectors/meta/schemas/instagram.profile.json) | None | instagram-playwright (stable) | +| linkedin | `linkedin.connections` | Your LinkedIn connections with names, headlines, profile URLs, and date connected | [JSON Schema](connectors/linkedin/schemas/linkedin.connections.json) | None | linkedin-playwright (stable) | +| linkedin | `linkedin.education` | Your education history including schools, degrees, years, and grades | [JSON Schema](connectors/linkedin/schemas/linkedin.education.json) | None | linkedin-playwright (stable) | +| linkedin | `linkedin.experience` | Your work experience including job titles, companies, dates, locations, and descriptions | [JSON Schema](connectors/linkedin/schemas/linkedin.experience.json) | None | linkedin-playwright (stable) | +| linkedin | `linkedin.languages` | Your languages and proficiency levels | [JSON Schema](connectors/linkedin/schemas/linkedin.languages.json) | None | linkedin-playwright (stable) | +| linkedin | `linkedin.profile` | Your LinkedIn profile including name, headline, location, about, and profile picture | [JSON Schema](connectors/linkedin/schemas/linkedin.profile.json) | None | linkedin-playwright (stable) | +| linkedin | `linkedin.skills` | Your skills and endorsement counts | [JSON Schema](connectors/linkedin/schemas/linkedin.skills.json) | None | linkedin-playwright (stable) | +| oura | `oura.activity` | Daily activity scores, step counts, calorie burn, and activity level breakdowns | [JSON Schema](connectors/oura/schemas/oura.activity.json) | None | oura-playwright (stable) | +| oura | `oura.readiness` | Daily readiness scores and contributing factors like HRV balance, resting heart rate, recovery, and body temperature | [JSON Schema](connectors/oura/schemas/oura.readiness.json) | None | oura-playwright (stable) | +| oura | `oura.sleep` | Daily sleep scores with contributor breakdowns and detailed sleep period data including duration, stages, heart rate, and HRV | [JSON Schema](connectors/oura/schemas/oura.sleep.json) | None | oura-playwright (stable) | +| shop | `shop.orders` | Complete order history from the Shop app including merchants, items, totals, and delivery status | [JSON Schema](connectors/shopify/schemas/shop.orders.json) | None | shop-playwright (beta) | +| spotify | `spotify.playlists` | Playlists and their tracks | [JSON Schema](connectors/spotify/schemas/spotify.playlists.json) | None | spotify-playwright (stable) | +| spotify | `spotify.profile` | Profile information | [JSON Schema](connectors/spotify/schemas/spotify.profile.json) | None | spotify-playwright (stable) | +| spotify | `spotify.savedTracks` | Saved/liked tracks in your library | [JSON Schema](connectors/spotify/schemas/spotify.savedTracks.json) | None | spotify-playwright (stable) | +| steam | `steam.friends` | Friend list with persona names, avatars, and friendship dates | [JSON Schema](connectors/valve/schemas/steam.friends.json) | None | steam-playwright (experimental) | +| steam | `steam.games` | Owned games with playtime statistics, plus recently played games | [JSON Schema](connectors/valve/schemas/steam.games.json) | None | steam-playwright (experimental) | +| steam | `steam.profile` | Steam profile including persona name, avatar, level, country, and account age | [JSON Schema](connectors/valve/schemas/steam.profile.json) | None | steam-playwright (experimental) | +| uber | `uber.receipts` | Fare breakdowns and receipt details for each trip | [JSON Schema](connectors/uber/schemas/uber.receipts.json) | None | uber-playwright (beta) | +| uber | `uber.trips` | Complete history of your Uber rides including pickup/dropoff locations, dates, and driver info | [JSON Schema](connectors/uber/schemas/uber.trips.json) | None | uber-playwright (beta) | +| wholefoods | `wholefoods.nutrition` | Nutrition facts (calories, macros, sodium, fiber, vitamins) for each product ordered. Aligned to Schema.org NutritionInformation. | [JSON Schema](connectors/wholefoods/schemas/wholefoods.nutrition.json) | None | wholefoods-playwright (experimental) | +| wholefoods | `wholefoods.orders` | Whole Foods delivery and pickup orders with item names, quantities, prices, and order dates | [JSON Schema](connectors/wholefoods/schemas/wholefoods.orders.json) | None | wholefoods-playwright (experimental) | +| wholefoods | `wholefoods.profile` | Your Amazon account name and email associated with Whole Foods orders | [JSON Schema](connectors/wholefoods/schemas/wholefoods.profile.json) | None | wholefoods-playwright (experimental) | +| youtube | `youtube.history` | Raw watch history entries for the top 50 most recent items | [JSON Schema](connectors/google/schemas/youtube.history.json) | Desktop (local; youtube-playwright): The Desktop connector returns at most the 50 most recent watch history items. | youtube-playwright (beta) | +| youtube | `youtube.likes` | Videos you have liked | [JSON Schema](connectors/google/schemas/youtube.likes.json) | None | youtube-playwright (beta) | +| youtube | `youtube.playlistItems` | Videos inside your playlists, including titles and URLs | [JSON Schema](connectors/google/schemas/youtube.playlistItems.json) | None | youtube-playwright (beta) | +| youtube | `youtube.playlists` | Playlists visible in your YouTube Library | [JSON Schema](connectors/google/schemas/youtube.playlists.json) | None | youtube-playwright (beta) | +| youtube | `youtube.profile` | Email, channel URL, handle, and joined date (plus optional channel fields like title, avatar, and stats) | [JSON Schema](connectors/google/schemas/youtube.profile.json) | None | youtube-playwright (beta) | +| youtube | `youtube.subscriptions` | Channels you are subscribed to | [JSON Schema](connectors/google/schemas/youtube.subscriptions.json) | None | youtube-playwright (beta) | +| youtube | `youtube.watchLater` | Videos saved to Watch Later | [JSON Schema](connectors/google/schemas/youtube.watchLater.json) | None | youtube-playwright (beta) | ## Maintenance -Do not edit this file. Update connector manifests/`registry.json` for Vana Desktop truth or `scopes/web-capabilities.json` for Vana Web truth, then run `node scripts/generate-scope-catalog.mjs`. +Do not edit this file. Update connector manifests/`registry.json`, then run `node scripts/generate-scope-catalog.mjs`. diff --git a/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.0.tgz b/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.0.tgz deleted file mode 100644 index c00e130..0000000 Binary files a/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.0.tgz and /dev/null differ diff --git a/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.1.tgz b/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.1.tgz new file mode 100644 index 0000000..9679570 Binary files /dev/null and b/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.1.tgz differ diff --git a/artifacts/claude-export-playwright/claude-export-playwright-2.0.0.tgz b/artifacts/claude-export-playwright/claude-export-playwright-2.0.0.tgz deleted file mode 100644 index 9ff578e..0000000 Binary files a/artifacts/claude-export-playwright/claude-export-playwright-2.0.0.tgz and /dev/null differ diff --git a/artifacts/claude-export-playwright/claude-export-playwright-2.0.1.tgz b/artifacts/claude-export-playwright/claude-export-playwright-2.0.1.tgz new file mode 100644 index 0000000..cfc18ec Binary files /dev/null and b/artifacts/claude-export-playwright/claude-export-playwright-2.0.1.tgz differ diff --git a/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.0.tgz b/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.0.tgz deleted file mode 100644 index f8e10c5..0000000 Binary files a/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.0.tgz and /dev/null differ diff --git a/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.1.tgz b/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.1.tgz new file mode 100644 index 0000000..4a84591 Binary files /dev/null and b/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.1.tgz differ diff --git a/connector-index.json b/connector-index.json index b7b0af4..9d37de5 100644 --- a/connector-index.json +++ b/connector-index.json @@ -1,6 +1,6 @@ { "indexVersion": "2.0", - "sourceRepo": "https://github.com/vana-com/data-connectors", + "sourceRepo": "https://github.com/PDP-Connect/data-connectors", "generatedAt": "2026-07-13T00:00:00Z", "signature": null, "connectors": { @@ -110,6 +110,42 @@ "iconKey": "chatgpt", "defaultScope": "chatgpt.conversations" } + }, + { + "connectorId": "chatgpt-playwright", + "company": "openai", + "version": "3.0.1", + "name": "ChatGPT", + "status": "stable", + "description": "Exports your email, memories, and all conversations from ChatGPT.", + "sourceFiles": { + "script": "openai/chatgpt-playwright.js", + "metadata": "openai/chatgpt-playwright.json" + }, + "publishedAt": "2026-07-13T00:00:00Z", + "sourceTag": "chore/apache-license-and-neutralize", + "sourceCommit": "37b45c33d159281f718bf2f1da325520ce0f50c4", + "releaseId": "connectors-37b45c33d159", + "pageApiVersion": 1, + "manifestSha256": "sha256:6b3df1f5ce03e588c23f1e26e35494526166baee99e69c384b72b31f7b306371", + "scriptSha256": "sha256:f2f33ef300e8b89ecbc13fcfa80ee88b0ec0d7849f2018a7a16623b5e1146527", + "artifactSha256": "sha256:e99ba585710615ae3a4afcd1babe21097bdd52acc06bfa79837e32334e5ee03e", + "artifactPath": "artifacts/chatgpt-playwright/chatgpt-playwright-3.0.1.tgz", + "artifactUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/37b45c33d159281f718bf2f1da325520ce0f50c4/artifacts/chatgpt-playwright/chatgpt-playwright-3.0.1.tgz", + "scopes": [ + "chatgpt.memories", + "chatgpt.conversations" + ], + "consumerMetadata": { + "sourceId": "chatgpt", + "displayName": "ChatGPT", + "brandDomain": "chatgpt.com", + "aliases": [ + "openai" + ], + "iconKey": "chatgpt", + "defaultScope": "chatgpt.conversations" + } } ], "claude-export-playwright": [ @@ -139,6 +175,33 @@ "claude.projects" ], "consumerMetadata": null + }, + { + "connectorId": "claude-export-playwright", + "company": "Anthropic", + "version": "2.0.1", + "name": "Claude", + "status": "experimental", + "description": "Exports your Claude conversations and projects via Anthropic's official data export (one complete archive, no per-conversation rate limiting).", + "sourceFiles": { + "script": "anthropic/claude-export-playwright.js", + "metadata": "anthropic/claude-export-playwright.json" + }, + "publishedAt": "2026-07-13T00:00:00Z", + "sourceTag": "chore/apache-license-and-neutralize", + "sourceCommit": "37b45c33d159281f718bf2f1da325520ce0f50c4", + "releaseId": "connectors-37b45c33d159", + "pageApiVersion": 1, + "manifestSha256": "sha256:7213ef42d1bcbde91db4972ae22a8f0b445f2ff47d85c95548b34ce20054c315", + "scriptSha256": "sha256:12a068a5186aececdd13e999b84f14a281d8e3d256c1eec70b812afd17142152", + "artifactSha256": "sha256:23c8b63a91e3ea3bafce80aa1a6efd5afb79625cdc65719776480fa1b2f8e320", + "artifactPath": "artifacts/claude-export-playwright/claude-export-playwright-2.0.1.tgz", + "artifactUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/37b45c33d159281f718bf2f1da325520ce0f50c4/artifacts/claude-export-playwright/claude-export-playwright-2.0.1.tgz", + "scopes": [ + "claude.conversations", + "claude.projects" + ], + "consumerMetadata": null } ], "doordash-playwright": [ @@ -911,6 +974,43 @@ "iconKey": "wholefoods", "defaultScope": "wholefoods.orders" } + }, + { + "connectorId": "wholefoods-playwright", + "company": "wholefoods", + "version": "1.0.1", + "name": "Whole Foods Market", + "status": "experimental", + "description": "Exports your Whole Foods Market order history (via Amazon) and product nutrition data.", + "sourceFiles": { + "script": "wholefoods/wholefoods-playwright.js", + "metadata": "wholefoods/wholefoods-playwright.json" + }, + "publishedAt": "2026-07-13T00:00:00Z", + "sourceTag": "chore/apache-license-and-neutralize", + "sourceCommit": "37b45c33d159281f718bf2f1da325520ce0f50c4", + "releaseId": "connectors-37b45c33d159", + "pageApiVersion": 1, + "manifestSha256": "sha256:b374a547bacfe49c0fcce7cea41db0630e8335c9f655e644a43a11b2eae9e0e1", + "scriptSha256": "sha256:2ebb4426258b87a8e3699b9609c36ff69b03f050e747ed530c54ee1e7f84c59b", + "artifactSha256": "sha256:5f1262b008e96360385af589b2cee71e8512fde6505382ed6e4c39087181ddd0", + "artifactPath": "artifacts/wholefoods-playwright/wholefoods-playwright-1.0.1.tgz", + "artifactUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/37b45c33d159281f718bf2f1da325520ce0f50c4/artifacts/wholefoods-playwright/wholefoods-playwright-1.0.1.tgz", + "scopes": [ + "wholefoods.profile", + "wholefoods.orders", + "wholefoods.nutrition" + ], + "consumerMetadata": { + "sourceId": "wholefoods", + "displayName": "Whole Foods Market", + "brandDomain": "wholefoodsmarket.com", + "aliases": [ + "whole foods" + ], + "iconKey": "wholefoods", + "defaultScope": "wholefoods.orders" + } } ], "youtube-playwright": [ diff --git a/connectors/_conformance/conformance-playwright.json b/connectors/_conformance/conformance-playwright.json index fa675b5..c39dbb8 100644 --- a/connectors/_conformance/conformance-playwright.json +++ b/connectors/_conformance/conformance-playwright.json @@ -4,7 +4,7 @@ "source_id": "conformance", "version": "1.0.0", "name": "Page API Conformance", - "company": "Vana", + "company": "Conformance", "description": "Synthetic connector that exercises every method in the canonical page API minimum surface. Used by CI to detect runtime drift between shells (Context Gateway, DataConnect, future runners).", "runtime": "playwright", "page_api_version": 1, diff --git a/connectors/anthropic/CLAUDE_CONNECTOR_PLAN.md b/connectors/anthropic/CLAUDE_CONNECTOR_PLAN.md index 30824e1..d7b2181 100644 --- a/connectors/anthropic/CLAUDE_CONNECTOR_PLAN.md +++ b/connectors/anthropic/CLAUDE_CONNECTOR_PLAN.md @@ -56,10 +56,10 @@ Initial target: Source docs: -- `skills/vana-connect/SKILL.md` -- `skills/vana-connect/CREATE.md` -- `skills/vana-connect/reference/PATTERNS.md` -- `skills/vana-connect/reference/PAGE-API.md` +- `skills/pdp-connect/SKILL.md` +- `skills/pdp-connect/CREATE.md` +- `skills/pdp-connect/reference/PATTERNS.md` +- `skills/pdp-connect/reference/PAGE-API.md` Important repo constraints: @@ -249,7 +249,7 @@ Once files exist, the working loop is: What is working now: -- connector is scaffolded, registered, and runnable through `vana` +- connector is scaffolded, registered, and runnable through `run-connector.cjs` - manual-login-first Claude session detection works - full thread/message export works for conversations via: - `GET /api/organizations/:orgId/chat_conversations_v2?...` @@ -318,15 +318,15 @@ Structural validation: Register updated connector locally: -- `node skills/vana-connect/scripts/register.cjs connectors/anthropic/claude-playwright.js` +- `node skills/pdp-connect/scripts/register.cjs connectors/anthropic/claude-playwright.js` Run the real connector: -- `vana connect claude --json --no-input` +- `node run-connector.cjs connectors/anthropic/claude-playwright.js --inputs '{...}'` Validate the exported result: -- `node scripts/validate-connector.cjs connectors/anthropic/claude-playwright.js --check-result ~/.vana/results/claude.json` +- `node scripts/validate-connector.cjs connectors/anthropic/claude-playwright.js --check-result ~/.pdp-connect/desktop/last-result.json` ## Next Steps @@ -386,15 +386,15 @@ Reason: ## Claude-Specific Learnings -- the local repo can be exercised directly through `vana` when run from the repo root; this is the right execution path for this project +- the local repo can be exercised directly through `run-connector.cjs` when run from the repo root; this is the right execution path for this project - Claude session state is reusable through the existing local browser profile, which makes manual-login-first iteration practical - the active organization id is recoverable from `lastActiveOrg` in cookie or storage, and that is enough to unlock the main authenticated API paths - the conversation-detail API returns message trees under `chat_messages` - flattening message text requires handling Claude content blocks rather than assuming a single plain-text field - sidebar DOM scraping is still useful as a fallback, but API collection is the primary path now - fetching every conversation serially is too slow; batching detail requests is necessary for practical export times -- after changing the connector, re-run the repo registration step so `vana` picks up the new script hash -- prefer validating with a real `vana connect claude --json --no-input` run, not only the structural validator +- after changing the connector, re-run the repo registration step so the registry picks up the new script hash +- prefer validating with a real `run-connector.cjs` run, not only the structural validator - if a field may legitimately be `null`, avoid marking it as required in the schema because the current validator can report it as missing - manual login is an acceptable and common connector tradeoff when auth is brittle or includes anti-bot checks, CAPTCHA, SSO, or third-party identity flows - for Claude specifically, email/password automation is a reasonable fast-follow candidate; Google automation should be treated as later work unless there is a very strong product reason to take it on diff --git a/connectors/anthropic/claude-export-playwright.js b/connectors/anthropic/claude-export-playwright.js index 121e18d..ef3b198 100644 --- a/connectors/anthropic/claude-export-playwright.js +++ b/connectors/anthropic/claude-export-playwright.js @@ -23,7 +23,7 @@ const CLAUDE_HOME_URL = 'https://claude.ai/new'; const CLAUDE_LOGIN_URL = 'https://claude.ai/login'; -const CKPT_DB = 'vana_claude_export_ckpt'; +const CKPT_DB = 'pdpconnect_claude_export_ckpt'; // The export is an async job; poll within the run so it completes in one click. const POLL_ATTEMPT_TIMEOUT_MS = 25000; // per attempt: navigate + wait for the download to fire const POLL_INTERVAL_MS = 8000; // pause between attempts while the job is still preparing diff --git a/connectors/anthropic/claude-export-playwright.json b/connectors/anthropic/claude-export-playwright.json index 2321ece..d2c7f80 100644 --- a/connectors/anthropic/claude-export-playwright.json +++ b/connectors/anthropic/claude-export-playwright.json @@ -2,7 +2,7 @@ "manifest_version": "1.0", "connector_id": "claude-export-playwright", "source_id": "claude", - "version": "2.0.0", + "version": "2.0.1", "name": "Claude", "company": "Anthropic", "description": "Exports your Claude conversations and projects via Anthropic's official data export (one complete archive, no per-conversation rate limiting).", diff --git a/connectors/openai/__tests__/chatgpt-resume.browser.test.cjs b/connectors/openai/__tests__/chatgpt-resume.browser.test.cjs index 09c3c43..1556230 100644 --- a/connectors/openai/__tests__/chatgpt-resume.browser.test.cjs +++ b/connectors/openai/__tests__/chatgpt-resume.browser.test.cjs @@ -176,7 +176,7 @@ async function runConnectorOnce(userDataDir, plan) { const returned = await connectorFn.call(null, api); // Read the on-disk checkpoint back the same way the connector does. const ckpt = await realPage.evaluate(`(async () => { - const db = await new Promise((res, rej) => { const r = indexedDB.open('vana_chatgpt_ckpt', 1); r.onsuccess = () => res(r.result); r.onerror = () => rej(r.error); }); + const db = await new Promise((res, rej) => { const r = indexedDB.open('pdpconnect_chatgpt_ckpt', 1); r.onsuccess = () => res(r.result); r.onerror = () => rej(r.error); }); const ids = await new Promise((res) => { const out = []; const cur = db.transaction('conversations').objectStore('conversations').openCursor(); cur.onsuccess = (e) => { const c = e.target.result; if (!c) return res(out); out.push(c.value.id); c.continue(); }; }); const meta = await new Promise((res) => { const g = db.transaction('meta').objectStore('meta').get('state'); g.onsuccess = () => res(g.result && g.result.v ? g.result.v : {}); }); return { ids: ids.sort(), meta }; diff --git a/connectors/openai/chatgpt-playwright.js b/connectors/openai/chatgpt-playwright.js index 31c6971..963e690 100644 --- a/connectors/openai/chatgpt-playwright.js +++ b/connectors/openai/chatgpt-playwright.js @@ -38,7 +38,7 @@ */ // ─── Tunables ──────────────────────────────────────────────────────── -const CKPT_DB = 'vana_chatgpt_ckpt'; +const CKPT_DB = 'pdpconnect_chatgpt_ckpt'; const CKPT_FORMAT = 1; // "Concurrency" now sizes the /conversations/batch request (ids per POST). The diff --git a/connectors/openai/chatgpt-playwright.json b/connectors/openai/chatgpt-playwright.json index cd23b38..6406fde 100644 --- a/connectors/openai/chatgpt-playwright.json +++ b/connectors/openai/chatgpt-playwright.json @@ -2,7 +2,7 @@ "manifest_version": "1.0", "connector_id": "chatgpt-playwright", "source_id": "chatgpt", - "version": "3.0.0", + "version": "3.0.1", "name": "ChatGPT", "company": "OpenAI", "description": "Exports your email, memories, and all conversations from ChatGPT.", diff --git a/connectors/wholefoods/schemas/wholefoods.nutrition.json b/connectors/wholefoods/schemas/wholefoods.nutrition.json index 7614ed6..9cf4743 100644 --- a/connectors/wholefoods/schemas/wholefoods.nutrition.json +++ b/connectors/wholefoods/schemas/wholefoods.nutrition.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scope": "wholefoods.nutrition", "dialect": "json", - "definitionUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/wholefoods/schemas/wholefoods.nutrition.json", + "definitionUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/wholefoods/schemas/wholefoods.nutrition.json", "description": "Nutrition facts per unique product ordered from Whole Foods Market, aligned to Schema.org NutritionInformation", "schema": { "type": "object", diff --git a/connectors/wholefoods/schemas/wholefoods.orders.json b/connectors/wholefoods/schemas/wholefoods.orders.json index 26bbd8f..20b4b63 100644 --- a/connectors/wholefoods/schemas/wholefoods.orders.json +++ b/connectors/wholefoods/schemas/wholefoods.orders.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scope": "wholefoods.orders", "dialect": "json", - "definitionUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/wholefoods/schemas/wholefoods.orders.json", + "definitionUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/wholefoods/schemas/wholefoods.orders.json", "description": "Whole Foods Market delivery and pickup order history with item details", "schema": { "type": "object", diff --git a/connectors/wholefoods/schemas/wholefoods.profile.json b/connectors/wholefoods/schemas/wholefoods.profile.json index 9db460c..8e76af8 100644 --- a/connectors/wholefoods/schemas/wholefoods.profile.json +++ b/connectors/wholefoods/schemas/wholefoods.profile.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scope": "wholefoods.profile", "dialect": "json", - "definitionUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/wholefoods/schemas/wholefoods.profile.json", + "definitionUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/wholefoods/schemas/wholefoods.profile.json", "description": "Amazon account information associated with Whole Foods Market orders", "schema": { "type": "object", diff --git a/connectors/wholefoods/wholefoods-playwright.json b/connectors/wholefoods/wholefoods-playwright.json index 87f1923..f391fd6 100644 --- a/connectors/wholefoods/wholefoods-playwright.json +++ b/connectors/wholefoods/wholefoods-playwright.json @@ -2,7 +2,7 @@ "manifest_version": "1.0", "connector_id": "wholefoods-playwright", "source_id": "wholefoods", - "version": "1.0.0", + "version": "1.0.1", "name": "Whole Foods Market", "company": "wholefoods", "description": "Exports your Whole Foods Market order history (via Amazon) and product nutrition data.", diff --git a/docs/260410-contract-freeze.md b/docs/260410-contract-freeze.md deleted file mode 100644 index c53199d..0000000 --- a/docs/260410-contract-freeze.md +++ /dev/null @@ -1,204 +0,0 @@ -# Connector Contract Freeze — Phase 0 Decisions - -Date: 2026-04-10 -Supersedes: ad-hoc manifest and page API conventions. -Related eval file: `context-gateway/docs/evals/260410-connector-contract-unification.evals.yaml` - -This document records the contract decisions that Phase 0 of the connector -unification plan locked in. Subsequent phases consume these as given. - -## 1. Canonical manifest shape - -The canonical contract is `schemas/manifest.schema.json` in this repository. -Every `*-playwright.json` manifest must validate against it. - -Required fields (additive from the pre-migration shape): - -| Field | Example | Notes | -| --- | --- | --- | -| `manifest_version` | `"1.0"` | Version of the manifest contract itself. | -| `connector_id` | `"instagram-playwright"` | Globally unique artifact id. | -| `source_id` | `"instagram"` | Stable product-facing source id. Must not change across releases without a versioned migration path. | -| `version` | `"1.0.0"` | Connector version (semver). Independent of the other versions. | -| `name` | `"Instagram"` | Display name. | -| `company` | `"Meta"` | Company / organization. | -| `description` | `"..."` | Human description. | -| `runtime` | `"playwright"` | Enum: `playwright`, `vanilla`, `network-capture`. | -| `page_api_version` | `1` | Major version of the page API this connector targets. Independent of `version`. | -| `connect_url` | `"https://..."` | Login / start URL. | -| `connect_selector` | `"..."` | Logged-in-state detection selector. | -| `scopes` | `[{scope, label, description}, ...]` | Canonical public data units. | - -Legacy aliases (`id`, `connectURL`, `connectSelector`, `iconURL`, -`exportFrequency`, `vectorize_config`) are kept in place during migration but -are no longer the source of truth. `scripts/normalize-manifests.mjs` keeps the -canonical and legacy fields in sync. - -### `connector_id` vs `source_id` - -- `connector_id` is the build/runtime artifact id. It includes runtime - qualifiers (e.g. `-playwright`) and secondary suffixes (e.g. `-ads`). -- `source_id` is the product-facing source id as users see it (e.g. the CG - source dropdown). Multiple connectors MAY emit into the same `source_id` if - they produce scopes under the same platform namespace. - -Explicit override: `instagram-ads-playwright` has `source_id: "instagram"` -because it is a second artifact producing `instagram.*` scopes, not a distinct -source. - -### Scope id format - -Every scope id must match `^[a-z0-9_-]+\.[A-Za-z0-9_.-]+$` and must begin with -the manifest's `source_id` followed by a dot. Historical camelCase scope tails -(`youtube.playlistItems`, `spotify.savedTracks`) are preserved during the -migration. - -## 2. Page API minimum surface - -`types/connector.d.ts` exposes the canonical page API. Phase 0 decision: the -following methods become canonical (they were missing or de facto only in -Context Gateway's CG proxy, not in the typed contract): - -- `click(selector, options?)` -- `fill(selector, value, options?)` -- `press(selector, key, options?)` -- `waitForSelector(selector, options?)` -- `url()` - -Rationale: CG connector scripts already rely on these methods, and DataConnect -can implement them as thin pass-throughs to the underlying Playwright page -object. Adding them to the canonical surface is easier than rewriting CG -scripts to the narrower upstream style, and it costs DC only a handful of -one-line method wrappers. - -`requestInput` is canonical. `getInput` is a shell-runtime adapter concern -only. - -Versioning: `types/page-api-version.ts` exports `PAGE_API_VERSION = 1`. Within -a major version the contract is additive-only. Breaking changes require both a -constant bump and a matching `page_api_version` bump in every manifest. CI -enforces this via `scripts/check-page-api-additive.mjs`. - -## 3. Instagram contract decision - -CG has historically exposed Instagram via a local script that scraped extra -fields (`following_accounts`, ad `targeting_categories`) and did NOT emit -posts. The canonical `data-connectors` Instagram connector emits profile, -posts, and ads but did not previously emit following. - -### `instagram.following` — upstreamed into the canonical script - -- `scrapeFollowingAccounts` from CG's `public/automations/instagram-headless.js` - has been ported into the canonical `connectors/meta/instagram-playwright.js` - (see `state.followingAccounts` + the `scrapeFollowingAccounts` helper near - the top of the file). -- The canonical script now emits `instagram.following` alongside profile, - posts, and ads. -- Schema: `schemas/instagram.following.json`. -- Rationale: keeping CG-only fields in a CG-only fork would perpetuate the - drift that this whole plan exists to eliminate. The port was mechanical — - the function only uses `page.evaluate` — so there was no reason to defer. - -### Ad targeting categories — open contract gap - -- `schemas/instagram.ads.json` declares an optional `categories` field, but - neither the canonical Instagram script nor the current CG script actually - emits it under that name. CG collects a similar concept as - `targeting_categories`, but that is not the canonical field name and is - not currently exported by either script in this cycle. -- Phase 3 deliberately does NOT attempt to rename or populate `categories`. - If product wants ad targeting categories on the public contract, that is - a follow-up task: implement canonical collection and then either populate - the existing optional `categories` field or declare a new scope. -- The earlier version of this doc incorrectly claimed `categories` was - "already present" on `instagram.ads`. It is present in the schema as an - optional field, but no emitter populates it. - -### Canonical Instagram script is now activated in CG - -Phase 3 replaces CG's `public/automations/instagram-headless.js` with the -canonical `connectors/meta/instagram-playwright.js` (byte-for-byte via the -generator's snapshot emission). CG users gain `instagram.posts` (which they -never had before), retain `instagram.profile`, `instagram.following`, and -`instagram.ads`. The runtime method gap is closed by new `setProgress`, -`showBrowser`, `goHeadless` shims on CG's `PlaywrightPageProxy`. - -### GitHub is also activated - -Phase 3 replaces `public/automations/github.js` with the canonical -`connectors/github/github-playwright.js`. Same mechanism. The CG proxy shims -also cover GitHub's method surface. - -### iCloud Notes is upstreamed but NOT yet runtime-canonical - -- The iCloud Notes manifest, schemas, and script are committed to - `data-connectors`, satisfying HC-PHASE3-ICLOUD-UPSTREAM-001. -- HOWEVER: the script depends on CG-runtime-only page methods that are - intentionally NOT part of the canonical typed Page API: - - `page.getInput(inputSchema)` — CG-specific credential prompt primitive - - `page.frame_click`, `page.frame_fill`, `page.frame_evaluate`, - `page.frame_waitForSelector` — CG-specific iframe helpers (used for - Apple's auth widget iframe) - - `page.keyboard_press`, `page.keyboard_type` — CG-specific keyboard - primitives (used for 2FA OTP entry) -- These methods are NOT in `types/connector.d.ts`, they are NOT implemented - in `data-connect/playwright-runner/index.cjs`, and the upstream script - will NOT run under the canonical DataConnect runtime without further - convergence. -- The manifest advertises this explicitly via the - `capabilities: ["cg-legacy-page-api"]` flag. Any runner that checks - capabilities before activating a connector should reject iCloud Notes - if it does not implement the CG-legacy surface. -- This is an honest "metadata converged, script runtime pending" state. - Full canonical convergence requires either (a) rewriting the iCloud Notes - login flow to use `requestInput` and eliminate the iframe helpers, or - (b) promoting `frame_*` and `keyboard_*` into the canonical Page API. - Neither is in scope for this PR set. - -### Oura is NOT activated - -- The canonical Oura script uses email + password login. -- The existing CG Oura script uses email + OTP-only login. -- Some Oura accounts are OTP-only (no password set); activating the canonical - script would block them from connecting. -- Phase 3 keeps `public/automations/oura.js` as the existing CG script with - an exception comment. The canonical script is still pinned in the snapshot - at `config/data-connectors-snapshot/scripts/oura-playwright.js` so that - future convergence has a stable baseline to diff against. -- The CG overlay flags this with `activate_canonical_script: false`. -- Runtime convergence follow-up: either add OTP as an option in the canonical - script, or have CG detect OTP accounts and fall through to a CG-only - adapter. Not in this cycle. - -## 4. Shell overlay scope - -Per HC-SHELL-OVERLAY-001, shell overlays (Context Gateway, DataConnect display -registry) own ONLY: - -- Availability / whitelist -- Canonical source or connector references (pointers, not definitions) -- Color / theme treatment -- Privacy note copy -- Route config -- Embed copy -- Optional icon override - -They do NOT own: - -- Source descriptions (live in manifests) -- Public scope or stream definitions (live in manifests) -- Public schema definitions (live in `schemas/`) - -## 5. Versioning axes - -Per HC-VERSIONING-INDEPENDENT-001, four version dimensions are tracked -independently: - -| Axis | Location | Bumps when | -| --- | --- | --- | -| Manifest schema | `manifest_version` field | The manifest contract itself changes. | -| Connector | `version` field | A connector's behavior changes. | -| Page API | `page_api_version` field + `PAGE_API_VERSION` constant | The page API contract changes. | -| Scope schema | `version` field inside each `schemas/*.json` | A public scope schema changes. | - -No two of these are allowed to be collapsed into a single field. diff --git a/docs/specs/2026-05-29-scope-coverage-catalog-design.md b/docs/specs/2026-05-29-scope-coverage-catalog-design.md deleted file mode 100644 index 5efe2a4..0000000 --- a/docs/specs/2026-05-29-scope-coverage-catalog-design.md +++ /dev/null @@ -1,78 +0,0 @@ -# Scope Coverage Registry — Design Spec - -**Date:** 2026-05-29 -**Author:** Maciej (brainstormed 2026-05-29) -**Status:** Accepted -**Linear:** [BUI-395 — Instagram scope mismatch found in CG lab E2E](https://linear.app/vana-team/issue/BUI-395/instagram-scope-mismatch-found-in-cg-lab-e2e) -**Superseded by BUI-705:** `SCOPES.md` is now generated from the public -machine-readable catalog. This document records the earlier BUI-395 decision -and is not the current maintenance contract. - -**Historical artifact:** `SCOPES.md` (repo root) - ---- - -## 1. Summary - -A single, **hand-maintained** reference file — `SCOPES.md` — in the -`data-connectors` repo. Open it and you see every data scope, whether the **web** -(ODL Data Pipe API) and/or **desktop** (Playwright connectors) flow can fulfill -it, and the exact connector(s) behind each scope. - -It is documentation, not machinery: no generator, no codegen, no automated sync. - -## 2. Background — the bug - -During a CG → Vana Web → Personal Server lab E2E, the lab requested `read:ads` -(→ `instagram.ads`). CG forwarded it correctly, but the **web (light)** flow can -only produce `instagram.profile`, so the Personal Server had no `instagram.ads` -data and the read returned a valid 404. - -The architecture is two-tier: -- **Desktop / heavy** — Playwright connectors in `data-connectors`. Already canonical. -- **Web / light** — the ODL-run Data Pipe API (`data-pipe.vana.org`) used by the - Vana Web app. Produces only `*.profile` today, for five wired sources. - -"Which tier can fulfill which scope" was undocumented; the only place the web set -existed was a hardcoded list in `unity-surfaces` -(`web-writable-scopes.ts`). The open question on BUI-395 was *where* this should -be documented. This spec answers: one file in data-connectors. - -## 3. Decision — manual file, deliberately not generated - -We considered generating the catalog from the connector manifests + a declared -web-capabilities input, and generating `unity-surfaces`' -`web-writable-scopes.ts` from it. **Rejected for now**: the web app's set of -web-fulfillable scopes changes infrequently, so the cost/complexity of a -generator, a vendored snapshot, and a sync step is not worth it. A single file a -human reads and edits is enough. - -Consequences: -- No scripts are added to any repo. -- `unity-surfaces` `web-writable-scopes.ts` is unchanged and stays - hand-maintained. Keeping it consistent with `SCOPES.md`'s Web column is a - manual responsibility. If it drifts, we fix it by hand. - -## 4. The file - -`SCOPES.md` at the `data-connectors` repo root. One table: - -| Column | Meaning | -|---|---| -| **Source** | source id (e.g. `instagram`) | -| **Scope** | canonical scope id (e.g. `instagram.ads`) | -| **Web** | ✅ if the ODL Data Pipe API can produce it (today: `*.profile` for instagram, spotify, youtube, linkedin, github) | -| **Desktop** | ✅ if a Playwright connector produces it | -| **Connector(s)** | the exact connector id(s) + status that fulfill the scope | - -Plus a header explaining how to read it and a footer explaining how to maintain -it (update Desktop rows when a connector changes scopes; flip Web cells when the -Data Pipe API gains a scope; keep `web-writable-scopes.ts` consistent by hand). - -## 5. Out of scope - -- Any generator, sync script, or codegen. -- Changes to `unity-surfaces` (`web-writable-scopes.ts` stays as-is). -- A Data Pipe API capabilities endpoint. -- CG surfacing the tier at request time (e.g. warning "ads needs Desktop" in the - lab) — a separate follow-on if desired. diff --git a/docs/superpowers/plans/2026-03-31-connector-smoke-test.md b/docs/superpowers/plans/2026-03-31-connector-smoke-test.md deleted file mode 100644 index 1ffe516..0000000 --- a/docs/superpowers/plans/2026-03-31-connector-smoke-test.md +++ /dev/null @@ -1,878 +0,0 @@ -# Connector Smoke Test Harness — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build a batch test harness that runs stable connectors headlessly, validates output against declared scopes, and reports per-connector health as a colored terminal table + JSON report. - -**Architecture:** A single CJS script (`scripts/test-connectors.cjs`) that reads `registry.json` and connector metadata, loops over connectors, spawns `run-connector.cjs` for each with a unique `--output` path, classifies outcomes from exit codes + stdout, validates result files against metadata scopes, and prints a summary table + writes a JSON report. - -**Tech Stack:** Node.js (CJS), child_process.spawn, fs, path. No external dependencies. - ---- - -## File Structure - -| File | Responsibility | -|---|---| -| `scripts/test-connectors.cjs` (create) | CLI entrypoint: arg parsing, connector resolution from registry, orchestration loop, result validation, terminal + JSON reporting | -| `.gitignore` (modify) | Add `test-results/` | - -All logic lives in one file. The harness is ~300 lines with no external dependencies. - ---- - -### Task 1: Gitignore and test-results directory - -**Files:** -- Modify: `.gitignore` - -- [ ] **Step 1: Add test-results/ to .gitignore** - -Add this line to `.gitignore` after the existing `connector-result.json` entry: - -``` -test-results/ -``` - -The full `# Test runner output` section becomes: - -```gitignore -# Test runner output -connector-result.json -test-results/ -``` - -- [ ] **Step 2: Commit** - -```bash -git add .gitignore -git commit -m "chore: gitignore test-results directory" -``` - ---- - -### Task 2: Arg parsing and connector resolution - -**Files:** -- Create: `scripts/test-connectors.cjs` - -- [ ] **Step 1: Write the test** - -Create `scripts/test-connectors.test.cjs` (we'll run it with `node --test`): - -```js -const { describe, it } = require('node:test'); -const assert = require('node:assert'); - -// We'll extract pure functions from the main script for testing. -// For now, test the module by requiring it with a test flag. -// The main script exports { parseArgs, resolveConnectors, classifyOutcome, validateResult } -// when require.main !== module. - -const { - parseArgs, - resolveConnectors, - classifyOutcome, - validateResult, -} = require('./test-connectors.cjs'); - -describe('parseArgs', () => { - it('returns stable defaults with no args', () => { - const result = parseArgs([]); - assert.strictEqual(result.includeBeta, false); - assert.strictEqual(result.validateSchemas, false); - assert.deepStrictEqual(result.connectors, null); - }); - - it('parses --connectors flag', () => { - const result = parseArgs(['--connectors', 'instagram-playwright,spotify-playwright']); - assert.deepStrictEqual(result.connectors, ['instagram-playwright', 'spotify-playwright']); - }); - - it('parses --include-beta flag', () => { - const result = parseArgs(['--include-beta']); - assert.strictEqual(result.includeBeta, true); - }); - - it('parses --validate-schemas flag', () => { - const result = parseArgs(['--validate-schemas']); - assert.strictEqual(result.validateSchemas, true); - }); -}); - -describe('resolveConnectors', () => { - const registry = { - connectors: [ - { id: 'a-playwright', status: 'stable', files: { script: 'a/a.js', metadata: 'a/a.json' } }, - { id: 'b-playwright', status: 'beta', files: { script: 'b/b.js', metadata: 'b/b.json' } }, - { id: 'c-playwright', status: 'experimental', files: { script: 'c/c.js', metadata: 'c/c.json' } }, - ], - }; - - it('returns only stable by default', () => { - const result = resolveConnectors(registry, { connectors: null, includeBeta: false }); - assert.deepStrictEqual(result.map(c => c.id), ['a-playwright']); - }); - - it('includes beta when flag set', () => { - const result = resolveConnectors(registry, { connectors: null, includeBeta: true }); - assert.deepStrictEqual(result.map(c => c.id), ['a-playwright', 'b-playwright']); - }); - - it('filters to specific connectors', () => { - const result = resolveConnectors(registry, { connectors: ['b-playwright'], includeBeta: false }); - assert.deepStrictEqual(result.map(c => c.id), ['b-playwright']); - }); - - it('throws on unknown connector id', () => { - assert.throws( - () => resolveConnectors(registry, { connectors: ['nonexistent'], includeBeta: false }), - /not found in registry: nonexistent/ - ); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: FAIL — `Cannot find module './test-connectors.cjs'` - -- [ ] **Step 3: Write arg parsing and connector resolution** - -Create `scripts/test-connectors.cjs`: - -```js -#!/usr/bin/env node -/** - * test-connectors.cjs — Batch smoke test for data connectors. - * - * Wraps run-connector.cjs to run each connector headlessly, validates output - * against declared scopes, and reports per-connector health. - * - * Usage: - * node scripts/test-connectors.cjs [options] - * - * Options: - * --connectors , Override default connector set (comma-separated IDs) - * --include-beta Include beta connectors (default: stable only) - * --validate-schemas Validate result data against schemas/*.json - * - * Exit codes: 0 all pass/warn, 1 any fail/auth/timeout - */ - -const { spawn } = require('child_process'); -const fs = require('fs'); -const path = require('path'); - -const ROOT = path.resolve(__dirname, '..'); -const REGISTRY_PATH = path.join(ROOT, 'registry.json'); -const CONNECTORS_DIR = path.join(ROOT, 'connectors'); -const RESULTS_DIR = path.join(ROOT, 'test-results'); -const RUN_CONNECTOR = path.join(ROOT, 'run-connector.cjs'); - -// ─── ANSI Colors ──────────────────────────────────────────── - -const c = { - reset: '\x1b[0m', bold: '\x1b[1m', dim: '\x1b[2m', - red: '\x1b[31m', green: '\x1b[32m', yellow: '\x1b[33m', - blue: '\x1b[34m', magenta: '\x1b[35m', cyan: '\x1b[36m', gray: '\x1b[90m', -}; - -// ─── Arg Parsing ──────────────────────────────────────────── - -function parseArgs(argv) { - const opts = { connectors: null, includeBeta: false, validateSchemas: false }; - for (let i = 0; i < argv.length; i++) { - if (argv[i] === '--connectors' && argv[i + 1]) { - opts.connectors = argv[++i].split(',').map(s => s.trim()).filter(Boolean); - } else if (argv[i] === '--include-beta') { - opts.includeBeta = true; - } else if (argv[i] === '--validate-schemas') { - opts.validateSchemas = true; - } - } - return opts; -} - -// ─── Connector Resolution ─────────────────────────────────── - -function resolveConnectors(registry, opts) { - if (opts.connectors) { - // Explicit list — find each by ID, regardless of status - return opts.connectors.map(id => { - const entry = registry.connectors.find(c => c.id === id); - if (!entry) throw new Error(`Connector not found in registry: ${id}`); - return entry; - }); - } - // Default: stable, or stable + beta - const allowed = opts.includeBeta ? ['stable', 'beta'] : ['stable']; - return registry.connectors.filter(c => allowed.includes(c.status)); -} - -// ─── Exports for testing ──────────────────────────────────── - -if (require.main !== module) { - module.exports = { parseArgs, resolveConnectors }; -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: All `parseArgs` and `resolveConnectors` tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add scripts/test-connectors.cjs scripts/test-connectors.test.cjs -git commit -m "feat(test-connectors): arg parsing and connector resolution" -``` - ---- - -### Task 3: Outcome classification and result validation - -**Files:** -- Modify: `scripts/test-connectors.test.cjs` -- Modify: `scripts/test-connectors.cjs` - -- [ ] **Step 1: Write the tests** - -Append to `scripts/test-connectors.test.cjs`: - -```js -describe('classifyOutcome', () => { - it('returns AUTH for exit code 2', () => { - const result = classifyOutcome(2, []); - assert.strictEqual(result.status, 'auth'); - assert.match(result.error, /need-input/); - }); - - it('returns AUTH for exit code 3', () => { - const result = classifyOutcome(3, []); - assert.strictEqual(result.status, 'auth'); - assert.match(result.error, /legacy-auth/); - }); - - it('returns TIMEOUT for exit 1 with timeout message', () => { - const stdout = [ - JSON.stringify({ type: 'error', message: 'Timeout after 5 minutes' }), - ]; - const result = classifyOutcome(1, stdout); - assert.strictEqual(result.status, 'timeout'); - }); - - it('returns FAIL for exit 1 without timeout', () => { - const stdout = [ - JSON.stringify({ type: 'error', message: 'Some other error' }), - ]; - const result = classifyOutcome(1, stdout); - assert.strictEqual(result.status, 'fail'); - }); - - it('returns needs-validation for exit 0', () => { - const result = classifyOutcome(0, []); - assert.strictEqual(result.status, 'needs-validation'); - }); -}); - -describe('validateResult', () => { - const metadata = { - scopes: [ - { scope: 'test.profile' }, - { scope: 'test.posts' }, - { scope: 'test.likes' }, - ], - }; - - it('returns PASS when all scopes present and non-empty', () => { - const data = { - 'test.profile': { name: 'Alice' }, - 'test.posts': [{ id: 1 }], - 'test.likes': [{ id: 2 }], - exportSummary: { count: 2 }, - }; - const result = validateResult(data, metadata); - assert.strictEqual(result.status, 'pass'); - assert.strictEqual(result.scopesFound.length, 3); - assert.strictEqual(result.scopesMissing.length, 0); - }); - - it('returns WARN when scope has empty array', () => { - const data = { - 'test.profile': { name: 'Alice' }, - 'test.posts': [], - 'test.likes': [{ id: 2 }], - }; - const result = validateResult(data, metadata); - assert.strictEqual(result.status, 'warn'); - assert.deepStrictEqual(result.warnings, ['test.posts: empty array']); - }); - - it('returns WARN when scope has empty object', () => { - const data = { - 'test.profile': {}, - 'test.posts': [{ id: 1 }], - 'test.likes': [{ id: 2 }], - }; - const result = validateResult(data, metadata); - assert.strictEqual(result.status, 'warn'); - assert.deepStrictEqual(result.warnings, ['test.profile: empty object']); - }); - - it('returns FAIL when scope is missing', () => { - const data = { - 'test.profile': { name: 'Alice' }, - 'test.posts': [{ id: 1 }], - }; - const result = validateResult(data, metadata); - assert.strictEqual(result.status, 'fail'); - assert.deepStrictEqual(result.scopesMissing, ['test.likes']); - }); - - it('returns FAIL when scope value is null', () => { - const data = { - 'test.profile': { name: 'Alice' }, - 'test.posts': [{ id: 1 }], - 'test.likes': null, - }; - const result = validateResult(data, metadata); - assert.strictEqual(result.status, 'fail'); - assert.deepStrictEqual(result.scopesMissing, ['test.likes']); - }); - - it('normalizes bare keys for github-style connectors', () => { - const githubMeta = { - scopes: [ - { scope: 'github.profile' }, - { scope: 'github.repositories' }, - ], - }; - const data = { - profile: { name: 'Alice' }, - repositories: [{ name: 'repo1' }], - }; - const result = validateResult(data, githubMeta); - assert.strictEqual(result.status, 'pass'); - assert.strictEqual(result.scopesFound.length, 2); - }); -}); -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: FAIL — `classifyOutcome is not a function`, `validateResult is not a function` - -- [ ] **Step 3: Implement classifyOutcome and validateResult** - -Add to `scripts/test-connectors.cjs`, before the `if (require.main !== module)` block: - -```js -// ─── Outcome Classification ───────────────────────────────── - -function classifyOutcome(exitCode, stdoutLines) { - if (exitCode === 2) return { status: 'auth', error: 'need-input: connector requires credentials' }; - if (exitCode === 3) return { status: 'auth', error: 'legacy-auth: connector uses headed login not supported in batch mode' }; - if (exitCode === 1) { - const isTimeout = stdoutLines.some(line => { - try { - const msg = JSON.parse(line); - return msg.type === 'error' && msg.message && msg.message.includes('Timeout after 5 minutes'); - } catch { return false; } - }); - return isTimeout - ? { status: 'timeout', error: 'Timed out after 5 minutes' } - : { status: 'fail', error: 'Connector exited with error' }; - } - if (exitCode === 0) return { status: 'needs-validation' }; - return { status: 'fail', error: `Unknown exit code: ${exitCode}` }; -} - -// ─── Result Validation ────────────────────────────────────── - -function validateResult(data, metadata) { - const expectedScopes = (metadata.scopes || []).map(s => s.scope); - const scopesFound = []; - const scopesMissing = []; - const warnings = []; - - for (const scope of expectedScopes) { - // Try exact scoped key first, then bare suffix (for github-style connectors) - const bareSuffix = scope.includes('.') ? scope.split('.').slice(1).join('.') : null; - const value = scope in data ? data[scope] : (bareSuffix && bareSuffix in data ? data[bareSuffix] : undefined); - - if (value === undefined || value === null) { - scopesMissing.push(scope); - continue; - } - - scopesFound.push(scope); - - if (Array.isArray(value) && value.length === 0) { - warnings.push(`${scope}: empty array`); - } else if (typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length === 0) { - warnings.push(`${scope}: empty object`); - } - } - - if (scopesMissing.length > 0) { - return { status: 'fail', scopesFound, scopesMissing, warnings }; - } - if (warnings.length > 0) { - return { status: 'warn', scopesFound, scopesMissing, warnings }; - } - return { status: 'pass', scopesFound, scopesMissing, warnings }; -} -``` - -Update the exports block: - -```js -if (require.main !== module) { - module.exports = { parseArgs, resolveConnectors, classifyOutcome, validateResult }; -} -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: All tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add scripts/test-connectors.cjs scripts/test-connectors.test.cjs -git commit -m "feat(test-connectors): outcome classification and result validation" -``` - ---- - -### Task 4: Orchestration loop and reporting - -**Files:** -- Modify: `scripts/test-connectors.cjs` - -- [ ] **Step 1: Implement the main function and connector runner** - -Add to `scripts/test-connectors.cjs`, replacing the `if (require.main !== module)` block with: - -```js -// ─── Run a single connector via run-connector.cjs ─────────── - -function runConnector(connectorEntry, opts) { - return new Promise((resolve) => { - const scriptPath = path.join(CONNECTORS_DIR, '..', connectorEntry.files.script); - const metadataPath = scriptPath.replace(/\.js$/, '.json'); - const outputPath = path.join(RESULTS_DIR, `${connectorEntry.id}.json`); - - // Load metadata for connectURL and scope info - let metadata = {}; - try { metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf-8')); } catch {} - const startUrl = metadata.connectURL || 'about:blank'; - - // Delete stale result file - try { fs.unlinkSync(outputPath); } catch {} - - const startTime = Date.now(); - const stdoutLines = []; - - const child = spawn(process.execPath, [ - RUN_CONNECTOR, scriptPath, startUrl, '--output', outputPath, - ], { stdio: ['pipe', 'pipe', 'pipe'] }); - - child.stdout.on('data', (chunk) => { - for (const line of chunk.toString().split('\n')) { - if (line.trim()) stdoutLines.push(line.trim()); - } - }); - - // Suppress stderr (runner debug output) - child.stderr.on('data', () => {}); - - child.on('close', (exitCode) => { - const duration = Date.now() - startTime; - const outcome = classifyOutcome(exitCode || 0, stdoutLines); - - if (outcome.status === 'needs-validation') { - // Read and validate result file - let data = null; - try { data = JSON.parse(fs.readFileSync(outputPath, 'utf-8')); } catch {} - - if (!data) { - resolve({ connector: connectorEntry.id, status: 'fail', exitCode: 0, duration, error: 'No result file produced' }); - return; - } - - const validation = validateResult(data, metadata); - resolve({ - connector: connectorEntry.id, - status: validation.status, - exitCode: 0, - duration, - scopesExpected: (metadata.scopes || []).map(s => s.scope), - scopesFound: validation.scopesFound, - scopesMissing: validation.scopesMissing, - warnings: validation.warnings.length > 0 ? validation.warnings : undefined, - }); - } else { - resolve({ - connector: connectorEntry.id, - status: outcome.status, - exitCode: exitCode || 0, - duration, - error: outcome.error, - }); - } - }); - }); -} - -// ─── Reporting ────────────────────────────────────────────── - -function printResults(results) { - const timestamp = new Date().toISOString(); - console.log(`\n${c.bold}Connector Smoke Test${c.reset} ${c.dim}— ${timestamp}${c.reset}\n`); - - for (const r of results) { - const dur = (r.duration / 1000).toFixed(1) + 's'; - const scopeCount = r.scopesExpected - ? `${r.scopesFound.length}/${r.scopesExpected.length} scopes` - : '—'; - - let detail = ''; - if (r.warnings && r.warnings.length > 0) detail = ` (${r.warnings.join(', ')})`; - if (r.scopesMissing && r.scopesMissing.length > 0) detail = ` (missing: ${r.scopesMissing.join(', ')})`; - if (r.error) detail = ` (${r.error})`; - - let statusStr; - switch (r.status) { - case 'pass': statusStr = `${c.green}PASS${c.reset} `; break; - case 'warn': statusStr = `${c.yellow}WARN${c.reset} `; break; - case 'auth': statusStr = `${c.magenta}AUTH${c.reset} `; break; - case 'fail': statusStr = `${c.red}FAIL${c.reset} `; break; - case 'timeout': statusStr = `${c.red}TIMEOUT${c.reset}`; break; - default: statusStr = `${c.gray}???${c.reset} `; - } - - const id = r.connector.padEnd(28); - const sc = scopeCount.padEnd(14); - console.log(` ${statusStr} ${id} ${sc} ${dur}${c.dim}${detail}${c.reset}`); - } - - const counts = { pass: 0, warn: 0, auth: 0, fail: 0, timeout: 0 }; - let totalDuration = 0; - for (const r of results) { - counts[r.status] = (counts[r.status] || 0) + 1; - totalDuration += r.duration; - } - - const parts = []; - if (counts.pass) parts.push(`${c.green}${counts.pass} pass${c.reset}`); - if (counts.warn) parts.push(`${c.yellow}${counts.warn} warn${c.reset}`); - if (counts.auth) parts.push(`${c.magenta}${counts.auth} auth${c.reset}`); - if (counts.fail) parts.push(`${c.red}${counts.fail} fail${c.reset}`); - if (counts.timeout) parts.push(`${c.red}${counts.timeout} timeout${c.reset}`); - - console.log(`\n${parts.join(' · ')} — ${(totalDuration / 1000).toFixed(1)}s total\n`); - - return counts; -} - -function writeJsonReport(results) { - const timestamp = new Date().toISOString(); - const counts = { pass: 0, warn: 0, auth: 0, fail: 0, timeout: 0 }; - for (const r of results) counts[r.status] = (counts[r.status] || 0) + 1; - - const report = { - timestamp, - summary: { ...counts, total: results.length }, - results, - }; - - const reportPath = path.join(RESULTS_DIR, `connector-smoke-${timestamp.replace(/[:.]/g, '-')}.json`); - fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); - console.log(`${c.dim}Report: ${reportPath}${c.reset}\n`); - return reportPath; -} - -// ─── Main ─────────────────────────────────────────────────── - -async function main() { - const opts = parseArgs(process.argv.slice(2)); - - // Load registry - if (!fs.existsSync(REGISTRY_PATH)) { - console.error(`${c.red}Registry not found: ${REGISTRY_PATH}${c.reset}`); - process.exit(1); - } - const registry = JSON.parse(fs.readFileSync(REGISTRY_PATH, 'utf-8')); - - // Resolve connectors - const connectors = resolveConnectors(registry, opts); - if (connectors.length === 0) { - console.error(`${c.red}No connectors matched.${c.reset}`); - process.exit(1); - } - - // Ensure results directory - fs.mkdirSync(RESULTS_DIR, { recursive: true }); - - // Verify run-connector.cjs exists - if (!fs.existsSync(RUN_CONNECTOR)) { - console.error(`${c.red}run-connector.cjs not found: ${RUN_CONNECTOR}${c.reset}`); - process.exit(1); - } - - // Run connectors sequentially (each needs its own browser) - const results = []; - for (const entry of connectors) { - const label = `${c.cyan}${entry.name}${c.reset} ${c.dim}(${entry.id})${c.reset}`; - process.stdout.write(` Running ${label}...`); - const result = await runConnector(entry, opts); - // Clear the "Running..." line - process.stdout.write('\r\x1b[K'); - results.push(result); - } - - // Report - const counts = printResults(results); - writeJsonReport(results); - - // Exit 1 if any fail/auth/timeout - const hasFailure = counts.fail > 0 || counts.auth > 0 || counts.timeout > 0; - process.exit(hasFailure ? 1 : 0); -} - -// ─── Exports for testing ──────────────────────────────────── - -if (require.main !== module) { - module.exports = { parseArgs, resolveConnectors, classifyOutcome, validateResult }; -} else { - main().catch((err) => { - console.error(`${c.red}Fatal: ${err.message}${c.reset}`); - process.exit(1); - }); -} -``` - -- [ ] **Step 2: Run existing tests to verify nothing broke** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: All previous tests still PASS - -- [ ] **Step 3: Manual smoke test** - -Run the harness against a single connector you have cookies for: - -```bash -node scripts/test-connectors.cjs --connectors github-playwright -``` - -Expected: A single-row result table showing PASS, WARN, AUTH, or FAIL with timing. - -- [ ] **Step 4: Commit** - -```bash -git add scripts/test-connectors.cjs -git commit -m "feat(test-connectors): orchestration loop and reporting" -``` - ---- - -### Task 5: Schema validation (opt-in) - -**Files:** -- Modify: `scripts/test-connectors.test.cjs` -- Modify: `scripts/test-connectors.cjs` - -- [ ] **Step 1: Write the test** - -Append to `scripts/test-connectors.test.cjs`: - -```js -describe('validateSchema', () => { - it('returns pass for data matching schema', () => { - const schema = { - type: 'object', - properties: { - username: { type: 'string' }, - followers: { type: 'number' }, - }, - required: ['username'], - }; - const data = { username: 'alice', followers: 42 }; - const errors = validateSchema(data, schema); - assert.deepStrictEqual(errors, []); - }); - - it('returns error for missing required field', () => { - const schema = { - type: 'object', - properties: { - username: { type: 'string' }, - }, - required: ['username'], - }; - const data = { followers: 42 }; - const errors = validateSchema(data, schema); - assert.strictEqual(errors.length, 1); - assert.match(errors[0], /required.*username/i); - }); - - it('returns error for wrong type', () => { - const schema = { - type: 'object', - properties: { - username: { type: 'string' }, - }, - required: ['username'], - }; - const data = { username: 123 }; - const errors = validateSchema(data, schema); - assert.strictEqual(errors.length, 1); - assert.match(errors[0], /username.*string/i); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: FAIL — `validateSchema is not a function` - -- [ ] **Step 3: Implement lightweight schema validator** - -Add to `scripts/test-connectors.cjs`, before the `classifyOutcome` function: - -```js -// ─── Lightweight JSON Schema Validator ────────────────────── -// Validates required fields and basic types. Not a full JSON Schema -// implementation — just enough for smoke testing connector output. - -function validateSchema(data, schema) { - const errors = []; - if (!schema || schema.type !== 'object' || !schema.properties) return errors; - - // Check required fields - for (const field of (schema.required || [])) { - if (!(field in data) || data[field] === undefined || data[field] === null) { - errors.push(`Required field missing: ${field}`); - } - } - - // Check types of present fields - for (const [field, spec] of Object.entries(schema.properties)) { - if (!(field in data) || data[field] === null || data[field] === undefined) continue; - const value = data[field]; - const expectedType = spec.type; - if (!expectedType) continue; - - let actual; - if (Array.isArray(value)) actual = 'array'; - else actual = typeof value; - - if (expectedType === 'array' && actual !== 'array') { - errors.push(`${field}: expected array, got ${actual}`); - } else if (expectedType !== 'array' && actual !== expectedType) { - errors.push(`${field}: expected ${expectedType}, got ${actual}`); - } - } - - return errors; -} -``` - -Update the exports: - -```js -if (require.main !== module) { - module.exports = { parseArgs, resolveConnectors, classifyOutcome, validateResult, validateSchema }; -} -``` - -- [ ] **Step 4: Wire schema validation into runConnector** - -In the `runConnector` function's `needs-validation` branch, after the `const validation = validateResult(data, metadata);` line, add: - -```js - // Optional schema validation - let schemaErrors = []; - if (opts.validateSchemas) { - const schemasDir = path.join(ROOT, 'schemas'); - for (const scope of validation.scopesFound) { - const schemaPath = path.join(schemasDir, `${scope}.json`); - if (!fs.existsSync(schemaPath)) continue; - try { - const schemaFile = JSON.parse(fs.readFileSync(schemaPath, 'utf-8')); - const scopeKey = scope in data ? scope : scope.split('.').slice(1).join('.'); - const errs = validateSchema(data[scopeKey], schemaFile.schema); - schemaErrors.push(...errs.map(e => `${scope}: ${e}`)); - } catch {} - } - } -``` - -Add `schemaErrors` to the resolved result object: - -```js - schemaErrors: schemaErrors.length > 0 ? schemaErrors : undefined, -``` - -- [ ] **Step 5: Run tests to verify all pass** - -Run: `node --test scripts/test-connectors.test.cjs` -Expected: All tests PASS - -- [ ] **Step 6: Commit** - -```bash -git add scripts/test-connectors.cjs scripts/test-connectors.test.cjs -git commit -m "feat(test-connectors): opt-in schema validation" -``` - ---- - -### Task 6: Final integration test and cleanup - -**Files:** -- Modify: `scripts/test-connectors.cjs` (if any issues found) - -- [ ] **Step 1: Run full harness with --connectors flag** - -Pick one connector you have valid cookies for: - -```bash -node scripts/test-connectors.cjs --connectors github-playwright -``` - -Verify: -- Terminal output shows a single-row result table -- `test-results/github-playwright.json` exists and contains valid connector output -- `test-results/connector-smoke-*.json` exists with the summary report -- Exit code is 0 for pass/warn, 1 for fail/auth/timeout - -- [ ] **Step 2: Run harness with no args to verify default set** - -```bash -node scripts/test-connectors.cjs -``` - -Verify it attempts all 6 stable connectors. (Some will AUTH if you don't have sessions — that's expected.) - -- [ ] **Step 3: Run harness with --validate-schemas** - -```bash -node scripts/test-connectors.cjs --connectors github-playwright --validate-schemas -``` - -Verify schema validation output appears in the report. - -- [ ] **Step 4: Final commit** - -```bash -git add scripts/test-connectors.cjs scripts/test-connectors.test.cjs -git commit -m "feat(test-connectors): batch connector smoke test harness" -``` diff --git a/docs/superpowers/specs/2026-03-31-connector-smoke-test-design.md b/docs/superpowers/specs/2026-03-31-connector-smoke-test-design.md deleted file mode 100644 index a6993a8..0000000 --- a/docs/superpowers/specs/2026-03-31-connector-smoke-test-design.md +++ /dev/null @@ -1,118 +0,0 @@ -# Connector Smoke Test Harness — Design Spec - -## Purpose - -A batch orchestrator that runs all stable connectors headlessly using pre-existing browser sessions, validates output against declared scopes, and reports per-connector health. Designed for local use with a path to CI. - -## Architecture - -The harness wraps `run-connector.cjs` — the maintained standalone runner that already handles runner resolution, auth classification, result capture, and process lifecycle. The test harness is a thin loop: resolve connectors from `registry.json`, spawn `run-connector.cjs` once per connector with a unique `--output` path, classify outcomes from exit codes + stdout, validate the result file against metadata scopes, and report. - -## Connector Resolution - -- Source: root `registry.json` (15 connectors: 6 stable, 5 beta, 4 experimental) -- Default policy: **stable connectors only** (chatgpt, github, instagram, linkedin, spotify, oura) -- `--connectors ,` overrides the default set, filtering by connector ID -- `--include-beta` expands default to stable + beta - -## Execution Per Connector - -1. Delete any previous `test-results/.json` (remove stale results) -2. Resolve connector script path from registry entry's `files.script` -3. Resolve start URL from connector metadata's `connectURL` -4. Spawn: `node run-connector.cjs --output test-results/.json` -5. Capture stdout (JSON lines) for timeout detection and logging -6. Wait for exit - -## Outcome Classification - -| Status | Condition | -|-----------|------------------------------------------------------------------| -| **PASS** | Exit 0 + all metadata scopes present in result with non-empty data | -| **WARN** | Exit 0 + all scopes present but some have empty arrays/objects | -| **AUTH** | Exit 2 (need-input) or exit 3 (legacy-auth) | -| **FAIL** | Exit 0 but missing scopes, or exit 1 without timeout message | -| **TIMEOUT** | Exit 1 + stdout contains `"Timeout after 5 minutes"` error event | - -## Validation Rules - -- For each scope in the connector's `.json` metadata, check that the result object contains a key matching the full scope (e.g., `github.profile`) -- Non-empty means the value is not `null`, not `undefined` -- Empty arrays (`[]`) or empty objects (`{}`) are **warnings**, not failures — a user could legitimately have zero starred repos -- Only validate result files from exit-0 runs - -### Scope Normalization (temporary) - -`github-playwright` currently returns bare keys (`profile`, `repositories`, `starred`) instead of scoped keys (`github.profile`, etc.). Until vana-com/data-connectors#52 lands, the harness checks both `` and the bare suffix (everything after the first dot) when matching result keys. This normalization applies only during validation, not to the result file itself. - -## Schema Validation (opt-in) - -When `--validate-schemas` is passed, the harness additionally validates each scope's data against the corresponding JSON Schema file in `schemas/.json`. This is not the default because schemas are strict and may flag acceptable variations in live data. - -## Output - -### Terminal (colored table) - -``` -Connector Smoke Test — 2026-03-31T12:00:00Z - - PASS chatgpt-playwright 2/2 scopes 18.2s - WARN instagram-playwright 3/3 scopes 12.1s (instagram.posts: empty array) - AUTH spotify-playwright — 2.3s (need-input: session expired) - FAIL github-playwright 2/3 scopes 6.4s (missing: github.starred) - PASS linkedin-playwright 6/6 scopes 22.1s - PASS oura-playwright 3/3 scopes 14.7s - -3 pass · 1 warn · 1 auth · 1 fail · 0 timeout — 75.8s total -``` - -### JSON report - -Written to `test-results/connector-smoke-.json`: - -```json -{ - "timestamp": "2026-03-31T12:00:00Z", - "summary": { "pass": 3, "warn": 1, "auth": 1, "fail": 1, "timeout": 0, "total": 6 }, - "results": [ - { - "connector": "spotify-playwright", - "status": "auth", - "exitCode": 2, - "duration": 2300, - "error": "need-input: session expired" - }, - { - "connector": "github-playwright", - "status": "fail", - "exitCode": 0, - "duration": 6400, - "scopesExpected": ["github.profile", "github.repositories", "github.starred"], - "scopesFound": ["github.profile", "github.repositories"], - "scopesMissing": ["github.starred"] - } - ] -} -``` - -## Invocation - -```bash -# Default: stable connectors only -node scripts/test-connectors.cjs - -# Specific connectors -node scripts/test-connectors.cjs --connectors instagram-playwright,spotify-playwright - -# Include beta -node scripts/test-connectors.cjs --include-beta - -# With schema validation -node scripts/test-connectors.cjs --validate-schemas -``` - -## Files - -- **New:** `scripts/test-connectors.cjs` (the harness) -- **New:** `test-results/` directory (gitignored) -- **Modified:** `.gitignore` (add `test-results/`) diff --git a/fixture-index.json b/fixture-index.json index 56f33f5..34baea8 100644 --- a/fixture-index.json +++ b/fixture-index.json @@ -1,7 +1,7 @@ { "$schema": "./schemas/fixture-index.schema.json", "indexVersion": "1.0", - "sourceRepo": "https://github.com/vana-com/data-connectors", + "sourceRepo": "https://github.com/PDP-Connect/data-connectors", "generatedAt": "2026-07-13T00:00:00Z", "fixtures": [ { @@ -12,7 +12,7 @@ "scenario": "empty", "recordCount": 0, "path": "connectors/meta/fixtures/instagram.profile.empty.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/meta/fixtures/instagram.profile.empty.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/meta/fixtures/instagram.profile.empty.json", "schemaPath": "connectors/meta/schemas/instagram.profile.json", "sizeBytes": 271, "sha256": "sha256:f294a6ca5ff524edf68110ef8b564c1df15284beccc1dc28bf8af7ff16a7e5f3" @@ -25,7 +25,7 @@ "scenario": "small", "recordCount": 0, "path": "connectors/meta/fixtures/instagram.profile.small.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/meta/fixtures/instagram.profile.small.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/meta/fixtures/instagram.profile.small.json", "schemaPath": "connectors/meta/schemas/instagram.profile.json", "sizeBytes": 274, "sha256": "sha256:1840789166c70c1468b7296b8842f0ff2b937da2a75f2d26d5a84a6d7c192efc" @@ -38,7 +38,7 @@ "scenario": "large", "recordCount": 0, "path": "connectors/meta/fixtures/instagram.profile.large.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/meta/fixtures/instagram.profile.large.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/meta/fixtures/instagram.profile.large.json", "schemaPath": "connectors/meta/schemas/instagram.profile.json", "sizeBytes": 484, "sha256": "sha256:9b304fea3be9c2e3937060d8374c1d1ba77cc66cfa578868950c2a7a7d246f9c" @@ -51,7 +51,7 @@ "scenario": "empty", "recordCount": 0, "path": "connectors/spotify/fixtures/spotify.savedTracks.empty.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/spotify/fixtures/spotify.savedTracks.empty.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/spotify/fixtures/spotify.savedTracks.empty.json", "schemaPath": "connectors/spotify/schemas/spotify.savedTracks.json", "sizeBytes": 38, "sha256": "sha256:1083809a5e7b2adb1c94cf74a067c80182da1b96b7fa90d4b648487e7bda7fef" @@ -64,7 +64,7 @@ "scenario": "small", "recordCount": 3, "path": "connectors/spotify/fixtures/spotify.savedTracks.small.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/spotify/fixtures/spotify.savedTracks.small.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/spotify/fixtures/spotify.savedTracks.small.json", "schemaPath": "connectors/spotify/schemas/spotify.savedTracks.json", "sizeBytes": 1355, "sha256": "sha256:0f577bfb2877e93b7b6e0b2c89b855906faaca373d2865c5a11d8ef47aa7063f" @@ -77,7 +77,7 @@ "scenario": "large", "recordCount": 15, "path": "connectors/spotify/fixtures/spotify.savedTracks.large.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/spotify/fixtures/spotify.savedTracks.large.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/spotify/fixtures/spotify.savedTracks.large.json", "schemaPath": "connectors/spotify/schemas/spotify.savedTracks.json", "sizeBytes": 6358, "sha256": "sha256:b1fe3c82003fef4f222161eed65c5a3e76c7cc13cc793b39940c3589c394c840" @@ -90,7 +90,7 @@ "scenario": "empty", "recordCount": 0, "path": "connectors/google/fixtures/youtube.history.empty.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/google/fixtures/youtube.history.empty.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/google/fixtures/youtube.history.empty.json", "schemaPath": "connectors/google/schemas/youtube.history.json", "sizeBytes": 64, "sha256": "sha256:e1920da9025417bba991f2edc74e91aa4444d86a13e402ad5625fbfc5e12e1aa" @@ -103,7 +103,7 @@ "scenario": "small", "recordCount": 4, "path": "connectors/google/fixtures/youtube.history.small.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/google/fixtures/youtube.history.small.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/google/fixtures/youtube.history.small.json", "schemaPath": "connectors/google/schemas/youtube.history.json", "sizeBytes": 1431, "sha256": "sha256:4d3fb2096b2a46741c42894095fdd12ffea69b3fbe905b1b507167236a43197f" @@ -116,7 +116,7 @@ "scenario": "large", "recordCount": 30, "path": "connectors/google/fixtures/youtube.history.large.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/google/fixtures/youtube.history.large.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/google/fixtures/youtube.history.large.json", "schemaPath": "connectors/google/schemas/youtube.history.json", "sizeBytes": 10107, "sha256": "sha256:fd202ae77b4f21f7bfe26f9b3b2f50a6f7af5cd76ab933f8ebe9ed762fc98eb6" @@ -129,7 +129,7 @@ "scenario": "empty", "recordCount": 0, "path": "connectors/google/fixtures/youtube.likes.empty.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/google/fixtures/youtube.likes.empty.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/google/fixtures/youtube.likes.empty.json", "schemaPath": "connectors/google/schemas/youtube.likes.json", "sizeBytes": 24, "sha256": "sha256:b142328617d8c8ecc8d81c3cee885930154515b95b131e9c53214e78d807434d" @@ -142,7 +142,7 @@ "scenario": "small", "recordCount": 4, "path": "connectors/google/fixtures/youtube.likes.small.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/google/fixtures/youtube.likes.small.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/google/fixtures/youtube.likes.small.json", "schemaPath": "connectors/google/schemas/youtube.likes.json", "sizeBytes": 1531, "sha256": "sha256:a134c8f9097f8cdbcce2acb3e18383595ea09a9cc729131ae9c3f7bda665c5a5" @@ -155,7 +155,7 @@ "scenario": "large", "recordCount": 30, "path": "connectors/google/fixtures/youtube.likes.large.json", - "rawUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors/google/fixtures/youtube.likes.large.json", + "rawUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors/google/fixtures/youtube.likes.large.json", "schemaPath": "connectors/google/schemas/youtube.likes.json", "sizeBytes": 11318, "sha256": "sha256:2d02bcef10dbb34b7a84e6a875b5b555fc0fe4fbb7712ce5067af6d9e9fb3f35" diff --git a/package.json b/package.json index a4cf082..8e031d0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "@opendatalabs/data-connectors-tools", "version": "0.1.0", + "license": "Apache-2.0", "private": true, "type": "module", "files": [ diff --git a/packages/connector-installer-core/index.mjs b/packages/connector-installer-core/index.mjs index 6506341..3298568 100644 --- a/packages/connector-installer-core/index.mjs +++ b/packages/connector-installer-core/index.mjs @@ -15,7 +15,7 @@ import { dirname, join, normalize, resolve as resolvePath } from "node:path"; import { verify as verifySigstoreBundle } from "sigstore"; export const DEFAULT_CONNECTOR_INDEX_URL = - "https://github.com/vana-com/data-connectors/releases/download/connectors-latest/connector-index.json"; + "https://github.com/PDP-Connect/data-connectors/releases/download/connectors-latest/connector-index.json"; export const DEFAULT_SIGSTORE_CERTIFICATE_ISSUER = "https://token.actions.githubusercontent.com"; export const DEFAULT_SIGSTORE_CERTIFICATE_IDENTITY = @@ -738,7 +738,7 @@ export async function generateLock({ sourceRepo: source.doc.sourceRepo ?? dependencies.source_repo ?? - "https://github.com/vana-com/data-connectors", + "https://github.com/PDP-Connect/data-connectors", sourceTag: sourceMeta.sourceTag, sourceCommit: sourceMeta.sourceCommit, index: { diff --git a/playwright-runner/index.cjs b/playwright-runner/index.cjs new file mode 100644 index 0000000..c7a88b4 --- /dev/null +++ b/playwright-runner/index.cjs @@ -0,0 +1,1251 @@ +/** + * Playwright Runner for PDP-Connect + * + * Runs as a sidecar process, receives commands via stdin, sends results via stdout. + * + * Commands: + * - { type: "run", runId, connectorPath, url, headless, allowHeaded, requestedScopes } + * - { type: "stop", runId } + * - { type: "evaluate", runId, script } + * - { type: "input-response", runId, requestId, data?, error? } + * - { type: "screenshot", runId } + * - { type: "quit" } + * + * Supports two-phase connectors: + * - Phase 1 (Browser): Login detection + credential extraction + * - Phase 2 (Background): Direct HTTP fetch without browser + */ + +const { chromium } = require('playwright'); +const fs = require('fs'); +const os = require('os'); +const readline = require('readline'); +const path = require('path'); +const { execSync } = require('child_process'); +const { readZipJsonEntries } = require('./zip-reader.cjs'); +const { classifyConnectorResult } = require('./result-classifier.cjs'); +const { + normalizeConnectorResult, + resolveHeadlessResumeUrl, +} = require('./runner-compat.cjs'); + +// System Chrome paths by platform +const CHROME_PATHS = { + darwin: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + win32: 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', + linux: '/usr/bin/google-chrome' +}; + +// Get browser cache directory - checks multiple candidate paths +function getBrowserCacheDir() { + if (process.env.PLAYWRIGHT_BROWSERS_PATH) { + log(`Using PLAYWRIGHT_BROWSERS_PATH: ${process.env.PLAYWRIGHT_BROWSERS_PATH}`); + return process.env.PLAYWRIGHT_BROWSERS_PATH; + } + const home = process.env.HOME || process.env.USERPROFILE || ''; + const candidates = [ + path.join(home, '.pdp-connect', 'browsers'), + path.join(home, '.pdp-connect', 'playwright-runner', 'node_modules', 'playwright-core', '.local-browsers'), + ]; + for (const dir of candidates) { + if (fs.existsSync(dir)) { + log(`Found browser cache: ${dir}`); + return dir; + } + } + return candidates[0]; +} + +// Check if system Chrome exists +function getSystemChromePath() { + const chromePath = CHROME_PATHS[process.platform]; + log(`Checking system Chrome at: ${chromePath}`); + if (chromePath && fs.existsSync(chromePath)) { + log(`Found system Chrome: ${chromePath}`); + return chromePath; + } + log(`System Chrome not found at default path`); + // Try alternative Windows paths + if (process.platform === 'win32') { + const altPaths = [ + 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', + path.join(process.env.LOCALAPPDATA || '', 'Google\\Chrome\\Application\\chrome.exe') + ]; + for (const p of altPaths) { + if (fs.existsSync(p)) return p; + } + } + // Try Edge on Windows + if (process.platform === 'win32') { + const edgePath = 'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe'; + if (fs.existsSync(edgePath)) return edgePath; + } + return null; +} + +// Check if Playwright Chromium is already downloaded (or bundled via PLAYWRIGHT_BROWSERS_PATH) +function getDownloadedChromiumPath() { + const cacheDir = getBrowserCacheDir(); + log(`Checking for Chromium in: ${cacheDir}`); + if (!fs.existsSync(cacheDir)) { + log(`Browser cache dir does not exist: ${cacheDir}`); + return null; + } + + // Look for chromium directory + const entries = fs.readdirSync(cacheDir); + const chromiumDir = entries.find(e => e.startsWith('chromium-') && !e.includes('headless')); + if (!chromiumDir) return null; + + const chromiumPath = path.join(cacheDir, chromiumDir); + + // Platform-specific executable paths (Playwright's "Chrome for Testing" structure) + if (process.platform === 'darwin') { + // Try arm64 first, then x64 + const paths = [ + path.join(chromiumPath, 'chrome-mac-arm64', 'Google Chrome for Testing.app', 'Contents', 'MacOS', 'Google Chrome for Testing'), + path.join(chromiumPath, 'chrome-mac', 'Google Chrome for Testing.app', 'Contents', 'MacOS', 'Google Chrome for Testing'), + // Legacy paths + path.join(chromiumPath, 'chrome-mac-arm64', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'), + path.join(chromiumPath, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'), + ]; + for (const p of paths) { + if (fs.existsSync(p)) return p; + } + } else if (process.platform === 'win32') { + const paths = [ + path.join(chromiumPath, 'chrome-win', 'chrome.exe'), + path.join(chromiumPath, 'chrome-win64', 'chrome.exe'), + ]; + for (const p of paths) { + if (fs.existsSync(p)) return p; + } + } else { + const paths = [ + path.join(chromiumPath, 'chrome-linux', 'chrome'), + path.join(chromiumPath, 'chrome-linux64', 'chrome'), + ]; + for (const p of paths) { + if (fs.existsSync(p)) return p; + } + } + + return null; +} + +// Default Chrome user-data directories by platform +const CHROME_PROFILE_DIRS = { + darwin: path.join(process.env.HOME || '', 'Library', 'Application Support', 'Google', 'Chrome'), + win32: path.join(process.env.LOCALAPPDATA || '', 'Google', 'Chrome', 'User Data'), + linux: path.join(process.env.HOME || '', '.config', 'google-chrome'), +}; + +// Check whether a browser path points to system Chrome (not Playwright Chromium). +function isSystemChrome(browserPath) { + if (!browserPath) return false; + const lower = browserPath.toLowerCase(); + if (lower.includes('.databridge') || lower.includes('chromium') || lower.includes('chrome for testing')) { + return false; + } + return true; +} + +// Get the Chrome last-used profile directory path. +function getChromeProfileDir(chromeRoot) { + const localStatePath = path.join(chromeRoot, 'Local State'); + if (fs.existsSync(localStatePath)) { + try { + const localState = JSON.parse(fs.readFileSync(localStatePath, 'utf-8')); + const lastUsed = localState?.profile?.last_used; + if (lastUsed) { + const profileDir = path.join(chromeRoot, lastUsed); + if (fs.existsSync(profileDir)) { + log(`Chrome last-used profile: "${lastUsed}"`); + return profileDir; + } + } + } catch (e) { + log(`Warning: could not read Chrome Local State: ${e.message}`); + } + } + + const defaultDir = path.join(chromeRoot, 'Default'); + if (fs.existsSync(defaultDir)) return defaultDir; + return null; +} + +// Import cookies from the user's Chrome profile into a running Playwright +// browser context's Cookies database. This is done AFTER Chrome creates its +// own fresh profile, so we INSERT into Chrome's own db rather than replacing it +// (which Chrome would wipe on startup). +// +// The encrypted_value blobs use the same Keychain key (v10 format), so Chrome +// can decrypt them transparently — no Keychain popup needed since Chrome itself +// is the one reading them. +function importChromecookies(userDataDir, browserPath) { + if (!isSystemChrome(browserPath)) return; + + // Only import once + const markerFile = path.join(userDataDir, '.cookies-imported'); + if (fs.existsSync(markerFile)) { + log('Skipping cookie import — already done'); + return; + } + + const chromeRoot = CHROME_PROFILE_DIRS[process.platform]; + if (!chromeRoot || !fs.existsSync(chromeRoot)) return; + + const sourceProfileDir = getChromeProfileDir(chromeRoot); + if (!sourceProfileDir) return; + + const sourceCookies = path.join(sourceProfileDir, 'Cookies'); + if (!fs.existsSync(sourceCookies)) return; + + // Find the target Cookies db — Chrome creates it inside "Default/" by default + const targetCookies = path.join(userDataDir, 'Default', 'Cookies'); + if (!fs.existsSync(targetCookies)) { + log('Skipping cookie import — target Cookies db not found yet'); + return; + } + + try { + // Use sqlite3 to INSERT cookies from source into the target db. + // ATTACH the source db, then INSERT OR IGNORE to avoid duplicates. + const sql = ` + ATTACH DATABASE '${sourceCookies.replace(/'/g, "''")}' AS src; + INSERT OR REPLACE INTO cookies + SELECT * FROM src.cookies; + DETACH DATABASE src; + `; + execSync(`sqlite3 "${targetCookies}" "${sql}"`, { + encoding: 'utf-8', + timeout: 10000, + }); + + // Verify + const count = execSync( + `sqlite3 "${targetCookies}" "SELECT COUNT(*) FROM cookies;"`, + { encoding: 'utf-8' } + ).trim(); + log(`Imported cookies into profile — total cookies now: ${count}`); + + fs.writeFileSync(markerFile, new Date().toISOString()); + } catch (e) { + log(`Warning: could not import Chrome cookies: ${e.message}`); + } +} + +// Download Chromium using Playwright +async function downloadChromium(sendStatus) { + const cacheDir = getBrowserCacheDir(); + + // Create cache directory + if (!fs.existsSync(cacheDir)) { + fs.mkdirSync(cacheDir, { recursive: true }); + } + + log('Downloading Chromium browser (one-time setup)...'); + if (sendStatus) { + sendStatus('DOWNLOADING_BROWSER'); + } + + // Set environment for Playwright to use our cache dir + process.env.PLAYWRIGHT_BROWSERS_PATH = cacheDir; + + try { + // Use Playwright's CLI to download Chromium + execSync('npx playwright install chromium', { + stdio: 'inherit', + env: { ...process.env, PLAYWRIGHT_BROWSERS_PATH: cacheDir } + }); + log('Chromium download complete'); + return getDownloadedChromiumPath(); + } catch (error) { + log('Failed to download Chromium:', error.message); + throw new Error('Failed to download browser. Please install Google Chrome or try again.'); + } +} + +// Active browser contexts by runId +const activeRuns = new Map(); + +// Send message to parent process +function send(msg) { + process.stdout.write(JSON.stringify(msg) + '\n'); +} + +function drainStdout() { + return new Promise(resolve => { + if (process.stdout.writableNeedDrain) { + process.stdout.once('drain', resolve); + } else { + process.stdout.write('', resolve); + } + }); +} + +// Log to stderr (doesn't interfere with JSON protocol) +function log(...args) { + console.error('[PlaywrightRunner]', ...args); +} + +function isCanonicalScopeId(scope) { + return ( + typeof scope === 'string' && + scope.length > 0 && + scope.includes('.') && + !scope.startsWith('.') && + !scope.endsWith('.') + ); +} + +function normalizeRequestedScopes(requestedScopes) { + if (!Array.isArray(requestedScopes) || requestedScopes.length === 0) { + throw new Error('Resolved requestedScopes must be a non-empty array'); + } + + const deduped = []; + const seen = new Set(); + for (const scope of requestedScopes) { + if (!isCanonicalScopeId(scope)) { + throw new Error(`Resolved requestedScopes contains a non-canonical scope id: ${String(scope)}`); + } + if (!seen.has(scope)) { + seen.add(scope); + deduped.push(scope); + } + } + + if (deduped.length === 0) { + throw new Error('Resolved requestedScopes must contain at least one canonical scope id'); + } + + return deduped; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function shouldRetryNavigationError(error) { + const message = String(error && error.message ? error.message : error || '').toLowerCase(); + return ( + message.includes('err_name_not_resolved') || + message.includes('err_internet_disconnected') || + message.includes('err_network_changed') || + message.includes('err_connection_reset') || + message.includes('err_connection_closed') || + message.includes('timeout') + ); +} + +async function gotoWithRetries(page, url, options = {}) { + const { + attempts = 3, + delayMs = 1500, + context = 'navigation', + waitUntil = 'domcontentloaded', + timeout, + } = options; + + const gotoOptions = { waitUntil }; + if (timeout != null) { + gotoOptions.timeout = timeout; + } + + let lastError = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + await page.goto(url, gotoOptions); + return; + } catch (error) { + lastError = error; + if (attempt === attempts || !shouldRetryNavigationError(error)) { + throw error; + } + log( + `${context} attempt ${attempt}/${attempts} failed for ${url}: ${error.message}. Retrying...`, + ); + await sleep(delayMs * attempt); + } + } + + throw lastError; +} + +function storeConnectorResult(runState, result) { + const normalizedResult = normalizeConnectorResult(result, { + requestedScopes: runState.requestedScopes, + }); + const classification = classifyConnectorResult(normalizedResult, { + expectedRequestedScopes: runState.requestedScopes, + }); + + if (classification.outcome === 'failure') { + log( + `Connector result classified as failure: ${classification.errorClass}` + + (classification.debug ? ` (${classification.debug})` : ''), + ); + } + + runState.resultEnvelope = { + classification, + rawResult: normalizedResult, + }; + + return runState.resultEnvelope; +} + +function emitStoredConnectorResult(runId, runState) { + if (!runState.resultEnvelope) { + return null; + } + + const { classification, rawResult } = runState.resultEnvelope; + const message = { + type: 'result', + runId, + classification, + }; + + if (classification.outcome === 'success' || classification.outcome === 'partial') { + message.data = rawResult; + } + + send(message); + return classification; +} + +// Resolve browser executable path +function resolveBrowserPath() { + let browserPath = null; + + if (!process.env.PDP_CONNECT_SIMULATE_NO_CHROME) { + browserPath = getSystemChromePath(); + } else { + log('PDP_CONNECT_SIMULATE_NO_CHROME is set, skipping system Chrome detection'); + } + + if (!browserPath) { + browserPath = getDownloadedChromiumPath(); + } + + if (!browserPath) { + throw new Error('No browser available. The Rust backend should have downloaded Chromium before starting the connector.'); + } + + return browserPath; +} + +// Launch a persistent browser context +async function launchPersistentContext(userDataDir, headless, browserPath) { + // Ensure profile directory exists + fs.mkdirSync(userDataDir, { recursive: true }); + + const launchOptions = { + headless, + args: [ + '--disable-blink-features=AutomationControlled', + '--disable-features=MediaRouter,DialMediaRouteProvider', + ], + viewport: { width: 1280, height: 800 }, + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36', + }; + + if (browserPath) { + launchOptions.executablePath = browserPath; + } + + // When using system Chrome, disable Playwright's mock keychain so Chrome + // uses the real macOS Keychain. This lets it decrypt cookies imported from + // the user's real Chrome profile (both use the same "Chrome Safe Storage" + // Keychain entry). No popup — Chrome itself is already authorized. + if (isSystemChrome(browserPath)) { + launchOptions.ignoreDefaultArgs = ['--use-mock-keychain']; + } + + log(`Launching ${headless ? 'headless' : 'headed'} browser with profile: ${userDataDir}`); + const context = await chromium.launchPersistentContext(userDataDir, launchOptions); + log('Browser launched successfully'); + return context; +} + +// Create the page API that connectors use +function createPageApi(runState, runId) { + const networkCaptures = new Map(); + const capturedResponses = new Map(); + + // Helper to get current page, throw if browser is closed + function requirePage() { + if (runState.browserClosed || !runState.page) { + throw new Error('Browser is closed. Use page.httpFetch() for HTTP requests.'); + } + return runState.page; + } + + // Set up network interception on current page + function setupNetworkCapture(page) { + page.on('response', async (response) => { + const url = response.url(); + + for (const [key, config] of networkCaptures.entries()) { + if (config.urlPattern && !url.includes(config.urlPattern)) continue; + + try { + const request = response.request(); + const postData = request.postData() || ''; + + if (config.bodyPattern) { + const patterns = config.bodyPattern.split('|'); + if (!patterns.some(p => postData.includes(p))) continue; + } + + const body = await response.json().catch(() => null); + if (body) { + capturedResponses.set(key, { url, data: body, timestamp: Date.now() }); + send({ type: 'network-captured', runId, key, url }); + } + } catch (e) { + // Ignore errors for non-JSON responses + } + } + }); + } + + // Set up network capture on initial page + if (runState.page) { + setupNetworkCapture(runState.page); + } + + return { + // ── Canonical page API minimum surface ───────────────────────────── + // These are thin pass-throughs to the underlying Playwright page object. + // See types/connector.d.ts in this repo for the canonical contract. + + requestedScopes: () => { + return [...runState.requestedScopes]; + }, + + url: async () => { + const page = requirePage(); + return page.url(); + }, + + click: async (selector, options = {}) => { + const page = requirePage(); + await page.click(selector, options); + }, + + fill: async (selector, value, options = {}) => { + const page = requirePage(); + await page.fill(selector, value, options); + }, + + press: async (selector, key, options = {}) => { + const page = requirePage(); + await page.press(selector, key, options); + }, + + waitForSelector: async (selector, options = {}) => { + const page = requirePage(); + await page.waitForSelector(selector, options); + }, + + goto: async (url, options = {}) => { + const page = requirePage(); + log(`pageApi.goto called with: ${url}`); + send({ type: 'log', runId, message: `Navigating to: ${url}` }); + const { waitUntil = 'domcontentloaded', timeout } = options; + const gotoOpts = { waitUntil }; + if (timeout != null) gotoOpts.timeout = timeout; + try { + await page.goto(url, gotoOpts); + log('pageApi.goto completed successfully'); + } catch (err) { + log(`pageApi.goto error: ${err.message}`); + throw err; + } + }, + + evaluate: async (script) => { + const page = requirePage(); + return await page.evaluate(script); + }, + + screenshot: async () => { + const page = requirePage(); + const buffer = await page.screenshot({ type: 'jpeg', quality: 70, timeout: 5000 }); + return buffer.toString('base64'); + }, + + sleep: (ms) => new Promise(resolve => setTimeout(resolve, ms)), + + setData: async (key, value) => { + if (key === 'status') { + send({ type: 'log', runId, message: value }); + log(`[status] ${value}`); + } else if (key === 'error') { + log(`[error] ${value}`); + } else if (key === 'result') { + storeConnectorResult(runState, value); + } + send({ type: 'data', runId, key, value }); + }, + + // Structured progress update — drives the frontend progress UI + setProgress: async ({ phase, message, count }) => { + send({ type: 'status', runId, status: { type: 'COLLECTING', message, phase, count } }); + if (message) log(`[progress] ${message}`); + }, + + promptUser: async (message, checkFn, interval = 2000) => { + send({ type: 'log', runId, message }); + send({ type: 'status', runId, status: 'WAITING_FOR_USER' }); + + // Poll until condition is met + while (true) { + await new Promise(resolve => setTimeout(resolve, interval)); + try { + const result = await checkFn(); + if (result) { + send({ type: 'log', runId, message: 'User action completed' }); + return; + } + } catch (e) { + // Keep waiting + } + } + }, + + captureNetwork: async (config) => { + networkCaptures.set(config.key, { + urlPattern: config.urlPattern || '', + bodyPattern: config.bodyPattern || '' + }); + log(`Registered network capture: ${config.key}`); + }, + + getCapturedResponse: async (key) => { + const captured = capturedResponses.get(key); + return captured ? captured : null; + }, + + clearNetworkCaptures: async () => { + networkCaptures.clear(); + capturedResponses.clear(); + }, + + hasCapturedResponse: (key) => { + return capturedResponses.has(key); + }, + + // Close the browser but keep the Node.js process alive for background HTTP work. + // Cookies/session persist in the profile directory for next run. + closeBrowser: async () => { + if (runState.browserClosed) { + log('Browser already closed'); + return; + } + + log('Closing browser (connector requested closeBrowser)'); + + // Extract cookies before closing so httpFetch can use them + if (runState.context) { + try { + runState.cookies = await runState.context.cookies(); + log(`Extracted ${runState.cookies.length} cookies for background HTTP requests`); + } catch (e) { + log('Warning: could not extract cookies:', e.message); + runState.cookies = []; + } + } + + runState.browserClosed = true; + runState.browserClosedByConnector = true; + + if (runState.context) { + try { + await runState.context.close(); + } catch (e) { + log('Error closing context:', e.message); + } + runState.context = null; + runState.page = null; + } + + send({ type: 'log', runId, message: 'Browser closed, continuing in background...' }); + log('Browser closed, process stays alive for background work'); + }, + + // Escalate to headed mode for live human interaction (e.g., interactive CAPTCHAs). + // Gated by allowHeaded capability — if the driver doesn't support headed mode, + // navigates in the existing headless browser and returns { headed: false }. + showBrowser: async (url) => { + log('showBrowser requested'); + + if (runState.browserClosed) { + log('showBrowser called but browser is already closed'); + return { headed: false }; + } + + if (!runState.allowHeaded) { + log('Headed mode not available — navigating headless'); + if (url && runState.page) { + try { + await runState.page.goto(url, { waitUntil: 'domcontentloaded' }); + } catch (e) { + log(`showBrowser headless navigation failed: ${e.message}`); + } + } + send({ type: 'log', runId, message: 'Headed interaction unavailable — staying headless' }); + return { headed: false }; + } + + // Close existing browser if open + if (runState.context && !runState.browserClosed) { + log('Closing existing browser before reopening headed'); + runState.browserClosedByConnector = true; + try { + await runState.context.close(); + } catch (e) { + log('Error closing existing context:', e.message); + } + runState.context = null; + runState.page = null; + } + + // Launch new headed browser with persistent context + runState.browserClosed = false; + runState.browserClosedByConnector = false; + runState.headless = false; + const context = await launchPersistentContext(runState.userDataDir, false, runState.browserPath); + const page = context.pages()[0] || await context.newPage(); + + // Set up disconnect handler + context.browser().on('disconnected', () => { + if (!runState.connectorCompleted && !runState.browserClosedByConnector) { + log(`Browser disconnected for run ${runId} (user closed window)`); + runState.browserClosed = true; + runState.context = null; + runState.page = null; + activeRuns.delete(runId); + send({ type: 'status', runId, status: 'STOPPED' }); + drainStdout().then(() => process.exit(0)); + } + }); + + // Update state + runState.context = context; + runState.page = page; + + // Re-setup network capture on new page + setupNetworkCapture(page); + + // Navigate to URL + if (url) { + await page.goto(url, { waitUntil: 'domcontentloaded' }); + } + + send({ type: 'log', runId, message: 'Browser opened for user interaction' }); + log('Headed browser opened'); + return { headed: true }; + }, + + // Switch to headless mode — browser becomes invisible but stays running. + // Use this after credentials are captured so the user doesn't see the browser + // during data collection, while preserving the TLS fingerprint for Cloudflare. + goHeadless: async (options = {}) => { + if (runState.headless && !runState.browserClosed) { + log('Already in headless mode'); + return; + } + + log('Switching to headless mode'); + const currentUrl = runState.page ? runState.page.url() : null; + const resumeUrl = resolveHeadlessResumeUrl({ + resumeUrl: options.resumeUrl, + currentUrl, + }); + + // Close existing headed browser + if (runState.context && !runState.browserClosed) { + runState.browserClosedByConnector = true; + try { + await runState.context.close(); + } catch (e) { + log('Error closing headed context:', e.message); + } + runState.context = null; + runState.page = null; + } + + // Reopen headless browser with persistent context + runState.browserClosed = false; + runState.browserClosedByConnector = false; + runState.headless = true; + const context = await launchPersistentContext(runState.userDataDir, true, runState.browserPath); + const page = context.pages()[0] || await context.newPage(); + + // Set up disconnect handler + context.browser().on('disconnected', () => { + if (!runState.connectorCompleted && !runState.browserClosedByConnector) { + log(`Browser disconnected for run ${runId}`); + runState.browserClosed = true; + runState.context = null; + runState.page = null; + activeRuns.delete(runId); + send({ type: 'status', runId, status: 'STOPPED' }); + drainStdout().then(() => process.exit(0)); + } + }); + + // Update state + runState.context = context; + runState.page = page; + + // Re-setup network capture on new page + setupNetworkCapture(page); + + // Restore the current browser location when possible so connectors that + // authenticate in headed mode can resume collection without re-implementing + // their own login handoff logic. + await gotoWithRetries(page, resumeUrl, { + context: 'goHeadless resume navigation', + }); + + send({ type: 'log', runId, message: 'Switched to headless mode for background data collection' }); + log(`Switched to headless mode (resumeUrl: ${resumeUrl})`); + }, + + // Direct HTTP fetch from Node.js — no browser needed. + // Works after closeBrowser() for background data collection. + // Automatically includes cookies extracted from the browser session. + httpFetch: async (url, options = {}) => { + const { timeout = 30000, ...fetchOptions } = options; + + // Auto-include cookies from the closed browser session + if (runState.cookies && runState.cookies.length > 0) { + try { + const urlObj = new URL(url); + const relevantCookies = runState.cookies + .filter(c => { + const cookieDomain = c.domain.startsWith('.') ? c.domain.slice(1) : c.domain; + return urlObj.hostname === cookieDomain || urlObj.hostname.endsWith('.' + cookieDomain); + }) + .map(c => `${c.name}=${c.value}`) + .join('; '); + if (relevantCookies) { + fetchOptions.headers = { ...fetchOptions.headers, cookie: relevantCookies }; + } + } catch (e) { + // Ignore cookie injection errors + } + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + try { + const response = await fetch(url, { + ...fetchOptions, + signal: controller.signal, + }); + clearTimeout(timeoutId); + const text = await response.text(); + let json = null; + try { json = JSON.parse(text); } catch {} + if (!response.ok) { + log(`[httpFetch] ${response.status} ${response.statusText} for ${url.substring(0, 100)}`); + log(`[httpFetch] Response body (first 200 chars): ${text.substring(0, 200)}`); + } + return { + ok: response.ok, + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + text, + json, + error: null, + }; + } catch (err) { + clearTimeout(timeoutId); + return { + ok: false, + status: 0, + headers: {}, + text: '', + json: null, + error: err.message, + }; + } + }, + + // Trigger a browser download by navigating to `url` and capture it to a temp + // file. Works for both direct attachment URLs (navigation aborts, download + // fires) and SPA-mediated downloads (the page hydrates and JS starts the + // download). Returns { ok, ready, path, name, size }. ready:false means no + // download fired within the timeout (e.g. an async export still preparing). + captureDownload: async (url, options = {}) => { + const page = requirePage(); + const timeout = options.timeout || 60000; + const downloadPromise = page.waitForEvent('download', { timeout }).catch(() => null); + try { + await page.goto(url, { timeout, waitUntil: 'commit' }); + } catch (e) { + // ERR_ABORTED is expected when the URL serves a direct attachment. + } + const download = await downloadPromise; + if (!download) return { ok: false, ready: false, error: 'no download within timeout' }; + try { + // Persist captured downloads to a durable, known location rather than a + // temp dir, so they survive as a local archive (e.g. the raw Claude + // export ZIP a connector keeps out of the Personal Server). Caller can + // override with options.persistDir. + const dir = options.persistDir || path.join(os.homedir(), '.pdp-connect', 'raw-exports'); + fs.mkdirSync(dir, { recursive: true }); + const name = download.suggestedFilename() || 'download.bin'; + const dest = path.join(dir, name); + await download.saveAs(dest); + const size = fs.statSync(dest).size; + log(`[captureDownload] saved ${name} (${size} bytes) to ${dest}`); + return { ok: true, ready: true, path: dest, name, size }; + } catch (err) { + return { ok: false, ready: true, error: err.message }; + } + }, + + // Read JSON entries out of a ZIP on disk (e.g. a captured export archive). + // `options.include` is an optional list of name substrings to select; when + // omitted, all .json entries are parsed. Returns { ok, names, json }. + extractZipEntries: async (zipPath, options = {}) => { + try { + const buffer = fs.readFileSync(zipPath); + const { names, json } = readZipJsonEntries(buffer, options.include || null); + log(`[extractZipEntries] ${names.length} entries, ${Object.keys(json).length} json parsed from ${path.basename(zipPath)}`); + return { ok: true, names, json }; + } catch (err) { + return { ok: false, error: err.message }; + } + }, + }; +} + +// Run a connector +async function runConnector(runId, connectorPath, url, headless = true, allowHeaded = true, requestedScopes) { + log(`Starting run ${runId} with connector ${connectorPath} (headless: ${headless}, allowHeaded: ${allowHeaded})`); + + // Derive connector ID for persistent browser profile + const connectorFileName = path.basename(connectorPath, path.extname(connectorPath)); + const home = process.env.HOME || process.env.USERPROFILE || ''; + const userDataDir = path.join(home, '.pdp-connect', 'browser-profiles', connectorFileName); + + // Mutable state shared with pageApi + const runState = { + context: null, + page: null, + browserClosed: false, + browserClosedByConnector: false, + connectorCompleted: false, + headless, + allowHeaded, + userDataDir, + browserPath: null, + requestCounter: 0, + pendingInputs: new Map(), + requestedScopes: normalizeRequestedScopes(requestedScopes), + resultEnvelope: null, + }; + + try { + // Read connector script + const connectorCode = fs.readFileSync(connectorPath, 'utf-8'); + + // Resolve browser executable + runState.browserPath = resolveBrowserPath(); + log(`Using browser: ${runState.browserPath}`); + + // On first run, we need to: + // 1. Launch Chrome briefly so it creates its profile/Cookies db + // 2. Close it + // 3. INSERT cookies from the user's Chrome profile into the db + // 4. Relaunch — now Chrome loads the imported cookies from disk + const markerFile = path.join(userDataDir, '.cookies-imported'); + if (isSystemChrome(runState.browserPath) && !fs.existsSync(markerFile)) { + log('First run: launching browser to initialize profile...'); + const tempCtx = await launchPersistentContext(userDataDir, true, runState.browserPath); + await tempCtx.close(); + log('Profile initialized, importing cookies...'); + importChromecookies(userDataDir, runState.browserPath); + } + + // Launch browser with persistent context (cookies already in db on first run) + const context = await launchPersistentContext(userDataDir, headless, runState.browserPath); + const page = context.pages()[0] || await context.newPage(); + + runState.context = context; + runState.page = page; + + // Handle browser disconnect (user closed browser window) + context.browser().on('disconnected', () => { + if (!runState.connectorCompleted && !runState.browserClosedByConnector && activeRuns.has(runId)) { + log(`Browser disconnected for run ${runId} (user closed window)`); + runState.browserClosed = true; + runState.context = null; + runState.page = null; + activeRuns.delete(runId); + send({ type: 'status', runId, status: 'STOPPED' }); + drainStdout().then(() => process.exit(0)); + } + }); + + // Store for cleanup + activeRuns.set(runId, { + runState, + setCompleted: () => { runState.connectorCompleted = true; }, + }); + + // Create page API + const pageApi = createPageApi(runState, runId); + + // Navigate to starting URL + log(`Navigating to initial URL: ${url}`); + await gotoWithRetries(page, url, { + context: 'initial navigation', + }); + log('Initial navigation complete'); + send({ type: 'status', runId, status: 'RUNNING' }); + + // Build the connector execution wrapper + // The connector has an IIFE at the end - we need to return its Promise + // Find the LAST IIFE and add 'return' before it (there may be inner IIFEs in helpers) + let modifiedCode = connectorCode; + + // Match both a leading-newline IIFE and one that starts on line 1 + // (e.g. steam-playwright.js). An earlier version required `\n` as a + // leading anchor, which silently false-succeeded any script whose only + // top-level IIFE was on line 1 — the AsyncFunction would resolve to + // undefined instantly while the IIFE ran fire-and-forget. + const iifePattern = /(?:^|\n)\(async\s*\(\)\s*=>\s*\{/g; + const matches = [...modifiedCode.matchAll(iifePattern)]; + + if (matches.length > 0) { + const lastMatch = matches[matches.length - 1]; + const matchedText = lastMatch[0]; + const leadingNewline = matchedText.startsWith('\n'); + const insertPos = lastMatch.index; + const replacement = leadingNewline + ? '\nreturn (async () => {' + : 'return (async () => {'; + modifiedCode = modifiedCode.substring(0, insertPos) + + replacement + + modifiedCode.substring(insertPos + matchedText.length); + log(`Added return before IIFE (match ${matches.length} of ${matches.length})`); + } else { + log('WARNING: Could not find IIFE pattern in connector code'); + } + + // Execute connector with page API in scope using AsyncFunction + log('Starting connector execution...'); + const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor; + const runConnectorFn = new AsyncFunction('page', modifiedCode); + + log('Calling connector function...'); + const result = await runConnectorFn.call(null, pageApi); + log('Connector function completed with result:', result != null ? 'has result' : 'undefined'); + + if (result !== undefined) { + storeConnectorResult(runState, result); + } else if (runState.resultEnvelope === null) { + storeConnectorResult(runState, result); + } + const classification = emitStoredConnectorResult(runId, runState); + + // Mark as completed to prevent disconnect handler from sending STOPPED + runState.connectorCompleted = true; + + // Close browser if still open + if (!runState.browserClosed && runState.context) { + await new Promise(resolve => setTimeout(resolve, 2000)); + try { + await runState.context.close(); + } catch (e) { + // Browser may already be closed + } + } + + activeRuns.delete(runId); + + // Exit process after successful completion + log( + 'Connector completed, classified outcome:', + classification ? classification.outcome : 'unknown', + ); + await drainStdout(); + process.exit(classification && classification.outcome === 'failure' ? 1 : 0); + + } catch (error) { + log(`Error in run ${runId}:`, error.message); + send({ type: 'error', runId, message: error.message }); + send({ type: 'status', runId, status: 'ERROR' }); + + // Cleanup on error + if (runState.context && !runState.browserClosed) { + try { + await runState.context.close(); + } catch (e) {} + } + activeRuns.delete(runId); + + // Exit process after error + log('Connector failed, exiting'); + await drainStdout(); + process.exit(1); + } +} + +// Stop a run +async function stopRun(runId) { + const run = activeRuns.get(runId); + if (run) { + log(`Stopping run ${runId}`); + // Reject any pending requestInput promises so the connector doesn't hang + for (const [, pending] of run.runState.pendingInputs) { + pending.reject(new Error('Run stopped')); + } + run.runState.pendingInputs.clear(); + if (run.runState && run.runState.context && !run.runState.browserClosed) { + await run.runState.context.close().catch(() => {}); + } + activeRuns.delete(runId); + send({ type: 'status', runId, status: 'STOPPED' }); + } +} + +// Main loop - read commands from stdin +async function main() { + log('Playwright runner started'); + send({ type: 'ready' }); + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + terminal: false + }); + + for await (const line of rl) { + try { + const cmd = JSON.parse(line); + + switch (cmd.type) { + case 'run': + runConnector( + cmd.runId, + cmd.connectorPath, + cmd.url, + cmd.headless !== false, + cmd.allowHeaded !== false, + cmd.requestedScopes, + ); + break; + + case 'stop': + await stopRun(cmd.runId); + break; + + case 'quit': + log('Quitting...'); + for (const [runId, run] of activeRuns) { + if (run.runState && run.runState.context && !run.runState.browserClosed) { + await run.runState.context.close().catch(() => {}); + } + } + process.exit(0); + break; + + case 'test': + // Simple test to prove Node.js is working + const os = require('os'); + send({ + type: 'test-result', + data: { + nodejs: process.version, + platform: process.platform, + arch: process.arch, + hostname: os.hostname(), + cpus: os.cpus().length, + memory: Math.round(os.totalmem() / 1024 / 1024 / 1024) + ' GB', + uptime: Math.round(os.uptime() / 60) + ' minutes' + } + }); + break; + + case 'evaluate': { + const evalRun = activeRuns.get(cmd.runId); + if (!evalRun) { + send({ type: 'evaluate-result', runId: cmd.runId, error: `No active run: ${cmd.runId}` }); + break; + } + const { runState: evalState } = evalRun; + if (evalState.browserClosed || !evalState.page) { + send({ type: 'evaluate-result', runId: cmd.runId, error: 'Browser is closed' }); + break; + } + // Non-blocking: don't await so stdin loop keeps processing other commands. + // Wrapped in try so synchronous throws (e.g. page torn down mid-call) + // always produce an evaluate-result instead of hanging the driver. + try { + evalState.page.evaluate(cmd.script) + .then(result => send({ type: 'evaluate-result', runId: cmd.runId, result })) + .catch(e => send({ type: 'evaluate-result', runId: cmd.runId, error: e.stack || e.message })); + } catch (e) { + send({ type: 'evaluate-result', runId: cmd.runId, error: e.stack || e.message }); + } + break; + } + + case 'input-response': { + const inputRun = activeRuns.get(cmd.runId); + if (!inputRun) break; + const pending = inputRun.runState.pendingInputs.get(cmd.requestId); + if (!pending) break; + inputRun.runState.pendingInputs.delete(cmd.requestId); + if (cmd.error) { + pending.reject(new Error(typeof cmd.error === 'string' ? cmd.error : JSON.stringify(cmd.error))); + } else { + pending.resolve(cmd.data); + } + break; + } + + case 'screenshot': { + const ssRun = activeRuns.get(cmd.runId); + if (!ssRun) { + send({ type: 'screenshot-result', runId: cmd.runId, error: `No active run: ${cmd.runId}` }); + break; + } + const { runState: ssState } = ssRun; + if (ssState.browserClosed || !ssState.page) { + send({ type: 'screenshot-result', runId: cmd.runId, error: 'Browser is closed' }); + break; + } + try { + ssState.page.screenshot({ type: 'jpeg', quality: 70, timeout: 5000 }) + .then(buffer => send({ type: 'screenshot-result', runId: cmd.runId, data: buffer.toString('base64') })) + .catch(e => send({ type: 'screenshot-result', runId: cmd.runId, error: e.stack || e.message })); + } catch (e) { + send({ type: 'screenshot-result', runId: cmd.runId, error: e.stack || e.message }); + } + break; + } + + default: + log(`Unknown command: ${cmd.type}`); + } + } catch (error) { + log(`Error parsing command: ${error.message}`); + } + } +} + +main().catch(err => { + log('Fatal error:', err); + process.exit(1); +}); diff --git a/playwright-runner/package.json b/playwright-runner/package.json new file mode 100644 index 0000000..cb2f7c5 --- /dev/null +++ b/playwright-runner/package.json @@ -0,0 +1,24 @@ +{ + "name": "pdp-connect-playwright-runner", + "version": "1.0.0", + "description": "Playwright sidecar for PDP-Connect connectors", + "main": "index.cjs", + "type": "module", + "scripts": { + "start": "node index.cjs", + "build": "node scripts/build.js", + "build:lean": "node scripts/build.js --lean", + "postinstall": "playwright install chromium" + }, + "pkg": { + "assets": [], + "targets": ["node20-macos-arm64", "node20-macos-x64", "node20-win-x64", "node20-linux-x64"], + "outputPath": "dist" + }, + "dependencies": { + "playwright": "^1.40.0" + }, + "devDependencies": { + "@yao-pkg/pkg": "^5.12.0" + } +} diff --git a/playwright-runner/result-classifier.cjs b/playwright-runner/result-classifier.cjs new file mode 100644 index 0000000..6786725 --- /dev/null +++ b/playwright-runner/result-classifier.cjs @@ -0,0 +1,294 @@ +const CONNECTOR_RESULT_RESERVED_KEYS = new Set([ + 'requestedScopes', + 'timestamp', + 'version', + 'platform', + 'exportSummary', + 'errors', +]); + +const TELEMETRY_ERROR_CLASS_PRIORITY = [ + 'protocol_violation', + 'auth_failed', + 'rate_limited', + 'upstream_error', + 'navigation_error', + 'network_error', + 'timeout', + 'selector_error', + 'runtime_error', + 'personal_server_unavailable', + 'unknown', +]; + +function isPlainRecord(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function protocolFailure(message, scopeSummary) { + return { + outcome: 'failure', + errorClass: 'protocol_violation', + scopeSummary: scopeSummary || { + requested: 0, + produced: 0, + degraded: 0, + omitted: 0, + }, + debug: `Protocol violation: ${message}`, + }; +} + +function dominantErrorClass(errors) { + for (const errorClass of TELEMETRY_ERROR_CLASS_PRIORITY) { + if (errors.some((error) => error.errorClass === errorClass)) { + return errorClass; + } + } + return 'unknown'; +} + +function hasSameMembers(a, b) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) { + return false; + } + + const left = new Set(a); + const right = new Set(b); + if (left.size !== a.length || right.size !== b.length) { + return false; + } + + for (const value of left) { + if (!right.has(value)) { + return false; + } + } + return true; +} + +function validateConnectorErrors(rawErrors, requestedScopes) { + if (!Array.isArray(rawErrors)) { + return { protocolMessage: 'errors must be an array' }; + } + + const errors = []; + for (const rawError of rawErrors) { + if (!isPlainRecord(rawError)) { + return { protocolMessage: 'errors entries must be objects' }; + } + + const { errorClass, reason, disposition, scope, phase } = rawError; + if ( + typeof errorClass !== 'string' || + !TELEMETRY_ERROR_CLASS_PRIORITY.includes(errorClass) + ) { + return { protocolMessage: 'errors entries must use a known errorClass' }; + } + if (typeof reason !== 'string' || reason.trim().length === 0) { + return { protocolMessage: 'errors entries must include a non-empty reason' }; + } + if (disposition !== 'omitted' && disposition !== 'degraded' && disposition !== 'fatal') { + return { protocolMessage: 'errors entries must use omitted, degraded, or fatal disposition' }; + } + if ((disposition === 'omitted' || disposition === 'degraded') && typeof scope !== 'string') { + return { protocolMessage: `${disposition} errors must include a scope` }; + } + if (typeof scope === 'string' && scope.length > 0 && !requestedScopes.has(scope)) { + return { protocolMessage: `errors scope '${scope}' is outside requestedScopes` }; + } + if (phase !== undefined && typeof phase !== 'string') { + return { protocolMessage: 'errors phase must be a string when present' }; + } + + errors.push({ + errorClass, + reason, + disposition, + ...(typeof scope === 'string' ? { scope } : {}), + ...(typeof phase === 'string' ? { phase } : {}), + }); + } + + return { errors }; +} + +function isCanonicalScopeId(scope) { + return ( + typeof scope === 'string' && + scope.length > 0 && + scope.includes('.') && + !scope.startsWith('.') && + !scope.endsWith('.') + ); +} + +function classifyConnectorResult(result, options) { + if (!isPlainRecord(result)) { + return protocolFailure('connector result must be an object'); + } + + if ( + typeof result.success === 'boolean' && + (Object.prototype.hasOwnProperty.call(result, 'data') || + Object.prototype.hasOwnProperty.call(result, 'error')) + ) { + return protocolFailure('legacy { success, data } wrapper is not supported'); + } + + const requestedScopesValue = result.requestedScopes; + if (!Array.isArray(requestedScopesValue) || requestedScopesValue.length === 0) { + return protocolFailure('requestedScopes must be a non-empty array'); + } + + if (requestedScopesValue.some((scope) => !isCanonicalScopeId(scope))) { + return protocolFailure('requestedScopes must contain canonical scope ids'); + } + + const requestedScopes = requestedScopesValue.slice(); + const requestedScopeSet = new Set(requestedScopes); + if (requestedScopeSet.size !== requestedScopes.length) { + return protocolFailure('requestedScopes must be deduplicated'); + } + + if ( + options && + Array.isArray(options.expectedRequestedScopes) && + !hasSameMembers(requestedScopes, options.expectedRequestedScopes) + ) { + return protocolFailure('requestedScopes does not match the resolved execution scope set'); + } + + if (typeof result.timestamp !== 'string' || result.timestamp.length === 0) { + return protocolFailure('timestamp is required'); + } + if (typeof result.version !== 'string' || result.version.length === 0) { + return protocolFailure('version is required'); + } + if (typeof result.platform !== 'string' || result.platform.length === 0) { + return protocolFailure('platform is required'); + } + + if (!isPlainRecord(result.exportSummary)) { + return protocolFailure('exportSummary is required'); + } + + const exportSummary = result.exportSummary; + if ( + typeof exportSummary.count !== 'number' || + !Number.isFinite(exportSummary.count) || + exportSummary.count < 0 + ) { + return protocolFailure('exportSummary.count must be a non-negative number'); + } + if (typeof exportSummary.label !== 'string' || exportSummary.label.length === 0) { + return protocolFailure('exportSummary.label must be a string'); + } + if ( + exportSummary.details !== undefined && + !isPlainRecord(exportSummary.details) + ) { + return protocolFailure('exportSummary.details must be an object when present'); + } + + const { errors, protocolMessage } = validateConnectorErrors(result.errors, requestedScopeSet); + if (!errors) { + return protocolFailure(protocolMessage || 'errors is malformed'); + } + + const producedScopes = Object.keys(result).filter( + (key) => !CONNECTOR_RESULT_RESERVED_KEYS.has(key), + ); + for (const producedScope of producedScopes) { + if (!isCanonicalScopeId(producedScope)) { + return protocolFailure(`produced scope key '${producedScope}' is not canonical`); + } + if (!requestedScopeSet.has(producedScope)) { + return protocolFailure(`produced scope key '${producedScope}' is outside requestedScopes`); + } + } + + const producedScopeSet = new Set(producedScopes); + const degradedScopes = new Set( + errors + .filter((error) => error.disposition === 'degraded' && error.scope) + .map((error) => error.scope), + ); + const omittedScopes = new Set( + errors + .filter((error) => error.disposition === 'omitted' && error.scope) + .map((error) => error.scope), + ); + const fatalErrors = errors.filter((error) => error.disposition === 'fatal'); + + for (const scope of degradedScopes) { + if (!producedScopeSet.has(scope)) { + return protocolFailure(`degraded scope '${scope}' must also be present in the result`); + } + if (omittedScopes.has(scope)) { + return protocolFailure(`scope '${scope}' cannot be both degraded and omitted`); + } + } + + for (const scope of omittedScopes) { + if (producedScopeSet.has(scope)) { + return protocolFailure(`omitted scope '${scope}' must not be present in the result`); + } + } + + const scopeSummary = { + requested: requestedScopeSet.size, + produced: producedScopeSet.size, + degraded: degradedScopes.size, + omitted: omittedScopes.size, + }; + + if (fatalErrors.length > 0) { + return { + outcome: 'failure', + errorClass: dominantErrorClass(fatalErrors), + recordCount: exportSummary.count, + scopeSummary, + debug: fatalErrors[0] ? fatalErrors[0].reason : undefined, + }; + } + + for (const scope of requestedScopeSet) { + if (!producedScopeSet.has(scope) && !omittedScopes.has(scope)) { + return protocolFailure( + `requested scope '${scope}' must be produced or marked omitted`, + scopeSummary, + ); + } + } + + if (producedScopeSet.size === 0) { + return { + outcome: 'failure', + errorClass: dominantErrorClass(errors), + recordCount: exportSummary.count, + scopeSummary, + debug: errors[0] ? errors[0].reason : undefined, + }; + } + + if (errors.length === 0) { + return { + outcome: 'success', + recordCount: exportSummary.count, + scopeSummary, + }; + } + + return { + outcome: 'partial', + errorClass: dominantErrorClass(errors), + recordCount: exportSummary.count, + scopeSummary, + debug: errors[0] ? errors[0].reason : undefined, + }; +} + +module.exports = { + classifyConnectorResult, +}; diff --git a/playwright-runner/result-classifier.conformance.test.cjs b/playwright-runner/result-classifier.conformance.test.cjs new file mode 100644 index 0000000..725f600 --- /dev/null +++ b/playwright-runner/result-classifier.conformance.test.cjs @@ -0,0 +1,51 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { join } = require('node:path'); + +const { classifyConnectorResult } = require('./result-classifier.cjs'); + +const corpusDir = join(__dirname, 'conformance', 'honest-telemetry'); +const corpus = JSON.parse(readFileSync(join(corpusDir, 'corpus.json'), 'utf8')); + +for (const testCase of corpus.cases) { + const fixture = JSON.parse( + readFileSync(join(corpusDir, testCase.fixture), 'utf8'), + ); + const classification = classifyConnectorResult(fixture, { + expectedRequestedScopes: fixture.requestedScopes, + }); + + assert.equal( + classification.outcome, + testCase.expected.classification.outcome, + `${testCase.id}: outcome mismatch`, + ); + + if (testCase.expected.classification.errorClass) { + assert.equal( + classification.errorClass, + testCase.expected.classification.errorClass, + `${testCase.id}: errorClass mismatch`, + ); + } + + if (testCase.expected.classification.scopeSummary) { + assert.deepEqual( + classification.scopeSummary, + testCase.expected.classification.scopeSummary, + `${testCase.id}: scopeSummary mismatch`, + ); + } + + if (testCase.expected.validity === 'protocol_violation') { + assert.equal( + classification.errorClass, + 'protocol_violation', + `${testCase.id}: expected protocol violation`, + ); + } +} + +console.log( + `honest telemetry conformance passed (${corpus.cases.length} cases)`, +); diff --git a/playwright-runner/result-classifier.test.cjs b/playwright-runner/result-classifier.test.cjs new file mode 100644 index 0000000..4230ebc --- /dev/null +++ b/playwright-runner/result-classifier.test.cjs @@ -0,0 +1,116 @@ +const assert = require('node:assert/strict'); + +const { classifyConnectorResult } = require('./result-classifier.cjs'); + +function makeResult(overrides = {}) { + return { + requestedScopes: ['chatgpt.conversations'], + timestamp: '2026-04-14T12:00:00.000Z', + version: '1.2.3', + platform: 'chatgpt', + exportSummary: { + count: 1, + label: '1 conversation', + }, + errors: [], + 'chatgpt.conversations': [{ id: 'conv-1' }], + ...overrides, + }; +} + +function run() { + const success = classifyConnectorResult(makeResult(), { + expectedRequestedScopes: ['chatgpt.conversations'], + }); + assert.equal(success.outcome, 'success'); + assert.deepEqual(success.scopeSummary, { + requested: 1, + produced: 1, + degraded: 0, + omitted: 0, + }); + + const partial = classifyConnectorResult( + makeResult({ + requestedScopes: ['chatgpt.conversations', 'chatgpt.memories'], + errors: [ + { + errorClass: 'selector_error', + reason: 'Memories selector drifted', + disposition: 'omitted', + scope: 'chatgpt.memories', + }, + ], + }), + { + expectedRequestedScopes: ['chatgpt.conversations', 'chatgpt.memories'], + }, + ); + assert.equal(partial.outcome, 'partial'); + assert.equal(partial.errorClass, 'selector_error'); + assert.deepEqual(partial.scopeSummary, { + requested: 2, + produced: 1, + degraded: 0, + omitted: 1, + }); + + const fatalFailure = classifyConnectorResult( + makeResult({ + errors: [ + { + errorClass: 'auth_failed', + reason: 'Auth expired', + disposition: 'fatal', + }, + ], + }), + { + expectedRequestedScopes: ['chatgpt.conversations'], + }, + ); + assert.equal(fatalFailure.outcome, 'failure'); + assert.equal(fatalFailure.errorClass, 'auth_failed'); + + const legacyWrapper = classifyConnectorResult({ + success: true, + data: makeResult(), + }); + assert.equal(legacyWrapper.outcome, 'failure'); + assert.equal(legacyWrapper.errorClass, 'protocol_violation'); + assert.match(legacyWrapper.debug || '', /legacy \{ success, data \} wrapper/i); + + const overProduced = classifyConnectorResult( + makeResult({ + 'chatgpt.memories': [{ id: 'memory-1' }], + }), + { + expectedRequestedScopes: ['chatgpt.conversations'], + }, + ); + assert.equal(overProduced.outcome, 'failure'); + assert.equal(overProduced.errorClass, 'protocol_violation'); + assert.match(overProduced.debug || '', /outside requestedScopes/i); + + const malformedErrors = classifyConnectorResult( + makeResult({ + errors: [{ errorClass: 'selector_error', reason: 'bad', disposition: 'omitted' }], + }), + { + expectedRequestedScopes: ['chatgpt.conversations'], + }, + ); + assert.equal(malformedErrors.outcome, 'failure'); + assert.equal(malformedErrors.errorClass, 'protocol_violation'); + assert.match(malformedErrors.debug || '', /must include a scope/i); + + const scopeMismatch = classifyConnectorResult(makeResult(), { + expectedRequestedScopes: ['chatgpt.memories'], + }); + assert.equal(scopeMismatch.outcome, 'failure'); + assert.equal(scopeMismatch.errorClass, 'protocol_violation'); + assert.match(scopeMismatch.debug || '', /resolved execution scope set/i); +} + +run(); +console.log('result-classifier tests passed'); diff --git a/playwright-runner/runner-compat.cjs b/playwright-runner/runner-compat.cjs new file mode 100644 index 0000000..47818c9 --- /dev/null +++ b/playwright-runner/runner-compat.cjs @@ -0,0 +1,217 @@ +function isPlainRecord(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function inferPlatformFromScopes(requestedScopes) { + if (!Array.isArray(requestedScopes)) { + return null; + } + + for (const scope of requestedScopes) { + if (typeof scope !== 'string') { + continue; + } + + const [platform] = scope.split('.', 1); + if (platform) { + return platform; + } + } + + return null; +} + +function isMeaningfulResumeUrl(url) { + if (typeof url !== 'string' || url.trim().length === 0) { + return false; + } + + try { + const parsed = new URL(url); + return parsed.protocol === 'http:' || parsed.protocol === 'https:'; + } catch { + return false; + } +} + +function resolveHeadlessResumeUrl(options = {}) { + const { resumeUrl, currentUrl } = options; + + if (isMeaningfulResumeUrl(resumeUrl)) { + return resumeUrl; + } + + if (isMeaningfulResumeUrl(currentUrl)) { + return currentUrl; + } + + return 'about:blank'; +} + +function inferLegacyErrorClass(reason) { + const normalizedReason = String(reason || '').toLowerCase(); + + if (/(login|sign in|sign-in|authenticate|authentication|auth|2fa|password|captcha|verification)/.test(normalizedReason)) { + return 'auth_failed'; + } + if (/(rate limit|too many requests|429)/.test(normalizedReason)) { + return 'rate_limited'; + } + if (/(timeout|timed out)/.test(normalizedReason)) { + return 'timeout'; + } + if (/(network|fetch failed|econn|socket|dns)/.test(normalizedReason)) { + return 'network_error'; + } + if (/(selector|locator)/.test(normalizedReason)) { + return 'selector_error'; + } + if (/(navigation|redirect|page crashed|about:blank)/.test(normalizedReason)) { + return 'navigation_error'; + } + + return 'runtime_error'; +} + +function normalizeProtocolDefaults(result, options) { + if (!isPlainRecord(result)) { + return result; + } + + const { requestedScopes = [], inferredPlatform } = options; + let changed = false; + const normalized = { ...result }; + + if (!Array.isArray(normalized.requestedScopes) || normalized.requestedScopes.length === 0) { + normalized.requestedScopes = [...requestedScopes]; + changed = true; + } + + if (!Array.isArray(normalized.errors)) { + normalized.errors = []; + changed = true; + } + + if ( + (typeof normalized.platform !== 'string' || normalized.platform.length === 0) && + inferredPlatform + ) { + normalized.platform = inferredPlatform; + changed = true; + } + + if (isPlainRecord(normalized.exportSummary) && normalized.exportSummary.details !== undefined) { + const details = normalized.exportSummary.details; + if (!isPlainRecord(details)) { + normalized.exportSummary = { ...normalized.exportSummary }; + if (typeof details === 'string') { + normalized.exportSummary.details = { text: details }; + } else if (Array.isArray(details)) { + normalized.exportSummary.details = { items: details }; + } else { + normalized.exportSummary.details = { value: details }; + } + changed = true; + } + } + + return changed ? normalized : result; +} + +function normalizeFailureReason(errorValue) { + if (typeof errorValue === 'string' && errorValue.trim().length > 0) { + return errorValue.trim(); + } + + if ( + isPlainRecord(errorValue) && + typeof errorValue.message === 'string' && + errorValue.message.trim().length > 0 + ) { + return errorValue.message.trim(); + } + + return 'Connector reported failure'; +} + +function synthesizeLegacyFailureResult(result, options) { + const { requestedScopes = [], inferredPlatform } = options; + const reason = normalizeFailureReason(result.error); + + return { + requestedScopes: [...requestedScopes], + timestamp: new Date().toISOString(), + version: + typeof result.version === 'string' && result.version.length > 0 + ? result.version + : 'legacy-compat', + platform: + typeof result.platform === 'string' && result.platform.length > 0 + ? result.platform + : inferredPlatform || 'unknown', + exportSummary: { + count: 0, + label: 'items', + }, + errors: [ + { + errorClass: inferLegacyErrorClass(reason), + reason, + disposition: 'fatal', + }, + ], + }; +} + +function normalizeConnectorResult(result, options = {}) { + const requestedScopes = Array.isArray(options.requestedScopes) + ? options.requestedScopes.filter((scope) => typeof scope === 'string' && scope.length > 0) + : []; + const inferredPlatform = inferPlatformFromScopes(requestedScopes); + + if (!isPlainRecord(result)) { + return result; + } + + const hasLegacyWrapper = + typeof result.success === 'boolean' && + (Object.prototype.hasOwnProperty.call(result, 'data') || + Object.prototype.hasOwnProperty.call(result, 'error')); + + if (!hasLegacyWrapper) { + return normalizeProtocolDefaults(result, { + requestedScopes, + inferredPlatform, + }); + } + + if (result.success) { + if (!isPlainRecord(result.data)) { + return synthesizeLegacyFailureResult( + { + ...result, + error: 'Legacy success wrapper must contain an object data payload', + }, + { + requestedScopes, + inferredPlatform, + }, + ); + } + + return normalizeProtocolDefaults(result.data, { + requestedScopes, + inferredPlatform, + }); + } + + return synthesizeLegacyFailureResult(result, { + requestedScopes, + inferredPlatform, + }); +} + +module.exports = { + normalizeConnectorResult, + resolveHeadlessResumeUrl, +}; diff --git a/playwright-runner/runner-compat.test.cjs b/playwright-runner/runner-compat.test.cjs new file mode 100644 index 0000000..4ccf3a3 --- /dev/null +++ b/playwright-runner/runner-compat.test.cjs @@ -0,0 +1,143 @@ +const assert = require('node:assert/strict'); + +const { classifyConnectorResult } = require('./result-classifier.cjs'); +const { + normalizeConnectorResult, + resolveHeadlessResumeUrl, +} = require('./runner-compat.cjs'); + +function makeLegacyPayload(overrides = {}) { + return { + timestamp: '2026-04-21T18:00:00.000Z', + version: '1.0.0-playwright', + platform: 'linkedin', + exportSummary: { + count: 1, + label: 'items', + }, + 'linkedin.profile': { + headline: 'Builder', + }, + ...overrides, + }; +} + +function run() { + assert.equal( + resolveHeadlessResumeUrl({ + resumeUrl: 'https://github.com/settings/profile', + currentUrl: 'https://github.com/', + }), + 'https://github.com/settings/profile', + ); + + assert.equal( + resolveHeadlessResumeUrl({ + resumeUrl: 'about:blank', + currentUrl: 'https://www.instagram.com/example/', + }), + 'https://www.instagram.com/example/', + ); + + assert.equal( + resolveHeadlessResumeUrl({ + currentUrl: 'chrome://settings', + }), + 'about:blank', + ); + + const normalizedWrappedSuccess = normalizeConnectorResult( + { + success: true, + data: makeLegacyPayload(), + }, + { + requestedScopes: ['linkedin.profile'], + }, + ); + assert.deepEqual(normalizedWrappedSuccess.requestedScopes, ['linkedin.profile']); + assert.deepEqual(normalizedWrappedSuccess.errors, []); + assert.equal( + classifyConnectorResult(normalizedWrappedSuccess, { + expectedRequestedScopes: ['linkedin.profile'], + }).outcome, + 'success', + ); + + const normalizedStringDetails = normalizeConnectorResult( + makeLegacyPayload({ + exportSummary: { + count: 1, + label: 'items', + details: 'x', + }, + }), + { + requestedScopes: ['linkedin.profile'], + }, + ); + assert.deepEqual(normalizedStringDetails.requestedScopes, ['linkedin.profile']); + assert.deepEqual(normalizedStringDetails.errors, []); + assert.equal( + classifyConnectorResult(normalizedStringDetails, { + expectedRequestedScopes: ['linkedin.profile'], + }).outcome, + 'success', + ); + assert.deepEqual(normalizedStringDetails.exportSummary.details, { text: 'x' }); + + const normalizedArrayDetails = normalizeConnectorResult( + makeLegacyPayload({ + exportSummary: { + count: 2, + label: 'items', + details: ['2 orders', '1 refund'], + }, + }), + { + requestedScopes: ['linkedin.profile'], + }, + ); + assert.deepEqual(normalizedArrayDetails.exportSummary.details, { + items: ['2 orders', '1 refund'], + }); + assert.equal( + classifyConnectorResult(normalizedArrayDetails, { + expectedRequestedScopes: ['linkedin.profile'], + }).outcome, + 'success', + ); + + const normalizedFailure = normalizeConnectorResult( + { + success: false, + error: 'Login requires a headed browser or requestInput support.', + }, + { + requestedScopes: ['instagram.profile'], + }, + ); + const failureClassification = classifyConnectorResult(normalizedFailure, { + expectedRequestedScopes: ['instagram.profile'], + }); + assert.equal(failureClassification.outcome, 'failure'); + assert.equal(failureClassification.errorClass, 'auth_failed'); + + const malformedLegacySuccess = normalizeConnectorResult( + { + success: true, + data: null, + }, + { + requestedScopes: ['github.profile'], + }, + ); + const malformedClassification = classifyConnectorResult(malformedLegacySuccess, { + expectedRequestedScopes: ['github.profile'], + }); + assert.equal(malformedClassification.outcome, 'failure'); + assert.equal(malformedClassification.errorClass, 'runtime_error'); +} + +run(); +console.log('runner-compat tests passed'); diff --git a/playwright-runner/zip-reader.cjs b/playwright-runner/zip-reader.cjs new file mode 100644 index 0000000..5c31965 --- /dev/null +++ b/playwright-runner/zip-reader.cjs @@ -0,0 +1,54 @@ +'use strict'; + +const zlib = require('zlib'); + +// Minimal, dependency-free ZIP reader (stored + deflate entries). +// Parses the central directory so it works on real archives without shelling +// out to an `unzip` binary (cross-platform). Returns { names, json } where +// `json` holds parsed JSON entries matching `include` (substring list, or all +// .json when null). +function readZipJsonEntries(buffer, include) { + const EOCD_SIG = 0x06054b50; + const CEN_SIG = 0x02014b50; + // Locate End Of Central Directory (scan backwards; comment is usually empty). + let eocd = -1; + for (let i = buffer.length - 22; i >= 0 && i >= buffer.length - 22 - 0x10000; i--) { + if (buffer.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; } + } + if (eocd < 0) throw new Error('not a zip (no EOCD)'); + const cdCount = buffer.readUInt16LE(eocd + 10); + let off = buffer.readUInt32LE(eocd + 16); + const names = []; + const json = {}; + for (let n = 0; n < cdCount; n++) { + if (buffer.readUInt32LE(off) !== CEN_SIG) break; + const method = buffer.readUInt16LE(off + 10); + const compSize = buffer.readUInt32LE(off + 20); + const nameLen = buffer.readUInt16LE(off + 28); + const extraLen = buffer.readUInt16LE(off + 30); + const commentLen = buffer.readUInt16LE(off + 32); + const localOff = buffer.readUInt32LE(off + 42); + const name = buffer.toString('utf8', off + 46, off + 46 + nameLen); + off += 46 + nameLen + extraLen + commentLen; + + if (name.endsWith('/')) { names.push(name); continue; } // directory entry + names.push(name); + const wanted = name.endsWith('.json') && (!include || include.some((s) => name.includes(s))); + if (!wanted) continue; + + // Resolve the data offset from the local file header (its name/extra lengths + // can differ from the central record). + const lNameLen = buffer.readUInt16LE(localOff + 26); + const lExtraLen = buffer.readUInt16LE(localOff + 28); + const dataStart = localOff + 30 + lNameLen + lExtraLen; + const comp = buffer.subarray(dataStart, dataStart + compSize); + let raw; + if (method === 0) raw = comp; // stored + else if (method === 8) raw = zlib.inflateRawSync(comp); // deflate + else continue; // unsupported method — skip rather than crash the run + try { json[name] = JSON.parse(raw.toString('utf8')); } catch (e) { /* skip unparseable */ } + } + return { names, json }; +} + +module.exports = { readZipJsonEntries }; diff --git a/playwright-runner/zip-reader.test.cjs b/playwright-runner/zip-reader.test.cjs new file mode 100644 index 0000000..e214531 --- /dev/null +++ b/playwright-runner/zip-reader.test.cjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +/** + * Unit test for the runner's dependency-free ZIP reader (readZipJsonEntries), + * which backs page.extractZipEntries(). Builds real ZIP archives in-memory + * (stored + deflate entries) and asserts JSON entries are parsed and filtered. + */ + +const assert = require('assert'); +const zlib = require('zlib'); +const { readZipJsonEntries } = require('./zip-reader.cjs'); + +// Build a minimal but spec-valid ZIP from { name: Buffer } entries. +// Uses deflate (method 8) when it helps, stored (method 0) otherwise. +function buildZip(entries) { + const crcTable = (() => { + const t = []; + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + t[n] = c >>> 0; + } + return t; + })(); + const crc32 = (buf) => { + let c = 0xffffffff; + for (let i = 0; i < buf.length; i++) c = crcTable[(c ^ buf[i]) & 0xff] ^ (c >>> 8); + return (c ^ 0xffffffff) >>> 0; + }; + + const locals = []; + const centrals = []; + let offset = 0; + for (const [name, content] of Object.entries(entries)) { + const nameBuf = Buffer.from(name, 'utf8'); + const deflated = zlib.deflateRawSync(content); + const useDeflate = deflated.length < content.length; + const method = useDeflate ? 8 : 0; + const data = useDeflate ? deflated : content; + const crc = crc32(content); + + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(0, 6); + local.writeUInt16LE(method, 8); + local.writeUInt16LE(0, 10); + local.writeUInt16LE(0, 12); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(content.length, 22); + local.writeUInt16LE(nameBuf.length, 26); + local.writeUInt16LE(0, 28); + locals.push(local, nameBuf, data); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(0, 8); + central.writeUInt16LE(method, 10); + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(data.length, 20); + central.writeUInt32LE(content.length, 24); + central.writeUInt16LE(nameBuf.length, 28); + central.writeUInt32LE(offset, 42); + centrals.push({ header: central, nameBuf }); + + offset += local.length + nameBuf.length + data.length; + } + + const cdStart = offset; + const cdParts = []; + for (const { header, nameBuf } of centrals) { + cdParts.push(header, nameBuf); + } + const cd = Buffer.concat(cdParts); + + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(centrals.length, 8); + eocd.writeUInt16LE(centrals.length, 10); + eocd.writeUInt32LE(cd.length, 12); + eocd.writeUInt32LE(cdStart, 16); + + return Buffer.concat([...locals, cd, eocd]); +} + +const cases = []; +function test(name, fn) { + try { fn(); cases.push({ name, ok: true }); } + catch (err) { cases.push({ name, ok: false, err: err.message }); } +} + +test('parses stored + deflate JSON entries', () => { + // A large, compressible payload forces the deflate path; a tiny one stays stored. + const big = JSON.stringify(Array.from({ length: 500 }, (_, i) => ({ id: i, text: 'hello world' }))); + const zip = buildZip({ + 'conversations.json': Buffer.from(big), + 'users.json': Buffer.from('[{"full_name":"Volod I"}]'), + 'notes.txt': Buffer.from('ignore me'), + }); + const { names, json } = readZipJsonEntries(zip, null); + assert.ok(names.includes('conversations.json') && names.includes('notes.txt')); + assert.strictEqual(json['conversations.json'].length, 500); + assert.strictEqual(json['users.json'][0].full_name, 'Volod I'); + assert.ok(!('notes.txt' in json), 'non-json entries are not parsed'); +}); + +test('include filter selects only matching entries', () => { + const zip = buildZip({ + 'conversations.json': Buffer.from('[1,2,3]'), + 'projects/p1.json': Buffer.from('{"uuid":"p1"}'), + 'users.json': Buffer.from('[{}]'), + }); + const { json } = readZipJsonEntries(zip, ['projects/']); + assert.deepStrictEqual(Object.keys(json), ['projects/p1.json']); +}); + +test('throws on a non-zip buffer', () => { + assert.throws(() => readZipJsonEntries(Buffer.from('not a zip'), null)); +}); + +const failed = cases.filter((c) => !c.ok); +for (const c of cases) console.log(`${c.ok ? 'ok ' : 'FAIL'} ${c.name}${c.ok ? '' : ' — ' + c.err}`); +console.log(`\n${cases.length - failed.length}/${cases.length} passed.`); +process.exit(failed.length === 0 ? 0 : 1); diff --git a/registry.json b/registry.json index 33d289e..480efc9 100644 --- a/registry.json +++ b/registry.json @@ -1,12 +1,12 @@ { "version": "2.0.0", "lastUpdated": "2026-07-13T00:00:00Z", - "baseUrl": "https://raw.githubusercontent.com/vana-com/data-connectors/main/connectors", + "baseUrl": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/connectors", "connectors": [ { "id": "chatgpt-playwright", "company": "openai", - "version": "3.0.0", + "version": "3.0.1", "name": "ChatGPT", "status": "stable", "description": "Exports your email, memories, and all conversations from ChatGPT.", @@ -25,8 +25,8 @@ "metadata": "openai/chatgpt-playwright.json" }, "checksums": { - "script": "sha256:f0bebc7055a0b49c87cece29bdcd078e447d88a575e643d02f30f5b6daa2e95b", - "metadata": "sha256:b4ced371d0a87c5d3d202ff0835b361caa0124732da179fb913688a6b29fc914" + "script": "sha256:f2f33ef300e8b89ecbc13fcfa80ee88b0ec0d7849f2018a7a16623b5e1146527", + "metadata": "sha256:6b3df1f5ce03e588c23f1e26e35494526166baee99e69c384b72b31f7b306371" } }, { @@ -225,7 +225,7 @@ { "id": "wholefoods-playwright", "company": "wholefoods", - "version": "1.0.0", + "version": "1.0.1", "name": "Whole Foods Market", "status": "experimental", "description": "Exports your Whole Foods Market order history (via Amazon) and product nutrition data.", @@ -245,7 +245,7 @@ }, "checksums": { "script": "sha256:2ebb4426258b87a8e3699b9609c36ff69b03f050e747ed530c54ee1e7f84c59b", - "metadata": "sha256:bbeb1748b6964a1b564788744eed1ddbbf5ea9c7d0a423639dfa5313c653fbe9" + "metadata": "sha256:b374a547bacfe49c0fcce7cea41db0630e8335c9f655e644a43a11b2eae9e0e1" } }, { @@ -338,7 +338,7 @@ { "id": "claude-export-playwright", "company": "Anthropic", - "version": "2.0.0", + "version": "2.0.1", "name": "Claude", "status": "experimental", "description": "Exports your Claude conversations and projects via Anthropic's official data export (one complete archive, no per-conversation rate limiting).", @@ -347,8 +347,8 @@ "metadata": "anthropic/claude-export-playwright.json" }, "checksums": { - "script": "sha256:1b8b4caf80b5e976822fa201b8232edd0d562858ce71f976dd1ebe449da909c1", - "metadata": "sha256:083af470ca0a590fff2b4929716c10332db230ab8ed7d83aa974cf5f183e2f53" + "script": "sha256:12a068a5186aececdd13e999b84f14a281d8e3d256c1eec70b812afd17142152", + "metadata": "sha256:7213ef42d1bcbde91db4972ae22a8f0b445f2ff47d85c95548b34ce20054c315" } }, { diff --git a/run-connector.cjs b/run-connector.cjs index f44adcf..9501f34 120000 --- a/run-connector.cjs +++ b/run-connector.cjs @@ -1 +1 @@ -skills/vana-connect/scripts/run-connector.cjs \ No newline at end of file +skills/pdp-connect/scripts/run-connector.cjs \ No newline at end of file diff --git a/schemas/connector-index.schema.json b/schemas/connector-index.schema.json index 75fc630..a802b2b 100644 --- a/schemas/connector-index.schema.json +++ b/schemas/connector-index.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/vana-com/data-connectors/schemas/connector-index.schema.json", + "$id": "https://github.com/PDP-Connect/data-connectors/schemas/connector-index.schema.json", "title": "Connector Index v2", "type": "object", "required": [ diff --git a/schemas/fixture-index.schema.json b/schemas/fixture-index.schema.json index f6fc2c3..c750a76 100644 --- a/schemas/fixture-index.schema.json +++ b/schemas/fixture-index.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/vana-com/data-connectors/schemas/fixture-index.schema.json", + "$id": "https://github.com/PDP-Connect/data-connectors/schemas/fixture-index.schema.json", "title": "Connector Fixture Index v1", "type": "object", "required": [ diff --git a/schemas/manifest.schema.json b/schemas/manifest.schema.json index c89bf24..617fd42 100644 --- a/schemas/manifest.schema.json +++ b/schemas/manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/vana-com/data-connectors/main/schemas/manifest.schema.json", + "$id": "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/schemas/manifest.schema.json", "title": "Connector Manifest", "description": "Canonical manifest contract for data-connectors. Every *-playwright.json file must validate against this schema.", "type": "object", diff --git a/schemas/scope-catalog.schema.json b/schemas/scope-catalog.schema.json index fbb282b..5a5c536 100644 --- a/schemas/scope-catalog.schema.json +++ b/schemas/scope-catalog.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:vana:data-connectors:scope-catalog:1.0.0", - "title": "Vana public source and scope catalog", + "$id": "urn:pdp-connect:data-connectors:scope-catalog:1.0.0", + "title": "PDP-Connect public source and scope catalog", "type": "object", "required": ["catalogSchema", "distribution", "catalogVersion", "generatedBy", "generatedFrom", "scopes"], "properties": { @@ -19,7 +19,7 @@ "type": "object", "required": ["repository"], "properties": { - "repository": { "const": "https://github.com/vana-com/data-connectors" }, + "repository": { "const": "https://github.com/PDP-Connect/data-connectors" }, "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, "releaseTag": { "type": "string", "pattern": "^connectors-[0-9a-f]{12}$" } }, @@ -32,7 +32,7 @@ "generatedBy": { "const": "scripts/generate-scope-catalog.mjs" }, "generatedFrom": { "type": "object", - "required": ["publishability", "manifests", "webCapabilities"], + "required": ["publishability", "manifests"], "properties": { "publishability": { "type": "object", @@ -47,8 +47,7 @@ "type": "array", "items": { "type": "string", "pattern": "^connectors/.+-playwright\\.json$" }, "uniqueItems": true - }, - "webCapabilities": { "const": "scopes/web-capabilities.json" } + } }, "additionalProperties": false }, @@ -83,20 +82,6 @@ }, "additionalProperties": false }, - "blocker": { - "type": "object", - "required": ["id", "description", "requiredCapture"], - "properties": { - "id": { "type": "string", "minLength": 1 }, - "description": { "type": "string", "minLength": 1 }, - "requiredCapture": { - "type": "array", - "minItems": 1, - "items": { "type": "string", "minLength": 1 } - } - }, - "additionalProperties": false - }, "scope": { "type": "object", "required": ["sourceId", "scopeId", "description", "schema", "maturity", "fulfillment"], @@ -109,14 +94,14 @@ "required": ["path"], "properties": { "path": { "type": "string", "pattern": "^connectors/.+/schemas/.+\\.json$" }, - "url": { "type": "string", "pattern": "^https://raw\\.githubusercontent\\.com/vana-com/data-connectors/[0-9a-f]{40}/connectors/.+/schemas/.+\\.json$" } + "url": { "type": "string", "pattern": "^https://raw\\.githubusercontent\\.com/PDP-Connect/data-connectors/[0-9a-f]{40}/connectors/.+/schemas/.+\\.json$" } }, "additionalProperties": false }, "maturity": { "type": "string", "enum": ["stable", "beta", "experimental"] }, "fulfillment": { "type": "object", - "required": ["desktop", "web"], + "required": ["desktop"], "properties": { "desktop": { "type": "object", @@ -131,39 +116,6 @@ } }, "additionalProperties": false - }, - "web": { - "type": "object", - "required": ["status"], - "properties": { - "status": { "type": "string", "enum": ["supported", "unsupported", "blocked"] }, - "limits": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/limit" } - }, - "blocker": { "$ref": "#/$defs/blocker" } - }, - "allOf": [ - { - "if": { "properties": { "status": { "const": "blocked" } }, "required": ["status"] }, - "then": { - "properties": { "blocker": { "$ref": "#/$defs/blocker" } }, - "required": ["blocker"] - }, - "else": { "not": { "required": ["blocker"] } } - }, - { - "if": { "properties": { "status": { "const": "supported" } }, "required": ["status"] }, - "else": { - "not": { - "properties": { "limits": { "type": "array" } }, - "required": ["limits"] - } - } - } - ], - "additionalProperties": false } }, "additionalProperties": false diff --git a/schemas/web-scope-capabilities.schema.json b/schemas/web-scope-capabilities.schema.json deleted file mode 100644 index f032bce..0000000 --- a/schemas/web-scope-capabilities.schema.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:vana:data-connectors:web-scope-capabilities:1.0.0", - "title": "Web scope capabilities input", - "type": "object", - "required": ["$schema", "schemaVersion", "contract", "provenance", "blockers", "scopes"], - "properties": { - "$schema": { "const": "../schemas/web-scope-capabilities.schema.json" }, - "schemaVersion": { "const": "1.0.0" }, - "contract": { "type": "string", "minLength": 1 }, - "provenance": { - "type": "object", - "required": ["issue", "baselineCommit", "note"], - "properties": { - "issue": { "type": "string", "minLength": 1 }, - "baselineCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, - "note": { "type": "string", "minLength": 1 } - }, - "additionalProperties": false - }, - "blockers": { - "type": "array", - "items": { "$ref": "#/$defs/blocker" } - }, - "scopes": { - "type": "array", - "items": { "$ref": "#/$defs/webScope" } - } - }, - "$defs": { - "limit": { - "type": "object", - "required": ["type", "value", "unit", "description"], - "properties": { - "type": { "type": "string", "enum": ["maxItems", "timeWindow"] }, - "value": { "type": "integer", "minimum": 1 }, - "unit": { "type": "string", "minLength": 1 }, - "description": { "type": "string", "minLength": 1 } - }, - "additionalProperties": false - }, - "blocker": { - "type": "object", - "required": ["id", "description", "requiredCapture"], - "properties": { - "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, - "description": { "type": "string", "minLength": 1 }, - "requiredCapture": { - "type": "array", - "minItems": 1, - "items": { "type": "string", "minLength": 1 } - } - }, - "additionalProperties": false - }, - "webScope": { - "type": "object", - "required": ["scopeId", "status"], - "properties": { - "scopeId": { "type": "string", "pattern": "^[a-z0-9_-]+\\.[A-Za-z0-9_.-]+$" }, - "status": { "type": "string", "enum": ["supported", "unsupported", "blocked"] }, - "limits": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/limit" } - }, - "blockerId": { "type": "string", "pattern": "^[a-z0-9-]+$" } - }, - "allOf": [ - { - "if": { "properties": { "status": { "const": "blocked" } }, "required": ["status"] }, - "then": { - "properties": { "blockerId": { "type": "string", "pattern": "^[a-z0-9-]+$" } }, - "required": ["blockerId"] - }, - "else": { "not": { "required": ["blockerId"] } } - }, - { - "if": { "properties": { "status": { "const": "supported" } }, "required": ["status"] }, - "else": { - "not": { - "properties": { "limits": { "type": "array" } }, - "required": ["limits"] - } - } - } - ], - "additionalProperties": false - } - }, - "additionalProperties": false -} diff --git a/scope-catalog.json b/scope-catalog.json index 49b5657..96e31c2 100644 --- a/scope-catalog.json +++ b/scope-catalog.json @@ -4,7 +4,7 @@ "releaseAsset": "scope-catalog.schema.json" }, "distribution": { - "repository": "https://github.com/vana-com/data-connectors" + "repository": "https://github.com/PDP-Connect/data-connectors" }, "catalogVersion": "1.0.0", "generatedBy": "scripts/generate-scope-catalog.mjs", @@ -31,8 +31,7 @@ "connectors/uber/uber-playwright.json", "connectors/valve/steam-playwright.json", "connectors/wholefoods/wholefoods-playwright.json" - ], - "webCapabilities": "scopes/web-capabilities.json" + ] }, "scopes": [ { @@ -52,9 +51,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -75,9 +71,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -98,9 +91,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -121,9 +111,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -144,9 +131,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -167,9 +151,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -190,9 +171,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -221,9 +199,6 @@ ] } ] - }, - "web": { - "status": "unsupported" } } }, @@ -258,9 +233,6 @@ ] } ] - }, - "web": { - "status": "unsupported" } } }, @@ -289,9 +261,6 @@ ] } ] - }, - "web": { - "status": "unsupported" } } }, @@ -312,9 +281,6 @@ "status": "stable" } ] - }, - "web": { - "status": "supported" } } }, @@ -335,9 +301,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -358,9 +321,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -381,9 +341,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -404,9 +361,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -427,9 +381,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -450,9 +401,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -473,9 +421,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -500,9 +445,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -523,9 +465,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -546,17 +485,6 @@ "status": "stable" } ] - }, - "web": { - "status": "supported", - "limits": [ - { - "type": "maxItems", - "value": 150, - "unit": "posts", - "description": "The Web path returns at most the 150 most recent posts." - } - ] } } }, @@ -577,9 +505,6 @@ "status": "stable" } ] - }, - "web": { - "status": "supported" } } }, @@ -600,17 +525,6 @@ "status": "stable" } ] - }, - "web": { - "status": "blocked", - "blocker": { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } } } }, @@ -631,17 +545,6 @@ "status": "stable" } ] - }, - "web": { - "status": "blocked", - "blocker": { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } } } }, @@ -662,17 +565,6 @@ "status": "stable" } ] - }, - "web": { - "status": "blocked", - "blocker": { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } } } }, @@ -693,17 +585,6 @@ "status": "stable" } ] - }, - "web": { - "status": "blocked", - "blocker": { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } } } }, @@ -724,17 +605,6 @@ "status": "stable" } ] - }, - "web": { - "status": "blocked", - "blocker": { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } } } }, @@ -755,17 +625,6 @@ "status": "stable" } ] - }, - "web": { - "status": "blocked", - "blocker": { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } } } }, @@ -786,9 +645,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -809,9 +665,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -832,9 +685,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -855,9 +705,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -878,9 +725,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -901,9 +745,6 @@ "status": "stable" } ] - }, - "web": { - "status": "supported" } } }, @@ -924,9 +765,6 @@ "status": "stable" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -947,9 +785,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -970,9 +805,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -993,9 +825,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1016,9 +845,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1039,9 +865,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1062,9 +885,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1085,9 +905,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1108,9 +925,6 @@ "status": "experimental" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1139,9 +953,6 @@ ] } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1162,9 +973,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1185,9 +993,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1208,9 +1013,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1231,9 +1033,6 @@ "status": "beta" } ] - }, - "web": { - "status": "supported" } } }, @@ -1254,9 +1053,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } }, @@ -1277,9 +1073,6 @@ "status": "beta" } ] - }, - "web": { - "status": "unsupported" } } } diff --git a/scopes/web-capabilities.json b/scopes/web-capabilities.json deleted file mode 100644 index 6d3e826..0000000 --- a/scopes/web-capabilities.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "$schema": "../schemas/web-scope-capabilities.schema.json", - "schemaVersion": "1.0.0", - "contract": "ODL Data Pipe API", - "provenance": { - "issue": "BUI-705", - "baselineCommit": "d0a537a7d2fc7606f8a8492a22d9c2cd6d9dc6e8", - "note": "Non-LinkedIn values preserve the Web capability assertions in SCOPES.md at the baseline commit. LinkedIn is blocked pending live Data Pipe and Personal Server evidence." - }, - "blockers": [ - { - "id": "linkedin-live-output", - "description": "LinkedIn Web scope capability is not classified without live end-to-end evidence.", - "requiredCapture": [ - "A captured successful LinkedIn Data Pipe result showing the exact returned scope-key set.", - "The resulting Personal Server scope inventory for the same run and user after sync." - ] - } - ], - "scopes": [ - { "scopeId": "amazon.orders", "status": "unsupported" }, - { "scopeId": "amazon.profile", "status": "unsupported" }, - { "scopeId": "chatgpt.conversations", "status": "unsupported" }, - { "scopeId": "chatgpt.memories", "status": "unsupported" }, - { "scopeId": "claude.conversations", "status": "unsupported" }, - { "scopeId": "claude.projects", "status": "unsupported" }, - { "scopeId": "doordash.orders", "status": "unsupported" }, - { "scopeId": "github.contributions", "status": "unsupported" }, - { "scopeId": "github.events", "status": "unsupported" }, - { "scopeId": "github.history", "status": "unsupported" }, - { "scopeId": "github.profile", "status": "supported" }, - { "scopeId": "github.repositories", "status": "unsupported" }, - { "scopeId": "github.starred", "status": "unsupported" }, - { "scopeId": "heb.nutrition", "status": "unsupported" }, - { "scopeId": "heb.orders", "status": "unsupported" }, - { "scopeId": "heb.profile", "status": "unsupported" }, - { "scopeId": "icloud_notes.folders", "status": "unsupported" }, - { "scopeId": "icloud_notes.notes", "status": "unsupported" }, - { "scopeId": "instagram.ads", "status": "unsupported" }, - { "scopeId": "instagram.following", "status": "unsupported" }, - { - "scopeId": "instagram.posts", - "status": "supported", - "limits": [ - { - "type": "maxItems", - "value": 150, - "unit": "posts", - "description": "The Web path returns at most the 150 most recent posts." - } - ] - }, - { "scopeId": "instagram.profile", "status": "supported" }, - { "scopeId": "linkedin.connections", "status": "blocked", "blockerId": "linkedin-live-output" }, - { "scopeId": "linkedin.education", "status": "blocked", "blockerId": "linkedin-live-output" }, - { "scopeId": "linkedin.experience", "status": "blocked", "blockerId": "linkedin-live-output" }, - { "scopeId": "linkedin.languages", "status": "blocked", "blockerId": "linkedin-live-output" }, - { "scopeId": "linkedin.profile", "status": "blocked", "blockerId": "linkedin-live-output" }, - { "scopeId": "linkedin.skills", "status": "blocked", "blockerId": "linkedin-live-output" }, - { "scopeId": "oura.activity", "status": "unsupported" }, - { "scopeId": "oura.readiness", "status": "unsupported" }, - { "scopeId": "oura.sleep", "status": "unsupported" }, - { "scopeId": "shop.orders", "status": "unsupported" }, - { "scopeId": "spotify.playlists", "status": "unsupported" }, - { "scopeId": "spotify.profile", "status": "supported" }, - { "scopeId": "spotify.savedTracks", "status": "unsupported" }, - { "scopeId": "steam.friends", "status": "unsupported" }, - { "scopeId": "steam.games", "status": "unsupported" }, - { "scopeId": "steam.profile", "status": "unsupported" }, - { "scopeId": "uber.receipts", "status": "unsupported" }, - { "scopeId": "uber.trips", "status": "unsupported" }, - { "scopeId": "wholefoods.nutrition", "status": "unsupported" }, - { "scopeId": "wholefoods.orders", "status": "unsupported" }, - { "scopeId": "wholefoods.profile", "status": "unsupported" }, - { "scopeId": "youtube.history", "status": "unsupported" }, - { "scopeId": "youtube.likes", "status": "unsupported" }, - { "scopeId": "youtube.playlistItems", "status": "unsupported" }, - { "scopeId": "youtube.playlists", "status": "unsupported" }, - { "scopeId": "youtube.profile", "status": "supported" }, - { "scopeId": "youtube.subscriptions", "status": "unsupported" }, - { "scopeId": "youtube.watchLater", "status": "unsupported" } - ] -} diff --git a/scripts/.claude/skills/auto-create-connector/SKILL.md b/scripts/.claude/skills/auto-create-connector/SKILL.md index 48e810a..057a98f 100644 --- a/scripts/.claude/skills/auto-create-connector/SKILL.md +++ b/scripts/.claude/skills/auto-create-connector/SKILL.md @@ -49,7 +49,7 @@ node capture-session.cjs ``` This opens a headed browser. The user logs in manually (handles CAPTCHAs, 2FA, OAuth). -The session persists in `~/.vana/desktop/browser-profiles/-playwright/` and is +The session persists in `~/.pdp-connect/desktop/browser-profiles/-playwright/` and is automatically reused by the connector during testing. ## Workflow @@ -117,7 +117,7 @@ If web search doesn't reveal clear APIs, use Chrome browser automation tools to: 2. Check if the browser profile exists: ```bash - ls ~/.vana/desktop/browser-profiles/-playwright/ 2>/dev/null && echo "Profile exists" || echo "No profile" + ls ~/.pdp-connect/desktop/browser-profiles/-playwright/ 2>/dev/null && echo "Profile exists" || echo "No profile" ``` 3. If no session/profile, tell the user and run capture: @@ -266,7 +266,7 @@ Re-run the validator after each fix until the report shows `"valid": true`. **Check if session or credentials are available:** ```bash # Check for captured session (from Step 1.5) -ls ~/.vana/desktop/browser-profiles/-playwright/ 2>/dev/null && echo "Session profile exists" || echo "No session profile" +ls ~/.pdp-connect/desktop/browser-profiles/-playwright/ 2>/dev/null && echo "Session profile exists" || echo "No session profile" # Check for .env credentials grep -q "USER_LOGIN_" .env 2>/dev/null && echo "Credentials found" || echo "No credentials" diff --git a/scripts/CLAUDE.md b/scripts/CLAUDE.md index 645118b..5982a94 100644 --- a/scripts/CLAUDE.md +++ b/scripts/CLAUDE.md @@ -67,7 +67,7 @@ For platforms where automated login fails (CAPTCHAs, Cloudflare, 2FA, OAuth): 1. Run `node capture-session.cjs ` — opens a headed browser 2. Log in manually in the browser window (handles any auth complexity) 3. Login is detected automatically via URL change + cookie increase -4. Session persists in `~/.vana/desktop/browser-profiles/-playwright/` +4. Session persists in `~/.pdp-connect/desktop/browser-profiles/-playwright/` 5. Subsequent connector tests automatically reuse the session (shared profile) The browser profile is shared by filename: both `sessions/-playwright.js` (capture) diff --git a/scripts/capture-session.cjs b/scripts/capture-session.cjs index df04686..15255c9 100755 --- a/scripts/capture-session.cjs +++ b/scripts/capture-session.cjs @@ -55,6 +55,8 @@ function resolveRunnerDir() { } const candidates = [ + // Bundled runner in this repo + path.resolve(__dirname, '..', 'playwright-runner'), path.resolve(__dirname, '..', 'data-dt-app', 'playwright-runner'), path.join(os.homedir(), 'Documents', 'GitHub', 'data-dt-app', 'playwright-runner'), path.join(os.homedir(), 'Documents', 'Github', 'data-dt-app', 'playwright-runner'), @@ -126,7 +128,7 @@ ${c.cyan}How it works:${c.reset} 4. Session is persisted in browser profile for connector reuse ${c.cyan}Profile sharing:${c.reset} - The browser profile is stored at ~/.vana/desktop/browser-profiles/-playwright/ + The browser profile is stored at ~/.pdp-connect/desktop/browser-profiles/-playwright/ Any connector named -playwright.js will automatically reuse this session. `); } @@ -266,7 +268,7 @@ function generateLoginConnector(platform, loginUrl, timeout, checkUrl) { lines.push(' sessionStorageKeys: Object.keys(sessionStorageData || {}).length'); lines.push(' },'); lines.push(' note: \'Full session (including HttpOnly cookies) is persisted in the browser profile. This file contains browser-accessible data only.\','); - lines.push(' profileDir: \'~/.vana/desktop/browser-profiles/' + platform + '-playwright/\''); + lines.push(' profileDir: \'~/.pdp-connect/desktop/browser-profiles/' + platform + '-playwright/\''); lines.push(' };'); lines.push(''); lines.push(' await page.setData(\'result\', result);'); @@ -318,7 +320,7 @@ async function main() { console.log(` ${c.cyan}Platform:${c.reset} ${args.platform}`); console.log(` ${c.cyan}Login URL:${c.reset} ${args.loginUrl}`); console.log(` ${c.cyan}Timeout:${c.reset} ${args.timeout}s`); - console.log(` ${c.cyan}Profile:${c.reset} ~/.vana/desktop/browser-profiles/${connectorName}/`); + console.log(` ${c.cyan}Profile:${c.reset} ~/.pdp-connect/desktop/browser-profiles/${connectorName}/`); console.log('─'.repeat(50)); console.log(''); console.log(`${c.yellow}A browser window will open. Please log in manually.${c.reset}`); @@ -411,7 +413,7 @@ async function main() { const sessionFile = path.join(sessionsDir, `${args.platform}.json`); fs.writeFileSync(sessionFile, JSON.stringify(resultRef.data, null, 2)); print(c.green, '[saved]', `Session data: ${sessionFile}`); - print(c.green, '[saved]', `Browser profile: ~/.vana/desktop/browser-profiles/${connectorName}/`); + print(c.green, '[saved]', `Browser profile: ~/.pdp-connect/desktop/browser-profiles/${connectorName}/`); const stats = resultRef.data.stats || {}; console.log(''); diff --git a/scripts/connector-artifact-contract.test.mjs b/scripts/connector-artifact-contract.test.mjs index 94e7f05..ce131e0 100644 --- a/scripts/connector-artifact-contract.test.mjs +++ b/scripts/connector-artifact-contract.test.mjs @@ -61,7 +61,7 @@ test("a fully signed index passes the publish signature gate", () => { type: "sigstoreBundle", bundlePath: "alpha-playwright-1.0.0.tgz.sigstore.json", bundleUrl: - "https://github.com/vana-com/data-connectors/releases/download/connectors-test/alpha-playwright-1.0.0.tgz.sigstore.json", + "https://github.com/PDP-Connect/data-connectors/releases/download/connectors-test/alpha-playwright-1.0.0.tgz.sigstore.json", }, }, { diff --git a/scripts/create-connector.sh b/scripts/create-connector.sh index 7ea0686..1e03a84 100755 --- a/scripts/create-connector.sh +++ b/scripts/create-connector.sh @@ -115,7 +115,7 @@ if [ -n "${!LOGIN_VAR}" ] && [ -n "${!PASSWORD_VAR}" ]; then HAS_CREDENTIALS=true fi -PROFILE_DIR="$HOME/.vana/desktop/browser-profiles/${PLATFORM}-playwright" +PROFILE_DIR="$HOME/.pdp-connect/desktop/browser-profiles/${PLATFORM}-playwright" if [ -d "$PROFILE_DIR" ]; then HAS_SESSION=true fi diff --git a/scripts/generate-connector-index.mjs b/scripts/generate-connector-index.mjs index 015e069..43e852e 100644 --- a/scripts/generate-connector-index.mjs +++ b/scripts/generate-connector-index.mjs @@ -65,7 +65,7 @@ function resolveReleaseMetadata(sourceCommit) { `connectors-${sourceCommit.slice(0, 12)}`; const releaseId = process.env.CONNECTOR_RELEASE_ID?.trim() || releaseTag; const repo = - process.env.GITHUB_REPOSITORY?.trim() || "vana-com/data-connectors"; + process.env.GITHUB_REPOSITORY?.trim() || "PDP-Connect/data-connectors"; return { releaseTag, @@ -455,7 +455,7 @@ async function main() { const releaseMetadata = resolveReleaseMetadata(sourceCommit); const nextIndex = { indexVersion: "2.0", - sourceRepo: "https://github.com/vana-com/data-connectors", + sourceRepo: "https://github.com/PDP-Connect/data-connectors", generatedAt: registry.lastUpdated ?? new Date().toISOString(), signature: buildSigstoreBundleMetadata( "connector-index.json.sigstore.json", diff --git a/scripts/generate-fixture-index.mjs b/scripts/generate-fixture-index.mjs index 1f89c15..e5b495d 100644 --- a/scripts/generate-fixture-index.mjs +++ b/scripts/generate-fixture-index.mjs @@ -18,7 +18,7 @@ const indexPath = join(repoRoot, "fixture-index.json"); const registryPath = join(repoRoot, "registry.json"); const rawBaseUrl = ( process.env.FIXTURE_RAW_BASE_URL || - "https://raw.githubusercontent.com/vana-com/data-connectors/main" + "https://raw.githubusercontent.com/PDP-Connect/data-connectors/main" ).replace(/\/$/, ""); const CONNECTOR_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const SCOPE_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/; @@ -491,7 +491,7 @@ function buildFixtureIndex() { return { $schema: "./schemas/fixture-index.schema.json", indexVersion: "1.0", - sourceRepo: "https://github.com/vana-com/data-connectors", + sourceRepo: "https://github.com/PDP-Connect/data-connectors", generatedAt: generatedAt(), fixtures, }; diff --git a/scripts/generate-scope-catalog.mjs b/scripts/generate-scope-catalog.mjs index b167dbc..ebd00d0 100644 --- a/scripts/generate-scope-catalog.mjs +++ b/scripts/generate-scope-catalog.mjs @@ -8,8 +8,8 @@ import Ajv2020 from "ajv/dist/2020.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const defaultRepoRoot = join(scriptDir, ".."); -const repositoryUrl = "https://github.com/vana-com/data-connectors"; -const rawRepositoryUrl = "https://raw.githubusercontent.com/vana-com/data-connectors"; +const repositoryUrl = "https://github.com/PDP-Connect/data-connectors"; +const rawRepositoryUrl = "https://raw.githubusercontent.com/PDP-Connect/data-connectors"; const readmeOverviewStart = ""; const readmeOverviewEnd = ""; const maturityRank = new Map([ @@ -147,25 +147,6 @@ function loadPublishedScopes(repoRoot) { }; } -function validateExactWebSet(publishedScopes, webInput) { - const webIds = webInput.scopes.map(({ scopeId }) => scopeId); - const duplicateIds = webIds.filter((scopeId, index) => webIds.indexOf(scopeId) !== index); - if (duplicateIds.length > 0) { - throw new Error(`Duplicate Web capability entries: ${[...new Set(duplicateIds)].sort().join(", ")}`); - } - - const publishedIds = new Set(publishedScopes.keys()); - const webIdSet = new Set(webIds); - const missing = [...publishedIds].filter((scopeId) => !webIdSet.has(scopeId)).sort(); - const extra = [...webIdSet].filter((scopeId) => !publishedIds.has(scopeId)).sort(); - if (missing.length > 0) { - throw new Error(`Missing Web capability entries: ${missing.join(", ")}`); - } - if (extra.length > 0) { - throw new Error(`Extra Web capability entries: ${extra.join(", ")}`); - } -} - function buildCatalog(repoRoot, { sourceCommit, releaseTag } = {}) { if ((sourceCommit && !releaseTag) || (!sourceCommit && releaseTag)) { throw new Error("sourceCommit and releaseTag must be provided together"); @@ -176,24 +157,7 @@ function buildCatalog(repoRoot, { sourceCommit, releaseTag } = {}) { if (sourceCommit && releaseTag !== `connectors-${sourceCommit.slice(0, 12)}`) { throw new Error("releaseTag must match connectors-"); } - const webInputPath = join(repoRoot, "scopes", "web-capabilities.json"); - const webInput = readJson(webInputPath); - const webInputSchema = readJson( - join(repoRoot, "schemas", "web-scope-capabilities.schema.json"), - ); - validateAgainstSchema(webInput, webInputSchema, "web-capabilities.json"); - const { groupedScopes: publishedScopes, manifestPaths } = loadPublishedScopes(repoRoot); - validateExactWebSet(publishedScopes, webInput); - - const blockerById = new Map(); - for (const blocker of webInput.blockers) { - if (blockerById.has(blocker.id)) { - throw new Error(`Duplicate Web blocker id: ${blocker.id}`); - } - blockerById.set(blocker.id, blocker); - } - const webByScope = new Map(webInput.scopes.map((entry) => [entry.scopeId, entry])); const scopes = [...publishedScopes.entries()] .sort(([left], [right]) => left.localeCompare(right)) @@ -218,21 +182,6 @@ function buildCatalog(repoRoot, { sourceCommit, releaseTag } = {}) { ); } - const webInputEntry = webByScope.get(scopeId); - const web = { status: webInputEntry.status }; - if (webInputEntry.limits) { - web.limits = webInputEntry.limits; - } - if (webInputEntry.blockerId) { - const blocker = blockerById.get(webInputEntry.blockerId); - if (!blocker) { - throw new Error( - `Web capability ${scopeId} references unknown blocker ${webInputEntry.blockerId}`, - ); - } - web.blocker = blocker; - } - return { sourceId: primary.sourceId, scopeId, @@ -249,7 +198,6 @@ function buildCatalog(repoRoot, { sourceCommit, releaseTag } = {}) { status: "supported", connectors: candidates.map(({ connector }) => connector), }, - web, }, }; }); @@ -271,7 +219,6 @@ function buildCatalog(repoRoot, { sourceCommit, releaseTag } = {}) { manifestSelector: "connectors[].files.metadata", }, manifests: manifestPaths, - webCapabilities: "scopes/web-capabilities.json", }, scopes, }; @@ -280,20 +227,11 @@ function buildCatalog(repoRoot, { sourceCommit, releaseTag } = {}) { return catalog; } -function renderWebStatus(web) { - if (web.status === "supported") return "Yes"; - if (web.status === "blocked") return `Blocked (${web.blocker.id})`; - return "No"; -} - function renderLimits(scope) { const limits = []; - for (const limit of scope.fulfillment.web.limits ?? []) { - limits.push(`Vana Web (hosted): ${limit.description}`); - } for (const connector of scope.fulfillment.desktop.connectors) { for (const limit of connector.limits ?? []) { - limits.push(`Vana Desktop (local; ${connector.id}): ${limit.description}`); + limits.push(`Desktop (local; ${connector.id}): ${limit.description}`); } } return limits.join("; ") || "None"; @@ -351,7 +289,7 @@ function renderScopesMarkdown(catalog) { const connectors = scope.fulfillment.desktop.connectors .map(({ id, status }) => `${id} (${status})`) .join("; "); - return `| ${escapeCell(scope.sourceId)} | \`${scope.scopeId}\` | ${escapeCell(scope.description)} | [JSON Schema](${scope.schema.path}) | ${renderWebStatus(scope.fulfillment.web)} | Yes | ${escapeCell(renderLimits(scope))} | ${escapeCell(connectors)} |`; + return `| ${escapeCell(scope.sourceId)} | \`${scope.scopeId}\` | ${escapeCell(scope.description)} | [JSON Schema](${scope.schema.path}) | ${escapeCell(renderLimits(scope))} | ${escapeCell(connectors)} |`; }); return ` @@ -359,12 +297,7 @@ function renderScopesMarkdown(catalog) { The public human-readable view of [\`scope-catalog.json\`](scope-catalog.json). -This catalog shows two collection paths. When both paths support a scope, they use the same scope ID and payload schema: - -- **Vana Desktop (local):** A registered Playwright connector runs on your device and uses your browser session. Connector manifests plus \`registry.json\` own this column. -- **Vana Web (hosted):** Vana Web requests data from Open Data Labs' hosted collection service, called Data Pipe. [\`scopes/web-capabilities.json\`](scopes/web-capabilities.json) owns this column and its limits. - -A website cannot run the local Node.js/Playwright connector or read your signed-in session on another site. Vana Web could ask an installed Vana Desktop app to collect the data, but that is still Desktop collection. The table uses **Yes**, **No**, or **Blocked** for each path. **Blocked** names the evidence required before making a claim. +This catalog describes the local connector collection path: a registered Playwright connector runs on your device and uses your browser session. Connector manifests plus \`registry.json\` own this data. ## Versioned artifact contract @@ -372,20 +305,13 @@ Every immutable \`connectors-\` GitHub release includes \`scope-catalo ## Coverage -| Source | Scope | Description | Schema | Vana Web (hosted) | Vana Desktop (local) | Material limits | Desktop connector(s) / maturity | -|---|---|---|---|:--:|:--:|---|---| +| Source | Scope | Description | Schema | Material limits | Desktop connector(s) / maturity | +|---|---|---|---|---|---| ${rows.join("\n")} -## Blocked evidence - -${catalog.scopes - .filter(({ fulfillment }) => fulfillment.web.status === "blocked") - .map(({ scopeId, fulfillment }) => `- \`${scopeId}\`: ${fulfillment.web.blocker.description} Required: ${fulfillment.web.blocker.requiredCapture.join(" ")}`) - .join("\n") || "None."} - ## Maintenance -Do not edit this file. Update connector manifests/\`registry.json\` for Vana Desktop truth or \`scopes/web-capabilities.json\` for Vana Web truth, then run \`node scripts/generate-scope-catalog.mjs\`. +Do not edit this file. Update connector manifests/\`registry.json\`, then run \`node scripts/generate-scope-catalog.mjs\`. `; } diff --git a/scripts/generate-scope-catalog.test.mjs b/scripts/generate-scope-catalog.test.mjs index 61a7e57..f4d8082 100644 --- a/scripts/generate-scope-catalog.test.mjs +++ b/scripts/generate-scope-catalog.test.mjs @@ -19,10 +19,6 @@ function writeJson(path, value) { function makeFixture() { const root = mkdtempSync(join(tmpdir(), "scope-catalog-test-")); mkdirSync(join(root, "schemas"), { recursive: true }); - cpSync( - join(repoRoot, "schemas", "web-scope-capabilities.schema.json"), - join(root, "schemas", "web-scope-capabilities.schema.json"), - ); cpSync( join(repoRoot, "schemas", "scope-catalog.schema.json"), join(root, "schemas", "scope-catalog.schema.json"), @@ -68,18 +64,6 @@ function makeFixture() { source_id: "ignored", scopes: [{ scope: "ignored.profile", description: "Not published." }], }); - writeJson(join(root, "scopes", "web-capabilities.json"), { - $schema: "../schemas/web-scope-capabilities.schema.json", - schemaVersion: "1.0.0", - contract: "Test Web API", - provenance: { - issue: "TEST-1", - baselineCommit: "a".repeat(40), - note: "Test evidence.", - }, - blockers: [], - scopes: [{ scopeId: "alpha.profile", status: "unsupported" }], - }); writeFileSync( join(root, "README.md"), `# Fixture @@ -116,12 +100,9 @@ test("generated files are clean and exclude unregistered manifests", () => { manifestSelector: "connectors[].files.metadata", }, manifests: ["connectors/alpha/alpha-playwright.json"], - webCapabilities: "scopes/web-capabilities.json", }); - assert.match(markdown, /Vana Web \(hosted\)/); - assert.match(markdown, /Vana Desktop \(local\)/); - assert.match(markdown, /A website cannot run the local Node\.js\/Playwright connector/); - assert.match(markdown, /\| No \| Yes \|/); + assert.match(markdown, /local connector collection path/); + assert.doesNotMatch(markdown, /Vana/); assert.doesNotMatch(markdown, /✅|—/); assert.match( readme, @@ -137,65 +118,6 @@ test("generated files are clean and exclude unregistered manifests", () => { ); }); -test("missing Web capability entries fail exact-set validation", () => { - const root = makeFixture(); - const inputPath = join(root, "scopes", "web-capabilities.json"); - const input = JSON.parse(readFileSync(inputPath)); - input.scopes = []; - writeJson(inputPath, input); - - assert.throws( - () => generateScopeCatalog({ repoRoot: root }), - /Missing Web capability entries: alpha\.profile/, - ); -}); - -test("extra Web capability entries fail exact-set validation", () => { - const root = makeFixture(); - const inputPath = join(root, "scopes", "web-capabilities.json"); - const input = JSON.parse(readFileSync(inputPath)); - input.scopes.push({ scopeId: "extra.profile", status: "unsupported" }); - writeJson(inputPath, input); - - assert.throws( - () => generateScopeCatalog({ repoRoot: root }), - /Extra Web capability entries: extra\.profile/, - ); -}); - -test("malformed Web capability entries fail schema validation", () => { - const root = makeFixture(); - const inputPath = join(root, "scopes", "web-capabilities.json"); - const input = JSON.parse(readFileSync(inputPath)); - input.scopes[0].status = "maybe"; - writeJson(inputPath, input); - - assert.throws( - () => generateScopeCatalog({ repoRoot: root }), - /web-capabilities\.json failed schema validation/, - ); -}); - -test("unsupported Web capability entries cannot declare limits", () => { - const root = makeFixture(); - const inputPath = join(root, "scopes", "web-capabilities.json"); - const input = JSON.parse(readFileSync(inputPath)); - input.scopes[0].limits = [ - { - type: "maxItems", - value: 1, - unit: "item", - description: "Contradictory unsupported limit.", - }, - ]; - writeJson(inputPath, input); - - assert.throws( - () => generateScopeCatalog({ repoRoot: root }), - /web-capabilities\.json failed schema validation/, - ); -}); - test("conflicting registered schema evidence fails", () => { const root = makeFixture(); const registryPath = join(root, "registry.json"); @@ -275,13 +197,13 @@ test("release generation embeds immutable payload-schema URLs", () => { const catalog = JSON.parse(readFileSync(join(root, "scope-catalog.json"))); assert.deepEqual(catalog.distribution, { - repository: "https://github.com/vana-com/data-connectors", + repository: "https://github.com/PDP-Connect/data-connectors", sourceCommit, releaseTag, }); assert.equal( catalog.scopes[0].schema.url, - `https://raw.githubusercontent.com/vana-com/data-connectors/${sourceCommit}/connectors/alpha/schemas/alpha.profile.json`, + `https://raw.githubusercontent.com/PDP-Connect/data-connectors/${sourceCommit}/connectors/alpha/schemas/alpha.profile.json`, ); }); diff --git a/scripts/test-connector.cjs b/scripts/test-connector.cjs index be17362..243aaf3 100755 --- a/scripts/test-connector.cjs +++ b/scripts/test-connector.cjs @@ -55,6 +55,8 @@ function resolveRunnerDir() { // 2. Common locations relative to this script const candidates = [ + // Bundled runner in this repo + path.resolve(__dirname, '..', 'playwright-runner'), // Sibling repo (same parent directory) path.resolve(__dirname, '..', 'data-dt-app', 'playwright-runner'), // Home directory common paths @@ -250,9 +252,17 @@ async function main() { const runnerDir = resolveRunnerDir(); const metadata = loadMetadata(connectorPath); - const connectUrl = args.url || (metadata && metadata.connectURL) || 'about:blank'; + const connectUrl = args.url || (metadata && (metadata.connectURL || metadata.connect_url)) || 'about:blank'; const connectorName = metadata?.name || path.basename(connectorPath, '.js'); const connectorVersion = metadata?.version || 'unknown'; + const requestedScopes = Array.isArray(metadata?.scopes) + ? metadata.scopes.map((s) => s.scope).filter((s) => typeof s === 'string' && s.length > 0) + : []; + + if (requestedScopes.length === 0) { + console.error(`${c.red}Could not resolve requestedScopes for ${connectorPath} — no sibling metadata JSON with a non-empty "scopes" array found.${c.reset}`); + process.exit(1); + } console.log(''); console.log(`${c.bold}Connector Test Runner${c.reset}`); @@ -290,6 +300,7 @@ async function main() { url: connectUrl, headless: args.headless, forceHeaded: !args.headless, + requestedScopes, }); child.stdin.write(runCmd + '\n'); print(c.green, '[runner]', 'Connected, starting connector...'); diff --git a/skills/vana-connect/CREATE.md b/skills/pdp-connect/CREATE.md similarity index 93% rename from skills/vana-connect/CREATE.md rename to skills/pdp-connect/CREATE.md index f1ceb74..b099aa3 100644 --- a/skills/vana-connect/CREATE.md +++ b/skills/pdp-connect/CREATE.md @@ -7,7 +7,7 @@ Build a data connector for a platform that isn't in the registry yet. - `reference/PAGE-API.md` -- full `page` object API - `reference/PATTERNS.md` -- data extraction approaches and code examples -All `node scripts/...` commands refer to `skills/vana-connect/scripts/` in the data-connectors repo. `run-connector.cjs` is at `~/.vana/desktop/run-connector.cjs` (installed by SETUP.md). +All `node scripts/...` commands refer to `skills/pdp-connect/scripts/` in the data-connectors repo. `run-connector.cjs` is the bundled runner at the repo root (a symlink to `skills/pdp-connect/scripts/run-connector.cjs`); see `SETUP.md` for the one-time install. ## Connector Format @@ -31,7 +31,7 @@ Scripts are plain JavaScript (CJS), no imports, no require. The runner injects a | GitHub | DOM extraction | 3 | Server-rendered, no client API | | Spotify | In-page fetch | 1 | Well-documented public API | -Look at existing connectors in `~/.vana/desktop/connectors/` for working examples. +Look at existing connectors in `~/.pdp-connect/desktop/connectors/` for working examples. --- @@ -167,8 +167,8 @@ Run the connector and validate in one step: ```bash node scripts/validate.cjs connectors//-playwright.js && \ - node ~/.vana/desktop/run-connector.cjs connectors//-playwright.js [start-url] && \ - node scripts/validate.cjs connectors//-playwright.js --check-result ~/.vana/desktop/last-result.json + node run-connector.cjs connectors//-playwright.js [start-url] && \ + node scripts/validate.cjs connectors//-playwright.js --check-result ~/.pdp-connect/desktop/last-result.json ``` The validator checks structure, output quality, debug code, data cleanliness, schema descriptions, and login method diversity. Fix all reported issues and re-run. @@ -184,7 +184,7 @@ Schemas are an API contract — app developers build against them. ### Generate the skeleton ```bash -node scripts/generate-schemas.cjs ~/.vana/desktop/last-result.json [output-dir] +node scripts/generate-schemas.cjs ~/.pdp-connect/desktop/last-result.json [output-dir] ``` ### Enrich from what you know diff --git a/skills/vana-connect/RECIPES.md b/skills/pdp-connect/RECIPES.md similarity index 91% rename from skills/vana-connect/RECIPES.md rename to skills/pdp-connect/RECIPES.md index 55ff8a8..405e306 100644 --- a/skills/vana-connect/RECIPES.md +++ b/skills/pdp-connect/RECIPES.md @@ -1,6 +1,6 @@ # Recipes -What to do with collected data. Each recipe starts from `~/.vana/desktop/last-result.json`. +What to do with collected data. Each recipe starts from `~/.pdp-connect/desktop/last-result.json`. --- @@ -56,7 +56,7 @@ Show the profile before saving. Ask what to change. Where to save: - **Claude Code:** User memory or CLAUDE.md - **OpenClaw/Kimi:** `USER.md` in the agent's workspace -- **Generic:** `~/.vana/desktop/user-profile.md` +- **Generic:** `~/.pdp-connect/desktop/user-profile.md` When adding a new platform to an existing profile, merge. Don't overwrite. @@ -109,7 +109,7 @@ Export personal data to portable formats. The result file is already JSON. Copy it: ```bash -cp ~/.vana/desktop/last-result.json ~/backups/github-export-2026-03-10.json +cp ~/.pdp-connect/desktop/last-result.json ~/backups/github-export-2026-03-10.json ``` ### CSV (for tabular data) @@ -131,7 +131,7 @@ Run the connector on a schedule (cron, agent heartbeat, etc.) and timestamp each ```bash node run-connector.cjs -cp ~/.vana/desktop/last-result.json ~/backups/-$(date +%Y-%m-%d).json +cp ~/.pdp-connect/desktop/last-result.json ~/backups/-$(date +%Y-%m-%d).json ``` --- @@ -143,7 +143,7 @@ Combine data from multiple platforms. ### Approach 1. Connect platforms one at a time. Each run produces a separate result file. -2. Before each run, copy the previous result: `cp ~/.vana/desktop/last-result.json ~/.vana/desktop/-result.json` +2. Before each run, copy the previous result: `cp ~/.pdp-connect/desktop/last-result.json ~/.pdp-connect/desktop/-result.json` 3. After all platforms are connected, read all result files and synthesize. ### What cross-referencing reveals diff --git a/skills/pdp-connect/SETUP.md b/skills/pdp-connect/SETUP.md new file mode 100644 index 0000000..fc0c57c --- /dev/null +++ b/skills/pdp-connect/SETUP.md @@ -0,0 +1,40 @@ +# Connect -- Setup + +This repo bundles its own Playwright runner (`playwright-runner/`), so connectors run standalone -- no other repo required. + +## Prerequisites + +- Node.js +- `git` and (optionally) the `gh` CLI for contributing new connectors + +## Install the runtime + +```bash +bash skills/pdp-connect/scripts/setup.sh +``` + +This installs the bundled `playwright-runner/`'s dependencies (just Playwright) and downloads a Chromium build. This is a one-time step. Equivalent to running `npm install` inside `playwright-runner/` yourself. + +## Verify + +```bash +ls playwright-runner/index.cjs +ls playwright-runner/node_modules/playwright/package.json +``` + +If either is missing, re-run `setup.sh` and check its output for the step that failed. + +## File Locations + +| Path | Purpose | +|------|---------| +| `run-connector.cjs` (repo root, symlinked to `skills/pdp-connect/scripts/run-connector.cjs`) | Runs a connector headlessly | +| `playwright-runner/` (repo root) | Bundled runner the above resolves by default | +| `~/.pdp-connect/desktop/connectors/` | Connector scripts (when using the desktop app layout) | +| `~/.pdp-connect/desktop/browser-profiles/` | Persistent sessions (cookies) | +| `~/.pdp-connect/desktop/last-result.json` | Most recent result | +| `~/.pdp-connect/desktop/logs/` | Setup, fetch, and run logs | + +## Optional: override the runner location + +If you want to point at a different runner build (e.g. while developing the runner itself), pass `--runner-dir ` to `run-connector.cjs` or set `PLAYWRIGHT_RUNNER_DIR`. Neither is required for normal use. diff --git a/skills/vana-connect/SKILL.md b/skills/pdp-connect/SKILL.md similarity index 50% rename from skills/vana-connect/SKILL.md rename to skills/pdp-connect/SKILL.md index 83efcec..2c16886 100644 --- a/skills/vana-connect/SKILL.md +++ b/skills/pdp-connect/SKILL.md @@ -1,5 +1,5 @@ --- -name: vana-connect +name: pdp-connect description: > Connect personal data from any web platform using browser automation. Use when: (1) user wants to connect a data source like ChatGPT, Instagram, @@ -10,87 +10,31 @@ description: > # Connect -Connect personal data from web platforms using the `vana` CLI and local browser automation. +Connect personal data from web platforms using the in-repo scripts under `skills/pdp-connect/scripts/` and local browser automation. ## Setup -Prefer an installed `vana` CLI on `PATH`: +Check whether the bundled runner's dependencies are already installed: ```bash -command -v vana +ls playwright-runner/node_modules/playwright/package.json ``` -If that succeeds, use: +If it's missing, follow `SETUP.md` in this folder before continuing. Tell the user: "I need to do a one-time setup first. This installs a browser engine and some dependencies and usually takes about a minute." -```bash -vana -``` - -If `vana` is unavailable, install the current published canary. Prefer: - -macOS with Homebrew: - -```bash -brew tap vana-com/vana -brew install vana -``` - -macOS and Linux: - -```bash -curl -fsSL https://raw.githubusercontent.com/vana-com/vana-connect/feat/connect-cli-v1/install/install.sh | sh -s -- --version canary-feat-connect-cli-v1 -``` - -Only if the installed CLI path is unavailable or blocked, fall back to: - -```bash -npx -y @opendatalabs/connect@canary -``` - -If the user is explicitly testing local changes, fall back to: - -```bash -node /home/tnunamak/code/vana-connect/dist/cli/bin.js -``` - -If neither path is available, follow `SETUP.md` in this folder. - -Before connecting a source, check runtime state with: - -```bash -vana status --json -``` - -If the user needs install, path, or upgrade diagnostics, use: - -```bash -vana doctor -``` - -If the user needs recent setup, fetch, or run logs, use: - -```bash -vana logs -vana logs -``` - -If the runtime is missing, tell the user: "I need to do a one-time setup first. This downloads a browser engine and some dependencies into `~/.vana/desktop/` and usually takes about a minute." Then run: - -```bash -vana setup --yes -``` +If setup, fetch, or run output is truncated, check `~/.pdp-connect/desktop/logs/` directly. ## Flow ### 1. Explore available sources ```bash -vana sources --json +node skills/pdp-connect/scripts/fetch-connector.cjs ``` -This is the source of truth for what the CLI can currently connect. Prefer it over inspecting repo files manually. +or inspect `registry.json` / the `connectors/` directory directly for the source of truth on what can currently be connected. -If the requested platform is present, use the CLI flow below. +If the requested platform is present, use the run flow below. **If no connector exists for the platform,** tell the user you'll build one — this involves researching the platform's data APIs, writing the extraction code, and testing it. Let them know it'll take a bit and they're welcome to do something else while you work. Then read `CREATE.md` and follow it. @@ -98,7 +42,7 @@ If the user is building or testing an app and needs sample data, prefer the public fixture flow over pasting large JSON into the agent: ```bash -curl -fsSL https://raw.githubusercontent.com/vana-com/data-connectors/main/fixture-index.json +curl -fsSL https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/fixture-index.json ``` Find the matching `sourceId`, `scope`, and `scenario`, then download the entry's @@ -107,72 +51,41 @@ payloads that conform to the connector schema. The raw URL points at latest `main`; compare the downloaded file against the entry's `sha256` if exact bytes matter. -### 2. Connect with the CLI +### 2. Run the connector -Start with the agent-safe probe: +Start with the agent-safe, non-interactive probe: ```bash -vana connect --json --no-input +node run-connector.cjs connectors//-playwright.js [start-url] ``` This will: -- ensure the runtime is installed -- resolve and cache the connector -- try a saved session if one exists -- return structured events and a final outcome such as `needs_input`, `legacy_auth`, `connected_local_only`, or `connected_and_ingested` +- resolve the playwright-runner (auto-detected, or pass `--runner-dir`) +- try a saved browser-profile session if one exists +- stream structured line-delimited JSON events on stdout: `need-input`, `legacy-auth`, `log`, `result`, `error` -If the outcome is `needs_input`, rerun interactively: +If the connector emits `need-input`, either supply the value up front with `--inputs '{"key":"val"}'` or rerun with `--pretty` for a human-readable interactive session so the user can respond. -```bash -vana connect -``` - -If the user specifically wants to inspect current state before rerunning, use: - -```bash -vana status -``` +If the connector emits `legacy-auth`, it still depends on an older `showBrowser` / `promptUser` pattern and may need a headed/manual session path — flag this to the user rather than retrying blindly. ### 3. Handle outcomes -The CLI emits structured JSON events in `--json` mode. - -Key outcomes: - -- `needs_input` - The connector needs a live login or another manual step. Explain that you'll rerun interactively. -- `legacy_auth` - The connector still depends on `showBrowser` / `promptUser`. Explain that this source still needs a headed/manual session path and may not work in fully headless batch mode yet. -- `connected_local_only` - Data was collected locally but no Personal Server target was available. -- `connected_and_ingested` - Data was collected and synced to the Personal Server. - -If setup, fetch, or run output is truncated, use: - -```bash -vana logs -vana logs -``` +- `need-input` — the connector needs a live login or another manual step. Explain that you'll rerun interactively or with `--inputs`. +- `legacy-auth` — this source still needs a headed/manual session and may not work in fully headless batch mode yet. +- `result` — data was collected and written to the output path (default `~/.pdp-connect/desktop/last-result.json`, override with `--output`). +- `error` — the run failed; inspect the emitted message and `~/.pdp-connect/desktop/logs/` if present. -Prefer that over manually hunting through `~/.vana/desktop/logs/` or rerunning blindly. +If output is truncated or unclear, check `~/.pdp-connect/desktop/logs/` directly rather than rerunning blindly. -After a successful connect, prefer the CLI data surfaces over raw file inspection when possible: - -```bash -vana data list -vana data show -vana data path -vana logs -``` +After a successful run, read the result file at the output path to inspect the collected data. ### 4. Validate, present results, and offer to contribute If you built or modified a connector, immediately run validation — before presenting results to the user: ```bash -node scripts/validate.cjs connectors//-playwright.js --check-result ~/.vana/desktop/last-result.json +node scripts/validate.cjs connectors//-playwright.js --check-result ~/.pdp-connect/desktop/last-result.json ``` Fix any issues the validator reports. The validator checks debug code, login method diversity, schema descriptions, data cleanliness, and more — it is the quality gate. Iterate until validation passes. @@ -201,8 +114,8 @@ The user can't see what you're doing behind the scenes. Keep them informed at ke 3. **After collection**, summarize results in human terms — not file paths: - Good: "Connected! I collected 249 issues, 63 projects, 9 teams, and your profile from Linear." - - Bad: "Data saved to ~/.vana/desktop/last-result.json" - - Prefer the CLI outcome plus the result file. Build the summary from `exportSummary` and the scoped keys. + - Bad: "Data saved to ~/.pdp-connect/desktop/last-result.json" + - Build the summary from the result's `exportSummary` and the scoped keys. 4. **On failure**, explain what went wrong and what the user can do: - Auth failed → "Login didn't work. Can you double-check your credentials?" @@ -215,13 +128,4 @@ The user can't see what you're doing behind the scenes. Keep them informed at ke 3. **One platform at a time** 4. **Check session first** -- try without credentials if a browser profile exists 5. **Read connectors before running them** -6. **Use the CLI as the primary interface** -- only drop to raw scripts when debugging or updating connector internals - -## CLI fallback order - -Use this order when choosing the CLI entrypoint: - -1. installed `vana` -2. official installer path for the current canary -3. `npx -y @opendatalabs/connect@canary` -4. `node /home/tnunamak/code/vana-connect/dist/cli/bin.js` only for local development or debugging +6. **Run connectors via `run-connector.cjs`** -- it is the sole documented entrypoint for executing a connector; see `SETUP.md` for the one-time runtime install diff --git a/skills/vana-connect/reference/PAGE-API.md b/skills/pdp-connect/reference/PAGE-API.md similarity index 100% rename from skills/vana-connect/reference/PAGE-API.md rename to skills/pdp-connect/reference/PAGE-API.md diff --git a/skills/vana-connect/reference/PATTERNS.md b/skills/pdp-connect/reference/PATTERNS.md similarity index 100% rename from skills/vana-connect/reference/PATTERNS.md rename to skills/pdp-connect/reference/PATTERNS.md diff --git a/skills/vana-connect/scripts/fetch-connector.cjs b/skills/pdp-connect/scripts/fetch-connector.cjs similarity index 91% rename from skills/vana-connect/scripts/fetch-connector.cjs rename to skills/pdp-connect/scripts/fetch-connector.cjs index a621251..c572975 100644 --- a/skills/vana-connect/scripts/fetch-connector.cjs +++ b/skills/pdp-connect/scripts/fetch-connector.cjs @@ -5,7 +5,7 @@ * Usage: node scripts/fetch-connector.cjs * * Searches the registry for the platform, downloads the connector script, - * metadata, and connector-local schemas to ~/.vana/desktop/connectors/. + * metadata, and connector-local schemas to ~/.pdp-connect/desktop/connectors/. * Prints the local path on success. * * Exit codes: 0 = found and downloaded, 1 = not found or error. @@ -22,9 +22,9 @@ if (!platform) { process.exit(1); } -const REGISTRY_URL = 'https://raw.githubusercontent.com/vana-com/data-connectors/main/registry.json'; -const BASE_URL = 'https://raw.githubusercontent.com/vana-com/data-connectors/main'; -const CONNECTORS_DIR = path.join(os.homedir(), '.vana', 'desktop', 'connectors'); +const REGISTRY_URL = 'https://raw.githubusercontent.com/PDP-Connect/data-connectors/main/registry.json'; +const BASE_URL = 'https://raw.githubusercontent.com/PDP-Connect/data-connectors/main'; +const CONNECTORS_DIR = path.join(os.homedir(), '.pdp-connect', 'desktop', 'connectors'); function fetch(url) { return new Promise((resolve, reject) => { diff --git a/skills/vana-connect/scripts/generate-schemas.cjs b/skills/pdp-connect/scripts/generate-schemas.cjs similarity index 100% rename from skills/vana-connect/scripts/generate-schemas.cjs rename to skills/pdp-connect/scripts/generate-schemas.cjs diff --git a/skills/vana-connect/scripts/register.cjs b/skills/pdp-connect/scripts/register.cjs similarity index 100% rename from skills/vana-connect/scripts/register.cjs rename to skills/pdp-connect/scripts/register.cjs diff --git a/skills/vana-connect/scripts/run-connector.cjs b/skills/pdp-connect/scripts/run-connector.cjs similarity index 86% rename from skills/vana-connect/scripts/run-connector.cjs rename to skills/pdp-connect/scripts/run-connector.cjs index a864726..8750553 100644 --- a/skills/vana-connect/scripts/run-connector.cjs +++ b/skills/pdp-connect/scripts/run-connector.cjs @@ -6,7 +6,7 @@ * * Options: * --inputs '{"key":"val"}' Pre-supply credentials/2FA - * --output Result file path (default: ~/.vana/desktop/last-result.json) + * --output Result file path (default: ~/.pdp-connect/desktop/last-result.json) * --pretty Human-readable colored output instead of JSON * --runner-dir Path to playwright-runner (auto-detected if not set) * @@ -37,7 +37,7 @@ const rawArgs = process.argv.slice(2); const positional = []; let preSuppliedInputs = {}; let pretty = false; -let outputPath = path.join(homedir, '.vana', 'desktop', 'last-result.json'); +let outputPath = path.join(homedir, '.pdp-connect', 'desktop', 'last-result.json'); let runnerDir = null; for (let i = 0; i < rawArgs.length; i++) { @@ -56,18 +56,45 @@ for (let i = 0; i < rawArgs.length; i++) { } const connectorPath = positional[0]; -const startUrl = positional[1] || 'about:blank'; if (!connectorPath) { console.error('Usage: node run-connector.cjs [start-url] [--inputs \'{"key":"val"}\'] [--pretty]'); process.exit(1); } +// ─── Metadata resolution ───────────────────────────────────── +// Connector metadata lives in a sibling .json next to the .js script. +// It supplies the canonical requestedScopes the runner requires, and (when +// --url isn't passed) the starting connectURL. +function loadMetadata(resolvedConnectorPath) { + const metadataPath = resolvedConnectorPath.replace(/\.js$/, '.json'); + if (!fs.existsSync(metadataPath)) return null; + try { + return JSON.parse(fs.readFileSync(metadataPath, 'utf-8')); + } catch (e) { + console.error(`Warning: could not parse metadata at ${metadataPath}: ${e.message}`); + return null; + } +} + +const resolvedConnectorPath = path.resolve(connectorPath); +const metadata = loadMetadata(resolvedConnectorPath); +const requestedScopes = Array.isArray(metadata?.scopes) + ? metadata.scopes.map((s) => s.scope).filter((s) => typeof s === 'string' && s.length > 0) + : []; + +if (requestedScopes.length === 0) { + console.error(`Could not resolve requestedScopes for ${connectorPath} — no sibling metadata JSON with a non-empty "scopes" array found.`); + process.exit(1); +} + +const startUrl = positional[1] || metadata?.connectURL || metadata?.connect_url || 'about:blank'; + // Scope IPC files by connector name + timestamp so multiple runs never collide. const connectorSlug = path.basename(connectorPath, path.extname(connectorPath)); const runId = `${connectorSlug}-${Date.now()}`; -const PENDING_INPUT_PATH = path.join(homedir, '.vana', 'desktop', `pending-input-${runId}.json`); -const INPUT_RESPONSE_PATH = path.join(homedir, '.vana', 'desktop', `input-response-${runId}.json`); +const PENDING_INPUT_PATH = path.join(homedir, '.pdp-connect', 'desktop', `pending-input-${runId}.json`); +const INPUT_RESPONSE_PATH = path.join(homedir, '.pdp-connect', 'desktop', `input-response-${runId}.json`); // ─── Pretty output helpers ─────────────────────────────────── @@ -122,7 +149,8 @@ function resolveRunnerDir() { } const candidates = [ - path.join(homedir, '.vana', 'desktop', 'playwright-runner'), + path.resolve(__dirname, '..', '..', '..', 'playwright-runner'), + path.join(homedir, '.pdp-connect', 'desktop', 'playwright-runner'), process.env.PLAYWRIGHT_RUNNER_DIR, path.resolve(__dirname, '..', '..', '..', 'data-dt-app', 'playwright-runner'), ].filter(Boolean); @@ -187,8 +215,9 @@ function handleMessage(msg) { case 'ready': runner.stdin.write(JSON.stringify({ type: 'run', runId, - connectorPath: path.resolve(connectorPath), + connectorPath: resolvedConnectorPath, url: startUrl, headless: true, allowHeaded: false, + requestedScopes, }) + '\n'); if (pretty) prettyPrint(c.green, '[ready]', 'Connected, starting connector...'); break; diff --git a/skills/vana-connect/scripts/scaffold.cjs b/skills/pdp-connect/scripts/scaffold.cjs similarity index 94% rename from skills/vana-connect/scripts/scaffold.cjs rename to skills/pdp-connect/scripts/scaffold.cjs index 0a8198c..4e17a01 100644 --- a/skills/vana-connect/scripts/scaffold.cjs +++ b/skills/pdp-connect/scripts/scaffold.cjs @@ -4,7 +4,7 @@ * * Usage: node scaffold.cjs [company] [output-dir] * - * Defaults: company = platform, output-dir = ~/.vana/desktop/connectors + * Defaults: company = platform, output-dir = ~/.pdp-connect/desktop/connectors */ const fs = require('fs'); @@ -12,7 +12,7 @@ const path = require('path'); const platform = process.argv[2]; const company = process.argv[3] || platform; -const outputDir = process.argv[4] || path.join(require('os').homedir(), '.vana', 'desktop', 'connectors'); +const outputDir = process.argv[4] || path.join(require('os').homedir(), '.pdp-connect', 'desktop', 'connectors'); if (!platform) { console.error('Usage: node scaffold.cjs [company] [output-dir]'); diff --git a/skills/pdp-connect/scripts/setup.sh b/skills/pdp-connect/scripts/setup.sh new file mode 100755 index 0000000..38fa18f --- /dev/null +++ b/skills/pdp-connect/scripts/setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# setup.sh — One-shot setup for the pdp-connect skill. +# Installs the bundled playwright-runner's dependencies and Chromium. +# +# Usage: bash skills/pdp-connect/scripts/setup.sh +# Run from the data-connectors repo root (where playwright-runner/ lives). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +RUNNER_DIR="$REPO_ROOT/playwright-runner" + +if [[ ! -f "$RUNNER_DIR/index.cjs" ]]; then + echo "Could not find playwright-runner/ at $RUNNER_DIR — is this the data-connectors repo root?" + exit 1 +fi + +# Skip if already set up +if [[ -f "$RUNNER_DIR/node_modules/playwright/package.json" ]]; then + echo "Already set up. To reinstall, remove $RUNNER_DIR/node_modules/ first." + exit 0 +fi + +echo "Setting up the bundled playwright-runner..." + +# 1. Install runner dependencies (just playwright) +echo "Installing dependencies..." +cd "$RUNNER_DIR" && npm install 2>&1 + +# 2. Install Chromium (also runs automatically via the runner's postinstall hook) +echo "Installing Chromium (this may take a minute)..." +npx playwright install chromium 2>&1 || { + echo "Note: Chromium install may need extra system deps. Trying 'playwright install-deps'..." + npx playwright install-deps chromium 2>&1 || true + npx playwright install chromium 2>&1 || true +} + +# Verify +echo "" +if [[ -d "$RUNNER_DIR/node_modules/playwright" ]]; then + echo "Setup complete! The bundled runner at $RUNNER_DIR is ready." +else + echo "Setup may be incomplete. Check $RUNNER_DIR for missing files." + exit 1 +fi diff --git a/skills/vana-connect/scripts/validate.cjs b/skills/pdp-connect/scripts/validate.cjs similarity index 98% rename from skills/vana-connect/scripts/validate.cjs rename to skills/pdp-connect/scripts/validate.cjs index d79cf18..a7e9254 100644 --- a/skills/vana-connect/scripts/validate.cjs +++ b/skills/pdp-connect/scripts/validate.cjs @@ -637,7 +637,7 @@ function contribute(connectorPath, connectorDir, metadataPath) { if (!useGh) { try { - execSync('git ls-remote https://github.com/vana-com/data-connectors.git HEAD', { stdio: 'pipe' }); + execSync('git ls-remote https://github.com/PDP-Connect/data-connectors.git HEAD', { stdio: 'pipe' }); } catch { console.error('Cannot authenticate with GitHub. Install and auth `gh` CLI, or configure git credentials.'); process.exit(1); @@ -672,7 +672,7 @@ function contribute(connectorPath, connectorDir, metadataPath) { try { // Fork if needed if (useGh) { - try { execSync('gh repo fork vana-com/data-connectors --clone=false', { stdio: 'pipe', cwd: repoRoot }); } catch {} + try { execSync('gh repo fork PDP-Connect/data-connectors --clone=false', { stdio: 'pipe', cwd: repoRoot }); } catch {} } execSync(`git checkout -b ${branchName}`, { stdio: 'pipe', cwd: repoRoot }); @@ -715,7 +715,7 @@ function contribute(connectorPath, connectorDir, metadataPath) { reportSection, ``, `---`, - `Generated by the [vana-connect skill](https://github.com/vana-com/data-connectors/tree/main/skills/vana-connect).`, + `Generated by the [pdp-connect skill](https://github.com/PDP-Connect/data-connectors/tree/main/skills/pdp-connect).`, ].join('\n'); const prBodyFile = path.join(repoRoot, '.pr-body.tmp'); @@ -732,7 +732,7 @@ function contribute(connectorPath, connectorDir, metadataPath) { } else { execSync(`git push -u origin ${branchName}`, { stdio: 'pipe', cwd: repoRoot }); console.log(`\nBranch pushed: ${branchName}`); - console.log(`Open a PR at: https://github.com/vana-com/data-connectors/compare/${branchName}`); + console.log(`Open a PR at: https://github.com/PDP-Connect/data-connectors/compare/${branchName}`); } } catch (e) { console.error('Contribution failed:', e.message); diff --git a/skills/vana-connect/templates/connector-metadata.json b/skills/pdp-connect/templates/connector-metadata.json similarity index 100% rename from skills/vana-connect/templates/connector-metadata.json rename to skills/pdp-connect/templates/connector-metadata.json diff --git a/skills/vana-connect/templates/connector-script.js b/skills/pdp-connect/templates/connector-script.js similarity index 100% rename from skills/vana-connect/templates/connector-script.js rename to skills/pdp-connect/templates/connector-script.js diff --git a/skills/vana-connect/templates/schema.json b/skills/pdp-connect/templates/schema.json similarity index 100% rename from skills/vana-connect/templates/schema.json rename to skills/pdp-connect/templates/schema.json diff --git a/skills/vana-connect/SETUP.md b/skills/vana-connect/SETUP.md deleted file mode 100644 index 2b35b6f..0000000 --- a/skills/vana-connect/SETUP.md +++ /dev/null @@ -1,95 +0,0 @@ -# Connect -- Setup - -This setup exists to let the skill use an installed `vana` CLI. - -```bash -command -v vana -``` - -Skip runtime setup if `vana status --json` reports `"runtime":"installed"` or `"runtime":{"installed":true,...}`. - -## Prerequisites - -For the normal installed CLI path: - -- Homebrew on macOS, or -- `curl` and `sh` on macOS/Linux - -Only local development and `npx` fallback require Node.js. - -## Install the published CLI - -Preferred on macOS: - -```bash -brew tap vana-com/vana -brew install vana -``` - -macOS and Linux: - -```bash -curl -fsSL https://raw.githubusercontent.com/vana-com/vana-connect/feat/connect-cli-v1/install/install.sh | sh -s -- --version canary-feat-connect-cli-v1 -``` - -Fallback if installer paths are blocked: - -```bash -npx -y @opendatalabs/connect@canary --help -``` - -## Local development fallback - -From `/home/tnunamak/code/vana-connect`: - -```bash -pnpm install -pnpm build -``` - -Verify: - -```bash -ls /home/tnunamak/code/vana-connect/dist/cli/bin.js -``` - -## Install the runtime - -Use the published CLI to provision the runtime: - -```bash -vana setup --yes -``` - -Before running, tell the user this downloads a browser engine and some dependencies into `~/.vana/desktop/`. This is a one-time step. - -## Verify - -```bash -vana status -vana doctor -vana logs -``` - -You should see `Runtime: installed`. If not, inspect the log path surfaced by the CLI or use `vana logs`, and only fall back to the older script-level flow if the CLI setup path is blocked. - -## Legacy fallback - -Only use this if the CLI setup path is broken and you are debugging the underlying runtime: - -```bash -bash skills/vana-connect/scripts/setup.sh -``` - -## File Locations - -| Path | Purpose | -|------|---------| -| `vana` | Preferred installed CLI entrypoint | -| `npx -y @opendatalabs/connect@canary` | Published canary CLI fallback | -| `/home/tnunamak/code/vana-connect/dist/cli/bin.js` | Local development fallback | -| `~/.local/share/vana/` | Hosted-installer release payloads | -| `~/.vana/desktop/connectors/` | Connector scripts | -| `~/.vana/desktop/browser-profiles/` | Persistent sessions (cookies) | -| `~/.vana/desktop/last-result.json` | Most recent result | -| `~/.vana/desktop/logs/` | Setup, fetch, and run logs | diff --git a/skills/vana-connect/scripts/setup.sh b/skills/vana-connect/scripts/setup.sh deleted file mode 100755 index f114e18..0000000 --- a/skills/vana-connect/scripts/setup.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# setup.sh — One-shot setup for the vana-connect skill. -# Installs playwright-runner, Chromium, and run-connector.cjs. -# -# Usage: bash scripts/setup.sh -# Run from the data-connectors repo root (where skills/ lives). - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -VANA_DIR="$HOME/.vana/desktop" - -# Skip if already set up -if [[ -f "$VANA_DIR/playwright-runner/index.cjs" && -f "$VANA_DIR/run-connector.cjs" ]]; then - echo "Already set up. To reinstall, remove ~/.vana/desktop/playwright-runner/ first." - exit 0 -fi - -echo "Setting up vana-connect..." - -mkdir -p "$VANA_DIR/connectors" - -# 1. Clone playwright-runner (sparse checkout, minimal download) -echo "Downloading playwright-runner..." -cd "$VANA_DIR" -rm -rf _data-connect -git clone --depth 1 --filter=blob:none --sparse --branch main \ - https://github.com/vana-com/data-connect.git _data-connect 2>&1 -cd _data-connect && git sparse-checkout set playwright-runner 2>&1 -cp -r playwright-runner ../playwright-runner -cd .. && rm -rf _data-connect - -# 2. Install dependencies -echo "Installing dependencies..." -cd "$VANA_DIR/playwright-runner" && npm install 2>&1 - -# 3. Install Chromium -echo "Installing Chromium (this may take a minute)..." -npx playwright install chromium 2>&1 || { - echo "Note: 'playwright install --with-deps' may need root. Trying without system deps..." - npx playwright install chromium 2>&1 || true -} - -# 4. Copy run-connector.cjs -if [[ -f "$SCRIPT_DIR/run-connector.cjs" ]]; then - cp "$SCRIPT_DIR/run-connector.cjs" "$VANA_DIR/run-connector.cjs" -else - echo "Warning: run-connector.cjs not found at $SCRIPT_DIR/run-connector.cjs" - echo "Copy it manually: cp skills/vana-connect/scripts/run-connector.cjs ~/.vana/desktop/run-connector.cjs" -fi - -# Verify -echo "" -if [[ -f "$VANA_DIR/playwright-runner/index.cjs" && -f "$VANA_DIR/run-connector.cjs" ]]; then - echo "Setup complete!" -else - echo "Setup may be incomplete. Check ~/.vana/desktop/ for missing files." - exit 1 -fi diff --git a/test-connector.cjs b/test-connector.cjs index c431efb..e9fd9ed 100644 --- a/test-connector.cjs +++ b/test-connector.cjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // TODO: Migrate .claude/skills/ and scripts/ to use run-connector.cjs -// (skills/vana-connect/scripts/run-connector.cjs), then delete this file. +// (skills/pdp-connect/scripts/run-connector.cjs), then delete this file. /** * Standalone Connector Test Runner @@ -58,6 +58,8 @@ function resolveRunnerDir() { // 2. Common locations relative to this script const candidates = [ + // Bundled runner in this repo + path.resolve(__dirname, 'playwright-runner'), // Sibling repo (same parent directory) path.resolve(__dirname, '..', 'data-dt-app', 'playwright-runner'), path.resolve(__dirname, '..', 'data-connect', 'playwright-runner'), @@ -301,9 +303,17 @@ async function main() { const runnerDir = resolveRunnerDir(); const metadata = loadMetadata(connectorPath); - const connectUrl = args.url || (metadata && metadata.connectURL) || 'about:blank'; + const connectUrl = args.url || (metadata && (metadata.connectURL || metadata.connect_url)) || 'about:blank'; const connectorName = metadata?.name || path.basename(connectorPath, '.js'); const connectorVersion = metadata?.version || 'unknown'; + const requestedScopes = Array.isArray(metadata?.scopes) + ? metadata.scopes.map((s) => s.scope).filter((s) => typeof s === 'string' && s.length > 0) + : []; + + if (requestedScopes.length === 0) { + console.error(`${c.red}Could not resolve requestedScopes for ${connectorPath} — no sibling metadata JSON with a non-empty "scopes" array found.${c.reset}`); + process.exit(1); + } console.log(''); console.log(`${c.bold}Connector Test Runner${c.reset}`); @@ -341,6 +351,7 @@ async function main() { url: connectUrl, headless: args.headless, forceHeaded: !args.headless, + requestedScopes, }); child.stdin.write(runCmd + '\n'); print(c.green, '[runner]', 'Connected, starting connector...');