diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0e73a6a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install dependencies + run: npm ci + + - name: Run packaging tests + run: npm test + + - name: Run installed skill tests + run: npm run test:skills + + - name: Release-safety gate + run: node scripts/check-release-safety.mjs + + - name: Skill-closure verification + run: node scripts/verify-skill-closure.mjs + + - name: Dependency-matrix verification + run: node scripts/verify-dependency-matrix.mjs + + - name: Validate compose.yaml + run: docker compose config --quiet + + - name: Check doc commands + run: node scripts/check-doc-commands.mjs + + - name: Check required doc sections + run: node scripts/check-required-doc-sections.mjs + + compose-rehearsal: + runs-on: ubuntu-latest + needs: validate + steps: + - uses: actions/checkout@v4 + + - name: Run compose rehearsal + run: bash scripts/rehearse-compose.sh + timeout-minutes: 10 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b2c5c3f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,91 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + build: + runs-on: ubuntu-latest + outputs: + archive: ${{ steps.archive.outputs.path }} + checksum: ${{ steps.checksum.outputs.sha256 }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install dependencies + run: npm ci + + - name: Run packaging tests + run: npm test + + - name: Run installed skill tests + run: npm run test:skills + + - name: Release verification (source tree) + run: npm run verify:release + + - name: Build source archive + id: archive + run: | + ARCHIVE="job-hunter-${GITHUB_REF_NAME}.tar.gz" + git archive --prefix="job-hunter-${GITHUB_REF_NAME}/" -o "$ARCHIVE" HEAD + echo "path=$ARCHIVE" >> "$GITHUB_OUTPUT" + + - name: SHA-256 checksum + id: checksum + run: | + ARCHIVE="${{ steps.archive.outputs.path }}" + SHA=$(sha256sum "$ARCHIVE" | awk '{print $1}') + echo "sha256=$SHA" >> "$GITHUB_OUTPUT" + echo "$SHA $ARCHIVE" > "${ARCHIVE}.sha256" + + - name: Extract and safety-scan archive + run: | + ARCHIVE="${{ steps.archive.outputs.path }}" + mkdir /tmp/archive-scan + tar -xzf "$ARCHIVE" -C /tmp/archive-scan + node scripts/check-release-safety.mjs /tmp/archive-scan/job-hunter-${GITHUB_REF_NAME} + + - name: Upload archive + uses: actions/upload-artifact@v4 + with: + name: source-archive + path: ${{ steps.archive.outputs.path }} + + - name: Upload checksum + uses: actions/upload-artifact@v4 + with: + name: source-archive-sha256 + path: ${{ steps.archive.outputs.path }}.sha256 + + release: + runs-on: ubuntu-latest + needs: build + steps: + - name: Download archive + uses: actions/download-artifact@v4 + with: + name: source-archive + + - name: Download checksum + uses: actions/download-artifact@v4 + with: + name: source-archive-sha256 + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + files: | + *.tar.gz + *.sha256 + draft: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4c63f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# Dependencies +node_modules/ + +# Python +__pycache__/ +*.pyc +.venv/ +uv.lock + +# Runtime artifacts +*.sqlite +*.sqlite3 +*.docx +*.pdf +*.xlsx +*.csv +!examples/**/*.docx +personal-info-cache.json +apply_logs/ +logs/ +backups/ +screenshots/ +*.png +*.jpg +*.jpeg +*.webp +*.log + +# Browser state +chromium-profile/ +chromium_profile/ +selenium_chrome_profile/ +searxng/ +*.bak + +# OS +.DS_Store +Thumbs.db + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# Environment +.env +.env.local +.env.*.local + +# Job-hunter runtime (never tracked) +.job-hunter/ + +# Agent evidence (local, never published) +agent-output/ + +# Build/release +dist/ +tmp/ +*.tar.gz +*.zip diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b614b63 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025-2026 Deviad + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f4f4d95 --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +# Job Hunter + +Complete job-search pipeline for [Pi Agent](https://github.com/earendil-works/pi) — search, score, salary enrichment, and assisted applications. + +## Prerequisites + +| Layer | Requirement | Stage | +|-------|-------------|-------| +| Runtime | Node.js ≥ 22, `uv` ≥ 0.4 | Install | +| Code | `npm ci` in repo root | Install | +| Workspace | `$JOBHUNTER_HOME` (default `~/.job-hunter`) with `jobhunter.sqlite`, `CV.docx`, `personal-info-cache.json` | Init | +| Browser | Chromium CDP on `127.0.0.1:9225` (Selenium container) | Search, Apply | +| Search | SearXNG on `127.0.0.1:8888` | Discover | +| Vision (optional) | LM Studio on `127.0.0.1:1234` with Qwen VLM | CAPTCHA, Debug | +| MCP (optional) | Obscura, Apple Mail MCP servers | Browse, Outreach | + +Host tools (`docker`, `sqlite3`, `python3`) must be installed and on `PATH` — the doctor (`jh-doctor`) diagnoses which are missing but does not install them. + +## Installation + +```bash +git clone https://github.com/Deviad/job-hunter.git && cd job-hunter +npm ci +node scripts/install.mjs # copies skills into ${PI_AGENT_HOME:-~/.pi/agent}/skills/ +node scripts/install.mjs --dry-run # preview without writing +``` + +Re-run is idempotent and never overwrites `CV.docx`, `personal-info-cache.json`, or `jobhunter.sqlite`. + +## Initialization + +```bash +node ~/.pi/agent/skills/job-hunter/scripts/jh-init.mjs +``` + +Before applying, populate `~/.job-hunter/CV.docx` and the `profile.firstName`, `profile.lastName`, `profile.email`, and `profile.phone` fields in `~/.job-hunter/personal-info-cache.json`. Application helpers fail closed when required identity fields are absent; the repository contains no maintainer identity defaults. + +### Doctor + +Run the doctor before each search or application session. It distinguishes required failures from optional degraded capabilities. + +```bash +node ~/.pi/agent/skills/job-hunter/scripts/jh-doctor.mjs +``` + +## External Services + +Start Selenium and SearXNG: + +```bash +export SEARXNG_SECRET="$(node -e "process.stdout.write(require('node:crypto').randomBytes(32).toString('hex'))")" +docker compose up -d +bash scripts/rehearse-compose.sh # isolated health-check rehearsal with teardown +``` + +See `docs/prerequisites.md` for full host-tool list and `docs/security-and-privacy.md` for security boundaries. + +## Workflow + +### Search + +```bash +node ~/.pi/agent/skills/job-hunter/scripts/jh-search.mjs --source linkedin --country GB --role "Software Engineer" +node ~/.pi/agent/skills/job-hunter/scripts/jh-search.mjs --source indeed --country GB --query "Software Engineer" +node ~/.pi/agent/skills/job-hunter/scripts/jh-discover.mjs --locations "United Kingdom" --queries "Software Engineer" +``` + +### Score + +Ask Pi to score the saved jobs. The `job-match-scorer` skill reads the user's CV and workspace profile rather than repository defaults. + +### Salary + +```bash +node ~/.pi/agent/skills/salary-calculator/scripts/enrich-job-salary.mjs --db ~/.job-hunter/jobhunter.sqlite --all-unsalaried +``` + +### Apply + +Ask Pi to apply to a scored job. The `auto-job-application` skill uses the user's local CV, cache, and authenticated Chromium session; there is no repository-level submit command. + +### Pipeline status and backup + +```bash +node ~/.pi/agent/skills/job-hunter/scripts/jh-status.mjs +node ~/.pi/agent/skills/job-hunter/scripts/jh-backup.mjs +``` + +## Update + +```bash +git pull && npm ci && node scripts/install.mjs +``` + +## Uninstall + +```bash +node scripts/install.mjs --uninstall +``` + +Removes only files recorded in the installation manifest. User data (`CV.docx`, DB, cache) is untouched. + +## Troubleshooting + +```bash +node ~/.pi/agent/skills/job-hunter/scripts/jh-doctor.mjs # full diagnostic +``` + +Common issues: + +- **CDP not reachable**: ensure `docker compose up -d` ran and `curl -s http://127.0.0.1:9225/json/version` returns JSON. +- **SearXNG 502**: SearXNG needs a few seconds after container start; retry after 5s. +- **better-sqlite3 build failure**: ensure `python3` and a C compiler (Xcode CLT on macOS) are installed. +- **MCP "Not connected"**: run the repair skill or `bash scripts/rehearse-compose.sh` to verify transport. + +## Security and Privacy + +- All job-search data, credentials, and browser sessions stay on your machine. +- The installer never reads or transmits personal files. +- Authenticated browser sessions (LinkedIn, Indeed) belong to you — this project does not bypass CAPTCHA, MFA, or site access controls. +- See `docs/security-and-privacy.md` for the full policy. + +## License + +MIT — see `LICENSE`. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..5a35d8c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,72 @@ +# Job Hunter — Selenium Chromium + SearXNG +# +# Usage: +# docker compose up -d # start services +# docker compose down # stop (no data loss) +# bash scripts/rehearse-compose.sh # bounded health check + teardown +# +# Prerequisites: Docker with Compose support. +# No credentials are embedded. Authenticated browser sessions are user-managed. + +services: + selenium-chromium: + image: selenium/standalone-chromium:4.44@sha256:8c5a8629c96104c0d73df94c6437af9ab9059c4e16aa32e35b330b7d77defe0b + shm_size: "2gb" + restart: unless-stopped + ports: + - "127.0.0.1:${SELENIUM_PORT:-4444}:4444" # Selenium WebDriver + - "127.0.0.1:${NOVNC_PORT:-7900}:7900" # noVNC web UI + - "127.0.0.1:${CDP_PORT:-9225}:9222" # Chrome DevTools Protocol (CDP) + entrypoint: + - "/bin/sh" + - "-c" + - | + # Ensure xdotool is available for visual-click-recovery workflow. + if ! command -v xdotool >/dev/null 2>&1; then + (sudo apt-get update -qq && sudo apt-get install -y -qq xdotool) \ + || echo "WARN: xdotool install failed; visual click recovery degraded" + fi + exec /opt/bin/entry_point.sh + environment: + - SE_START_XVFB=true + - SE_START_VNC=true + - SE_START_NO_VNC=true + - SE_VNC_NO_PASSWORD=true + - SE_NO_VNC_PORT=7900 + - SE_VNC_PORT=5900 + - SE_SCREEN_WIDTH=1920 + - SE_SCREEN_HEIGHT=1080 + volumes: + - "${JOBHUNTER_HOME:-${HOME}/.job-hunter}:/home/seluser/job-hunter" + - "${SELENIUM_PROFILE_DIR:-${JOBHUNTER_HOME:-${HOME}/.job-hunter}/chromium-profile}:/home/seluser/.config/google-chrome" + - "./docker/selenium/cdp_proxy.py:/opt/selenium/cdp_proxy.py:ro" + - "./docker/selenium/manual-chrome.conf:/etc/supervisor/conf.d/manual-chrome.conf:ro" + networks: + - jobhunter + + searxng: + image: searxng/searxng:latest@sha256:b36af7984b87191b595bc5301418ed6432c047668a4547ab531a7439b816fac3 + restart: unless-stopped + ports: + - "127.0.0.1:${SEARXNG_PORT:-8888}:8080" + environment: + SEARXNG_BASE_URL: http://localhost:${SEARXNG_PORT:-8888}/ + SEARXNG_LIMITER: "false" + SEARXNG_SECRET: ${SEARXNG_SECRET:-} + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - "${JOBHUNTER_HOME:-${HOME}/.job-hunter}/searxng:/etc/searxng" + tmpfs: + - /tmp + networks: + - jobhunter + +networks: + jobhunter: + name: jobhunter + driver: bridge diff --git a/docker/selenium/cdp_proxy.py b/docker/selenium/cdp_proxy.py new file mode 100644 index 0000000..5a0311a --- /dev/null +++ b/docker/selenium/cdp_proxy.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import select +import socket +import threading + +LISTEN_HOST = "0.0.0.0" +LISTEN_PORT = 9222 +TARGET_HOST = "127.0.0.1" +TARGET_PORT = 9223 +BUFFER_SIZE = 65536 + + +def pipe(client): + target = socket.create_connection((TARGET_HOST, TARGET_PORT)) + sockets = [client, target] + try: + while True: + readable, _, _ = select.select(sockets, [], []) + for source in readable: + data = source.recv(BUFFER_SIZE) + if not data: + return + destination = target if source is client else client + destination.sendall(data) + finally: + client.close() + target.close() + + +def main(): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((LISTEN_HOST, LISTEN_PORT)) + listener.listen(50) + while True: + client, _ = listener.accept() + threading.Thread(target=pipe, args=(client,), daemon=True).start() + + +if __name__ == "__main__": + main() diff --git a/docker/selenium/manual-chrome.conf b/docker/selenium/manual-chrome.conf new file mode 100644 index 0000000..3f13357 --- /dev/null +++ b/docker/selenium/manual-chrome.conf @@ -0,0 +1,27 @@ +[program:manual-chrome] +priority=16 +command=bash -lc "sleep 3; exec /usr/lib/chromium/chromium --no-sandbox --disable-dev-shm-usage --remote-debugging-address=127.0.0.1 --remote-debugging-port=9223 --no-first-run --no-default-browser-check --user-data-dir=/home/seluser/.config/google-chrome about:blank" +autostart=true +autorestart=true +startsecs=3 +startretries=5 +stopsignal=TERM +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 + +[program:cdp-proxy] +priority=17 +command=python3 /opt/selenium/cdp_proxy.py +autostart=true +autorestart=true +startsecs=0 +startretries=5 +stopsignal=TERM +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 diff --git a/docs/dependency-matrix.md b/docs/dependency-matrix.md new file mode 100644 index 0000000..86caad2 --- /dev/null +++ b/docs/dependency-matrix.md @@ -0,0 +1,81 @@ +# Dependency Matrix + +This matrix covers every bundled skill and classifies code dependencies, executables, services, MCP integrations, and workflow-stage optionality. **Required** means the named workflow cannot run without it. **Conditional** means only specific stages need it. **Optional** means its absence degrades a recovery or convenience path. + +## Skills and Packages + +| Skill | Class | Node packages | Python dependencies | Direct skill dependencies | +|---|---|---|---|---| +| `job-hunter` | Core | `better-sqlite3`, `ws` | None | LinkedIn, Indeed, scorer, salary, auto-apply | +| `linkedin-job-search` | Core | `ws`, workspace `better-sqlite3` | None | CAPTCHA and browser repair paths | +| `indeed-job-search` | Core | `ws`, workspace `better-sqlite3` | None | LinkedIn SQLite saver, CAPTCHA and browser repair paths | +| `job-match-scorer` | Core | workspace `better-sqlite3` | Standard-library scorer | DOCX input contract, salary handoff | +| `salary-calculator` | Core | `better-sqlite3` | None | Browser session bridge for posted-salary evidence | +| `auto-job-application` | Core | `better-sqlite3`, `ws` | Standard-library helpers | CAPTCHA, Qwen recovery, Selenium visual recovery, DOCX/PDF | +| `captcha-resolution` | Required support | `ws` | `Pillow` through its helper environment | Qwen recovery when visual interpretation is needed | +| `qwen-screenshot-debug` | Required support | `ws` | Standard library | LM Studio/Qwen service | +| `selenium-container-visual-click-recovery` | Required support | None | Standard library | Qwen screenshot debugging | +| `obscura-mcp-repair` | Required support | None | None | Pi MCP repair | +| `pi-mcp-repair` | Required support | None | None | Pi MCP gateway | +| `brave-obscura-session` | Required support | `ws` | Standard library | Obscura MCP | +| `docx` | Required support | None | PEP 723: `python-docx`, shared document helper | `uv` | +| `pdf` | Required support | None | PEP 723: `pypdf`, `reportlab`, shared document helper | `uv` | + +The root `package.json` and `package-lock.json` pin the shared Node packages. DOCX and PDF helpers declare isolated Python dependencies in their PEP 723 scripts, run with `uv run`, and share the bundled `skills/_document_common/document_common.py` helper. + +## Executables + +| Executable | Requirement | Used by | +|---|---|---| +| Node.js 22 or newer and npm | Required | All JavaScript skills and bootstrap | +| `sqlite3` | Required | Workspace diagnostics and database operations | +| `python3` | Conditional | Scoring and ATS/document helpers | +| `uv` | Conditional | DOCX and PDF helpers | +| Docker with Compose | Conditional | Selenium Chromium and SearXNG services | +| `ffmpeg` and `xdotool` | Conditional, inside recovery environment | Selenium visual click recovery | +| Git | Required for source installation and updates | Bootstrap workflow | + +## Services + +| Service | Default endpoint | Requirement | Workflow | +|---|---|---|---| +| Selenium Chromium/CDP | `127.0.0.1:9225` | Conditional | LinkedIn/Indeed search and applications | +| SearXNG | `127.0.0.1:8888` | Conditional | External job discovery | +| LM Studio with Qwen VLM | `127.0.0.1:1234` | Optional | Screenshot interpretation and visual recovery | + +`compose.yaml` provides Selenium Chromium and SearXNG. The installer verifies heavyweight prerequisites but does not install Docker, start services, or download model weights. + +## MCP Integrations + +| Integration | Requirement | Used by | +|---|---|---| +| Obscura MCP | Conditional | Alternate authorized browser flows and repair | +| Pi MCP gateway | Conditional | MCP repair skills | +| Apple Mail MCP | Optional | Authorized email-code retrieval during applications | +| Context-mode MCP | Optional | Large-output analysis by the controlling Pi agent | + +## Workspace Dependencies + +`$JOBHUNTER_HOME` defaults to `~/.job-hunter` and contains user-owned runtime state: + +- `jobhunter.sqlite` +- `CV.docx` +- `personal-info-cache.json` +- `node_modules/better-sqlite3` +- `node_modules/ws` +- generated logs, backups, and run artifacts + +These files are installed or created locally and are excluded from the publication repository. + +## Workflow-Stage Optionality + +| Stage | Active skills | Required service or integration | +|---|---|---| +| Initialize and doctor | `job-hunter` | Local Node, npm, SQLite | +| Search | `job-hunter`, LinkedIn, Indeed | Authenticated Selenium Chromium/CDP | +| Discover | `job-hunter` | SearXNG | +| Score | `job-match-scorer`, DOCX | Local CV and workspace profile | +| Salary | `salary-calculator` | Database; browser access only for posted evidence | +| Apply | `auto-job-application`, CAPTCHA support | Authenticated Selenium Chromium/CDP | +| Visual recovery | Qwen and Selenium recovery skills | Optional LM Studio/Qwen, `ffmpeg`, `xdotool` | +| MCP repair | Obscura and Pi MCP repair skills | Corresponding MCP server | diff --git a/docs/prerequisites.md b/docs/prerequisites.md new file mode 100644 index 0000000..0124f87 --- /dev/null +++ b/docs/prerequisites.md @@ -0,0 +1,45 @@ +# Prerequisites + +Host tools are required for the pipeline to operate. They must be installed manually — the installer and doctor verify their presence but never install host-level software silently. + +## Automatically Installed Code Dependencies + +The global bootstrap copies the bundled skills and runs `npm ci` from the shipped workspace lockfile. It does not install host tools, containers, MCP servers, authenticated browser sessions, or model weights. + +## Required Host Tools + +| Tool | Purpose | Verify | Install (macOS) | +|------|---------|--------|------------------| +| Node.js ≥ 22 | Script runtime | `node --version` | `brew install node` | +| npm | Package manager | `npm --version` | bundled with Node | +| uv ≥ 0.4 | PEP 723 Python helpers (docx, pdf) | `uv --version` | `brew install uv` | +| python3 ≥ 3.11 | DOCX/PDF tools, inline extractions | `python3 --version` | bundled / `brew install python` | +| sqlite3 CLI | Schema inspections, ad-hoc queries | `sqlite3 --version` | bundled / `brew install sqlite` | +| docker + compose | Selenium Chromium, SearXNG | `docker compose version` | [Docker Desktop](https://docs.docker.com/desktop/) | +| git | Installation, updates | `git --version` | `xcode-select --install` | + +## Container Services and Conditional Requirements + +| Tool | Stage | Verify | Notes | +|------|-------|--------|-------| +| Chromium CDP (`127.0.0.1:9225`) | Search, Apply | `bash scripts/rehearse-compose.sh` | Provided by `compose.yaml` | +| SearXNG (`127.0.0.1:8888`) | Discover | `curl -sf http://127.0.0.1:8888/search?q=test` | Provided by `compose.yaml`; set `SEARXNG_SECRET` before startup | +| Authenticated LinkedIn session | LinkedIn search | Manual login in Selenium browser | Session is user-managed | +| Authenticated Indeed session | Indeed search | Manual login in Selenium browser | Session is user-managed | + +## Service Data and Secrets + +The installer creates `chromium-profile/` and `searxng/` under `$JOBHUNTER_HOME`; Compose bind-mounts them for browser state and SearXNG configuration. Set `SEARXNG_SECRET` to a random local value before `docker compose up -d`. The repository contains no default credential, and the installer never starts containers. + +## Optional + +| Tool | Stage | Purpose | Notes | +|------|-------|---------|-------| +| LM Studio (`127.0.0.1:1234`) | CAPTCHA solve, Visual debug | Qwen VLM inference | Never downloaded by installer | +| Obscura MCP | Browse, Apply | Headless browsing alternative | Pi MCP integration; see `obscura-mcp-repair` skill | +| Apple Mail MCP | Outreach | Cold email sending | Pi MCP integration | +| Brave browser | Brave session bridge | Cookie relay to Obscura | `brave-obscura-session` skill | + +## Doctor + +`jh-doctor.mjs` checks all required and conditional prerequisites, reporting which are present, missing, or degraded. It distinguishes required failures (exit non-zero) from optional degradations (warning only). diff --git a/docs/security-and-privacy.md b/docs/security-and-privacy.md new file mode 100644 index 0000000..089e2e4 --- /dev/null +++ b/docs/security-and-privacy.md @@ -0,0 +1,54 @@ +# Security and Privacy + +## Local Data Locations + +All job-hunting data lives under `$JOBHUNTER_HOME` (default `~/.job-hunter`): + +| File | Content | Sensitive? | +|------|---------|------------| +| `jobhunter.sqlite` | Jobs, scores, salary, applications | Yes — contains work history, salary expectations | +| `personal-info-cache.json` | Work auth, salary, notice, demographics | Yes — PII and employment details | +| `CV.docx` | Resume | Yes | +| `optional_documents/` | Cover letters, uploads | Yes | +| `apply_logs/` | Per-application run logs | May contain form answers | +| `backups/` | Timestamped DB snapshots | Yes — mirror of DB | + +None of these files are tracked in Git. `.gitignore` excludes them. + +## Sensitive-File Exclusions + +The release-safety gate (`scripts/check-release-safety.mjs`) scans all tracked files and rejects: + +- Filepaths matching personal-data patterns (`.sqlite`, `.docx` outside `examples/`, `.env`, `personal-info`) +- File content containing real email addresses, phone numbers, API keys, or tokens +- Absolute host-specific paths (`/Users/`, `/home/`) in bundled skill files + +## Credential Handling + +- Credentials live in `personal-info-cache.json` and authenticated browser cookies — never in this repository. +- The installer copies only skill code, never workspace data. +- Docker Compose binds `$JOBHUNTER_HOME` into the Selenium container for CV uploads; the Compose file itself contains no credentials. + +## Authenticated-Browser Boundaries + +- LinkedIn and Indeed sessions are user-managed. The project does not store, export, or replay credentials. +- CAPTCHA, MFA, and access-control challenges are the user's responsibility. The `captcha-resolution` skill provides a local-vision assist for image-grid CAPTCHAs but does not bypass site security measures. +- Brave-to-Obscura cookie relay (`brave-obscura-session`) operates on same-origin cookies within the user's own browser session. It never transmits cookies to third parties. + +## CAPTCHA and Access-Control Policy + +- `captcha-resolution` uses a local Qwen VLM (LM Studio) to identify image-grid tiles. It runs entirely on your machine. +- The project does not use third-party CAPTCHA-solving services. +- If a site blocks automation, the skill halts and surfaces the block. No retries that could violate terms of service. + +## Backup Expectations + +- `jh-backup.mjs` creates timestamped snapshots in `$JOBHUNTER_HOME/backups/`. +- Backups are local-only. The project never uploads backups anywhere. +- The installer does not create or modify backups. + +## Disclosure Implications + +- Applying to jobs is an externally visible action. The project applies only when the user instructs it to. +- Outreach emails (via Apple Mail MCP or manual) are sent only with explicit user authorization. +- No application data is shared with this project's maintainers or any third party. diff --git a/examples/synthetic-workspace/README.md b/examples/synthetic-workspace/README.md new file mode 100644 index 0000000..f0f22d7 --- /dev/null +++ b/examples/synthetic-workspace/README.md @@ -0,0 +1,11 @@ +# Synthetic Workspace Fixtures + +These files are synthetic test data for CI and local validation. +They contain no real personal information. + +## Files + +- `sample-personal-info-cache.json` — minimal cache fixture with placeholder values. +- `sample-cv.md` — plain-text CV stub for extraction tests. + +All files in `examples/` are scanned by the release-safety gate to confirm they contain no real data. diff --git a/examples/synthetic-workspace/sample-cv.md b/examples/synthetic-workspace/sample-cv.md new file mode 100644 index 0000000..350e09e --- /dev/null +++ b/examples/synthetic-workspace/sample-cv.md @@ -0,0 +1,25 @@ +# Jane Doe + +London, United Kingdom · jane.doe@example.com · +44 7700 000000 + +## Summary + +Synthetic CV for Job Hunter pipeline testing. No real data. + +## Experience + +### Senior Engineer — Acme Corp (2020–Present) +- Led backend platform migration to event-driven architecture. +- Reduced p99 latency by 40%. + +### Software Engineer — Beta Inc (2016–2020) +- Built REST APIs serving 10k requests/second. +- Introduced CI/CD pipeline reducing deploy time from hours to minutes. + +## Education + +BSc Computer Science — University of Example (2016) + +## Skills + +Python, TypeScript, PostgreSQL, Docker, Kubernetes, AWS, system design. diff --git a/examples/synthetic-workspace/sample-personal-info-cache.json b/examples/synthetic-workspace/sample-personal-info-cache.json new file mode 100644 index 0000000..5a4e7ff --- /dev/null +++ b/examples/synthetic-workspace/sample-personal-info-cache.json @@ -0,0 +1,13 @@ +{ + "fullName": "Jane Doe", + "email": "jane.doe@example.com", + "phone": "+44 7700 000000", + "location": "London, United Kingdom", + "workAuthorization": "Right to work in UK", + "noticePeriod": "4 weeks", + "currentSalary": 0, + "desiredSalary": 0, + "demographics": { + "gender": "Prefer not to say" + } +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..848e121 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,492 @@ +{ + "name": "job-hunter", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "job-hunter", + "version": "0.0.1", + "dependencies": { + "better-sqlite3": "12.10.0", + "ws": "8.21.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "12.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.10.0.tgz", + "integrity": "sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x || 26.x" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.96.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz", + "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..5ed642d --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "job-hunter", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "Pi Job Hunter — complete skill closure with idempotent installer", + "scripts": { + "test": "node --test test/*.test.mjs", + "test:skills": "node scripts/test-installed-skills.mjs", + "install:global": "node scripts/install.mjs", + "verify-runtime": "node scripts/verify-runtime-dependencies.mjs", + "verify:release": "node scripts/check-release-safety.mjs && node scripts/check-local-profile-leaks.mjs && node scripts/verify-skill-closure.mjs && node scripts/verify-dependency-matrix.mjs && node scripts/check-doc-commands.mjs && node scripts/check-required-doc-sections.mjs", + "rehearse:compose": "bash scripts/rehearse-compose.sh" + }, + "dependencies": { + "better-sqlite3": "12.10.0", + "ws": "8.21.0" + }, + "engines": { + "node": ">=22.0.0" + } +} diff --git a/scripts/check-doc-commands.mjs b/scripts/check-doc-commands.mjs new file mode 100644 index 0000000..cf0bc74 --- /dev/null +++ b/scripts/check-doc-commands.mjs @@ -0,0 +1,85 @@ +import { stat, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** + * Parse README.md for command references and verify each resolves to a + * shipped file or package script. + */ +export async function run(rootDir) { + const findings = []; + const output = []; + const readmePath = join(rootDir, 'README.md'); + + const s = await stat(readmePath).catch(() => null); + if (!s || !s.isFile()) { + findings.push({ type: 'missing-readme', path: 'README.md', detail: 'README.md not found' }); + output.push('[FAIL] missing-readme: README.md — README.md not found'); + return { exitCode: 1, output, findings }; + } + + const content = await readFile(readmePath, 'utf8'); + + // Extract file paths from code blocks and inline code + // Matches: node scripts/foo.mjs, ./scripts/foo.sh, scripts/foo.mjs + const fileRefRe = /(?:^|\s)(?:node |bash |\.\/)?(scripts\/[\w./-]+\.(?:mjs|js|sh))/gm; + const refs = new Set(); + let m; + while ((m = fileRefRe.exec(content)) !== null) { + refs.add(m[1]); + } + + // Also match `npm run AI Architect`, + 'AI Architect', + 'AI Architect', +); +assert.equal(queryTitleBackfill.title, '', 'backfill must not use the search query as the job title evidence'); +assert.equal(classifyListing({ title: queryTitleBackfill.title, descriptionText: queryTitleBackfill.description_text, provisional: false }).label, 'Out of scope'); + +const provisionalLeadership = rowFromResult({ + url: 'https://jobs.lever.co/example/head-ai-engineering', + title: 'Head of AI Engineering', + content: 'Lead hiring, people management, and roadmap ownership for an AI engineering function.', +}, 'AI Architect', 'Ireland'); +assert.ok(provisionalLeadership, 'AI-central provisional out-of-scope stubs must survive for JD backfill'); +assert.equal(provisionalLeadership.role_family_inferred, 'Out of scope'); +assert.equal(classifyListing({ + title: 'Head of AI Engineering', + descriptionText: 'Own technical direction, architecture quality, engineering standards, and platform strategy for an AI platform. Manage 10 engineers across 2 teams.', + provisional: false, +}).label, 'Leadership progression'); + +const provisionalData = rowFromResult({ + url: 'https://jobs.lever.co/example/data-ai-architect', + title: 'Data & AI Architect', + content: 'Own data modelling, governance, lakehouse, warehouse, Spark, Databricks, and Snowflake.', +}, 'AI Architect', 'Ireland'); +assert.ok(provisionalData, 'Data & AI stubs must survive until the full JD distinguishes stretch from data-dominated scope'); +assert.equal(provisionalData.role_family_inferred, 'Out of scope'); +assert.equal(classifyListing({ + title: 'Data & AI Architect', + descriptionText: 'Design AI system architecture; data governance and warehouse are secondary responsibilities.', + provisional: false, +}).label, 'Data-domain stretch'); + +const newRow = { source: 'external', job_id: 'new-job' }; +const existingRow = { source: 'external', job_id: 'existing-job' }; +const insertDb = { + prepare(sql) { + if (sql.includes('SELECT 1')) return { get: (_source, jobId) => jobId === 'existing-job' ? { 1: 1 } : undefined }; + if (sql.includes('INSERT OR IGNORE')) return { run: () => ({ changes: 1 }) }; + throw new Error(`Unexpected SQL in insert fixture: ${sql}`); + }, + transaction(fn) { return fn; }, +}; +const insertResult = insertDiscoveredRows(insertDb, [existingRow, newRow], false); +assert.equal(insertResult.inserted, 1); +assert.deepEqual(insertResult.insertedRows, [newRow]); + +const fullLeadershipDescription = 'Own technical direction, architecture quality, engineering standards, and platform strategy for an AI platform. Manage 10 engineers across 2 teams. Hire and mentor engineers while leading cross-team production delivery and the technical roadmap.'; +const html = ``; +const server = createServer((req, res) => { + if (req.url === '/fail') { + res.writeHead(503); + res.end('unavailable'); + return; + } + res.writeHead(200, { 'content-type': 'text/html' }); + res.end(html); +}); +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); +try { + const { port } = server.address(); + const updates = []; + const backfillDb = { + prepare(sql) { + if (sql.includes('language_filter_reason = NULL')) return { run: (params) => updates.push({ type: 'success', ...params }) }; + if (sql.includes('language_filter_reason = @reason')) return { run: (params) => updates.push({ type: 'failure', ...params }) }; + throw new Error(`Unexpected SQL in backfill fixture: ${sql}`); + }, + }; + const backfillResult = await backfillRows(backfillDb, { dryRun: false, json: true }, [{ + source: 'external', + job_id: 'backfill-job', + url: `http://127.0.0.1:${port}/head-ai-engineering`, + title: 'Head of AI Engineering', + company: 'Example Co', + searched_keywords: 'AI Architect', + }]); + assert.equal(backfillResult[0].ok, true); + assert.equal(backfillResult[0].role_family_inferred, 'Leadership progression'); + assert.equal(updates[0].type, 'success'); + assert.equal(updates[0].role_family_inferred, 'Leadership progression'); + assert.equal(updates[0].language_filter_reason, undefined); + + const failed = await backfillRows(backfillDb, { dryRun: false, json: true }, [{ + source: 'external', + job_id: 'failed-job', + url: `http://127.0.0.1:${port}/fail`, + title: 'AI Architect', + company: 'Example Co', + searched_keywords: 'AI Architect', + }]); + assert.equal(failed[0].ok, false); + assert.match(updates[1].reason, /^backfill-failed:/); + assert.equal(updates[1].role_family_inferred, undefined, 'transport failures must not become role labels'); +} finally { + await new Promise((resolve) => server.close(resolve)); +} + +console.log('jh-discover tests: PASS'); diff --git a/skills/job-hunter/scripts/test-jh-freshness.mjs b/skills/job-hunter/scripts/test-jh-freshness.mjs new file mode 100644 index 0000000..1bd30ee --- /dev/null +++ b/skills/job-hunter/scripts/test-jh-freshness.mjs @@ -0,0 +1,118 @@ +#!/usr/bin/env node +/** + * Unit tests for the jh-freshness.mjs posting-age filter. + * Rows older than 30 days are excluded by default. Fixtures are synthetic + * absolute-date, relative-date, and missing-signal cases. + */ +import assert from 'node:assert/strict'; +import { resolvePostingAge, classifyFreshness, filterByFreshness } from './jh-freshness.mjs'; + +const NOW = new Date('2026-07-21T12:00:00Z'); + +// ── ISO posting_date: straightforward age math ──────────────────────── +{ + // Synthetic stale posting — posted 2026-03-02, about 141 days old + const resolved = resolvePostingAge('2026-03-02T13:01:27.407Z', '2026-06-07 15:26:36', NOW); + assert.equal(resolved.hasRealSignal, true); + assert.equal(resolved.method, 'posting_date_iso'); + assert.ok(resolved.ageDays > 140 && resolved.ageDays < 142, `expected ~141 days, got ${resolved.ageDays}`); + console.log('✓ ISO posting_date resolves a synthetic stale age (~141d)'); +} + +// ── Relative text anchored to created_at (scrape time) ──────────────── +{ + // LinkedIn "Reposted 4 days ago" scraped 2026-07-20 -> posted ~2026-07-16 + const resolved = resolvePostingAge('London, England, United Kingdom · Reposted 4 days ago · 47 people clicked apply', '2026-07-20 07:47:58', NOW); + assert.equal(resolved.hasRealSignal, true); + assert.equal(resolved.method, 'relative_day'); + assert.ok(resolved.ageDays >= 4 && resolved.ageDays <= 6, `expected ~5 days, got ${resolved.ageDays}`); + console.log('✓ relative "Reposted N days ago" text anchors to created_at correctly'); +} + +{ + const resolved = resolvePostingAge('European Economic Area · Reposted 38 minutes ago · Over 100 people clicked apply', '2026-07-20 08:09:57', NOW); + assert.equal(resolved.hasRealSignal, true); + assert.equal(resolved.method, 'relative_minute'); + console.log('✓ relative minutes-ago text parses'); +} + +{ + const resolved = resolvePostingAge('Posted yesterday', '2026-07-02 05:25:02', NOW); + assert.equal(resolved.hasRealSignal, true); + assert.equal(resolved.method, 'yesterday'); + console.log('✓ "Posted yesterday" resolves via the yesterday branch'); +} + +// ── No real signal at all: created_at is a lower bound, never asserted as fact ── +{ + // Synthetic no-signal job — job_posting_date null, discovered 71 days ago + const resolved = resolvePostingAge(null, '2026-05-11 20:06:45', NOW); + assert.equal(resolved.hasRealSignal, false); + assert.equal(resolved.ageDays, null, 'no fabricated ageDays when there is no real signal'); + assert.ok(resolved.createdAgeDays > 69 && resolved.createdAgeDays < 72); + console.log('✓ no-signal case returns ageDays=null (never fabricates a posting date), exposes createdAgeDays as a lower bound only'); +} + +// ── classifyFreshness: real signal beyond cutoff -> stale ───────────── +{ + const row = { job_posting_date: '2026-03-02T13:01:27.407Z', created_at: '2026-06-07 15:26:36' }; + const c = classifyFreshness(row, 30, NOW); + assert.equal(c.verdict, 'stale'); + console.log('✓ classifyFreshness: 141-day-old real posting date -> stale'); +} + +// ── classifyFreshness: real signal within cutoff -> fresh ───────────── +{ + const row = { job_posting_date: '2026-07-07T13:36:45.14Z', created_at: '2026-07-07 21:01:46' }; + const c = classifyFreshness(row, 30, NOW); + assert.equal(c.verdict, 'fresh'); + console.log('✓ classifyFreshness: 14-day-old real posting date -> fresh'); +} + +// ── classifyFreshness: no signal, but discovery date alone exceeds cutoff -> stale ── +{ + // Example Company "AI Agents Solutions Architect" — no posting_date, discovered 52 days ago + const row = { job_posting_date: null, created_at: '2026-05-30 23:04:50' }; + const c = classifyFreshness(row, 30, NOW); + assert.equal(c.verdict, 'stale', 'discovery date alone is a lower bound; if it already exceeds cutoff, job cannot be fresher'); + assert.equal(c.method, 'no_signal_but_discovery_exceeds_cutoff'); + console.log('✓ classifyFreshness: no signal + discovery date alone > cutoff -> certainly stale (lower-bound logic)'); +} + +// ── classifyFreshness: no signal, discovery date within cutoff -> unverified (kept, flagged) ── +{ + // "AI Systems Architect (Dutch & English)" — no posting_date, discovered 6 days ago + const row = { job_posting_date: null, created_at: '2026-07-15 23:44:28' }; + const c = classifyFreshness(row, 30, NOW); + assert.equal(c.verdict, 'unverified', 'no signal but not provably stale -> unverified, not silently fresh'); + console.log('✓ classifyFreshness: no signal but discovery date within cutoff -> unverified (not asserted fresh)'); +} + +// ── filterByFreshness: buckets a mixed batch correctly, default cutoff 30d ── +{ + const jobs = [ + { title: 'Synthetic stale role (real signal)', job_posting_date: '2026-03-02T13:01:27.407Z', created_at: '2026-06-07 15:26:36' }, + { title: 'Synthetic fresh role', job_posting_date: '2026-07-07T13:36:45.14Z', created_at: '2026-07-07 21:01:46' }, + { title: 'Example Company (no signal, certainly stale by discovery date)', job_posting_date: null, created_at: '2026-05-30 23:04:50' }, + { title: 'AI Systems Architect (no signal, unverified)', job_posting_date: null, created_at: '2026-07-15 23:44:28' }, + ]; + const { fresh, stale, unverified } = filterByFreshness(jobs, { now: NOW }); + assert.equal(fresh.length, 1); + assert.equal(fresh[0].title, 'Synthetic fresh role'); + assert.equal(stale.length, 2, 'both the real-signal-stale and no-signal-certainly-stale jobs land in stale'); + assert.equal(unverified.length, 1); + assert.equal(unverified[0].title, 'AI Systems Architect (no signal, unverified)'); + console.log('✓ filterByFreshness buckets a mixed batch into fresh/stale/unverified correctly'); +} + +// ── Custom cutoff is respected (not hardcoded to 30) ────────────────── +{ + const row = { job_posting_date: '2026-06-23T21:19:08.538Z', created_at: '2026-07-21 05:36:17' }; // ~27-28 days old + const c30 = classifyFreshness(row, 30, NOW); + const c14 = classifyFreshness(row, 14, NOW); + assert.equal(c30.verdict, 'fresh', '27-28 days old passes a 30-day cutoff'); + assert.equal(c14.verdict, 'stale', '27-28 days old fails a stricter 14-day cutoff'); + console.log('✓ cutoffDays is a real parameter, not hardcoded — same job classifies differently at 30d vs 14d'); +} + +console.log('\n── All jh-freshness.mjs tests passed ──\n'); diff --git a/skills/job-hunter/scripts/test-jh-migrate.mjs b/skills/job-hunter/scripts/test-jh-migrate.mjs new file mode 100644 index 0000000..6855dc0 --- /dev/null +++ b/skills/job-hunter/scripts/test-jh-migrate.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { migrate } from './jh-migrate.mjs'; + +const home = process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter'); +const req = createRequire(path.join(home, 'package.json')); +const Database = req('better-sqlite3'); +const root = mkdtempSync(path.join(tmpdir(), 'jh-migrate-test-')); +const dbPath = path.join(root, 'fixture.sqlite'); + +try { + const db = new Database(dbPath); + db.exec(` + CREATE TABLE jobs ( + source TEXT NOT NULL, + job_id TEXT NOT NULL, + application_status TEXT, + applied_at TEXT, + role_family_inferred TEXT, + PRIMARY KEY (source, job_id) + ); + INSERT INTO jobs (source, job_id, application_status, role_family_inferred) + VALUES ('linkedin', 'legacy-1', 'saved', 'ai_architecture'); + `); + db.close(); + + const first = migrate(dbPath); + const second = migrate(dbPath); + assert.equal(first.taxonomyColumnAdded, true); + assert.equal(second.taxonomyColumnAdded, false); + + const check = new Database(dbPath, { readonly: true }); + const columns = check.prepare('PRAGMA table_info(jobs)').all().map((column) => column.name); + assert.equal(columns.filter((name) => name === 'role_taxonomy_version').length, 1); + const legacy = check.prepare('SELECT role_family_inferred, role_taxonomy_version FROM jobs WHERE job_id = ?').get('legacy-1'); + assert.deepEqual(legacy, { role_family_inferred: 'ai_architecture', role_taxonomy_version: null }); + assert.ok(check.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='application_stage_events'").get()); + assert.ok(check.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='jh_meta'").get()); + check.close(); + + console.log('jh-migrate tests: PASS'); +} finally { + rmSync(root, { recursive: true, force: true }); +} diff --git a/skills/job-hunter/scripts/test-jh-report-gate.mjs b/skills/job-hunter/scripts/test-jh-report-gate.mjs new file mode 100644 index 0000000..a696b6d --- /dev/null +++ b/skills/job-hunter/scripts/test-jh-report-gate.mjs @@ -0,0 +1,219 @@ +#!/usr/bin/env node +/** + * Unit tests for jh-report-gate.mjs — remediation R6 final-report gate. + * Uses a temp SQLite DB seeded with jobs/match_results/job_salary_observations; + * does NOT touch ~/.job-hunter/jobhunter.sqlite. + */ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, writeFileSync, unlinkSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { stripPartialSuffix, partialSuffixPath } from './jh-report-gate.mjs'; + +const SCRIPT_DIR = resolve(fileURLToPath(import.meta.url), '..'); +const GATE_SCRIPT = join(SCRIPT_DIR, 'jh-report-gate.mjs'); +const DAY_MS = 86_400_000; +const daysAgo = (days) => new Date(Date.now() - days * DAY_MS).toISOString(); +const today = new Date().toISOString(); + +function makeTempDb() { + const dbPath = join(tmpdir(), `test-report-gate-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.sqlite`); + const schema = ` + CREATE TABLE jobs ( + source TEXT, job_id TEXT, title TEXT, company TEXT, + job_posting_date TEXT, created_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (source, job_id) + ); + CREATE TABLE match_results (search_id TEXT, source TEXT, job_id TEXT, cta TEXT, fit_score REAL); + CREATE TABLE job_salary_observations ( + job_source TEXT, job_id TEXT, observation_id TEXT, + is_posted_salary INTEGER, benchmark_id TEXT, observed_at TEXT, + PRIMARY KEY (job_source, job_id, observation_id) + ); + `; + execFileSync('sqlite3', [dbPath], { input: schema }); + return dbPath; +} + +function sql(dbPath, statement) { + execFileSync('sqlite3', [dbPath], { input: statement }); +} + +function cleanupDb(dbPath) { + try { unlinkSync(dbPath); } catch {} +} + +function runGate(args) { + try { + const out = execFileSync('node', [GATE_SCRIPT, ...args, '--json'], { encoding: 'utf8' }); + return { status: 0, result: JSON.parse(out) }; + } catch (e) { + return { status: e.status, result: e.stdout ? JSON.parse(e.stdout) : null, stderr: e.stderr }; + } +} + +// ── Pure helpers: PARTIAL suffix round-trip ─────────────────────────── +{ + assert.equal(partialSuffixPath('/tmp/report.md'), '/tmp/report.PARTIAL.md'); + assert.equal(partialSuffixPath('/tmp/report.PARTIAL.md'), '/tmp/report.PARTIAL.md', 'idempotent on already-PARTIAL'); + assert.equal(stripPartialSuffix('/tmp/report.PARTIAL.md'), '/tmp/report.md'); + assert.equal(stripPartialSuffix('/tmp/report.md'), '/tmp/report.md', 'no-op on non-PARTIAL'); + console.log('✓ PARTIAL suffix helpers round-trip correctly'); +} + +// ── Gate passes when every Apply row has provenance ─────────────────── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company) VALUES ('linkedin','1','AI Architect','Acme'); + INSERT INTO match_results VALUES ('search-1','linkedin','1','Apply',85); + INSERT INTO job_salary_observations VALUES ('linkedin','1','obs-1',1,NULL,'${today}'); + `); + const { status, result } = runGate(['--search-id', 'search-1', '--db', db]); + assert.equal(status, 0, 'gate exits 0 when all Apply rows have provenance'); + assert.equal(result.passed, true); + assert.equal(result.missingProvenance.length, 0); + cleanupDb(db); + console.log('✓ gate PASSes when every Apply row has salary provenance'); +} + +// ── Gate fails when an Apply row has no observation at all ──────────── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company) VALUES ('linkedin','2','AI Architect','Acme'); + INSERT INTO match_results VALUES ('search-2','linkedin','2','Apply',85); + `); + const { status, result } = runGate(['--search-id', 'search-2', '--db', db]); + assert.equal(status, 1, 'gate exits 1 when an Apply row has no salary observation'); + assert.equal(result.passed, false); + assert.equal(result.missingProvenance.length, 1); + assert.equal(result.missingProvenance[0].job_id, '2'); + cleanupDb(db); + console.log('✓ gate fails when an Apply row has no salary observation at all'); +} + +// ── Estimate with benchmark_id counts as provenance; estimate WITHOUT benchmark_id does not ── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company) VALUES ('linkedin','3','AI Architect','Acme'), ('linkedin','4','AI Architect','Beta'); + INSERT INTO match_results VALUES ('search-3','linkedin','3','Apply',85), ('search-3','linkedin','4','Apply',80); + INSERT INTO job_salary_observations VALUES ('linkedin','3','obs-3',0,'bench-1','${today}'); + INSERT INTO job_salary_observations VALUES ('linkedin','4','obs-4',0,NULL,'${today}'); + `); + const { status, result } = runGate(['--search-id', 'search-3', '--db', db]); + assert.equal(status, 1); + assert.equal(result.missingProvenance.length, 1, 'only the benchmark_id-less estimate is missing provenance'); + assert.equal(result.missingProvenance[0].job_id, '4'); + cleanupDb(db); + console.log('✓ estimate requires a real benchmark_id to count as provenance'); +} + +// ── Skip/Maybe rows never gate the report (only Apply rows matter) ──── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company) VALUES ('linkedin','5','AI Architect','Acme'); + INSERT INTO match_results VALUES ('search-4','linkedin','5','Skip',30); + `); + const { status, result } = runGate(['--search-id', 'search-4', '--db', db]); + assert.equal(status, 0, 'no Apply rows means nothing to gate on'); + assert.equal(result.totalApplyRows, 0); + cleanupDb(db); + console.log('✓ Skip/Maybe rows do not block the gate; only Apply rows are checked'); +} + +// ── Degraded --run source status fails the gate even with clean provenance ── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company) VALUES ('linkedin','6','AI Architect','Acme'); + INSERT INTO match_results VALUES ('search-5','linkedin','6','Apply',85); + INSERT INTO job_salary_observations VALUES ('linkedin','6','obs-6',1,NULL,'${today}'); + `); + const { status, result } = runGate(['--search-id', 'search-5', '--db', db, '--run', 'nonexistent-run-id']); + assert.equal(status, 1, 'gate fails when a named run has no checkpoint (degraded source)'); + assert.equal(result.degradedSources.length, 1); + cleanupDb(db); + console.log('✓ a --run with no/non-ok checkpoint degrades the gate regardless of provenance'); +} + +// ── Report file is renamed to .PARTIAL.md on failure, with findings appended ── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company) VALUES ('linkedin','7','AI Architect','Acme'); + INSERT INTO match_results VALUES ('search-6','linkedin','7','Apply',85); + `); + const reportDir = join(tmpdir(), `gate-report-test-${Date.now()}`); + mkdirSync(reportDir, { recursive: true }); + const reportPath = join(reportDir, 'report.md'); + writeFileSync(reportPath, '# Report\nbody\n'); + + runGate(['--search-id', 'search-6', '--db', db, '--report', reportPath]); + assert.equal(existsSync(reportPath), false, 'original report.md renamed away'); + const partialPath = join(reportDir, 'report.PARTIAL.md'); + assert.equal(existsSync(partialPath), true, 'report.PARTIAL.md created'); + const content = readFileSync(partialPath, 'utf8'); + assert.match(content, /Degraded sources \/ missing provenance/); + assert.match(content, /linkedin:7/); + + cleanupDb(db); + rmSync(reportDir, { recursive: true, force: true }); + console.log('✓ failing gate renames report.md -> report.PARTIAL.md with findings appended'); +} + +// ── Stale posting (>30d, real signal) fails the gate even with clean provenance ── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company,job_posting_date,created_at) + VALUES ('indeed','8','AI Architect','Acme','${daysAgo(60)}','${daysAgo(60)}'); + INSERT INTO match_results VALUES ('search-7','indeed','8','Apply',85); + INSERT INTO job_salary_observations VALUES ('indeed','8','obs-8',1,NULL,'${today}'); + `); + const { status, result } = runGate(['--search-id', 'search-7', '--db', db]); + assert.equal(status, 1, 'gate fails when an Apply row has a real posting date older than the default 30-day cutoff'); + assert.equal(result.stalePostings.length, 1); + assert.equal(result.stalePostings[0].job_id, '8'); + cleanupDb(db); + console.log('✓ gate fails on a >30-day-old real posting date even with clean salary provenance'); +} + +// ── Fresh posting (<30d) passes the freshness check ── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company,job_posting_date,created_at) + VALUES ('indeed','9','AI Architect','Acme','${daysAgo(5)}','${daysAgo(5)}'); + INSERT INTO match_results VALUES ('search-8','indeed','9','Apply',85); + INSERT INTO job_salary_observations VALUES ('indeed','9','obs-9',1,NULL,'${today}'); + `); + const { status } = runGate(['--search-id', 'search-8', '--db', db]); + assert.equal(status, 0, 'a recently posted Apply row with clean provenance passes'); + cleanupDb(db); + console.log('✓ gate PASSes on a fresh (<30d) posting with clean provenance'); +} + +// ── Custom --max-age-days is honored ── +{ + const db = makeTempDb(); + sql(db, ` + INSERT INTO jobs (source,job_id,title,company,job_posting_date,created_at) + VALUES ('indeed','10','AI Architect','Acme','${daysAgo(26)}','${daysAgo(26)}'); + INSERT INTO match_results VALUES ('search-9','indeed','10','Apply',85); + INSERT INTO job_salary_observations VALUES ('indeed','10','obs-10',1,NULL,'${today}'); + `); + // ~26 days old: passes default 30-day cutoff, fails a stricter 14-day cutoff + const { status: status30 } = runGate(['--search-id', 'search-9', '--db', db]); + assert.equal(status30, 0, '~26 days old passes the default 30-day cutoff'); + const { status: status14 } = runGate(['--search-id', 'search-9', '--db', db, '--max-age-days', '14']); + assert.equal(status14, 1, '~26 days old fails a stricter --max-age-days 14'); + cleanupDb(db); + console.log('✓ --max-age-days is a real parameter, not hardcoded to 30'); +} + +console.log('\n── All jh-report-gate.mjs tests passed ──\n'); diff --git a/skills/job-hunter/scripts/test-jh-search.mjs b/skills/job-hunter/scripts/test-jh-search.mjs new file mode 100644 index 0000000..bde0e1e --- /dev/null +++ b/skills/job-hunter/scripts/test-jh-search.mjs @@ -0,0 +1,143 @@ +#!/usr/bin/env node +/** + * Unit tests for jh-search.mjs — the sanctioned single entry point for + * LinkedIn/Indeed searches (remediation R2/R3/R4/R5). + * Pure module surface — no CDP/browser/network. Exercises argument parsing, + * per-country domain/location derivation, and outcome classification. + */ +import assert from 'node:assert/strict'; +import { + parseArgs, + parseRefreshJobIds, + buildLinkedInArgs, + runId, + classifySearchOutcome, + INDEED_DOMAIN_BY_COUNTRY, + LOCATION_BY_COUNTRY, + EXIT, +} from './jh-search.mjs'; + +// ── One source + one country enforced ────────────────────────────── +{ + const opts = parseArgs(['--source', 'indeed', '--country', 'nl']); + assert.equal(opts.source, 'indeed'); + assert.equal(opts.country, 'NL'); + assert.equal(opts.domain, INDEED_DOMAIN_BY_COUNTRY.get('NL')); + assert.equal(opts.location, LOCATION_BY_COUNTRY.get('NL')); + console.log('✓ parseArgs derives Indeed domain + location from country code'); +} + +{ + assert.throws(() => parseArgs(['--source', 'indeed', '--country', 'ZZ']), + /no known Indeed domain/, 'unknown country without --domain throws'); + const explicit = parseArgs(['--source', 'indeed', '--country', 'ZZ', '--domain', 'https://zz.indeed.com']); + assert.equal(explicit.domain, 'https://zz.indeed.com'); + console.log('✓ unknown country requires explicit --domain'); +} + +// ── Query-batch cap: default 3, hard cap 5 ─────────────────────────── +{ + assert.equal(parseArgs(['--source', 'indeed', '--country', 'GB']).maxQueries, 3, 'default max-queries is 3'); + assert.equal(parseArgs(['--source', 'indeed', '--country', 'GB', '--max-queries', '5']).maxQueries, 5); + assert.equal(parseArgs(['--source', 'indeed', '--country', 'GB', '--max-queries', '9']).maxQueries, 5, 'hard cap at 5'); + assert.equal(parseArgs(['--source', 'indeed', '--country', 'GB', '--max-queries', '0']).maxQueries, 3, '0 is falsy, falls back to default 3'); + console.log('✓ --max-queries is bounded to [1,5], default 3'); +} + +// ── LinkedIn requires no --domain ───────────────────────────────────── +{ + const opts = parseArgs(['--source', 'linkedin', '--country', 'IE']); + assert.equal(opts.domain, null); + assert.equal(opts.location, 'Ireland'); + console.log('✓ LinkedIn invocation derives location without requiring a domain'); +} + +// ── LinkedIn refresh IDs are bounded and forwarded ─────────────────── +{ + const opts = parseArgs(['--source', 'linkedin', '--country', 'IE', '--refresh-job-ids', '9000001003,9000001004,9000001003']); + assert.deepEqual(opts.refreshJobIds, ['9000001003', '9000001004']); + assert.deepEqual(parseRefreshJobIds(' 9000001003,9000001004 '), ['9000001003', '9000001004']); + assert.throws(() => parseRefreshJobIds('9000001003,abc'), /invalid LinkedIn job ID/); + assert.throws(() => parseRefreshJobIds(Array.from({ length: 51 }, (_, index) => String(index + 1)).join(',')), /max 50/); + assert.throws(() => parseArgs(['--source', 'indeed', '--country', 'GB', '--refresh-job-ids', '9000001003']), /only for LinkedIn/); + const args = buildLinkedInArgs(opts, '/tmp/out.json', '/tmp/summary.json'); + const refreshIndex = args.indexOf('--refresh-job-ids'); + assert.ok(refreshIndex >= 0); + assert.equal(args[refreshIndex + 1], '9000001003,9000001004'); + console.log('✓ LinkedIn refresh IDs are validated, capped, deduplicated, and forwarded'); +} + +// ── runId is stable across --resume, fresh otherwise ───────────────── +{ + const resumed = parseArgs(['--source', 'linkedin', '--country', 'GB', '--resume', 'linkedin-GB-fixed-id']); + assert.equal(runId(resumed), 'linkedin-GB-fixed-id', '--resume pins the run id'); + const fresh1 = runId(parseArgs(['--source', 'linkedin', '--country', 'GB'])); + assert.match(fresh1, /^linkedin-GB-\d{8}-\d{6}$/, 'fresh run id is source-country-timestamp'); + console.log('✓ runId: --resume pins id, fresh run derives a timestamped id'); +} + +// ── classifySearchOutcome: LinkedIn blocking states map to 'blocked' ── +{ + const opts = { source: 'linkedin' }; + const blockedSummary = { + terminalStatuses: { + searchQueries: [{ query: 'AI Architect', status: 'active_challenge' }], + detailPages: { status: 'healthy' }, + }, + }; + const outcome = classifySearchOutcome(opts, { status: 2 }, blockedSummary); + assert.equal(outcome.verdict, 'blocked'); + assert.match(outcome.reason, /active_challenge/); + console.log('✓ classifySearchOutcome: LinkedIn active_challenge -> blocked verdict'); +} + +{ + const opts = { source: 'linkedin' }; + const rateLimitedDetail = { + terminalStatuses: { + searchQueries: [{ query: 'AI Architect', status: 'healthy' }], + detailPages: { status: 'blocked' }, + }, + }; + const outcome = classifySearchOutcome(opts, { status: 2 }, rateLimitedDetail); + assert.equal(outcome.verdict, 'blocked', 'detail-page block also counts as blocked'); + console.log('✓ classifySearchOutcome: LinkedIn detail-page block -> blocked verdict'); +} + +{ + const opts = { source: 'linkedin' }; + const cancelledOutcome = classifySearchOutcome(opts, { status: 3 }, { terminalStatuses: { searchQueries: [], detailPages: {} } }); + assert.equal(cancelledOutcome.verdict, 'cancelled'); + console.log('✓ classifySearchOutcome: exit 3 -> cancelled verdict'); +} + +// ── classifySearchOutcome: Indeed verification/CAPTCHA text -> blocked ── +{ + const opts = { source: 'indeed', domain: 'https://nl.indeed.com' }; + const spawnResult = { status: 1, stderr: '❌ Indeed verification/CAPTCHA page detected at https://nl.indeed.com/jobs; user intervention required.' }; + const outcome = classifySearchOutcome(opts, spawnResult, null); + assert.equal(outcome.verdict, 'blocked'); + assert.match(outcome.reason, /verification\/CAPTCHA/); + console.log('✓ classifySearchOutcome: Indeed CAPTCHA stderr -> blocked verdict, not retried'); +} + +// ── classifySearchOutcome: non-blocking failures stay 'fatal', never silently retried ── +{ + const opts = { source: 'indeed', domain: 'https://uk.indeed.com' }; + const outcome = classifySearchOutcome(opts, { status: 1, stderr: 'TypeError: cannot read property x' }, null); + assert.equal(outcome.verdict, 'fatal'); + console.log('✓ classifySearchOutcome: ordinary script error -> fatal (not blocked, not auto-retried)'); +} + +// ── Exit code table matches the remediation plan contract ──────────── +{ + assert.equal(EXIT.OK, 0); + assert.equal(EXIT.USAGE, 1); + assert.equal(EXIT.FATAL, 2); + assert.equal(EXIT.BUDGET, 3); + assert.equal(EXIT.BLOCKED, 4); + assert.equal(EXIT.PREFLIGHT, 5); + console.log('✓ EXIT codes match documented contract (0 ok / 1 usage / 2 fatal / 3 budget / 4 blocked / 5 preflight)'); +} + +console.log('\n── All jh-search.mjs tests passed ──\n'); diff --git a/skills/job-hunter/scripts/test-jh-semantic-shadow.mjs b/skills/job-hunter/scripts/test-jh-semantic-shadow.mjs new file mode 100644 index 0000000..386abd6 --- /dev/null +++ b/skills/job-hunter/scripts/test-jh-semantic-shadow.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import http from 'node:http'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { + REQUIRED_CATEGORIES, + cosineSimilarity, + validateFixture, +} from './jh-semantic-shadow.mjs'; + +const root = mkdtempSync(path.join(tmpdir(), 'jh-semantic-shadow-test-')); +const fixturePath = new URL('../fixtures/role-classification-gold-v1.json', import.meta.url).pathname; +const scriptPath = new URL('./jh-semantic-shadow.mjs', import.meta.url).pathname; +const outPath = path.join(root, 'shadow-report.json'); +const copiedDb = path.join(root, 'jobhunter.sqlite'); +const canonicalDb = path.join(process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter'), 'jobhunter.sqlite'); +const sha256 = (file) => createHash('sha256').update(readFileSync(file)).digest('hex'); + +function categoryVector(text) { + const value = text.toLowerCase(); + let index; + if (/bim|revit|construction|structural building/.test(value)) index = 7; + else if (/quota|revenue target|sales pipeline|commercial deals/.test(value)) index = 6; + else if (/backend software engineer|build web services|application features/.test(value)) index = 5; + else if (/lakehouse|snowflake|databricks|data governance/.test(value)) index = 4; + else if (/head of|director of|multiple ai engineering teams|manage three/.test(value)) index = 2; + else if (/forward deployed|staff ai engineer/.test(value)) index = 1; + else if (/customer engineer|solutions engineer/.test(value)) index = 3; + else index = 0; + return Array.from({ length: 8 }, (_, candidate) => candidate === index ? 1 : 0); +} + +function runCli(args) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [scriptPath, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('error', reject); + child.on('close', (code) => code === 0 ? resolve({ stdout, stderr }) : reject(new Error(`shadow CLI exited ${code}: ${stderr}`))); + }); +} + +const server = http.createServer((request, response) => { + let raw = ''; + request.setEncoding('utf8'); + request.on('data', (chunk) => { raw += chunk; }); + request.on('end', () => { + try { + const payload = JSON.parse(raw); + const inputs = Array.isArray(payload.input) ? payload.input : [payload.input]; + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + model: payload.model, + data: inputs.map((text, index) => ({ index, embedding: categoryVector(String(text)) })), + })); + } catch (error) { + response.writeHead(400, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ error: error.message })); + } + }); +}); + +try { + const fixture = validateFixture(JSON.parse(readFileSync(fixturePath, 'utf8'))); + assert.equal(fixture.reviewed, true); + assert.deepEqual(new Set(fixture.cases.map((item) => item.expectedCategory)), REQUIRED_CATEGORIES); + assert.ok(fixture.cases.every((item) => item.reviewReason)); + assert.equal(cosineSimilarity([1, 0], [1, 0]), 1); + assert.equal(cosineSimilarity([1, 0], [0, 1]), 0); + + if (existsSync(canonicalDb)) copyFileSync(canonicalDb, copiedDb); + else writeFileSync(copiedDb, 'portable database sentinel'); + const dbBefore = sha256(copiedDb); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + const endpoint = `http://127.0.0.1:${address.port}/v1/embeddings`; + await runCli([ + '--fixture', fixturePath, + '--endpoint', endpoint, + '--model', 'stub-role-embeddings', + '--threshold', '0.5', + '--margin', '0.1', + '--out', outPath, + ]); + + const report = JSON.parse(readFileSync(outPath, 'utf8')); + assert.equal(report.schemaVersion, 1); + assert.equal(report.mode, 'shadow'); + assert.equal(report.modelId, 'stub-role-embeddings'); + assert.ok(report.taxonomyVersion); + assert.match(report.fixture.sha256, /^[a-f0-9]{64}$/); + assert.equal(report.embeddingDimensions, 8); + assert.deepEqual(report.confusionMatrix, { truePositive: 5, trueNegative: 3, falsePositive: 0, falseNegative: 0 }); + assert.deepEqual(report.unknownTitleRecovery, { total: 3, recovered: 3 }); + assert.equal(report.evaluations.length, 8); + assert.equal(sha256(copiedDb), dbBefore, 'semantic shadow run leaves copied DB byte-identical'); + console.log('jh-semantic-shadow tests: PASS'); +} finally { + await new Promise((resolve) => server.close(() => resolve())); + rmSync(root, { recursive: true, force: true }); +} diff --git a/skills/job-hunter/scripts/test-role-taxonomy.mjs b/skills/job-hunter/scripts/test-role-taxonomy.mjs new file mode 100644 index 0000000..8799992 --- /dev/null +++ b/skills/job-hunter/scripts/test-role-taxonomy.mjs @@ -0,0 +1,400 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { + ROLE_LABELS, + ROLE_TAXONOMY_VERSION, + assertRoleClassification, + classifyRole, + expandRoleQueries, +} from './role-taxonomy.mjs'; + +assert.equal(typeof ROLE_TAXONOMY_VERSION, 'string'); +assert.ok(ROLE_TAXONOMY_VERSION.length > 0, 'taxonomy version is non-empty'); + +assert.deepEqual(ROLE_LABELS, [ + 'Exact architecture', + 'Adjacent technical', + 'Leadership progression', + 'Leadership lateral', + 'Conditional', + 'Data-domain stretch', + 'Out of scope', +]); + +const cases = [ + { + name: 'core AI Architect', + input: { title: 'Senior AI Architect', descriptionText: 'Own the architecture and technical direction for production AI platforms.' }, + label: 'Exact architecture', + }, + { + name: 'core Generative AI Architect', + input: { title: 'Generative AI / LLM Architect', descriptionText: 'Design enterprise generative AI and agentic system architecture.' }, + label: 'Exact architecture', + }, + { + name: 'Applied AI Architect software building is not construction', + input: { + title: 'Applied AI Architect, Digital Natives', + descriptionText: 'Define AI architectural patterns, deploy OpenAI APIs, and build production prototypes, including building and presenting demos.', + }, + label: 'Exact architecture', + }, + { + name: 'technical Forward Deployed Architect is eligible', + input: { title: 'Forward Deployed Architect', descriptionText: 'Design and deliver production LLM systems while owning customer system architecture and implementation decisions.' }, + label: 'Exact architecture', + }, + { + name: 'sales-only Forward Deployed Architect is excluded', + input: { title: 'Forward Deployed Architect', descriptionText: 'Own sales pipeline, customer presentations, revenue targets, and quota without implementation responsibility.' }, + label: 'Out of scope', + }, + { + name: 'strong technical Solutions Engineer is eligible', + input: { title: 'Solutions Engineer — AI', descriptionText: 'Own production AI system architecture, implementation design, cloud integration, and technical decisions.' }, + label: 'Exact architecture', + }, + { + name: 'adjacent Principal AI Engineer with architecture evidence', + input: { title: 'Principal AI Engineer', descriptionText: 'Lead system design, production architecture, and platform decisions for AI services.' }, + label: 'Adjacent technical', + }, + { + name: 'feature-only adjacent title rejected', + input: { title: 'Staff AI Engineer', descriptionText: 'Implement product features and train models; no architecture or platform ownership.' }, + label: 'Out of scope', + }, + { + name: 'architect-named adjacent role still needs technical evidence', + input: { title: 'AI Enablement Architect', descriptionText: 'Coordinate adoption workshops and stakeholder communications; no system design or platform ownership.' }, + label: 'Out of scope', + }, + { + name: 'small technical AI manager is lateral', + input: { title: 'Engineering Manager — AI/ML', descriptionText: 'Manage a team of 4 engineers and own technical direction, architecture quality, and engineering standards.' }, + label: 'Leadership lateral', + }, + { + name: 'eight engineers is progression', + input: { title: 'Engineering Manager — AI Platform', descriptionText: 'Lead 8 engineers across two teams. Own platform strategy, technical direction, hiring, and cross-team architecture.' }, + label: 'Leadership progression', + }, + { + name: 'written eight engineers is progression', + input: { title: 'Engineering Manager — AI Platform', descriptionText: 'Own technical direction for eight engineers across two teams.' }, + label: 'Leadership progression', + }, + { + name: 'cross-team collaboration alone stays lateral', + input: { title: 'Engineering Manager — AI/ML', descriptionText: 'Manage a team of 4 engineers, collaborate cross-team, and own technical direction.' }, + label: 'Leadership lateral', + }, + { + name: 'delivery manager without technical authority rejected', + input: { title: 'Technical Delivery Manager — AI', descriptionText: 'Own milestones, status reporting, ceremonies, and headcount planning for an AI delivery team.' }, + label: 'Out of scope', + }, + { + name: 'AI leadership without technical direction rejected', + input: { title: 'Head of AI Engineering', descriptionText: 'Own hiring, budgets, people development, and delivery roadmap; no architecture or technical decision authority.' }, + label: 'Out of scope', + }, + { + name: 'AI leadership needs technical authority evidence', + input: { title: 'Head of AI Engineering', descriptionText: 'Own hiring, budgets, people development, and delivery roadmap for an AI team.' }, + label: 'Out of scope', + }, + { + name: 'hybrid AI engineering leadership', + input: { title: 'Director of AI Engineering', descriptionText: 'Own architecture standards and technical direction for 12 engineers across multiple teams while remaining hands-on in system design.' }, + label: 'Leadership progression', + check(result) { + assert.equal(result.leadershipEvaluation.hybridArchitectureLeadership, true); + assert.equal(result.leadershipEvaluation.teamScope.engineers, 12); + }, + }, + { + name: 'commercial role with authority is conditional', + input: { title: 'AI Pre-Sales Solutions Architect', descriptionText: 'Lead customer AI system architecture, implementation design, and technical discovery. No quota.' }, + label: 'Conditional', + }, + { + name: 'commercial role without authority is excluded', + input: { title: 'AI Pre-Sales Solutions Architect', descriptionText: 'Own pipeline, customer relationships, presentations, and revenue growth. No architecture or implementation responsibility.' }, + label: 'Out of scope', + }, + { + name: 'quota-led sales is excluded', + input: { title: 'AI Solutions Architect — Quota Carrying', descriptionText: 'Own revenue quota and sales targets.' }, + label: 'Out of scope', + }, + { + name: 'policy-only governance is excluded', + input: { title: 'AI Governance Lead', descriptionText: 'Write responsible AI policy, compliance frameworks, and ethics guidance.' }, + label: 'Out of scope', + }, + { + name: 'technical governance is conditional', + input: { title: 'AI Governance Lead', descriptionText: 'Own implementation of AI governance controls, platform architecture, and technical risk decisions.' }, + label: 'Conditional', + }, + { + name: 'data and AI stretch', + input: { title: 'Data & AI Architect', descriptionText: 'Design AI systems; data modelling and governance are secondary responsibilities.' }, + label: 'Data-domain stretch', + check(result) { + assert.ok(result.dataGap.missingSkills.includes('lakehouse')); + assert.ok(result.dataGap.missingSkills.includes('Databricks')); + assert.ok(result.reason.gaps.includes('lakehouse')); + }, + }, + { + name: 'data and AI present skills are normalized', + input: { title: 'Data & AI Architect', descriptionText: 'Design AI architecture; data modelling and Databricks are secondary responsibilities.' }, + label: 'Data-domain stretch', + check(result) { + assert.deepEqual(result.dataGap.presentSkills, ['data modelling', 'Databricks']); + assert.ok(!result.dataGap.missingSkills.includes('Databricks')); + }, + }, + { + name: 'ordinary AI architect with incidental warehouse integration stays exact', + input: { title: 'AI Architect', descriptionText: 'Design production AI systems that integrate with a data warehouse.' }, + label: 'Exact architecture', + }, + { + name: 'data-dominated AI architect is not ordinary architecture', + input: { title: 'AI Architect', descriptionText: 'Primarily own the data lakehouse, warehouse, Spark, Databricks, Snowflake, data modelling, and data governance platform.' }, + label: 'Out of scope', + }, + { + name: 'generic data architect is excluded', + input: { title: 'Principal Data Architect', descriptionText: 'Own enterprise data architecture and warehouse strategy.' }, + label: 'Out of scope', + }, + { + name: 'software engineer title always excluded', + input: { title: 'Lead Software Engineer — AI Platform', descriptionText: 'Own AI platform architecture and technical direction.' }, + label: 'Out of scope', + }, + { + name: 'scientist titles excluded', + input: { title: 'Research Scientist — Generative AI', descriptionText: 'Research large language models.' }, + label: 'Out of scope', + }, + { + name: 'non-software architect excluded', + input: { title: 'AI Building Architect', descriptionText: 'Design construction and BIM projects.' }, + label: 'Out of scope', + }, + { + name: 'generic architect without AI central rejected', + input: { title: 'Enterprise Architect', descriptionText: 'Own ERP, infrastructure, and business systems architecture.' }, + label: 'Out of scope', + }, + { + name: 'incidental AI mention is not central architecture evidence', + input: { title: 'Enterprise Architect', descriptionText: 'Occasionally advise the AI team while owning ERP architecture.' }, + label: 'Out of scope', + }, + { + name: 'governance architect remains core architecture', + input: { title: 'AI Governance Architect', descriptionText: 'Design responsible AI governance architecture and implementation controls.' }, + label: 'Exact architecture', + }, + { + name: 'core title family coverage', + input: { title: 'Chief AI Architect', descriptionText: 'Own enterprise AI architecture and technical direction.' }, + label: 'Exact architecture', + }, + { + name: 'core AI ML title family coverage', + input: { title: 'Principal Architect, AI/ML', descriptionText: 'Define production machine-learning architecture.' }, + label: 'Exact architecture', + }, + { + name: 'core enterprise title family coverage', + input: { title: 'Enterprise AI Architect', descriptionText: 'Own enterprise AI system architecture.' }, + label: 'Exact architecture', + }, + { + name: 'core solutions title family coverage', + input: { title: 'AI Solutions Architect', descriptionText: 'Design and govern production AI solution architecture.' }, + label: 'Exact architecture', + }, + { + name: 'core platform title family coverage', + input: { title: 'AI Platform Architect', descriptionText: 'Own architecture for a production AI platform.' }, + label: 'Exact architecture', + }, + { + name: 'core infrastructure title family coverage', + input: { title: 'AI Infrastructure Architect', descriptionText: 'Design cloud infrastructure architecture for AI workloads.' }, + label: 'Exact architecture', + }, + { + name: 'core integration title family coverage', + input: { title: 'AI Integration Architect', descriptionText: 'Own integration architecture for enterprise AI systems.' }, + label: 'Exact architecture', + }, + { + name: 'core security architect title family coverage', + input: { title: 'AI Security Architect', descriptionText: 'Design secure AI system architecture and implementation controls.' }, + label: 'Exact architecture', + }, + { + name: 'adjacent lead family coverage', + input: { title: 'AI Engineering Lead', descriptionText: 'Lead technical direction, system design, and production AI delivery.' }, + label: 'Adjacent technical', + }, + { + name: 'lead with people scope becomes leadership progression', + input: { title: 'AI Engineering Lead', descriptionText: 'Manage 10 engineers across two teams and own technical direction and architecture quality.' }, + label: 'Leadership progression', + }, + { + name: 'adjacent applied AI family coverage', + input: { title: 'Lead Applied AI Engineer', descriptionText: 'Own production architecture and platform decisions for applied AI.' }, + label: 'Adjacent technical', + }, + { + name: 'adjacent platform engineer family coverage', + input: { title: 'Staff AI Platform Engineer', descriptionText: 'Lead platform architecture and production system design.' }, + label: 'Adjacent technical', + }, + { + name: 'adjacent ML platform family coverage', + input: { title: 'ML Platform Lead', descriptionText: 'Own ML platform decisions, system design, and production architecture.' }, + label: 'Adjacent technical', + }, + { + name: 'adjacent MLOps family coverage', + input: { title: 'MLOps Architect', descriptionText: 'Design production ML operations architecture.' }, + label: 'Adjacent technical', + }, + { + name: 'adjacent enablement family coverage', + input: { title: 'AI Enablement Architect', descriptionText: 'Define architecture and platform enablement for production AI.' }, + label: 'Adjacent technical', + }, + { + name: 'leadership senior manager family coverage', + input: { title: 'Senior Engineering Manager — AI/ML', descriptionText: 'Own technical direction and architecture quality for 10 engineers.' }, + label: 'Leadership progression', + }, + { + name: 'leadership head platform family coverage', + input: { title: 'Head of AI Platform', descriptionText: 'Own platform strategy and technical direction for a department.' }, + label: 'Leadership progression', + }, + { + name: 'leadership director family coverage', + input: { title: 'AI Engineering Director', descriptionText: 'Own engineering standards, architecture quality, and organizational AI platform strategy.' }, + label: 'Leadership progression', + }, + { + name: 'leadership solutions architecture manager family coverage', + input: { title: 'Solutions Architecture Manager — AI', descriptionText: 'Own AI architecture quality and technical direction for a small team.' }, + label: 'Leadership lateral', + }, + { + name: 'conditional security specialist family coverage', + input: { title: 'Principal AI Security Specialist', descriptionText: 'Own secure AI system architecture and implementation governance.' }, + label: 'Conditional', + }, + { + name: 'conditional responsible AI family coverage', + input: { title: 'Responsible AI Lead', descriptionText: 'Define technical controls, implementation governance, and architecture for responsible AI.' }, + label: 'Conditional', + }, + { + name: 'conditional strategy family coverage', + input: { title: 'AI Strategy Lead', descriptionText: 'Own AI platform architecture, technical roadmap, and implementation decisions.' }, + label: 'Conditional', + }, + { + name: 'strategy leadership remit remains conditional', + input: { title: 'AI Strategy Director', descriptionText: 'Own AI platform architecture and technical roadmap for the organization.' }, + label: 'Conditional', + }, + { + name: 'commercial solutions architecture manager uses conditional remit', + input: { title: 'AI Solutions Architecture Manager — Pre-Sales', descriptionText: 'Own customer AI architecture and implementation design, without quota ownership.' }, + label: 'Conditional', + }, + { + name: 'generic data warehouse architect excluded', + input: { title: 'Data Warehouse Architect', descriptionText: 'Own warehouse architecture and data pipelines.' }, + label: 'Out of scope', + }, + { + name: 'sales executive excluded', + input: { title: 'AI Sales Executive', descriptionText: 'Own revenue targets and customer acquisition.' }, + label: 'Out of scope', + }, + { + name: 'policy-only strategy excluded', + input: { title: 'AI Strategy Lead', descriptionText: 'Write policy and provide non-technical strategic advice.' }, + label: 'Out of scope', + }, + { + name: 'product manager without engineering ownership rejected', + input: { title: 'AI Product Manager', descriptionText: 'Own product roadmap, launches, and stakeholder communications.' }, + label: 'Out of scope', + }, +]; + +for (const testCase of cases) { + const result = assertRoleClassification(classifyRole(testCase.input)); + assert.equal(result.label, testCase.label, testCase.name); + assert.ok(result.confidence >= 0 && result.confidence <= 1, `${testCase.name}: confidence bounds`); + assert.equal(result.reason.queryUsedAsEvidence, false, `${testCase.name}: query excluded from evidence`); + assert.equal(typeof JSON.stringify(result.reason), 'string', `${testCase.name}: reason serializes`); + testCase.check?.(result); +} + +const provisional = assertRoleClassification(classifyRole({ title: 'AI Architect', provisional: true })); +assert.equal(provisional.provisional, true, 'provisional flag is preserved'); +assert.ok(provisional.confidence <= 0.65, 'provisional confidence is capped'); + +const withQuery = classifyRole({ + title: 'Enterprise Architect', + descriptionText: 'Own ERP architecture.', + query: 'AI Architect', +}); +const withoutQuery = classifyRole({ + title: 'Enterprise Architect', + descriptionText: 'Own ERP architecture.', +}); +assert.deepEqual(withQuery, withoutQuery, 'query text never changes classification'); + +const expanded = expandRoleQueries({ + targetRole: 'AI Architect', + similarRoles: ['AI Architect', 'Principal AI Engineer', 'ai architect'], + maxQueries: 8, +}); +assert.equal(expanded.length, 8, 'query expansion respects maxQueries'); +assert.equal(new Set(expanded.map((query) => query.toLowerCase())).size, expanded.length, 'query expansion is deduplicated'); +assert.equal(expanded[0], 'AI Architect', 'target query is first'); +assert.ok(expanded.includes('Principal AI Engineer'), 'similar query is retained'); +const fullExpansion = expandRoleQueries({ targetRole: 'AI Architect', maxQueries: 32 }); +for (const query of [ + 'Applied AI Architect', + 'Forward Deployed Architect', + 'Forward Deployed Engineer', + 'AI Customer Engineer', + 'AI Field Engineer', +]) { + assert.ok(fullExpansion.includes(query), `query expansion includes ${query}`); +} +assert.deepEqual(expandRoleQueries({ targetRole: 'AI Architect', maxQueries: 0 }), [], 'zero query bound returns empty'); +assert.deepEqual(expandRoleQueries({}), [], 'missing query seed returns empty'); +assert.ok(!expandRoleQueries({ targetRole: 'AI Architect', similarRoles: ['Lead Software Engineer — AI', 'Principal Data Architect'], maxQueries: 32 }).some((query) => /software engineer|data architect/i.test(query)), 'forbidden query families are filtered'); + +assert.deepEqual(classifyRole(withQuery), classifyRole(withQuery), 'classification and reason are deterministic'); +assert.throws(() => assertRoleClassification({ ...withQuery, confidence: 2 }), /confidence/); +assert.throws(() => assertRoleClassification({ ...withQuery, label: 'unknown' }), /Unknown/); +assert.throws(() => assertRoleClassification({ ...withQuery, reason: { ...withQuery.reason, evidence: 'not-an-array' } }), /evidence/); + +console.log(`role-taxonomy tests: PASS (${cases.length} table cases)`); diff --git a/skills/job-hunter/scripts/workspace-dependencies.mjs b/skills/job-hunter/scripts/workspace-dependencies.mjs new file mode 100644 index 0000000..d7d08cd --- /dev/null +++ b/skills/job-hunter/scripts/workspace-dependencies.mjs @@ -0,0 +1,11 @@ +import { createRequire } from 'node:module'; +import { join } from 'node:path'; + +/** Load shared runtime packages from the canonical Job Hunter workspace. */ +const jobHunterHome = process.env.JOBHUNTER_HOME || join(process.env.HOME || process.cwd(), '.job-hunter'); +const requireWorkspace = createRequire(join(jobHunterHome, 'package.json')); + +export const WebSocketModule = requireWorkspace('ws'); +export const WebSocket = WebSocketModule.WebSocket || WebSocketModule; +export const WebSocketServer = WebSocketModule.WebSocketServer; +export const Database = requireWorkspace('better-sqlite3'); diff --git a/skills/job-match-scorer/SKILL.md b/skills/job-match-scorer/SKILL.md new file mode 100644 index 0000000..05d7589 --- /dev/null +++ b/skills/job-match-scorer/SKILL.md @@ -0,0 +1,105 @@ +--- +name: job-match-scorer +description: Scores saved jobs against the installing user's CV and local profile, persists fit and blocker evidence, and produces deterministic Apply/Skip recommendations. Use after job search and before salary enrichment or application. +allowed-tools: read bash +--- + +# Job Match Scorer + +Score only from the installing user's local evidence. The repository contains no maintainer CV, work authorization, languages, salary, notice, relocation, or disclosure defaults. + +## Inputs + +Resolve `JOBHUNTER_HOME`, defaulting to `~/.job-hunter`: + +- `CV.docx` +- `personal-info-cache.json` +- `jobhunter.sqlite` + +Read the cache before evaluating languages, location, work authorization, sponsorship, relocation, notice, or other eligibility. Unknown values remain unknown. + +## Workflow + +### 1. Select Candidates + +Score jobs saved by the current search that have a non-empty title and description. Do not rescore an existing result for the same search unless the caller requested it. + +External discoveries require a successfully backfilled description. Report rows with missing descriptions rather than assigning a confident score. + +### 2. Classify the Role + +Use the shared classifier in `../job-hunter/scripts/role-taxonomy.mjs`. Query text is discovery context, not evidence. Listing-only classifications are provisional; refresh from the full job description when possible. + +Keep the classifier label, confidence, reason, and taxonomy version with the result. Generic AI/cloud role taxonomy is product code; user eligibility and experience are runtime data. + +### 3. Extract Requirements + +Separate: + +- mandatory skills and experience; +- preferred or nice-to-have criteria; +- language requirements; +- location and work-mode requirements; +- work-authorization, citizenship, clearance, and sponsorship requirements; +- compensation or travel constraints. + +Do not turn a preferred criterion into a blocker. Preserve the source text supporting every mandatory requirement. + +### 4. Match Against Runtime Evidence + +A criterion is matched only when supported by the CV or explicit cache data. Do not infer a technology, employer, certification, language, citizenship, or authorization from related experience. + +Work-authorization logic is country-specific: + +- read the user's value for the job's country; +- explicit no-sponsorship or citizenship requirements are blockers only when they conflict with that value; +- ambiguous eligibility wording remains unresolved; +- never derive authorization from nationality or residence. + +Mandatory language requirements become blockers only when the user's cache does not support the language. Nice-to-have languages are gaps, not blockers. + +### 5. Calculate the Score + +Use the actual criterion counts produced by the scorer. Recompute every ratio and percentage from those counts before persisting or presenting it. Do not narrate a score separately from the formula. + +The default application handoff threshold is `fit_score >= 60` when no hard blocker exists. Role-family labels and tailoring gaps inform the recommendation but do not silently override the numeric threshold. + +### 6. Persist Evidence + +Store the result in `match_results` with: + +- `fit_score` +- `cta` (`Apply` or `Skip`) +- `stretch_label` +- blocker list +- mandatory criteria found and matched +- missing or unclear must-haves +- tailoring suggestions +- scoring basis and role-classification evidence + +JSON columns must contain stringified JSON arrays or objects, not language-native arrays passed directly to SQLite. + +### 7. Hand Off + +After scoring: + +1. Enrich salary evidence with `salary-calculator`. +2. Exclude stale jobs through the Job Hunter freshness gate. +3. Present every visible row with fit/category, role, salary provenance, posting age, reason/gaps, location/source, and clickable URL. +4. Hand application candidates to `auto-job-application` only after blockers and unknown required answers are resolved. + +## Commands + +The deterministic scorer is `scripts/score_jobs_inline.py`. It reads jobs from JSON and writes scored JSON; callers should use temporary files or pipes and then persist through argv-safe database code. + +Run its focused tests with: + +```bash +python3 scripts/test-score_jobs_inline.py +``` + +## References + +- [Adjacent role threshold policy](references/adjacent-role-threshold-policy.md) +- [Adjacent AI role evidence](references/adjacent-ai-roles-title-gate.md) +- [Application handoff threshold](references/user-threshold-ge60-application-handoff.md) diff --git a/skills/job-match-scorer/references/adjacent-ai-roles-title-gate.md b/skills/job-match-scorer/references/adjacent-ai-roles-title-gate.md new file mode 100644 index 0000000..3177c89 --- /dev/null +++ b/skills/job-match-scorer/references/adjacent-ai-roles-title-gate.md @@ -0,0 +1,5 @@ +# Adjacent AI Role Evidence + +Classify adjacent AI roles from the title plus job-description evidence. Relevant evidence includes technical direction, production architecture, platform ownership, system design, and engineering leadership. + +Listing-only results remain provisional. Refresh the detail page before treating an adjacent role as settled. Do not use the search query as evidence and do not assume the installing user's skills; match requirements against the runtime CV/cache. diff --git a/skills/job-match-scorer/references/adjacent-role-threshold-policy.md b/skills/job-match-scorer/references/adjacent-role-threshold-policy.md new file mode 100644 index 0000000..77c89af --- /dev/null +++ b/skills/job-match-scorer/references/adjacent-role-threshold-policy.md @@ -0,0 +1,7 @@ +# Adjacent Role Threshold Policy + +Adjacent role titles are evidence labels, not automatic vetoes. A job with `fit_score >= 60` remains an application candidate when no hard blocker exists. + +A full description may support adjacent technical or leadership alignment through scope, architecture ownership, platform responsibility, and seniority. Query text alone is never classification evidence. + +Record the adjacent-role reason and tailoring gaps so the user can distinguish a core fit from a stretch application. diff --git a/skills/job-match-scorer/references/user-threshold-ge60-application-handoff.md b/skills/job-match-scorer/references/user-threshold-ge60-application-handoff.md new file mode 100644 index 0000000..62dba3f --- /dev/null +++ b/skills/job-match-scorer/references/user-threshold-ge60-application-handoff.md @@ -0,0 +1,12 @@ +# Application Handoff Threshold + +The default handoff policy is: + +```text +fit_score >= 60 and no hard blocker -> Apply candidate +otherwise -> Skip or resolve the blocker +``` + +The score must be computed from persisted matched/total criterion counts. Hard blockers include only verified conflicts such as mandatory unsupported language, incompatible country-specific work authorization, explicit no-sponsorship requirements, or a clearly out-of-scope role. + +Unknown eligibility is not silently converted into either approval or rejection; surface it for resolution. diff --git a/skills/job-match-scorer/scripts/score_jobs_inline.py b/skills/job-match-scorer/scripts/score_jobs_inline.py new file mode 100644 index 0000000..03000a4 --- /dev/null +++ b/skills/job-match-scorer/scripts/score_jobs_inline.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +"""Inline scoring template for unscored jobs. + +Use this when the unscored queue exceeds 15 jobs (a leaf subagent times out +around 40+ jobs at the 600s leaf budget). The script reads the CV, the +unscored jobs JSON, and the personal-info cache, then emits a JSON array of +score objects in the exact match_results INSERT format. + +Inputs (paths can be overridden via env vars): + $SCORE_CV_PATH default: $PWD/CV.docx + $SCORE_JOBS_PATH default: /tmp/jobs-to-score.json + $SCORE_OUTPUT_PATH default: /tmp/scores.json + $SCORE_CACHE_PATH default: $PWD/personal-info-cache.json + $SCORE_REQUIRE_TITLE default: "0" — retained for CLI compatibility; when + "1", add an adjacent-title tailoring note but never + veto a score-driven Apply decision. + $SCORE_TARGET_ROLE default: first few preferred roles from rolePreferences, + or "" when the cache has no rolePreferences (also used in + the deterministic search ID) + $SCORE_SEARCH_ID optional explicit override. Without it, the script derives + one stable ID from the target role, country scope, CV bytes, + scoring version, and canonical job payload. + +Usage: + python3 score_jobs_inline.py + SCORE_TARGET_ROLE="AI Architect; AI Lead" python3 score_jobs_inline.py +""" +import hashlib +import html +import json +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path + +# --- CV skill inventory (from SKILL.md "For each mandatory requirement") --- +CV_SKILLS = { + "aws", "azure", "cosmos db", "cosmosdb", "blob storage", "azure ai search", + "azure web service", "azure functions", "entra id", "microsoft entra", + "langgraph", "langchain", "rag", "retrieval-augmented generation", + "openai", "ai agents", "agentic ai", "agentic", "prompt engineering", + "anthropic", "claude", "claude code", "github copilot", "copilot", + "java", "spring boot", "python", "typescript", "javascript", + "docker", "kubernetes", "k8s", "kafka", + "microservices", "api-driven design", "api design", "ddd", + "domain-driven design", "event-driven", "event-driven architecture", + "jenkins", "gitlab ci", "github actions", + "sql", "nosql", "dynamodb", "elastic search", "elasticsearch", + "oauth2", "sso", "oidc", "gdpr", + "az-900", "az-305", "safer agilist", "safer", "safe agilist", + "ci/cd", "ci-cd", "continuous integration", "continuous deployment", + "cloud migration", "microservice", "rest", "restful", "openapi", + "azure cloud", "aws lambda", + "ai architect", "solution architect", "cloud architect", "enterprise architect", + "integration architect", "technical architect", "software architect", + "platform architect", "ai engineer", "llm", +} + +NON_CV_SKILLS = { + "databricks", "pyspark", "spark", "mlflow", "terraform", "pulumi", + "salesforce", "servicenow", "mulesoft", "sap", "oracle database", + "snowflake", "airflow", "hadoop", "react", "angular", "node.js", "nodejs", + "go", "golang", "rust", "c++", "c#", ".net", "power bi", "tableau", + "grafana", "prometheus", "istio", "linkerd", "helm", "argocd", + "murex", "calypso", "bloomberg", "figma", "redux", "vue", "vue.js", + "flutter", "swift", "kotlin", "android", "ios", "react native", + "jquery", "sass", "scss", "less", "graphql", "trpc", "deno", "bun", + "postman", "swagger", "fastapi", "django", "flask", "express", + "laravel", "rails", "ruby", "php", "perl", "scala", "groovy", + "r language", "matlab", "sas", "spss", "looker", "metabase", + "datadog", "splunk", "newrelic", "appdynamics", "dynatrace", + "selenium", "cypress", "playwright", "jest", "mocha", "junit", + "testng", "cucumber", "jmeter", "loadrunner", "soapui", + "rest-assured", "pytest", +} + +MANDATORY_MARKERS = re.compile( + r"\b(required|must have|essential|strong background in|strong experience with|" + r"significant experience with|deep experience with|deep knowledge of|" + r"deep understanding of|proven experience in|proven track record with|" + r"demonstrated experience with|hands-on experience with|hands-on expertise in|" + r"expertise in|proficient in|solid experience with|you will work with|" + r"you will use|you will be responsible for|experience with)\b", + re.IGNORECASE, +) +NICE_TO_HAVE_MARKERS = re.compile( + r"\b(nice to have|preferred|bonus|plus|optional|familiarity with|" + r"exposure to|good to have|advantageous)\b", + re.IGNORECASE, +) +BUILDING_ARCH_TITLE = re.compile( + r"\b(architekt|architecte|architetto|bim|riba|part ii|revit|autocad|" + r"construction architect|building architect|innenarchitekt|" + r"architecte d.int.rieur|architetto d.interni|interior architect|" + r"draftsman|zeichner|bauleiter|praktikant|construction manager)\b", + re.IGNORECASE, +) +# Backward-compatible title matcher used when the cache has no usable taxonomy. +FALLBACK_TITLE_MATCH = re.compile( + r"\b(ai architect|solution architect|cloud architect|enterprise architect|" + r"integration architect|technical architect|software architect|" + r"platform architect|ai lead|head of ai|ai engineer|principal architect|" + r"lead architect|senior architect|chief architect|pre.?sales.*architect|" + r"data architect|gen.*ai architect|generative ai architect|" + r"agentic.*ai architect|ai knowledge architect|ai solution architect|" + r"ai strategy|head of architecture|director of architecture|ai principal)\b", + re.IGNORECASE, +) +LANG_PATTERNS = { + "German": re.compile(r"\b(german|deutsch|fluent in german|german required)\b", re.IGNORECASE), + "French": re.compile(r"\b(french|français|fluent in french|french required)\b", re.IGNORECASE), + "Spanish": re.compile(r"\b(spanish|español|fluent in spanish)\b", re.IGNORECASE), + "English": re.compile(r"\b(english|fluent in english)\b", re.IGNORECASE), + "Italian": re.compile(r"\b(italian|italiano|fluent in italian)\b", re.IGNORECASE), +} +USER_SPOKEN = {"English", "Italian"} +CLASSIFIER_CLI = Path(__file__).resolve().parents[2] / "job-hunter" / "scripts" / "role-classifier-cli.mjs" + + +def load_role_preferences() -> dict | None: + """Load the optional machine-readable role taxonomy from the canonical cache.""" + home = Path(os.environ.get("JOBHUNTER_HOME") or (Path.home() / ".job-hunter")).expanduser() + try: + with (home / "personal-info-cache.json").open(encoding="utf-8") as handle: + cache = json.load(handle) + preferences = cache.get("rolePreferences") + return preferences if isinstance(preferences, dict) else None + except (OSError, TypeError, ValueError): + return None + + +def _role_titles(preferences: dict | None) -> list[str]: + if not preferences: + return [] + titles: list[str] = [] + groups = [ + preferences.get("preferredPrimaryRoles"), + (preferences.get("adjacentRoles") or {}).get("adjacentTechnicalLeadership"), + (preferences.get("adjacentRoles") or {}).get("leadershipProgression"), + ] + for group in groups: + if not isinstance(group, list): + continue + for title in group: + if isinstance(title, str) and title.strip() and title.strip().lower() not in { + existing.lower() for existing in titles + }: + titles.append(title.strip()) + return titles + + +def build_title_match(preferences: dict | None) -> re.Pattern[str]: + """Build a title matcher from cached role titles, allowing spacing or hyphens.""" + patterns = [] + for title in _role_titles(preferences): + words = re.findall(r"[a-z0-9]+", title.lower()) + if words: + patterns.append(r"\b" + r"[\s\-\u2013\u2014/,]+".join(re.escape(word) for word in words) + r"\b") + return re.compile("(?:" + "|".join(patterns) + ")", re.IGNORECASE) if patterns else FALLBACK_TITLE_MATCH + + +ROLE_PREFERENCES = load_role_preferences() +TITLE_MATCH = build_title_match(ROLE_PREFERENCES) if ROLE_PREFERENCES else FALLBACK_TITLE_MATCH +PREFERRED_PRIMARY_ROLES = ( + ROLE_PREFERENCES.get("preferredPrimaryRoles", []) + if ROLE_PREFERENCES and isinstance(ROLE_PREFERENCES.get("preferredPrimaryRoles"), list) + else [] +) +DEFAULT_TARGET_ROLE = "; ".join( + role for role in PREFERRED_PRIMARY_ROLES[:3] if isinstance(role, str) and role.strip() +) if PREFERRED_PRIMARY_ROLES else "" + + +def job_description(job: dict) -> str: + """Return JD evidence without falling back to query/search metadata.""" + for field in ("description", "description_text"): + value = job.get(field) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def classify_jobs(jobs: list[dict]) -> list[dict]: + """Classify a batch through the shared JavaScript taxonomy.""" + if not jobs: + return [] + payload = [ + { + "title": job.get("title") or "", + "descriptionText": job_description(job), + "jobFunction": job.get("jobFunction") or job.get("job_function") or "", + "industries": job.get("industries") or "", + } + for job in jobs + ] + if not CLASSIFIER_CLI.exists(): + raise RuntimeError(f"Shared role classifier CLI not found: {CLASSIFIER_CLI}") + completed = subprocess.run( + ["node", str(CLASSIFIER_CLI)], + input=json.dumps(payload, sort_keys=True, separators=(",", ":")), + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout).strip() + raise RuntimeError(f"Role classifier bridge failed: {detail}") + try: + results = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("Role classifier bridge returned invalid JSON") from exc + if not isinstance(results, list) or len(results) != len(jobs): + raise RuntimeError("Role classifier bridge returned the wrong batch length") + return results + + +def extract_cv_text(path: str) -> str: + with zipfile.ZipFile(path) as z: + data = z.read("word/document.xml").decode("utf-8", "ignore") + data = re.sub(r"]*/>", " ", data) + data = re.sub(r"", "\n", data) + data = re.sub(r"<[^>]+>", "", data) + return html.unescape(data) + + +def extract_mandatory_skills(desc: str) -> list[str]: + if not desc: + return [] + skills: list[str] = [] + for sent in re.split(r"(?<=[.!?;])\s+|\n", desc): + if not sent.strip(): + continue + has_mandatory = bool(MANDATORY_MARKERS.search(sent)) + if not has_mandatory: + continue + sent_lower = sent.lower() + candidates: set[str] = set() + for skill in CV_SKILLS | NON_CV_SKILLS: + if re.search(r"\b" + re.escape(skill) + r"\b", sent_lower): + candidates.add(skill) + for m in re.finditer( + r"experience (?:with|in)\s+([a-zA-Z][\w\s./+#-]{1,40}?)(?=[,;.]|\sand\s|\sor\s|$)", + sent, re.IGNORECASE, + ): + tok = m.group(1).strip().lower() + if 2 < len(tok) < 50: + candidates.add(tok) + for m in re.finditer( + r"knowledge of\s+([a-zA-Z][\w\s./+#-]{1,40}?)(?=[,;.]|\sand\s|\sor\s|$)", + sent, re.IGNORECASE, + ): + tok = m.group(1).strip().lower() + if 2 < len(tok) < 50: + candidates.add(tok) + for c in candidates: + if c not in skills: + skills.append(c) + return skills + + +def check_cv_has_skill(skill: str) -> bool: + s = skill.lower().strip() + if s in CV_SKILLS: + return True + for cv_skill in CV_SKILLS: + if s in cv_skill or cv_skill in s: + return True + return False + + +def detect_language_blocker(desc: str) -> bool: + if not desc: + return False + for lang, pattern in LANG_PATTERNS.items(): + if lang in USER_SPOKEN: + continue + if not pattern.search(desc): + continue + for sent in re.split(r"(?<=[.!?;])\s+|\n", desc): + if pattern.search(sent): + if MANDATORY_MARKERS.search(sent) and not NICE_TO_HAVE_MARKERS.search(sent): + return True + if re.search(r"\b(must|required|essential|fluent|proficient|mandatory).*" + + pattern.pattern, sent, re.IGNORECASE): + return True + return False + + +def role_reason_text(classification: dict) -> str: + reason = classification.get("reason") or {} + parts = [reason.get("summary") or classification.get("label") or "Role classification unavailable"] + evidence = reason.get("evidence") or [] + gaps = reason.get("gaps") or [] + if evidence: + parts.append(f"Evidence: {', '.join(evidence)}.") + if gaps: + parts.append(f"Gaps: {', '.join(gaps)}.") + evaluation = classification.get("leadershipEvaluation") or {} + if evaluation.get("isLeadership"): + scope = evaluation.get("teamScope") or {} + technical = evaluation.get("technicalOwnership") or {} + influence = evaluation.get("organizationalInfluence") or {} + growth = evaluation.get("growthPath") or {} + parts.append( + "Leadership evaluation: " + f"team scope engineers={scope.get('engineers')}, teams={scope.get('teams')}, " + f"multiple_teams={scope.get('multipleTeams')}, organizational_remit={scope.get('organizationalRemit')}; " + f"technical_ownership={technical.get('present')}; " + f"organizational_influence={influence.get('present')}; " + f"growth_path={growth.get('present')}; " + f"hands_on_transition_risk={evaluation.get('handsOnTransitionRisk')}; " + f"hybrid_architecture_leadership={evaluation.get('hybridArchitectureLeadership')}." + ) + data_gap = classification.get("dataGap") or {} + if data_gap.get("applies"): + missing = data_gap.get("missingSkills") or [] + if missing: + parts.append(f"Data-domain gaps: {', '.join(missing)}.") + return " ".join(parts) + + +def _append_unique(items: list[str], value: str) -> None: + if value and value not in items: + items.append(value) + + +def scored_search_ids(job: dict) -> set[str]: + """Read optional exported score metadata without requiring a DB write.""" + values = [] + for key in ("scored_search_ids", "existing_search_ids", "match_result_search_ids"): + value = job.get(key) + values.extend(value if isinstance(value, list) else [value] if value else []) + value = job.get("scored_search_id") + if value: + values.append(value) + match_results = job.get("match_results") + if isinstance(match_results, list): + values.extend( + row.get("search_id") for row in match_results + if isinstance(row, dict) and row.get("search_id") + ) + elif isinstance(match_results, dict): + values.extend( + row.get("search_id") for row in match_results.values() + if isinstance(row, dict) and row.get("search_id") + ) + return {str(value) for value in values if value} + + +def jobs_unscored_for_search(jobs: list[dict], search_id: str) -> list[dict]: + """Keep rows without a result for this search, not merely without history.""" + return [job for job in jobs if search_id not in scored_search_ids(job)] + + +def score_job( + job: dict, + require_title_match: bool = False, + search_id: str | None = None, + role_classification: dict | None = None, +) -> dict: + title = (job.get("title") or "").strip() + desc = job_description(job) + classification = role_classification or classify_jobs([job])[0] + role_label = classification["label"] + role_reason = role_reason_text(classification) + # jobs.role_family_reason is an existing structured TEXT field populated by + # discovery with the shared classifier's reason object. Keep that shape so + # scorer output can flow through the existing jobs-field integration path. + role_reason_json = json.dumps(classification.get("reason") or {}, sort_keys=True) + + title_match = bool(TITLE_MATCH.search(title)) + domain_match = bool(re.search( + r"\b(banking|fintech|finance|financial services|insurance|" + r"investment|asset management|wealth|trading|" + r"cloud|azure|aws|microservices|saas|enterprise|" + r"architect|architecture)\b", desc, re.IGNORECASE, + )) or title_match + + mandatory_skills = extract_mandatory_skills(desc) + mandatory_matched = [s for s in mandatory_skills if check_cv_has_skill(s)] + tech_total = len(mandatory_skills) + tech_matched = len(mandatory_matched) + tech_ratio = tech_matched / tech_total if tech_total else 1.0 + tech_stack_pass = (tech_total == 0) or (tech_ratio >= 0.5) + if tech_total >= 3 and tech_ratio < 0.3: + tech_stack_pass = False + + seniority_match = bool(re.search( + r"\b(architect|principal|lead|head|director|chief|senior|staff)\b", + title, re.IGNORECASE, + )) or title_match + language_pass = not detect_language_blocker(desc) + + blockers: list[str] = [] + if not language_pass: + blockers.append("Language blocker (German/French/Spanish required)") + if BUILDING_ARCH_TITLE.search(title): + blockers.append("Role mismatch (building/construction architect)") + + must_have_matched = sum([ + 1 if title_match else 0, + 1 if domain_match else 0, + 1 if tech_stack_pass else 0, + 1 if seniority_match else 0, + 1 if language_pass else 0, + ]) + fit_score = (must_have_matched / 5) * 100.0 + + if role_label == "Out of scope": + blockers.append(f"Role family blocker (Out of scope): {role_reason}") + has_blocker = bool(blockers) + # The application threshold is score-driven. Adjacent titles are tailoring + # signals only; they do not veto a score of 60 or higher. + cta = "Apply" if (fit_score >= 60 and not has_blocker) else "Skip" + + # Stretch label — DB CHECK values only (see SKILL.md) + if has_blocker: + stretch = "Blocked" + elif must_have_matched == 5 and tech_ratio >= 0.8: + stretch = "Core fit" + elif must_have_matched >= 4 and not has_blocker: + stretch = "Stretch" + elif not title_match: + stretch = "Major domain stretch" + elif not tech_stack_pass and tech_total > 0: + stretch = "Major cloud stretch" + else: + stretch = "Stretch" + + tailoring: list[str] = [] + if require_title_match and not title_match: + tailoring.append(f"Title '{title}' is adjacent to the target role tier — emphasize transferable architecture evidence") + if tech_total > 0 and tech_matched < tech_total: + missing = [s for s in mandatory_skills if s not in mandatory_matched] + tailoring.append(f"CV lacks {len(missing)} of {tech_total} mandatory skills: {', '.join(missing[:5])}") + tailoring_effort = ( + "high" if (tech_total > 0 and tech_matched < tech_total * 0.5) else + "medium" if tech_matched < tech_total else "low" + ) + + matched_must: list[str] = [] + if title_match: matched_must.append("Title alignment") + if domain_match: matched_must.append("Domain overlap (banking/cloud/enterprise)") + if tech_stack_pass: matched_must.append(f"Tech stack: {tech_matched}/{tech_total} mandatory skills matched") + if seniority_match: matched_must.append("Seniority match") + if language_pass: matched_must.append("Language alignment") + missing_must: list[str] = [] + if not title_match: missing_must.append(f"Title '{title}' not strictly AI/Solution/Cloud Architect") + if not tech_stack_pass: + if tech_total > 0: + missing_must.append(f"Tech stack: missing {', '.join([s for s in mandatory_skills if s not in mandatory_matched][:5])}") + else: + missing_must.append("Tech stack: no explicit mandatory skills listed") + if not seniority_match: missing_must.append("Seniority: title suggests IC/mid rather than architect/lead") + if not language_pass: missing_must.append("Language blocker") + + if role_label == "Data-domain stretch": + data_gap = classification.get("dataGap") or {} + data_missing = data_gap.get("missingSkills") or [] + if data_missing: + _append_unique(missing_must, f"Data-domain stretch gaps: {', '.join(data_missing)}") + _append_unique(tailoring, f"Data-domain stretch: explicitly address missing specialist skills ({', '.join(data_missing)})") + if (classification.get("leadershipEvaluation") or {}).get("isLeadership"): + evaluation = classification["leadershipEvaluation"] + scope = evaluation.get("teamScope") or {} + technical = evaluation.get("technicalOwnership") or {} + influence = evaluation.get("organizationalInfluence") or {} + growth = evaluation.get("growthPath") or {} + leadership_fact = ( + "Leadership evaluation: " + f"team scope engineers={scope.get('engineers')}, teams={scope.get('teams')}, " + f"technical ownership={technical.get('present')}, " + f"organizational influence={influence.get('present')}, " + f"growth path={growth.get('present')}, " + f"hands-on transition risk={evaluation.get('handsOnTransitionRisk')}" + ) + _append_unique(missing_must, leadership_fact) + _append_unique(tailoring, leadership_fact) + + has_blocker = bool(blockers) + if has_blocker: + cta = "Skip" + + return { + "search_id": search_id, + "source": job.get("source"), + "job_id": job.get("job_id"), + "fit_score": round(fit_score, 1), + "cta": cta, + "stretch_label": stretch, + "role_family_inferred": role_label, + "role_family_confidence": classification.get("confidence"), + "role_family_reason": role_reason_json, + "must_have_total": 5, + "must_have_matched": must_have_matched, + "tech_stack_total": tech_total, + "tech_stack_matched": tech_matched, + "mandatory_skills_found_json": json.dumps(mandatory_skills), + "mandatory_skills_matched_json": json.dumps(mandatory_matched), + "nice_to_have_total": 0, + "nice_to_have_matched": 0, + "has_language_blocker": 0 if language_pass else 1, + "has_country_mismatch": 0, + "has_work_mode_mismatch": 0, + "tailoring_effort": tailoring_effort, + "matched_must_haves_json": json.dumps(matched_must), + "missing_or_unclear_must_haves_json": json.dumps(missing_must), + "matched_nice_to_haves_json": json.dumps([]), + "tailoring_suggestions_json": json.dumps(tailoring), + "blockers_json": json.dumps(blockers), + } + + +def slug(value: str, fallback: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return normalized[:32] or fallback + + +def deterministic_search_id(jobs: list[dict], cv_path: str, require_title: bool) -> str: + explicit = os.environ.get("SCORE_SEARCH_ID") + if explicit: + return explicit + + target_role = os.environ.get("SCORE_TARGET_ROLE") or DEFAULT_TARGET_ROLE + countries = sorted({ + str(job.get("country_code") or job.get("countryCode") or "").upper() + for job in jobs + if job.get("country_code") or job.get("countryCode") + }) + canonical_jobs = sorted( + ({ + "source": job.get("source"), + "job_id": job.get("job_id"), + "title": job.get("title"), + "company": job.get("company"), + "country_code": job.get("country_code") or job.get("countryCode"), + "description": job.get("description") or job.get("description_text"), + } for job in jobs), + key=lambda job: (str(job["source"] or ""), str(job["job_id"] or "")), + ) + payload = { + "scorer_version": "inline-v2", + "target_role": target_role, + "require_title": require_title, + "cv_sha256": hashlib.sha256(Path(cv_path).read_bytes()).hexdigest(), + "jobs": canonical_jobs, + } + digest = hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + ).hexdigest()[:12] + role_part = slug(target_role, "targeted") + country_part = "-".join(countries).lower() or "global" + return f"score-{role_part}-{country_part}-{digest}" + + +def main(): + cv_path = os.environ.get("SCORE_CV_PATH", str(Path.cwd() / "CV.docx")) + jobs_path = os.environ.get("SCORE_JOBS_PATH", "/tmp/jobs-to-score.json") + output_path = os.environ.get("SCORE_OUTPUT_PATH", "/tmp/scores.json") + require_title = os.environ.get("SCORE_REQUIRE_TITLE", "0") == "1" + + if not Path(cv_path).exists(): + sys.exit(f"CV not found: {cv_path}") + if not Path(jobs_path).exists(): + sys.exit(f"Jobs JSON not found: {jobs_path}") + extract_cv_text(cv_path) # sanity check parse + + jobs = json.loads(Path(jobs_path).read_text()) + if isinstance(jobs, dict) and isinstance(jobs.get("jobs"), list): + jobs = jobs["jobs"] + if not isinstance(jobs, list): + sys.exit("Jobs JSON must contain an array of jobs") + search_id = deterministic_search_id(jobs, cv_path, require_title) + jobs_to_score = jobs_unscored_for_search(jobs, search_id) + classifications = classify_jobs(jobs_to_score) + print(f"Search ID: {search_id}", file=sys.stderr) + print(f"Scoring {len(jobs_to_score)} of {len(jobs)} jobs (require_title={require_title}) ...", file=sys.stderr) + scored = [ + score_job(j, require_title_match=require_title, search_id=search_id, role_classification=classification) + for j, classification in zip(jobs_to_score, classifications) + ] + apply = sum(1 for s in scored if s["cta"] == "Apply") + skip = len(scored) - apply + print(f"Apply: {apply} Skip: {skip}", file=sys.stderr) + print("\nTop Apply jobs:", file=sys.stderr) + by_id = {(j.get("source"), j.get("job_id")): j for j in jobs} + for s in sorted([x for x in scored if x["cta"] == "Apply"], key=lambda x: -x["fit_score"])[:10]: + j = by_id.get((s["source"], s["job_id"]), {}) + print(f" {s['fit_score']:.0f}% | {j.get('title','')} | {j.get('company','')} | {s['stretch_label']}", file=sys.stderr) + Path(output_path).write_text(json.dumps(scored, indent=2)) + print(f"\nWrote {output_path} ({len(scored)} scores)", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/job-match-scorer/scripts/test-score_jobs_inline.py b/skills/job-match-scorer/scripts/test-score_jobs_inline.py new file mode 100644 index 0000000..16dbf1a --- /dev/null +++ b/skills/job-match-scorer/scripts/test-score_jobs_inline.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).with_name("score_jobs_inline.py") +CLI = SCRIPT.parents[2] / "job-hunter" / "scripts" / "role-classifier-cli.mjs" +MODULE_HOME = tempfile.TemporaryDirectory() +os.environ["JOBHUNTER_HOME"] = MODULE_HOME.name +spec = importlib.util.spec_from_file_location("score_jobs_inline", SCRIPT) +scorer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(scorer) + + +def title_match_in_home(home: Path, title: str) -> bool: + env = os.environ.copy() + env["JOBHUNTER_HOME"] = str(home) + probe = ( + "import importlib.util, sys; " + "spec = importlib.util.spec_from_file_location('scorer', sys.argv[1]); " + "module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); " + "raise SystemExit(0 if module.TITLE_MATCH.search(sys.argv[2]) else 1)" + ) + completed = subprocess.run( + [sys.executable, "-c", probe, str(SCRIPT), title], + env=env, + check=False, + ) + return completed.returncode == 0 + + +with tempfile.TemporaryDirectory() as temp_home: + home = Path(temp_home) + (home / "personal-info-cache.json").write_text( + json.dumps({"rolePreferences": {"preferredPrimaryRoles": ["Head of AI Platform"]}}), + encoding="utf-8", + ) + assert title_match_in_home(home, "Head of AI Platform") + +with tempfile.TemporaryDirectory() as empty_home: + assert title_match_in_home(Path(empty_home), "AI Architect") + + +def score(case): + classification = scorer.classify_jobs([case])[0] + return scorer.score_job(case, search_id="score-test", role_classification=classification) + + +cases = [ + { + "name": "cached adjacent title matches", + "job": { + "source": "test", + "job_id": "adjacent-60", + "title": "ML Platform Lead", + "description": "Lead technical direction for ML products.", + }, + "label": "Adjacent technical", + "fit_score": 60.0, + "cta": "Apply", + }, + { + "name": "out of scope is a role-family skip", + "job": { + "source": "test", + "job_id": "out-of-scope", + "title": "Data Scientist", + "description": "Research machine learning models and publish findings.", + }, + "label": "Out of scope", + "cta": "Skip", + "blocker": "Role family blocker (Out of scope)", + }, + { + "name": "data-domain stretch remains eligible and discloses gaps", + "job": { + "source": "test", + "job_id": "data-stretch", + "title": "Data & AI Architect", + "description": "Design AI architecture; data modelling and Databricks are secondary responsibilities.", + }, + "label": "Data-domain stretch", + "cta": "Apply", + "missing": "Data-domain stretch gaps:", + "tailoring": "Data-domain stretch:", + }, + { + "name": "leadership evaluation is carried into score fields", + "job": { + "source": "test", + "job_id": "leadership", + "title": "Engineering Manager — AI Platform", + "description": "Lead 8 engineers across two teams. Own platform strategy, technical direction, hiring, and cross-team architecture.", + }, + "label": "Leadership progression", + "cta": "Apply", + "missing": "Leadership evaluation:", + "tailoring": "Leadership evaluation:", + "reason_field": "technicalOwnership", + }, +] + +for case in cases: + result = score(case["job"]) + if case["name"] == "cached adjacent title matches": + gated_compat = scorer.score_job( + case["job"], require_title_match=True, role_classification=scorer.classify_jobs([case["job"]])[0] + ) + assert gated_compat["fit_score"] == 60.0 + assert gated_compat["cta"] == "Apply" + assert result["role_family_inferred"] == case["label"], case["name"] + if "fit_score" in case: + assert result["fit_score"] == case["fit_score"], case["name"] + assert result["cta"] == case["cta"], case["name"] + if "blocker" in case: + assert any(case["blocker"] in value for value in json.loads(result["blockers_json"])), case["name"] + if "missing" in case: + assert any(case["missing"] in value for value in json.loads(result["missing_or_unclear_must_haves_json"])), case["name"] + if "tailoring" in case: + assert any(case["tailoring"] in value for value in json.loads(result["tailoring_suggestions_json"])), case["name"] + if "reason_field" in case: + role_reason = json.loads(result["role_family_reason"]) + assert case["reason_field"] in role_reason, case["name"] + +fallback = { + "source": "test", + "job_id": "description-fallback", + "title": "Enterprise Architect", + "description": "", + "description_text": "Own ERP architecture.", + "query": "AI Architect", +} +fallback_result = score(fallback) +assert fallback_result["role_family_inferred"] == "Out of scope" +assert "AI Architect" not in fallback_result["role_family_reason"] + +search_id = "score-current" +filter_cases = [ + {"job_id": "other", "scored_search_ids": ["score-other"]}, + {"job_id": "current", "scored_search_ids": [search_id]}, + {"job_id": "nested", "match_results": [{"search_id": search_id}]}, + {"job_id": "new"}, +] +assert [job["job_id"] for job in scorer.jobs_unscored_for_search(filter_cases, search_id)] == ["other", "new"] + +bridge_input = [case["job"] for case in cases] +bridge_payload = [ + {"title": job.get("title", ""), "descriptionText": scorer.job_description(job)} + for job in bridge_input +] +completed = subprocess.run( + ["node", str(CLI)], + input=json.dumps(bridge_payload), + text=True, + capture_output=True, + check=True, +) +cli_results = json.loads(completed.stdout) +python_results = scorer.classify_jobs(bridge_input) +assert [result["label"] for result in python_results] == [result["label"] for result in cli_results] +assert [result["confidence"] for result in python_results] == [result["confidence"] for result in cli_results] +assert [result["reason"] for result in python_results] == [result["reason"] for result in cli_results] + +alias_bridge = subprocess.run( + ["node", str(CLI)], + input=json.dumps([{ + "title": "AI Architect", + "description": "Own production AI architecture.", + "query": "Data Architect", + }]), + text=True, + capture_output=True, + check=True, +) +alias_result = json.loads(alias_bridge.stdout)[0] +assert alias_result["label"] == "Exact architecture" +assert alias_result["reason"]["queryUsedAsEvidence"] is False + +MODULE_HOME.cleanup() +print(f"score_jobs_inline tests: PASS ({len(cases)} table cases + fallback/filter/bridge parity)") diff --git a/skills/linkedin-job-search/SKILL.md b/skills/linkedin-job-search/SKILL.md new file mode 100644 index 0000000..52ad9d9 --- /dev/null +++ b/skills/linkedin-job-search/SKILL.md @@ -0,0 +1,85 @@ +--- +name: linkedin-job-search +description: Searches LinkedIn jobs through the user's authorized Chromium CDP session, extracts job evidence, filters configured blockers, and saves normalized results to the canonical Job Hunter database. Use for LinkedIn-specific search and refresh work. +allowed-tools: read bash +--- + +# LinkedIn Job Search + +Use the logged-in Chromium session on `127.0.0.1:9225`. Do not use raw HTTP, exported cookies, or a second browser for routine search. + +## Normal Entry Point + +Run through the Job Hunter wrapper: + +```bash +node ~/.pi/agent/skills/job-hunter/scripts/jh-search.mjs \ + --source linkedin \ + --country \ + --role "" +``` + +The wrapper performs the doctor check, exact-search-URL CDP preflight, bounded query expansion, checkpointing, and blocker handling described in `../job-hunter/references/search-safety-contract.md`. + +Direct calls to `scripts/search-linkedin-jobs.mjs` are for focused debugging and tests. + +## Workspace + +Resolve `JOBHUNTER_HOME`, defaulting to `~/.job-hunter`. Save to `jobhunter.sqlite`; never resolve job data from the launch directory. + +## Search Contract + +- Run one source and one country per invocation. +- Keep query batches bounded and resumable. +- Probe the exact search URL, not merely an already-open job-detail tab. +- Send CDP keepalives during long browser work. +- Stop on LinkedIn security checks, login walls, CAPTCHA, or verification pages. +- Do not report a blocker as zero results. + +## Extraction + +LinkedIn search pages are client-rendered and may expose IDs through HTML or embedded data even when cards are incomplete. Extract unique job IDs from the authorized browser target, then fetch detail evidence through the same CDP session. + +For each job preserve: + +- source and job ID; +- title, company, location, and posting date; +- description text; +- application URL and external ATS URL when visible; +- applicant count and recruiter evidence when visible; +- language and work-mode evidence; +- extraction timestamp and search ID. + +Query text is not role-classification evidence. Listing-only role classifications remain provisional until detail extraction succeeds. + +## Filters + +Filter only from the installing user's runtime policy: + +- mandatory unsupported language; +- explicit location/work-mode exclusions; +- stale posting cutoff; +- clearly irrelevant role family. + +Do not encode maintainer citizenship, authorization, sponsorship, relocation, salary, or language defaults in this skill. Pass eligibility evidence to `job-match-scorer`. + +## Persistence + +Use `scripts/save-to-sqlite.mjs` for normalized writes. It loads shared packages from the canonical workspace and preserves existing detail evidence on sparse updates. + +Run focused tests with the scripts named `test-*.mjs` under `scripts/` and the taxonomy tests under `tests/`. + +## Blockers and Recovery + +- LinkedIn security or verification page: stop and ask the user to resolve it in Chromium. +- CDP unavailable: run `obscura-mcp-repair` only when the user chose an Obscura path; routine search remains direct Chromium CDP. +- Unclear page state: use `qwen-screenshot-debug` before retrying navigation. +- Application action: hand off to `auto-job-application`; the search skill does not answer screening questions. + +## References + +- [LinkedIn selectors](references/linkedin-selectors.md) +- [Public listing fallback](references/public-listing-fallback.md) +- [International screening evidence](references/us-remote-international-screening.md) +- [Easy Apply interest loop](references/linkedin-easy-apply-interest-loop.md) +- [Profile review privacy](references/linkedin-profile-review-cdp.md) diff --git a/skills/linkedin-job-search/package.json b/skills/linkedin-job-search/package.json new file mode 100644 index 0000000..53b90aa --- /dev/null +++ b/skills/linkedin-job-search/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "better-sqlite3": "^12.9.0" + } +} diff --git a/skills/linkedin-job-search/references/fixtures/job-card-1-expected.json b/skills/linkedin-job-search/references/fixtures/job-card-1-expected.json new file mode 100644 index 0000000..87d919a --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-card-1-expected.json @@ -0,0 +1,16 @@ +{ + "title": "Senior Software Engineer", + "company": "Acme Corp", + "location": "San Francisco, CA", + "applicants": 25, + "description": "We are looking for a Senior Software Engineer with strong experience in distributed systems. Required: Fluent in English. Nice to have: German language skills.", + "applicationLinks": ["https://apply.acme.com/12345"], + "recruiter": "Jane Smith", + "recruiterEmail": "candidate@example.invalid", + "recruiterProfileLink": "https://www.linkedin.com/in/janesmith", + "jobPostingDate": null, + "languageRequirements": { + "required": ["English"], + "niceToHave": ["German"] + } +} \ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-card-1.html b/skills/linkedin-job-search/references/fixtures/job-card-1.html new file mode 100644 index 0000000..b5fb8f5 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-card-1.html @@ -0,0 +1,15 @@ +
+ Senior Software Engineer + Acme Corp + San Francisco, CA + 25 applicants +
+

We are looking for a Senior Software Engineer with strong experience in distributed systems.

+

Required: Fluent in English. Nice to have: German language skills.

+
+ Apply externally +
+ Jane Smith + candidate@example.invalid +
+
\ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-de-expected.json b/skills/linkedin-job-search/references/fixtures/job-detail-de-expected.json new file mode 100644 index 0000000..5117ddb --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-de-expected.json @@ -0,0 +1,16 @@ +{ + "title": "Produktmanager", + "company": "BerlinTech GmbH", + "location": "Berlin, Germany", + "applicants": 8, + "description": "Verantwortlich für die Produktentwicklung und Roadmap. Required: Fluent in German (C1). Must speak German. English is a plus.", + "applicationLinks": ["https://berlintech.apply.com/123"], + "recruiter": "Anna Mueller", + "recruiterEmail": "candidate@example.invalid", + "recruiterProfileLink": "https://www.linkedin.com/in/annamueller", + "jobPostingDate": "Posted on 2025-04-15", + "languageRequirements": { + "required": ["German"], + "niceToHave": ["English"] + } +} \ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-de.html b/skills/linkedin-job-search/references/fixtures/job-detail-de.html new file mode 100644 index 0000000..cfcf984 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-de.html @@ -0,0 +1,18 @@ +
+

Produktmanager

+ BerlinTech GmbH + Berlin, Germany +
8 applicants
+ Posted on 2025-04-15 +
+

Verantwortlich für die Produktentwicklung und Roadmap.

+

Required: Fluent in German (C1). Must speak German. English is a plus.

+
+
+ Apply +
+
+ Anna Mueller + candidate@example.invalid +
+
\ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-en-expected.json b/skills/linkedin-job-search/references/fixtures/job-detail-en-expected.json new file mode 100644 index 0000000..ff0f5f1 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-en-expected.json @@ -0,0 +1,16 @@ +{ + "title": "Data Analyst", + "company": "DataWorks Inc", + "location": "New York, NY", + "applicants": 12, + "description": "Analyze data to drive business decisions. Requirements: Fluent English. No other language required.", + "applicationLinks": [], + "recruiter": "Bob Johnson", + "recruiterEmail": "candidate@example.invalid", + "recruiterProfileLink": "https://www.linkedin.com/in/bobjohnson", + "jobPostingDate": "Posted 2 weeks ago", + "languageRequirements": { + "required": ["English"], + "niceToHave": [] + } +} \ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-en.html b/skills/linkedin-job-search/references/fixtures/job-detail-en.html new file mode 100644 index 0000000..fe1b12a --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-en.html @@ -0,0 +1,18 @@ +
+

Data Analyst

+ DataWorks Inc + New York, NY +
12 applicants
+ Posted 2 weeks ago +
+

Analyze data to drive business decisions.

+

Requirements: Fluent English. No other language required.

+
+
+ Easy Apply +
+
+ Bob Johnson + candidate@example.invalid +
+
\ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-fr-nice-to-have-expected.json b/skills/linkedin-job-search/references/fixtures/job-detail-fr-nice-to-have-expected.json new file mode 100644 index 0000000..b34d8b6 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-fr-nice-to-have-expected.json @@ -0,0 +1,16 @@ +{ + "title": "Marketing Coordinator", + "company": "GlobalBrands", + "location": "Paris, France", + "applicants": 3, + "description": "Coordinate marketing campaigns across European markets. Required: Fluent English. French is a nice to have but not required.", + "applicationLinks": [], + "recruiter": "Claire Dubois", + "recruiterEmail": "candidate@example.invalid", + "recruiterProfileLink": "https://www.linkedin.com/in/clairedubois", + "jobPostingDate": "Posted 1 week ago", + "languageRequirements": { + "required": ["English"], + "niceToHave": ["French"] + } +} \ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-fr-nice-to-have.html b/skills/linkedin-job-search/references/fixtures/job-detail-fr-nice-to-have.html new file mode 100644 index 0000000..0006ac1 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-fr-nice-to-have.html @@ -0,0 +1,18 @@ +
+

Marketing Coordinator

+ GlobalBrands + Paris, France +
3 applicants
+ Posted 1 week ago +
+

Coordinate marketing campaigns across European markets.

+

Required: Fluent English. French is a nice to have but not required.

+
+
+ Easy Apply +
+
+ Claire Dubois + candidate@example.invalid +
+
\ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-no-applicants-expected.json b/skills/linkedin-job-search/references/fixtures/job-detail-no-applicants-expected.json new file mode 100644 index 0000000..b658fd7 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-no-applicants-expected.json @@ -0,0 +1,16 @@ +{ + "title": "Junior Developer", + "company": "StartupCo", + "location": "Remote", + "applicants": null, + "description": "Build and maintain web applications. Requirements: English proficiency. No foreign language requirement.", + "applicationLinks": ["https://startupco.apply.io/apply"], + "recruiter": null, + "recruiterEmail": null, + "recruiterProfileLink": null, + "jobPostingDate": "Posted 3 days ago", + "languageRequirements": { + "required": ["English"], + "niceToHave": [] + } +} \ No newline at end of file diff --git a/skills/linkedin-job-search/references/fixtures/job-detail-no-applicants.html b/skills/linkedin-job-search/references/fixtures/job-detail-no-applicants.html new file mode 100644 index 0000000..b209415 --- /dev/null +++ b/skills/linkedin-job-search/references/fixtures/job-detail-no-applicants.html @@ -0,0 +1,15 @@ +
+

Junior Developer

+ StartupCo + Remote +
+ Posted 3 days ago +
+

Build and maintain web applications.

+

Requirements: English proficiency. No foreign language requirement.

+
+
+ Apply now +
+
+
\ No newline at end of file diff --git a/skills/linkedin-job-search/references/linkedin-easy-apply-interest-loop.md b/skills/linkedin-job-search/references/linkedin-easy-apply-interest-loop.md new file mode 100644 index 0000000..fa249b2 --- /dev/null +++ b/skills/linkedin-job-search/references/linkedin-easy-apply-interest-loop.md @@ -0,0 +1,5 @@ +# LinkedIn Easy Apply Interest Loop + +Easy Apply may return to an interest or review step after an attempted action. Detect the current dialog stage from visible headings and controls rather than assuming navigation succeeded. + +Do not answer screening questions in the search skill. Hand the job to `auto-job-application`, which loads the user's cache and asks for unknown required answers. A repeated dialog with no visible state change is a blocker, not a successful application. diff --git a/skills/linkedin-job-search/references/linkedin-profile-review-cdp.md b/skills/linkedin-job-search/references/linkedin-profile-review-cdp.md new file mode 100644 index 0000000..389d093 --- /dev/null +++ b/skills/linkedin-job-search/references/linkedin-profile-review-cdp.md @@ -0,0 +1,5 @@ +# LinkedIn Profile Review over CDP + +Use the authorized Chromium CDP session when saved-job, login, or regional profile context matters. Inspect only the profile fields needed for the user's request and do not copy profile content into fixtures, skill documentation, or logs. + +Profile review is read-only. Application answers still come from the canonical cache and CV; LinkedIn profile text is not a substitute for work authorization, salary, disclosure, or consent answers. diff --git a/skills/linkedin-job-search/references/linkedin-selectors.md b/skills/linkedin-job-search/references/linkedin-selectors.md new file mode 100644 index 0000000..f23ae16 --- /dev/null +++ b/skills/linkedin-job-search/references/linkedin-selectors.md @@ -0,0 +1,146 @@ +# LinkedIn Job Search — Selectors & Patterns + +> These selectors and patterns were validated against fixture HTML via the TDD test harness (`scripts/test-extraction-and-filter.mjs`). LinkedIn changes their DOM periodically — if extraction fails, update the patterns in `scripts/linkedin-extractor.mjs` and re-run tests. + +## URL Patterns + +| Purpose | URL | +|---------|-----| +| Job search (keyword + location) | `https://www.linkedin.com/jobs/search/?keywords={keywords}&location={location}` | +| Job detail | `https://www.linkedin.com/jobs/view/{jobId}` | +| Login (if session bridge fails) | `https://www.linkedin.com/uas/login` | + +## CSS Selectors (for Obscura `wait` / `click` / `type`) + +### Job Detail Page (current working selectors) + +| Element | Selector | Notes | +|---------|----------|-------| +| Title | `h1.top-card-layout__title` | Large heading at top of detail page | +| Company | `a.topcard__org-name-link` | Link next to company logo | +| Location | `span.topcard__flavor--bullet` | Bullet-separated location text | +| Applicants | `figure.num-applicants__figure` | Contains applicant count text | +| Description | `div.description__text` | Main content block with paragraphs | +| Posting date | `span.job-posted-date` | Text like "Posted 2 weeks ago" | +| Apply button | `.apply-button`, `.apply-link`, `.easy-apply` | | +| Recruiter/hirer | `.hirer-info` > `.hirer-name` | May be absent | +| Recruiter email | `.hirer-email` | Email address in hiring panel | +| Recruiter profile link | `a.hirer-name` | `href` attribute on hirer name link | + +### Job Search Results (still valid) + +| Element | Selector | Notes | +|---------|----------|-------| +| Search keywords input | `#search-jobs-keywords` or `input[aria-label*="keyword"]` | | +| Search location input | `#search-jobs-location` or `input[aria-label*="location"]` | | +| Search submit button | `button[aria-label*="Search"]` or `.jobs-search-box__submit-button` | | +| Job card in results | `.job-card-container` | Each card is a container | +| Job title link | `.job-card-list__title` or `a.job-title` | Contains the job detail URL | +| Company name | `.job-card-list__company-name` | | +| Location | `.job-card-list__location` | | +| Applicant count | `.job-card-list__applicant-count` | May be empty | + +### Deprecated Selectors (kept as fallbacks) + +| Element | Old Selector | New Replacement | +|---------|-------------|----------------| +| Title | `.job-details-jobs-v2__main-job-title` | `h1.top-card-layout__title` | +| Company | `.job-details-jobs-v2__company-name` | `a.topcard__org-name-link` | +| Location | `.job-details-jobs-v2__location` | `span.topcard__flavor--bullet` | +| Applicants | `.job-details-jobs-v2__applicant-count` | `figure.num-applicants__figure` | +| Description | `.show-more-less__description` | `div.description__text` | + +## Extraction Patterns (for Obscura `extract` with JS eval) + +Use these JavaScript expressions with `obscura_browse_session` action `extract` or `obscura_browse_page` with `eval`. + +### Extract all job cards from search results + +```js +// Get all job card containers and extract their data +Array.from(document.querySelectorAll('.job-card-container')).map(card => ({ + title: card.querySelector('.job-card-list__title')?.textContent?.trim(), + company: card.querySelector('.job-card-list__company-name')?.textContent?.trim(), + location: card.querySelector('.job-card-list__location')?.textContent?.trim(), + applicants: card.querySelector('.job-card-list__applicant-count')?.textContent?.trim(), + url: card.querySelector('.job-card-list__title')?.href || card.querySelector('a')?.href, + recruiter: card.querySelector('.hirer-name')?.textContent?.trim(), +})) +``` + +### Extract detail from a single job page (current selectors) + +```js +// Extract all fields from the job detail page using current LinkedIn DOM +({ + title: document.querySelector('h1.top-card-layout__title')?.textContent?.trim(), + company: document.querySelector('a.topcard__org-name-link')?.textContent?.trim(), + location: document.querySelector('span.topcard__flavor--bullet')?.textContent?.trim(), + applicants: document.querySelector('figure.num-applicants__figure')?.textContent?.trim(), + description: document.querySelector('div.description__text')?.textContent?.trim(), + applyLinks: Array.from(document.querySelectorAll('.apply-link, .apply-button')).map(a => a.href || a.textContent), + recruiter: document.querySelector('.hirer-name')?.textContent?.trim(), + jobPostingDate: document.querySelector('span.job-posted-date')?.textContent?.trim(), + recruiterEmail: document.querySelector('.hirer-email')?.textContent?.trim(), + recruiterProfileLink: document.querySelector('a.hirer-name')?.href, +}) +``` + +### Fallback extraction (old selectors — kept for compatibility) + +```js +// Fallback: old LinkedIn DOM selectors +({ + title: document.querySelector('.job-details-jobs-v2__main-job-title')?.textContent?.trim(), + company: document.querySelector('.job-details-jobs-v2__company-name')?.textContent?.trim(), + location: document.querySelector('.job-details-jobs-v2__location')?.textContent?.trim(), + applicants: document.querySelector('.job-details-jobs-v2__applicant-count')?.textContent?.trim(), + description: document.querySelector('.show-more-less__description')?.textContent?.trim(), + applyLinks: Array.from(document.querySelectorAll('.apply-link, .apply-button')).map(a => a.href || a.textContent), + recruiter: document.querySelector('.hirer-name')?.textContent?.trim(), + jobPostingDate: document.querySelector('.job-posted-date')?.textContent?.trim(), + recruiterEmail: document.querySelector('.hirer-email')?.textContent?.trim(), + recruiterProfileLink: document.querySelector('a.hirer-name')?.href, +}) +``` + +## Language Requirement Patterns + +These regex patterns (validated in `scripts/linkedin-extractor.mjs`) detect language requirements in job descriptions: + +### Required language indicators +- `Required: {Language}` +- `Requirements: {Language}` +- `Must speak {Language}` +- `Fluent in {Language}` +- `{Language} is required` +- `Proficient in {Language}` +- `Mandatory: {Language}` +- `Essential: {Language}` + +### Nice-to-have indicators +- `Nice to have: {Language}` +- `{Language} is a plus` +- `{Language} is nice to have` +- `Bonus: {Language}` +- `Preferred: {Language}` +- `Optional: {Language}` +- `Good to have: {Language}` + +### Filter logic (validated in tests) + +A job **passes** the language filter if: +1. No required languages are specified → pass +2. All required languages match the user's specified language → pass +3. Any required language does NOT match → skip + +A "nice to have" language is **not** a requirement — jobs with only nice-to-have languages that differ from the user's language still pass. + +## Known LinkedIn Behaviors + +- Job search results are dynamically loaded — wait for `.job-card-container` to appear after search +- Some job descriptions are truncated with "Show more" — may need to click that button +- "Easy Apply" jobs have an inline application form; external apply links go to the employer's site +- Applicant count may be hidden or show "Be one of the first applicants" +- Recruiter info is often absent or shows "Hiring team" instead of a named person +- LinkedIn may show a login wall if the session cookie is not properly injected diff --git a/skills/linkedin-job-search/references/public-listing-fallback.md b/skills/linkedin-job-search/references/public-listing-fallback.md new file mode 100644 index 0000000..adafecf --- /dev/null +++ b/skills/linkedin-job-search/references/public-listing-fallback.md @@ -0,0 +1,56 @@ +# LinkedIn public listing fallback for exact query searches + +Use this when the normal logged-in LinkedIn CDP runner repeatedly hits CAPTCHA/security pages but the user asks for a simple listing-level search such as `AI Architect AND remote`. + +## When to use + +- The runner logs repeated `[captcha] CAPTCHA detected on search page` and backs off/timeouts. +- The user asked for an exact LinkedIn query and needs candidate listings quickly. +- Full JD extraction/scoring can wait until after the shortlist is identified. + +## Pattern + +1. Open the public LinkedIn jobs search URL in the browser: + +```text +https://www.linkedin.com/jobs/search/?keywords=&location=&f_TPR=r2592000 +``` + +Example: + +```text +https://www.linkedin.com/jobs/search/?keywords=AI%20Architect%20AND%20remote&location=Remote&f_TPR=r2592000 +``` + +2. Dismiss any sign-in modal if visible. If a cookie dialog appears, choose the visible preference needed to expose listings. + +3. Extract listing cards from the DOM rather than relying on the logged-in runner. A useful browser-console expression is: + +```js +Array.from(document.querySelectorAll('main li')).map(li => { + const jobLink = li.querySelector('a[href*="/jobs/view/"]'); + const title = (li.querySelector('h3')?.innerText || jobLink?.innerText || '').trim(); + const company = (li.querySelector('h4')?.innerText || '').trim(); + const txt = (li.innerText || '').split('\n').map(s => s.trim()).filter(Boolean); + const loc = txt.find(s => s && s !== title && s !== company && !/ago$|applicant|benefits|insurance/i.test(s)) || ''; + return { title, company, location: loc, text: txt.slice(0, 8), url: jobLink ? new URL(jobLink.href, location.href).href.split('?')[0] : '' }; +}).filter(x => x.title && x.url).slice(0, 30) +``` + +4. Save listing-level results to SQLite as `source='linkedin'` records with: + +- native LinkedIn numeric job id extracted from `/jobs/view/...-` +- title, company, location, country code if inferable +- `applicationLinks: [url]` +- `descriptionText` and `descriptionRaw` empty if the JD was not fetched +- `languageFilterReason` clearly noting listing-only extraction +- `workModes: [{ mode: 'remote', isPrimary: true }]` only if the query or title/location explicitly indicates remote +- `searchedKeywords` and `searchedLocation` set to the exact query context + +5. Tell the user the result is listing-level only. Do not claim JD/language/application-link extraction succeeded. The next step is detail fetch/backfill, then scoring, then application. + +## Pitfalls + +- Public LinkedIn may show a large count and visible listings even when the logged-in CDP runner is CAPTCHA-blocked. Do not stop at “runner blocked” if a public listing search can satisfy the user's immediate request. +- Boolean-like query strings such as `AI Architect AND remote` may work well enough in LinkedIn's public keyword field, but treat the results as search-engine matches, not guaranteed Boolean semantics. +- Avoid marking public listing fallback results as fully screened. They need JD fetch/backfill before `job-match-scorer` and `auto-job-application` can safely proceed. diff --git a/skills/linkedin-job-search/references/us-remote-international-screening.md b/skills/linkedin-job-search/references/us-remote-international-screening.md new file mode 100644 index 0000000..4e866f6 --- /dev/null +++ b/skills/linkedin-job-search/references/us-remote-international-screening.md @@ -0,0 +1,5 @@ +# International and Remote Screening + +Remote labels do not prove cross-border eligibility. Preserve the job's stated location, residency, citizenship, work-authorization, and sponsorship requirements as evidence for the scorer. + +Do not encode a maintainer's eligibility in the search skill. The scorer reads the installing user's `workAuthorization` cache and treats explicit no-sponsorship or citizenship requirements as blockers when applicable. Ambiguous requirements remain unresolved until verified. diff --git a/skills/linkedin-job-search/schema.sql b/skills/linkedin-job-search/schema.sql new file mode 100644 index 0000000..587c911 --- /dev/null +++ b/skills/linkedin-job-search/schema.sql @@ -0,0 +1,186 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE jobs ( + source TEXT NOT NULL, + job_id TEXT NOT NULL, + url TEXT, + title TEXT, + company TEXT, + description_raw TEXT, + description_text TEXT, + location_raw TEXT, + country_code TEXT, + region TEXT, + city TEXT, + job_posting_date TEXT, + applicants_raw TEXT, + applicants_count INTEGER CHECK (applicants_count IS NULL OR applicants_count >= 0), + application_links_json TEXT, + recruiter TEXT, + recruiter_email TEXT, + recruiter_profile_link TEXT, + role_family_inferred TEXT, + role_family_confidence REAL CHECK ( + role_family_confidence IS NULL + OR (role_family_confidence >= 0 AND role_family_confidence <= 1) + ), + role_family_reason TEXT, + language_filter_reason TEXT, + work_mode_reason TEXT, + searched_keywords TEXT, + searched_location TEXT, + application_status TEXT NOT NULL DEFAULT 'saved' + CHECK (application_status IN ('saved', 'applying', 'applied', 'failed', 'withdrawn')), + applied_at TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source, job_id) +); + +CREATE INDEX IF NOT EXISTS idx_jobs_application_status +ON jobs(application_status, applied_at); + +CREATE TRIGGER jobs_touch_updated_at +AFTER UPDATE ON jobs +FOR EACH ROW +WHEN NEW.updated_at = OLD.updated_at +BEGIN + UPDATE jobs + SET updated_at = CURRENT_TIMESTAMP + WHERE source = NEW.source AND job_id = NEW.job_id; +END; + +CREATE TABLE job_languages ( + source TEXT NOT NULL, + job_id TEXT NOT NULL, + language TEXT NOT NULL, + importance TEXT NOT NULL CHECK (importance IN ('required', 'nice_to_have')), + PRIMARY KEY (source, job_id, language, importance), + FOREIGN KEY (source, job_id) REFERENCES jobs(source, job_id) ON DELETE CASCADE +); + +CREATE TABLE job_work_modes ( + source TEXT NOT NULL, + job_id TEXT NOT NULL, + work_mode TEXT NOT NULL CHECK (work_mode IN ('onsite', 'hybrid', 'remote')), + is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0, 1)), + PRIMARY KEY (source, job_id, work_mode), + FOREIGN KEY (source, job_id) REFERENCES jobs(source, job_id) ON DELETE CASCADE +); + +CREATE TABLE users ( + user_id TEXT PRIMARY KEY, + name TEXT, + default_stretch_tolerance TEXT NOT NULL DEFAULT 'medium' CHECK ( + default_stretch_tolerance IN ('low', 'medium', 'high') + ), + current_country_code TEXT, + current_city TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE user_countries ( + user_id TEXT NOT NULL, + country_code TEXT NOT NULL, + PRIMARY KEY (user_id, country_code), + FOREIGN KEY (user_id) REFERENCES users(user_id) ON DELETE CASCADE +); + +CREATE TABLE user_languages ( + user_id TEXT NOT NULL, + language TEXT NOT NULL, + proficiency TEXT NOT NULL CHECK ( + proficiency IN ('basic', 'conversational', 'professional', 'fluent', 'native') + ), + PRIMARY KEY (user_id, language), + FOREIGN KEY (user_id) REFERENCES users(user_id) ON DELETE CASCADE +); + +CREATE TABLE resumes ( + resume_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + label TEXT, + file_path TEXT NOT NULL, + file_language TEXT, + parsed_text TEXT, + is_default INTEGER NOT NULL DEFAULT 0 CHECK (is_default IN (0, 1)), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(user_id) ON DELETE CASCADE +); + +CREATE TABLE search_profiles ( + search_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + resume_id TEXT NOT NULL, + target_role_input TEXT NOT NULL, + target_role_confirmed TEXT, + role_family_suggestions_json TEXT, + stretch_tolerance_override TEXT CHECK ( + stretch_tolerance_override IS NULL + OR stretch_tolerance_override IN ('low', 'medium', 'high') + ), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(user_id) ON DELETE CASCADE, + FOREIGN KEY (resume_id) REFERENCES resumes(resume_id) ON DELETE CASCADE +); + +CREATE TABLE match_results ( + search_id TEXT NOT NULL, + source TEXT NOT NULL, + job_id TEXT NOT NULL, + fit_score REAL NOT NULL CHECK (fit_score >= 0 AND fit_score <= 100), + cta TEXT NOT NULL CHECK (cta IN ('Apply', 'Maybe', 'Skip')), + stretch_label TEXT NOT NULL CHECK ( + stretch_label IN ('Core fit', 'Stretch', 'Blocked') + ), + must_have_total INTEGER NOT NULL DEFAULT 0 CHECK (must_have_total >= 0), + must_have_matched INTEGER NOT NULL DEFAULT 0 CHECK ( + must_have_matched >= 0 AND must_have_matched <= must_have_total + ), + nice_to_have_total INTEGER NOT NULL DEFAULT 0 CHECK (nice_to_have_total >= 0), + nice_to_have_matched INTEGER NOT NULL DEFAULT 0 CHECK ( + nice_to_have_matched >= 0 AND nice_to_have_matched <= nice_to_have_total + ), + has_language_blocker INTEGER NOT NULL DEFAULT 0 CHECK (has_language_blocker IN (0, 1)), + has_country_mismatch INTEGER NOT NULL DEFAULT 0 CHECK (has_country_mismatch IN (0, 1)), + has_work_mode_mismatch INTEGER NOT NULL DEFAULT 0 CHECK (has_work_mode_mismatch IN (0, 1)), + tailoring_effort TEXT CHECK (tailoring_effort IN ('low', 'medium', 'high')), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + matched_must_haves_json TEXT, + missing_or_unclear_must_haves_json TEXT, + matched_nice_to_haves_json TEXT, + tailoring_suggestions_json TEXT, + blockers_json TEXT, + PRIMARY KEY (search_id, source, job_id), + FOREIGN KEY (search_id) REFERENCES search_profiles(search_id) ON DELETE CASCADE, + FOREIGN KEY (source, job_id) REFERENCES jobs(source, job_id) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX ux_job_work_modes_primary +ON job_work_modes(source, job_id) +WHERE is_primary = 1; + +CREATE UNIQUE INDEX ux_resumes_default_per_user +ON resumes(user_id) +WHERE is_default = 1; + +CREATE INDEX idx_jobs_country_region_city +ON jobs(country_code, region, city); + +CREATE INDEX idx_jobs_role_family +ON jobs(role_family_inferred); + +CREATE INDEX idx_job_languages_language_importance +ON job_languages(language, importance); + +CREATE INDEX idx_job_work_modes_mode +ON job_work_modes(work_mode); + +CREATE INDEX idx_user_languages_language +ON user_languages(language); + +CREATE INDEX idx_match_results_cta_score +ON match_results(cta, fit_score DESC); + +CREATE INDEX idx_match_results_flags +ON match_results(has_language_blocker, has_country_mismatch, has_work_mode_mismatch); diff --git a/skills/linkedin-job-search/scripts/batch-fetch-jds.mjs b/skills/linkedin-job-search/scripts/batch-fetch-jds.mjs new file mode 100644 index 0000000..69288d2 --- /dev/null +++ b/skills/linkedin-job-search/scripts/batch-fetch-jds.mjs @@ -0,0 +1,805 @@ +#!/usr/bin/env node +import { Database } from '../../job-hunter/scripts/workspace-dependencies.mjs'; +/** + * Batch-backfill LinkedIn job descriptions already saved in SQLite. + * + * This is the skill-integrated version of the project-local helper that was + * previously run from a project-local scripts folder. It uses the + * browser-based Obscura CLI text path (not raw HTTP to LinkedIn), extracts the + * JD segment, refreshes language-filter decisions, and updates job_languages. + * + * Usage: + * node ../../linkedin-job-search/scripts/batch-fetch-jds.mjs \ + * --db $PWD/jobhunter.sqlite \ + * --speaks "English,Italian" \ + * --exclude-languages "German,French" \ + * --batch-size 5 --timeout 25 + * + * The --db flag is optional; the script defaults to + * path.join(process.cwd(), 'jobhunter.sqlite') — i.e. the directory + * Pi Agent was launched from. + */ + +import { execFileSync, spawn } from 'node:child_process'; +import path from 'node:path'; + +import { startCdpKeepAlive } from './cdp-keepalive.mjs'; +import { detectLinkedInBlockPage } from './linkedin-page-state.mjs'; + +const JOBHUNTER_HOME = process.env.JOBHUNTER_HOME || path.join(process.env.HOME || process.cwd(), '.job-hunter'); +const DEFAULT_DB = process.env.JOBHUNTER_DB || path.join(JOBHUNTER_HOME, 'jobhunter.sqlite'); +const DEFAULT_OBSCURA_BIN = process.env.OBSCURA_BIN || 'obscura'; +const DEFAULT_PORT = parsePositiveInt(process.env.LINKEDIN_CDP_PORT || process.env.BROWSER_CDP_PORT || process.env.OBSCURA_CDP_PORT || process.env.OBSCURA_PORT, 9225); +const COMMON_OBSCURA_PORTS = [9225, 9222, 9224, 9226, 9227, 9228, 9230]; + +function usage() { + console.log(`Usage: + batch-fetch-jds.mjs [options] + +Purpose: + Backfill description_text/description_raw and language_filter_reason for + LinkedIn jobs already saved in SQLite, using the active browser CDP text + fetch path. This is intended for jobs whose detail fetch timed out or whose + description was missing after a search run. + +Options: + --db SQLite DB (default ${DEFAULT_DB}) + --source job source to process (default linkedin) + --speaks comma-separated languages user speaks (default English,Italian) + --exclude-languages comma-separated required languages that block (default German,French) + --batch-size concurrent detail fetches per batch (default 2) + --limit process at most n jobs + --timeout browser per-page timeout (default 25) + --wait browser wait before dump (default 2) + --keepalive-seconds CDP heartbeat interval; 0 disables (default 15) + --obscura-bin Obscura CLI binary, used only for --start-obscura (default obscura or $OBSCURA_BIN) + --port browser CDP port to use/check (default ${DEFAULT_PORT}) + --obscura-port alias for --port + --start-obscura start 'obscura serve' only if no existing CDP is reachable + --retry-failed also retry prior JD extraction failures + --all process all saved source jobs, not only missing descriptions + --dry-run fetch and classify, but do not write SQLite + --help show this help + +Notes: + - LinkedIn pages are fetched through the existing browser CDP session. + - This script does not use curl/raw HTTP against LinkedIn or spawn per-page Obscura fetch processes. + - It updates normalized job_languages rows to match the refreshed JD parse. +`); +} + +function splitList(value) { + return String(value || '') + .split(/[;,]/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function parsePositiveInt(value, fallback) { + const n = Number.parseInt(value, 10); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +function parseArgs(argv) { + const opts = { + db: DEFAULT_DB, + source: 'linkedin', + speaks: ['English', 'Italian'], + excludeLanguages: ['German', 'French'], + batchSize: 2, + limit: null, + timeout: 25, + wait: 2, + keepAliveSeconds: Math.max(0, Number(process.env.CDP_KEEPALIVE_SECONDS ?? 15) || 0), + obscuraBin: DEFAULT_OBSCURA_BIN, + port: DEFAULT_PORT, + startObscura: false, + retryFailed: false, + all: false, + dryRun: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--help' || arg === '-h') { + usage(); + process.exit(0); + } else if (arg === '--db') { + opts.db = argv[++i]; + } else if (arg === '--source') { + opts.source = argv[++i] || opts.source; + } else if (arg === '--speaks') { + opts.speaks = splitList(argv[++i]); + } else if (arg === '--exclude-languages') { + opts.excludeLanguages = splitList(argv[++i]); + } else if (arg === '--batch-size') { + opts.batchSize = parsePositiveInt(argv[++i], opts.batchSize); + } else if (arg === '--limit') { + opts.limit = parsePositiveInt(argv[++i], opts.limit || 0) || null; + } else if (arg === '--timeout') { + opts.timeout = parsePositiveInt(argv[++i], opts.timeout); + } else if (arg === '--wait') { + opts.wait = parsePositiveInt(argv[++i], opts.wait); + } else if (arg === '--keepalive-seconds') { + opts.keepAliveSeconds = Math.max(0, Number(argv[++i]) || 0); + } else if (arg === '--obscura-bin') { + opts.obscuraBin = argv[++i] || opts.obscuraBin; + } else if (arg === '--port' || arg === '--obscura-port') { + opts.port = parsePositiveInt(argv[++i], opts.port); + } else if (arg === '--start-obscura') { + opts.startObscura = true; + } else if (arg === '--retry-failed') { + opts.retryFailed = true; + } else if (arg === '--all') { + opts.all = true; + } else if (arg === '--dry-run') { + opts.dryRun = true; + } else { + console.error(`Unknown argument: ${arg}`); + usage(); + process.exit(2); + } + } + + if (!opts.speaks.length) { + console.error('At least one spoken language is required via --speaks.'); + process.exit(2); + } + return opts; +} + +function shQuiet(cmd, args, options = {}) { + return execFileSync(cmd, args, { stdio: 'ignore', ...options }); +} + +function readCdpVersion(port) { + try { + const out = execFileSync('curl', ['-sS', '-f', '--max-time', '3', `http://127.0.0.1:${port}/json/version`], { encoding: 'utf8', timeout: 5000 }); + return JSON.parse(out); + } catch { + return null; + } +} + +function uniq(values) { + return [...new Set(values.filter(Boolean))]; +} + +function candidateObscuraPorts(preferred) { + return uniq([ + preferred, + process.env.OBSCURA_CDP_PORT, + process.env.OBSCURA_PORT, + DEFAULT_PORT, + ...COMMON_OBSCURA_PORTS, + ].map((value) => parsePositiveInt(value, null)).filter(Boolean)); +} + +function findExistingObscuraCdp(preferred) { + for (const port of candidateObscuraPorts(preferred)) { + const version = readCdpVersion(port); + if (version?.webSocketDebuggerUrl) return { port, version }; + } + return null; +} + +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function ensureObscura(opts) { + const existing = findExistingObscuraCdp(opts.port); + if (existing) { + opts.port = existing.port; + console.log(`Using existing Docker Chromium/CDP browser on 127.0.0.1:${opts.port}.`); + return opts.port; + } + if (!opts.startObscura) { + const ports = candidateObscuraPorts(opts.port).join(', '); + throw new Error(`Browser CDP is not reachable on checked port(s): ${ports}. Start the Docker Chromium session, pass --port, or use --start-obscura only for the optional legacy fallback.`); + } + + console.log(`Starting Obscura on port ${opts.port}...`); + const child = spawn(opts.obscuraBin, ['serve', '-p', String(opts.port), '--stealth', '--workers', '8'], { + detached: true, + stdio: 'ignore', + }); + child.unref(); + + for (let i = 0; i < 30; i++) { + if (readCdpVersion(opts.port)?.webSocketDebuggerUrl) { + console.log(`Obscura started on 127.0.0.1:${opts.port}.`); + return opts.port; + } + sleepSync(500); + } + throw new Error(`Timed out waiting for Obscura on 127.0.0.1:${opts.port}.`); +} + +function withTimeout(promise, ms, label) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function randomDelayMs(minSeconds = 1, maxSeconds = 2) { + const min = Math.min(minSeconds, maxSeconds); + const max = Math.max(minSeconds, maxSeconds); + return Math.round((min + Math.random() * (max - min)) * 1000); +} + +// ── Anti-detection utilities ────────────────────────────────────────────── + +function jitter(min, max) { + return min + Math.random() * (max - min); +} + +async function humanDelay(context, minMs, maxMs) { + const ms = Math.round(jitter(minMs, maxMs)); + console.log(` [human] ${context} (${(ms / 1000).toFixed(1)}s)`); + await sleep(ms); +} + +const ANTI_DETECT_EXPR = `(() => { + try { + Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + window.chrome = { runtime: {} }; + const origQuery = navigator.permissions.query; + navigator.permissions.query = (p) => p.name === 'notifications' + ? Promise.resolve({ state: Notification.permission }) + : origQuery(p); + Object.defineProperty(navigator, 'plugins', { get: () => ({ length: 5 }) }); + Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); + Object.defineProperty(navigator, 'platform', { get: () => 'MacIntel' }); + return { ok: true }; + } catch(e) { return { ok: false, error: e.message }; } +})()`; + +async function injectAntiDetection(client, sessionId) { + try { + const result = await withTimeout( + client.send('Runtime.evaluate', { + expression: ANTI_DETECT_EXPR, + returnByValue: true, + awaitPromise: true, + }, sessionId), + 5000, + 'inject anti-detect' + ); + return result?.result?.value?.ok === true; + } catch { + return false; + } +} + +async function simulateMouseMovement(client, sessionId) { + const moves = 3 + Math.floor(Math.random() * 4); + const vw = 800 + Math.floor(Math.random() * 400); + const vh = 500 + Math.floor(Math.random() * 300); + for (let i = 0; i < moves; i++) { + const x = Math.floor(jitter(vw * 0.1, vw * 0.9)); + const y = Math.floor(jitter(vh * 0.1, vh * 0.7)); + await client.send('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }, sessionId).catch(() => {}); + await sleep(Math.round(jitter(150, 400))); + } +} + +async function simulateScroll(client, sessionId) { + const scrolls = 3 + Math.floor(Math.random() * 3); + await client.send('Runtime.evaluate', { + expression: `(async () => { + for (let i = 0; i < ${scrolls}; i++) { + const amount = ${200 + Math.floor(Math.random() * 500)}; + window.scrollBy({ top: amount, behavior: 'smooth' }); + await new Promise(r => setTimeout(r, ${200 + Math.floor(Math.random() * 300)})); + } + return 'ok'; + })()`, + awaitPromise: true, + returnByValue: true, + }, sessionId).catch(() => {}); +} + +const detectBlockPage = detectLinkedInBlockPage; + +function backoffDelay(attempt, baseMs = 10000, maxMs = 120000) { + return Math.round(Math.min(baseMs * Math.pow(2, Math.min(attempt, 4)) + jitter(0, 3000), maxMs)); +} + +const rateTracker = { requests: [], windowMs: 60000, maxRequests: 12 }; + +function noteRequest() { + const now = Date.now(); + rateTracker.requests.push(now); + while (rateTracker.requests.length && rateTracker.requests[0] < now - rateTracker.windowMs) { + rateTracker.requests.shift(); + } +} + +async function enforceRateLimit() { + noteRequest(); + while (rateTracker.requests.length > rateTracker.maxRequests) { + const oldest = rateTracker.requests[0]; + const waitMs = oldest + rateTracker.windowMs - Date.now() + 1000; + if (waitMs > 0) { + console.log(` [rate-limit] ${rateTracker.requests.length} requests in window, pausing ${(waitMs / 1000).toFixed(1)}s`); + await sleep(waitMs); + } + const now = Date.now(); + while (rateTracker.requests.length && rateTracker.requests[0] < now - rateTracker.windowMs) { + rateTracker.requests.shift(); + } + } +} + +class CdpClient { + constructor(wsUrl) { + this.wsUrl = wsUrl; + this.seq = 0; + this.pending = new Map(); + } + + async connect() { + this.ws = new WebSocket(this.wsUrl); + await new Promise((resolve, reject) => { + this.ws.onopen = resolve; + this.ws.onerror = reject; + }); + this.ws.onmessage = (ev) => { + const msg = JSON.parse(ev.data); + if (msg.id && this.pending.has(msg.id)) { + const p = this.pending.get(msg.id); + this.pending.delete(msg.id); + msg.error ? p.reject(new Error(JSON.stringify(msg.error))) : p.resolve(msg.result); + } + }; + } + + send(method, params = {}, sessionId) { + return new Promise((resolve, reject) => { + const id = ++this.seq; + this.pending.set(id, { resolve, reject }); + const msg = { id, method, params }; + if (sessionId) msg.sessionId = sessionId; + this.ws.send(JSON.stringify(msg)); + }); + } + + close() { + try { this.ws?.close(); } catch {} + } +} + +async function connectCdp(opts) { + const version = readCdpVersion(opts.port); + if (!version?.webSocketDebuggerUrl) throw new Error(`Browser CDP is not reachable on 127.0.0.1:${opts.port}`); + const client = new CdpClient(version.webSocketDebuggerUrl); + await client.connect(); + return client; +} + +async function mapLimit(items, limit, fn, interDelayMs = 0) { + const out = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length || 1) }, async () => { + while (true) { + const i = next++; + if (i >= items.length) return; + if (interDelayMs > 0 && i > 0) { + const delayMs = Math.round(jitter(interDelayMs * 0.5, interDelayMs * 1.5)); + await sleep(delayMs); + } + out[i] = await fn(items[i], i); + } + }); + await Promise.all(workers); + return out; +} + +async function fetchJobText(job, opts, client) { + const url = job.url || `https://www.linkedin.com/jobs/view/${job.job_id}`; + let targetId; + try { + const timeoutMs = Math.max(5000, Number(opts.timeout || 25) * 1000); + const target = await withTimeout(client.send('Target.createTarget', { url: 'about:blank' }), 10000, 'Target.createTarget'); + targetId = target.targetId; + const attached = await withTimeout(client.send('Target.attachToTarget', { targetId, flatten: true }), 10000, 'Target.attachToTarget'); + const sessionId = attached.sessionId; + await withTimeout(client.send('Page.enable', {}, sessionId), 8000, 'Page.enable'); + await withTimeout(client.send('Runtime.enable', {}, sessionId), 8000, 'Runtime.enable'); + await injectAntiDetection(client, sessionId); + await humanDelay('batch-detail nav', 2000, 5000); + const nav = await withTimeout(client.send('Page.navigate', { url, referrer: 'https://www.linkedin.com/jobs/search/' }, sessionId), timeoutMs, 'Page.navigate'); + if (nav.errorText) throw new Error(nav.errorText); + + await simulateMouseMovement(client, sessionId); + await simulateScroll(client, sessionId); + await humanDelay('batch-detail read', 3000, 6000); + + let text = ''; + for (let i = 0; i < 18; i++) { + if (i > 0) await humanDelay('batch-detail retry', 2000, 4000); + const result = await withTimeout(client.send('Runtime.evaluate', { + expression: 'document.body ? document.body.innerText : document.documentElement.innerText || ""', + returnByValue: true, + awaitPromise: true, + }, sessionId), timeoutMs, 'Runtime.evaluate'); + text = String(result.result?.value || ''); + const block = detectBlockPage(text); + if (block.blocked) { + return { ok: true, text, url, blocked: true, isCaptcha: block.isCaptcha, blockReason: block.reason }; + } + if (text.includes('Report this job') || text.includes('Seniority level') || text.includes('About the job')) break; + } + if (!text) throw new Error('empty detail page text'); + return { ok: true, text, url, blocked: false, isCaptcha: false, blockReason: null }; + } catch (e) { + return { ok: false, text: '', url, error: e.message, blocked: false, isCaptcha: false, blockReason: null }; + } finally { + if (targetId) await client.send('Target.closeTarget', { targetId }).catch(() => {}); + } +} + +function publicTextLines(text) { + const lines = []; + for (const part of String(text || '').split(/\n+/)) { + const line = part.trim().replace(/\s+/g, ' '); + if (!line) continue; + if (line === lines[lines.length - 1]) continue; + if (/^window\.__/.test(line)) continue; + lines.push(line); + } + return lines; +} + +function extractDescriptionFromLines(lines) { + const start = lines.findIndex((line) => line === 'Report this job'); + const end = lines.findIndex((line, index) => index > start && line === 'Seniority level'); + let descLines = start >= 0 && end > start ? lines.slice(start + 1, end) : []; + if (!descLines.length) { + const aboutStart = lines.findIndex((line) => /^About the job$/i.test(line)); + const aboutEnds = [/^Show less$/i, /^Show more$/i, /^Seniority level$/i, /^Employment type$/i, /^Job function$/i, /^Industries$/i, /^Skills$/i, /^Similar jobs$/i, /^People also viewed$/i, /^Set alert/i]; + const aboutEnd = lines.findIndex((line, index) => index > aboutStart && aboutEnds.some((re) => re.test(line))); + if (aboutStart >= 0) descLines = lines.slice(aboutStart + 1, aboutEnd > aboutStart ? aboutEnd : undefined); + } + + const actualStart = descLines.findIndex((line) => /^(About|Responsibilities|Requirements|Qualifications|Job description|The Role|Role Overview|Your role|What you|We are|We’re|Our client|Company Description|Minimum qualifications|Basic qualifications|Overview|Locations?|Aufgaben|Profil|Deine|Ihre|Your mission)/i.test(line)); + if (actualStart > 0) descLines = descLines.slice(actualStart); + + return descLines.join('\n').trim(); +} + +function extractDescriptionByMarkers(text) { + const markers = [ + ['Report this job', 'Seniority level'], + ['Report this job', 'Employment type'], + ['Report this job', 'Show more'], + ['About this job', 'Show more'], + ['About this job', 'Show less'], + ['About the job', 'Show more'], + ['About the job', 'Show less'], + ['Job description', 'Show more'], + ]; + + for (const [startMarker, endMarker] of markers) { + const start = text.indexOf(startMarker); + const end = text.indexOf(endMarker, start + startMarker.length); + if (start >= 0 && end > start) { + const jd = cleanDescription(text.slice(start + startMarker.length, end)); + if (jd.length > 100) return jd; + } + } + return null; +} + +function cleanDescription(text) { + return String(text || '') + .replace(/\b(?:Join now|Sign in|Apply|Easy Apply|Save)\b/g, ' ') + .replace(/\n{3,}/g, '\n\n') + .replace(/[ \t]{2,}/g, ' ') + .trim(); +} + +function extractJD(text) { + if (!text || text.length < 100) return null; + const lines = publicTextLines(text); + const fromLines = cleanDescription(extractDescriptionFromLines(lines)); + if (fromLines.length > 100) return fromLines; + return extractDescriptionByMarkers(text); +} + +function escapeRe(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function sentenceSplit(text) { + return String(text || '') + .replace(/\n+/g, '. ') + .split(/(?<=[.!?;])\s+|\s+[-–—]\s+/) + .map((part) => part.trim()) + .filter(Boolean); +} + +const LANGUAGE_MAP = { + English: ['English', 'Englisch', 'anglais', 'inglese'], + Italian: ['Italian', 'Italienisch', 'italien', 'italiano'], + German: ['German', 'Deutsch', 'Allemand', 'Tedesco', 'Deutschkenntnisse'], + French: ['French', 'Français', 'Francais', 'Französisch', 'Franzoesisch', 'francese'], + Spanish: ['Spanish', 'Spanisch', 'Espagnol', 'Spagnolo'], + Dutch: ['Dutch', 'Niederländisch', 'Nederlands'], + Portuguese: ['Portuguese', 'Portugiesisch', 'Portugais'], +}; + +const NICE_CONTEXT = /\b(nice[- ]to[- ]have|plus|bonus|preferred|advantage|asset|optional|beneficial|would be a plus|good to have|desirable)\b/i; +const REQUIRED_CONTEXT = /\b(required|must|mandatory|essential|fluent|fluency|proficient|excellent|strong|native|business fluent|c1|c2|kenntnisse|maîtrise|maitrise|obligatoire|nécessaire|necessaire|erforderlich|voraussetzung|zwingend)\b/i; +const LANGUAGE_CONTEXT = /\b(language skills|sprachkenntnisse|langues?|languages?|written and spoken|spoken and written)\b/i; + +function parseLanguages(description) { + const required = new Set(); + const niceToHave = new Set(); + + for (const sentence of sentenceSplit(description)) { + for (const [language, aliases] of Object.entries(LANGUAGE_MAP)) { + if (!aliases.some((alias) => new RegExp(`\\b${escapeRe(alias)}\\b`, 'i').test(sentence))) continue; + if (NICE_CONTEXT.test(sentence) && !REQUIRED_CONTEXT.test(sentence.replace(NICE_CONTEXT, ''))) { + niceToHave.add(language); + } else if (REQUIRED_CONTEXT.test(sentence) || LANGUAGE_CONTEXT.test(sentence)) { + required.add(language); + } + } + } + + return { + required: [...required], + niceToHave: [...niceToHave].filter((language) => !required.has(language)), + }; +} + +function canonicalLanguage(value) { + const text = String(value || '').trim().toLowerCase(); + for (const [language, aliases] of Object.entries(LANGUAGE_MAP)) { + if (language.toLowerCase() === text || aliases.some((alias) => alias.toLowerCase() === text)) return language; + } + return text ? text.charAt(0).toUpperCase() + text.slice(1) : null; +} + +function checkLanguageFilter(requirements, opts) { + const speaks = new Set(opts.speaks.map((language) => canonicalLanguage(language)?.toLowerCase()).filter(Boolean)); + const blockedExplicit = new Set(opts.excludeLanguages.map((language) => canonicalLanguage(language)?.toLowerCase()).filter(Boolean)); + const missing = requirements.required.filter((language) => !speaks.has(language.toLowerCase())); + const blocked = requirements.required.filter((language) => blockedExplicit.has(language.toLowerCase())); + + if (!requirements.required.length) { + return { pass: true, reason: 'PASS: No language requirements detected in JD' }; + } + if (blocked.length) { + return { pass: false, reason: `FAIL: JD requires ${blocked.join(', ')} (user does not speak it/them)` }; + } + if (missing.length) { + return { pass: false, reason: `FAIL: JD requires ${missing.join(', ')} (not in user spoken languages)` }; + } + return { pass: true, reason: `PASS: JD requires ${requirements.required.join(', ')} (user speaks all)` }; +} + +const BLOCKED_TITLE_RE = /\b(architekt(?:in)?|architecte|architetto|projektleiter(?:in)?|zeichner(?:in)?|bauleiter(?:in)?|innenarchitekt|architecte d.intérieur|architetto d.interni|praktikant(?:in)?)\b/i; +const AI_SOFTWARE_RE = /\b(ai|a\.i\.|artificial intelligence|genai|generative ai|llm|machine learning|ml\b|data\s*&\s*ai|data and ai|ai\s*&\s*data|solution architect|solutions architect|enterprise architect|software architect|cloud architect)\b/i; + +function checkTitle(title) { + const text = String(title || ''); + if (BLOCKED_TITLE_RE.test(text) && !AI_SOFTWARE_RE.test(text)) { + return { pass: false, reason: `FAIL: Title appears to be a non-IT/building architect role (${text})` }; + } + return { pass: true, reason: null }; +} + +function buildJobQuery(opts) { + const where = ['source = @source']; + if (!opts.all) { + where.push(`( + description_text IS NULL + OR trim(description_text) = '' + ${opts.retryFailed ? "OR language_filter_reason LIKE 'FAIL: Could not extract JD%' OR language_filter_reason LIKE 'FAIL: Error fetching page:%' OR language_filter_reason LIKE 'scrape_failed:%'" : ''} + )`); + } + const limitSql = opts.limit ? 'LIMIT @limit' : ''; + return ` + SELECT source, job_id, url, title, company, location_raw, language_filter_reason + FROM jobs + WHERE ${where.join(' AND ')} + ORDER BY updated_at DESC, title COLLATE NOCASE + ${limitSql} + `; +} + +function ensureTables(db) { + db.exec(` + CREATE TABLE IF NOT EXISTS job_languages ( + source TEXT NOT NULL, + job_id TEXT NOT NULL, + language TEXT NOT NULL, + importance TEXT NOT NULL CHECK (importance IN ('required', 'nice_to_have')), + PRIMARY KEY (source, job_id, language, importance), + FOREIGN KEY (source, job_id) REFERENCES jobs(source, job_id) ON DELETE CASCADE + ); + `); +} + +function makeUpdater(db) { + const updateJob = db.prepare(` + UPDATE jobs + SET description_text = ?, + description_raw = ?, + language_filter_reason = ?, + updated_at = CURRENT_TIMESTAMP + WHERE source = ? AND job_id = ? + `); + const deleteLanguages = db.prepare('DELETE FROM job_languages WHERE source = ? AND job_id = ?'); + const insertLanguage = db.prepare(` + INSERT OR IGNORE INTO job_languages (source, job_id, language, importance) + VALUES (?, ?, ?, ?) + `); + + return db.transaction((job, descriptionText, descriptionRaw, languageFilterReason, requirements) => { + updateJob.run(descriptionText, descriptionRaw, languageFilterReason, job.source, job.job_id); + deleteLanguages.run(job.source, job.job_id); + for (const language of requirements.required || []) { + insertLanguage.run(job.source, job.job_id, language, 'required'); + } + for (const language of requirements.niceToHave || []) { + insertLanguage.run(job.source, job.job_id, language, 'nice_to_have'); + } + }); +} + +async function processJob(job, opts, client) { + let blockAttempt = 0; + while (true) { + await enforceRateLimit(); + const fetched = await fetchJobText(job, opts, client); + + if (fetched.blocked) { + blockAttempt++; + if (fetched.isCaptcha) { + console.warn(` [captcha] CAPTCHA on detail ${job.job_id}: ${fetched.blockReason}`); + } else { + console.warn(` [blocked] LinkedIn warning on detail ${job.job_id}: ${fetched.blockReason}`); + } + const backoff = backoffDelay(blockAttempt, 15000, 120000); + console.warn(` [batch-backoff] ${job.job_id} attempt=${blockAttempt} waiting ${(backoff / 1000).toFixed(1)}s`); + await sleep(backoff); + continue; + } + + const jd = extractJD(fetched.text); + const requirements = jd ? parseLanguages(jd) : { required: [], niceToHave: [] }; + + if (!fetched.ok) { + return { + job, + jd: null, + raw: fetched.text ? fetched.text.slice(0, 10000) : null, + requirements, + pass: false, + reason: `FAIL: Error fetching page: ${fetched.error}`, + }; + } + + if (!jd) { + return { + job, + jd: null, + raw: fetched.text ? fetched.text.slice(0, 10000) : null, + requirements, + pass: false, + reason: 'FAIL: Could not extract JD from page', + }; + } + + const titleCheck = checkTitle(job.title); + if (!titleCheck.pass) { + return { + job, + jd, + raw: jd, + requirements, + pass: false, + reason: titleCheck.reason, + }; + } + + const languageCheck = checkLanguageFilter(requirements, opts); + const suffix = requirements.niceToHave.length ? ` Nice-to-have: ${requirements.niceToHave.join(', ')}.` : ''; + return { + job, + jd, + raw: jd, + requirements, + pass: languageCheck.pass, + reason: `${languageCheck.reason}.${suffix}`.replace('..', '.'), + }; + } +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + console.log('Batch JD backfill started'); + console.log(`DB: ${opts.db}`); + console.log(`Source: ${opts.source}`); + console.log(`Speaks: ${opts.speaks.join(', ')}`); + console.log(`Blocked required languages: ${opts.excludeLanguages.join(', ') || '(none)'}`); + console.log(`Batch size: ${opts.batchSize}; timeout: ${opts.timeout}s; dry-run: ${opts.dryRun ? 'yes' : 'no'}`); + + ensureObscura(opts); + const client = await connectCdp(opts); + const keepAlive = startCdpKeepAlive(client, { + intervalMs: opts.keepAliveSeconds * 1000, + label: 'LinkedIn JD CDP', + onStart: ({ intervalMs }) => console.log(`[cdp] heartbeat enabled every ${intervalMs / 1000}s on 127.0.0.1:${opts.port}`), + onFailure: (error) => console.warn(`[cdp] heartbeat failed: ${error.message}`), + }); + + const db = new Database(opts.db); + ensureTables(db); + const jobs = db.prepare(buildJobQuery(opts)).all({ source: opts.source, limit: opts.limit }); + console.log(`Found ${jobs.length} job(s) to process.`); + + if (!jobs.length) { + keepAlive.stop(); + client.close(); + db.close(); + return; + } + + const update = makeUpdater(db); + const totals = { processed: 0, passed: 0, failed: 0, errors: 0, written: 0 }; + + for (let i = 0; i < jobs.length; i += opts.batchSize) { + const batch = jobs.slice(i, i + opts.batchSize); + console.log(`\nBatch ${Math.floor(i / opts.batchSize) + 1}: fetching ${batch.length} job detail page(s)...`); + const results = await mapLimit(batch, opts.batchSize, (job) => processJob(job, opts, client), 5000); + + for (const result of results) { + totals.processed++; + if (result.pass) totals.passed++; + else totals.failed++; + if (result.reason.startsWith('FAIL: Error fetching page:')) totals.errors++; + + if (!opts.dryRun) { + update(result.job, result.jd, result.raw, result.reason, result.requirements); + totals.written++; + } + + const mark = result.pass ? '✓ PASS' : '✗ FAIL'; + const title = result.job.title || result.job.job_id; + const company = result.job.company ? ` @ ${result.job.company}` : ''; + const langs = [...result.requirements.required.map((l) => `${l}:required`), ...result.requirements.niceToHave.map((l) => `${l}:nice`)]; + const langText = langs.length ? ` [${langs.join(', ')}]` : ''; + console.log(` [${totals.processed}/${jobs.length}] ${mark} ${title}${company} — ${result.reason}${langText}`); + } + + if (i + opts.batchSize < jobs.length) { + await humanDelay('between batches', 5000, 10000); + } + } + + keepAlive.stop(); + client.close(); + db.close(); + console.log('\n=== Summary ==='); + console.log(`Total processed: ${totals.processed}`); + console.log(`Passed language filter: ${totals.passed}`); + console.log(`Failed language/title/JD filter: ${totals.failed}`); + console.log(`Fetch errors: ${totals.errors}`); + console.log(`SQLite rows updated: ${totals.written}`); +} + +main().catch((error) => { + console.error(error?.stack || error?.message || String(error)); + process.exit(1); +}); diff --git a/skills/linkedin-job-search/scripts/cdp-keepalive.mjs b/skills/linkedin-job-search/scripts/cdp-keepalive.mjs new file mode 100644 index 0000000..9c31cab --- /dev/null +++ b/skills/linkedin-job-search/scripts/cdp-keepalive.mjs @@ -0,0 +1,52 @@ +function withTimeout(promise, ms, label) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); + timer.unref?.(); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +export function startCdpKeepAlive(client, options = {}) { + const intervalMs = Math.max(0, Number(options.intervalMs ?? 15000)); + const timeoutMs = Math.max(1000, Number(options.timeoutMs ?? 5000)); + const label = options.label || 'CDP'; + const stats = { sent: 0, failed: 0, lastSuccessAt: null, lastError: null }; + let stopped = false; + let busy = false; + let timer = null; + + const beat = async () => { + if (stopped || busy) return; + busy = true; + try { + await withTimeout(client.send('Browser.getVersion', {}), timeoutMs, `${label} keepalive`); + stats.sent++; + stats.lastSuccessAt = new Date().toISOString(); + stats.lastError = null; + } catch (error) { + stats.failed++; + stats.lastError = error?.message || String(error); + options.onFailure?.(error, { ...stats }); + } finally { + busy = false; + } + }; + + if (intervalMs > 0) { + void beat(); + timer = setInterval(() => { void beat(); }, intervalMs); + timer.unref?.(); + options.onStart?.({ intervalMs, timeoutMs }); + } + + return { + stats, + async beat() { await beat(); }, + stop() { + stopped = true; + if (timer) clearInterval(timer); + timer = null; + }, + }; +} diff --git a/skills/linkedin-job-search/scripts/cdp-lease.mjs b/skills/linkedin-job-search/scripts/cdp-lease.mjs new file mode 100644 index 0000000..31d4757 --- /dev/null +++ b/skills/linkedin-job-search/scripts/cdp-lease.mjs @@ -0,0 +1,469 @@ +/** + * Cross-process CDP coordination: lease + shared request budget + tab registry. + * + * All state is stored in files under a lock directory (default ~/.job-hunter/locks/). + * Every operation is fail-open: a corrupt/missing file never crashes the caller. + * + * LEASE + * const h = acquireLease({ lockDir, leaseName: 'linkedin-search:9225', runId: 'x' }); + * refreshHeartbeat(h); + * registerTab(h, targetId); + * releaseLease(h); // call on cleanup + * + * Stale leases (heartbeat > 60 s, or dead PID) may be taken over by a new process. + * + * SHARED BUDGET + * const b = createSharedBudget({ lockDir, budgetName: 'linkedin.com:9225', windowMs: 60000, maxRequests: 12 }); + * await b.waitForSlot(); // sleep until a slot is available + * // On cleanup: b.destroy() (optional) + * + * Budged file corruption or I/O errors → warn and fail open (caller proceeds). + */ + +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import path from 'node:path'; + +// ── Helpers ────────────────────────────────────────────────────────── + +function sleep(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +function pidAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function ensureDir(dir) { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); +} + +function safeReadJson(filePath) { + try { + if (!existsSync(filePath)) return null; + return JSON.parse(readFileSync(filePath, 'utf8')); + } catch { + return null; + } +} + +function safeWriteJson(filePath, obj) { + try { + writeFileSync(filePath, JSON.stringify(obj, null, 2), 'utf8'); + return true; + } catch { + return false; + } +} + +function safeUnlink(filePath) { + try { + if (existsSync(filePath)) unlinkSync(filePath); + } catch { + // best-effort + } +} + +/** + * Atomic file creation: returns true if the file was newly created, false + * if it already existed. Uses the O_EXCL | O_CREAT trick on Unix. + */ +function atomicCreate(filePath, content) { + let fd; + try { + fd = openSync(filePath, 'wx'); + writeFileSync(fd, content, 'utf8'); + return true; + } catch (err) { + if (err.code === 'EEXIST') return false; + throw err; + } finally { + if (fd !== undefined) { + try { closeSync(fd); } catch {} + } + } +} + +// ── Mutex (advisory, file-based) ───────────────────────────────────── + +const MUTEX_RETRY_MS = 50; +const MUTEX_MAX_WAIT_MS = 3000; + +async function withMutex(lockFile, fn) { + const started = Date.now(); + while (true) { + if (atomicCreate(lockFile, String(process.pid))) { + try { + return await fn(); + } finally { + safeUnlink(lockFile); + } + } + // Break stale mutex left by a crashed process (dead PID). + try { + const holderPid = parseInt(readFileSync(lockFile, 'utf8').trim(), 10); + if (Number.isInteger(holderPid) && holderPid > 0 && !pidAlive(holderPid)) { + safeUnlink(lockFile); + continue; + } + } catch { + // Lockfile vanished between create attempt and read — retry. + continue; + } + if (Date.now() - started > MUTEX_MAX_WAIT_MS) { + throw new Error(`mutex timeout on ${lockFile}`); + } + await sleep(MUTEX_RETRY_MS + Math.random() * 30); + } +} + +// ── Lease ──────────────────────────────────────────────────────────── + +const DEFAULT_HEARTBEAT_THRESHOLD_MS = 60_000; +const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000; + +/** + * Acquire a named lease for CDP coordination. + * + * @param {object} opts + * @param {string} [opts.lockDir] directory for lock files (default ~/.job-hunter/locks) + * @param {string} opts.leaseName unique lease identifier, e.g. "linkedin-search:9225" + * @param {string} [opts.runId] optional run identifier for diagnostics + * @param {number} [opts.heartbeatThresholdMs] max heartbeat age before lease considered stale (default 60000) + * @returns {object} lease handle { leaseName, leasePath, destroy } + */ +export function acquireLease({ + lockDir, + leaseName, + runId = `${Date.now()}-${Math.random().toString(36).slice(2, 6)}`, + heartbeatThresholdMs = DEFAULT_HEARTBEAT_THRESHOLD_MS, +} = {}) { + if (!leaseName) throw new Error('leaseName is required'); + + const dir = lockDir || path.join(homedir(), '.job-hunter', 'locks'); + ensureDir(dir); + + const safeName = leaseName.replace(/[^a-zA-Z0-9:_-]/g, '_'); + const leasePath = path.join(dir, `${safeName}.lease`); + const lockPath = path.join(dir, `${safeName}.lease.lock`); + + function readLease() { + return safeReadJson(leasePath); + } + + function writeLease(data, ignoreStale = false) { + // Only write if we still own the lease (or if we're forcing) + if (!ignoreStale) { + const current = readLease(); + if (!current || current.pid !== process.pid) return false; + } + return safeWriteJson(leasePath, data); + } + + function isStale(data) { + if (!data) return true; + // Dead PID → stale + if (!pidAlive(data.pid)) return true; + // Heartbeat too old → stale + const heartbeat = data.heartbeat ? new Date(data.heartbeat).getTime() : 0; + if (Date.now() - heartbeat > heartbeatThresholdMs) return true; + return false; + } + + // ── Acquire (try-create, then take-over if stale) ────────────────── + let data = readLease(); + if (data && !isStale(data)) { + throw new Error( + `Lease "${leaseName}" already held by PID ${data.pid} (run ${data.runId || '?'}) since ${data.acquiredAt}` + ); + } + + if (data && isStale(data)) { + // Take over stale lease + console.warn(` [cdp-lease] Taking over stale lease "${leaseName}" (PID ${data.pid} ${pidAlive(data.pid) ? 'alive but heartbeat stale' : 'dead'})`); + safeUnlink(leasePath); + } + + const now = new Date().toISOString(); + const leaseData = { + leaseName, + pid: process.pid, + runId, + acquiredAt: now, + heartbeat: now, + tabs: [], + }; + + const created = atomicCreate(leasePath, JSON.stringify(leaseData, null, 2)); + if (!created) { + // Raced — re-read and check + data = readLease(); + if (data && !isStale(data)) { + throw new Error( + `Lease "${leaseName}" already held by PID ${data.pid} (run ${data.runId || '?'}) — lost race` + ); + } + // Stale after all — force it + safeUnlink(leasePath); + const created2 = atomicCreate(leasePath, JSON.stringify(leaseData, null, 2)); + if (!created2) { + throw new Error(`Could not acquire lease "${leaseName}" — atomic create failed after cleanup`); + } + } + + console.log(` [cdp-lease] Acquired lease "${leaseName}" (run ${runId})`); + + // ── Heartbeat starter ────────────────────────────────────────────── + let heartbeatTimer = null; + + function startHeartbeat(intervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS) { + if (heartbeatTimer) return; + heartbeatTimer = setInterval(() => { + try { + const current = readLease(); + if (!current || current.pid !== process.pid) return; // lost lease + current.heartbeat = new Date().toISOString(); + writeLease(current, true); + } catch { + // heartbeat failure is non-fatal + } + }, intervalMs); + heartbeatTimer.unref(); + } + + startHeartbeat(); + + // ── Handle ───────────────────────────────────────────────────────── + const handle = { + leaseName, + leasePath, + /** Refresh heartbeat immediately. */ + refreshHeartbeat() { + const current = readLease(); + if (!current || current.pid !== process.pid) return false; + current.heartbeat = new Date().toISOString(); + return writeLease(current, true); + }, + /** Register a CDP target (tab) as owned by this lease. */ + registerTab(targetId) { + if (!targetId) return; + try { + const current = readLease(); + if (!current || current.pid !== process.pid) return; + if (!Array.isArray(current.tabs)) current.tabs = []; + if (!current.tabs.includes(targetId)) { + current.tabs.push(targetId); + writeLease(current, true); + } + } catch { + // best-effort + } + }, + /** Get all tabs registered under this lease. */ + getTabs() { + const current = readLease(); + if (!current || !Array.isArray(current.tabs)) return []; + return [...current.tabs]; + }, + /** Release the lease and stop heartbeat. */ + release() { + if (heartbeatTimer) { + clearInterval(heartbeatTimer); + heartbeatTimer = null; + } + try { + const current = readLease(); + if (current && current.pid === process.pid) { + safeUnlink(leasePath); + console.log(` [cdp-lease] Released lease "${leaseName}"`); + } + // Also clean up mutex lockfile if we left it + safeUnlink(lockPath); + } catch { + // best-effort + } + }, + }; + + return handle; +} + +/** + * Non-throwing wrapper: returns the handle or null on failure. + */ +export function tryAcquireLease(opts) { + try { + return acquireLease(opts); + } catch (err) { + console.warn(` [cdp-lease] Could not acquire lease "${opts?.leaseName}": ${err.message}`); + return null; + } +} + +// ── Shared cross-process request budget ────────────────────────────── + +/** + * Create a shared rate budget manager keyed by budgetName. + * + * @param {object} opts + * @param {string} [opts.lockDir] + * @param {string} opts.budgetName e.g. "linkedin.com:9225" + * @param {number} [opts.windowMs] rolling window in ms (default 60000) + * @param {number} [opts.maxRequests] max requests in window (default 12) + * @returns {{ waitForSlot: () => Promise, noteRequest: () => void, destroy: () => void }} + */ +export function createSharedBudget({ + lockDir, + budgetName, + windowMs = 60_000, + maxRequests = 12, +} = {}) { + if (!budgetName) throw new Error('budgetName is required'); + + const dir = lockDir || path.join(homedir(), '.job-hunter', 'locks'); + ensureDir(dir); + + const safeName = budgetName.replace(/[^a-zA-Z0-9:_-]/g, '_'); + const budgetPath = path.join(dir, `budget-${safeName}.json`); + const lockPath = path.join(dir, `budget-${safeName}.lock`); + + function readBudget() { + try { + if (!existsSync(budgetPath)) return { windowMs, maxRequests, requests: [] }; + const data = JSON.parse(readFileSync(budgetPath, 'utf8')); + return { + windowMs: data.windowMs || windowMs, + maxRequests: data.maxRequests || maxRequests, + requests: Array.isArray(data.requests) ? data.requests : [], + }; + } catch { + // Corrupt file → fail open + return { windowMs, maxRequests, requests: [] }; + } + } + + function writeBudget(budget) { + try { + writeFileSync(budgetPath, JSON.stringify(budget, null, 2), 'utf8'); + return true; + } catch { + return false; + } + } + + /** + * Prune timestamps outside the window. Returns the pruned array. + */ + function pruneRequests(requests) { + const cutoff = Date.now() - windowMs; + return requests.filter((ts) => ts > cutoff); + } + + const handle = { + /** + * Record a request in the shared budget. Best-effort; never throws. + */ + noteRequest() { + try { + let budget = readBudget(); + budget.requests = pruneRequests(budget.requests); + budget.requests.push(Date.now()); + // Keep from growing unbounded (windowMs worth at most) + budget.requests = budget.requests.slice(-maxRequests * 3); + writeBudget(budget); + } catch { + // fail open + } + }, + + /** + * Wait until a request slot is available under the shared budget. + * Returns once a slot is acquired (our timestamp is recorded). + * On persistent budget file errors, fails open immediately. + */ + async waitForSlot() { + while (true) { + let outcome; + try { + // Read-modify-write under the advisory lockfile so concurrent + // processes don't both claim the last slot. Small races after a + // mutex timeout are accepted (fail open). + outcome = await withMutex(lockPath, async () => { + const budget = readBudget(); + budget.requests = pruneRequests(budget.requests); + if (budget.requests.length < maxRequests) { + budget.requests.push(Date.now()); + budget.requests = budget.requests.slice(-maxRequests * 3); + const ok = writeBudget(budget); + return { acquired: true, writeOk: ok }; + } + return { acquired: false, oldest: budget.requests[0], used: budget.requests.length }; + }); + } catch { + // Mutex timeout or I/O error — fail open to per-process limiting + console.warn(' [cdp-lease] Budget lock/IO error — proceeding without shared rate limit'); + return; + } + + if (outcome.acquired) { + if (!outcome.writeOk) { + console.warn(' [cdp-lease] Budget file write error — proceeding without shared rate limit'); + } + return; + } + + // Budget exhausted — compute wait + const waitMs = outcome.oldest + windowMs - Date.now() + 100; + if (waitMs <= 0) { + // Window already expired for oldest entry — retry prune+write + continue; + } + + console.log( + ` [cdp-lease:budget] ${outcome.used}/${maxRequests} slots used, pausing ${(waitMs / 1000).toFixed(1)}s` + ); + // Sleep for the full wait time so we don't spin. Maximum one log line per slot wait. + await sleep(waitMs); + } + }, + + /** + * Detach from the budget. Leaves the budget file in place — sibling + * processes may still be sharing it and it self-prunes by window — + * but clears any advisory lockfile we may have left behind. + */ + destroy() { + try { + const holder = existsSync(lockPath) ? readFileSync(lockPath, 'utf8').trim() : null; + if (holder === String(process.pid)) safeUnlink(lockPath); + } catch {} + }, + }; + + return handle; +} + +/** + * Non-throwing wrapper for createSharedBudget. + */ +export function tryCreateSharedBudget(opts) { + try { + return createSharedBudget(opts); + } catch (err) { + console.warn(` [cdp-lease] Could not create shared budget "${opts?.budgetName}": ${err.message}`); + return null; + } +} diff --git a/skills/linkedin-job-search/scripts/cdp-preflight.mjs b/skills/linkedin-job-search/scripts/cdp-preflight.mjs new file mode 100644 index 0000000..6009dd8 --- /dev/null +++ b/skills/linkedin-job-search/scripts/cdp-preflight.mjs @@ -0,0 +1,209 @@ +#!/usr/bin/env node +import http from 'node:http'; +import { classifyLinkedInPage } from './linkedin-page-state.mjs'; + +function parseArgs(argv) { + const opts = { + port: Number(process.env.BROWSER_CDP_PORT || process.env.LINKEDIN_CDP_PORT || 9225) || 9225, + json: false, + watchSeconds: 0, + keepAliveSeconds: Number(process.env.CDP_KEEPALIVE_SECONDS || 15) || 15, + probeUrls: [], + waitMs: 3000, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + if (i + 1 >= argv.length) throw new Error(`${arg} requires a value`); + return argv[++i]; + }; + if (arg === '--port' || arg === '--cdp-port') opts.port = Number(next()) || opts.port; + else if (arg === '--json') opts.json = true; + else if (arg === '--watch-seconds') opts.watchSeconds = Math.max(0, Number(next()) || 0); + else if (arg === '--keepalive-seconds') opts.keepAliveSeconds = Math.max(1, Number(next()) || 15); + else if (arg === '--probe-url') opts.probeUrls.push(next()); + else if (arg === '--wait-ms') opts.waitMs = Math.max(500, Number(next()) || 3000); + else if (arg === '--help' || arg === '-h') { + console.log('Usage: cdp-preflight.mjs [--port 9225] [--json] [--probe-url URL] [--wait-ms 3000] [--watch-seconds N] [--keepalive-seconds N]'); + process.exit(0); + } else throw new Error(`Unknown option: ${arg}`); + } + return opts; +} + +function requestJson(port, path, method = 'GET') { + return new Promise((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port, path, method, timeout: 5000 }, (res) => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => { body += chunk; }); + res.on('end', () => { + try { resolve(JSON.parse(body)); } + catch (error) { reject(new Error(`Invalid CDP JSON at ${path}: ${error.message}`)); } + }); + }); + req.on('timeout', () => req.destroy(new Error(`CDP timeout at ${path}`))); + req.on('error', reject); + req.end(); + }); +} + +function getJson(port, path) { + return requestJson(port, path, 'GET'); +} + +async function evaluateTarget(target) { + if (!target.webSocketDebuggerUrl) return null; + const ws = new WebSocket(target.webSocketDebuggerUrl); + const pending = new Map(); + let seq = 0; + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('target WebSocket open timeout')), 5000); + ws.onopen = () => { clearTimeout(timer); resolve(); }; + ws.onerror = () => { clearTimeout(timer); reject(new Error('target WebSocket error')); }; + }); + ws.onmessage = (event) => { + const message = JSON.parse(event.data); + const item = pending.get(message.id); + if (!item) return; + pending.delete(message.id); + message.error ? item.reject(new Error(message.error.message || JSON.stringify(message.error))) : item.resolve(message.result); + }; + const send = (method, params = {}) => new Promise((resolve, reject) => { + const id = ++seq; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`${method} timeout`)); + }, 5000); + pending.set(id, { + resolve: (value) => { clearTimeout(timer); resolve(value); }, + reject: (error) => { clearTimeout(timer); reject(error); }, + }); + ws.send(JSON.stringify({ id, method, params })); + }); + try { + const result = await send('Runtime.evaluate', { + expression: 'JSON.stringify({title:document.title||"",url:location.href,text:(document.body?.innerText||"").slice(0,5000)})', + returnByValue: true, + }); + return JSON.parse(result.result?.value || '{}'); + } finally { + ws.close(); + } +} + +function classifyIndeedPage(page) { + const joined = `${page?.title || ''}\n${page?.text || ''}`; + if (/verify you are human|unusual traffic|security check|additional verification|required verification/i.test(joined)) { + return { state: 'blocked', reason: 'Indeed verification page detected' }; + } + if (/indeed\.com/i.test(page?.url || '') && page?.text) return { state: 'healthy', reason: null }; + return { state: 'unknown', reason: 'No readable Indeed page body' }; +} + +async function browserKeepAlive(wsUrl, durationMs, intervalMs) { + if (!durationMs) return { sent: 0, failed: 0 }; + const ws = new WebSocket(wsUrl); + let seq = 0; + let sent = 0; + let failed = 0; + const pending = new Map(); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('browser WebSocket open timeout')), 5000); + ws.onopen = () => { clearTimeout(timer); resolve(); }; + ws.onerror = () => { clearTimeout(timer); reject(new Error('browser WebSocket error')); }; + }); + ws.onmessage = (event) => { + const message = JSON.parse(event.data); + const item = pending.get(message.id); + if (!item) return; + pending.delete(message.id); + message.error ? item.reject(new Error(message.error.message || JSON.stringify(message.error))) : item.resolve(message.result); + }; + const ping = () => new Promise((resolve, reject) => { + const id = ++seq; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error('heartbeat timeout')); + }, 5000); + pending.set(id, { + resolve: (value) => { clearTimeout(timer); resolve(value); }, + reject: (error) => { clearTimeout(timer); reject(error); }, + }); + ws.send(JSON.stringify({ id, method: 'Browser.getVersion', params: {} })); + }); + const deadline = Date.now() + durationMs; + try { + while (Date.now() < deadline) { + try { await ping(); sent++; } + catch { failed++; } + const wait = Math.min(intervalMs, Math.max(0, deadline - Date.now())); + if (wait) await new Promise((resolve) => setTimeout(resolve, wait)); + } + } finally { + ws.close(); + } + return { sent, failed }; +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + const version = await getJson(opts.port, '/json/version'); + const targets = await getJson(opts.port, '/json/list'); + const relevant = targets.filter((target) => target.type === 'page' && /linkedin\.com|indeed\.com/i.test(target.url || '')); + const pages = []; + for (const target of relevant) { + try { + const page = await evaluateTarget(target); + const site = /linkedin\.com/i.test(page?.url || target.url || '') ? 'linkedin' : 'indeed'; + const check = site === 'linkedin' ? classifyLinkedInPage(page) : classifyIndeedPage(page); + pages.push({ site, state: check.state, reason: check.reason, title: page?.title || target.title, url: page?.url || target.url }); + } catch (error) { + pages.push({ site: /linkedin\.com/i.test(target.url || '') ? 'linkedin' : 'indeed', state: 'error', reason: error.message, title: target.title, url: target.url }); + } + } + for (const url of opts.probeUrls) { + let target; + try { + target = await requestJson(opts.port, `/json/new?${encodeURIComponent(url)}`, 'PUT'); + await new Promise((resolve) => setTimeout(resolve, opts.waitMs)); + const page = await evaluateTarget(target); + const site = /linkedin\.com/i.test(page?.url || url) ? 'linkedin' : /indeed\.com/i.test(page?.url || url) ? 'indeed' : 'other'; + const check = site === 'linkedin' + ? classifyLinkedInPage(page) + : site === 'indeed' + ? classifyIndeedPage(page) + : { state: page?.text ? 'healthy' : 'unknown', reason: page?.text ? null : 'No readable page body' }; + pages.push({ site, state: check.state, reason: check.reason, title: page?.title || target.title, url: page?.url || url, probe: true }); + } catch (error) { + pages.push({ site: /linkedin\.com/i.test(url) ? 'linkedin' : /indeed\.com/i.test(url) ? 'indeed' : 'other', state: 'error', reason: error.message, title: '', url, probe: true }); + } finally { + if (target?.id) await requestJson(opts.port, `/json/close/${target.id}`, 'PUT').catch(() => {}); + } + } + const heartbeat = await browserKeepAlive( + version.webSocketDebuggerUrl, + opts.watchSeconds * 1000, + opts.keepAliveSeconds * 1000, + ); + const result = { + ok: Boolean(version.webSocketDebuggerUrl) && !pages.some((page) => ['blocked', 'captcha', 'login_required', 'error'].includes(page.state)), + port: opts.port, + browser: version.Browser || version.browser || null, + pageCount: targets.filter((target) => target.type === 'page').length, + pages, + heartbeat, + }; + if (opts.json) console.log(JSON.stringify(result)); + else { + console.log(`CDP ${result.port}: ${result.browser || 'reachable'}; ${result.pageCount} page target(s)`); + for (const page of pages) console.log(`${page.site.toUpperCase()} ${page.state}: ${page.title} — ${page.url}${page.reason ? ` — ${page.reason}` : ''}`); + if (opts.watchSeconds) console.log(`Heartbeat: ${heartbeat.sent} sent, ${heartbeat.failed} failed`); + } + process.exitCode = result.ok ? 0 : 3; +} + +main().catch((error) => { + console.error(JSON.stringify({ ok: false, error: error.message })); + process.exitCode = 2; +}); diff --git a/skills/linkedin-job-search/scripts/direct-jd-fetch.mjs b/skills/linkedin-job-search/scripts/direct-jd-fetch.mjs new file mode 100644 index 0000000..ab59572 --- /dev/null +++ b/skills/linkedin-job-search/scripts/direct-jd-fetch.mjs @@ -0,0 +1,176 @@ +import { Database, WebSocket } from '../../job-hunter/scripts/workspace-dependencies.mjs'; +// Direct CDP JD backfill for jobs the batch script couldn't extract. +// Opens each LinkedIn job URL, extracts JD via DOM, saves to DB. + +import http from 'http'; + +import { writeFileSync } from 'node:fs'; +import { startCdpKeepAlive } from './cdp-keepalive.mjs'; + +const JH = process.env.JOBHUNTER_HOME || `${process.env.HOME}/.job-hunter`; +const DB = process.env.JOBHUNTER_DB || `${JH}/jobhunter.sqlite`; +const CDP_PORT = 9225; +const JOBS = process.argv.slice(2); +if (JOBS.length === 0) { console.error('Usage: node direct-jd-fetch.mjs [job_id2] ...'); process.exit(1); } + +const db = new Database(DB); + +function httpJson(method, path) { + return new Promise((resolve, reject) => { + const req = http.request({ method, host: '127.0.0.1', port: CDP_PORT, path }, res => { + let d = ''; res.on('data', c => d += c); res.on('end', () => { + try { resolve(JSON.parse(d)); } catch { reject(new Error(d.slice(0,200))); } + }); + }); + req.on('error', reject); req.end(); + }); +} + +class Cdp { + constructor(wsUrl) { this.wsUrl = wsUrl; this.nid = 1; this.pending = new Map(); } + async connect() { + this.ws = new WebSocket(this.wsUrl); + await new Promise((resolve, reject) => { + const t = setTimeout(() => reject(new Error('WS open timeout')), 10000); + this.ws.on('open', () => { clearTimeout(t); resolve(); }); + this.ws.on('error', e => { clearTimeout(t); reject(e); }); + }); + this.ws.on('message', raw => { + try { + const m = JSON.parse(raw); + const p = this.pending.get(m.id); + if (!p) return; + clearTimeout(p.t); + this.pending.delete(m.id); + if (m.error) p.reject(new Error(m.error.message)); + else p.resolve(m.result); + } catch {} + }); + } + send(method, params = {}, timeout = 20000) { + const id = this.nid++; + return new Promise((resolve, reject) => { + const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`Timeout: ${method}`)); }, timeout); + this.pending.set(id, { resolve, reject, t }); + this.ws.send(JSON.stringify({ id, method, params })); + }); + } + async eval(expr, timeout = 20000) { + const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: false }, timeout); + if (r?.result?.value !== undefined) return r.result.value; + if (r?.result?.result?.value !== undefined) return r.result.result.value; + if (r?.exceptionDetails) throw new Error('JS: ' + JSON.stringify(r.exceptionDetails).slice(0,200)); + return r?.result; + } + close() { try { this.ws.close(); } catch {} } +} + +const delay = ms => new Promise(r => setTimeout(r, ms)); + +async function fetchOne(jobId) { + // Look up URL from DB + const row = db.prepare('SELECT url, title, company FROM jobs WHERE source = ? AND job_id = ?').get('linkedin', jobId); + if (!row) { console.log(` ${jobId}: not in DB`); return { jobId, status: 'not_in_db' }; } + const url = row.url; + // Strip /uk prefix to get the canonical /jobs/view/ form + let jobUrl = url; + const m = url.match(/linkedin\.com\/jobs\/view\/\d+/); + if (m) jobUrl = 'https://www.' + m[0]; + + console.log(` ${jobId} (${row.title?.slice(0,30)}): navigating to ${jobUrl.slice(0,80)}`); + const tab = await httpJson('PUT', `/json/new?url=${encodeURIComponent(jobUrl)}`); + const tabId = tab.id; + const cdp = new Cdp(tab.webSocketDebuggerUrl); + let keepAlive; + let result = { jobId, status: 'unknown' }; + try { + await cdp.connect(); + keepAlive = startCdpKeepAlive(cdp, { + intervalMs: Math.max(0, Number(process.env.CDP_KEEPALIVE_SECONDS ?? 15) || 0) * 1000, + label: 'LinkedIn direct-JD CDP', + }); + await cdp.send('Page.enable'); + await cdp.send('Runtime.enable'); + await cdp.send('Page.navigate', { url: jobUrl }); + await delay(7000); + + // Extract JD text — try multiple selectors + const jd = await cdp.eval(`(function() { + // LinkedIn JD containers (multiple variants across redesigns) + const selectors = [ + '.jobs-description__content .jobs-box__html', + '.jobs-description__content', + '.jobs-description .jobs-box__html', + '.description__text--rich', + '.description__text', + 'article.jobs-description', + '[data-testid="expandable-text-box"]', + 'div[class*="jobs-description"]', + 'div[class*="show-more-less-html"]', + 'div[class*="job-details"]', + '#job-details', + ]; + let best = ''; + let bestLen = 0; + for (const sel of selectors) { + const els = document.querySelectorAll(sel); + for (const el of els) { + const txt = (el.innerText || el.textContent || '').trim(); + if (txt.length > bestLen && txt.length > 200) { best = txt; bestLen = txt.length; } + } + } + // Fallback: largest text block on page + if (bestLen < 200) { + const all = Array.from(document.querySelectorAll('div, section, article')); + for (const el of all) { + const txt = (el.innerText || '').trim(); + if (txt.length > bestLen && txt.length > 500 && txt.length < 50000) { best = txt; bestLen = txt.length; } + } + } + return { jd: best, jd_len: best.length, title: document.title, url: location.href }; + })()`); + + if (jd.jd && jd.jd.length > 200) { + // Save to DB + db.prepare('UPDATE jobs SET description_text = ?, description_raw = ?, language_filter_reason = COALESCE(language_filter_reason, ?) WHERE source = ? AND job_id = ?') + .run(jd.jd, jd.jd, 'PASS: extracted via direct CDP', 'linkedin', jobId); + console.log(` ✓ saved ${jd.jd_len} chars`); + result = { jobId, status: 'ok', jd_len: jd.jd_len }; + } else { + console.log(` ✗ extracted only ${jd.jd_len} chars (title: ${jd.title?.slice(0,60)})`); + // Save screenshot + try { + const ss = await cdp.send('Page.captureScreenshot', { format: 'png' }); + writeFileSync(`/tmp/jd-FAIL-${jobId}.png`, Buffer.from(ss.data, 'base64')); + } catch {} + result = { jobId, status: 'fail', jd_len: jd.jd_len, page_title: jd.title, page_url: jd.url }; + } + } catch (e) { + console.log(` ✗ ERR: ${e.message}`); + result = { jobId, status: 'error', err: e.message }; + } finally { + keepAlive?.stop(); + cdp.close(); + // Close tab + try { + await httpJson('PUT', `/json/close/${tabId}`); + } catch {} + } + return result; +} + +async function main() { + const results = []; + for (const jid of JOBS) { + const r = await fetchOne(jid); + results.push(r); + await delay(2000); // human-like pause + } + console.log('\n=== Summary ==='); + for (const r of results) { + console.log(` ${r.jobId}: ${r.status}` + (r.jd_len ? ` (${r.jd_len} chars)` : '') + (r.err ? ` — ${r.err}` : '')); + } + process.exit(0); +} + +main().catch(e => { console.error('FATAL:', e); process.exit(1); }); diff --git a/skills/linkedin-job-search/scripts/linkedin-extractor.mjs b/skills/linkedin-job-search/scripts/linkedin-extractor.mjs new file mode 100644 index 0000000..f7ebc4a --- /dev/null +++ b/skills/linkedin-job-search/scripts/linkedin-extractor.mjs @@ -0,0 +1,313 @@ +/** + * LinkedIn Job Extractor + * + * Extracts fields from LinkedIn job HTML using regex patterns. + * Used by both the test harness and the skill instructions. + * + * Patterns are derived from common LinkedIn DOM structures. + * When LinkedIn changes their DOM, update these patterns. + */ + +/** + * Extract the job title from HTML. + * @param {string} html + * @returns {string|null} + */ +export function extractTitle(html) { + const patterns = [ + /class="top-card-layout__title"[^>]*>([^<]+)([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>(\d+)[^<]*]*>(\d+)\s*applicants?/, + /class="job-details-jobs-v2__applicant-count"[^>]*>(\d+)\s*applicants?/, + /over\s+(\d+)\s*applicants?/i, + /(\d+)\+?\s*applicants?\s*(?:applied|have applied)?/i, + ]; + for (const p of patterns) { + const m = html.match(p); + if (m) return parseInt(m[1], 10); + } + return null; +} + +/** + * Extract the full description text from HTML. + * @param {string} html + * @returns {string} + */ +export function extractDescription(html) { + const patterns = [ + /class="description__text"[^>]*>([\s\S]*?)<\/div>/, + /class="show-more-less__description"[^>]*>([\s\S]*?)<\/div>/, + /class="job-card-list__description"[^>]*>([\s\S]*?)<\/div>/, + ]; + for (const p of patterns) { + const m = html.match(p); + if (m) { + // Strip HTML tags from description + return m[1].replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim(); + } + } + return ''; +} + +/** + * Extract application links from HTML. + * @param {string} html + * @returns {string[]} + */ +export function extractApplicationLinks(html) { + const links = []; + const patterns = [ + /class="apply-link"[^>]*href="([^"]+)"/, + /class="apply-link external"[^>]*href="([^"]+)"/, + /class="apply-button"[^>]*href="([^"]+)"/, + ]; + for (const p of patterns) { + const m = html.match(p); + if (m && m[1] && m[1] !== '#') links.push(m[1]); + } + return links; +} + +/** + * Extract recruiter/hirer name. + * @param {string} html + * @returns {string|null} + */ +export function extractRecruiter(html) { + const patterns = [ + /class="hirer-name"[^>]*>([^<]+)]*>([^<]+)]*>[\s\S]*?class="hirer-name"[^>]*>([^<]+)]*>([^<]+)]*>([^<]+)]*datetime="([^"]+)"/, + /class="job-posted-date"[^>]*datetime="([^"]+)"/, + /class="job-details-jobs-v2__posted-date"[^>]*>([^<]+)]*>([^<]+)]*>([^<]+)]*>[\s\S]*?([\w.+-]+@[\w.-]+\.[a-zA-Z]{2,})[\s\S]*?]*href="([^"]+)"/, + /class="recruiter-name"[^>]*href="([^"]+)"/, + /class="hirer-info"[^>]*>[\s\S]*?class="hirer-name"[^>]*href="([^"]+)"/, + ]; + for (const p of patterns) { + const m = html.match(p); + if (m) return m[1].trim(); + } + return null; +} + +/** + * Parse language requirements from a description string. + * + * Returns an object with: + * - required: string[] of languages that are required + * - niceToHave: string[] of languages that are nice-to-have + * + * @param {string} description + * @returns {{ required: string[], niceToHave: string[] }} + */ +export function parseLanguageRequirements(description) { + const result = { required: [], niceToHave: [] }; + if (!description) return result; + + // Known language keywords to detect + const languageKeywords = [ + 'English', 'German', 'French', 'Spanish', 'Italian', + 'Portuguese', 'Dutch', 'Japanese', 'Chinese', 'Korean', + 'Arabic', 'Example Company 094n', 'Polish', 'Swedish', 'Danish', + 'Norwegian', 'Finnish', 'Turkish', 'Hindi', 'Bengali', + ]; + + // Build regex that matches any of these languages + const langPattern = languageKeywords.join('|'); + + // Required patterns: "Required: Fluent in X", "Must speak X", "X is required", "Requirements: X" + const requiredRegex = new RegExp( + `(?:Required|Requirement|Must|Fluent|Proficient|Mandatory|Essential)` + + `[^.]*?(?:in|speak|speaking|language|fluent|proficient)` + + `[^.]*?\\b(${langPattern})\\b`, + 'gi' + ); + let m; + while ((m = requiredRegex.exec(description)) !== null) { + const lang = capitalizeLang(m[1]); + if (!result.required.includes(lang)) result.required.push(lang); + } + + // Also catch simpler patterns like "Required: English" or "Requirements: English" + const simpleRequired = new RegExp( + `(?:Required|Requirements):\\s*(${langPattern})`, + 'gi' + ); + while ((m = simpleRequired.exec(description)) !== null) { + const lang = capitalizeLang(m[1]); + if (!result.required.includes(lang)) result.required.push(lang); + } + + // Nice-to-have patterns: "Nice to have: X", "X is a plus", "X is nice to have" + const niceRegex = new RegExp( + `(?:Nice|Plus|Bonus|Preferred|Optional|Good to have)` + + `[^.]*?(?:to have|skill|language|speak|speaking)` + + `[^.]*?\\b(${langPattern})\\b`, + 'gi' + ); + while ((m = niceRegex.exec(description)) !== null) { + const lang = capitalizeLang(m[1]); + if (!result.niceToHave.includes(lang)) result.niceToHave.push(lang); + } + + // Also catch "X is a plus", "X is nice to have" + const simpleNice = new RegExp( + `(${langPattern})[^.]*?(?:is a plus|is nice to have|nice to have)`, + 'gi' + ); + while ((m = simpleNice.exec(description)) !== null) { + const lang = capitalizeLang(m[1]); + if (!result.niceToHave.includes(lang)) result.niceToHave.push(lang); + } + + return result; +} + +/** + * Determine if a job passes the language filter. + * + * A job passes if: + * - No required languages are specified (pass by default) + * - All required languages match the user's language + * - A required language is listed as "nice to have" (not strictly required) + * + * @param {{ required: string[], niceToHave: string[] }} requirements + * @param {string} userLanguage - e.g. "English", "German" + * @returns {boolean} true = include job, false = skip + */ +export function passesLanguageFilter(requirements, userLanguage) { + if (!requirements || !userLanguage) return true; + + // Normalize user language + const userLang = capitalizeLang(userLanguage); + + // If no required languages, job passes + if (requirements.required.length === 0) return true; + + // Check if any required language does NOT match user's language + const allMatch = requirements.required.every( + (req) => req.toLowerCase() === userLang.toLowerCase() + ); + + return allMatch; +} + +/** + * Capitalize a language name properly. + * @param {string} lang + * @returns {string} + */ +function capitalizeLang(lang) { + if (!lang) return ''; + return lang.charAt(0).toUpperCase() + lang.slice(1).toLowerCase(); +} diff --git a/skills/linkedin-job-search/scripts/linkedin-page-state.mjs b/skills/linkedin-job-search/scripts/linkedin-page-state.mjs new file mode 100644 index 0000000..e9058d1 --- /dev/null +++ b/skills/linkedin-job-search/scripts/linkedin-page-state.mjs @@ -0,0 +1,230 @@ +const PASSIVE_HTML_RE = /<(?:script|style|iframe|noscript)\b[^>]*>[\s\S]*?<\/(?:script|style|iframe|noscript)>|<(?:iframe|meta|link)\b[^>]*\/?\s*>/gi; + +const BLOCK_SIGNATURES = [ + // ── Captcha / active challenge (checked first — these take precedence) ── + { re: /(?:captcha|recaptcha|hcaptcha).{0,120}(?:solve|verify|verification|challenge|continue)/i, state: 'active_challenge' }, + { re: /(?:solve|verify|verification|challenge).{0,120}(?:captcha|recaptcha|hcaptcha)/i, state: 'active_challenge' }, + { re: /please solve/i, state: 'active_challenge' }, + // Adjacency required: LinkedIn's checkpoint markup uses "challengePlatform" / + // "challenge verification". A {0,30} gap matched ordinary job-description prose + // ("challenges into platform") and broke healthy pages. + { re: /challenge[-_\s]?platform\b/i, state: 'active_challenge' }, + { re: /challenge[-_\s]?verification\b/i, state: 'active_challenge' }, + // ── Rate limiting ── + { re: /too many requests/i, state: 'rate_limited' }, + { re: /rate limit/i, state: 'rate_limited' }, + { re: /temporarily restricted/i, state: 'rate_limited' }, + // ── Other blocks (catch-all) ── + { re: /security (?:verification|check)/i, state: 'blocked' }, + { re: /verify you.{0,40}(?:human|real person)/i, state: 'blocked' }, + { re: /are you a (?:real person|human)/i, state: 'blocked' }, + { re: /we.{0,30}detected.{0,30}(?:unusual|suspicious)/i, state: 'blocked' }, + { re: /unusual activity/i, state: 'blocked' }, + { re: /sign.?in to confirm you.{0,30}(?:real|not a bot)/i, state: 'blocked' }, +]; + +/** + * Canonical page state values shared across the runner and its callers. + */ +export const PAGE_STATE = { + HEALTHY: 'healthy', + EMPTY: 'empty', + LOGIN_REQUIRED: 'login_required', + ACTIVE_CHALLENGE: 'active_challenge', + RATE_LIMITED: 'rate_limited', + BLOCKED: 'blocked', + TRANSIENT_ERROR: 'transient_error', +}; + +/** + * Terminal run-level statuses (page states + run-level only). + */ +export const RUN_STATUS = { + ...PAGE_STATE, + FAILED: 'failed', + CANCELLED: 'cancelled', +}; + +function visibleLikeText(input) { + const raw = String(input || ''); + if (!/<(?:html|body|script|iframe|style)\b/i.test(raw)) return raw; + return raw + .replace(PASSIVE_HTML_RE, ' ') + .replace(//g, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ | /gi, ' ') + .replace(/&(?:amp|lt|gt|quot);/gi, ' ') + .replace(/\s+/g, ' '); +} + +/** + * True when the payload carries rendered LinkedIn job-result data. Mirrors the + * result markers search-linkedin-jobs.mjs uses to accept a search page. + * + * @param {string} htmlOrText - raw HTML or visible text of the page + * @returns {boolean} + */ +export function hasJobResultMarkers(htmlOrText) { + const raw = String(htmlOrText || ''); + if (!raw) return false; + return /"job_id":\d+|\/jobs\/view\/\d+|jobCardPrefetchQueries|totalResultSize|data-job-id=|job-card-container/i.test(raw); +} + +/** + * True when the payload is a rendered job-detail page. Mirrors the JD markers + * search-linkedin-jobs.mjs waits for before accepting detail text. + * + * @param {string} htmlOrText - raw HTML or visible text of the page + * @returns {boolean} + */ +export function hasJobDetailMarkers(htmlOrText) { + const raw = String(htmlOrText || ''); + if (!raw) return false; + return /Report this job|Seniority level|About the job|Employment type|Job function/i.test(raw); +} + +/** + * Detect whether a LinkedIn HTML page is a block/warning/challenge. + * CRITICAL INVARIANT: passive invisible reCAPTCHA scripts/iframes present on + * HEALTHY pages must NOT be classified as a challenge. + * + * @param {string} htmlOrText - raw HTML or visible-text of the page + * @returns {{ blocked: boolean, isCaptcha: boolean, reason: string|null }} + * blocked = any block/warning page (true for all block types) + * isCaptcha = specifically a CAPTCHA challenge (otherwise generic block) + * reason = human-readable explanation + */ +export function detectLinkedInBlockPage(htmlOrText) { + const hay = visibleLikeText(htmlOrText); + for (const signature of BLOCK_SIGNATURES) { + if (signature.re.test(hay)) { + return { + blocked: true, + isCaptcha: signature.state === 'active_challenge', + reason: `matched visible page text: ${signature.re.source.slice(0, 70)}`, + }; + } + } + return { blocked: false, isCaptcha: false, reason: null }; +} + +/** + * Classify a LinkedIn page into one of the canonical PAGE_STATE values. + * Classification is based on visible/accessible page text and title, never + * on raw +AI Architect`; +assert.equal(detectLinkedInBlockPage(passiveRecaptcha).blocked, false, 'passive invisible reCAPTCHA must not block a healthy jobs page'); + +const activeCaptcha = '
Please solve the CAPTCHA challenge to continue
'; +const captchaResult = detectLinkedInBlockPage(activeCaptcha); +assert.equal(captchaResult.blocked, true); +assert.equal(captchaResult.isCaptcha, true); + +const securityCheck = detectLinkedInBlockPage('Security verification required. Verify you are a real person.'); +assert.equal(securityCheck.blocked, true); +assert.equal(securityCheck.isCaptcha, false); + +const healthyFixture = classifyLinkedInPage({ + title: 'Example Role Jobs in Example Region | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/?keywords=Example%20Role', + text: 'Home My Network Jobs Messaging Notifications Example Role results', +}); +assert.equal(healthyFixture.state, 'healthy'); + +const login = classifyLinkedInPage({ + title: 'LinkedIn Login', + url: 'https://www.linkedin.com/login', + text: 'Sign in to LinkedIn', +}); +assert.equal(login.state, 'login_required'); + +// ── New: page state model extensions ───────────────────────────────── + +// active_challenge (was captcha) +const challenge = classifyLinkedInPage({ + title: 'Security Check | LinkedIn', + url: 'https://www.linkedin.com/checkpoint/challenge/', + text: 'Please solve the CAPTCHA challenge to verify you are human. This is required to continue.', +}); +assert.equal(challenge.state, PAGE_STATE.ACTIVE_CHALLENGE, 'CAPTCHA page => active_challenge'); +assert.equal(challenge.blocked, true); +assert.equal(challenge.isCaptcha, true); + +// challengePlatform marker in LinkedIn checkpoint markup still detected +const challengePlatform = classifyLinkedInPage({ + title: 'Security Verification | LinkedIn', + url: 'https://www.linkedin.com/checkpoint/challenge/', + text: 'window.challengePlatform = { type: "captcha" };', +}); +assert.equal(challengePlatform.state, PAGE_STATE.ACTIVE_CHALLENGE, 'challengePlatform => active_challenge'); + +// Job-description prose must NOT trip challenge detection. Real text from a +// Celonis posting that tripped the old /challenge.{0,30}(platform|verification)/ +// rule and broke a healthy 32-card search page via the 3-strike circuit breaker. +const jdProse = classifyLinkedInPage({ + title: '(16) Artificial Intelligence Architect Jobs in United Kingdom | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/?keywords=AI%20Architect&start=56', + text: 'Translate customer business challenges into Celonis platform strategies, ' + + 'identifying high-impact use cases and defining success metrics. Support pre-sales ' + + 'by providing technical credibility in complex deals.', + isSearch: true, + hasResults: true, +}); +assert.equal(jdProse.blocked, false, 'JD prose "challenges into platform" must not be blocked'); +assert.equal(jdProse.isCaptcha, false, 'JD prose must not be flagged as CAPTCHA'); +assert.equal(jdProse.state, PAGE_STATE.HEALTHY, 'healthy search page with JD prose => healthy'); + +// A page carrying job-result markers is serving content, so block signatures +// appearing in JD prose must not classify it as a wall. Real case: a posting +// mentioning "security verification" on a page with 32 job cards. +const securityProse = classifyLinkedInPage({ + title: '(16) Enterprise AI Architect Jobs in United Kingdom | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/?keywords=Enterprise%20AI%20Architect&start=161', + text: '
' + + 'Lead security verification workflows for enterprise identity platforms.' + + '
"job_id":9000000001', + isSearch: true, +}); +assert.equal(securityProse.blocked, false, 'page with job results must not be blocked by JD prose'); +assert.equal(securityProse.state, PAGE_STATE.HEALTHY, 'page with job-result markers => healthy'); + +// The results guard must NOT mask a real interstitial: /checkpoint/ URLs stay +// blocked even if result-shaped markers appear in the payload. +const realCheckpoint = classifyLinkedInPage({ + title: 'Security Verification | LinkedIn', + url: 'https://www.linkedin.com/checkpoint/challenge/AgFxyz', + text: 'Security verification required. data-job-id="123" Please verify you are human.', + isSearch: true, +}); +assert.equal(realCheckpoint.blocked, true, 'checkpoint URL must stay blocked despite result markers'); + +// Rendered JD page: UK security-clearance boilerplate must not read as a wall. +// Real case: "Lead AI Architect" at Synthetic Grid Operator, whose JD +// says "The level of clearance associated with the role is Security Check (SC)". +const clearanceProse = classifyLinkedInPage({ + title: 'Lead AI Architect | Synthetic Grid Operator | LinkedIn', + url: 'https://linkedin.example/jobs/view/9999999999/', + text: 'About the job. This role is designated as requiring a National Security ' + + 'Vetting (NSV) clearance. The level of clearance associated with the role is ' + + 'Security Check (SC). You will usually need to have been a resident in the UK ' + + 'for the last five years to apply for an SC clearance.', +}); +assert.equal(clearanceProse.blocked, false, 'JD clearance boilerplate must not be blocked'); +assert.equal(clearanceProse.state, PAGE_STATE.HEALTHY, 'rendered JD page => healthy'); + +// A real interstitial with no JD markers must still block. +const detailWall = classifyLinkedInPage({ + title: 'Security Verification | LinkedIn', + url: 'https://linkedin.example/jobs/view/9999999999/', + text: 'Security verification required. Please verify you are human to continue.', +}); +assert.equal(detailWall.blocked, true, 'detail-page wall without JD markers must stay blocked'); + +// hasResults===true is an explicit caller signal that results rendered. +const explicitResults = classifyLinkedInPage({ + title: 'AI Architect Jobs | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/?keywords=AI%20Architect', + text: 'Manage unusual activity detection for the fraud platform.', + isSearch: true, + hasResults: true, +}); +assert.equal(explicitResults.state, PAGE_STATE.HEALTHY, 'hasResults=true => healthy despite prose match'); + +// rate_limited +const rateLimited = classifyLinkedInPage({ + title: 'Too Many Requests | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/', + text: 'You have made too many requests. Please try again later.', +}); +assert.equal(rateLimited.state, PAGE_STATE.RATE_LIMITED, 'rate-limit page => rate_limited'); + +// blocked (generic) +const genericBlock = classifyLinkedInPage({ + title: 'LinkedIn', + url: 'https://www.linkedin.com/', + text: 'We detected unusual activity from your network. Please verify your identity to continue.', +}); +assert.equal(genericBlock.state, PAGE_STATE.BLOCKED, 'unusual activity => blocked'); +assert.equal(genericBlock.isCaptcha, false); + +// empty search results +const emptySearch = classifyLinkedInPage({ + title: 'No matching jobs in Example Location 013 for AI Architect | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/?keywords=AI%20Architect&location=Example Location 013&start=0', + text: 'AI Architect jobs in Example Location 013. No matching jobs found. Try adjusting your search.', + isSearch: true, + hasResults: false, +}); +assert.equal(emptySearch.state, PAGE_STATE.EMPTY, 'empty search page => empty'); + +// empty with explicit "0 results" text +const zeroResults = classifyLinkedInPage({ + title: '0 results for AI Architect in Mauritius | LinkedIn', + url: 'https://www.linkedin.com/jobs/search/?keywords=AI%20Architect&location=Mauritius', + text: '0 results for AI Architect in Mauritius. Check your spelling or try broader keywords.', + isSearch: true, +}); +assert.equal(zeroResults.state, PAGE_STATE.EMPTY, '0 results page => empty'); + +// login wall on detail page +const loginDetail = classifyLinkedInPage({ + title: 'Sign In | LinkedIn', + url: 'https://www.linkedin.com/jobs/view/12345', + text: 'Sign in to LinkedIn to view this job.', +}); +assert.equal(loginDetail.state, PAGE_STATE.LOGIN_REQUIRED); + +// login page BUT with logged-in nav bar (should be healthy) +const loggedInHomepage = classifyLinkedInPage({ + title: 'LinkedIn', + url: 'https://www.linkedin.com/feed/', + text: 'Home My Network Jobs Messaging Notifications Sign out', +}); +assert.equal(loggedInHomepage.state, PAGE_STATE.HEALTHY, 'logged-in nav => healthy, not login_required'); + +// ── New: classifyRunStatus ─────────────────────────────────────────── + +assert.equal(classifyRunStatus([PAGE_STATE.HEALTHY, PAGE_STATE.HEALTHY]), PAGE_STATE.HEALTHY); +assert.equal(classifyRunStatus([PAGE_STATE.HEALTHY, PAGE_STATE.EMPTY]), PAGE_STATE.EMPTY); +assert.equal(classifyRunStatus([PAGE_STATE.ACTIVE_CHALLENGE]), PAGE_STATE.ACTIVE_CHALLENGE); +assert.equal(classifyRunStatus([PAGE_STATE.HEALTHY, PAGE_STATE.ACTIVE_CHALLENGE]), PAGE_STATE.ACTIVE_CHALLENGE); +assert.equal(classifyRunStatus([PAGE_STATE.HEALTHY, PAGE_STATE.RATE_LIMITED]), PAGE_STATE.RATE_LIMITED); +assert.equal(classifyRunStatus([PAGE_STATE.HEALTHY, PAGE_STATE.LOGIN_REQUIRED]), PAGE_STATE.LOGIN_REQUIRED); +assert.equal(classifyRunStatus([PAGE_STATE.HEALTHY, PAGE_STATE.TRANSIENT_ERROR]), PAGE_STATE.TRANSIENT_ERROR); +assert.equal(classifyRunStatus([]), PAGE_STATE.EMPTY); + +// ── New: isBlockingState / isCircuitBreakerState ───────────────────── + +assert.equal(isBlockingState(PAGE_STATE.ACTIVE_CHALLENGE), true); +assert.equal(isBlockingState(PAGE_STATE.BLOCKED), true); +assert.equal(isBlockingState(PAGE_STATE.RATE_LIMITED), true); +assert.equal(isBlockingState(PAGE_STATE.TRANSIENT_ERROR), true); +assert.equal(isBlockingState(PAGE_STATE.HEALTHY), false); +assert.equal(isBlockingState(PAGE_STATE.EMPTY), false); +assert.equal(isBlockingState(PAGE_STATE.LOGIN_REQUIRED), false); + +assert.equal(isCircuitBreakerState(PAGE_STATE.ACTIVE_CHALLENGE), true); +assert.equal(isCircuitBreakerState(PAGE_STATE.BLOCKED), true); +assert.equal(isCircuitBreakerState(PAGE_STATE.RATE_LIMITED), true); +assert.equal(isCircuitBreakerState(PAGE_STATE.TRANSIENT_ERROR), false); +assert.equal(isCircuitBreakerState(PAGE_STATE.HEALTHY), false); + +console.log('page-state tests: PASS'); diff --git a/skills/linkedin-job-search/scripts/test-retry-policy.mjs b/skills/linkedin-job-search/scripts/test-retry-policy.mjs new file mode 100644 index 0000000..0050907 --- /dev/null +++ b/skills/linkedin-job-search/scripts/test-retry-policy.mjs @@ -0,0 +1,123 @@ +#!/usr/bin/env node +/** + * Unit tests for retry-policy.mjs — pure module, no CDP/browser/network. + */ +import assert from 'node:assert/strict'; +import { createRetryPolicy } from './retry-policy.mjs'; + +// ── Basic: first attempt always allowed ────────────────────────────── +{ + const rp = createRetryPolicy({ maxRetries: 3, circuitBreakerThreshold: 3 }); + const d = rp.canRetry('job-1', 'linkedin.com'); + assert.equal(d.allowed, true, 'first attempt allowed'); +} + +// ── Retry exhaustion: key eventually runs out ──────────────────────── +{ + const rp = createRetryPolicy({ maxRetries: 3, circuitBreakerThreshold: 10 }); + const key = 'job-exhaust'; + const origin = 'linkedin.com'; + + assert.equal(rp.canRetry(key, origin).allowed, true); + rp.recordResult(key, origin, 'active_challenge', 'captcha'); + assert.equal(rp.canRetry(key, origin).allowed, true); + rp.recordResult(key, origin, 'active_challenge', 'captcha'); + assert.equal(rp.canRetry(key, origin).allowed, true); + rp.recordResult(key, origin, 'active_challenge', 'captcha'); + // 3 attempts exhausted + const d = rp.canRetry(key, origin); + assert.equal(d.allowed, false, 'exhausted after maxRetries'); + assert.ok(d.reason.includes('exhausted'), 'reason mentions exhausted'); +} + +// ── Distinct keys are tracked independently ────────────────────────── +{ + const rp = createRetryPolicy({ maxRetries: 3, circuitBreakerThreshold: 10 }); + const origin = 'linkedin.com'; + + rp.recordResult('a', origin, 'active_challenge', 'x'); + rp.recordResult('a', origin, 'active_challenge', 'x'); + rp.recordResult('a', origin, 'active_challenge', 'x'); + assert.equal(rp.canRetry('a', origin).allowed, false, 'key a exhausted'); + + assert.equal(rp.canRetry('b', origin).allowed, true, 'key b still allowed'); + rp.recordResult('b', origin, 'healthy', null); + assert.equal(rp.canRetry('b', origin).allowed, true, 'key b has 1 attempt left'); +} + +// ── Circuit breaker trips on consecutive blocking states ───────────── +{ + const rp = createRetryPolicy({ maxRetries: 10, circuitBreakerThreshold: 3 }); + const origin = 'linkedin.com'; + + // 3 consecutive blocking states from different keys => circuit broken + rp.recordResult('page-1', origin, 'active_challenge', 'captcha on page 1'); + assert.equal(rp.isCircuitBroken(origin), false); + rp.recordResult('page-2', origin, 'blocked', 'blocked on page 2'); + assert.equal(rp.isCircuitBroken(origin), false); + rp.recordResult('page-3', origin, 'blocked', 'blocked on page 3'); + assert.equal(rp.isCircuitBroken(origin), true, 'circuit broken after 3 consecutive blocks'); + + const d = rp.canRetry('page-4', origin); + assert.equal(d.allowed, false, 'new key denied because circuit broken'); + assert.ok(d.reason.includes('Circuit broken'), 'reason mentions circuit broken'); +} + +// ── Circuit breaker does NOT trip on non-blocking states ───────────── +{ + const rp = createRetryPolicy({ maxRetries: 10, circuitBreakerThreshold: 3 }); + const origin = 'linkedin.com'; + + rp.recordResult('page-1', origin, 'active_challenge', 'x'); + rp.recordResult('page-2', origin, 'healthy', null); + assert.equal(rp.isCircuitBroken(origin), false, 'healthy resets consecutive count'); + + rp.recordResult('page-3', origin, 'transient_error', 'timeout'); + assert.equal(rp.isCircuitBroken(origin), false, 'transient_error not a circuit-breaker state'); + + rp.recordResult('page-4', origin, 'active_challenge', 'x'); + rp.recordResult('page-5', origin, 'rate_limited', 'x'); + rp.recordResult('page-6', origin, 'blocked', 'x'); + assert.equal(rp.isCircuitBroken(origin), true, '3 consecutive blocks after reset => broken'); +} + +// ── Different origins have independent circuits ────────────────────── +{ + const rp = createRetryPolicy({ maxRetries: 10, circuitBreakerThreshold: 2 }); + const origin1 = 'linkedin.com'; + const origin2 = 'linkedin-detail'; + + rp.recordResult('a', origin1, 'active_challenge', 'x'); + rp.recordResult('b', origin1, 'blocked', 'x'); + assert.equal(rp.isCircuitBroken(origin1), true, 'origin1 broken'); + assert.equal(rp.isCircuitBroken(origin2), false, 'origin2 still healthy'); +} + +// ── getStats returns correct snapshot ───────────────────────────────── +{ + const rp = createRetryPolicy({ maxRetries: 3, circuitBreakerThreshold: 3 }); + rp.recordResult('k1', 'o1', 'active_challenge', 'test'); + rp.recordResult('k1', 'o1', 'healthy', null); + rp.recordResult('k2', 'o2', 'blocked', 'test'); + + const stats = rp.getStats(); + assert.equal(stats.keys.length, 2); + assert.equal(stats.origins.length, 2); + assert.equal(stats.maxRetries, 3); + assert.equal(stats.circuitBreakerThreshold, 3); + + const k1 = stats.keys.find((k) => k.key === 'k1'); + assert.equal(k1.attempts, 2); + assert.deepEqual(k1.states, ['active_challenge', 'healthy']); + assert.equal(k1.exhausted, false); + + const o1 = stats.origins.find((o) => o.origin === 'o1'); + assert.equal(o1.consecutiveBlocking, 0, 'healthy reset consecutive'); + assert.equal(o1.broken, false); + + const o2 = stats.origins.find((o) => o.origin === 'o2'); + assert.equal(o2.consecutiveBlocking, 1); + assert.equal(o2.broken, false); +} + +console.log('retry-policy tests: PASS'); diff --git a/skills/linkedin-job-search/scripts/test-save-to-sqlite.mjs b/skills/linkedin-job-search/scripts/test-save-to-sqlite.mjs new file mode 100644 index 0000000..f0e9f6d --- /dev/null +++ b/skills/linkedin-job-search/scripts/test-save-to-sqlite.mjs @@ -0,0 +1,342 @@ +#!/usr/bin/env node +import { Database } from '../../job-hunter/scripts/workspace-dependencies.mjs'; +/** + * Focused tests for save-to-sqlite.mjs: role_taxonomy_version column + * migration, persistence with new/refreshed classifications, and + * preservation on descriptionless updates. + * + * Uses :memory: databases only — no canonical DB is touched. + */ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { execSync } from 'node:child_process'; + +import { normalizeJob } from './save-to-sqlite.mjs'; +import * as roleTaxonomy from '../../job-hunter/scripts/role-taxonomy.mjs'; + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + passed++; + console.log(` ✓ ${name}`); + } catch (err) { + failed++; + console.error(` ✗ ${name}`); + console.error(` ${err.message}`); + } +} + +const SCHEMA_SQL = `PRAGMA foreign_keys = ON; + CREATE TABLE jobs ( + source TEXT NOT NULL, job_id TEXT NOT NULL, + url TEXT, title TEXT, company TEXT, + description_raw TEXT, description_text TEXT, + location_raw TEXT, country_code TEXT, region TEXT, city TEXT, + job_posting_date TEXT, applicants_raw TEXT, applicants_count INTEGER, + application_links_json TEXT, recruiter TEXT, recruiter_email TEXT, + recruiter_profile_link TEXT, role_family_inferred TEXT, + role_family_confidence REAL, role_family_reason TEXT, + language_filter_reason TEXT, work_mode_reason TEXT, + searched_keywords TEXT, searched_location TEXT, + application_status TEXT NOT NULL DEFAULT 'saved', applied_at TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source, job_id) + ); + CREATE TABLE job_languages ( + source TEXT NOT NULL, job_id TEXT NOT NULL, language TEXT NOT NULL, + importance TEXT NOT NULL CHECK (importance IN ('required', 'nice_to_have')), + PRIMARY KEY (source, job_id, language, importance) + ); + CREATE TABLE job_work_modes ( + source TEXT NOT NULL, job_id TEXT NOT NULL, + work_mode TEXT NOT NULL, is_primary INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (source, job_id, work_mode) + );`; + +const ROLE_LABELS = roleTaxonomy.ROLE_LABELS; +const ROLE_LABEL_SQL = ROLE_LABELS.map((l) => `'${l.replaceAll("'", "''")}'`).join(', '); + +const UPSERT_SQL = ` + INSERT INTO jobs ( + source, job_id, url, title, company, description_raw, description_text, + location_raw, country_code, region, city, job_posting_date, + applicants_raw, applicants_count, application_links_json, + recruiter, recruiter_email, recruiter_profile_link, + role_family_inferred, role_family_confidence, role_family_reason, + language_filter_reason, work_mode_reason, + searched_keywords, searched_location, role_taxonomy_version + ) VALUES ( + @source, @jobId, @url, @title, @company, @descriptionRaw, @descriptionText, + @locationRaw, @countryCode, @region, @city, @jobPostingDate, + @applicantsRaw, @applicantsCount, @applicationLinksJson, + @recruiter, @recruiterEmail, @recruiterProfileLink, + @roleFamilyInferred, @roleFamilyConfidence, @roleFamilyReason, + @languageFilterReason, @workModeReason, + @searchedKeywords, @searchedLocation, @roleTaxonomyVersion + ) + ON CONFLICT(source, job_id) DO UPDATE SET + url = excluded.url, + title = excluded.title, + company = excluded.company, + description_raw = excluded.description_raw, + description_text = excluded.description_text, + location_raw = excluded.location_raw, + country_code = excluded.country_code, + region = excluded.region, + city = excluded.city, + job_posting_date = excluded.job_posting_date, + applicants_raw = excluded.applicants_raw, + applicants_count = excluded.applicants_count, + application_links_json = excluded.application_links_json, + recruiter = excluded.recruiter, + recruiter_email = excluded.recruiter_email, + recruiter_profile_link = excluded.recruiter_profile_link, + role_family_inferred = CASE + WHEN NULLIF(trim(excluded.description_text), '') IS NULL + AND role_family_inferred IN (${ROLE_LABEL_SQL}) + THEN COALESCE(role_family_inferred, excluded.role_family_inferred) + ELSE COALESCE(excluded.role_family_inferred, role_family_inferred) + END, + role_family_confidence = CASE + WHEN NULLIF(trim(excluded.description_text), '') IS NULL + AND role_family_inferred IN (${ROLE_LABEL_SQL}) + THEN COALESCE(role_family_confidence, excluded.role_family_confidence) + ELSE COALESCE(excluded.role_family_confidence, role_family_confidence) + END, + role_family_reason = CASE + WHEN NULLIF(trim(excluded.description_text), '') IS NULL + AND role_family_inferred IN (${ROLE_LABEL_SQL}) + THEN COALESCE(role_family_reason, excluded.role_family_reason) + ELSE COALESCE(excluded.role_family_reason, role_family_reason) + END, + role_taxonomy_version = CASE + WHEN NULLIF(trim(excluded.description_text), '') IS NULL + AND role_family_inferred IN (${ROLE_LABEL_SQL}) + THEN COALESCE(role_taxonomy_version, excluded.role_taxonomy_version) + ELSE COALESCE(excluded.role_taxonomy_version, role_taxonomy_version) + END, + language_filter_reason = excluded.language_filter_reason, + work_mode_reason = excluded.work_mode_reason, + searched_keywords = excluded.searched_keywords, + searched_location = excluded.searched_location +`; + +// ── normalizeJob includes roleTaxonomyVersion ────────────────────────── + +test('normalizeJob includes roleTaxonomyVersion field', () => { + const job = normalizeJob({ + source: 'linkedin', + job_id: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', + descriptionText: 'Own AI architecture for production systems', + searchedKeywords: 'AI Architect', + }); + assert.ok('row' in job, 'normalizeJob returns { row, languages, workModes }'); + assert.ok('roleTaxonomyVersion' in job.row, 'row must have roleTaxonomyVersion'); +}); + +test('normalizeJob respects explicit roleTaxonomyVersion from input', () => { + const job = normalizeJob({ + source: 'linkedin', + job_id: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', + descriptionText: 'Own AI architecture', + searchedKeywords: 'AI Architect', + roleTaxonomyVersion: 'test-v42', + }); + assert.equal(job.row.roleTaxonomyVersion, 'test-v42'); +}); + +// ── Column migration: ALTER TABLE adds role_taxonomy_version ────────── + +test('ALTER TABLE adds role_taxonomy_version when missing', () => { + const db = new Database(':memory:'); + db.exec(SCHEMA_SQL); + const colsBefore = db.prepare(`PRAGMA table_info(jobs)`).all().map((c) => c.name); + assert.ok(!colsBefore.includes('role_taxonomy_version'), 'column must not exist before migration'); + db.exec(`ALTER TABLE jobs ADD COLUMN role_taxonomy_version TEXT`); + const colsAfter = db.prepare(`PRAGMA table_info(jobs)`).all().map((c) => c.name); + assert.ok(colsAfter.includes('role_taxonomy_version'), 'column must exist after ALTER TABLE'); + db.close(); +}); + +// ── New rows get ROLE_TAXONOMY_VERSION ──────────────────────────────── + +test('new row with description gets role_taxonomy_version on insert', () => { + const db = new Database(':memory:'); + db.exec(SCHEMA_SQL); + db.exec(`ALTER TABLE jobs ADD COLUMN role_taxonomy_version TEXT`); + const stmt = db.prepare(UPSERT_SQL); + stmt.run({ + source: 'linkedin', jobId: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', company: 'TestCo', + descriptionRaw: '
desc
', descriptionText: 'AI architecture for production', + locationRaw: null, countryCode: null, region: null, city: null, jobPostingDate: null, + applicantsRaw: null, applicantsCount: null, applicationLinksJson: null, + recruiter: null, recruiterEmail: null, recruiterProfileLink: null, + roleFamilyInferred: 'Exact architecture', roleFamilyConfidence: 0.85, + roleFamilyReason: 'AI architecture explicit', + languageFilterReason: null, workModeReason: null, + searchedKeywords: 'AI Architect', searchedLocation: 'Switzerland', + roleTaxonomyVersion: 'v1.0', + }); + const row = db.prepare(`SELECT role_taxonomy_version FROM jobs WHERE source = 'linkedin' AND job_id = '9000001003'`).get(); + assert.equal(row.role_taxonomy_version, 'v1.0'); + db.close(); +}); + +// ── Descriptionless update preserves existing role_taxonomy_version ── + +test('descriptionless update preserves existing role_taxonomy_version', () => { + const db = new Database(':memory:'); + db.exec(SCHEMA_SQL); + db.exec(`ALTER TABLE jobs ADD COLUMN role_taxonomy_version TEXT`); + const stmt = db.prepare(UPSERT_SQL); + + // Insert with description and version + stmt.run({ + source: 'linkedin', jobId: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', company: 'TestCo', + descriptionRaw: '
desc
', descriptionText: 'AI architecture for production', + locationRaw: null, countryCode: null, region: null, city: null, jobPostingDate: null, + applicantsRaw: null, applicantsCount: null, applicationLinksJson: null, + recruiter: null, recruiterEmail: null, recruiterProfileLink: null, + roleFamilyInferred: 'Exact architecture', roleFamilyConfidence: 0.85, + roleFamilyReason: 'AI architecture explicit', + languageFilterReason: null, workModeReason: null, + searchedKeywords: 'AI Architect', searchedLocation: 'Switzerland', + roleTaxonomyVersion: 'v1.0', + }); + + // Update with empty description (listing-only refresh) and null version + stmt.run({ + source: 'linkedin', jobId: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', company: 'TestCo', + descriptionRaw: null, descriptionText: '', + locationRaw: null, countryCode: null, region: null, city: null, jobPostingDate: null, + applicantsRaw: null, applicantsCount: null, applicationLinksJson: null, + recruiter: null, recruiterEmail: null, recruiterProfileLink: null, + roleFamilyInferred: 'Exact architecture', roleFamilyConfidence: 0.85, + roleFamilyReason: 'AI architecture explicit', + languageFilterReason: null, workModeReason: null, + searchedKeywords: 'AI Architect', searchedLocation: 'Switzerland', + roleTaxonomyVersion: null, + }); + + const row = db.prepare(`SELECT role_taxonomy_version FROM jobs WHERE source = 'linkedin' AND job_id = '9000001003'`).get(); + assert.equal(row.role_taxonomy_version, 'v1.0', 'version must be preserved on descriptionless update'); + db.close(); +}); + +// ── Refreshed (with description) update sets new role_taxonomy_version ─ + +test('refreshed row with description gets updated role_taxonomy_version', () => { + const db = new Database(':memory:'); + db.exec(SCHEMA_SQL); + db.exec(`ALTER TABLE jobs ADD COLUMN role_taxonomy_version TEXT`); + const stmt = db.prepare(UPSERT_SQL); + + // Insert with old version + stmt.run({ + source: 'linkedin', jobId: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'Stale Title', company: 'TestCo', + descriptionRaw: '
old
', descriptionText: 'old description', + locationRaw: null, countryCode: null, region: null, city: null, jobPostingDate: null, + applicantsRaw: null, applicantsCount: null, applicationLinksJson: null, + recruiter: null, recruiterEmail: null, recruiterProfileLink: null, + roleFamilyInferred: 'Exact architecture', roleFamilyConfidence: 0.8, + roleFamilyReason: 'old reason', + languageFilterReason: null, workModeReason: null, + searchedKeywords: 'AI Architect', searchedLocation: 'Switzerland', + roleTaxonomyVersion: 'v1.0', + }); + + // Refresh with new description and new version + stmt.run({ + source: 'linkedin', jobId: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', company: 'TestCo', + descriptionRaw: '
new
', descriptionText: 'updated AI architecture description', + locationRaw: null, countryCode: null, region: null, city: null, jobPostingDate: null, + applicantsRaw: null, applicantsCount: null, applicationLinksJson: null, + recruiter: null, recruiterEmail: null, recruiterProfileLink: null, + roleFamilyInferred: 'Exact architecture', roleFamilyConfidence: 0.9, + roleFamilyReason: 'updated reason', + languageFilterReason: null, workModeReason: null, + searchedKeywords: 'AI Architect', searchedLocation: 'Switzerland', + roleTaxonomyVersion: 'v2.0', + }); + + const row = db.prepare(`SELECT role_taxonomy_version FROM jobs WHERE source = 'linkedin' AND job_id = '9000001003'`).get(); + assert.equal(row.role_taxonomy_version, 'v2.0', 'version must be updated when description is present'); + db.close(); +}); + +// ── Old rows remain null until next touch (no bulk rewrite) ────────── + +test('old rows without role_taxonomy_version remain null', () => { + const db = new Database(':memory:'); + db.exec(SCHEMA_SQL); + db.exec(`ALTER TABLE jobs ADD COLUMN role_taxonomy_version TEXT`); + + // Insert a historical row directly (without using the upsert, simulating old data) + db.prepare(`INSERT INTO jobs (source, job_id, title, description_text, application_status) + VALUES ('linkedin', '9999999999', 'Old Job', 'old desc', 'saved')`).run(); + + const row = db.prepare(`SELECT role_taxonomy_version FROM jobs WHERE source = 'linkedin' AND job_id = '9999999999'`).get(); + assert.equal(row.role_taxonomy_version, null, 'historical row must have null version until next touch'); + db.close(); +}); + +// ── CLI integration: save-to-sqlite.mjs end-to-end with temp DB ─────── + +test('CLI save-to-sqlite.mjs adds column and persists version', () => { + const tmpDir = mkdtempSync(join(tmpdir(), 'save-test-')); + const dbPath = join(tmpDir, 'test.sqlite'); + const jsonPath = join(tmpDir, 'jobs.json'); + + const jobs = [{ + source: 'linkedin', + job_id: '9000001003', + url: 'https://linkedin.example/jobs/view/9999999999', + title: 'AI Architect', + company: 'TestCo', + descriptionText: 'AI architecture for production systems', + searchedKeywords: 'AI Architect', + searchedLocation: 'Switzerland', + roleTaxonomyVersion: 'cli-test-v1', + }]; + writeFileSync(jsonPath, JSON.stringify(jobs)); + + const schemaPath = join(process.cwd(), 'skills/linkedin-job-search/schema.sql'); + execSync( + `node skills/linkedin-job-search/scripts/save-to-sqlite.mjs "${jsonPath}" --db "${dbPath}" --schema "${schemaPath}"`, + { encoding: 'utf8', cwd: process.cwd() }, + ); + + const db = new Database(dbPath); + const cols = db.prepare(`PRAGMA table_info(jobs)`).all().map((c) => c.name); + assert.ok(cols.includes('role_taxonomy_version'), 'column must exist after CLI save'); + const row = db.prepare(`SELECT role_taxonomy_version FROM jobs WHERE source = 'linkedin' AND job_id = '9000001003'`).get(); + assert.equal(row.role_taxonomy_version, 'cli-test-v1', 'version must be persisted from input'); + db.close(); + rmSync(tmpDir, { recursive: true, force: true }); +}); + +// ── Run ─────────────────────────────────────────────────────────────── + +console.log('\n── test-save-to-sqlite.mjs ──'); +console.log(` Passed: ${passed}, Failed: ${failed}\n`); +if (failed > 0) process.exit(1); diff --git a/skills/linkedin-job-search/scripts/test-search-linkedin-jobs.mjs b/skills/linkedin-job-search/scripts/test-search-linkedin-jobs.mjs new file mode 100644 index 0000000..05f00a6 --- /dev/null +++ b/skills/linkedin-job-search/scripts/test-search-linkedin-jobs.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node +/** + * Focused tests for the LinkedIn search slice: shared query expansion, + * --refresh-job-ids parsing/validation, query-family provenance, and + * existing-ID suppression bypass for refresh IDs. + * + * No live CDP calls — all fixture-based. + */ +import assert from 'node:assert/strict'; + +import { + buildQueries, + bypassRefreshIds, + mergeExplicitRefreshIds, + getQueryFamily, + parseRefreshJobIds, + toDbRecord, +} from './search-linkedin-jobs.mjs'; +import * as roleTaxonomy from '../../job-hunter/scripts/role-taxonomy.mjs'; + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + passed++; + console.log(` ✓ ${name}`); + } catch (err) { + failed++; + console.error(` ✗ ${name}`); + console.error(` ${err.message}`); + } +} + +// ── buildQueries: consumes shared taxonomy expansion ────────────────── + +const taxonomyQueries = roleTaxonomy.expandRoleQueries({ + targetRole: 'AI Architect', + similarRoles: [], + maxQueries: 32, +}); + +test('buildQueries returns taxonomy-expanded queries for AI Architect role', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: [], + roleVariants: true, + }); + assert.ok(queries.length > 5, 'should produce more than the old 5-title list'); + for (const tq of taxonomyQueries) { + assert.ok(queries.includes(tq), `taxonomy query "${tq}" missing from buildQueries output`); + } +}); + +test('buildQueries no longer maintains a private five-title list', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: [], + roleVariants: true, + }); + // Old private list: Artificial Intelligence Architect, AI Solution Architect, + // Enterprise AI Architect, GenAI Architect, Data AI Architect. + // These must now come from the taxonomy expansion, not a private list. + assert.ok(queries.includes('Enterprise AI Architect'), 'Enterprise AI Architect from taxonomy'); + assert.ok(queries.includes('GenAI Architect'), 'GenAI Architect from taxonomy'); +}); + +test('buildQueries respects 32-query hard cap', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: [], + roleVariants: true, + }); + assert.ok(queries.length <= 32, `got ${queries.length}, expected <= 32`); +}); + +test('buildQueries includes supplementary families not in taxonomy', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: [], + roleVariants: true, + }); + assert.ok(queries.includes('Applied AI Architect'), 'Applied AI Architect supplementary'); + assert.ok(queries.includes('Forward Deployed Architect'), 'Forward Deployed Architect supplementary'); + assert.ok(queries.includes('Forward Deployed Engineer'), 'Forward Deployed Engineer supplementary'); +}); + +test('buildQueries with --no-role-variants returns only user-supplied', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: ['AI Platform Lead'], + roleVariants: false, + }); + assert.deepEqual(queries, ['AI Architect', 'AI Platform Lead']); +}); + +test('buildQueries with explicit queries bypasses expansion', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: [], + roleVariants: true, + queries: ['Custom Query', 'Another Query'], + }); + assert.deepEqual(queries, ['Custom Query', 'Another Query']); +}); + +// ── getQueryFamily: query-family provenance ──────────────────────────── + +test('getQueryFamily returns correct families for known queries', () => { + assert.equal(getQueryFamily('AI Architect'), 'core-architect'); + assert.equal(getQueryFamily('AI Platform Architect'), 'platform-mlops'); + assert.equal(getQueryFamily('MLOps Architect'), 'platform-mlops'); + assert.equal(getQueryFamily('Principal AI Engineer'), 'principal-staff-lead'); + assert.equal(getQueryFamily('AI Solutions Architect'), 'solutions-field'); + assert.equal(getQueryFamily('Head of AI Engineering'), 'leadership'); + assert.equal(getQueryFamily('AI Security Specialist'), 'security-governance'); + assert.equal(getQueryFamily('Data & AI Architect'), 'data-ai'); + assert.equal(getQueryFamily('Enterprise AI Architect'), 'enterprise'); + assert.equal(getQueryFamily('GenAI Architect'), 'generative-ai'); + assert.equal(getQueryFamily('Applied AI Architect'), 'applied-ai-architect'); + assert.equal(getQueryFamily('Forward Deployed Architect'), 'forward-deployed'); +}); + +test('getQueryFamily returns user-supplied for unrecognized queries', () => { + assert.equal(getQueryFamily('Some Random Title'), 'user-supplied'); + assert.equal(getQueryFamily(''), 'user-supplied'); +}); + +// ── parseRefreshJobIds: parsing, validation, dedup, cap ──────────────── + +test('parseRefreshJobIds parses comma-separated numeric IDs', () => { + const ids = parseRefreshJobIds('9000000002,9000000003'); + assert.deepEqual(ids, ['9000000002', '9000000003']); +}); + +test('parseRefreshJobIds handles whitespace around IDs', () => { + const ids = parseRefreshJobIds(' 9000000002 , 9000000003 '); + assert.deepEqual(ids, ['9000000002', '9000000003']); +}); + +test('parseRefreshJobIds deduplicates', () => { + const ids = parseRefreshJobIds('9000000002,9000000002,9000000003'); + assert.deepEqual(ids, ['9000000002', '9000000003']); +}); + +test('parseRefreshJobIds rejects non-numeric IDs', () => { + assert.throws( + () => parseRefreshJobIds('9000000002,abc'), + /must be numeric/, + ); +}); + +test('parseRefreshJobIds enforces 50-ID cap', () => { + const many = Array.from({ length: 51 }, (_, i) => String(9000001001 + i)).join(','); + assert.throws( + () => parseRefreshJobIds(many), + /max 50/, + ); +}); + +test('parseRefreshJobIds accepts exactly 50 IDs', () => { + const fifty = Array.from({ length: 50 }, (_, i) => String(9000001001 + i)).join(','); + const ids = parseRefreshJobIds(fifty); + assert.equal(ids.length, 50); +}); + +test('parseRefreshJobIds returns empty array for empty/null input', () => { + assert.deepEqual(parseRefreshJobIds(''), []); + assert.deepEqual(parseRefreshJobIds(null), []); + assert.deepEqual(parseRefreshJobIds(undefined), []); +}); + +// ── bypassRefreshIds: existing-ID suppression bypass ────────────────── + +test('bypassRefreshIds removes refresh IDs from existing set', () => { + const existing = new Set(['9000000002', '9000001005', '9999999999']); + const result = bypassRefreshIds(existing, new Set(['9000000002'])); + assert.ok(!result.has('9000000002'), 'refresh ID must be removed from existing set'); + assert.ok(result.has('9000001005'), 'non-refresh ID must remain'); + assert.ok(result.has('9999999999'), 'non-refresh ID must remain'); +}); + +test('bypassRefreshIds does not alter normal deduplication for other jobs', () => { + const existing = new Set(['111', '222', '333']); + const result = bypassRefreshIds(existing, new Set(['222'])); + assert.equal(result.size, 2); + assert.ok(result.has('111')); + assert.ok(result.has('333')); + assert.ok(!result.has('222')); +}); + +test('bypassRefreshIds returns original set when no refresh IDs', () => { + const existing = new Set(['111', '222']); + const result = bypassRefreshIds(existing, null); + assert.equal(result.size, 2); + assert.ok(result.has('111')); + assert.ok(result.has('222')); +}); + +test('mergeExplicitRefreshIds schedules and prioritizes every requested ID', () => { + const results = mergeExplicitRefreshIds([ + { query: 'AI Architect', ids: ['111', '9000000002'], pages: [], status: 'healthy' }, + ], ['9000000002', '9000000003']); + assert.equal(results.length, 2); + assert.deepEqual(results[0], { + query: 'Explicit LinkedIn ID refresh', + ids: ['9000000002', '9000000003'], + pages: [], + status: 'healthy', + }); + assert.deepEqual(results[1].ids, ['111']); + assert.equal(getQueryFamily(results[0].query), 'explicit-refresh'); +}); + +test('mergeExplicitRefreshIds is a no-op when no IDs are requested', () => { + const source = [{ query: 'AI Architect', ids: ['9000000002'], pages: [], status: 'healthy' }]; + const results = mergeExplicitRefreshIds(source, []); + assert.deepEqual(results, source); +}); + +// ── toDbRecord: includes roleTaxonomyVersion ─────────────────────────── + +test('toDbRecord includes roleTaxonomyVersion in output', () => { + const record = toDbRecord({ + source: 'linkedin', + job_id: '123', + url: 'https://www.linkedin.com/jobs/view/123', + title: 'AI Architect', + descriptionText: 'AI architecture for production systems', + searchedKeywords: 'AI Architect', + }); + assert.ok('roleTaxonomyVersion' in record, 'roleTaxonomyVersion field must exist'); + // Value is SHARED_TAXONOMY_VERSION — null if taxonomy worker hasn't exported it yet, + // otherwise a non-empty string. + if (record.roleTaxonomyVersion !== null) { + assert.equal(typeof record.roleTaxonomyVersion, 'string'); + assert.ok(record.roleTaxonomyVersion.length > 0); + } +}); + +// ── Summary structure: queryFamilies and refresh counters ───────────── + +test('queryFamilies array maps each query to a family', () => { + const queries = buildQueries({ + role: 'AI Architect', + similarRoles: [], + roleVariants: true, + }); + const queryFamilies = queries.map((q) => ({ query: q, family: getQueryFamily(q) })); + assert.equal(queryFamilies.length, queries.length); + for (const entry of queryFamilies) { + assert.ok(entry.query, 'query must be non-empty'); + assert.ok(entry.family, 'family must be non-empty'); + assert.equal(typeof entry.family, 'string'); + } +}); + +test('refresh counters structure has requested/found/refreshed/missing', () => { + const refreshJobIds = ['9000000002', '9000000003']; + const ids = ['9000000002', '9999999999']; + const scrapedRecords = [ + { job_id: '9000000002', descriptionText: 'refreshed description' }, + ]; + const refresh = { + requested: refreshJobIds.length, + found: refreshJobIds.filter((id) => ids.includes(id)).length, + refreshed: refreshJobIds.filter((id) => scrapedRecords.some((r) => r.job_id === id && r.descriptionText)).length, + missing: refreshJobIds.filter((id) => !scrapedRecords.some((r) => r.job_id === id)).length, + }; + assert.equal(refresh.requested, 2); + assert.equal(refresh.found, 1); + assert.equal(refresh.refreshed, 1); + assert.equal(refresh.missing, 1); +}); + +// ── Run ─────────────────────────────────────────────────────────────── + +console.log('\n── test-search-linkedin-jobs.mjs ──'); +console.log(` Passed: ${passed}, Failed: ${failed}\n`); +if (failed > 0) process.exit(1); diff --git a/skills/linkedin-job-search/scripts/test-target-lifecycle.mjs b/skills/linkedin-job-search/scripts/test-target-lifecycle.mjs new file mode 100644 index 0000000..6e58a63 --- /dev/null +++ b/skills/linkedin-job-search/scripts/test-target-lifecycle.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node +/** + * Unit tests for target-registry.mjs — pure module, no CDP/browser/network. + */ +import assert from 'node:assert/strict'; +import { createTargetRegistry, createCleanup } from './target-registry.mjs'; + +// ── Helpers ────────────────────────────────────────────────────────── +function fakeClient() { + const closed = []; + const calls = []; + let disconnected = false; + return { + closed, + calls, + async send(method, params) { + if (disconnected) throw new Error('WebSocket is not open'); + calls.push({ method, params }); + return {}; + }, + disconnect() { disconnected = true; }, + }; +} + +// ── Registry: register/unregister owned ────────────────────────────── +{ + const reg = createTargetRegistry(); + assert.deepStrictEqual(reg.getOwned(), [], 'no owned targets initially'); + + reg.registerOwned('t1', 'search page'); + reg.registerOwned('t2', 'detail page'); + assert.deepStrictEqual(reg.getOwned().sort(), ['t1', 't2'], 'two owned registered'); + + reg.unregister('t1'); + assert.deepStrictEqual(reg.getOwned(), ['t2'], 'unregister removes owned'); +} + +// ── Registry: register/unregister borrowed ─────────────────────────── +{ + const reg = createTargetRegistry(); + reg.registerBorrowed('tab-1'); + reg.registerBorrowed('tab-2'); + assert.deepStrictEqual(reg.getBorrowed().sort(), ['tab-1', 'tab-2'], 'two borrowed'); + + reg.unregister('tab-1'); + assert.deepStrictEqual(reg.getBorrowed(), ['tab-2'], 'unregister removes borrowed'); +} + +// ── Registry: isOwned / isBorrowed ─────────────────────────────────── +{ + const reg = createTargetRegistry(); + reg.registerOwned('a'); + reg.registerBorrowed('b'); + assert.equal(reg.isOwned('a'), true); + assert.equal(reg.isOwned('b'), false); + assert.equal(reg.isBorrowed('b'), true); + assert.equal(reg.isBorrowed('a'), false); + assert.equal(reg.isOwned('unknown'), false); + assert.equal(reg.isBorrowed('unknown'), false); +} + +// ── Registry: null/undefined targetId is a no-op ───────────────────── +{ + const reg = createTargetRegistry(); + reg.registerOwned(null); + reg.registerOwned(undefined); + reg.registerBorrowed(null); + assert.deepStrictEqual(reg.getOwned(), []); + assert.deepStrictEqual(reg.getBorrowed(), []); +} + +// ── Cleanup: owned targets are closed ──────────────────────────────── +{ + const reg = createTargetRegistry(); + reg.registerOwned('to-close-1'); + reg.registerOwned('to-close-2'); + reg.registerBorrowed('keep-me'); + + const client = fakeClient(); + const cleanup = createCleanup({ registry: reg, client }); + + await cleanup(); + + // Verify owned targets were closed via CDP + const closeCalls = client.calls.filter((c) => c.method === 'Target.closeTarget'); + assert.equal(closeCalls.length, 2, 'two closeTarget calls'); + const closedIds = closeCalls.map((c) => c.params.targetId).sort(); + assert.deepStrictEqual(closedIds, ['to-close-1', 'to-close-2'], 'owned targets closed'); + + // Borrowed target was never touched + assert.deepStrictEqual(reg.getBorrowed(), ['keep-me'], 'borrowed target preserved'); + assert.deepStrictEqual(reg.getOwned(), [], 'owned targets drained after cleanup'); +} + +// ── Cleanup: idempotent — second call is a no-op ───────────────────── +{ + const reg = createTargetRegistry(); + reg.registerOwned('once'); + + const client = fakeClient(); + const cleanup = createCleanup({ registry: reg, client }); + + await cleanup(); + assert.equal(client.calls.length, 1, 'one call on first cleanup'); + + // Second call — should be a no-op + await cleanup(); + assert.equal(client.calls.length, 1, 'no extra calls on second cleanup (idempotent)'); +} + +// ── Cleanup: swallows "browser already gone" errors ────────────────── +{ + const reg = createTargetRegistry(); + reg.registerOwned('ghost'); + + const client = fakeClient(); + client.disconnect(); // simulate dead browser + + const cleanup = createCleanup({ registry: reg, client }); + + // Must not throw + await cleanup(); + // Registry still drains + assert.deepStrictEqual(reg.getOwned(), [], 'owned drained even when browser is gone'); +} + +// ── Cleanup: keepalive stop invoked ────────────────────────────────── +{ + const reg = createTargetRegistry(); + let keepaliveStopped = false; + const stopKeepalive = () => { keepaliveStopped = true; }; + + const cleanup = createCleanup({ registry: reg, stopKeepalive }); + await cleanup(); + assert.equal(keepaliveStopped, true, 'keepalive stop was called'); +} + +// ── Cleanup: client close invoked ──────────────────────────────────── +{ + const reg = createTargetRegistry(); + let clientClosed = false; + const closeClient = () => { clientClosed = true; }; + + const cleanup = createCleanup({ registry: reg, closeClient }); + await cleanup(); + assert.equal(clientClosed, true, 'client close was called'); +} + +// ── Cleanup: handles missing client/stop/close gracefully ──────────── +{ + const reg = createTargetRegistry(); + reg.registerOwned('orphan'); + + // No client provided — cleanup must still work (best-effort, no throw). + // Owned targets remain in registry because CDP closure is unavailable. + const cleanup = createCleanup({ registry: reg }); + await cleanup(); + assert.deepStrictEqual(reg.getOwned(), ['orphan'], 'owned remains when no client available'); +} + +// ── Cleanup: partial failure in one target does not block others ───── +{ + const reg = createTargetRegistry(); + reg.registerOwned('ok'); + reg.registerOwned('bad'); + reg.registerOwned('also-ok'); + + // Client that fails for 'bad' but succeeds for others + const client = { + closed: [], + calls: [], + async send(method, params) { + this.calls.push({ method, params }); + if (params.targetId === 'bad') throw new Error('Target not found'); + }, + }; + const cleanup = createCleanup({ registry: reg, client }); + await cleanup(); + + const closedIds = client.calls + .filter((c) => c.method === 'Target.closeTarget') + .map((c) => c.params.targetId) + .sort(); + assert.deepStrictEqual(closedIds, ['also-ok', 'bad', 'ok'], 'all three targets attempted'); + assert.deepStrictEqual(reg.getOwned(), [], 'all owned drained'); +} + +console.log('target-lifecycle tests: PASS'); diff --git a/skills/linkedin-job-search/scripts/visible-listing-extract.mjs b/skills/linkedin-job-search/scripts/visible-listing-extract.mjs new file mode 100644 index 0000000..943a2d7 --- /dev/null +++ b/skills/linkedin-job-search/scripts/visible-listing-extract.mjs @@ -0,0 +1,216 @@ +#!/usr/bin/env node +import { WebSocketModule } from '../../job-hunter/scripts/workspace-dependencies.mjs'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +const require = createRequire(import.meta.url); +let WebSocket; +try { WebSocket = WebSocketModule; } +catch { WebSocket = createRequire(process.env.JOBHUNTER_HOME ? path.join(process.env.JOBHUNTER_HOME, 'package.json') : path.join(process.env.HOME, '.job-hunter/package.json'))('ws'); } + +function argValue(name, fallback = null) { + const idx = process.argv.indexOf(name); + return idx >= 0 && idx + 1 < process.argv.length ? process.argv[idx + 1] : fallback; +} +function splitList(value, fallback) { + return String(value || '').split(/[;,]/).map(s => s.trim()).filter(Boolean).length + ? String(value).split(/[;,]/).map(s => s.trim()).filter(Boolean) + : fallback; +} +const RUN_DIR = argValue('--run-dir', process.env.RUN_DIR || process.cwd()); +const OUT = argValue('--out', path.join(RUN_DIR, 'visible-listings.json')); +const SUMMARY = argValue('--summary', path.join(RUN_DIR, 'visible-listings-summary.json')); +const PORT = Number(argValue('--cdp-port', process.env.BROWSER_CDP_PORT || 9225)); +const locations = splitList(argValue('--locations', process.env.LOCATIONS || ''), ['United Kingdom', 'Ireland', 'Denmark', 'Netherlands']); +const queries = splitList(argValue('--queries', process.env.QUERIES || ''), ['AI Architect', 'AI Solution Architect', 'Enterprise AI Architect', 'Generative AI Architect']); +const countryByLocation = new Map([ + ['United Kingdom', 'GB'], ['Ireland', 'IE'], ['Denmark', 'DK'], ['Netherlands', 'NL'], +]); + +const sleep = ms => new Promise(r => setTimeout(r, ms)); +async function jget(url) { const r = await fetch(url); if (!r.ok) throw new Error(`${url} HTTP ${r.status}`); return r.json(); } +async function openTab(url) { + const list = await jget(`http://127.0.0.1:${PORT}/json/list`); + let tab = list.find(t => (t.url || '').includes('linkedin.com/jobs/search') && t.webSocketDebuggerUrl); + if (!tab) { + const r = await fetch(`http://127.0.0.1:${PORT}/json/new?url=${encodeURIComponent(url)}`, { method: 'PUT' }); + tab = await r.json(); + } + return tab; +} +class Cdp { + constructor(wsUrl) { this.wsUrl = wsUrl; this.id = 1; this.pending = new Map(); } + async init() { + this.ws = new WebSocket(this.wsUrl); + await new Promise((res, rej) => { this.ws.on('open', res); this.ws.on('error', rej); }); + this.ws.on('message', raw => { + let msg; try { msg = JSON.parse(raw); } catch { return; } + if (msg.id && this.pending.has(msg.id)) { + const p = this.pending.get(msg.id); clearTimeout(p.t); this.pending.delete(msg.id); + msg.error ? p.reject(new Error(msg.error.message || JSON.stringify(msg.error))) : p.resolve(msg.result); + } + }); + } + send(method, params = {}, timeout = 30000) { + const id = this.id++; + return new Promise((resolve, reject) => { + const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`timeout ${method}`)); }, timeout); + this.pending.set(id, { resolve, reject, t }); + this.ws.send(JSON.stringify({ id, method, params })); + }); + } + close() { try { this.ws?.close(); } catch {} } +} + +function searchUrl(query, location) { + const u = new URL('https://www.linkedin.com/jobs/search/'); + u.searchParams.set('keywords', query); + u.searchParams.set('location', location); + u.searchParams.set('f_TPR', 'r604800'); + return u.toString(); +} + +async function evalJson(cdp, expression, timeout = 30000) { + const res = await cdp.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, timeout); + if (res.exceptionDetails) throw new Error(`Runtime exception: ${res.exceptionDetails.text || JSON.stringify(res.exceptionDetails)}`); + const val = res.result?.value; + return typeof val === 'string' ? JSON.parse(val) : val; +} + +const blockerExpr = `JSON.stringify((() => { + const text = (document.body?.innerText || '').slice(0, 3000); + const title = document.title || ''; + const lower = (title + '\\n' + text).toLowerCase(); + const visibleRecaptcha = Array.from(document.querySelectorAll('iframe[src*="recaptcha"], iframe[src*="checkpoint"], iframe[src*="challenge"]')).some(f => { + const r = f.getBoundingClientRect(); return r.width > 20 && r.height > 20; + }); + const blocked = visibleRecaptcha || /security verification|unusual activity|captcha|verify you|verify your|checkpoint|sign in to linkedin|join linkedin|authwall|login/.test(lower); + return { title, href: location.href, blocked, visibleRecaptcha, text: text.slice(0, 500) }; +})())`; + +const extractExpr = `JSON.stringify((() => { + const candidates = Array.from(document.querySelectorAll('li[data-occludable-job-id], [data-job-id], div.job-card-container, li.jobs-search-results__list-item, main li')).slice(0, 500); + const rows = []; + function cleanLines(text) { return (text || '').split(String.fromCharCode(10)).map(s => s.trim()).filter(Boolean); } + function idFromHref(href) { + const path = (() => { try { return new URL(href, location.href).pathname; } catch { return href || ''; } })(); + const nums = path.split(/[^0-9]+/).filter(s => s.length >= 6); + return nums.length ? nums[nums.length - 1] : ''; + } + for (const el of candidates) { + const link = el.querySelector('a[href*="/jobs/view/"]'); + let href = link ? new URL(link.href, location.href).href.split('?')[0].split('#')[0] : ''; + if (href.endsWith('/')) href = href.slice(0, -1); + const idFromAttr = el.getAttribute('data-job-id') || el.getAttribute('data-occludable-job-id') || ''; + const id = idFromAttr || idFromHref(href); + if (!href && id) href = 'https://www.linkedin.com/jobs/view/' + id; + const txt = cleanLines(el.innerText); + let title = (el.querySelector('a.job-card-container__link, a.job-card-list__title, a[href*="/jobs/view/"]')?.innerText || '').trim(); + if (!title) title = txt.find(s => s.length > 2 && !/^Promoted$|^Viewed$|^Saved$|Easy Apply|applicant/i.test(s)) || ''; + let company = (el.querySelector('.artdeco-entity-lockup__subtitle, .job-card-container__primary-description, h4')?.innerText || '').trim(); + if (!company) company = txt.find(s => s !== title && !/ago|applicant|Easy Apply|Promoted|Viewed|Saved|Remote|Hybrid|United Kingdom|Ireland|Denmark|Netherlands/i.test(s)) || ''; + let loc = (el.querySelector('.job-card-container__metadata-item, .job-card-container__metadata-wrapper')?.innerText || '').trim(); + if (!loc) loc = txt.find(s => /Remote|Hybrid|United Kingdom|Ireland|Denmark|Netherlands|UK|,/.test(s)) || ''; + const applicants = txt.find(s => /applicant/i.test(s)) || ''; + const posted = txt.find(s => /ago|reposted|promoted/i.test(s)) || ''; + if (id && title && href) rows.push({ id, title, company, locationRaw: loc, applicantsRaw: applicants, jobPostingDate: posted, url: href, text: txt.slice(0, 10) }); + } + const seen = new Set(); + return rows.filter(r => !seen.has(r.id) && seen.add(r.id)).slice(0, 50); +})())`; + +async function dismissOverlays(cdp) { + await cdp.send('Runtime.evaluate', { expression: `(() => { + for (const b of Array.from(document.querySelectorAll('button, [role=button]'))) { + const t = (b.innerText || b.ariaLabel || '').trim(); + if (/^(dismiss|close|maybe later|not now|skip)$/i.test(t)) { try { b.click(); } catch {} } + } + })()`, returnByValue: true }, 10000).catch(() => {}); +} + +async function scrollAndExtract(cdp) { + const all = []; + for (let i = 0; i < 7; i++) { + const rows = await evalJson(cdp, extractExpr, 20000).catch(e => { console.log(`[extract-warn] ${e.message}`); return []; }); + all.push(...rows); + await cdp.send('Runtime.evaluate', { expression: `(() => { + const scrollers = [document.querySelector('.jobs-search-results-list'), document.querySelector('.scaffold-layout__list'), document.querySelector('main'), document.scrollingElement].filter(Boolean); + for (const s of scrollers) { try { s.scrollTop = (s.scrollTop || 0) + 900; } catch {} } + window.scrollBy(0, 500); + })()`, returnByValue: true }, 10000).catch(() => {}); + await sleep(1200); + } + const seen = new Set(); + return all.filter(r => !seen.has(r.id) && seen.add(r.id)); +} + +const startUrl = searchUrl(queries[0], locations[0]); +const tab = await openTab(startUrl); +const cdp = new Cdp(tab.webSocketDebuggerUrl); +await cdp.init(); +await cdp.send('Page.enable', {}, 10000).catch(() => {}); + +const recordsById = new Map(); +const searchSummaries = []; +for (const location of locations) { + for (const query of queries) { + const url = searchUrl(query, location); + console.log(`\n=== visible fallback: ${query} / ${location} ===`); + console.log(url); + await cdp.send('Page.navigate', { url }, 30000).catch(e => console.log(`[navigate-warn] ${e.message}`)); + await sleep(7500); + await dismissOverlays(cdp); + const state = await evalJson(cdp, blockerExpr, 15000).catch(e => ({ blocked: true, error: e.message })); + console.log(`[state] title=${JSON.stringify(state.title)} blocked=${state.blocked} visibleRecaptcha=${state.visibleRecaptcha || false}`); + if (state.blocked) { + console.log(`[blocked-visible] ${JSON.stringify(state).slice(0, 1000)}`); + searchSummaries.push({ query, location, blocked: true, state }); + continue; + } + const rows = await scrollAndExtract(cdp); + console.log(`[extract] ${rows.length} unique visible cards`); + searchSummaries.push({ query, location, blocked: false, count: rows.length, ids: rows.map(r => r.id).slice(0, 20) }); + for (const r of rows) { + if (recordsById.has(r.id)) continue; + const textJoined = (r.text || []).join(' | '); + const modes = []; + if (/remote/i.test(`${r.locationRaw} ${textJoined}`)) modes.push({ mode: 'remote', isPrimary: true }); + if (/hybrid/i.test(`${r.locationRaw} ${textJoined}`)) modes.push({ mode: 'hybrid', isPrimary: modes.length === 0 }); + if (!modes.length && /onsite|on-site/i.test(`${r.locationRaw} ${textJoined}`)) modes.push({ mode: 'onsite', isPrimary: true }); + recordsById.set(r.id, { + source: 'linkedin', + job_id: r.id, + url: r.url, + title: r.title, + company: r.company || null, + locationRaw: r.locationRaw || location, + countryCode: countryByLocation.get(location) || null, + region: null, + city: null, + applicantsRaw: r.applicantsRaw || null, + applicantsCount: r.applicantsRaw ? Number((r.applicantsRaw.match(/\d+/)||[])[0] || 0) || null : null, + descriptionRaw: '', + descriptionText: '', + applicationLinks: [r.url], + recruiter: null, + recruiterEmail: null, + recruiterProfileLink: null, + jobPostingDate: r.jobPostingDate || null, + languageRequirements: { required: [], niceToHave: [] }, + workModes: modes, + languageFilterReason: 'listing-only visible LinkedIn fallback; JD backfill pending', + roleFamilyInferred: null, + roleFamilyReason: 'visible LinkedIn search card fallback after CDP runner security/backoff signal', + searchedKeywords: query, + searchedLocation: location, + }); + } + await sleep(1500); + } +} +cdp.close(); +const records = Array.from(recordsById.values()); +fs.writeFileSync(OUT, JSON.stringify(records, null, 2)); +fs.writeFileSync(SUMMARY, JSON.stringify({ generatedAt: new Date().toISOString(), count: records.length, searches: searchSummaries }, null, 2)); +console.log(`\nWrote ${OUT} (${records.length} records)`); +console.log(`Wrote ${SUMMARY}`); diff --git a/skills/obscura-mcp-repair/SKILL.md b/skills/obscura-mcp-repair/SKILL.md new file mode 100644 index 0000000..673d334 --- /dev/null +++ b/skills/obscura-mcp-repair/SKILL.md @@ -0,0 +1,149 @@ +--- +name: obscura-mcp-repair +description: Repairs Pi MCP configuration for Obscura browser automation when calls fail with "Not connected", "CDP connection closed", "CDP connection is not open", or metadata shows Obscura tools but runtime calls fail. Use before Obscura browsing, scraping, hotel searches, or LinkedIn/login flows when Obscura MCP is broken. +allowed-tools: read bash edit write mcp +--- + +# Obscura MCP Repair + +## Global Obscura Reuse Rule + +Before repairing, check for an existing local Obscura CDP browser and preserve it if reachable. Do not kill or start a second `obscura serve` instance just to repair MCP metadata. Use `--kill-stale` only when the existing Obscura/Obscura-MCP processes are confirmed stale or broken. + +Use this skill when the Pi MCP gateway can list Obscura tools but calls such as `obscura_browse_page` fail, especially with: + +- `Failed to call tool: Not connected` +- `Execution Error: CDP connection closed` +- `Execution Error: CDP connection is not open` +- `Obscura CDP client is not connected` +- `mcp({})` shows cached/connected Obscura metadata, but no `obscura-mcp` process is running + +## Key Findings + +- **Generic variant**: the stale-connection root cause affects ALL Pi MCP servers, not just obscura. For non-obscura servers (apple-mail, searxng, ...) or a pure stale-connection repair, use the `pi-mcp-repair` skill (`~pi-mcp-repair skill`) and its `repair-pi-mcp.sh [server]`. This obscura script additionally manages the obscura wrapper + browser processes. +- **Adapter upgrades wipe the patch** in `pi-mcp-adapter/proxy-modes.ts`. Re-run the repair after every Pi / pi-mcp-adapter update (verify: `grep -c "pi-mcp-repair" /proxy-modes.ts` should be 2). +- Pi’s global MCP config is normally `~/.pi/agent/mcp.json`, not `~/.pi/mcp.json`. +- Obscura works reliably when launched through a wrapper that: + - uses absolute paths, + - sets `OBSCURA_PATH`, + - sets `OBSCURA_STEALTH=true`, + - logs startup/CDP errors to `/tmp/obscura-mcp-pi-wrapper.log`. +- Clearing `~/.pi/agent/mcp-cache.json` forces fresh MCP metadata discovery. +- The running Pi session may keep stale MCP client state; after repair, ask the user to run `/reload` if the tool state was already initialized. +- If `mcp({ connect: "obscura" })` only returns cached tools and does not spawn a process, patching `pi-mcp-adapter` to close before explicit reconnect fixes stale-client reuse. + +## One-command Repair + +Run the bundled script: + +```bash +../obscura-mcp-repair/scripts/repair-obscura-mcp.sh +``` + +The script is idempotent. It will: + +1. find `obscura-mcp` and the Obscura browser binary, +2. create/update `~/.local/bin/obscura-mcp-pi-wrapper`, +3. merge the `obscura` server into `~/.pi/agent/mcp.json`, preserving other MCP servers, +4. back up and remove `~/.pi/agent/mcp-cache.json`, +5. preserve existing Obscura CDP browser processes by default, +6. patch `pi-mcp-adapter` so explicit `mcp({ connect: "obscura" })` forces a fresh child process. + +If stale processes must be stopped, run `repair-obscura-mcp.sh --kill-stale` after confirming no healthy Obscura CDP browser should be reused. + +## Post-repair Workflow + +After running the script: + +1. If already inside Pi, tell the user to send: + + ```text + /reload + ``` + +2. Then verify status: + + ```js + mcp({}) + ``` + + Expected before connecting: + + ```text + MCP: 0/1 servers, 0 tools + ○ obscura (not connected) + ``` + +3. Connect: + + ```js + mcp({ connect: "obscura" }) + ``` + +4. Check the wrapper log and process list if needed: + + ```bash + tail -80 /tmp/obscura-mcp-pi-wrapper.log + ps -Ao pid,ppid,etime,command | rg -i 'obscura-mcp|/.local/bin/obscura serve' + lsof -iTCP:9222 -sTCP:LISTEN -n -P + ``` + +5. Test the actual tool path: + + ```js + mcp({ tool: "obscura_browse_page", args: '{"url":"https://example.com","format":"text"}' }) + ``` + + Expected result contains `Example Domain`. + +## Config Shape + +The repaired `~/.pi/agent/mcp.json` should contain an `obscura` server like this: + +```json +{ + "mcpServers": { + "obscura": { + "command": "$HOME/.local/bin/obscura-mcp-pi-wrapper", + "args": ["--transport", "stdio"], + "lifecycle": "lazy", + "idleTimeout": 1, + "env": { + "OBSCURA_PATH": "$HOME/.local/bin/obscura", + "OBSCURA_STEALTH": "true" + } + } + } +} +``` + +Use `$HOME`-relative equivalents on other machines. + +## LinkedIn Finding + +For LinkedIn login flows, `https://www.linkedin.com/login` may render a stripped/challenged shell with no inputs. Use: + +```text +https://www.linkedin.com/uas/login +``` + +Expected selectors: + +- email: `#username` +- password: `#password` +- submit: `button.btn__primary--large` + +Do not ask users to paste passwords unless they explicitly choose that flow. Prefer cookie/session injection for authenticated automation. + +## If Repair Still Fails + +Inspect in this order: + +1. `~/.pi/agent/mcp.json` +2. `/tmp/obscura-mcp-pi-wrapper.log` +3. `ps` for `obscura-mcp` and `obscura serve` +4. `lsof -iTCP:9222 -sTCP:LISTEN -n -P` +5. direct wrapper test using the MCP SDK if available +6. whether the Pi session needs `/reload` + +See `references/findings.md` for the detailed incident notes. diff --git a/skills/obscura-mcp-repair/references/findings.md b/skills/obscura-mcp-repair/references/findings.md new file mode 100644 index 0000000..3472b63 --- /dev/null +++ b/skills/obscura-mcp-repair/references/findings.md @@ -0,0 +1,96 @@ +# Obscura MCP Repair Findings + +## Symptoms Observed + +- `mcp({})` showed Obscura with 4 tools, but tool calls failed with `Failed to call tool: Not connected`. +- `mcp({ connect: "obscura" })` listed tools but did not spawn or keep an `obscura-mcp` process. +- Previous errors included: + - `Execution Error: CDP connection closed` + - `Execution Error: CDP connection is not open` + - `Obscura CDP client is not connected` + - `Cannot read properties of null (reading 'send')` +- Booking.com was blocked by robot/JS checks. Trip.com JSON extraction worked for hotel-price scraping. +- LinkedIn `/login` rendered a shell with zero inputs/buttons/forms; `/uas/login` rendered the classic login form. + +## Confirmed Working Direct MCP Test + +A standalone MCP SDK test using `StdioClientTransport` worked with: + +- command: `$HOME/.local/bin/obscura-mcp-pi-wrapper` (created by repair script) +- args: `--transport stdio` +- env: + - `OBSCURA_PATH=$HOME/.local/bin/obscura` + - `OBSCURA_STEALTH=true` + +It listed: + +- `browse_page` +- `browse_interact` +- `browse_session` +- `browse_scrape` + +And `browse_page` against `https://example.com` returned `Example Domain`. + +## Effective Fix + +1. Create wrapper: `~/.local/bin/obscura-mcp-pi-wrapper` +2. Point Pi MCP config at wrapper: `~/.pi/agent/mcp.json` +3. Clear stale metadata cache: `~/.pi/agent/mcp-cache.json` +4. Reload Pi so the MCP adapter drops stale in-memory state. +5. Patch `pi-mcp-adapter/proxy-modes.ts` so explicit connect closes any existing cached client first. + +## Why the Wrapper Helps + +Pi launches MCP servers under its extension runtime, where PATH/env can differ from an interactive shell. The wrapper makes startup deterministic and writes logs without polluting MCP stdout. + +Important: MCP stdio protocol uses stdout, so wrapper diagnostics must go to stderr or a file only. + +## Wrapper Log + +Default log path: + +```text +/tmp/obscura-mcp-pi-wrapper.log +``` + +Working startup includes: + +```text +Starting Obscura service... +Using Obscura binary: $HOME/.local/bin/obscura +CDP server: ws://127.0.0.1:9222/devtools/browser +Connected to Obscura CDP at ws://127.0.0.1:9222/devtools/browser +Obscura MCP Server running on stdio +``` + +## Good Verification Sequence + +```js +mcp({}) +mcp({ connect: "obscura" }) +mcp({ tool: "obscura_browse_page", args: '{"url":"https://example.com","format":"text"}' }) +``` + +Expected final output contains: + +```text +Example Domain +``` + +## LinkedIn Selectors + +Use: + +```text +https://www.linkedin.com/uas/login +``` + +Expected selectors: + +```text +#username +#password +button.btn__primary--large +``` + +Avoid collecting credentials unless the user explicitly chooses that route. Prefer cookie injection for logged-in flows. diff --git a/skills/obscura-mcp-repair/scripts/repair-obscura-mcp.sh b/skills/obscura-mcp-repair/scripts/repair-obscura-mcp.sh new file mode 100755 index 0000000..b4a0ed6 --- /dev/null +++ b/skills/obscura-mcp-repair/scripts/repair-obscura-mcp.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +set -euo pipefail + +PATCH_ADAPTER=1 +KILL_STALE=0 +DRY_RUN=0 + +usage() { + cat <<'USAGE' +Repair Pi + Obscura MCP integration. + +Usage: + repair-obscura-mcp.sh [--no-patch-adapter] [--kill-stale] [--dry-run] + +What it does: + - creates ~/.local/bin/obscura-mcp-pi-wrapper + - merges an obscura server into ~/.pi/agent/mcp.json + - backs up and removes ~/.pi/agent/mcp-cache.json + - preserves existing Obscura CDP browsers by default + - optionally stops stale obscura-mcp / obscura serve processes with --kill-stale + - optionally patches pi-mcp-adapter so explicit connect forces a fresh spawn + +After running inside an existing Pi session, send /reload. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-patch-adapter) PATCH_ADAPTER=0 ;; + --kill-stale) KILL_STALE=1 ;; + --no-kill) KILL_STALE=0 ;; + --dry-run) DRY_RUN=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown arg: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +home_dir="${HOME:-$(cd ~ && pwd)}" +pi_agent_dir="${PI_CODING_AGENT_DIR:-$home_dir/.pi/agent}" +mcp_config="$pi_agent_dir/mcp.json" +mcp_cache="$pi_agent_dir/mcp-cache.json" +wrapper="$home_dir/.local/bin/obscura-mcp-pi-wrapper" +log_file="/tmp/obscura-mcp-pi-wrapper.log" +obscura_bin="${OBSCURA_PATH:-$home_dir/.local/bin/obscura}" + +say() { printf '[obscura-mcp-repair] %s\n' "$*"; } +run() { + if [[ "$DRY_RUN" == 1 ]]; then + printf '[dry-run] %q ' "$@"; printf '\n' + else + "$@" + fi +} + +resolve_realpath() { + python3 - "$1" <<'PY' +import os, sys +print(os.path.realpath(sys.argv[1])) +PY +} + +find_obscura_mcp() { + if [[ -n "${OBSCURA_MCP_BIN:-}" && -x "${OBSCURA_MCP_BIN}" ]]; then + printf '%s\n' "${OBSCURA_MCP_BIN}" + return 0 + fi + if command -v obscura-mcp >/dev/null 2>&1; then + command -v obscura-mcp + return 0 + fi + local candidate + for candidate in \ + "$home_dir/.nvm/versions/node"/*/bin/obscura-mcp \ + "$home_dir/.brew/bin/obscura-mcp" \ + "/opt/homebrew/bin/obscura-mcp" \ + "/usr/local/bin/obscura-mcp"; do + if [[ -x "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + return 1 +} + +obscura_mcp_bin="$(find_obscura_mcp || true)" +if [[ -z "$obscura_mcp_bin" ]]; then + echo "ERROR: Could not find obscura-mcp. Install it or set OBSCURA_MCP_BIN=/path/to/obscura-mcp" >&2 + exit 1 +fi +obscura_mcp_bin="$(resolve_realpath "$obscura_mcp_bin")" + +if [[ ! -x "$obscura_bin" ]]; then + echo "ERROR: Obscura binary not executable at $obscura_bin. Set OBSCURA_PATH=/path/to/obscura" >&2 + exit 1 +fi + +say "Using obscura-mcp: $obscura_mcp_bin" +say "Using obscura binary: $obscura_bin" +say "Pi MCP config: $mcp_config" + +if [[ "$KILL_STALE" == 1 ]]; then + say "Stopping stale Obscura processes if present" + if [[ "$DRY_RUN" == 0 ]]; then + pkill -f 'obscura-mcp' 2>/dev/null || true + pkill -f "$obscura_bin serve" 2>/dev/null || true + pids="$(lsof -tiTCP:9222 -sTCP:LISTEN 2>/dev/null || true)" + if [[ -n "$pids" ]]; then + while IFS= read -r pid; do + [[ -z "$pid" ]] && continue + cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)" + if [[ "$cmd" == *obscura* ]]; then + kill "$pid" 2>/dev/null || true + fi + done <<< "$pids" + fi + else + say "Would pkill obscura-mcp and stale obscura serve processes" + fi +fi + +say "Writing wrapper: $wrapper" +if [[ "$DRY_RUN" == 0 ]]; then + mkdir -p "$(dirname "$wrapper")" + cat > "$wrapper" <> "\$LOG" 2>&1 + +export OBSCURA_PATH="\${OBSCURA_PATH:-$obscura_bin}" +export OBSCURA_STEALTH="\${OBSCURA_STEALTH:-true}" +export PATH="$(dirname "$obscura_mcp_bin"):$home_dir/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:\${PATH:-}" + +exec "$obscura_mcp_bin" "\$@" 2>> "\$LOG" +EOF + chmod +x "$wrapper" +fi + +say "Merging Obscura server into MCP config" +if [[ "$DRY_RUN" == 0 ]]; then + mkdir -p "$pi_agent_dir" + ts="$(date +%Y%m%d%H%M%S)" + if [[ -f "$mcp_config" ]]; then + cp "$mcp_config" "$mcp_config.bak.$ts" + say "Backed up config to $mcp_config.bak.$ts" + fi + python3 - "$mcp_config" "$wrapper" "$obscura_bin" <<'PY' +import json, os, sys +path, wrapper, obscura = sys.argv[1:4] +raw = {} +if os.path.exists(path): + try: + with open(path, 'r', encoding='utf-8') as f: + raw = json.load(f) + except Exception as e: + backup = path + '.invalid' + os.replace(path, backup) + print(f'Invalid JSON moved to {backup}', file=sys.stderr) + raw = {} +if not isinstance(raw, dict): + raw = {} +servers = raw.get('mcpServers') or raw.get('mcp-servers') or {} +if not isinstance(servers, dict): + servers = {} +servers['obscura'] = { + 'command': wrapper, + 'args': ['--transport', 'stdio'], + 'lifecycle': 'lazy', + 'idleTimeout': 1, + 'env': { + 'OBSCURA_PATH': obscura, + 'OBSCURA_STEALTH': 'true', + }, +} +raw.pop('mcp-servers', None) +raw['mcpServers'] = servers +tmp = path + f'.{os.getpid()}.tmp' +with open(tmp, 'w', encoding='utf-8') as f: + json.dump(raw, f, indent=2) + f.write('\n') +os.replace(tmp, path) +PY +fi + +if [[ -f "$mcp_cache" ]]; then + say "Backing up and removing stale MCP metadata cache" + if [[ "$DRY_RUN" == 0 ]]; then + ts="$(date +%Y%m%d%H%M%S)" + cp "$mcp_cache" "$mcp_cache.bak.$ts" + rm -f "$mcp_cache" + say "Backed up cache to $mcp_cache.bak.$ts" + fi +fi + +find_adapter_root() { + local bin real root candidate + if command -v pi-mcp-adapter >/dev/null 2>&1; then + bin="$(command -v pi-mcp-adapter)" + real="$(resolve_realpath "$bin")" + root="$(dirname "$real")" + if [[ -f "$root/proxy-modes.ts" ]]; then + printf '%s\n' "$root" + return 0 + fi + fi + if command -v npm >/dev/null 2>&1; then + root="$(npm root -g 2>/dev/null || true)/pi-mcp-adapter" + if [[ -f "$root/proxy-modes.ts" ]]; then + printf '%s\n' "$root" + return 0 + fi + fi + for candidate in "$home_dir/.nvm/versions/node"/*/lib/node_modules/pi-mcp-adapter; do + if [[ -f "$candidate/proxy-modes.ts" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + return 1 +} + +if [[ "$PATCH_ADAPTER" == 1 ]]; then + adapter_root="$(find_adapter_root || true)" + if [[ -n "$adapter_root" ]]; then + proxy_file="$adapter_root/proxy-modes.ts" + say "Patching adapter stale-client reconnect behavior: $proxy_file" + if [[ "$DRY_RUN" == 0 ]]; then + ts="$(date +%Y%m%d%H%M%S)" + cp "$proxy_file" "$proxy_file.bak.$ts" + python3 - "$proxy_file" <<'PY' +from pathlib import Path +import sys +path = Path(sys.argv[1]) +text = path.read_text() +changed = False + +needle = ''' if (state.ui) { + state.ui.setStatus("mcp", `MCP: connecting to ${serverName}...`); + } + let connection = await state.manager.connect(serverName, definition);''' +replacement = ''' if (state.ui) { + state.ui.setStatus("mcp", `MCP: connecting to ${serverName}...`); + } + // Treat explicit connect requests as reconnects. A child MCP process can exit + // while the SDK Client object remains cached as "connected", causing later + // calls to fail with "Not connected". Closing first forces a fresh spawn. + await state.manager.close(serverName); + let connection = await state.manager.connect(serverName, definition);''' +if 'Treat explicit connect requests as reconnects' not in text: + if needle in text: + text = text.replace(needle, replacement, 1) + changed = True + else: + print('WARN: explicit-connect patch anchor not found', file=sys.stderr) + +needle2 = ''' } catch (error) { + const message = error instanceof Error ? error.message : String(error); + uiSession?.sendToolCancelled(message); + + let errorWithSchema = `Failed to call tool: ${message}`;''' +replacement2 = ''' } catch (error) { + const message = error instanceof Error ? error.message : String(error); + uiSession?.sendToolCancelled(message); + + if (/not connected|connection closed|connection is not open/i.test(message)) { + await state.manager.close(serverName).catch(() => {}); + } + + let errorWithSchema = `Failed to call tool: ${message}`;''' +if 'connection is not open/i.test(message)' not in text: + if needle2 in text: + text = text.replace(needle2, replacement2, 1) + changed = True + else: + print('WARN: call-failure cleanup patch anchor not found', file=sys.stderr) + +if changed: + path.write_text(text) + print('patched') +else: + print('already patched or no changes') +PY + fi + else + say "pi-mcp-adapter package not found; skipping adapter patch" + fi +fi + +say "Repair complete." +say "Next inside Pi: send /reload, then run mcp({ connect: \"obscura\" }) and test browse_page." +say "Wrapper log: $log_file" diff --git a/skills/pdf/SKILL.md b/skills/pdf/SKILL.md new file mode 100644 index 0000000..1d7af2f --- /dev/null +++ b/skills/pdf/SKILL.md @@ -0,0 +1,42 @@ +--- +name: pdf +description: Create, inspect, validate, merge, split, or extract text from PDF files. Use whenever PDF is the primary input or deliverable, including reports, forms, manuscripts, page operations, and searchable-text checks. +allowed-tools: read bash write edit +--- + +# PDF documents + +Use this Pi-native skill for PDF work. It uses open Python libraries declared directly in its script. + +## Tool + +```bash +uv run scripts/pdf_tool.py create INPUT.md OUTPUT.pdf --title "Title" --author "Author" +uv run scripts/pdf_tool.py inspect FILE.pdf +uv run scripts/pdf_tool.py validate FILE.pdf +uv run scripts/pdf_tool.py merge OUTPUT.pdf INPUT1.pdf INPUT2.pdf +uv run scripts/pdf_tool.py split INPUT.pdf OUTPUT_DIRECTORY +``` + +For quick independent extraction checks, this machine also provides: + +```bash +pdftotext FILE.pdf - +pdfinfo FILE.pdf +``` + +## Workflow + +1. Inspect the PDF before editing or transforming it. +2. For new PDFs, retain the editable Markdown source. +3. Perform page operations into a new output unless overwrite was requested. +4. Validate page count and extractable text after generation. +5. When dealing with scans, determine whether OCR is needed; do not call an image-only PDF searchable. +6. Report output path, pages, bytes, SHA-256, and whether text extraction succeeds. + +## Quality requirements + +- Structural validation does not prove visual layout; render or open the result when visual fidelity matters. +- Preserve metadata and page order for merges. +- Never silently remove encryption, signatures, annotations, or form fields. +- Use `create-book` for multi-format book publication; this skill owns the PDF-specific operation. diff --git a/skills/pdf/scripts/pdf_tool.py b/skills/pdf/scripts/pdf_tool.py new file mode 100755 index 0000000..80c6e41 --- /dev/null +++ b/skills/pdf/scripts/pdf_tool.py @@ -0,0 +1,135 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = ["pypdf>=5", "reportlab>=4"] +# /// +"""Create, inspect, validate, merge, and split PDF files.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from pypdf import PdfReader, PdfWriter +from reportlab.lib.enums import TA_CENTER +from reportlab.lib.pagesizes import LETTER +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import inch +from reportlab.platypus import ListFlowable, ListItem, PageBreak, Paragraph, SimpleDocTemplate, Spacer + +SKILLS = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(SKILLS / "_document_common")) +from document_common import markdown_blocks, parse_frontmatter, sha256_file, strip_inline_markdown # noqa: E402 + + +def create(source: Path, output: Path, title: str | None, author: str | None) -> dict: + text = source.read_text(encoding="utf-8") + metadata, _ = parse_frontmatter(text) + resolved_title = title or metadata.get("title") + resolved_author = author or metadata.get("author") + output.parent.mkdir(parents=True, exist_ok=True) + styles = getSampleStyleSheet() + styles.add(ParagraphStyle(name="BookTitle", parent=styles["Title"], alignment=TA_CENTER, fontSize=26, leading=32, spaceAfter=18)) + styles.add(ParagraphStyle(name="Byline", parent=styles["Normal"], alignment=TA_CENTER, fontSize=12, spaceAfter=18)) + story = [] + if resolved_title: + story.extend([Paragraph(resolved_title, styles["BookTitle"])]) + if resolved_author: + story.append(Paragraph(resolved_author, styles["Byline"])) + story.append(PageBreak()) + for block in markdown_blocks(text): + value = strip_inline_markdown(block["text"]) + if block["type"] == "heading": + style = styles["Heading1"] if block["level"] == 1 else styles["Heading2"] + story.extend([Spacer(1, 8), Paragraph(value, style)]) + elif block["type"] in {"bullet", "number"}: + story.append(ListFlowable([ListItem(Paragraph(value, styles["BodyText"]))], bulletType="bullet" if block["type"] == "bullet" else "1")) + elif block["type"] == "quote": + story.append(Paragraph(f"{value}", styles["Italic"])) + else: + story.extend([Paragraph(value, styles["BodyText"]), Spacer(1, 6)]) + document = SimpleDocTemplate( + str(output), pagesize=LETTER, rightMargin=0.8 * inch, leftMargin=0.8 * inch, + topMargin=0.75 * inch, bottomMargin=0.75 * inch, + title=resolved_title or source.stem, author=resolved_author or "", + ) + document.build(story) + return inspect(output) + + +def inspect(path: Path) -> dict: + reader = PdfReader(path) + text = "\n".join((page.extract_text() or "") for page in reader.pages) + return { + "path": str(path.resolve()), + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + "pages": len(reader.pages), + "encrypted": reader.is_encrypted, + "metadata": {str(k): str(v) for k, v in (reader.metadata or {}).items()}, + "text_chars": len(text), + "text_preview": text[:500], + } + + +def merge(inputs: list[Path], output: Path) -> dict: + writer = PdfWriter() + for source in inputs: + for page in PdfReader(source).pages: + writer.add_page(page) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as stream: + writer.write(stream) + return inspect(output) + + +def split(source: Path, output_dir: Path) -> list[dict]: + output_dir.mkdir(parents=True, exist_ok=True) + results = [] + for index, page in enumerate(PdfReader(source).pages, 1): + target = output_dir / f"page-{index:04d}.pdf" + writer = PdfWriter() + writer.add_page(page) + with target.open("wb") as stream: + writer.write(stream) + results.append(inspect(target)) + return results + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + make = sub.add_parser("create") + make.add_argument("source", type=Path) + make.add_argument("output", type=Path) + make.add_argument("--title") + make.add_argument("--author") + check = sub.add_parser("inspect") + check.add_argument("path", type=Path) + validate = sub.add_parser("validate") + validate.add_argument("path", type=Path) + join = sub.add_parser("merge") + join.add_argument("output", type=Path) + join.add_argument("inputs", type=Path, nargs="+") + divide = sub.add_parser("split") + divide.add_argument("source", type=Path) + divide.add_argument("output_dir", type=Path) + args = parser.parse_args() + + if args.command == "create": + result = create(args.source, args.output, args.title, args.author) + elif args.command in {"inspect", "validate"}: + result = inspect(args.path) + if args.command == "validate" and (result["pages"] == 0 or result["text_chars"] == 0): + raise SystemExit("invalid PDF: no pages or extractable text") + elif args.command == "merge": + result = merge(args.inputs, args.output) + else: + result = split(args.source, args.output_dir) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/skills/pi-mcp-repair/SKILL.md b/skills/pi-mcp-repair/SKILL.md new file mode 100644 index 0000000..b19d4c9 --- /dev/null +++ b/skills/pi-mcp-repair/SKILL.md @@ -0,0 +1,81 @@ +--- +name: pi-mcp-repair +description: Repairs stale Pi MCP gateway connections for ANY MCP server (apple-mail, obscura, searxng, context-mode, ...) when mcp({}) or metadata shows tools/connected but tool calls fail with "Not connected", "connection closed", "Request timed out" followed by "Connection closed", or mcp({connect}) lists tools without spawning a server process. Also the canonical place to re-apply the pi-mcp-adapter stale-client patch after any pi-mcp-adapter upgrade. Use BEFORE declaring an MCP server unusable or falling back to workarounds. +allowed-tools: read bash edit write mcp +--- + +# Pi MCP Repair (generic, any server) + +Generic repair for the Pi MCP gateway's stale-connection failure mode. The +obscura-specific wrapper/browser repair stays in `obscura-mcp-repair`; this +skill covers the shared root cause and works for every server in +`~/.pi/agent/mcp.json`. + +## Failure signature (must match before repairing) + +- `mcp({})` or a `connect` call shows the server as connected with N tools, + but tool calls fail with `Failed to call tool: Not connected`, + `connection closed`, or `connection is not open`. +- A `Request timed out` on a slow tool call, after which every further call + fails and `connect` stops working. +- `mcp({ connect: "" })` returns the tool list but `ps` shows no + child process for that server's command. + +Root cause: the MCP child process exits (timeout, crash, or killed) but the +adapter's cached client still reads `status === "connected"`. +`ServerManager.connect()` reuses a connection whose status is `"connected"` +without checking whether the child is alive, so it never respawns. Details +and incident history: `references/findings.md`. + +## One-command repair + +```bash +../pi-mcp-repair/scripts/repair-pi-mcp.sh [server-name] [--kill-stale] [--dry-run] [--no-patch-adapter] [--no-smoke-test] +``` + +Resolve `../pi-mcp-repair/scripts/repair-pi-mcp.sh` against this skill's +directory. The script is idempotent. With a `server-name` (key from +`~/.pi/agent/mcp.json`, e.g. `apple-mail`) it additionally: + +1. resolves and verifies the configured command is executable, +2. with `--kill-stale`, pkills processes matching that command path, +3. runs a 3-second launch smoke test (process must survive with stdin held + open) and reports its log. + +Always, it: + +4. backs up and removes `~/.pi/agent/mcp-cache.json` (forces fresh metadata + discovery), +5. patches `pi-mcp-adapter/proxy-modes.ts` (idempotent, version-tolerant + regex anchors) so explicit connects close any stale cached client first, + and call failures matching connection-dead errors reset the stale state. + +## Post-repair workflow + +1. The patch only takes effect after the extension reloads. Tell the user to + send `/reload` (or restart Pi). +2. Verify: `mcp({})` should show the server not connected, then + `mcp({ connect: "" })` must spawn the process — confirm with + `ps aux | grep `. +3. Test one real tool call against the server. + +## UPGRADE WIPE — re-run after every pi-mcp-adapter update + +The patch lives inside the `pi-mcp-adapter` package, so ANY update/reinstall +of that package removes it. This is why the failure recurs across servers. +Re-run `scripts/repair-pi-mcp.sh` after upgrading Pi or pi-mcp-adapter +(verify with `grep -c "pi-mcp-repair" /proxy-modes.ts` — expect 2). + +## While MCP is broken: use the direct backend + +MCP servers are thin wrappers. If the user needs results immediately, drive +the underlying system directly instead of waiting for the repair: +apple-mail → Mail.app via `osascript` (accounts by Mail.app display name, +mailboxes often differ from IMAP names, e.g. Hotmail uses `Inbox` not +`INBOX`, Gmail has `Sent Mail`); obscura → the `obscura` binary; etc. + +## If repair still fails + +Inspect in order: `~/.pi/agent/mcp.json` command path → smoke-test log in +`/tmp/pi-mcp-repair-smoke.*.log` → adapter root resolution (script prints +it) → whether `/reload` happened after patching → `references/findings.md`. diff --git a/skills/pi-mcp-repair/references/findings.md b/skills/pi-mcp-repair/references/findings.md new file mode 100644 index 0000000..517584f --- /dev/null +++ b/skills/pi-mcp-repair/references/findings.md @@ -0,0 +1,96 @@ +# Pi MCP Repair — Findings + +## Root cause (confirmed twice: obscura May 2026, apple-mail 2026-08-17) + +Pi's MCP adapter (`pi-mcp-adapter` package) keeps one `ServerConnection` +object per server in an in-memory map. When the stdio child process dies — +request timeout, crash, or kill — the cached object's `status` often stays +`"connected"` because nothing watches the transport EOF. + +Two consequences in `server-manager.ts` / `proxy-modes.ts`: + +1. `ServerManager.connect()` (v2.11.0 lines ~105-110) reuses any connection + whose status is `"connected"` without verifying the child is alive: + "Reuse existing connection if healthy" — the health check is status-only. + So `mcp({ connect: ... })` and `/mcp reconnect` return the stale + connection's cached tool list and never spawn a new process. +2. The call path's auto-reconnect only fires when + `!connection || connection.status !== "connected"`. With the stale status + it proceeds straight to `client.callTool(...)`, which throws the SDK's + transport error, surfaced as `Failed to call tool: Not connected`. + +`ServerManager.close(name)` (v2.11.0 line ~417) is the safe reset: no-op when +absent, deletes from the map before async cleanup (race-safe by design). + +## The fix: two patches in proxy-modes.ts + +Both are applied by `scripts/repair-pi-mcp.sh` with version-tolerant regex +anchors and idempotency markers (`pi-mcp-repair: ...`). + +- **Patch A (connect path)**: in `executeConnect`, insert + `await state.manager.close(serverName).catch(() => {});` immediately before + `let connection = await state.manager.connect(serverName, definition...)`. + Explicit connects now always force a fresh spawn. Anchor: the + `state.ui.setStatus("mcp", \`MCP: connecting to ...\`)` line precedes it. + Note v2.11.0 passes a `signal` arg; the older patched version did not — + the regex accepts both. +- **Patch B (call path)**: in the generic tool-call catch block (identified + by `uiSession?.sendToolCancelled(message);` followed by + `const schemaText =`), close the connection when the error matches + `/not connected|connection closed|connection is not open|transport closed|broken pipe/i`. + The next call then hits the auto-reconnect branch and succeeds. Deliberately + NOT matching "timed out": a slow-but-healthy server answering a long request + must not have its connection torn down. + +Verify patch presence: +`grep -c "pi-mcp-repair" ~/.pi/agent/npm/node_modules/pi-mcp-adapter/proxy-modes.ts` +expect `2`. + +## Why it recurs: adapter upgrades wipe the patch + +The patch edits files inside the `pi-mcp-adapter` package. Any upgrade or +reinstall of the package replaces those files and silently removes the patch. +Evidence on 2026-08-17: installed v2.11.0 had zero `pi-mcp-repair`/ +`Treat explicit connect requests as reconnects` markers and no +`proxy-modes.ts.bak.*` files, although the obscura incident had been patched +previously; registry latest was 2.26.0. Therefore: **re-run +`repair-pi-mcp.sh` after every Pi / pi-mcp-adapter update** (same posture as +`pi-anthropic-toolid-patch` after `brew upgrade pi-coding-agent`). + +Anchor text changes between adapter versions (v2.11.0 added `, signal`, +`UrlElicitationRequiredError` handling, and `guardMcpOutput` in the catch). +If both WARN anchors appear, read the current `proxy-modes.ts` +(`executeConnect` + the generic call catch) and update the regexes here. + +## Incident log + +### obscura (2026-05, original incident) +Symptoms: `Not connected`, `CDP connection closed`, connect listed tools but +no process. Fixed via wrapper + cache clear + adapter patch; documented in +`obscura-mcp-repair`. + +### apple-mail (2026-08-17) +- `apple_mail_search_messages` on the Google account timed out + (`MCP error -32001: Request timed out`), after which the connection + closed and every call failed with `Not connected`. +- `mcp({})` showed `✓ apple-mail (24 tools)`; `ps` had no + `apple-mail-fast-mcp` process; `mcp({ connect })` returned the cached tool + list three times without spawning. +- Server binary itself was healthy: launched standalone it stays alive with + stdin open (server binary at configured `command` path). +- Workaround that unblocked the user immediately: drive Mail.app directly + with `osascript` (the MCP server is a thin wrapper over AppleScript/IMAP). + Gotchas: Mail.app account names differ from the MCP `name` field + (`candidate@example.invalid`, `candidate@example.invalid`); mailbox names are + Mail.app's (`Inbox`, `Sent Mail`, `Sent Items`), not IMAP's (`INBOX`); + AppleScript handlers that read Mail properties must run inside a + `tell application "Mail"` block or `date received` fails to resolve. +- AppleScript `whose sender contains` on inbox-sized mailboxes was fast + (seconds); avoid `content contains` sweeps on large mailboxes. + +## Known non-fixes + +- Repeated `mcp({ connect })` calls: returns stale cached tools, never spawns. +- `/mcp reconnect `: same code path (`lazyConnect`), same result. +- Waiting for backoff to expire: backoff only applies after a *failed + connect*; the stale path never fails the connect. diff --git a/skills/pi-mcp-repair/scripts/repair-pi-mcp.sh b/skills/pi-mcp-repair/scripts/repair-pi-mcp.sh new file mode 100755 index 0000000..d09d3d3 --- /dev/null +++ b/skills/pi-mcp-repair/scripts/repair-pi-mcp.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generic Pi MCP stale-connection repair. Works for any server in mcp.json. +# See the sibling SKILL.md for when/how to use this. + +PATCH_ADAPTER=1 +KILL_STALE=0 +DRY_RUN=0 +SMOKE_TEST=1 +SERVER="" + +usage() { + cat <<'USAGE' +Repair stale Pi MCP gateway connections (any server). + +Usage: + repair-pi-mcp.sh [server-name] [--kill-stale] [--no-patch-adapter] \ + [--no-smoke-test] [--dry-run] + + server-name optional key from ~/.pi/agent/mcp.json (e.g. apple-mail). + Enables command verification, stale-process kill, and a + launch smoke test for that server. + --kill-stale pkill processes matching the server's configured command. + --no-patch-adapter skip the pi-mcp-adapter patch (cache clear only). + --no-smoke-test skip the 3s launch smoke test. + --dry-run print actions without changing anything. + +Always: backs up + removes ~/.pi/agent/mcp-cache.json and patches +pi-mcp-adapter/proxy-modes.ts so explicit connects drop stale cached clients +and connection-dead call failures reset state. Re-run after any +pi-mcp-adapter upgrade (the patch lives inside the package). + +After running inside a live Pi session, send /reload. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --kill-stale) KILL_STALE=1 ;; + --no-kill) KILL_STALE=0 ;; + --no-patch-adapter) PATCH_ADAPTER=0 ;; + --no-smoke-test) SMOKE_TEST=0 ;; + --dry-run) DRY_RUN=1 ;; + -h|--help) usage; exit 0 ;; + -*) echo "Unknown flag: $1" >&2; usage >&2; exit 2 ;; + *) SERVER="$1" ;; + esac + shift +done + +home_dir="${HOME:-$(cd ~ && pwd)}" +pi_agent_dir="${PI_CODING_AGENT_DIR:-$home_dir/.pi/agent}" +mcp_config="$pi_agent_dir/mcp.json" +mcp_cache="$pi_agent_dir/mcp-cache.json" + +say() { printf '[pi-mcp-repair] %s\n' "$*"; } +run() { + if [[ "$DRY_RUN" == 1 ]]; then + printf '[dry-run]'; printf ' %q' "$@"; printf '\n' + else + "$@" + fi +} + +if [[ ! -f "$mcp_config" ]]; then + echo "ERROR: MCP config not found at $mcp_config" >&2 + exit 1 +fi +say "Pi MCP config: $mcp_config" + +# --------------------------------------------------------------------------- +# Optional: resolve, verify, kill-stale, and smoke-test one named server +# --------------------------------------------------------------------------- +server_command="" +if [[ -n "$SERVER" ]]; then + say "Resolving server: $SERVER" + server_command="$(python3 - "$mcp_config" "$SERVER" <<'PY' +import json, sys +path, name = sys.argv[1], sys.argv[2] +with open(path, encoding="utf-8") as f: + cfg = json.load(f) +servers = cfg.get("mcpServers") or cfg.get("mcp-servers") or {} +srv = servers.get(name) +if not srv: + sys.exit(f"server '{name}' not in {path}; keys: {sorted(servers)}") +cmd = srv.get("command") +if not cmd: + sys.exit(f"server '{name}' has no command") +print(cmd) +PY + )" || { echo "ERROR: $server_command" >&2; exit 1; } + + if [[ ! -x "$server_command" && ! -f "$server_command" ]]; then + say "WARNING: command not executable at $server_command (may resolve via PATH)" + else + say "Command: $server_command" + fi + + if [[ "$KILL_STALE" == 1 ]]; then + say "Killing stale processes matching: $server_command" + run pkill -f "$server_command" || true + fi + + if [[ "$SMOKE_TEST" == 1 ]]; then + server_args="$(python3 - "$mcp_config" "$SERVER" <<'PY' +import json, sys +path, name = sys.argv[1], sys.argv[2] +with open(path, encoding="utf-8") as f: + cfg = json.load(f) +servers = cfg.get("mcpServers") or cfg.get("mcp-servers") or {} +print(" ".join(servers.get(name, {}).get("args") or [])) +PY + )" + smoke_log="/tmp/pi-mcp-repair-smoke.$$.log" + say "Smoke-testing launch (3s, log: $smoke_log)" + if [[ "$DRY_RUN" == 1 ]]; then + say "[dry-run] would launch: $server_command $server_args" + else + # Hold stdin open so a stdio server stays alive; check it survives 3s. + # shellcheck disable=SC2086 + ( sleep 6 | "$server_command" $server_args > "$smoke_log" 2>&1 ) & + launcher_pid=$! + sleep 3 + if kill -0 "$launcher_pid" 2>/dev/null && pgrep -P "$launcher_pid" >/dev/null 2>&1; then + alive_pid="$(pgrep -P "$launcher_pid" | while read -r p; do + [[ "$(ps -p "$p" -o comm= 2>/dev/null)" != *sleep* ]] && echo "$p" + done | head -1)" + if [[ -n "$alive_pid" ]]; then + say "OK: server process alive after 3s (pid $alive_pid)" + else + say "WARN: launcher alive but no child process yet" + fi + else + say "FAIL: server process exited within 3s — see $smoke_log" + tail -20 "$smoke_log" >&2 || true + fi + # Clean up the smoke-test process tree. + pkill -P "$launcher_pid" 2>/dev/null || true + kill "$launcher_pid" 2>/dev/null || true + fi + fi +fi + +# --------------------------------------------------------------------------- +# Always: clear stale metadata cache +# --------------------------------------------------------------------------- +if [[ -f "$mcp_cache" ]]; then + ts="$(date +%Y%m%d%H%M%S)" + say "Backing up + removing stale MCP metadata cache" + if [[ "$DRY_RUN" == 0 ]]; then + cp "$mcp_cache" "$mcp_cache.bak.$ts" + rm -f "$mcp_cache" + say "Backed up cache to $mcp_cache.bak.$ts" + fi +else + say "No MCP cache to clear ($mcp_cache absent)" +fi + +# --------------------------------------------------------------------------- +# Locate pi-mcp-adapter package root +# --------------------------------------------------------------------------- +resolve_realpath() { python3 -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$1"; } + +find_adapter_root() { + local bin real root candidate + # Pi's bundled install location first. + candidate="$pi_agent_dir/npm/node_modules/pi-mcp-adapter" + if [[ -f "$candidate/proxy-modes.ts" ]]; then printf '%s\n' "$candidate"; return 0; fi + if command -v pi-mcp-adapter >/dev/null 2>&1; then + bin="$(command -v pi-mcp-adapter)" + real="$(resolve_realpath "$bin")" + root="$(dirname "$real")" + if [[ -f "$root/proxy-modes.ts" ]]; then printf '%s\n' "$root"; return 0; fi + fi + if command -v npm >/dev/null 2>&1; then + root="$(npm root -g 2>/dev/null || true)/pi-mcp-adapter" + if [[ -f "$root/proxy-modes.ts" ]]; then printf '%s\n' "$root"; return 0; fi + fi + for candidate in "$home_dir/.nvm/versions/node"/*/lib/node_modules/pi-mcp-adapter; do + if [[ -f "$candidate/proxy-modes.ts" ]]; then printf '%s\n' "$candidate"; return 0; fi + done + return 1 +} + +# --------------------------------------------------------------------------- +# Patch the adapter (idempotent, version-tolerant regex anchors) +# --------------------------------------------------------------------------- +if [[ "$PATCH_ADAPTER" == 1 ]]; then + adapter_root="$(find_adapter_root || true)" + if [[ -z "$adapter_root" ]]; then + say "WARN: pi-mcp-adapter not found; skipping patch" + else + proxy_file="$adapter_root/proxy-modes.ts" + say "Adapter root: $adapter_root" + say "Patching stale-client behavior: $proxy_file" + if [[ "$DRY_RUN" == 0 ]]; then + ts="$(date +%Y%m%d%H%M%S)" + cp "$proxy_file" "$proxy_file.bak.$ts" + say "Backed up proxy-modes.ts to $proxy_file.bak.$ts" + python3 - "$proxy_file" <<'PY' +from pathlib import Path +import re, sys + +path = Path(sys.argv[1]) +text = path.read_text() +changed = False + +# Patch A: explicit connect must drop any stale cached client first. +# Anchor: the `let connection = await state.manager.connect(...)` inside +# executeConnect, which is preceded by the "MCP: connecting to" setStatus. +MARK_A = "pi-mcp-repair: explicit connect forces a fresh spawn" +if MARK_A not in text: + pat_a = re.compile( + r"([ \t]*)let connection = await state\.manager\.connect\(" + r"serverName, definition(?:, signal)?\);" + ) + def repl_a(m): + indent = m.group(1) + return ( + f"{indent}// {MARK_A}\n" + f"{indent}// A child MCP process can exit while the cached client still\n" + f"{indent}// reads \"connected\"; close first so connect() respawns it.\n" + f"{indent}await state.manager.close(serverName).catch(() => {{}});\n" + f"{m.group(0)}" + ) + # Only patch the occurrence that follows the executeConnect setStatus. + anchor = 'state.ui.setStatus("mcp", `MCP: connecting to ${serverName}...`);' + anchor_pos = text.find(anchor) + if anchor_pos >= 0: + region_start = anchor_pos + # Find the first connect() call after the anchor. + match = None + for match in pat_a.finditer(text, region_start): + break + if match: + text = text[:match.start()] + repl_a(match) + text[match.end():] + changed = True + else: + print("WARN: Patch A anchor (connect after setStatus) not found", file=sys.stderr) + else: + print("WARN: Patch A setStatus anchor not found", file=sys.stderr) +else: + print("Patch A already applied") + +# Patch B: on a connection-dead call failure, reset the stale connection so +# the next call auto-reconnects. Anchor: the generic catch that sets +# `const message = ...` then `uiSession?.sendToolCancelled(message);` +# immediately followed by `const schemaText =`. +MARK_B = "pi-mcp-repair: reset stale connection on call failure" +if MARK_B not in text: + pat_b = re.compile( + r"(const message = error instanceof Error \? error\.message : String\(error\);\n" + r"[ \t]*uiSession\?\.sendToolCancelled\(message\);\n)" + r"(\n[ \t]*const schemaText =)" + ) + def repl_b(m): + return ( + f"{m.group(1)}" + " if (/not connected|connection closed|connection is not open|" + "transport closed|broken pipe/i.test(message)) {\n" + f" // {MARK_B}\n" + " await state.manager.close(serverName).catch(() => {});\n" + " }\n" + f"{m.group(2)}" + ) + new_text, n = pat_b.subn(repl_b, text, count=1) + if n: + text = new_text + changed = True + else: + print("WARN: Patch B anchor (catch + schemaText) not found", file=sys.stderr) +else: + print("Patch B already applied") + +if changed: + path.write_text(text) + print("patched") +else: + print("already patched or no changes") +PY + fi + fi +fi + +say "Repair complete." +if [[ -n "$SERVER" ]]; then + say "Next: send /reload, then mcp({ connect: \"$SERVER\" }) and test one real tool call." +else + say "Next: send /reload, then reconnect the affected server and test one real tool call." +fi diff --git a/skills/qwen-screenshot-debug/SKILL.md b/skills/qwen-screenshot-debug/SKILL.md new file mode 100644 index 0000000..97e68f1 --- /dev/null +++ b/skills/qwen-screenshot-debug/SKILL.md @@ -0,0 +1,133 @@ +--- +name: qwen-screenshot-debug +description: Browser automation visual debugging with screenshot + local Qwen VLM. Use when a page state is uncertain, an expected upload/submit/navigation does not appear, CDP/DOM checks time out, automation seems stuck waiting for a browser change, or a local screenshot-helper approval/denial needs clarification. See references/local-qwen-helper-approval.md. +allowed-tools: read bash write +--- +# Qwen Screenshot Debug + +Use this skill whenever browser automation is blocked by uncertainty about what the user-visible page actually shows. + +## Trigger rule + +Before declaring an application step failed, retrying blindly, or waiting again, **capture a screenshot and ask the local Qwen VLM to inspect it** when any of these happen: + +- an upload, submit, save, continue, or navigation was expected but DOM/CDP checks do not confirm it +- `Runtime.evaluate`, accessibility snapshots, or DOM polling time out after an action +- a spinner/progress state appears to hang +- a hidden input/file upload was manipulated and the visible UI state is unclear +- a page is technically reachable but automation cannot identify the next visible control +- the user says the visible browser shows success but automation did not detect it + +The screenshot is the source of truth for visible UI state. Do not infer failure from missing DOM text alone. + +**This is the DEFAULT stuck-flow protocol**, not a last resort: any time a click, form fill, or submit cannot progress — in LinkedIn Easy Apply, any ATS, or any browser flow — run visual recovery BEFORE retrying the same DOM action, before switching strategies blindly, and before marking anything failed/blocked. + +## One-command entry point: `scripts/visual-recover.mjs` + +Prefer this over hand-rolling the capture→ask→click→verify steps. It pairs the capture method with the correct coordinate system automatically and verifies after clicking. + +```bash +# Inspect what the stuck page actually shows (CDP viewport, first page tab) +node ../qwen-screenshot-debug/scripts/visual-recover.mjs + +# Pick the tab and ask a specific question +node ../qwen-screenshot-debug/scripts/visual-recover.mjs \ + --url-substr linkedin.com/jobs --prompt "Did the Easy Apply modal advance past the resume step? Any validation errors?" + +# Inspect + click the next control Qwen identifies + verify the state changed (exit 3 if unchanged) +node ../qwen-screenshot-debug/scripts/visual-recover.mjs --url-substr linkedin.com --click + +# Container display capture + xdotool click — for JS alerts/native modals CDP screenshots can't see +node ../qwen-screenshot-debug/scripts/visual-recover.mjs --capture ffmpeg --click +``` + +Coordinate-system contract (the historical #1 failure when mixed): `--capture cdp` → viewport px → CDP `Input.dispatchMouseEvent`; `--capture ffmpeg` → X-display px → `xdotool` in the container. Never feed ffmpeg coordinates to CDP or vice versa. + +Escalation ladder when stuck: +1. `visual-recover.mjs` (inspect) — what does the page actually show? +2. If Qwen sees a clickable control but DOM selectors can't reach it → `--click` (CDP first). +3. If CDP screenshots look fine but the click has no effect, or a native alert/modal is suspected → `--capture ffmpeg --click` (full `selenium-container-visual-click-recovery` skill for pitfalls). +4. CAPTCHA visible → `captcha-resolution` skill. +5. Only after these: pause and escalate to the user with the screenshot path and Qwen's description. + +Env overrides: `QWEN_VLM_ENDPOINT`, `QWEN_VLM_MODEL`, `BROWSER_CDP_PORT`, `SELENIUM_CONTAINER`. + +## Local Qwen VLM endpoint + +Use the existing local LM Studio vision endpoint documented by `captcha-resolution`: + +- Endpoint: `http://localhost:1234/v1/chat/completions` +- Model: `qwen3.6-35b-a3b-holo3-qwopus-instruct-qx64-hi-mlx` +- Do **not** use DeepSeek on port `8002` for images; it is text-only and silently ignores screenshots. + +## Workflow + +1. Capture a screenshot of the relevant browser viewport or element. +2. Save it under `/tmp`, e.g. `/tmp/qwen-debug-.png`. +3. Send the screenshot to Qwen with a concrete visual-inspection prompt. +4. Prefer the bundled helper `scripts/qwen-vlm-inspect.mjs` instead of shell pipelines such as `curl ... | python3 -c ...`; Pi Agent/Tirith can flag network-output-to-interpreter pipelines as high-risk even when the endpoint is local. +5. Use Qwen's answer to decide the next action. + - For coordinate requests, treat Qwen's answer as a hypothesis, not proof. When DOM/CDP is reachable, verify the coordinate with `document.elementFromPoint()` and/or the element's `getBoundingClientRect()` converted to the screenshot/xdotool coordinate system before clicking. Qwen can misidentify a nearby text/form area as a button when layout changes or DevTools has just been closed. +6. Record the finding in the application notes / skill known issues if it solves a reusable form issue. + +If a tool approval result says a command was denied but the user did not intentionally deny it, do not abandon the workflow. Explain the exact action, ask for/accept explicit approval, then retry or use the safer bundled helper. + +## CDP screenshot capture example + +```js +const shot = await c.send('Page.captureScreenshot', { + format: 'png', + captureBeyondViewport: false +}); +fs.writeFileSync('/tmp/qwen-debug.png', Buffer.from(shot.data, 'base64')); +``` + +If CDP on the target tab is wedged, use an alternate visual path instead of stopping: + +- Selenium/WebDriver screenshot endpoint if the browser is a Selenium container +- OS screenshot (`screencapture`) if the visible browser window is on the desktop +- VNC/noVNC screenshot if the Selenium browser is visible there + +For Selenium Chromium containers, use the `selenium-container-visual-click-recovery` skill: capture Xvfb with `ffmpeg`, ask Qwen for visible state/coordinates, and use `xdotool` inside the container to click blocked alerts or modal buttons. + +## Qwen call example + +Use the bundled helper to avoid approval friction from `curl | python`-style pipelines: + +```bash +node ../qwen-screenshot-debug/scripts/qwen-vlm-inspect.mjs \ + /tmp/qwen-debug.png \ + "Inspect this browser screenshot. Answer only with: visible status, any error text, whether the expected action appears complete, and the next visible button/control to click." +``` + +The helper calls `http://localhost:1234/v1/chat/completions` directly from Node, parses JSON internally, and prints only the model answer. + +## Prompt templates + +### Upload state + +> Inspect this browser screenshot. Did the resume/CV upload succeed? Quote any visible filename, success text, error text, or upload/remove button. If successful, what is the next visible control to continue? + +### Stuck wait / timeout + +> Inspect this browser screenshot. The automation waited for a state change but DOM checks timed out. What state is visibly shown? Is there a spinner, modal, validation error, file picker, success message, or next-step button? + +### Form validation + +> Inspect this application form screenshot. Identify visible validation errors, required fields still blank, and the next action the automation should take. + +Workday-specific checkbox lesson: if a Workday step reports `Field and Value required` for a required acknowledgement/terms checkbox, do not trust prior programmatic checkbox handling. Capture a screenshot, ask Qwen whether the checkbox is visibly checked/unchecked, then click the actual visible input coordinates and verify `checked=true` / `aria-checked=true` before pressing Next. + +### Verify filled field values + +> For each field below, state the EXACT VALUE SHOWN and whether there's a red 'Value is required' error: 1) [field name], 2) [field name], ... Also: is the submit/next button enabled? Are any fields still empty? + +Use this after filling a form with autocomplete/dropdown fields to confirm the correct option was selected — not just "something was typed." The keyboard+Enter fallback often selects the wrong dropdown entry; the VLM screenshot is the only reliable verification. + +## Important lesson from NTT DATA application + +The NTT DATA resume upload looked failed from DOM/CDP polling because `Runtime.evaluate` timed out after file selection. The visible page actually showed the upload had succeeded. The correct response was to take a screenshot and ask Qwen, not to assume the upload failed or retry with different file formats. + +In the Selenium Chromium container, the alert was dismissed with `xdotool` after Qwen identified the `OK` button coordinates. See `selenium-container-visual-click-recovery` for the reusable workflow. + +Base directory for this skill: this directory diff --git a/skills/qwen-screenshot-debug/references/local-qwen-helper-approval.md b/skills/qwen-screenshot-debug/references/local-qwen-helper-approval.md new file mode 100644 index 0000000..733a8ca --- /dev/null +++ b/skills/qwen-screenshot-debug/references/local-qwen-helper-approval.md @@ -0,0 +1,19 @@ +# Qwen screenshot workflow approval and safety notes + +Use the local helper instead of ad-hoc shell pipelines: + +```bash +node ../../qwen-screenshot-debug/scripts/qwen-vlm-inspect.mjs "" +``` + +Why: +- Avoids Tirith-flagged `curl | python` / network-output-to-interpreter patterns. +- Keeps screenshot inspection local to LM Studio/Qwen. +- Produces direct model output without requiring manual JSON parsing. + +User workflow preference: +- If the tool layer reports a denial for this workflow but the user did not intentionally deny it, do not abandon the task. +- Explain the exact command/action that was blocked and ask for explicit approval. +- After explicit approval, retry through the helper rather than rephrasing the risky pipeline. + +Use this helper for browser/application screenshots, CAPTCHA page diagnosis, and visual verification of form state. Do not use platform `vision_analyze` for this user’s job-automation screenshot workflow. \ No newline at end of file diff --git a/skills/qwen-screenshot-debug/scripts/qwen-vlm-inspect.mjs b/skills/qwen-screenshot-debug/scripts/qwen-vlm-inspect.mjs new file mode 100755 index 0000000..7768a5e --- /dev/null +++ b/skills/qwen-screenshot-debug/scripts/qwen-vlm-inspect.mjs @@ -0,0 +1,55 @@ +#!/usr/bin/env node +// Inspect an image with the local Qwen VLM via LM Studio without using a shell pipeline. +// Usage: +// node ../../qwen-screenshot-debug/scripts/qwen-vlm-inspect.mjs /tmp/screenshot.png "Inspect this browser screenshot..." +import fs from 'node:fs'; + +const imagePath = process.argv[2]; +const prompt = process.argv.slice(3).join(' ') || 'Inspect this browser screenshot. Answer with visible status, any error text, whether the expected action appears complete, and the next visible button/control to click.'; +const endpoint = process.env.QWEN_VLM_ENDPOINT || 'http://localhost:1234/v1/chat/completions'; +const model = process.env.QWEN_VLM_MODEL || 'qwen3.6-35b-a3b-holo3-qwopus-instruct-qx64-hi-mlx'; + +if (!imagePath) { + console.error('Usage: qwen-vlm-inspect.mjs [prompt]'); + process.exit(2); +} + +const b64 = fs.readFileSync(imagePath).toString('base64'); +const body = { + model, + messages: [{ + role: 'user', + content: [ + { type: 'image_url', image_url: { url: `data:image/png;base64,${b64}` } }, + { type: 'text', text: prompt }, + ], + }], + max_tokens: Number(process.env.QWEN_VLM_MAX_TOKENS || 300), + temperature: Number(process.env.QWEN_VLM_TEMPERATURE || 0.1), +}; + +const res = await fetch(endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), +}); + +const text = await res.text(); +if (!res.ok) { + console.error(`Qwen VLM HTTP ${res.status}: ${text.slice(0, 1000)}`); + process.exit(1); +} + +let data; +try { data = JSON.parse(text); } +catch (err) { + console.error(`Invalid JSON from Qwen VLM: ${text.slice(0, 1000)}`); + process.exit(1); +} + +const answer = data?.choices?.[0]?.message?.content; +if (!answer) { + console.error(`No Qwen answer in response: ${JSON.stringify(data).slice(0, 1000)}`); + process.exit(1); +} +console.log(answer.trim()); diff --git a/skills/qwen-screenshot-debug/scripts/visual-recover.mjs b/skills/qwen-screenshot-debug/scripts/visual-recover.mjs new file mode 100644 index 0000000..5dfc2b8 --- /dev/null +++ b/skills/qwen-screenshot-debug/scripts/visual-recover.mjs @@ -0,0 +1,169 @@ +#!/usr/bin/env node +// visual-recover.mjs — one-command visual recovery for stuck browser automation. +// +// Default protocol whenever a click / form-fill / submit / navigation cannot progress: +// 1. Capture what the browser ACTUALLY shows (CDP viewport or container X display). +// 2. Ask the local Qwen VLM what state is visible and where the next control is. +// 3. Optionally click that control (coordinate system paired with the capture method). +// 4. Re-capture and verify the state changed. +// +// Usage: +// node visual-recover.mjs # inspect via CDP (default) +// node visual-recover.mjs --prompt "Did the CV upload complete?" +// node visual-recover.mjs --url-substr linkedin.com/jobs # pick tab by URL fragment +// node visual-recover.mjs --click # inspect + click next control + verify +// node visual-recover.mjs --capture ffmpeg --click # container display + xdotool (for modals/alerts CDP can't see) +// node visual-recover.mjs --ask "Is there a validation error under the email field?" +// +// Coordinate-system contract (the #1 historical failure when mixed): +// --capture cdp -> screenshot = viewport px -> click via CDP Input.dispatchMouseEvent +// --capture ffmpeg -> screenshot = X display px -> click via xdotool inside the container +// +// Exit codes: 0 ok / progressed, 1 qwen or capture error, 3 clicked but state did not change. +import fs from 'node:fs'; +import path from 'node:path'; +import http from 'node:http'; +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; + +const JH = process.env.JOBHUNTER_HOME || `${process.env.HOME}/.job-hunter`; +const CDP_PORT = Number(process.env.BROWSER_CDP_PORT || 9225); +const CONTAINER = process.env.SELENIUM_CONTAINER || 'selenium-chromium'; +const ENDPOINT = process.env.QWEN_VLM_ENDPOINT || 'http://localhost:1234/v1/chat/completions'; +const MODEL = process.env.QWEN_VLM_MODEL || 'qwen3.6-35b-a3b-holo3-qwopus-instruct-qx64-hi-mlx'; + +const args = process.argv.slice(2); +const opt = { capture: 'cdp', click: false, prompt: null, urlSubstr: null, tabId: null }; +for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === '--capture') opt.capture = args[++i]; + else if (a === '--click') opt.click = true; + else if (a === '--prompt' || a === '--ask') opt.prompt = args[++i]; + else if (a === '--url-substr') opt.urlSubstr = args[++i]; + else if (a === '--tab-id') opt.tabId = args[++i]; + else if (a === '--cdp-port') { /* override */ } +} + +const stamp = Date.now(); +const shot = (n) => `/tmp/visual-recover-${stamp}-${n}.png`; + +// ---------- capture ---------- +function httpJson(port, p) { + return new Promise((resolve, reject) => { + const req = http.request({ host: '127.0.0.1', port, path: p, method: 'GET' }, (res) => { + let d = ''; res.on('data', (c) => (d += c)); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(e); } }); + }); + req.on('error', reject); req.end(); + }); +} + +let WebSocketImpl = globalThis.WebSocket; +if (!WebSocketImpl) { + try { WebSocketImpl = createRequire(path.join(JH, 'package.json'))('ws'); } catch { /* noop */ } +} + +async function cdpSession() { + const tabs = await httpJson(CDP_PORT, '/json'); + const pages = tabs.filter((t) => t.type === 'page'); + let tab = opt.tabId ? pages.find((t) => t.id === opt.tabId) : null; + if (!tab && opt.urlSubstr) tab = pages.find((t) => (t.url || '').includes(opt.urlSubstr)); + if (!tab) tab = pages[0]; + if (!tab) throw new Error('no CDP page tab found'); + const ws = new WebSocketImpl(tab.webSocketDebuggerUrl); + await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; }); + let id = 0; const pending = new Map(); + ws.onmessage = (ev) => { + const m = JSON.parse(ev.data.toString()); + if (m.id && pending.has(m.id)) { const { res, rej } = pending.get(m.id); pending.delete(m.id); m.error ? rej(new Error(m.error.message)) : res(m.result); } + }; + const send = (method, params = {}) => new Promise((res, rej) => { const mid = ++id; pending.set(mid, { res, rej }); ws.send(JSON.stringify({ id: mid, method, params })); }); + return { send, close: () => ws.close(), url: tab.url }; +} + +async function captureCdp(session, file) { + const r = await session.send('Page.captureScreenshot', { format: 'png' }); + fs.writeFileSync(file, Buffer.from(r.data, 'base64')); +} + +function captureFfmpeg(file) { + execFileSync('docker', ['exec', CONTAINER, 'sh', '-lc', + `ffmpeg -y -f x11grab -video_size 1920x1080 -i :99.0 -frames:v 1 /tmp/vr.png >/tmp/vr-ffmpeg.log 2>&1`]); + execFileSync('docker', ['cp', `${CONTAINER}:/tmp/vr.png`, file]); +} + +// ---------- qwen ---------- +async function askQwen(imageFile, prompt, wantJson = false) { + const b64 = fs.readFileSync(imageFile).toString('base64'); + const body = { + model: MODEL, + messages: [{ role: 'user', content: [ + { type: 'image_url', image_url: { url: `data:image/png;base64,${b64}` } }, + { type: 'text', text: prompt }, + ] }], + max_tokens: Number(process.env.QWEN_VLM_MAX_TOKENS || 400), + temperature: 0.1, + }; + const res = await fetch(ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const text = await res.text(); + if (!res.ok) throw new Error(`Qwen VLM HTTP ${res.status}: ${text.slice(0, 500)}`); + const content = JSON.parse(text).choices?.[0]?.message?.content ?? ''; + if (!wantJson) return content; + const m = content.match(/\{[\s\S]*\}/); + if (!m) throw new Error(`Qwen did not return JSON: ${content.slice(0, 300)}`); + return JSON.parse(m[0]); +} + +const INSPECT_PROMPT = opt.prompt || + 'Inspect this browser screenshot from a stuck automation flow (a click, form fill, or submit did not visibly progress). Report: (1) what state is visibly shown, (2) any error/validation text verbatim, (3) whether the intended action appears complete, (4) the next visible button or control to click to make progress, if any.'; + +const CLICK_PROMPT = (base) => + `${base}\n\nThe screenshot is ${opt.capture === 'ffmpeg' ? '1920x1080 (full display)' : 'the browser viewport'}. ` + + 'Respond ONLY with strict JSON: {"state": "", "action_needed": true|false, "label": "", "x": , "y": }. ' + + 'If no click is needed or no control is visible, use action_needed=false and x=y=0.'; + +// ---------- main ---------- +try { + let session = null; + const capture = async (file) => { + if (opt.capture === 'ffmpeg') return captureFfmpeg(file); + if (!session) session = await cdpSession(); + return captureCdp(session, file); + }; + + await capture(shot(1)); + console.log(`screenshot: ${shot(1)}${session ? ` (tab: ${session.url.slice(0, 90)})` : ''}`); + + if (!opt.click) { + const answer = await askQwen(shot(1), INSPECT_PROMPT); + console.log('\n--- Qwen inspection ---\n' + answer); + session?.close(); + process.exit(0); + } + + // click mode + const plan = await askQwen(shot(1), CLICK_PROMPT(opt.prompt || 'Identify the next control to click to make this stuck flow progress (dismiss modal/alert, press Next/Submit/Continue/OK, close overlay).'), true); + console.log(`\nQwen: ${JSON.stringify(plan)}`); + if (!plan.action_needed) { console.log('No click needed per Qwen — inspect output above and handle in DOM.'); session?.close(); process.exit(0); } + + if (opt.capture === 'ffmpeg') { + execFileSync('docker', ['exec', CONTAINER, 'sh', '-lc', `DISPLAY=:99.0 xdotool mousemove ${plan.x} ${plan.y} click 1`]); + console.log(`xdotool click at ${plan.x},${plan.y} ("${plan.label}")`); + } else { + for (const type of ['mousePressed', 'mouseReleased']) { + await session.send('Input.dispatchMouseEvent', { type, x: plan.x, y: plan.y, button: 'left', clickCount: 1 }); + } + console.log(`CDP click at ${plan.x},${plan.y} ("${plan.label}")`); + } + + await new Promise((r) => setTimeout(r, 2500)); + await capture(shot(2)); + const verify = await askQwen(shot(2), + `Compare with the previous state: the automation just clicked "${plan.label}" at ${plan.x},${plan.y}. ` + + 'Respond ONLY with strict JSON: {"changed": true|false, "state": ""}.', true); + console.log(`verify: ${JSON.stringify(verify)} (screenshot: ${shot(2)})`); + session?.close(); + process.exit(verify.changed ? 0 : 3); +} catch (err) { + console.error(`visual-recover error: ${err.message}`); + process.exit(1); +} diff --git a/skills/salary-calculator/SKILL.md b/skills/salary-calculator/SKILL.md new file mode 100644 index 0000000..398ad13 --- /dev/null +++ b/skills/salary-calculator/SKILL.md @@ -0,0 +1,153 @@ +--- +name: salary-calculator +description: Enriches saved jobs in jobhunter.sqlite with either a verified posted salary or a clearly-labeled market-rate estimate, never confusing the two. Uses LinkedIn for posted salaries and ITJobsWatch for UK benchmarks. Preserves provenance from estimate to benchmark snapshot across cache refreshes. Single-writer concurrency via salary_writer_lock. Use for one-shot enrichment, batch passes over unsalaried jobs, or extraction-rate health checks. +allowed-tools: read bash write +--- +## Workspace + +All paths in this skill are written against `${WORKSPACE}` — the canonical job-hunter home `$JOBHUNTER_HOME`, default `~/.job-hunter` (NOT the launch directory; contract updated 2026-07-06). Resolve to `process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter')` in Node and `"${JOBHUNTER_HOME:-$HOME/.job-hunter}"` in bash. The SQLite DB is at `~/.job-hunter/jobhunter.sqlite`. Pass `--db "$HOME/.job-hunter/jobhunter.sqlite"` to helper scripts (their built-in default may still resolve from cwd). Do NOT hardcode `$JOBHUNTER_HOME`. + +# Salary Calculator + +## Purpose + +For every saved job in `jobhunter.sqlite`, the salary-calculator skill surfaces either a verified posted salary or a market-rate estimate — and **never confuses the two**. Provenance from every estimate back to the source benchmark snapshot is preserved across cache refreshes; historical observations are never rewritten when the benchmark cache is refreshed. + +The skill operates a two-class observation model enforced at the SQL level (CHECK partition on `job_salary_observations`). Every row is either an **exact** observation (`is_posted_salary=1`, `benchmark_id IS NULL`) sourced from a job page or ATS API, or an **estimate** (`is_posted_salary=0`, `benchmark_id NOT NULL`) derived from an immutable cohort snapshot in `salary_benchmarks`. This is the core correctness property of the skill: an estimated salary is never presented as posted, and the lineage from estimate to benchmark is always queryable. + +The skill is invoked by Pi (or directly via `npm run` / `node scripts/...`) to enrich one job at a time, batch over all unsalaried jobs, refresh benchmarks without inserting estimates, or run an extraction-rate health check. + +## Golden Rule + +Never present an estimated salary as posted. Every observation belongs to exactly one of two classes — `is_posted_salary=1` with `benchmark_id IS NULL` (exact), or `is_posted_salary=0` with a real `benchmark_id` (estimate). The CHECK partition in SQLite enforces this at the database level; no application code path can bypass it. + +The database is the source of truth for salary presentation. Do not cache salary amounts in skill Markdown, prompt examples, or static report templates; they change. Templates may define columns only. At render time, fetch salary values from `job_salary_observations` / `salary_benchmarks` or run fresh enrichment, then display the current value with posted-vs-estimated provenance. If a generated Markdown/CSV artifact includes salary, treat it as a timestamped snapshot, not a reusable source. + +## Workspace resolution rule + +`${WORKSPACE}` in every example, bash/SQL snippet, and `DEFAULT_DB` constant in this skill means the canonical job-hunter home `$JOBHUNTER_HOME`, default `~/.job-hunter` — not the launch directory (contract updated 2026-07-06). Resolve to `process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter')` in Node and `"${JOBHUNTER_HOME:-$HOME/.job-hunter}"` in bash. If you find a hardcoded `WORKSPACE/` or any other project-specific path anywhere in this skill (including references and scripts), replace it with `${WORKSPACE}/...` and `path.join(process.cwd(), '...')` respectively. Canonical reference: `auto-job-application` skill → Core Rule 17 + Workspace section. + +## Workflow + +1. Acquire the writer lock (`salary_writer_lock`) via `BEGIN IMMEDIATE` — fail fast if another writer holds it (exit 4). +2. Fetch the job page via the chosen adapter (`scripts/sources/linkedin.mjs` or `scripts/sources/itjobswatch.mjs`). +3. Parse salary candidates from the response (regex + JSON-LD), with explicit boundary exclusion for LinkedIn's "Similar jobs" / "People also viewed" / "More searches" / "Explore top content" / "Show more jobs like this" sections (PARSE-08). +4. If the LinkedIn/job-board page has no posted salary and exposes an external advertiser/ATS apply URL, follow that URL in read-only mode before falling back to market benchmarks. Do **not** submit or fill an application; only load the advertiser/ATS job detail page and parse a salary range that is clearly tied to the same job. Persist it as `is_posted_salary=1` with `data_source_url` pointing to the advertiser/ATS page. Do not mark a job as applied merely because the external apply link was opened. +5. Select the best observation via the deterministic 8-level comparator in `scripts/lib/salary-selector.mjs` (confidence > exactness > predicted > observed_at > location > currency > source > observation_id). +6. Persist via `INSERT OR IGNORE` against `job_salary_observations`; for estimates, look up or store the cohort snapshot in `salary_benchmarks` first. +7. Apply per-axis retry state via `applyRetryTransition` on `job_enrichment_state` (RETRY-04 axis independence — exact and benchmark axes never share counters). +8. Release the writer lock; emit a stdout envelope (canonical text or `--json`). + +## Commands + +Use the active Pi skill path (`../salary-calculator/...`) for these commands unless the user gives a different active profile path. + +| Goal | Command | +|------|---------| +| Enrich one LinkedIn job | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --source linkedin --job-id 12345` | +| Advertiser/ATS exact salary scan for a scored queue | `node scripts/external-salary-scan.mjs --search-id score-YYYYMMDD --limit 10` | +| Batch salary research for an Apply queue | `node scripts/batch-salary-research.mjs --search-id score-YYYYMMDD` | +| Batch pass over unsalaried | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --all-unsalaried --limit 50` | +| Refresh benchmark cache without inserting estimates | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --all-unsalaried --refresh-benchmarks` | +| Benchmark-only (skip exact pass entirely) | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --all-unsalaried --benchmark-only` | +| Force exact retry on stuck jobs | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --all-unsalaried --force-exact-retry` | +| Force benchmark retry | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --all-unsalaried --force-benchmark-retry` | +| Check extraction-rate health | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --health` | +| JSON envelope (single job) | `node scripts/enrich-job-salary.mjs --source linkedin --job-id 12345 --json` | +| Health report (JSON) | `node scripts/enrich-job-salary.mjs --db jobhunter.sqlite --health --json` | +| Dry-run (no DB writes) | `node scripts/enrich-job-salary.mjs --source linkedin --job-id 12345 --dry-run` | + +`batch-salary-research.mjs` routes `source='external'` jobs through `external-salary-scan.mjs`; `enrich-job-salary.mjs` remains the adapter pipeline for board/benchmark sources. + +## Source Priority + +| Source | Exact salary? | Benchmark? | Regions | Notes | +|--------|---------------|------------|---------|-------| +| linkedin | yes | no | GB, IE, US, CA, AU, EU, CH, AE | Excludes 'Similar jobs' / 'People also viewed' / 'More searches' / 'Explore top content' / 'Show more jobs like this' sections (PARSE-08) | +| advertiser_ats | yes | no | Any | Use `scripts/external-salary-scan.mjs` to follow the LinkedIn/job-board external apply URL in read-only mode and parse salary from the advertiser/ATS job detail page before using market estimates. Do not submit/fill forms; exact only when the salary is clearly tied to the same job. | +| itjobswatch | no | yes | GB primarily | UK technology-role benchmarks; normalizer-stamped (`normalizer_version` column on every persisted row) | +| indeed | no | yes | IE, GB, US, CA, AU, CH, AE, major EU | Public Indeed Career salary pages; 401/403/no parse returns no-data so fallback sources can run. | +| levels_fyi | no | yes | IE, GB, US, CA, AU, CH, major EU, AE opportunistic | Tech compensation range pages; generally total compensation. | +| robert_half | no | yes | IE, GB, US, CA, AU, CH, AE opportunistic | Public salary-guide HTML; role-adjacent extraction when ranges are exposed. | +| salaryexpert | no | yes | IE, GB, US, CA, AU, CH, AE, major EU opportunistic | Public SalaryExpert pages when accessible; frequent 403 is treated as no-data fallback, not a hard failure. | + +Benchmark sources are tried in country-code order; the routing table is encoded in `scripts/lib/source-priority.mjs`. Each source advertises `supports.exactSalary` / `supports.benchmark` flags and per-source rate limits (`limits.perHostRps`, `limits.maxConcurrent`). + +## Supported Regions + +| Country | Currency | Sample sources | +|---------|----------|----------------| +| GB | GBP | linkedin, itjobswatch | +| IE | EUR | linkedin, indeed, levels_fyi, robert_half, salaryexpert | +| US | USD | linkedin, levels_fyi, indeed, robert_half, salaryexpert | +| CA | CAD | linkedin, levels_fyi, indeed, robert_half, salaryexpert | +| AU | AUD | linkedin, levels_fyi, indeed, robert_half, salaryexpert | +| CH | CHF | linkedin, levels_fyi, indeed, robert_half, salaryexpert | +| AE | AED | linkedin, indeed, levels_fyi, robert_half, salaryexpert | +| EU (other) | EUR | linkedin, levels_fyi, indeed, salaryexpert | + +Cross-currency numeric comparison is intentionally NOT performed in v1 — FX columns (`amount_*`, `annualized_*`, `fx_*`) are display-only and the selector cannot rank across currencies. This is a deliberate v2 boundary. + +## SQLite Tables + +| Table | Purpose | +|-------|---------| +| `jobs` | Read-only source of saved jobs (managed by the `linkedin-job-search` skill). The salary-calculator never mutates this table. | +| `job_salary_observations` | Insert-only observation log; CHECK partition separates exact (`is_posted_salary=1`, `benchmark_id IS NULL`) from estimate (`is_posted_salary=0`, `benchmark_id NOT NULL`). `INSERT OR IGNORE` conflict policy. | +| `salary_benchmarks` | Immutable benchmark snapshots; `normalizer_version` column is a schema-migration trigger (see Normalizer Version subsection below). UNIQUE(`benchmark_series_id`, `payload_hash`) deduplication. | +| `job_enrichment_state` | Per-job retry/backoff state for exact + benchmark axes (RETRY-04 axis independence). `trg_state_updated` trigger maintains `updated_at`. | +| `salary_writer_lock` | Advisory single-writer lock acquired via `BEGIN IMMEDIATE`; 10-min stale-window recovery via `acquired_at` heartbeat. | + +### Normalizer Version + +The salary normalization layer (titles, seniority, industry) is rule-driven by [references/normalization.md](references/normalization.md). Benchmark source behavior, fallback semantics, and verification patterns are documented in [references/benchmark-adapters.md](references/benchmark-adapters.md). The integer `NORMALIZER_VERSION` exported from `scripts/lib/normalize/rules-loader.mjs` is a **schema-migration trigger**: every row inserted into `salary_benchmarks` carries this stamp in the `normalizer_version` column. + +**Current value:** `1` + +**Bump policy:** Any semantic edit to `references/normalization.md` (adding/removing/changing a synonym, reordering the seniority table, adding/renaming an industry code, expanding the location list) MUST be paired with an increment of the integer constant in `scripts/lib/normalize/rules-loader.mjs` in the SAME commit. Whitespace-only or comment-only edits do not bump. + +**Downstream invalidation:** Benchmarks stamped with an older `normalizer_version` are NOT retroactively re-normalized. They remain valid identity within their version cohort. The benchmark cache (Phase v1.0-06) treats `(benchmark_series_id, normalizer_version)` as the cohort key — cross-version matches are explicit, not silent. + +**Schema migration:** Adding the `normalizer_version` column to an existing v1.0-01 database is handled idempotently by `ensureNormalizerVersionColumn()` in `scripts/lib/normalize/benchmark-stamp.mjs`, called from the schema installer. Backfilled rows default to version 1. + +## Retry Semantics + +- **Exact axis:** `next_exact_retry_at = now + min(2^(attempt_count-1), 7)` days (capped at 7); after attempt 5 → `NULL` (manual reset via `--force-exact-retry`). +- **Benchmark axis:** independent columns (`benchmark_status`, `benchmark_attempt_count`, `next_benchmark_retry_at`) — same math, independent counter (RETRY-04). +- **`not_found` floor:** exact = 14 days, benchmark = 30 days. Prevents thrash on jobs that genuinely have no posted salary. +- **Reproducibility:** behaviour is reproducible from persisted state alone (RETRY-03 — no in-memory counters, no module-scope mutable state, `nowIso` injected into pure functions). +- **Mixed-status arrays:** when `classifyError` receives a non-empty array with no `markerNotFound` token and at least one failure, the result is `not_found` (opencode improvement #2 / Branch C), not `unrecoverable_error`. + +## Concurrency + +Single-writer model. Each writer acquires `salary_writer_lock` via `BEGIN IMMEDIATE` before any insert; contention exits non-zero (exit 4) with "Another writer holds the lock; try again in minutes". A crashed writer's lock is reclaimable after 10 minutes (`acquired_at` heartbeat). + +Reads (`--health`, batch-count queries) do NOT acquire the writer lock — health checks can run concurrently with an in-flight enrichment. The CLI verifies job-existence BEFORE acquiring the writer lock so that exit 2 (precondition) takes precedence over exit 4 (contention). + +## Manual benchmark updates + +When the automated benchmark pass returns `not_found` but the user asks for salary updates across countries or main locations, perform a manual market-benchmark update rather than leaving jobs unsalaried. Keep the core partition strict: manual market numbers are estimates (`is_posted_salary=0`, `is_predicted=1`) and must first be persisted as `salary_benchmarks`, then linked from `job_salary_observations`. See `references/manual-market-benchmark-updates.md` for the researched-source hierarchy, SQLite insert pattern, migrated-DB compatibility cautions, and verification queries. + +## Lessons / Guardrails + +### Do not replace source-backed salary scraping with manual curated estimates + +When asked to update salary information, run the salary-calculator workflow and source adapters first. Do not manually insert a batch of ad-hoc estimates from search snippets as if that satisfied the skill. The current pipeline supports LinkedIn exact salary extraction and ITJobsWatch UK benchmarks; non-UK market benchmarks need explicit source adapters (for example Indeed salary pages, Levels.fyi, Example Location 013.ai/Robert Half, SalaryExpert if accessible) or a clearly documented fallback. If a region lacks an adapter, say that directly, add/patch the adapter or store any fallback as `estimated_market` with full provenance, and update this skill so the gap is not rediscovered. + +### Schema side-effects must be explicit and fixed properly + +If inserts fail because a legacy trigger/view/table is broken (for example a trigger references a missing `match_results_old` table), do not silently create compatibility tables as the final fix. Call out the schema bug, prefer a proper migration/trigger fix, and only use a compatibility shim as a temporary unblocker with clear reporting. + +## Troubleshooting + +| Symptom | Likely cause | Remedy | +|---------|--------------|--------| +| Exit 4 + "Another writer holds the lock" | Concurrent CLI invocation | Wait the printed minutes, or rerun later. `--force-exact-retry` does NOT bypass the lock — it only resets retry state once the lock is acquired. | +| Parser returns 0 candidates on a known-salaried page | LinkedIn DOM drift or HTML→text gap | Run `--health` — if hit-rate dropped, see HEALTH-03 warning. Inspect `evidence_snippet` in `job_salary_observations` for the last successful row, then compare to the failing page. | +| Exit 5 + "401/403" | LinkedIn auth expired | Re-run the `brave-obscura-session` skill to refresh cookies; rerun with `--force-exact-retry` once auth is back. | +| Exit 6 + "transient exhausted" | 5xx retry budget exhausted (5 attempts) | Wait for `next_exact_retry_at` (printed in stderr); rerun with `--force-exact-retry` to reset the counter. Check `references/benchmark-adapters.md` for supported source behavior. | +| Exit 7 + per-job synthetic envelope in batch | Unhandled exception in pipeline for a single job | Inspect the offending job via single-job CLI (`--job-id`); 7-dominates-6 priority means the batch may still surface this even if other jobs schedule transient retries. | +| `ensureNormalizerVersionColumn` errors | Schema drift between repo and DB | Run `npm run ensure-schema -- --db ` to install the migration. Backfill defaults to version 1; cross-version matches in the cache are explicit. | +| `.pi/agents/` (plural) in install path | Typo (canonical path is singular `.pi/agent/`) | `install-skill.mjs` refuses with exit 2 + stderr "plural". Re-run with the correct path: `../salary-calculator/`. | + +Base directory for this skill: this directory diff --git a/skills/salary-calculator/package.json b/skills/salary-calculator/package.json new file mode 100644 index 0000000..e6e0bfc --- /dev/null +++ b/skills/salary-calculator/package.json @@ -0,0 +1,7 @@ +{ + "name": "salary-calculator", + "type": "module", + "dependencies": { + "better-sqlite3": "^12.9.0" + } +} diff --git a/skills/salary-calculator/references/benchmark-adapters.md b/skills/salary-calculator/references/benchmark-adapters.md new file mode 100644 index 0000000..c51fc19 --- /dev/null +++ b/skills/salary-calculator/references/benchmark-adapters.md @@ -0,0 +1,93 @@ +# Benchmark adapter notes + +This reference captures durable implementation details for the market-salary benchmark adapters in `scripts/sources/`. + +## Adapter contract + +Each benchmark adapter should export: + +- `sourceName` +- `supports = { exactSalary: false, benchmark: true }` +- `limits = { perHostRps, maxConcurrent }` +- `fetchExactSalary()` returning `[]` +- `fetchBenchmark(query, ctx)` returning either: + - a `storeBenchmarkSnapshot`-shaped benchmark object, or + - `null` for clean no-data / inaccessible / no parseable public range. + +Do not throw for common public-page no-data conditions such as 401/403 blocks from opportunistic fallback sources. Returning `null` lets the country-priority chain continue to later adapters. + +## Current adapters + +- `indeed.mjs`: public Indeed Career salary pages (`/career//salaries`). Extracts average salary from meta description / JSON-LD / nearby page text. Treat 401/403/no parse as `null`. +- `levels_fyi.mjs`: public Levels.fyi role/location pages. Extracts ranges from meta description or embedded Next.js payload. These values are generally total compensation; set `compensationType: 'total_compensation'`. +- `robert_half.mjs`: public Robert Half salary guides. Opportunistic role-adjacent range extraction when static HTML exposes ranges. Return `null` if guide content is too interactive or unparseable. +- `salaryexpert.mjs`: public SalaryExpert pages when accessible. SalaryExpert commonly returns 403 to automation; treat as `null`, not a hard benchmark failure. +- `itjobswatch.mjs`: UK benchmark adapter; keep GB first to preserve existing UK behavior. + +## Country priority + +`source-priority.mjs` owns benchmark ordering. Current durable defaults: + +- GB: `itjobswatch`, `indeed`, `levels_fyi`, `robert_half`, `salaryexpert` +- IE: `indeed`, `levels_fyi`, `robert_half`, `salaryexpert` +- US/CA/AU/CH: `levels_fyi`, `indeed`, `robert_half`, `salaryexpert` +- AE: `indeed`, `levels_fyi`, `robert_half`, `salaryexpert` +- major EU: `levels_fyi`, `indeed`, `salaryexpert` + +The pipeline should try adapters in order until one returns a benchmark object. `null`/`undefined` means clean no-data and should not poison retry state while later priority sources remain available. + +## When the benchmark-only pass returns 0 across a batch + +A `--all-unsalaried --limit 30 --refresh-benchmarks` run that ends with +`found=0 not_found=0 error=0` is the common first-run outcome when: + +- The benchmark cache is empty (no `salary_benchmarks` rows yet). +- Adapters are returning `null` for the cohort (e.g. the role/location + combo has no public range pages, or all 4 fallback sources 401/403). +- The `cohort_id` normalizer has a synonym gap (e.g. "AI Architect" is + not yet mapped to any benchmark cohort). + +In all three cases, the right next step is **not** to keep looping the +`--all-unsalaried` pass — it will keep returning 0 until the cache +warms or the cohort is fixed. Instead: + +1. **Inspect the benchmark cache** with + `sqlite3 ${WORKSPACE}/jobhunter.sqlite "SELECT COUNT(*) FROM salary_benchmarks;"`. + If 0 rows, the cache is cold — pre-warm by running `--refresh-benchmarks` + against a small hand-picked set of representative jobs (1 per role + family × 1 per country). +2. **Check the cohort extractor** for the failing role. The runner prints + `fetching benchmark cohort: ` per job; if the slug is gibberish + (e.g. `chief_technology officer` instead of `ai-architect`), the + `references/normalization.md` rules need a synonym. Edit and re-run. +3. **If cache is warm and cohorts are sane but adapters still return null** + for a specific region, the region lacks a public benchmark source for + that role. The right move is to skip the benchmark pass for that + region and rely on posted-salary data only (run + `--all-unsalaried --force-exact-retry` to push the LinkedIn exact + path instead). + +A 30-job pass with 0 found is a SIGNAL to switch strategy, not a +failure to retry. + +## Verification pattern + +Use `node --check` for every changed `.mjs` file, then run a dry-run benchmark-only enrichment against a real saved job: + +```bash +node scripts/enrich-job-salary.mjs \ + --db ${WORKSPACE}/jobhunter.sqlite \ + --source indeed \ + --job-id \ + --benchmark-only \ + --dry-run \ + --json +``` + +Expected dry-run behavior when an adapter succeeds: + +- `state.benchmark.status = 'found'` +- `state.exitCode = 0` +- no DB writes are required in dry-run, so `selectedBest` may remain null. + +Live IE smoke-test lesson: Indeed may return no-data in the CLI path while Levels.fyi still provides a usable Ireland tech range. This is the reason adapter fallback must continue after `null` rather than converting the first inaccessible source into an unrecoverable error. diff --git a/skills/salary-calculator/references/manual-market-benchmark-updates.md b/skills/salary-calculator/references/manual-market-benchmark-updates.md new file mode 100644 index 0000000..0842d8d --- /dev/null +++ b/skills/salary-calculator/references/manual-market-benchmark-updates.md @@ -0,0 +1,61 @@ +# Manual market benchmark updates + +Use this reference when the built-in benchmark pass returns `not_found` but the user asks to update salary information across countries/locations. + +## Pattern + +1. Keep the salary model distinction intact: + - posted salaries: `job_salary_observations.is_posted_salary=1`, `benchmark_id IS NULL` + - market estimates: `is_posted_salary=0`, `is_predicted=1`, `benchmark_id` points to `salary_benchmarks` +2. Search multiple sources per country/location and save the provenance in both: + - `salary_benchmarks.evidence_snippet` + - `salary_benchmarks.raw_payload_json` + - the derived `job_salary_observations.evidence_snippet/raw_payload_json` +3. Insert immutable benchmark snapshots first, then per-job observations from those snapshots. +4. Prefer city-specific benchmark rows for job observations; fall back to country-level benchmarks when no city-specific benchmark exists. +5. Upsert `salary_market_medians` for country-level role mappings used by `jobs_with_salary`. + +## Useful source hierarchy observed + +- UK: ITJobsWatch is strong for AI Architect country/ex-London market benchmarks; Levels.fyi can help for London Solution Architect total-comp benchmarks. +- Ireland: Indeed AI Architect gives country averages/ranges; Levels.fyi Greater Dublin Solution Architect gives Dublin total-comp distribution. +- Switzerland: Example Location 013.ai and Robert Half can provide AI/AI-engineer ranges; use AI Architect as a labeled market estimate if exact architect benchmarks are sparse. +- UAE: Indeed UAE/Dubai AI Architect pages may show both low averages and active senior ranges; for senior AI Architect job searches prefer active senior ranges but label them as estimates. +- US: Indeed AI Architect gives country averages/ranges; Levels.fyi Solution Architect is useful for high-comp city markets such as New York and San Francisco. + +## SQLite cautions + +- Some migrated `jobhunter.sqlite` DBs may contain a legacy trigger `trg_salary_gate_check` that references `match_results_old`. If inserts into `job_salary_observations` fail with `no such table: main.match_results_old`, create a compatibility table from `match_results` before inserting: + +```sql +CREATE TABLE match_results_old AS SELECT * FROM match_results; +CREATE INDEX IF NOT EXISTS idx_match_results_old_job + ON match_results_old(source, job_id, search_id); +``` + +Do not create it as a view: the trigger performs an `UPDATE`, so a view will fail with `cannot modify match_results_old because it is a view`. + +- Normalize missing country codes before joining to salary medians. In particular, UAE Indeed rows may have city/location populated (`Dubai`, `Abu Dhabi`, `..., AE`) while `country_code` is NULL. Update those to `AE` so `jobs_with_salary` and benchmark matching can work. + +## Verification queries + +After a manual benchmark update, verify at least: + +```sql +SELECT country_code, currency, annual_p25, annual_median, annual_p75, source +FROM salary_market_medians +WHERE role_family='AI Architect' AND seniority='_any' +ORDER BY country_code; + +SELECT country_code, city, currency, amount_p25, amount_median, amount_p75, data_source +FROM salary_benchmarks +WHERE normalized_title='ai architect' +ORDER BY country_code, city, fetched_at DESC; + +SELECT country_code, city, currency, COUNT(*) n, + MIN(annualized_median) min_med, MAX(annualized_median) max_med +FROM job_salary_observations +WHERE confidence_label='estimated_market' +GROUP BY country_code, city, currency +ORDER BY country_code, city; +``` diff --git a/skills/salary-calculator/references/normalization.md b/skills/salary-calculator/references/normalization.md new file mode 100644 index 0000000..cb441a0 --- /dev/null +++ b/skills/salary-calculator/references/normalization.md @@ -0,0 +1,278 @@ +# Normalization Rules + +NORMALIZER_VERSION: 1 +Last bumped: 2026-05-12 +Changed: Initial version + +## compound_phrases + +```yaml +c++: cpp +c#: csharp +.net: dotnet +machine learning: machine_learning +artificial intelligence: artificial_intelligence +natural language processing: natural_language_processing +site reliability engineer: sre +software engineer: software_engineer +solutions architect: solutions_architect +backend engineer: backend_engineer +frontend engineer: frontend_engineer +frontend developer: frontend_engineer +full stack engineer: full_stack_engineer +senior engineer: senior_engineer +principal engineer: principal_engineer +staff engineer: staff_engineer +engineering manager: engineering_manager +technical lead: technical_lead +product manager: product_manager +project manager: project_manager +quality assurance: quality_assurance +user experience: user_experience +user interface: user_interface +cloud architect: cloud_architect +security engineer: security_engineer +network engineer: network_engineer +database administrator: database_administrator +systems administrator: systems_administrator +business analyst: business_analyst +systems engineer: systems_engineer +solutions engineer: solutions_engineer +technical architect: technical_architect +senior manager: senior_manager +vice president: vice_president +chief executive: chief_executive +chief financial: chief_financial +chief technology: chief_technology +chief operating: chief_operating +senior architect: senior_architect +lead architect: lead_architect +``` + +## synonym_map + +```yaml +sr: senior +sr.: senior +jr: junior +jr.: junior +eng: engineer +engr: engineer +engineers: engineer +swe: software_engineer +mgr: manager +managers: manager +dev: developer +developers: developer +vp: vice_president +ai: artificial_intelligence +ml: machine_learning +nlp: natural_language_processing +ui: user_interface +ux: user_experience +devops: development_operations +i: 1 +ii: 2 +iii: 3 +iv: 4 +v: 5 +développeur: developer +desarrollador: developer +entwickler: developer +architekt: architect +architekt: architect +architetto: architect +architeto: architect +ingeniero: engineer +ingegnere: engineer +senior: senior +principal: principal +manager: manager +director: director +lead: lead +architect: architect +analyst: analyst +engineer: engineer +administrator: administrator +specialist: specialist +coordinator: coordinator +consultant: consultant +associate: associate +junior: junior +intern: intern +trainee: trainee +programmer: programmer +coder: coder +technician: technician +operator: operator +officer: officer +executive: executive +president: president +vice: vice +chief: chief +officer: officer +architect: architect +analyst: analyst +administrator: administrator +supervisor: supervisor +coordinator: coordinator +designer: designer +writer: writer +``` + +## seniority_table + +```yaml +- bucket: cxo + keywords: [cxo, ceo, cfo, cto, coo, chief, chief_executive, chief_financial, chief_technology, chief_operating] +- bucket: vp + keywords: [vp, vice_president, vice president, vice] +- bucket: director + keywords: [director, directeur, direktor, direttore] +- bucket: manager + keywords: [manager, mgr, management, managing, engineering_manager, product_manager, project_manager, senior_manager] +- bucket: lead + keywords: [lead, tech_lead, technical_lead] +- bucket: principal + keywords: [principal, principal_engineer] +- bucket: staff + keywords: [staff, staff_engineer] +- bucket: senior + keywords: [senior, sr, senior_engineer, sr., iii, iv, v] +- bucket: mid + keywords: [mid, mid_level, intermediate, intermediate_level] +- bucket: junior + keywords: [junior, jr, jr., junior_engineer, entry_level, entry] +- bucket: intern + keywords: [intern, internship, trainee, apprentice] +- bucket: _any + keywords: [] +``` + +## industry_list + +```yaml +finance: [finance, banking, investment, trading, hedge_fund, fintech, capital, securities, forex, crypto, blockchain, financial, banking, payment, lending] +healthcare: [healthcare, medical, hospital, pharmacy, clinical, health, doctor, nurse, physician, surgery, dental, mental, wellness, telemedicine, biomedical] +biotech: [biotech, biotechnology, pharmaceutical, pharma, drug, clinical_trial, research, lab, laboratory, genetics, dna, rna, protein, cell, tissue, immunology, oncology] +manufacturing: [manufacturing, factory, industrial, production, operations, supply_chain, logistics, assembly, warehouse, distribution, fabrication, automotive, aerospace] +retail: [retail, ecommerce, commerce, shopping, store, customer_service, merchandising, inventory, sales, point_of_sale, pos, buying, merchandising] +education: [education, school, university, college, academic, training, course, teaching, instructor, professor, curriculum, lms] +consulting: [consulting, consulting, advisory, strategy, management_consulting, business_consulting, it_consulting, technical_consulting, engagement, client] +government: [government, public_sector, federal, state, municipal, local, civic, military, defense, administration] +media: [media, publishing, entertainment, broadcast, television, tv, film, movie, news, journalism, content, creative, design, art, music, studio] +energy: [energy, power, oil, gas, petroleum, renewable, solar, wind, utility, electricity, coal, nuclear, utilities, hydro] +telecom: [telecom, telecommunications, wireless, mobile, carrier, 5g, 4g, network, infrastructure, isp, internet_service] +transportation: [transportation, logistics, freight, shipping, delivery, supply_chain, automotive, airline, aviation, maritime, rail, railroad, vehicle, truck] +insurance: [insurance, underwriting, claims, policy, actuarial, risk_management, reinsurance, adjuster, broker, agent] +software: [software, saas, edtech, platform, cloud, programming, api, framework, library, application, web, mobile, backend, frontend, devops, data, full stack, fullstack, full_stack, database, sql, nosql, python, java, javascript, typescript, ruby, php, golang, rust, cpp, csharp, kotlin, swift, scala, .net, dotnet] +_any: [] +``` + +Note: Each industry keyword appears under exactly ONE industry code. No cross-list duplicates exist. This eliminates first-match ambiguity entirely — a token can only map to one industry. `fintech` is registered ONLY under `finance`, not under `software` or other codes. + +## stopwords + +```yaml +- a +- an +- the +- of +- for +- at +- in +- with +- and +- or +- to +``` + +Note: This list MUST NOT include any seniority keyword (manager, lead, principal, senior, staff, director, vp, cxo, etc.). The rules-loader module asserts at init time that `stopwords ∩ seniority_keywords = ∅` and throws fatally if violated. + +## filler_adjectives + +```yaml +- experienced +- seasoned +- talented +- passionate +- motivated +``` + +## remote_markers + +```yaml +- remote +- hybrid +- on-site +- onsite +- wfh +``` + +## locations + +```yaml +# High-frequency English-speaking locations (bootstrap set for v1) +# Expansion: full LLM-generated list (~500-1000 tokens) lands in a later commit; regeneration bumps NORMALIZER_VERSION. +- london +- manchester +- edinburgh +- glasgow +- dublin +- paris +- berlin +- munich +- example location 013 +- geneva +- amsterdam +- madrid +- barcelona +- milan +- rome +- stockholm +- copenhagen +- oslo +- helsinki +- warsaw +- prague +- vienna +- brussels +- lisbon +- new_york +- san_francisco +- los_angeles +- seattle +- boston +- chicago +- austin +- denver +- toronto +- vancouver +- montreal +- sydney +- melbourne +- tokyo +- singapore +- hong_kong +- bangalore +- mumbai +- dubai +- abu_dhabi +- uk +- usa +- eu +- emea +- apac +- na +``` + +--- + +## NORM-02 Anchor + +**Important:** `solutions architect` IS a compound phrase (compounded to `solutions_architect`), but `architect solutions` is NOT a recognized compound phrase. This distinction is load-bearing: + +- `normalizeTitle('Solutions Architect')` → compound expansion applies → `solutions_architect` +- `normalizeTitle('Architect Solutions')` → no compound phrase matches (compound phrases must match the raw input sequence before tokenization) → after tokenization: `architect solutions` (two separate tokens) + +Token-order preservation in Steps 5-8 of the pipeline ensures these outputs differ. This test is encoded in NORM-02 fixture entries and unit test assertions. diff --git a/skills/salary-calculator/scripts/batch-salary-research.mjs b/skills/salary-calculator/scripts/batch-salary-research.mjs new file mode 100644 index 0000000..f4a1713 --- /dev/null +++ b/skills/salary-calculator/scripts/batch-salary-research.mjs @@ -0,0 +1,163 @@ +#!/usr/bin/env node +// batch-salary-research.mjs — queue/search-id driver for enrich-job-salary.mjs. +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const JH = process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter'); +const DEFAULT_DB = process.env.JOBHUNTER_DB || path.join(JH, 'jobhunter.sqlite'); +const LOGS_DIR = path.join(JH, 'logs'); +const ENRICH = path.join(__dirname, 'enrich-job-salary.mjs'); +const EXTERNAL_SCAN = path.join(__dirname, 'external-salary-scan.mjs'); + +function stamp() { + return new Date().toISOString().replace(/[-:]/g, '').replace(/\..+/, '').replace('T', '-'); +} + +function usage(exitCode = 0) { + const out = exitCode === 0 ? process.stdout : process.stderr; + out.write(`Usage:\n`); + out.write(` batch-salary-research.mjs --search-id [options]\n`); + out.write(` batch-salary-research.mjs --queue [options]\n\n`); + out.write(`Options:\n`); + out.write(` --db SQLite DB, default ${DEFAULT_DB}\n`); + out.write(` --out progress NDJSON, default ~/.job-hunter/logs/batch-salary-research-*.ndjson\n`); + out.write(` --done done marker, default .done\n`); + out.write(` --limit max jobs\n`); + out.write(` --dry-run pass --dry-run through to enrich-job-salary\n`); + out.write(` --help show this help\n`); + process.exit(exitCode); +} + +function parseArgs(argv) { + const opts = { + db: DEFAULT_DB, + searchId: null, + queue: null, + out: path.join(LOGS_DIR, `batch-salary-research-${stamp()}.ndjson`), + done: null, + limit: 0, + dryRun: false, + }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + const next = () => { + if (i + 1 >= argv.length) throw new Error(`${a} requires a value`); + return argv[++i]; + }; + if (a === '--help' || a === '-h') usage(0); + else if (a === '--db') opts.db = next(); + else if (a === '--search-id') opts.searchId = next(); + else if (a === '--queue') opts.queue = next(); + else if (a === '--out') opts.out = next(); + else if (a === '--done') opts.done = next(); + else if (a === '--limit') opts.limit = Number(next()); + else if (a === '--dry-run') opts.dryRun = true; + else throw new Error(`unknown argument: ${a}`); + } + if (!opts.done) opts.done = `${opts.out}.done`; + if (!opts.searchId && opts.queue) { + const q = JSON.parse(fs.readFileSync(opts.queue, 'utf8')); + opts.searchId = q.search_id || q.searchId || null; + opts.queueJobs = Array.isArray(q.jobs) ? q.jobs : (Array.isArray(q) ? q : []); + } + if (!opts.searchId && !opts.queueJobs?.length) throw new Error('provide --search-id or --queue'); + return opts; +} + +function openDb(dbPath) { + const req = createRequire(path.join(JH, 'package.json')); + const Database = req('better-sqlite3'); + return new Database(dbPath, { readonly: true }); +} + +function jobsForQueue(db, opts) { + if (opts.searchId) { + return db.prepare(` + SELECT j.source, j.job_id, j.title, j.company, j.country_code, mr.fit_score, mr.stretch_label + FROM jobs j + JOIN match_results mr ON mr.source = j.source AND mr.job_id = j.job_id + WHERE mr.search_id = ? AND mr.cta = 'Apply' + ORDER BY CASE mr.stretch_label WHEN 'Core fit' THEN 0 WHEN 'Stretch' THEN 1 ELSE 2 END, + mr.fit_score DESC, j.title COLLATE NOCASE + `).all(opts.searchId); + } + const stmt = db.prepare(`SELECT source, job_id, title, company, country_code FROM jobs WHERE source=COALESCE(@source, source) AND job_id=@job_id`); + return (opts.queueJobs || []).map((j) => stmt.get({ source: j.source || null, job_id: String(j.job_id) })).filter(Boolean); +} + +function appendJsonLine(file, obj) { + fs.appendFileSync(file, `${JSON.stringify(obj)}\n`); +} + +function main() { + const opts = parseArgs(process.argv.slice(2)); + fs.mkdirSync(path.dirname(opts.out), { recursive: true }); + fs.writeFileSync(opts.out, ''); + const db = openDb(opts.db); + let jobs = jobsForQueue(db, opts); + db.close(); + if (opts.limit > 0) jobs = jobs.slice(0, opts.limit); + const totalJobs = jobs.length; + console.log(`salary research start jobs=${totalJobs} out=${opts.out} dry_run=${opts.dryRun}`); + let ok = 0; + let failed = 0; + + const externalJobs = jobs.filter((j) => j.source === 'external'); + if (externalJobs.length) { + const queuePath = path.join(path.dirname(opts.out), `batch-salary-external-${Date.now()}.json`); + const externalOut = opts.out.replace(/\.ndjson$/i, '') + '-external-salary-scan.ndjson'; + fs.writeFileSync(queuePath, JSON.stringify({ jobs: externalJobs.map((j) => ({ source: j.source, job_id: j.job_id })) }, null, 2)); + const extArgs = ['node', EXTERNAL_SCAN, '--queue', queuePath, '--db', opts.db, '--out', externalOut, '--limit', String(externalJobs.length)]; + if (opts.dryRun) extArgs.push('--dry-run'); + const p = spawnSync(extArgs[0], extArgs.slice(1), { cwd: JH, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024 }); + appendJsonLine(opts.out, { + at: new Date().toISOString(), + kind: 'external_salary_scan', + jobs: externalJobs.length, + returncode: p.status, + ok: p.status === 0, + out: externalOut, + stdout: p.stdout.slice(-4000), + stderr: p.stderr.slice(-4000), + }); + if (p.status === 0) ok += externalJobs.length; + else failed += externalJobs.length; + jobs = jobs.filter((j) => j.source !== 'external'); + } + + for (let i = 0; i < jobs.length; i += 1) { + const job = jobs[i]; + console.log(`[${i + 1}/${jobs.length}] ${job.source}:${job.job_id} ${job.title} — ${job.company}`); + const args = ['node', ENRICH, '--db', opts.db, '--source', job.source, '--job-id', job.job_id, '--json']; + if (opts.dryRun) args.push('--dry-run'); + const startedAt = new Date().toISOString(); + const p = spawnSync(args[0], args.slice(1), { cwd: JH, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024 }); + let parsed = null; + try { parsed = JSON.parse(p.stdout); } catch {} + const rec = { + at: new Date().toISOString(), + started_at: startedAt, + job, + returncode: p.status, + ok: p.status === 0, + stdout: parsed || p.stdout.slice(-4000), + stderr: p.stderr.slice(-4000), + }; + if (rec.ok) ok += 1; + else failed += 1; + appendJsonLine(opts.out, rec); + } + fs.writeFileSync(opts.done, JSON.stringify({ done_at: new Date().toISOString(), jobs: totalJobs, ok, failed }, null, 2)); + console.log(`salary research done jobs=${totalJobs} ok=${ok} failed=${failed} done=${opts.done}`); + process.exitCode = failed ? 1 : 0; +} + +try { main(); } +catch (err) { + console.error(`[fatal] ${err.stack || err}`); + process.exit(2); +} diff --git a/skills/salary-calculator/scripts/enrich-job-salary.mjs b/skills/salary-calculator/scripts/enrich-job-salary.mjs new file mode 100644 index 0000000..a885d21 --- /dev/null +++ b/skills/salary-calculator/scripts/enrich-job-salary.mjs @@ -0,0 +1,872 @@ +#!/usr/bin/env node +import { Database } from '../../job-hunter/scripts/workspace-dependencies.mjs'; +// Phase v1.0-09 Plan 05 + Phase v1.0-10 Plan 04 — CLI entry point for salary enrichment. +// +// Two modes: +// 1. Single-job mode (v1.0-09): --source X --job-id Y → runs enrichOneJob once. +// 2. Batch mode (v1.0-10): --all-unsalaried [--limit N] → iterates candidates. +// +// Glue layer. Opens the SQLite DB, acquires the writer lock ONCE (fail-fast), +// builds the adapter registry + httpClient, dispatches to runSingle or runBatch, +// releases the lock and closes the DB in a try/finally. +// +// ===================================================================== +// EXIT CODE MATRIX (CLI-10) — MIRRORED FROM Plan 04 deriveExitCode +// ===================================================================== +// The CLI consumes envelope.state.exitCode VERBATIM. The pipeline's +// deriveExitCode function (scripts/lib/enrich-pipeline.mjs) is the +// SINGLE SOURCE OF TRUTH. This comment block exists so reviewers can +// verify rule-table agreement at a grep level. +// +// | Condition | Exit | +// |--------------------------------------------------------------------|------| +// | event.kind === 'unrecoverable_error' (exact OR benchmark) | 5 | +// | event.kind === 'transient_error' AND nextRetryAtIso === null | 6 | +// | selectedBest !== null | 0 | +// | selectedBest === null AND stateExact.status === 'not_found' | 0 | +// | event.kind === 'transient_error' AND nextRetryAtIso !== null | 0 | +// | | | +// | CLI-only exit codes (set BEFORE/AROUND enrichOneJob): | | +// | precondition failure (missing arg, unknown source, DB open fails, | 2 | +// | job row missing, invalid --limit, mutual-exclusion violation) | | +// | lock contention (handle.acquired === false) | 4 | +// | uncaught exception in main() try/catch | 1 | +// | per-job pipeline exception (batch-mode CLI try/catch) | 7 | +// +// Batch exit-code aggregation rules (worst-case, never short-circuits): +// 5 if any per-job envelope.state.exitCode === 5 (unrecoverable from pipeline) +// 7 else if any per-job envelope.state.exitCode === 7 (per-job synthetic envelope: CLI try/catch around enrichOneJob) +// 6 else if any per-job envelope.state.exitCode === 6 (transient exhaustion) +// 0 otherwise (every job either succeeded or scheduled a future retry) +// CLI-level codes that short-circuit BEFORE the aggregator: +// 2 (preflight: argv validation), 4 (lock contention), 1 (CLI-level uncaught exception at top of main) +// Per-job exitCode is consumed VERBATIM from the envelope — no message-string parsing here +// (status routing already done by deriveExitCode in enrich-pipeline.mjs). +// +// CRITICAL: 'unrecoverable' (per locked CONTEXT decision) means +// 401/403/transient-exhaustion ONLY — NOT every failed attempt. +// +// EXPLICITLY FORBIDDEN at CLI layer (opencode improvement #5): +// - String(err) substring tests against HTTP status digits (the literal three-digit forbidden strings) +// - regex tests on err.message for status-code digits +// - Any logic that overrides envelope.state.exitCode based on error message content +// ===================================================================== + +import { parseArgs } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +import { + acquire, + makeIdentity, + formatContentionMessage, + installSignalHandlers, +} from './lib/writer-lock.mjs'; +import { createHttpClient } from './http-client.mjs'; +import { enrichOneJob } from './lib/enrich-pipeline.mjs'; +import { + selectBatchCandidates, + selectBatchCandidatesForceRetry, + countBatchCandidates, + countBatchCandidatesForceRetry, + getCurrent30DayHitRates, + getPrior30DayHitRates, +} from './lib/salary-db.mjs'; +import { forceResetAxis } from './lib/enrichment-state.mjs'; +import { evaluateHealth } from './lib/health-metrics.mjs'; +import * as linkedin from './sources/linkedin.mjs'; +import * as itjobswatch from './sources/itjobswatch.mjs'; +import * as indeed from './sources/indeed.mjs'; +import * as levelsFyi from './sources/levels_fyi.mjs'; +import * as robertHalf from './sources/robert_half.mjs'; +import * as salaryexpert from './sources/salaryexpert.mjs'; + +const ADAPTERS = { + linkedin, + itjobswatch, + indeed, + levels_fyi: levelsFyi, + robert_half: robertHalf, + salaryexpert, +}; + +/** + * Build the options object passed to createHttpClient. Exported so tests can + * import the production source-of-truth for per-host limits wiring without + * spawning the CLI. + * + * Host keys must be the literal URL host strings (URL(url).host returns the + * full host including the www. prefix). Adapter `.limits` objects supply + * per-host overrides; unlisted hosts inherit the top-level perHostRps default + * via the merge semantics in http-client.mjs createHttpClient(). + * + * @returns {object} Options ready for createHttpClient() + */ +export function buildHttpClientOptions() { + return { + perHostRps: 0.5, + maxConcurrent: 1, + maxRetries: 3, + limits: { + 'www.linkedin.com': linkedin.limits, + 'www.itjobswatch.co.uk': itjobswatch.limits, + 'ie.indeed.com': indeed.limits, + 'uk.indeed.com': indeed.limits, + 'www.indeed.com': indeed.limits, + 'ca.indeed.com': indeed.limits, + 'au.indeed.com': indeed.limits, + 'ch.indeed.com': indeed.limits, + 'ae.indeed.com': indeed.limits, + 'de.indeed.com': indeed.limits, + 'fr.indeed.com': indeed.limits, + 'nl.indeed.com': indeed.limits, + 'es.indeed.com': indeed.limits, + 'it.indeed.com': indeed.limits, + 'www.levels.fyi': levelsFyi.limits, + 'www.roberthalf.com': robertHalf.limits, + 'www.salaryexpert.com': salaryexpert.limits, + }, + }; +} + +// Canonical source list for KNOWN_SOURCES backfill in the --health report. +// SQL aggregates GROUP BY job_source so zero-observation sources are absent +// from raw SQL output; the CLI layer backfills with no_data rows so the +// operator always sees a row per supported adapter (HEALTH-03 contract). +const KNOWN_SOURCES = ['linkedin', 'itjobswatch', 'indeed', 'levels_fyi', 'robert_half', 'salaryexpert']; + +// ===================================================================== +// FAKE BENCHMARK ADAPTER (test-only) +// ===================================================================== +// When ENRICH_FAKE_BENCHMARK is set, the CLI overlays a stub benchmark +// adapter onto the registry that returns a canned benchmark payload +// WITHOUT a network call. Used by CLI-03 / CLI-06 integration tests so +// the salary_benchmarks cache can be observed end-to-end without +// depending on the live ITJobsWatch endpoint. Production runs leave it +// unset and use the real adapter. +function buildFakeBenchmarkOverlay(baseAdapters) { + if (!process.env.ENRICH_FAKE_BENCHMARK) return baseAdapters; + const fakeItjobswatch = { + ...itjobswatch, + supports: { exactSalary: false, benchmark: true }, + async fetchBenchmark(query, _ctx) { + return { + normalizedTitle: query.normalizedTitle || query.title || 'software_engineer', + seniority: query.seniority || 'unknown', + industry: query.industry || 'unknown', + countryCode: query.countryCode || 'GB', + region: query.region || '', + city: query.city || '', + currency: 'GBP', + period: 'year', + compensationType: 'base_salary', + dataSource: 'itjobswatch', + dataSourceUrl: 'https://fake.test/benchmark', + rawTitle: query.title || 'Software Engineer', + amountMin: 50000, + amountMax: 80000, + amountMedian: 65000, + amountP10: 45000, + amountP90: 90000, + sampleSize: 100, + confidenceScore: null, + rawPayloadJson: JSON.stringify({ fake: true, query }), + fetchedAt: new Date().toISOString(), + normalizerVersion: 'fake-1', + }; + }, + }; + return { ...baseAdapters, itjobswatch: fakeItjobswatch }; +} +// ===================================================================== + +const USAGE = `Usage: + Single-job mode: + node scripts/enrich-job-salary.mjs --db --source --job-id [flags] + Batch mode: + node scripts/enrich-job-salary.mjs --db --all-unsalaried [--limit N] [flags] + +Required (one mode): + --db Path to jobhunter.sqlite + --source Adapter name (linkedin, itjobswatch, indeed, levels_fyi, robert_half, salaryexpert) — single-job mode + --job-id jobs.job_id value — single-job mode + --all-unsalaried Batch mode: iterate unsalaried candidates + +Operational flags (Phase v1.0-10): + --limit N Batch size (default 50, max 100); ignored in single-job mode + --benchmark-only Skip exact pass entirely; only fetch/refresh benchmark + --refresh-benchmarks Force benchmark cache refresh (bypass isBenchmarkStale) + --force-exact-retry Reset exact axis to pending per-job before run + --force-benchmark-retry Reset benchmark axis to pending per-job + force refresh + +Other: + --dry-run Acquire lock + fetch + parse + select but skip DB writes + --verbose Multi-line text output (single-job mode) + --json Emit JSON envelope(s) on stdout; NDJSON in batch mode + --help Show this help and exit +`; + +function buildErrorEnvelope({ exitCode, error, dryRun = false }) { + return { + job: null, + observations: [], + selectedBest: null, + benchmarkUsed: null, + state: { + exact: null, + benchmark: null, + dryRun: !!dryRun, + exitCode, + error, + }, + }; +} + +function retryClause(nextRetryAtIso) { + return nextRetryAtIso + ? `next retry ${String(nextRetryAtIso).slice(0, 10)}` + : 'no further retries (manual)'; +} + +function formatRange(obs) { + if (!obs) return ''; + const cur = obs.currency ?? ''; + if (obs.amount_min != null && obs.amount_max != null) { + return `${cur} ${obs.amount_min}–${obs.amount_max}`.trim(); + } + if (obs.amount_single != null) { + return `${cur} ${obs.amount_single}`.trim(); + } + if (obs.amount_median != null) { + return `${cur} ~${obs.amount_median}`.trim(); + } + if (obs.annualized_value != null) { + return `${cur} ~${obs.annualized_value}`.trim(); + } + return cur; +} + +function renderSingleLine(envelope) { + const job = envelope.job ?? {}; + const source = job.source ?? '?'; + const jobId = job.job_id ?? '?'; + const sb = envelope.selectedBest; + + if (sb === null || sb === undefined) { + const exact = envelope.state?.exact; + if (exact?.status === 'not_found') { + return `No salary found (${source} ${jobId}) — ${retryClause(exact.next_retry_at)}`; + } + if (exact?.status === 'error') { + return `Error fetching ${source} ${jobId}: ${exact.error || 'unknown'} — ${retryClause(exact.next_retry_at)}`; + } + return `No salary found (${source} ${jobId})`; + } + + if (sb.is_posted_salary === 1) { + return `Posted salary ${formatRange(sb)} (${source} ${jobId})`; + } + const benchmark = envelope.benchmarkUsed || sb.benchmark_id || 'unknown'; + return `Best estimate ${formatRange(sb)} (${source} ${jobId} benchmark=${benchmark})`; +} + +function renderVerbose(envelope) { + const lines = []; + const job = envelope.job ?? {}; + lines.push(`Job: ${job.source ?? '?'} ${job.job_id ?? '?'} — ${job.title ?? ''} @ ${job.company ?? ''}`); + if (job.url) lines.push(`URL: ${job.url}`); + + const exact = envelope.state?.exact; + if (exact) { + lines.push(`Exact: status=${exact.status} attempts=${exact.attempt_count} ${retryClause(exact.next_retry_at)}`); + if (exact.error) lines.push(` error: ${exact.error}`); + } + const benchmark = envelope.state?.benchmark; + if (benchmark) { + lines.push(`Benchmark: status=${benchmark.status} attempts=${benchmark.attempt_count} ${retryClause(benchmark.next_retry_at)}`); + if (benchmark.error) lines.push(` error: ${benchmark.error}`); + } + + const sb = envelope.selectedBest; + if (sb === null || sb === undefined) { + if (exact?.status === 'not_found') { + lines.push(`Selected: No salary found`); + } else if (exact?.status === 'error') { + lines.push(`Selected: Error fetching: ${exact.error || 'unknown'}`); + } else { + lines.push(`Selected: No salary found`); + } + } else if (sb.is_posted_salary === 1) { + lines.push(`Selected: Posted salary ${formatRange(sb)}`); + } else { + const bm = envelope.benchmarkUsed || sb.benchmark_id || 'unknown'; + lines.push(`Selected: Best estimate ${formatRange(sb)} benchmark=${bm}`); + } + + lines.push(`Observations: ${envelope.observations?.length ?? 0}`); + lines.push(`Dry-run: ${envelope.state?.dryRun ? 'yes' : 'no'}`); + lines.push(`Exit code: ${envelope.state?.exitCode ?? 0}`); + return lines.join('\n'); +} + +// ===================================================================== +// FAKE OUTCOMES INJECTION (test-only) +// ===================================================================== +// When ENRICH_FAKE_OUTCOMES is set (csv list, one outcome per candidate), +// the CLI synthesizes per-job envelopes WITHOUT calling enrichOneJob — +// used by integration tests to deterministically exercise exit-code +// aggregation paths (5/6/7 priorities). Production runs leave it unset. +// +// Supported outcomes: success, not_found, unrecoverable, transient_exhausted, +// transient_scheduled, pipeline_throw +function getFakeOutcomes() { + const raw = process.env.ENRICH_FAKE_OUTCOMES; + if (!raw) return null; + return raw.split(',').map((s) => s.trim()).filter(Boolean); +} + +function fakeEnvelopeFor(outcome, candidate) { + const job = { + source: candidate.source, + job_id: candidate.job_id, + title: candidate.title, + company: candidate.company, + }; + const base = { + job, + observations: [], + selectedBest: null, + benchmarkUsed: null, + state: { + exact: { status: 'pending', attempt_count: 0, next_retry_at: null, error: null, last_attempt_at: null }, + benchmark: null, + dryRun: false, + exitCode: 0, + }, + }; + switch (outcome) { + case 'success': + base.selectedBest = { is_posted_salary: 1, currency: 'GBP', amount_median: 50000 }; + base.state.exact.status = 'found'; + base.state.exitCode = 0; + return base; + case 'not_found': + base.state.exact.status = 'not_found'; + base.state.exitCode = 0; + return base; + case 'unrecoverable': + base.state.exact.status = 'error'; + base.state.exact.error = 'unrecoverable (fake)'; + base.state.exitCode = 5; + return base; + case 'transient_exhausted': + base.state.exact.status = 'error'; + base.state.exact.error = 'transient exhausted (fake)'; + base.state.exitCode = 6; + return base; + case 'transient_scheduled': + base.state.exact.status = 'error'; + base.state.exact.error = 'transient scheduled (fake)'; + base.state.exact.next_retry_at = '2099-01-01T00:00:00.000Z'; + base.state.exitCode = 0; + return base; + case 'pipeline_throw': + // Simulate the CLI try/catch around enrichOneJob synthesising a 7-envelope. + throw new Error('pipeline throw (fake)'); + default: + base.state.exact.status = 'error'; + base.state.exact.error = `unknown fake outcome '${outcome}'`; + base.state.exitCode = 5; + return base; + } +} +// ===================================================================== + +// ===================================================================== +// HEALTH REPORT (Phase v1.0-11 — HEALTH-02 / HEALTH-03) +// ===================================================================== +// Read-only parser hit-rate report per job_source over the trailing 30 +// complete days (today excluded). Read-only path — dispatched BEFORE +// writer-lock acquisition. Two concurrent --health invocations both +// succeed (proof: cli-health-flag.test.mjs subtest 4 via async spawn + +// Promise.all). +// +// Locked 4-state taxonomy: ok / warning / no_data / insufficient_baseline. +// No degradation-state literal anywhere in the envelope vocabulary. + +function formatPct(rate) { + if (rate === null || rate === undefined) return '-'; + return `${(rate * 100).toFixed(1)}%`; +} + +function formatDeltaPct(delta) { + if (delta === null || delta === undefined) return '-'; + const sign = delta >= 0 ? '+' : ''; + return `${sign}${delta.toFixed(1)}%`; +} + +function renderHealthText(envelope) { + const lines = []; + lines.push('Source Obs(30d) Current Prior Delta State'); + lines.push('------------- -------- ------- ------ ------ ----------------------'); + for (const row of envelope.sources) { + const src = String(row.source).padEnd(13, ' '); + const obs = String(row.observations_last_30d).padStart(8, ' '); + const cur = formatPct(row.hit_rate_current).padStart(7, ' '); + const prior = formatPct(row.hit_rate_prior).padStart(6, ' '); + const delta = formatDeltaPct(row.delta_pct).padStart(6, ' '); + const stateSuffix = row.warning ? ' ⚠' : ''; + lines.push(`${src} ${obs} ${cur} ${prior} ${delta} ${row.state}${stateSuffix}`); + } + lines.push(`Generated: ${envelope.generated_at}`); + return lines.join('\n'); +} + +async function runHealth({ db, json }) { + const currentRows = getCurrent30DayHitRates({ db }); + const priorRows = getPrior30DayHitRates({ db }); + + const currentBySource = new Map(currentRows.map((r) => [r.job_source, r])); + const priorBySource = new Map(priorRows.map((r) => [r.job_source, r])); + + // KNOWN_SOURCES backfill: every supported adapter gets a row even when + // the raw SQL output is empty for that source. Deterministic sort. + const sortedSources = [...KNOWN_SOURCES].sort(); + const rows = sortedSources.map((src) => { + const current = currentBySource.get(src) || { job_source: src, total: 0, hits: 0 }; + const prior = priorBySource.get(src) || { job_source: src, total: 0, hits: 0 }; + const health = evaluateHealth({ + current: { total: current.total, hits: current.hits }, + prior: { total: prior.total, hits: prior.hits }, + }); + return { + source: src, + observations_last_30d: current.total, + hit_rate_current: health.hit_rate_current, + hit_rate_prior: health.hit_rate_prior, + delta_pct: health.delta_pct, + warning: health.warning, + state: health.state, + }; + }); + + const envelope = { + type: 'health', + generated_at: new Date().toISOString(), + sources: rows, + }; + + if (json) { + process.stdout.write(`${JSON.stringify(envelope)}\n`); + } else { + process.stdout.write(`${renderHealthText(envelope)}\n`); + } + // Locked CONTEXT: --health always exits 0; warning state surfaces via + // the warning flag + state literal in the envelope, not the exit code. + return 0; +} +// ===================================================================== + +function parseAndValidateArgs() { + let values; + try { + ({ values } = parseArgs({ + options: { + db: { type: 'string' }, + source: { type: 'string' }, + 'job-id': { type: 'string' }, + 'dry-run': { type: 'boolean', default: false }, + verbose: { type: 'boolean', default: false }, + json: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false }, + // v1.0-10 batch + operational flags + 'all-unsalaried': { type: 'boolean', default: false }, + limit: { type: 'string', default: '50' }, + 'benchmark-only': { type: 'boolean', default: false }, + 'refresh-benchmarks': { type: 'boolean', default: false }, + 'force-exact-retry': { type: 'boolean', default: false }, + 'force-benchmark-retry': { type: 'boolean', default: false }, + // v1.0-11 health-metrics flag. Read-only report path; does NOT + // acquire the writer-lock. NOTE: a `--days N` override was + // explicitly deferred (locked CONTEXT discretion) — the SQL + // queries are hardcoded 30/60-day literals. Registering `days` + // here under strict:true would silently accept the flag without + // effect AND mask unknown-flag typo errors. Keep this block + // minimal: only `health` is added in this phase. + health: { type: 'boolean', default: false }, + }, + strict: true, + })); + } catch (err) { + return { error: err.message, values: null }; + } + return { error: null, values }; +} + +function preconditionFail(msg, values) { + process.stderr.write(`${msg}\n`); + if (values?.json) { + process.stdout.write( + JSON.stringify(buildErrorEnvelope({ exitCode: 2, error: msg, dryRun: values['dry-run'] })) + '\n', + ); + } + process.exit(2); +} + +async function runSingle({ db, values, adapters, httpClient, handle }) { + const source = values.source; + const jobId = values['job-id']; + + // Operational flags: per-axis pre-run reset (single-job variant — mirrors batch path). + if (values['force-exact-retry']) forceResetAxis(db, source, jobId, 'exact'); + if (values['force-benchmark-retry']) forceResetAxis(db, source, jobId, 'benchmark'); + + const skipExact = values['benchmark-only']; + // IMPROVEMENT 1: include --force-benchmark-retry in the forceBenchmarkRefresh derivation + // so forceResetAxis('benchmark') isn't a no-op for fresh cached entries. + const forceBenchmarkRefresh = values['benchmark-only'] || values['refresh-benchmarks'] || values['force-benchmark-retry']; + const progress = (msg) => process.stderr.write(`${msg}\n`); + + const envelope = await enrichOneJob({ + db, + source, + jobId, + adapterRegistry: adapters, + dryRun: values['dry-run'], + nowIso: new Date().toISOString(), + httpClient, + enableBenchmark: true, + skipExact, + forceBenchmarkRefresh, + progress, + }); + + if (values.json) { + process.stdout.write(JSON.stringify(envelope) + '\n'); + } else { + const text = values.verbose ? renderVerbose(envelope) : renderSingleLine(envelope); + process.stdout.write(`${text}\n`); + } + + return envelope.state?.exitCode ?? 0; +} + +async function runBatch({ db, values, adapters, httpClient }) { + const limit = Number.parseInt(values.limit, 10); + const forceExactRetry = values['force-exact-retry']; + const forceBenchmarkRetry = values['force-benchmark-retry']; + const skipExact = values['benchmark-only']; + // IMPROVEMENT 1: include --force-benchmark-retry. Mirrors runSingle. + const forceBenchmarkRefresh = values['benchmark-only'] || values['refresh-benchmarks'] || values['force-benchmark-retry']; + + // Step 2: pick query + capture pre-batch total for more_eligible math. + // Snapshotting the count BEFORE the batch is necessary because --dry-run does not + // mutate retry state, so a post-batch count would equal the pre-batch count and + // give the wrong more_eligible (60 instead of 10 for a 60-jobs / limit-50 dry run). + const totalEligibleBefore = forceExactRetry + ? countBatchCandidatesForceRetry({ db }) + : countBatchCandidates({ db }); + const candidates = forceExactRetry + ? selectBatchCandidatesForceRetry({ db, limit }) + : selectBatchCandidates({ db, limit }); + + // Step 5: per-job loop + const results = []; + const fakeOutcomes = getFakeOutcomes(); // test-only synthesis path + + for (let i = 0; i < candidates.length; i++) { + const c = candidates[i]; + + if (forceExactRetry) forceResetAxis(db, c.source, c.job_id, 'exact'); + if (forceBenchmarkRetry) forceResetAxis(db, c.source, c.job_id, 'benchmark'); + + let envelope; + try { + if (fakeOutcomes) { + const outcome = fakeOutcomes[i] ?? 'success'; + envelope = fakeEnvelopeFor(outcome, c); + } else { + envelope = await enrichOneJob({ + db, + source: c.source, + jobId: c.job_id, + adapterRegistry: adapters, + dryRun: values['dry-run'], + nowIso: new Date().toISOString(), + httpClient, + enableBenchmark: true, + skipExact, + forceBenchmarkRefresh, + progress: (msg) => process.stderr.write(` ${msg}\n`), + }); + } + } catch (err) { + // IMPROVEMENT 4: per-job CLI-level try/catch around enrichOneJob. + // Use a DEDICATED exit code 7 (NOT 1) to distinguish this synthesised + // failure from a CLI-level uncaught exception (which would never reach + // this aggregator — it terminates the process at the top level). + envelope = buildErrorEnvelope({ exitCode: 7, error: err.message, dryRun: values['dry-run'] }); + envelope.job = { source: c.source, job_id: c.job_id }; + } + + results.push(envelope); + + // Per-job emission + if (values.json) { + process.stdout.write(JSON.stringify(envelope) + '\n'); + } + // stderr post-line with terminal status + const status = envelope.state?.exact?.status + ?? (envelope.state?.exitCode === 5 ? 'unrecoverable' + : envelope.state?.exitCode === 7 ? 'pipeline_exception' + : '?'); + process.stderr.write(`[${i + 1}/${candidates.length}] ${c.source}/${c.job_id} ${status}\n`); + } + + // Step 6: aggregate exit code (worst-case, NEVER short-circuits). + // IMPROVEMENT 4: priority 5 > 7 > 6 > 0. + const anyUnrecoverable = results.some(e => e.state?.exitCode === 5); + const anyPipelineException = results.some(e => e.state?.exitCode === 7); + const anyTransientExhaust = results.some(e => e.state?.exitCode === 6); + const batchExitCode = anyUnrecoverable ? 5 + : anyPipelineException ? 7 + : anyTransientExhaust ? 6 + : 0; + + // Step 7: more_eligible — `totalEligibleBefore - results.length` (RESEARCH `count - selected`). + // IMPROVEMENT 2: BATCH-01 vs BATCH-03 semantics. + // BATCH-01 (default --all-unsalaried): processed rows advance state and drop out of + // the eligible set, so this count equals "what the NEXT run would pick up". + // BATCH-03 (--force-exact-retry): per-job pre-run reset puts each row back to 'pending'. + // A transient/recoverable job that fails THIS run is reset to pending again on the + // next run, so it remains in the count. Documented divergence from `count - selected`. + // Implementation uses PRE-batch count minus selected to correctly handle --dry-run + // (where state isn't mutated and a post-batch count would equal the pre-batch count). + // This matches the contract from cli-batch-mode "60 jobs / limit 50 / dry-run → 10". + const moreEligible = Math.max(0, totalEligibleBefore - results.length); + + // Step 8: summary + const summary = { + total: results.length, + found: results.filter(e => e.state?.exact?.status === 'found').length, + not_found: results.filter(e => e.state?.exact?.status === 'not_found').length, + error: results.filter(e => e.state?.exact?.status === 'error' && e.state?.exitCode !== 5 && e.state?.exitCode !== 6 && e.state?.exitCode !== 7).length, + unrecoverable: results.filter(e => e.state?.exitCode === 5).length, + transient_exhausted: results.filter(e => e.state?.exitCode === 6).length, + pipeline_exception: results.filter(e => e.state?.exitCode === 7).length, + more_eligible: moreEligible, + }; + + if (values.json) { + process.stdout.write(JSON.stringify({ summary }) + '\n'); + } else { + process.stderr.write( + `Summary: total=${summary.total} found=${summary.found} not_found=${summary.not_found} ` + + `error=${summary.error} unrecoverable=${summary.unrecoverable} ` + + `transient_exhausted=${summary.transient_exhausted} pipeline_exception=${summary.pipeline_exception} ` + + `more_eligible=${summary.more_eligible}\n`, + ); + } + + return batchExitCode; +} + +async function main() { + // 1. parseArgs + const parsed = parseAndValidateArgs(); + if (parsed.error) { + process.stderr.write(`Error: ${parsed.error}\n`); + process.stderr.write(USAGE); + process.exit(2); + } + const values = parsed.values; + + if (values.help) { + process.stdout.write(USAGE); + process.exit(0); + } + + // ==== Preflight validation (exit 2 — BEFORE writer-lock acquisition) ==== + + if (!values.db) { + const jh = process.env.JOBHUNTER_HOME || `${process.env.HOME}/.job-hunter`; + values.db = process.env.JOBHUNTER_DB || `${jh}/jobhunter.sqlite`; + } + + // --health early dispatch — BEFORE the single/batch precondition gates and + // BEFORE the writer-lock acquire. Read-only path. The early process.exit + // pattern STRUCTURALLY guarantees no lock acquisition for --health. + if (values.health) { + if (values.source || values['job-id'] || values['all-unsalaried']) { + const msg = 'Error: --health is mutually exclusive with --source/--job-id/--all-unsalaried'; + process.stderr.write(`${msg}\n`); + if (values.json) { + process.stdout.write(`${JSON.stringify({ error: msg })}\n`); + } + process.exit(2); + } + // Use the existing `Database` import from 'better-sqlite3' (line 55) per + // the precedent at line 583 (writer-lock path). No wrapper helper + // exists in scripts/lib/salary-db.mjs — do not invent one. + let dbForHealth; + try { + dbForHealth = new Database(values.db); + dbForHealth.pragma('foreign_keys = ON'); + } catch (err) { + const msg = `Error: cannot open database ${values.db}: ${err.message}`; + process.stderr.write(`${msg}\n`); + if (values.json) { + process.stdout.write(`${JSON.stringify({ error: msg })}\n`); + } + process.exit(2); + } + try { + const exit = await runHealth({ db: dbForHealth, json: values.json }); + process.exit(exit); + } finally { + try { dbForHealth.close(); } catch { /* best-effort */ } + } + } + + const batchMode = values['all-unsalaried']; + const singleSpec = !!(values.source || values['job-id']); + + // Mode mutual exclusivity + if (batchMode && singleSpec) { + preconditionFail('Error: --all-unsalaried is mutually exclusive with --source/--job-id', values); + } + if (!batchMode && !singleSpec) { + preconditionFail('Error: specify either --all-unsalaried or both --source and --job-id', values); + } + if (!batchMode) { + if (!values.source) preconditionFail('Error: --source is required', values); + if (!values['job-id']) preconditionFail('Error: --job-id is required', values); + if (!ADAPTERS[values.source]) { + preconditionFail(`Error: unknown source: ${values.source} (known: linkedin, itjobswatch)`, values); + } + } + + // --limit numeric validation (Pitfall 7): always validated regardless of mode; + // silently no-op in single-job mode rather than reject — explicit-vs-default detection + // isn't reliable across --limit 50 and --limit=50 forms. + const limitInt = Number.parseInt(values.limit, 10); + if ( + !Number.isInteger(limitInt) || + String(limitInt) !== String(values.limit) || + limitInt < 1 || + limitInt > 100 + ) { + preconditionFail( + `Error: --limit must be a positive integer between 1 and 100; got '${values.limit}'`, + values, + ); + } + + // Flag combination contradictions + if (values['benchmark-only'] && values['force-exact-retry']) { + preconditionFail( + 'Error: --benchmark-only cannot be combined with --force-exact-retry (skipping exact while forcing exact retry is incoherent)', + values, + ); + } + + // 2. open DB + let db; + try { + db = new Database(values.db); + db.pragma('foreign_keys = ON'); + } catch (err) { + const msg = `Error: cannot open database ${values.db}: ${err.message}`; + process.stderr.write(`${msg}\n`); + if (values.json) { + process.stdout.write( + JSON.stringify(buildErrorEnvelope({ exitCode: 2, error: err.message, dryRun: values['dry-run'] })) + '\n', + ); + } + process.exit(2); + } + + // Single-job mode: verify the job row exists BEFORE acquiring the lock + // (precondition exit 2 takes precedence over exit 4 contention). + if (!batchMode) { + let jobExists; + try { + jobExists = db + .prepare('SELECT 1 FROM jobs WHERE source = ? AND job_id = ?') + .get(values.source, values['job-id']); + } catch (err) { + const msg = `Error: cannot query jobs table: ${err.message}`; + process.stderr.write(`${msg}\n`); + if (values.json) { + process.stdout.write( + JSON.stringify(buildErrorEnvelope({ exitCode: 2, error: err.message, dryRun: values['dry-run'] })) + '\n', + ); + } + try { db.close(); } catch { /* best-effort */ } + process.exit(2); + } + if (!jobExists) { + const msg = `Error: job not found: source=${values.source} job_id=${values['job-id']}`; + process.stderr.write(`${msg}\n`); + if (values.json) { + process.stdout.write( + JSON.stringify(buildErrorEnvelope({ exitCode: 2, error: 'job not found', dryRun: values['dry-run'] })) + '\n', + ); + } + try { db.close(); } catch { /* best-effort */ } + process.exit(2); + } + } + + // 3. acquire lock ONCE (fail-fast — no polling, no --wait). Held across the + // entire batch in batch mode; released in try/finally below. + const identity = makeIdentity(); + const handle = acquire(db, identity); + if (!handle.acquired) { + const msg = formatContentionMessage(handle.holder); + process.stderr.write(`${msg}\n`); + if (values.json) { + process.stdout.write( + JSON.stringify(buildErrorEnvelope({ exitCode: 4, error: msg, dryRun: values['dry-run'] })) + '\n', + ); + } + try { db.close(); } catch { /* best-effort */ } + process.exit(4); + } + const uninstall = installSignalHandlers(handle); + handle.startHeartbeat(); + + let exitCode = 0; + try { + // 4. build httpClient + const httpClient = createHttpClient(buildHttpClientOptions()); + + // 5. dispatch — branch on --all-unsalaried AFTER lock acquisition + const adapters = buildFakeBenchmarkOverlay(ADAPTERS); + if (batchMode) { + exitCode = await runBatch({ db, values, adapters, httpClient }); + } else { + exitCode = await runSingle({ db, values, adapters, httpClient, handle }); + } + } catch (err) { + process.stderr.write(`Unexpected error: ${err.stack || err.message}\n`); + if (values.json) { + process.stdout.write( + JSON.stringify(buildErrorEnvelope({ exitCode: 1, error: err.message, dryRun: values['dry-run'] })) + '\n', + ); + } + exitCode = 1; + } finally { + try { handle.release(); } catch { /* best-effort */ } + try { uninstall(); } catch { /* best-effort */ } + try { db.close(); } catch { /* best-effort */ } + } + + process.exit(exitCode); +} + +// Run main() only when this file is executed as a CLI script, not when imported. +// This lets tests import buildHttpClientOptions() (and other helpers) without +// spawning the CLI, which would open SQLite, parse argv, install signal handlers, etc. +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/skills/salary-calculator/scripts/ensure-salary-schema.mjs b/skills/salary-calculator/scripts/ensure-salary-schema.mjs new file mode 100644 index 0000000..39847c5 --- /dev/null +++ b/skills/salary-calculator/scripts/ensure-salary-schema.mjs @@ -0,0 +1,178 @@ +#!/usr/bin/env node +import { Database } from '../../job-hunter/scripts/workspace-dependencies.mjs'; +// CLI orchestrator for the salary-schema installer. +// Composes preflight + writer-lock + drift + DDL constants in the locked order. +// Translates Errors → categorized exit codes: +// 0 ok (incl. silent no-op re-run) +// 2 preflight failure (file missing, FK off, jobs uniqueness missing) +// 3 schema drift (pre- OR post-DDL) +// 4 lock contention or lost heartbeat +// 1 other / unexpected +import { parseArgs } from 'node:util'; +import { resolve } from 'node:path'; + +import { + assertDbFileExists, assertForeignKeysOn, assertJobsUnique +} from './lib/preflight.mjs'; +import { + makeIdentity, acquire, installSignalHandlers, formatContentionMessage +} from './lib/writer-lock.mjs'; +import { ALL_DDL, COUNTS } from './lib/salary-schema.mjs'; +import { detectDrift } from './lib/drift.mjs'; +import { ensureNormalizerVersionColumn } from './lib/normalize/benchmark-stamp.mjs'; + +function main() { + const { values } = parseArgs({ + options: { + db: { type: 'string' }, + 'dry-run': { type: 'boolean', default: false }, + verbose: { type: 'boolean', default: false }, + json: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false }, + }, + strict: true, + }); + + const verbose = !!values.verbose; + function vlog(...args) { + if (verbose) process.stderr.write(`[ensure-salary-schema] ${args.join(' ')}\n`); + } + + if (values.help) { + process.stdout.write( + `Usage: ensure-salary-schema.mjs --db [--dry-run] [--verbose] [--json]\n` + + `\n` + + `Install the salary schema (4 tables, 8 indexes, 1 trigger) idempotently into\n` + + `an existing SQLite database. Acquires the writer lock before DDL.\n` + + `\n` + + `Exit codes: 0 ok | 2 preflight | 3 drift | 4 lock | 1 other\n` + ); + return 0; + } + if (!values.db) { + const jh = process.env.JOBHUNTER_HOME || `${process.env.HOME}/.job-hunter`; + values.db = process.env.JOBHUNTER_DB || `${jh}/jobhunter.sqlite`; + } + const absPath = resolve(process.cwd(), values.db); + + vlog('preflight: start', `db=${absPath}`); + assertDbFileExists(absPath); + vlog('preflight: db file exists'); + + // Open. Do NOT touch journal_mode (persistent DB-header property; lock semantics + // work in any journal mode). Do NOT touch foreign_keys (SCHEMA-03 verifiability). + const db = new Database(absPath); + + assertForeignKeysOn(db); + vlog('preflight: foreign_keys is ON'); + + assertJobsUnique(db); + vlog('preflight: jobs(source, job_id) uniqueness OK'); + + // Pre-DDL drift check: only flag present-but-different. Missing objects are + // expected on a fresh install (DDL will create them). + const preDdlDrifts = detectDrift(db).filter(d => d.actual !== null); + if (preDdlDrifts.length > 0) { + process.stderr.write(`Schema drift detected before install:\n`); + for (const d of preDdlDrifts) { + process.stderr.write(` [${d.type}] ${d.name}\n expected: ${d.expected}\n actual: ${d.actual}\n`); + } + db.close(); + return 3; + } + vlog('preflight: pre-DDL drift check passed'); + + // Acquire writer lock + const handle = acquire(db, makeIdentity()); + if (!handle.acquired) { + const msg = formatContentionMessage(handle.holder); + process.stdout.write(msg + '\n'); + if (handle.holder) { + process.stderr.write(`holder: ${handle.holder.hostname}:${handle.holder.pid}\n`); + } + db.close(); + return 4; + } + vlog('lock: acquired', handle.reclaimed ? '(reclaimed stale lock)' : ''); + + const uninstall = installSignalHandlers(handle); + handle.startHeartbeat(); + vlog('lock: heartbeat started'); + + try { + if (values['dry-run']) { + for (const sql of ALL_DDL) process.stdout.write(sql + ';\n'); + vlog('dry-run: printed', String(ALL_DDL.length), 'DDL statements'); + handle.release(); + uninstall(); + db.close(); + return 0; + } + + // Exec DDL OUTSIDE any explicit transaction. Each IF NOT EXISTS DDL auto-commits. + for (const sql of ALL_DDL) { + vlog('exec:', sql.split('\n')[0].slice(0, 80)); + db.exec(sql); + } + vlog('ddl: exec complete'); + + // Ensure normalizer_version column exists (migrates existing v1.0-01 DBs) + const stampMigration = ensureNormalizerVersionColumn(db); + if (stampMigration.added) { + vlog('migration: added normalizer_version column to salary_benchmarks'); + } else { + vlog('migration: normalizer_version column already present'); + } + + // Post-DDL drift check (defense in depth — catches DDL bugs) + const diffs = detectDrift(db); + if (diffs.length > 0) { + process.stderr.write(`Schema drift detected after install (DDL bug?):\n`); + for (const d of diffs) { + process.stderr.write(` [${d.type}] ${d.name}\n expected: ${d.expected}\n actual: ${d.actual ?? ''}\n`); + } + handle.release(); + uninstall(); + db.close(); + return 3; + } + vlog('drift: post-DDL check passed (', String(diffs.length), 'diffs)'); + + const triggerWord = COUNTS.triggers === 1 ? 'trigger' : 'triggers'; + if (values.json) { + process.stdout.write(JSON.stringify({ + status: 'ok', + tables_created: COUNTS.tables, + indexes_created: COUNTS.indexes, + errors: [], + }) + '\n'); + } else { + process.stdout.write( + `Schema OK at ${absPath} (${COUNTS.tables} tables, ${COUNTS.indexes} indexes, ${COUNTS.triggers} ${triggerWord}).\n` + ); + } + + handle.release(); + uninstall(); + vlog('lock: released'); + db.close(); + return 0; + } catch (e) { + try { handle.release(); } catch {} + try { uninstall(); } catch {} + try { db.close(); } catch {} + throw e; + } +} + +try { + process.exit(main()); +} catch (e) { + if (e && typeof e.exitCode === 'number') { + process.stderr.write(`${e.message}\n`); + process.exit(e.exitCode); + } + process.stderr.write(`error: ${e.message ?? e}\n`); + if (process.env.VERBOSE_STACK) process.stderr.write((e.stack || '') + '\n'); + process.exit(1); +} diff --git a/skills/salary-calculator/scripts/external-salary-scan.mjs b/skills/salary-calculator/scripts/external-salary-scan.mjs new file mode 100644 index 0000000..2498dac --- /dev/null +++ b/skills/salary-calculator/scripts/external-salary-scan.mjs @@ -0,0 +1,451 @@ +#!/usr/bin/env node +// external-salary-scan.mjs — read-only advertiser/ATS salary scan for saved jobs. +import http from 'node:http'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { insertObservation } from './lib/salary-db.mjs'; +import { applyRetryTransition, getEnrichmentState } from './lib/enrichment-state.mjs'; +import { computeNextRetry } from './lib/retry-state-machine.mjs'; +import { acquire, formatContentionMessage, installSignalHandlers } from './lib/writer-lock.mjs'; + +const JH = process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter'); +const DEFAULT_DB = process.env.JOBHUNTER_DB || path.join(JH, 'jobhunter.sqlite'); +const LOGS_DIR = path.join(JH, 'logs'); + +function stamp() { + return new Date().toISOString().replace(/[-:]/g, '').replace(/\..+/, '').replace('T', '-'); +} + +function usage(exitCode = 0) { + const out = exitCode === 0 ? process.stdout : process.stderr; + out.write(`Usage:\n`); + out.write(` external-salary-scan.mjs --search-id [options]\n`); + out.write(` external-salary-scan.mjs --queue [options]\n\n`); + out.write(`Options:\n`); + out.write(` --db SQLite DB, default ${DEFAULT_DB}\n`); + out.write(` --out NDJSON output, default ~/.job-hunter/logs/external-salary-scan-*.ndjson\n`); + out.write(` --limit max jobs to inspect\n`); + out.write(` --cdp-port Chromium CDP port, default 9225\n`); + out.write(` --dry-run scan and write output but do not insert observations\n`); + out.write(` --help show this help\n`); + process.exit(exitCode); +} + +function parseArgs(argv) { + const opts = { + db: DEFAULT_DB, + queue: null, + searchId: null, + out: path.join(LOGS_DIR, `external-salary-scan-${stamp()}.ndjson`), + limit: 0, + cdpPort: Number(process.env.CDP_PORT || process.env.BROWSER_CDP_PORT || 9225), + dryRun: false, + }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + const next = () => { + if (i + 1 >= argv.length) throw new Error(`${a} requires a value`); + return argv[++i]; + }; + if (a === '--help' || a === '-h') usage(0); + else if (a === '--db') opts.db = next(); + else if (a === '--queue') opts.queue = next(); + else if (a === '--search-id') opts.searchId = next(); + else if (a === '--out') opts.out = next(); + else if (a === '--limit') opts.limit = Number(next()); + else if (a === '--cdp-port' || a === '--port') opts.cdpPort = Number(next()); + else if (a === '--dry-run') opts.dryRun = true; + else throw new Error(`unknown argument: ${a}`); + } + if (!opts.searchId && opts.queue) { + const q = JSON.parse(fs.readFileSync(opts.queue, 'utf8')); + opts.searchId = q.search_id || q.searchId || null; + opts.queueJobs = Array.isArray(q.jobs) ? q.jobs : (Array.isArray(q) ? q : []); + } + if (!opts.searchId && !opts.queueJobs?.length) throw new Error('provide --search-id or --queue'); + return opts; +} + +function requireFromWorkspace(id) { + return createRequire(path.join(JH, 'package.json'))(id); +} +const WebSocket = requireFromWorkspace('ws'); +const Database = requireFromWorkspace('better-sqlite3'); + +function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function makeCdpHttp(cdpPort) { + return function httpJson(method, reqPath) { + return new Promise((resolve, reject) => { + const req = http.request({ method, host: '127.0.0.1', port: cdpPort, path: reqPath, timeout: 10000 }, (res) => { + let d = ''; + res.on('data', (c) => { d += c; }); + res.on('end', () => { + try { resolve(JSON.parse(d)); } catch { reject(new Error(`bad JSON from CDP: ${d.slice(0, 200)}`)); } + }); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('CDP HTTP timeout')); }); + req.end(); + }); + }; +} + +class Cdp { + constructor(wsUrl) { this.wsUrl = wsUrl; this.id = 1; this.pending = new Map(); } + async connect() { + this.ws = new WebSocket(this.wsUrl); + await new Promise((resolve, reject) => { + const t = setTimeout(() => reject(new Error('ws timeout')), 10000); + this.ws.on('open', () => { clearTimeout(t); resolve(); }); + this.ws.on('error', (e) => { clearTimeout(t); reject(e); }); + }); + this.ws.on('message', (raw) => { + let m; + try { m = JSON.parse(raw.toString()); } catch { return; } + if (!m.id) return; + const p = this.pending.get(m.id); + if (!p) return; + clearTimeout(p.t); + this.pending.delete(m.id); + if (m.error) p.reject(new Error(m.error.message || 'CDP error')); + else p.resolve(m.result); + }); + } + send(method, params = {}, timeout = 20000) { + const id = this.id++; + return new Promise((resolve, reject) => { + const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`Timeout: ${method}`)); }, timeout); + this.pending.set(id, { resolve, reject, t }); + this.ws.send(JSON.stringify({ id, method, params })); + }); + } + async eval(expression, timeout = 20000) { + const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, timeout); + if (r?.exceptionDetails) throw new Error(`JS exception ${JSON.stringify(r.exceptionDetails).slice(0, 300)}`); + return r?.result?.value; + } + close() { try { this.ws.close(); } catch {} } +} + +async function newTab(httpJson, url) { + const tab = await httpJson('PUT', `/json/new?url=${encodeURIComponent(url)}`); + const c = new Cdp(tab.webSocketDebuggerUrl); + await c.connect(); + await c.send('Page.enable'); + await c.send('Runtime.enable'); + await c.send('Page.navigate', { url }); + return { tab, c }; +} +async function closeTab(httpJson, tabId) { try { await httpJson('GET', `/json/close/${tabId}`); } catch {} } + +function salaryCandidates(text) { + if (!text) return []; + const clean = text.replace(/\s+/g, ' '); + const patterns = [ + /(?:salary|base pay range|pay range|compensation|range|remuneration)?[^£€$A-Z]{0,30}((?:GBP|EUR|USD|DKK|CHF|£|€|\$)\s?[0-9][0-9.,]*(?:\s?[kK])?\s*(?:-|–|—|to)\s*(?:(?:GBP|EUR|USD|DKK|CHF|£|€|\$)\s?)?[0-9][0-9.,]*(?:\s?[kK])?(?:\s*(?:per year|\/yr|year|annually|annual|p\.a\.))?)/gi, + /((?:GBP|EUR|USD|DKK|CHF|£|€|\$)\s?[0-9][0-9.,]*(?:\s?[kK])?\s*(?:per year|\/yr|year|annually|annual|p\.a\.))/gi, + ]; + const out = []; + const seen = new Set(); + for (const re of patterns) { + for (const m of clean.matchAll(re)) { + const start = Math.max(0, m.index - 180); + const end = Math.min(clean.length, m.index + m[0].length + 220); + const snippet = clean.slice(start, end).trim(); + const key = `${m[1]}|${snippet.slice(0, 80)}`; + if (seen.has(key)) continue; + seen.add(key); + out.push({ match: m[1].trim(), snippet }); + if (out.length >= 8) return out; + } + } + return out; +} + +function parseRange(match) { + if (!match) return null; + let currency = ''; + if (/£|GBP/i.test(match)) currency = 'GBP'; + else if (/€|EUR/i.test(match)) currency = 'EUR'; + else if (/\$|USD/i.test(match)) currency = 'USD'; + else if (/DKK/i.test(match)) currency = 'DKK'; + else if (/CHF/i.test(match)) currency = 'CHF'; + const nums = [...match.matchAll(/(?:GBP|EUR|USD|DKK|CHF|£|€|\$)?\s*([0-9][0-9.,]*)\s*([kK])?/g)].map((x) => { + let n = x[1].replace(/,/g, ''); + if (/^\d+\.\d+$/.test(n) && Number(n) < 1000 && !x[2]) n = String(Number(n) * 1000); + let v = Number(n); + if (x[2]) v *= 1000; + return v; + }).filter((n) => Number.isFinite(n)); + if (!currency || nums.length === 0) return null; + const min = nums[0]; + const max = nums[1] || nums[0]; + return { currency, min, max, median: (min + max) / 2 }; +} + +function isLikelyExactCandidate(c, source) { + const s = (c.snippet || '').toLowerCase(); + if (/base pay range|provided pay range|salary\s*:|salary range|pay range/.test(s)) return true; + if (source === 'external' && /salary|compensation|remuneration|base/.test(s)) return true; + return false; +} + +async function pageText(c) { + return await c.eval(`(() => ({url: location.href, title: document.title, text: document.body ? document.body.innerText : ''}))()`); +} + +async function findExternal(c) { + return await c.eval(`(() => { + const linkObjs = Array.from(document.querySelectorAll('a')).map(a => ({text:(a.innerText||a.textContent||'').trim().slice(0,80), href:a.href||'', tracking:a.getAttribute('data-tracking-control-name')||'', aria:a.getAttribute('aria-label')||''})); + function isExternal(h){ try { const u=new URL(h, location.href); return !/(^|\\.)linkedin\\.com$/i.test(u.hostname); } catch { return false; } } + const explicit = linkObjs.find(x => /apply-link-offsite|offsite|external/i.test(x.tracking) && x.href); + const nonLiApply = linkObjs.find(x => /apply/i.test(x.text+' '+x.aria+' '+x.tracking) && x.href && (isExternal(x.href) || String(x.href).toLowerCase().includes('safety/go'))); + const buttons = Array.from(document.querySelectorAll('button, a')).map((b,idx)=>({idx, tag:b.tagName, text:(b.innerText||b.textContent||'').trim().replace(/\s+/g,' ').slice(0,80), cls:b.className||'', aria:b.getAttribute('aria-label')||''})).filter(x=>/apply/i.test(x.text+' '+x.aria+' '+x.cls)); + return {explicit: explicit||null, nonLiApply: nonLiApply||null, buttons: buttons.slice(0,10), allExternal: linkObjs.filter(x=>x.href && isExternal(x.href)).slice(0,20)}; + })()`); +} + +async function clickApplyAndDetect(c, beforeIds, httpJson) { + const clicked = await c.eval(`(() => { + const els=Array.from(document.querySelectorAll('button, a')); + const el=els.find(b => /^(apply|apply now)$/i.test((b.innerText||b.textContent||b.getAttribute('aria-label')||'').trim()) && !/easy apply/i.test((b.innerText||b.textContent||''))); + if (!el) return {clicked:false, reason:'no non-easy apply button'}; + el.click(); return {clicked:true, text:(el.innerText||el.textContent||el.getAttribute('aria-label')||'').trim()}; + })()`); + let current = null; + let tabs = []; + for (let i = 0; i < 14; i += 1) { + await delay(1000); + current = await c.eval(`(() => location.href)()`).catch(() => null); + tabs = await httpJson('GET', '/json/list').catch(() => []); + const newTabs = tabs.filter((t) => !beforeIds.has(t.id)); + const usefulNew = newTabs.find((t) => t.url && !/^about:blank$/i.test(t.url) && !/^chrome:\/\//i.test(t.url)); + const currentUseful = current && !/^about:blank$/i.test(current) && !/^chrome:\/\//i.test(current); + if (usefulNew || (currentUseful && !/linkedin\.com\/jobs\/view/i.test(current))) break; + } + const newTabs = tabs.filter((t) => !beforeIds.has(t.id)); + return { clicked, current, newTabs: newTabs.map((t) => ({ id: t.id, url: t.url, title: t.title, ws: t.webSocketDebuggerUrl })) }; +} + +function decodeLinkedinSafety(h) { + try { + const u = new URL(h); + const target = u.searchParams.get('url'); + if (target) return decodeURIComponent(target); + } catch {} + return h; +} + +function chooseExact(candidates, source) { + for (const c of candidates) { + const parsed = parseRange(c.match); + if (!parsed) continue; + if (isLikelyExactCandidate(c, source)) return { ...c, ...parsed, source }; + } + return null; +} + +function shapeObservation(row, exactCandidate, nowIso) { + const dataSource = exactCandidate.source === 'external' ? 'advertiser_ats' : 'linkedin'; + return { + job_source: row.source, + job_id: row.job_id, + data_source: dataSource, + data_source_url: exactCandidate.sourceUrl || row.url, + benchmark_id: null, + confidence_label: dataSource === 'advertiser_ats' ? 'external_exact' : 'posted_exact', + matched_by: 'exact_job', + is_posted_salary: 1, + is_predicted: 0, + currency: exactCandidate.currency, + amount_min: exactCandidate.min, + amount_max: exactCandidate.max, + amount_median: exactCandidate.median, + period: 'year', + compensation_type: 'base_salary', + annualized_min: exactCandidate.min, + annualized_max: exactCandidate.max, + annualized_median: exactCandidate.median, + annualization_note: 'posted annual salary range parsed from job page', + location_raw: row.location_raw ?? null, + country_code: row.country_code ?? null, + region: row.region ?? null, + city: row.city ?? null, + evidence_snippet: exactCandidate.evidence || exactCandidate.snippet || exactCandidate.match, + raw_payload_json: { exactCandidate, job: { source: row.source, job_id: row.job_id, title: row.title, company: row.company } }, + observed_at: nowIso, + }; +} + +function rowsForScan(db, opts) { + if (opts.searchId) { + return db.prepare(` + WITH latest_obs AS ( + SELECT o.*, ROW_NUMBER() OVER (PARTITION BY o.job_source,o.job_id ORDER BY o.is_posted_salary DESC,o.observed_at DESC,o.created_at DESC) rn + FROM job_salary_observations o + ) + SELECT j.source,j.job_id,j.title,j.company,j.country_code,j.region,j.city,j.location_raw,j.url,lo.is_posted_salary,lo.data_source + FROM jobs j JOIN match_results mr ON mr.source=j.source AND mr.job_id=j.job_id + LEFT JOIN latest_obs lo ON lo.job_source=j.source AND lo.job_id=j.job_id AND lo.rn=1 + WHERE mr.search_id=? AND mr.cta='Apply' AND COALESCE(lo.is_posted_salary,0)=0 + ORDER BY CASE j.country_code WHEN 'GB' THEN 0 WHEN 'IE' THEN 1 WHEN 'NL' THEN 2 WHEN 'DK' THEN 3 ELSE 9 END, j.company, j.title + `).all(opts.searchId); + } + const queueJobs = opts.queueJobs || []; + const stmt = db.prepare(` + SELECT j.source,j.job_id,j.title,j.company,j.country_code,j.region,j.city,j.location_raw,j.url + FROM jobs j + WHERE j.source = COALESCE(@source, j.source) AND j.job_id=@job_id + `); + return queueJobs.map((j) => stmt.get({ source: j.source || null, job_id: String(j.job_id) })).filter(Boolean); +} + +function isLinkedinJob(url) { + try { return /(^|\.)linkedin\.com$/i.test(new URL(url).hostname) && /\/jobs\/view/i.test(new URL(url).pathname); } catch { return false; } +} + +async function scanRow(row, httpJson) { + const rec = { + at: new Date().toISOString(), + job: row, + linkedinSalaryCandidates: [], + externalUrl: null, + externalTitle: null, + externalSalaryCandidates: [], + exactCandidate: null, + insertedObservation: null, + errors: [], + }; + let tab; + let c; + try { + const before = new Set((await httpJson('GET', '/json/list')).map((t) => t.id)); + ({ tab, c } = await newTab(httpJson, row.url)); + await delay(5500); + const first = await pageText(c); + if (isLinkedinJob(row.url)) { + rec.linkedinUrl = first.url; + rec.linkedinTitle = first.title; + rec.linkedinSalaryCandidates = salaryCandidates((first.text || '').split(/Show more jobs like this|Similar jobs|People also viewed/i)[0]); + const liExact = chooseExact(rec.linkedinSalaryCandidates, 'linkedin'); + if (liExact) rec.exactCandidate = { ...liExact, sourceUrl: first.url, evidence: liExact.snippet }; + const found = await findExternal(c); + rec.applyInspection = found; + let externalHref = found?.explicit?.href || found?.nonLiApply?.href || null; + if (externalHref) externalHref = decodeLinkedinSafety(externalHref); + if (!externalHref && !rec.exactCandidate) { + const det = await clickApplyAndDetect(c, before, httpJson); + rec.clickDetect = det; + const current = det.current || ''; + if (/^https?:\/\//i.test(current) && !/linkedin\.com/i.test(new URL(current).hostname)) externalHref = current; + if (!externalHref && det.newTabs?.length) { + const nt = det.newTabs.find((t) => t.url && /^https?:\/\//i.test(t.url) && !/linkedin\.com/i.test(new URL(t.url).hostname)); + if (nt) { externalHref = nt.url; rec.externalTabId = nt.id; } + } + } + if (externalHref && !/linkedin\.com\/signup|linkedin\.com\/login/i.test(externalHref)) { + rec.externalUrl = externalHref; + let extC = c; + let extTabId = tab.id; + const tabs = await httpJson('GET', '/json/list').catch(() => []); + const opened = tabs.find((t) => t.url === externalHref || (rec.externalTabId && t.id === rec.externalTabId)); + if (opened && opened.webSocketDebuggerUrl && opened.id !== tab.id) { + extC = new Cdp(opened.webSocketDebuggerUrl); + await extC.connect(); await extC.send('Page.enable'); await extC.send('Runtime.enable'); extTabId = opened.id; + } else { + await c.send('Page.navigate', { url: externalHref }); await delay(6500); + } + const ext = await pageText(extC); + rec.externalUrl = ext.url || externalHref; rec.externalTitle = ext.title; + rec.externalSalaryCandidates = salaryCandidates((ext.text || '').slice(0, 250000)); + const extExact = chooseExact(rec.externalSalaryCandidates, 'external'); + if (extExact) rec.exactCandidate = { ...extExact, sourceUrl: ext.url || externalHref, evidence: extExact.snippet }; + if (extC !== c) extC.close(); + if (extTabId !== tab.id) await closeTab(httpJson, extTabId); + } + } else { + rec.externalUrl = first.url || row.url; + rec.externalTitle = first.title; + rec.externalSalaryCandidates = salaryCandidates((first.text || '').slice(0, 250000)); + const extExact = chooseExact(rec.externalSalaryCandidates, 'external'); + if (extExact) rec.exactCandidate = { ...extExact, sourceUrl: first.url || row.url, evidence: extExact.snippet }; + } + } catch (e) { + rec.errors.push(String(e?.stack || e)); + } finally { + try { c?.close(); } catch {} + if (tab?.id) await closeTab(httpJson, tab.id); + } + return rec; +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + fs.mkdirSync(path.dirname(opts.out), { recursive: true }); + fs.writeFileSync(opts.out, ''); + const db = new Database(opts.db); + const rows = rowsForScan(db, opts); + console.log(`scan start ${rows.length} jobs; out=${opts.out}; dry_run=${opts.dryRun}`); + const httpJson = makeCdpHttp(opts.cdpPort); + let lock = null; + let uninstall = null; + if (!opts.dryRun) { + lock = acquire(db); + if (!lock.acquired) { + console.error(formatContentionMessage(lock.holder)); + process.exit(4); + } + uninstall = installSignalHandlers(lock); + lock.startHeartbeat(); + } + let count = 0; + let inserted = 0; + try { + for (const row of rows) { + if (opts.limit && count >= opts.limit) break; + count += 1; + console.log(`[${count}/${rows.length}] ${row.source}:${row.job_id} ${row.title} — ${row.company}`); + const rec = await scanRow(row, httpJson); + if (rec.exactCandidate) { + const obs = shapeObservation(row, rec.exactCandidate, new Date().toISOString()); + if (!opts.dryRun) { + try { + const res = insertObservation(db, obs); + const prior = getEnrichmentState(db, row.source, row.job_id) || { exact_status: 'pending', exact_attempt_count: 0 }; + const transition = computeNextRetry({ + axis: 'exact', + prevStatus: prior.exact_status, + prevAttemptCount: Number(prior.exact_attempt_count || 0), + event: { kind: 'success' }, + nowIso: obs.observed_at, + }); + applyRetryTransition(db, row.source, row.job_id, 'exact', transition, obs.observed_at); + rec.insertedObservation = res; + rec.enrichmentState = { exact: transition.status, attemptCount: transition.attemptCount }; + if (res.inserted) inserted += 1; + } catch (err) { + rec.errors.push(`insert failed: ${String(err?.message || err)}`); + } + } + console.log(` exact ${rec.exactCandidate.currency} ${rec.exactCandidate.min}-${rec.exactCandidate.max} from ${rec.exactCandidate.sourceUrl}`); + } else if (rec.externalUrl) console.log(` external no exact: ${rec.externalUrl.slice(0, 120)}`); + else console.log(' no external/exact found'); + fs.appendFileSync(opts.out, `${JSON.stringify(rec)}\n`); + await delay(800); + } + } finally { + if (lock) lock.release(); + if (uninstall) uninstall(); + db.close(); + } + const done = `${opts.out}.done`; + fs.writeFileSync(done, `${new Date().toISOString()}\n`); + console.log(`scan done inspected=${count} inserted=${inserted} done=${done}`); +} + +main().catch((err) => { + console.error(`[fatal] ${err.stack || err}`); + process.exit(2); +}); diff --git a/skills/salary-calculator/scripts/http-client.mjs b/skills/salary-calculator/scripts/http-client.mjs new file mode 100644 index 0000000..6b4c010 --- /dev/null +++ b/skills/salary-calculator/scripts/http-client.mjs @@ -0,0 +1,637 @@ +/** + * HTTP Client with Per-Host Rate Limiting, Concurrency Control, and Retry Logic + * + * Provides a single HTTP client for all outbound requests with: + * - Per-host token bucket rate limiting (configurable rps, default 0.5 = 2s between requests) + * - Per-host FIFO semaphore for concurrency control (configurable, default 1 sequential) + * - Automatic retry on 429, 5xx, and transient transport errors + * - Retry-After header parsing (both delta-seconds and HTTP-date RFC 2822 formats) + * - Exponential backoff with full jitter for retryable errors + * - AbortSignal support with prompt rejection and no retry budget consumption + * - Per-source limits override (configured at startup, sealed on first request per host) + * + * Main export: createHttpClient(options) → { get(url, opts), getJson(url, opts) } + * + * No external dependencies. Node.js 22+ only (uses native fetch, AbortController). + * Contract enforcement (HTTP-01): focused HTTP client checks ensure + * no source adapter calls global fetch() directly; all must use ctx.httpClient.get(). + */ + +/** + * TokenBucket: Per-host rate limiting via token bucket algorithm + * + * Maintains a fixed number of tokens (burst capacity). On each consume(), + * refills tokens at perHostRps rate, then blocks until sufficient tokens + * are available. + * + * @example + * const bucket = new TokenBucket({ perHostRps: 0.5, burst: 1 }); + * await bucket.consume(1); // Waits 2s if empty (0.5 rps = 2s per token) + */ +export class TokenBucket { + /** + * Create a token bucket with specified rate limit. + * + * @param {Object} options - Configuration + * @param {number} options.perHostRps - Tokens per second (default 0.5 = 2s between requests) + * @param {number} options.burst - Max token capacity (default 1) + */ + constructor({ perHostRps = 0.5, burst = 1 } = {}) { + this.perHostRps = perHostRps; + this.burst = burst; + this.tokens = burst; // Start with full capacity + this.lastRefill = Date.now(); + } + + /** + * Internal: Refill tokens based on elapsed time since last refill. + * Tokens are capped at burst capacity. + */ + refill() { + const now = Date.now(); + const elapsedSeconds = (now - this.lastRefill) / 1000; + const tokensToAdd = elapsedSeconds * this.perHostRps; + this.tokens = Math.min(this.burst, this.tokens + tokensToAdd); + this.lastRefill = now; + } + + /** + * Consume one or more tokens, blocking until available. + * + * If insufficient tokens, sleeps until the required tokens are available + * at the configured perHostRps rate, then decrements and returns. + * + * Honors an optional AbortSignal: if signal aborts mid-wait the throttle + * sleep rejects with AbortError and tokens are NOT decremented. + * + * @param {number} count - Number of tokens to consume (default 1) + * @param {Object} [options] - Options + * @param {AbortSignal} [options.signal] - Optional abort signal — when aborted, + * the throttle-wait rejects promptly with AbortError and no token is consumed. + */ + async consume(count = 1, { signal } = {}) { + if (signal?.aborted) { + throw signal.reason ?? new DOMException('Aborted', 'AbortError'); + } + this.refill(); + while (this.tokens < count) { + const deficit = count - this.tokens; + const waitMs = (deficit / this.perHostRps) * 1000; + // eslint-disable-next-line no-await-in-loop + await abortableSleep(waitMs, signal); + this.refill(); + } + this.tokens -= count; + } +} + +/** + * PerHostSemaphore: FIFO concurrency control per host + * + * Maintains a queue of pending requests. When inFlight < maxConcurrent, + * acquire() resolves immediately; otherwise, it waits in a FIFO queue. + * + * @example + * const sem = new PerHostSemaphore({ maxConcurrent: 2 }); + * const release = await sem.acquire(); + * try { + * // Do work + * } finally { + * release(); + * } + */ +export class PerHostSemaphore { + /** + * Create a semaphore with maximum concurrent acquisitions. + * + * @param {Object} options - Configuration + * @param {number} options.maxConcurrent - Max in-flight requests (default 1) + */ + constructor({ maxConcurrent = 1 } = {}) { + this.maxConcurrent = maxConcurrent; + this.inFlight = 0; + this.queue = []; // Array of { resolve, reject } + } + + /** + * Acquire a slot. Returns a release function to call when done. + * + * If inFlight < maxConcurrent, returns immediately. + * Otherwise, waits in FIFO queue until a slot becomes available. + * + * @returns {Promise} A release function to call when done with the slot + */ + async acquire() { + if (this.inFlight < this.maxConcurrent) { + this.inFlight++; + // Return a release function that decrements and unblocks next waiter + return () => { + this.release(); + }; + } + + // Queue this request and wait for its turn + return new Promise((resolve) => { + this.queue.push({ + resolve: () => { + resolve(() => { + this.release(); + }); + }, + }); + }); + } + + /** + * Internal: Release a slot and unblock the next waiter (if any). + */ + release() { + this.inFlight--; + const next = this.queue.shift(); + if (next) { + this.inFlight++; + next.resolve(); + } + } +} + +/** + * Calculate exponential backoff delay with full jitter. + * + * Formula: delay = random([0, min(baseMs * 2^(attemptNumber - 1), maxMs)]) + * + * Full jitter means the entire calculated delay is randomized, not just added + * as a random amount on top of a fixed base. This prevents thundering herds + * when many clients retry simultaneously. + * + * @param {number} attemptNumber - 1-indexed attempt number (first retry = 1) + * @param {Object} options - Configuration + * @param {number} options.baseDelayMs - Base delay in milliseconds (default 500) + * @param {number} options.maxDelayMs - Maximum delay cap (default 30000) + * @returns {number} Delay in milliseconds, randomized in [0, capped exponential) + * + * @example + * const delay = calculateBackoffMs(1); // [0, 500) + * const delay = calculateBackoffMs(2); // [0, 1000) + * const delay = calculateBackoffMs(3); // [0, 2000) + */ +export function calculateBackoffMs( + attemptNumber, + { baseDelayMs = 500, maxDelayMs = 30000 } = {} +) { + if (attemptNumber < 1) return 0; + const exponential = baseDelayMs * Math.pow(2, attemptNumber - 1); + const capped = Math.min(exponential, maxDelayMs); + return Math.random() * capped; +} + +/** + * Parse Retry-After header (both delta-seconds and HTTP-date formats). + * + * RFC 7231 Retry-After can be either: + * - Delta-seconds: "5" meaning 5 seconds + * - HTTP-date: "Wed, 21 Oct 2026 07:28:00 GMT" meaning that specific time + * + * Returns an object { delayMs, exceededCap } if valid; null if unparseable. + * If the calculated or server-stated delay exceeds maxDelayMs, sets exceededCap=true + * to signal that the caller should sleep this long then stop retrying. + * + * @param {string|null|undefined} headerValue - The Retry-After header value + * @param {Object} options - Configuration + * @param {number} options.maxDelayMs - Maximum acceptable delay (default 30000) + * @returns {Object|null} { delayMs: number, exceededCap: boolean } or null if unparseable + * + * @example + * parseRetryAfterMs('5') // => { delayMs: 5000, exceededCap: false } + * parseRetryAfterMs('120', { maxDelayMs: 30000 }) // => { delayMs: 120000, exceededCap: true } + * parseRetryAfterMs('Wed, 21 Oct 2026 07:28:00 GMT') // => { delayMs: ~10000, exceededCap: false } + * parseRetryAfterMs(null) // => null + * parseRetryAfterMs('garbage') // => null + */ +export function parseRetryAfterMs(headerValue, { maxDelayMs = 30000 } = {}) { + // Handle null, undefined, empty string + if (!headerValue || headerValue === '') { + return null; + } + + // Try parsing as delta-seconds (integer) + if (/^\s*\d+\s*$/.test(headerValue)) { + const seconds = parseInt(headerValue, 10); + const delayMs = seconds * 1000; + const exceededCap = delayMs > maxDelayMs; + return { + delayMs: exceededCap ? maxDelayMs : delayMs, + exceededCap, + }; + } + + // Try parsing as HTTP-date (RFC 2822 format) + const date = new Date(headerValue); + if (!isNaN(date.getTime())) { + const delayMs = Math.max(0, date.getTime() - Date.now()); + const exceededCap = delayMs > maxDelayMs; + return { + delayMs: exceededCap ? maxDelayMs : delayMs, + exceededCap, + }; + } + + // Unparseable + return null; +} + +/** + * Check if an HTTP status code is retryable. + * + * Retryable statuses: 429 (Too Many Requests), all 5xx (server errors). + * Non-retryable 4xx (400, 401, 403, 404, etc.) fail immediately. + * + * @param {number} status - HTTP status code + * @returns {boolean} True if the request should be retried + * + * @example + * isRetryableStatus(429) // => true + * isRetryableStatus(503) // => true + * isRetryableStatus(404) // => false + * isRetryableStatus(401) // => false + */ +export function isRetryableStatus(status) { + return status === 429 || (status >= 500 && status < 600); +} + +/** + * Check if a thrown error is retryable (transient transport error). + * + * Retryable errors: ECONNRESET (connection dropped), ETIMEDOUT (socket timeout), + * and fetch TypeError (usually network-related). + * + * Non-retryable: Invalid URLs, caller abort, etc. + * + * @param {Error} err - The thrown error + * @returns {boolean} True if the request should be retried + * + * @example + * const err = new Error('socket hang up'); + * err.code = 'ECONNRESET'; + * isRetryableError(err) // => true + * + * isRetryableError(new TypeError('fetch failed')) // => true + * isRetryableError(new DOMException('Aborted', 'AbortError')) // => false + */ +export function isRetryableError(err) { + if (!err) return false; + if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { + return true; + } + if (err instanceof TypeError) { + return true; + } + return false; +} + +/** + * Create a host registry for lazy-instantiating per-host TokenBucket and PerHostSemaphore. + * + * Ensures that each unique host has exactly one TokenBucket and one PerHostSemaphore, + * preventing one slow host from blocking requests to other hosts. + * + * @param {Object} options - Default bucket/semaphore options + * @param {number} options.perHostRps - Tokens per second (passed to TokenBucket) + * @param {number} options.maxConcurrent - Max concurrent requests per host + * @returns {Object} { getBucket(host, opts), getSemaphore(host, opts) } + * + * @example + * const registry = createHostRegistry({ perHostRps: 0.5, maxConcurrent: 2 }); + * const bucket = registry.getBucket('example.com'); // Lazy created + * const bucket2 = registry.getBucket('example.com'); // Same instance + * const bucket3 = registry.getBucket('other.com'); // Different instance + */ +export function createHostRegistry( + { perHostRps = 0.5, maxConcurrent = 1 } = {} +) { + const buckets = new Map(); + const semaphores = new Map(); + + return { + /** + * Get or create a TokenBucket for the given host. + * + * @param {string} host - The host (e.g., 'example.com') + * @param {Object} opts - Override options { perHostRps, burst } + * @returns {TokenBucket} The bucket instance for this host + */ + getBucket(host, opts = {}) { + if (!buckets.has(host)) { + buckets.set( + host, + new TokenBucket({ + perHostRps: opts.perHostRps ?? perHostRps, + burst: opts.burst ?? 1, + }) + ); + } + return buckets.get(host); + }, + + /** + * Get or create a PerHostSemaphore for the given host. + * + * @param {string} host - The host (e.g., 'example.com') + * @param {Object} opts - Override options { maxConcurrent } + * @returns {PerHostSemaphore} The semaphore instance for this host + */ + getSemaphore(host, opts = {}) { + if (!semaphores.has(host)) { + semaphores.set( + host, + new PerHostSemaphore({ + maxConcurrent: opts.maxConcurrent ?? maxConcurrent, + }) + ); + } + return semaphores.get(host); + }, + }; +} + +/** + * Helper: abortableSleep resolves after ms OR rejects with AbortError if signal aborts first. + * + * Clears timers and removes listeners on abort to prevent leaks. + * If signal is already aborted on entry, rejects synchronously. + * + * @param {number} ms - Milliseconds to sleep + * @param {AbortSignal} [signal] - Optional abort signal + * @returns {Promise} Resolves after sleep or rejects on abort + */ +function abortableSleep(ms, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + return reject(signal.reason ?? new DOMException('Aborted', 'AbortError')); + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(signal.reason ?? new DOMException('Aborted', 'AbortError')); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +/** + * Create an HTTP client with per-host rate limiting, concurrency control, and retry logic. + * + * Usage in a source adapter (e.g., scripts/sources/linkedin.mjs): + * + * export const limits = { perHostRps: 1, maxConcurrent: 2 }; + * export async function fetchExactSalary(job, ctx) { + * const response = await ctx.httpClient.get(`https://www.linkedin.com/jobs/view/${job.id}`); + * const html = await response.text(); + * return parseSalaryCandidates(html, ctx); + * } + * + * The orchestrator builds the client once at startup: + * const httpClient = createHttpClient({ limits: { 'www.linkedin.com': linkedin.limits } }); + * const ctx = { httpClient, logger }; + * + * Never call global fetch() from scripts/sources/**; use this client so rate limits and evidence remain consistent. + * + * Adapters MUST receive the httpClient via ctx and call ctx.httpClient.get() / ctx.httpClient.getJson(). + * Per-request limits override is NOT supported (D4 resolution). Limits are sealed at first request to each host. + * + * @param {Object} options - Configuration + * @param {number} options.perHostRps - Default tokens per second (default 0.5 = 2s between requests) + * @param {number} options.maxConcurrent - Default max concurrent requests per host (default 1) + * @param {number} options.maxRetries - Max retry attempts (default 3) + * @param {number} options.maxDelayMs - Maximum backoff delay (default 30000) + * @param {number} options.baseDelayMs - Base exponential backoff delay (default 500) + * @param {Object} options.limits - Map of host -> { perHostRps, maxConcurrent } overrides + * @returns {Object} { get(url, opts), getJson(url, opts) } + */ +export function createHttpClient(options = {}) { + const { + perHostRps = 0.5, + maxConcurrent = 1, + maxRetries = 3, + maxDelayMs = 30000, + baseDelayMs = 500, + limits = {}, // Maps { host: { perHostRps, maxConcurrent } } + } = options; + + // Create one host registry to cache buckets and semaphores per host + const registry = createHostRegistry({ perHostRps, maxConcurrent }); + + /** + * Core fetch logic with rate limiting, retry, and abort support. + * + * @param {string} url - URL to fetch + * @param {Object} opts - Request options + * @param {AbortSignal} [opts.abortSignal] - Optional abort signal. Either `signal` or `abortSignal` is accepted at the call site; both keys are honoured identically by all downstream sites (fetch, retry sleep, throttle wait) via a single hoisted alias inside the function body. + * @param {Object} opts.fetchInit - Additional fetch() options (headers, method, etc.) + * @param {Function} opts.logger - Optional logger for structured events + * @returns {Promise} The response object + */ + async function get(url, opts = {}) { + // Validate per-request limits constraint (D4) + if (opts.limits !== undefined) { + throw new Error( + 'Per-request limits override is not supported; configure limits at createHttpClient() time via limits[host]' + ); + } + + // Hoist a single local alias so both `signal` and `abortSignal` keys are honoured + // identically by every downstream call site (fetch, abortableSleep, bucket.consume, + // and the error-classification branches). Adding new sites? Reference `signal` only. + const signal = opts.signal ?? opts.abortSignal; + + const urlObj = new URL(url); + const host = urlObj.host; + + // Resolve effective limits: defaults < createHttpClient options < limits[host] + // Per-host limits are sealed on first request to that host; this is intentional (D4). + const hostOverride = limits[host]; + const effectiveRps = hostOverride?.perHostRps ?? perHostRps; + const effectiveConcurrent = hostOverride?.maxConcurrent ?? maxConcurrent; + + // Get the host's bucket and semaphore from the registry + const bucket = registry.getBucket(host, { perHostRps: effectiveRps }); + const semaphore = registry.getSemaphore(host, { maxConcurrent: effectiveConcurrent }); + + // Acquire semaphore slot + const releaseSlot = await semaphore.acquire(); + + try { + // Await rate limit token consumption (abortable — signal aborts mid-wait reject promptly) + await bucket.consume(1, { signal }); + + // Retry loop + let attempt = 0; + while (attempt <= maxRetries) { + // Check for abort before attempting + if (signal?.aborted) { + throw signal.reason ?? new DOMException('Aborted', 'AbortError'); + } + + try { + // Perform the fetch + const response = await fetch(url, { + ...(opts.fetchInit || {}), + signal, + }); + + // Check response status + if (response.ok || (response.status >= 400 && response.status !== 429 && response.status < 500)) { + // 2xx, 3xx, or 4xx (except 429) — return without retry + return response; + } + + // Check if status is retryable + if (!isRetryableStatus(response.status)) { + return response; + } + + // Status is retryable (429 or 5xx) + attempt++; + if (attempt > maxRetries) { + // Exhausted retries — throw the response as error + throw response; + } + + // Calculate delay + let delayMs; + let parsedRetryAfter = null; + let shouldStopRetrying = false; + + if (response.status === 429) { + const retryAfterHeader = response.headers.get('retry-after'); + parsedRetryAfter = parseRetryAfterMs(retryAfterHeader, { maxDelayMs }); + + if (parsedRetryAfter) { + delayMs = parsedRetryAfter.delayMs; + // If server delay exceeds cap, sleep that long then stop retrying + if (parsedRetryAfter.exceededCap) { + shouldStopRetrying = true; + // Log if logger provided + if (opts.logger) { + opts.logger({ + event: 'retry', + host, + attempt, + status: response.status, + retryAfterMs: parsedRetryAfter.delayMs, + delayMs, + reason: 'exceededCap', + }); + } + // Sleep then throw error + await abortableSleep(delayMs, signal); + throw response; // Throw the response as the error + } + } else { + // No Retry-After or unparseable — use backoff + delayMs = calculateBackoffMs(attempt, { baseDelayMs, maxDelayMs }); + } + } else { + // 5xx — use backoff + delayMs = calculateBackoffMs(attempt, { baseDelayMs, maxDelayMs }); + } + + // Log structured retry event if logger provided + if (opts.logger) { + opts.logger({ + event: 'retry', + host, + attempt, + status: response.status, + retryAfterMs: parsedRetryAfter?.delayMs ?? null, + delayMs, + }); + } + + // Sleep with abort support + await abortableSleep(delayMs, signal); + } catch (err) { + // Handle fetch errors + if (err.name === 'AbortError' || signal?.aborted) { + // Abort — rethrow immediately without incrementing retry budget + throw err; + } + + // Check if error is retryable + if (!isRetryableError(err)) { + // Non-retryable error — rethrow + throw err; + } + + // Retryable error + attempt++; + if (attempt > maxRetries) { + // Exhausted retries — rethrow + throw err; + } + + // Calculate backoff for retryable error + const delayMs = calculateBackoffMs(attempt, { baseDelayMs, maxDelayMs }); + + // Log structured retry event + if (opts.logger) { + opts.logger({ + event: 'retry', + host, + attempt, + error: err.code || err.message, + delayMs, + }); + } + + // Sleep with abort support + await abortableSleep(delayMs, signal); + } + } + + // Shouldn't reach here, but just in case + throw new Error('Retry loop exited unexpectedly'); + } finally { + releaseSlot(); + } + } + + /** + * Convenience wrapper that returns parsed JSON. + * + * @param {string} url - URL to fetch + * @param {Object} opts - Request options (same as get()) + * @returns {Promise} Parsed JSON response body + * @throws {Error} If response is not ok + */ + async function getJson(url, opts = {}) { + const response = await get(url, opts); + if (!response.ok) { + throw new Error(`HTTP ${response.status} for ${url}`); + } + return response.json(); + } + + return { get, getJson }; +} + +/** + * Internal exports for unit testing. + * + * Provides access to all primitives via a single __internal namespace, + * useful when tests prefer to import via destructuring. + */ +export const __internal = { + TokenBucket, + PerHostSemaphore, + calculateBackoffMs, + parseRetryAfterMs, + isRetryableStatus, + isRetryableError, + createHostRegistry, + abortableSleep, + createHttpClient, +}; diff --git a/skills/salary-calculator/scripts/lib/benchmark-cache.mjs b/skills/salary-calculator/scripts/lib/benchmark-cache.mjs new file mode 100644 index 0000000..6b35f2e --- /dev/null +++ b/skills/salary-calculator/scripts/lib/benchmark-cache.mjs @@ -0,0 +1,384 @@ +/** + * benchmark-cache.mjs - Benchmark Cache Layer (Phase v1.0-06) + * + * Two-layer identity architecture: + * 1. `benchmark_series_id` = deterministic 16-hex SHA-256 of canonical query cohort (NULL-safe). + * Canonicalized field order is part of the identity contract — changing order or adding/removing + * fields requires a `normalizer_version`-style migration. + * + * 2. `payload_hash` = deterministic 16-hex SHA-256 of canonicalized payload alone. Participates in + * UNIQUE (benchmark_series_id, payload_hash) — drives the per-series dedupe semantics for + * INSERT OR IGNORE. + * + * 3. `benchmark_id` = deterministic 16-hex SHA-256 of the TUPLE (benchmark_series_id, payload_hash, fetched_at). + * This is the salary_benchmarks PRIMARY KEY and MUST be globally unique across all series. Hashing + * the tuple (rather than the payload alone) prevents PK collisions when two different cohorts + * happen to share the same payload JSON. + * + * SHA-256 truncated to 64 bits (16 hex chars) — accepted up to ~5M benchmarks; widening to 128-bit + * is HARDEN-01 in v2. + * + * Snapshots are immutable: never UPDATE; new payload → new row via INSERT OR IGNORE against + * UNIQUE (benchmark_series_id, payload_hash). + * + * Field order in `deriveBenchmarkSeriesId` is part of the identity contract — changing order or + * adding/removing fields requires a `normalizer_version`-style migration. + */ + +import { createHash } from 'node:crypto'; +import { NORMALIZER_VERSION } from './normalizers.mjs'; + +/** + * Helper: Compute 16-character hex SHA-256 hash of a string. + * @param {string} input + * @returns {string} 16-char lowercase hex + */ +function hashSha256Truncated(input) { + return createHash('sha256').update(input).digest('hex').slice(0, 16); +} + +/** + * Derive benchmark_series_id from query cohort (deterministic, NULL-safe). + * + * Canonical field order (NEVER reorder; mirrors RESEARCH.md Pattern 1): + * 1. normalizedTitle (lowercase) + * 2. seniority (default: '_any', lowercase) + * 3. industry (default: '_any', lowercase) + * 4. countryCode (2-letter ISO, uppercased) + * 5. region (default: '', lowercase) + * 6. city (default: '', lowercase) + * 7. compensationType (default: 'unknown', lowercase) + * 8. period (default: 'year', lowercase) + * + * @param {object} query - Benchmark query with cohort fields + * @returns {string} 16-char lowercase hex hash + */ +export function deriveBenchmarkSeriesId(query) { + const parts = [ + (query.normalizedTitle || '').toLowerCase(), + (query.seniority || '_any').toLowerCase(), + (query.industry || '_any').toLowerCase(), + (query.countryCode || '').toUpperCase(), + (query.region || '').toLowerCase(), + (query.city || '').toLowerCase(), + (query.compensationType || 'unknown').toLowerCase(), + (query.period || 'year').toLowerCase() + ]; + const canonical = parts.join('|'); + return hashSha256Truncated(canonical); +} + +/** + * Derive payload_hash from benchmark payload (deterministic, canonicalized). + * + * Canonicalize the payload BEFORE hashing to avoid key-order fragmentation: + * - Parse JSON strings; fall back to raw string on parse failure. + * - Sort top-level keys and re-serialize without extra whitespace. + * - Hash the canonical string with SHA-256; truncate to 16 hex chars. + * + * This is the canonical content-hash for the `payload_hash` column on salary_benchmarks, + * which participates in the UNIQUE (benchmark_series_id, payload_hash) constraint. + * + * @param {object|string} payload - Benchmark payload (object or JSON string) + * @returns {string} 16-char lowercase hex hash + */ +export function derivePayloadHash(payload) { + let parsed; + + if (typeof payload === 'string') { + try { + parsed = JSON.parse(payload); + } catch { + // Parse failed; hash the raw string as-is. This allows fallback for + // non-JSON payloads while still providing deterministic hashing. + return hashSha256Truncated(payload); + } + } else { + parsed = payload; + } + + // Canonicalize: sort top-level keys, serialize without extra whitespace + const canonical = JSON.stringify(parsed, Object.keys(parsed).sort()); + return hashSha256Truncated(canonical); +} + +/** + * Derive benchmark_id from (seriesId, payloadHash, fetchedAt) tuple. + * + * benchmark_id is the salary_benchmarks PRIMARY KEY and MUST be globally unique. + * We hash the (benchmarkSeriesId, payloadHash, fetchedAt) tuple instead of the payload + * alone so that two different cohorts (series) carrying the same payload JSON do NOT + * collide on the global PK. The schema enforces dedupe via UNIQUE (benchmark_series_id, payload_hash) + * separately — INSERT OR IGNORE on that constraint no-ops before the benchmark_id PK is + * even compared. + * + * @param {string} benchmarkSeriesId - Series ID (from deriveBenchmarkSeriesId) + * @param {string} payloadHash - Payload hash (from derivePayloadHash) + * @param {string|Date} fetchedAt - ISO string or Date object + * @returns {string} 16-char lowercase hex hash + * @throws {Error} if inputs are invalid + */ +export function deriveBenchmarkId(benchmarkSeriesId, payloadHash, fetchedAt) { + // Validation + if (typeof benchmarkSeriesId !== 'string' || !benchmarkSeriesId.length) { + throw new Error('benchmarkSeriesId must be a non-empty string'); + } + if (typeof payloadHash !== 'string' || !payloadHash.length) { + throw new Error('payloadHash must be a non-empty string'); + } + + // Coerce fetchedAt to ISO string + let fetchedAtIso; + if (fetchedAt instanceof Date) { + fetchedAtIso = fetchedAt.toISOString(); + } else if (typeof fetchedAt === 'string' && fetchedAt.length) { + fetchedAtIso = fetchedAt; + } else { + throw new Error('fetchedAt must be a non-empty string or Date'); + } + + // Canonical form: serialize the tuple + const canonical = `${benchmarkSeriesId}|${payloadHash}|${fetchedAtIso}`; + return hashSha256Truncated(canonical); +} + +/** + * Check if a benchmark snapshot is stale. + * + * Returns true only if now is STRICTLY AFTER the boundary (latest.fetched_at + maxAgeDays). + * At the exact boundary → false. 1ms past → true. (CACHE-05) + * + * @param {object} latest - Benchmark row with fetched_at field + * @param {number} maxAgeDays - Maximum age in days (non-negative) + * @param {string|Date} now - Current timestamp (ISO string or Date) + * @returns {boolean} true if stale, false otherwise + * @throws {Error} if inputs are invalid + */ +export function isBenchmarkStale(latest, maxAgeDays, now) { + // Validation + if (!latest || typeof latest !== 'object' || !latest.fetched_at) { + throw new Error('latest benchmark row required with fetched_at field'); + } + + if (!Number.isFinite(maxAgeDays) || maxAgeDays < 0) { + throw new Error('maxAgeDays must be a non-negative number'); + } + + if (!now) { + throw new Error('now timestamp required (ISO string or Date)'); + } + + // Parse timestamps + const fetchedTime = new Date(latest.fetched_at).getTime(); + const nowTime = now instanceof Date ? now.getTime() : new Date(now).getTime(); + + // Compute stale boundary: fetched_at + maxAgeDays * 24 * 60 * 60 * 1000 + const staleBoundary = fetchedTime + maxAgeDays * 24 * 60 * 60 * 1000; + + // True only if now is STRICTLY AFTER the boundary — at-boundary returns false (CACHE-05). + return nowTime > staleBoundary; +} + +/** + * Find the latest benchmark snapshot for a query cohort. + * + * Computes benchmark_series_id from the query (re-uses deriveBenchmarkSeriesId) and + * returns the row with the max fetched_at. Secondary deterministic sort by benchmark_id ASC + * guarantees stable ordering when two snapshots happen to share fetched_at to the millisecond + * (mirrors v1.0-05 Plan-03's observation_id ASC secondary sort rule). + * + * @param {Database} db - better-sqlite3 connection + * @param {object} query - Benchmark query with cohort fields + * @returns {object|null} - The latest benchmark row, or null if no rows exist for the series + */ +export function findLatestBenchmark(db, query) { + const benchmarkSeriesId = deriveBenchmarkSeriesId(query); + // Secondary sort by benchmark_id ASC for deterministic ordering on identical-timestamp ties + const stmt = db.prepare( + 'SELECT * FROM salary_benchmarks WHERE benchmark_series_id = ? ORDER BY fetched_at DESC, benchmark_id ASC LIMIT 1' + ); + return stmt.get(benchmarkSeriesId) || null; +} + +/** + * Insert one benchmark snapshot using INSERT OR IGNORE semantics + * (CACHE-03 idempotency against UNIQUE (benchmark_series_id, payload_hash)). + * + * PRECONDITION: Caller MUST hold the `salary_writer_lock` (acquired via + * BEGIN IMMEDIATE in writer-lock.mjs) for this `db` connection. This function + * does NOT acquire the lock — see Phase v1.0-09 orchestrator (bin/salary-cli) + * for the lock-acquisition boundary. Calling this without holding the writer + * lock risks SQLITE_BUSY under concurrent writers. + * + * Idempotent immutable snapshot insert. Same series + same payload → silent no-op + * (changes()=0, existing row retained AND RETURNED in the row field per ROADMAP SC-2). + * Same series + different payload → new row with new benchmark_id (changes()=1). + * Different series + same payload → new row with new benchmark_id (the global PK is + * derived from the tuple, not the payload alone — BLOCKER-1 fix). + * + * @param {Database} db - better-sqlite3 connection inside an active writer-lock transaction + * @param {object} benchmark - benchmark payload with required cohort fields + rawPayloadJson + * @returns {{ benchmarkId: string, inserted: boolean, changes: number, row: object }} + */ +export function storeBenchmarkSnapshot(db, benchmark) { + // Pre-flight validation — aggregate errors before any DB roundtrip + if (!benchmark || typeof benchmark !== 'object') { + throw new Error('benchmark must be a non-null object'); + } + const errors = []; + if (typeof benchmark.normalizedTitle !== 'string' || !benchmark.normalizedTitle.length) { + errors.push('normalizedTitle is required (non-empty string)'); + } + if (typeof benchmark.countryCode !== 'string' || benchmark.countryCode.length !== 2) { + errors.push('countryCode is required (exactly 2 chars)'); + } + if (benchmark.currency !== undefined && benchmark.currency !== null && benchmark.currency.length !== 3) { + errors.push('currency must be exactly 3 chars'); + } + const validPeriods = ['hour', 'day', 'week', 'month', 'year']; + if (!benchmark.period || !validPeriods.includes(benchmark.period)) { + errors.push(`period is required (one of: ${validPeriods.join(', ')})`); + } + const validCompTypes = ['base_salary', 'total_compensation', 'ote', 'contract_rate', 'unknown']; + if (!benchmark.compensationType || !validCompTypes.includes(benchmark.compensationType)) { + errors.push(`compensationType is required (one of: ${validCompTypes.join(', ')})`); + } + if (benchmark.rawPayloadJson === undefined || benchmark.rawPayloadJson === null) { + errors.push('rawPayloadJson is required (string or object)'); + } + if (!benchmark.fetchedAt) { + errors.push('fetchedAt is required (ISO string)'); + } + if (errors.length) { + throw new Error(`storeBenchmarkSnapshot validation failed: ${errors.join('; ')}`); + } + + // Compute or verify series_id + const derivedSeriesId = deriveBenchmarkSeriesId({ + normalizedTitle: benchmark.normalizedTitle, + seniority: benchmark.seniority, + industry: benchmark.industry, + countryCode: benchmark.countryCode, + region: benchmark.region, + city: benchmark.city, + compensationType: benchmark.compensationType, + period: benchmark.period + }); + let benchmarkSeriesId; + if (benchmark.benchmarkSeriesId) { + if (benchmark.benchmarkSeriesId !== derivedSeriesId) { + throw new Error('benchmark.benchmarkSeriesId disagrees with derived value'); + } + benchmarkSeriesId = benchmark.benchmarkSeriesId; + } else { + benchmarkSeriesId = derivedSeriesId; + } + + // BLOCKER-1 fix: `benchmark_id` (global PK) and `payload_hash` (per-series UNIQUE column) + // are DIFFERENT SHA-256 truncated hashes computed from DIFFERENT inputs. `payload_hash` + // hashes the canonicalized payload alone. `benchmark_id` hashes the + // (benchmark_series_id, payload_hash, fetched_at) tuple so that two different series carrying + // the same payload do not collide on the global PRIMARY KEY. + const rawPayloadString = typeof benchmark.rawPayloadJson === 'string' + ? benchmark.rawPayloadJson + : JSON.stringify(benchmark.rawPayloadJson); + const payloadHash = derivePayloadHash(benchmark.rawPayloadJson); + const benchmarkId = deriveBenchmarkId(benchmarkSeriesId, payloadHash, benchmark.fetchedAt); + + // Bind named parameters matching every column in salary_benchmarks schema (lines 137-188) + // except created_at (schema default). Use null for optional columns the caller omits. + const bindings = { + benchmark_id: benchmarkId, + benchmark_series_id: benchmarkSeriesId, + data_source: benchmark.dataSource || 'unknown', + data_source_url: benchmark.dataSourceUrl ?? null, + raw_title: benchmark.rawTitle || benchmark.normalizedTitle, + normalized_title: benchmark.normalizedTitle, + role_family: benchmark.roleFamily ?? null, + seniority: benchmark.seniority || '_any', + industry: benchmark.industry || '_any', + country_code: benchmark.countryCode, + region: benchmark.region ?? '', + city: benchmark.city ?? '', + location_raw: benchmark.locationRaw ?? null, + currency: benchmark.currency ?? 'USD', + period: benchmark.period, + compensation_type: benchmark.compensationType, + amount_min: benchmark.amountMin ?? null, + amount_max: benchmark.amountMax ?? null, + amount_median: benchmark.amountMedian ?? null, + amount_p10: benchmark.amountP10 ?? null, + amount_p25: benchmark.amountP25 ?? null, + amount_p75: benchmark.amountP75 ?? null, + amount_p90: benchmark.amountP90 ?? null, + sample_size: benchmark.sampleSize ?? null, + confidence_score: benchmark.confidenceScore ?? null, + effective_from: benchmark.effectiveFrom ?? null, + effective_to: benchmark.effectiveTo ?? null, + fetched_at: benchmark.fetchedAt, + next_refresh_at: benchmark.nextRefreshAt ?? null, + refresh_frequency_days: benchmark.refreshFrequencyDays ?? 30, + payload_hash: payloadHash, + evidence_snippet: benchmark.evidenceSnippet ?? null, + raw_payload_json: rawPayloadString, + // MEDIUM-3 fix: default sourced from the imported NORMALIZER_VERSION constant. + // NEVER hardcode `?? 1`. Callers may override (e.g., for historical replay), but the + // default is the live version from scripts/lib/normalizers.mjs. + normalizer_version: benchmark.normalizerVersion ?? NORMALIZER_VERSION + }; + + const insertSql = `INSERT OR IGNORE INTO salary_benchmarks ( + benchmark_id, benchmark_series_id, data_source, data_source_url, + raw_title, normalized_title, role_family, seniority, industry, + country_code, region, city, location_raw, + currency, period, compensation_type, + amount_min, amount_max, amount_median, + amount_p10, amount_p25, amount_p75, amount_p90, + sample_size, confidence_score, + effective_from, effective_to, fetched_at, next_refresh_at, refresh_frequency_days, + payload_hash, evidence_snippet, raw_payload_json, + normalizer_version + ) VALUES ( + @benchmark_id, @benchmark_series_id, @data_source, @data_source_url, + @raw_title, @normalized_title, @role_family, @seniority, @industry, + @country_code, @region, @city, @location_raw, + @currency, @period, @compensation_type, + @amount_min, @amount_max, @amount_median, + @amount_p10, @amount_p25, @amount_p75, @amount_p90, + @sample_size, @confidence_score, + @effective_from, @effective_to, @fetched_at, @next_refresh_at, @refresh_frequency_days, + @payload_hash, @evidence_snippet, @raw_payload_json, + @normalizer_version + )`; + + let result; + try { + result = db.prepare(insertSql).run(bindings); + } catch (err) { + if (err && err.message && err.message.includes('FOREIGN KEY')) { + throw new Error(`storeBenchmarkSnapshot FOREIGN KEY violation: ${err.message}`); + } + throw err; + } + + // BLOCKER-2 fix: populate row on BOTH branches (ROADMAP SC-2) + let row; + if (result.changes === 1) { + // Fresh insert — re-query by the freshly-known PK + row = db.prepare('SELECT * FROM salary_benchmarks WHERE benchmark_id = ?').get(benchmarkId); + } else { + // INSERT OR IGNORE no-op (CACHE-03) — re-query the existing row by (series_id, payload_hash) + row = db.prepare( + 'SELECT * FROM salary_benchmarks WHERE benchmark_series_id = ? AND payload_hash = ? LIMIT 1' + ).get(benchmarkSeriesId, payloadHash); + } + + // On the no-op path, the existing row's benchmark_id is the AUTHORITATIVE PK. + const effectiveBenchmarkId = row ? row.benchmark_id : benchmarkId; + return { + benchmarkId: effectiveBenchmarkId, + inserted: result.changes === 1, + changes: result.changes, + row + }; +} diff --git a/skills/salary-calculator/scripts/lib/ddl-normalize.mjs b/skills/salary-calculator/scripts/lib/ddl-normalize.mjs new file mode 100644 index 0000000..ab61fd7 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/ddl-normalize.mjs @@ -0,0 +1,18 @@ +/** + * Normalize a SQL DDL string for equivalence comparison. + * - Convert CRLF/CR to LF, then all whitespace runs to single space + * - Strip whitespace around (), and ; + * - Lowercase the entire string (safe because we control all identifiers — no + * user-supplied strings; no quoted identifiers in our DDL) + * - Trim + * Idempotent: normalizeDdl(normalizeDdl(x)) === normalizeDdl(x) + */ +export function normalizeDdl(sql) { + if (typeof sql !== 'string') return ''; + return sql + .replace(/\r\n?/g, '\n') + .replace(/\s+/g, ' ') + .replace(/\s*([(),;])\s*/g, '$1') + .trim() + .toLowerCase(); +} diff --git a/skills/salary-calculator/scripts/lib/drift.mjs b/skills/salary-calculator/scripts/lib/drift.mjs new file mode 100644 index 0000000..b7396ea --- /dev/null +++ b/skills/salary-calculator/scripts/lib/drift.mjs @@ -0,0 +1,38 @@ +// Drift detector — compares stored sqlite_master DDL against canonical strings +// from salary-schema.mjs. Returns Array<{name, type, expected, actual}>; +// empty array means no drift. +import { normalizeDdl } from './ddl-normalize.mjs'; +import { TABLES, INDEXES, TRIGGERS, EXPECTED_NAMES } from './salary-schema.mjs'; + +// SQLite stores trigger/table DDL stripped of `IF NOT EXISTS`. Normalize that +// modifier away on BOTH sides so equivalence comparisons don't false-positive. +const stripIfNotExists = (s) => + s.replace(/create (table|index|trigger) if not exists /, 'create $1 '); + +export function detectDrift(db) { + const expected = new Map(); + for (const [name, sql] of Object.entries(TABLES)) expected.set(name, { type: 'table', sql }); + for (const [name, sql] of Object.entries(INDEXES)) expected.set(name, { type: 'index', sql }); + for (const [name, sql] of Object.entries(TRIGGERS)) expected.set(name, { type: 'trigger', sql }); + + const placeholders = EXPECTED_NAMES.map(() => '?').join(','); + const rows = db.prepare( + `SELECT type, name, sql FROM sqlite_master WHERE name IN (${placeholders})` + ).all(...EXPECTED_NAMES); + const stored = new Map(rows.map(r => [r.name, r])); + + const diffs = []; + for (const [name, exp] of expected) { + const got = stored.get(name); + if (!got) { + diffs.push({ name, type: exp.type, expected: exp.sql, actual: null }); + continue; + } + const a = stripIfNotExists(normalizeDdl(got.sql)); + const b = stripIfNotExists(normalizeDdl(exp.sql)); + if (a !== b) { + diffs.push({ name, type: exp.type, expected: exp.sql, actual: got.sql }); + } + } + return diffs; +} diff --git a/skills/salary-calculator/scripts/lib/enrich-pipeline.mjs b/skills/salary-calculator/scripts/lib/enrich-pipeline.mjs new file mode 100644 index 0000000..9fd8511 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/enrich-pipeline.mjs @@ -0,0 +1,647 @@ +// Phase v1.0-09 Plan 04 — Pipeline orchestrator. +// +// Single export: enrichOneJob({db, source, jobId, adapterRegistry, dryRun, nowIso, httpClient}). +// Pure in-process orchestrator. Caller (Plan 05 CLI) is responsible for the +// salary_writer_lock + argv parsing + exit-code → process.exit translation. +// +// Pipeline stages (in order): +// 0. Initialize envelope skeleton (RESEARCH Pitfall #6 — partial failures still parse). +// 1. Adapter lookup (unknown source → exit 2 via envelope). +// 2. LOAD job row (not found → exit 2 via envelope). +// 3. Read prior enrichment state (defaults if absent). +// 4. EXACT pass via adapter.fetchExactSalary → classifyError → computeNextRetry → applyRetryTransition. +// 5. SELECT best from observations. +// 6. BENCHMARK pass (only when selectedBest is null OR is an estimate already) +// → findLatestBenchmark → isBenchmarkStale → benchmarkAdapter.fetchBenchmark +// → storeBenchmarkSnapshot → insert estimate observation → applyRetryTransition. +// 7. Re-read + re-select after benchmark pass. +// 8. Build envelope.state from fresh row (or projected dryRun row) + deriveExitCode. +// +// HEALTH-01: both shapeObservationFromCandidate and shapeEstimateObservation +// populate evidence_snippet + raw_payload_json non-null. +// +// EXIT CODE DERIVATION (opencode improvement #5): deriveExitCode operates on +// STRUCTURED inputs only (event.kind, transition.nextRetryAtIso) — never parses +// error message strings for status substrings. +// +// PURITY: nowIso is INJECTED by the caller; this module performs no wall-clock reads. + +import { insertObservation, getObservationsByJob } from './salary-db.mjs'; +import { selectBestObservation } from './salary-selector.mjs'; +import { + findLatestBenchmark, + storeBenchmarkSnapshot, + isBenchmarkStale, +} from './benchmark-cache.mjs'; +import { + normalizeTitle, + normalizeSeniority, + normalizeIndustry, + NORMALIZER_VERSION, +} from './normalizers.mjs'; +import { benchmarkSourcesForCountry } from './source-priority.mjs'; +import { applyRetryTransition, getEnrichmentState } from './enrichment-state.mjs'; +import { computeNextRetry, classifyError } from './retry-state-machine.mjs'; + +const EXPECTED_CURRENCY_BY_COUNTRY = Object.freeze({ + GB: 'GBP', + US: 'USD', + CH: 'CHF', + AE: 'AED', + CA: 'CAD', + AU: 'AUD', + DE: 'EUR', + FR: 'EUR', + NL: 'EUR', + IE: 'EUR', + ES: 'EUR', + IT: 'EUR', +}); + +const BENCHMARK_MAX_AGE_DAYS = 30; + +export function expectedCurrencyFor(countryCode) { + if (!countryCode) return undefined; + return EXPECTED_CURRENCY_BY_COUNTRY[countryCode]; +} + +/** + * Enrich one saved job by orchestrating exact + benchmark passes through the + * retry state machine. Returns an envelope object regardless of outcome. + * + * @param {object} args + * @param {import('better-sqlite3').Database} args.db + * @param {string} args.source + * @param {string} args.jobId + * @param {object} args.adapterRegistry - { linkedin: adapter, itjobswatch: adapter, ... } + * @param {boolean} args.dryRun + * @param {string} args.nowIso + * @param {object} [args.httpClient] + * @param {(msg:string)=>void} [args.progress] + * @param {boolean} [args.skipExact=false] - CLI-04 (--benchmark-only): skip stage 4 entirely. + * When true, the pipeline does NOT call adapter.fetchExactSalary, does NOT insert any + * is_posted_salary=1 observation, and does NOT touch exact_status. The exact axis state + * remains at its prior persisted value. + * @param {boolean} [args.forceBenchmarkRefresh=false] - CLI-03 (--refresh-benchmarks): + * bypass the isBenchmarkStale early-return; always re-fetch the benchmark and refresh + * the salary_benchmarks cache. INVARIANT: if an exact observation already exists for the + * job, the benchmark cache is refreshed BUT no estimate observation is inserted into + * job_salary_observations (enforced via the split of shouldFetchBenchmark vs + * shouldInsertEstimateObs). This invariant lives at the pipeline level so it holds + * regardless of which CLI invokes enrichOneJob. + * @returns {Promise} envelope + */ +export async function enrichOneJob({ + db, + source, + jobId, + adapterRegistry, + dryRun, + nowIso, + httpClient, + enableBenchmark = false, + progress = () => {}, + // v1.0-10 batch-mode operational flags: + skipExact = false, // CLI-04: --benchmark-only — skip stage 4 + exact insert entirely. + forceBenchmarkRefresh = false, // CLI-03: --refresh-benchmarks — bypass isBenchmarkStale. +}) { + // 0. Initialize envelope skeleton (assigned upfront — partial failures parse). + const envelope = { + job: null, + observations: [], + selectedBest: null, + benchmarkUsed: null, + state: { + exact: null, + benchmark: null, + dryRun: !!dryRun, + exitCode: 0, + }, + }; + + // 1. Adapter lookup + const adapter = adapterRegistry?.[source]; + if (!adapter) { + envelope.state.exitCode = 2; + envelope.state.exact = { + status: 'pending', + attempt_count: 0, + next_retry_at: null, + error: `unknown source: ${source}`, + last_attempt_at: null, + }; + return envelope; + } + + // 2. LOAD job + const jobRow = db + .prepare('SELECT * FROM jobs WHERE source = ? AND job_id = ?') + .get(source, jobId); + if (!jobRow) { + envelope.state.exitCode = 2; + envelope.state.exact = { + status: 'pending', + attempt_count: 0, + next_retry_at: null, + error: 'job not found', + last_attempt_at: null, + }; + return envelope; + } + envelope.job = pickJobFields(jobRow); + + // 3. Prior enrichment state (defaults if absent) + const priorState = getEnrichmentState(db, source, jobId) ?? defaultEnrichmentState(); + + // 4. EXACT pass (if adapter supports it). + // CLI-04 invariant: when skipExact=true (--benchmark-only), this entire stage + // is bypassed — no adapter.fetchExactSalary call, no is_posted_salary=1 insert, + // no applyRetryTransition('exact', ...). exactEvent stays null; the persisted + // exact_status row is untouched. Stage 5 (SELECT) still runs so prior exact + // observations are read for downstream selectedBest computation. + let exactEvent = null; + let exactTransition = null; + if (!skipExact && adapter.supports?.exactSalary) { + progress(`fetching exact: ${jobRow.url}`); + let result; + try { + result = await adapter.fetchExactSalary(jobRow, { httpClient }); + } catch (err) { + result = err; + } + exactEvent = classifyError(result); + + if (exactEvent.kind === 'success' && Array.isArray(result)) { + // Filter no-extraction markers — only real candidates persist. + const realCandidates = result.filter( + (c) => c && (c.extraction_status === null || c.extraction_status === undefined), + ); + if (realCandidates.length === 0) { + // Defensive — classifyError shouldn't have returned 'success' for marker-only arrays, + // but keep a fallthrough for robustness. + exactEvent = { kind: 'not_found' }; + exactTransition = computeNextRetry({ + axis: 'exact', + prevStatus: priorState.exact_status, + prevAttemptCount: priorState.exact_attempt_count, + event: exactEvent, + nowIso, + }); + } else { + for (const c of realCandidates) { + const obs = shapeObservationFromCandidate(c, jobRow, nowIso); + progress(`inserting observation: ${obs.confidence_label} ${obs.currency}`); + if (!dryRun) insertObservation(db, obs); + } + exactTransition = computeNextRetry({ + axis: 'exact', + prevStatus: priorState.exact_status, + prevAttemptCount: priorState.exact_attempt_count, + event: { kind: 'success' }, + nowIso, + }); + } + } else { + exactTransition = computeNextRetry({ + axis: 'exact', + prevStatus: priorState.exact_status, + prevAttemptCount: priorState.exact_attempt_count, + event: exactEvent, + nowIso, + }); + } + + if (!dryRun) { + applyRetryTransition(db, source, jobId, 'exact', exactTransition, nowIso); + } + } + + // 5. SELECT best from current observations + let observations = dryRun + ? envelope.observations + : getObservationsByJob(db, source, jobId) ?? []; + let selectedBest = selectBestObservation(observations, { + jobCountryCode: jobRow.country_code, + expectedCurrency: expectedCurrencyFor(jobRow.country_code), + }); + + // 6. BENCHMARK pass — only when no exact selected OR selectedBest is itself an estimate. + // SKIPPED when exact pass produced an error event (unrecoverable or transient): the + // CLI should not consume another adapter call when the exact axis is already in + // a non-not_found error state. Benchmark axis remains untouched (RETRY-04 axis + // independence + locked CONTEXT: 'benchmark fallback after clean not_found only'). + let benchmarkEvent = null; + let benchmarkTransition = null; + const exactInError = + exactEvent && + (exactEvent.kind === 'unrecoverable_error' || exactEvent.kind === 'transient_error'); + const exactObservationExists = + selectedBest !== null && selectedBest !== undefined && selectedBest.is_posted_salary === 1; + + // v1.0-10 flag split (CLI-03 invariant lives here): + // shouldFetchBenchmark — gates adapter.fetchBenchmark + storeBenchmarkSnapshot + retry transition. + // shouldInsertEstimateObs — gates the insertObservation(...) call that writes the is_posted_salary=0 row. + // These diverge when forceBenchmarkRefresh=true AND an exact observation already exists: + // the cache is refreshed (shouldFetchBenchmark=true) but NO estimate observation is inserted + // (shouldInsertEstimateObs=false). This enforces CLI-03 at the pipeline level — independent of CLI. + // CLI-03/CLI-06 (v1.0-10): when the operator explicitly asks for benchmark work + // (--refresh-benchmarks / --force-benchmark-retry / --benchmark-only), the + // benchmark pass must run regardless of whether the exact pass errored. The + // exact-error short-circuit is only for the implicit-fallback path (no + // selectedBest). RETRY-04 axis independence is preserved because the benchmark + // transition uses its own classifyError result. + const operatorRequestedBenchmark = forceBenchmarkRefresh || skipExact; + const shouldFetchBenchmark = + enableBenchmark && + (operatorRequestedBenchmark || !exactInError) && + ( + // Existing condition: no exact selected (or selected is itself an estimate) — need benchmark fallback. + selectedBest === null || + (selectedBest && selectedBest.is_posted_salary === 0) || + // CLI-03: explicit refresh requested even when an exact observation exists. + forceBenchmarkRefresh || + // CLI-04 defensive: benchmark-only mode (CLI will also set forceBenchmarkRefresh, but defend here). + skipExact + ); + const shouldInsertEstimateObs = shouldFetchBenchmark && !exactObservationExists; + + if (shouldFetchBenchmark) { + const benchmarkAdapters = pickBenchmarkAdapters(adapterRegistry, jobRow.country_code); + if (benchmarkAdapters.length > 0) { + const query = buildBenchmarkQuery(jobRow); + let latest = findLatestBenchmark(db, query); + // CLI-03: forceBenchmarkRefresh bypasses the cache-fresh early-return. + const stale = + !latest || forceBenchmarkRefresh || isBenchmarkStale(latest, BENCHMARK_MAX_AGE_DAYS, nowIso); + + if (stale) { + progress(`fetching benchmark cohort: ${query.normalizedTitle}`); + let bmResult; + const noDataEvents = []; + for (const benchmarkAdapter of benchmarkAdapters) { + try { + // eslint-disable-next-line no-await-in-loop + bmResult = await benchmarkAdapter.fetchBenchmark(query, { httpClient, now: new Date(nowIso) }); + } catch (err) { + bmResult = err; + } + + // A null/undefined result means the adapter had no parseable public benchmark + // for this cohort. Try the next country-priority source before deciding the + // benchmark axis outcome. This lets opportunistic sources such as SalaryExpert + // fail closed without blocking Indeed/Levels.fyi fallbacks. + if (bmResult === null || bmResult === undefined) { + noDataEvents.push(benchmarkAdapter.sourceName || 'unknown'); + bmResult = null; + continue; + } + break; + } + + if (bmResult instanceof Error) { + benchmarkEvent = classifyError(bmResult); + } else if ( + bmResult && + typeof bmResult === 'object' && + typeof bmResult.status === 'number' && + bmResult.ok === false + ) { + benchmarkEvent = classifyError(bmResult); + } else if (bmResult && typeof bmResult === 'object') { + if (!dryRun) { + // The pipeline is the authority on the cohort identity (cohort fields + // come from buildBenchmarkQuery on the canonical jobRow). If the adapter + // included a benchmarkSeriesId, drop it — storeBenchmarkSnapshot will + // derive the canonical series_id from the cohort fields, avoiding + // disagreement errors when adapters return symbolic ids. + const { benchmarkSeriesId: _ignoredSeriesId, ...adapterPayload } = bmResult; + const storeResult = storeBenchmarkSnapshot(db, adapterPayload); + latest = storeResult.row; + } else { + latest = bmResult; + } + benchmarkEvent = { kind: 'success' }; + } else { + benchmarkEvent = { + kind: 'not_found', + message: `no benchmark from adapters: ${noDataEvents.join(', ')}`, + }; + } + } else { + // cache hit — fresh enough + benchmarkEvent = { kind: 'success' }; + } + + benchmarkTransition = computeNextRetry({ + axis: 'benchmark', + prevStatus: priorState.benchmark_status, + prevAttemptCount: priorState.benchmark_attempt_count, + event: benchmarkEvent, + nowIso, + }); + + if (benchmarkEvent.kind === 'success' && latest && latest.benchmark_id) { + // CLI-03 invariant: only insert the estimate observation when no exact + // observation already exists for this job. When forceBenchmarkRefresh=true + // AND exactObservationExists=true, we refreshed the cache above but MUST NOT + // write an estimate row — selectedBest would still resolve to the exact. + if (shouldInsertEstimateObs) { + const estimateObs = shapeEstimateObservation(latest, jobRow, nowIso); + if (!dryRun) insertObservation(db, estimateObs); + envelope.benchmarkUsed = latest.benchmark_id; + } + } + + if (!dryRun) { + applyRetryTransition(db, source, jobId, 'benchmark', benchmarkTransition, nowIso); + } + } + } + + // 7. Re-read + re-select after benchmark pass + observations = dryRun ? observations : getObservationsByJob(db, source, jobId) ?? []; + selectedBest = selectBestObservation(observations, { + jobCountryCode: jobRow.country_code, + expectedCurrency: expectedCurrencyFor(jobRow.country_code), + }); + + // 8. Build state block: fresh row (live) OR projected (dryRun). + const finalState = !dryRun + ? getEnrichmentState(db, source, jobId) ?? defaultEnrichmentState() + : projectDryRunState(priorState, exactTransition, benchmarkTransition, nowIso); + + envelope.observations = observations; + envelope.selectedBest = selectedBest; + envelope.state.exact = shapeStateForEnvelope(finalState, 'exact'); + envelope.state.benchmark = shapeStateForEnvelope(finalState, 'benchmark'); + envelope.state.exitCode = deriveExitCode({ + selectedBest, + exactEvent, + exactTransition, + benchmarkEvent, + benchmarkTransition, + stateExact: envelope.state.exact, + }); + + return envelope; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function pickJobFields(row) { + return { + source: row.source, + job_id: row.job_id, + title: row.title ?? null, + company: row.company ?? null, + url: row.url ?? null, + country_code: row.country_code ?? null, + region: row.region ?? null, + city: row.city ?? null, + location_raw: row.location_raw ?? null, + }; +} + +function defaultEnrichmentState() { + return { + exact_status: 'pending', + exact_attempt_count: 0, + next_exact_retry_at: null, + exact_last_attempt_at: null, + exact_error: null, + benchmark_status: 'pending', + benchmark_attempt_count: 0, + next_benchmark_retry_at: null, + benchmark_last_attempt_at: null, + benchmark_error: null, + }; +} + +/** + * Shape a parser candidate into an observation row for insertObservation. + * HEALTH-01: evidence_snippet + raw_payload_json populated non-null. + */ +export function shapeObservationFromCandidate(candidate, job, nowIso) { + const rawPayload = + typeof candidate.raw_payload_json === 'string' + ? candidate.raw_payload_json + : JSON.stringify(candidate.raw_payload_json ?? candidate); + const evidence = + typeof candidate.evidence_snippet === 'string' && candidate.evidence_snippet.length > 0 + ? candidate.evidence_snippet + : `${candidate.currency ?? ''} ${candidate.amount_min ?? ''}-${candidate.amount_max ?? ''}`.trim(); + + return { + job_source: job.source, + job_id: job.job_id, + data_source: job.source, + data_source_url: job.url ?? null, + benchmark_id: null, + confidence_label: candidate.confidence_label ?? 'posted_exact', + matched_by: candidate.matched_by ?? 'exact_job', + is_posted_salary: 1, + is_predicted: 0, + currency: candidate.currency, + amount_min: candidate.amount_min ?? null, + amount_max: candidate.amount_max ?? null, + amount_median: candidate.amount_median ?? null, + period: candidate.period ?? 'year', + compensation_type: candidate.compensation_type ?? 'base_salary', + location_raw: job.location_raw ?? null, + country_code: job.country_code ?? null, + region: job.region ?? null, + city: job.city ?? null, + evidence_snippet: evidence, + raw_payload_json: rawPayload, + observed_at: nowIso, + }; +} + +/** + * Shape a stored benchmark row into an estimate observation tied to benchmark_id. + * HEALTH-01: evidence_snippet + raw_payload_json populated non-null. + */ +export function shapeEstimateObservation(benchmarkRow, job, nowIso) { + const rawPayload = + typeof benchmarkRow.raw_payload_json === 'string' + ? benchmarkRow.raw_payload_json + : JSON.stringify(benchmarkRow.raw_payload_json ?? benchmarkRow); + const evidence = + typeof benchmarkRow.evidence_snippet === 'string' && benchmarkRow.evidence_snippet.length > 0 + ? benchmarkRow.evidence_snippet + : `benchmark ${benchmarkRow.normalized_title ?? ''} median ${benchmarkRow.amount_median ?? '?'}`; + + return { + job_source: job.source, + job_id: job.job_id, + data_source: benchmarkRow.data_source ?? 'benchmark', + data_source_url: benchmarkRow.data_source_url ?? null, + benchmark_id: benchmarkRow.benchmark_id, + confidence_label: 'estimated_market', + matched_by: 'role_location', + is_posted_salary: 0, + is_predicted: 1, + currency: benchmarkRow.currency ?? 'USD', + amount_min: benchmarkRow.amount_min ?? null, + amount_max: benchmarkRow.amount_max ?? null, + amount_median: benchmarkRow.amount_median ?? null, + period: benchmarkRow.period ?? 'year', + compensation_type: benchmarkRow.compensation_type ?? 'base_salary', + location_raw: job.location_raw ?? null, + country_code: job.country_code ?? null, + region: job.region ?? null, + city: job.city ?? null, + evidence_snippet: evidence, + raw_payload_json: rawPayload, + observed_at: nowIso, + }; +} + +/** + * Build a benchmark query cohort from the saved job row. Applies normalizers + * (title/seniority/industry) so the derived series_id is canonical across runs. + */ +export function buildBenchmarkQuery(job) { + return { + title: job.title ?? '', + normalizedTitle: normalizeTitle(job.title ?? ''), + seniority: normalizeSeniority(job.title ?? ''), + industry: normalizeIndustry(''), + countryCode: job.country_code ?? '', + region: job.region ?? '', + city: job.city ?? '', + compensationType: 'base_salary', + period: 'year', + normalizerVersion: NORMALIZER_VERSION, + }; +} + +/** + * Pick benchmark-providing adapters for the given country in priority order. + */ +function pickBenchmarkAdapters(registry, countryCode) { + const out = []; + for (const source of benchmarkSourcesForCountry(countryCode)) { + const candidate = registry?.[source]; + if (candidate && candidate.supports?.benchmark) out.push(candidate); + } + return out; +} + +function shapeStateForEnvelope(row, axis) { + if (axis === 'exact') { + return { + status: row.exact_status, + attempt_count: row.exact_attempt_count, + next_retry_at: row.next_exact_retry_at ?? null, + last_attempt_at: row.exact_last_attempt_at ?? null, + error: row.exact_error ?? null, + }; + } + return { + status: row.benchmark_status, + attempt_count: row.benchmark_attempt_count, + next_retry_at: row.next_benchmark_retry_at ?? null, + last_attempt_at: row.benchmark_last_attempt_at ?? null, + error: row.benchmark_error ?? null, + }; +} + +/** + * Project the state row that WOULD exist after the run, without writing. + * Mirrors CLI-07 dry-run preview semantics. + */ +function projectDryRunState(prior, exactTransition, benchmarkTransition, nowIso) { + const out = { ...prior }; + if (exactTransition) { + out.exact_status = exactTransition.status; + out.exact_attempt_count = exactTransition.attemptCount; + out.next_exact_retry_at = exactTransition.nextRetryAtIso ?? null; + out.exact_last_attempt_at = nowIso; + out.exact_error = exactTransition.errorMessage ?? null; + } + if (benchmarkTransition) { + out.benchmark_status = benchmarkTransition.status; + out.benchmark_attempt_count = benchmarkTransition.attemptCount; + out.next_benchmark_retry_at = benchmarkTransition.nextRetryAtIso ?? null; + out.benchmark_last_attempt_at = nowIso; + out.benchmark_error = benchmarkTransition.errorMessage ?? null; + } + return out; +} + +/** + * Derive process exit code from STRUCTURED retry-state-machine outputs. + * + * INPUTS ARE STRUCTURED — this function MUST NOT parse error message strings + * for substrings like '401', '403', or 'HTTP'. All routing happens on: + * - event.kind (closed-set: 'success'|'not_found'|'transient_error'|'unrecoverable_error') + * - transition.nextRetryAtIso (null === exhausted/unrecoverable) + * - selectedBest (null === no usable observation) + * - stateExact.status (string from persisted/projected row) + * + * Exit code matrix (CLI-10, locked CONTEXT decision): + * 0 — success: selectedBest non-null OR clean not_found + * OR transient_error WITH a scheduled retry (recording succeeded) + * 1 — uncaught exception (defensive fall-through) + * 2 — precondition failure (handled by CLI, not here) + * 4 — lock contention (handled by CLI, not here) + * 5 — unrecoverable_error (HTTP 401/403, unknown unrecoverable) + * 6 — transient_error exhaustion (nextRetryAtIso === null after >5 attempts) + */ +export function deriveExitCode({ + selectedBest, + exactEvent, + exactTransition, + benchmarkEvent, + benchmarkTransition, + stateExact, +}) { + // Rule 1: unrecoverable errors take precedence (401/403/unknown). + if (exactEvent && exactEvent.kind === 'unrecoverable_error') return 5; + if (benchmarkEvent && benchmarkEvent.kind === 'unrecoverable_error') return 5; + + // Rule 2: transient exhaustion (transition gave up — nextRetryAtIso === null). + // The `nextRetryAtIso === null` guard MUST appear on the line immediately preceding + // every `return 6` (verify gate: grep -B 1 'return 6' | grep 'nextRetryAtIso === null' >= 2). + if (exactEvent && exactEvent.kind === 'transient_error' && exactTransition && exactTransition.nextRetryAtIso === null) + return 6; + if (benchmarkEvent && benchmarkEvent.kind === 'transient_error' && benchmarkTransition && benchmarkTransition.nextRetryAtIso === null) + return 6; + + // Rule 3: success / "did the work" outcomes. + if (selectedBest !== null && selectedBest !== undefined) return 0; + if (stateExact && stateExact.status === 'not_found') return 0; + // Clean exact 'found' even with no selectedBest in envelope (e.g. dryRun: writes + // skipped, so observations array is empty but the run did the work). + if (stateExact && stateExact.status === 'found') return 0; + // Clean benchmark 'found' even with no selectedBest in envelope (e.g. dryRun: + // benchmark cache/estimate writes are skipped, so there is no persisted + // benchmark_id-backed observation to re-select, but the adapter work succeeded). + if (benchmarkEvent && benchmarkEvent.kind === 'success') return 0; + + // Rule 4: transient_error WITH a scheduled retry — CLI recorded the failure + // and scheduled the retry. Per locked CONTEXT + CLI-01, this is a SUCCESSFUL + // run → exit 0. + if ( + exactEvent && + exactEvent.kind === 'transient_error' && + exactTransition && + exactTransition.nextRetryAtIso !== null + ) { + return 0; + } + if ( + benchmarkEvent && + benchmarkEvent.kind === 'transient_error' && + benchmarkTransition && + benchmarkTransition.nextRetryAtIso !== null + ) { + return 0; + } + + // Rule 5: defensive fall-through. + return 1; +} diff --git a/skills/salary-calculator/scripts/lib/enrichment-state.mjs b/skills/salary-calculator/scripts/lib/enrichment-state.mjs new file mode 100644 index 0000000..e288ac3 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/enrichment-state.mjs @@ -0,0 +1,194 @@ +// Enrichment state UPSERT helper for job_enrichment_state. +// +// Core responsibility: apply per-axis retry transitions (exact OR benchmark) +// to job_enrichment_state without ever clobbering the other axis (RETRY-04 +// independence). First write inserts a default row; subsequent writes UPDATE +// only the 5 columns of the named axis. +// +// PRECONDITION: All write operations (applyRetryTransition) assume the caller +// holds the `salary_writer_lock` for this `db` connection. This module does +// NOT acquire the lock — see Phase v1.0-09 orchestrator for the lock- +// acquisition boundary. Calling without the lock risks SQLITE_BUSY under +// concurrent writers. +// +// SQL design: +// * Two hardcoded UPDATE statements (one per axis) — no dynamic SQL column- +// name interpolation (injection-safe + prepared-statement-cache-friendly). +// * `updated_at` is OMITTED from every UPDATE SET clause; the +// `trg_state_updated` trigger bumps it AFTER UPDATE (single source of +// truth — RESEARCH Pitfall #4). +// * Two-step INSERT-OR-IGNORE → UPDATE rather than `ON CONFLICT DO UPDATE` +// for sqlite3-version portability and easier reasoning about idempotence. + +const VALID_AXES = new Set(['exact', 'benchmark']); +const VALID_STATUSES = new Set(['pending', 'found', 'not_found', 'error']); + +const INSERT_DEFAULT_SQL = `INSERT OR IGNORE INTO job_enrichment_state + (job_source, job_id, exact_status, exact_attempt_count, benchmark_status, benchmark_attempt_count) + VALUES (?, ?, 'pending', 0, 'pending', 0)`; + +const UPDATE_EXACT_SQL = `UPDATE job_enrichment_state + SET exact_status = ?, + exact_attempt_count = ?, + next_exact_retry_at = ?, + exact_last_attempt_at = ?, + exact_error = ? + WHERE job_source = ? AND job_id = ?`; + +const UPDATE_BENCHMARK_SQL = `UPDATE job_enrichment_state + SET benchmark_status = ?, + benchmark_attempt_count = ?, + next_benchmark_retry_at = ?, + benchmark_last_attempt_at = ?, + benchmark_error = ? + WHERE job_source = ? AND job_id = ?`; + +// forceResetAxis SQL — two hardcoded per-axis statements (no dynamic column +// interpolation, mirrors v1.0-09 Plan 03 injection-safety pattern). The +// trg_state_updated trigger bumps updated_at on UPDATE (single source of +// truth — RESEARCH Pitfall #4); these SET clauses deliberately OMIT +// updated_at. Resets touch ONLY the named axis's 5 columns; the OTHER axis +// is preserved byte-for-byte (RETRY-04 axis independence). +const RESET_EXACT_SQL = `UPDATE job_enrichment_state + SET exact_status = 'pending', + exact_attempt_count = 0, + next_exact_retry_at = NULL, + exact_error = NULL, + exact_last_attempt_at = NULL + WHERE job_source = ? AND job_id = ?`; + +const RESET_BENCHMARK_SQL = `UPDATE job_enrichment_state + SET benchmark_status = 'pending', + benchmark_attempt_count = 0, + next_benchmark_retry_at = NULL, + benchmark_error = NULL, + benchmark_last_attempt_at = NULL + WHERE job_source = ? AND job_id = ?`; + +const SELECT_STATE_SQL = `SELECT * FROM job_enrichment_state WHERE job_source = ? AND job_id = ?`; + +// Prepared-statement cache, keyed by db handle (mirrors salary-db.mjs pattern). +const stmtCache = new WeakMap(); + +function getStmts(db) { + let cached = stmtCache.get(db); + if (!cached) { + cached = { + insertDefault: db.prepare(INSERT_DEFAULT_SQL), + updateExact: db.prepare(UPDATE_EXACT_SQL), + updateBenchmark: db.prepare(UPDATE_BENCHMARK_SQL), + resetExact: db.prepare(RESET_EXACT_SQL), + resetBenchmark: db.prepare(RESET_BENCHMARK_SQL), + selectState: db.prepare(SELECT_STATE_SQL), + }; + stmtCache.set(db, cached); + } + return cached; +} + +/** + * UPSERT job_enrichment_state for (jobSource, jobId), mutating ONLY the + * named axis. The other axis's columns are preserved verbatim, which is the + * storage-layer guarantee for RETRY-04 (axis independence). + * + * PRECONDITION: caller holds the salary_writer_lock. + * + * @param {import('better-sqlite3').Database} db + * @param {string} jobSource + * @param {string} jobId + * @param {'exact'|'benchmark'} axis + * @param {{status: string, attemptCount: number, nextRetryAtIso: (string|null), errorMessage: (string|null)}} transition + * @param {string} nowIso - ISO-8601 timestamp recorded as _last_attempt_at + * @returns {{inserted: boolean, axis: string}} + */ +export function applyRetryTransition(db, jobSource, jobId, axis, transition, nowIso) { + if (!VALID_AXES.has(axis)) { + throw new TypeError(`applyRetryTransition: invalid axis '${axis}' (must be 'exact' or 'benchmark')`); + } + if (!transition || typeof transition !== 'object') { + throw new TypeError('applyRetryTransition: transition must be a non-null object'); + } + if (!VALID_STATUSES.has(transition.status)) { + throw new TypeError(`applyRetryTransition: invalid transition.status '${transition.status}' (must be one of pending|found|not_found|error)`); + } + + const stmts = getStmts(db); + + const insertResult = stmts.insertDefault.run(jobSource, jobId); + const inserted = insertResult.changes === 1; + + const stmt = axis === 'exact' ? stmts.updateExact : stmts.updateBenchmark; + stmt.run( + transition.status, + transition.attemptCount, + transition.nextRetryAtIso ?? null, + nowIso, + transition.errorMessage ?? null, + jobSource, + jobId, + ); + + return { inserted, axis }; +} + +/** + * Force-reset the named axis to its pristine ('pending', count=0, next NULL) + * state, leaving the OTHER axis byte-for-byte unchanged. Used by the batch + * CLI's `--force-exact-retry` / `--force-benchmark-retry` flags. + * + * Behavior: + * 1. INSERT OR IGNORE default row (ensures UPDATE has a target row). + * 2. UPDATE the 5 columns of the named axis to their pristine defaults: + * _status = 'pending' + * _attempt_count = 0 + * next__retry_at = NULL + * _error = NULL + * _last_attempt_at = NULL + * + * Axis independence (RETRY-04): the OTHER axis's 5 columns are NEVER named in + * the UPDATE SET clause, so they remain byte-for-byte equal pre/post — proved + * by force-reset-axis.test.mjs snapshot tests. + * + * @precondition Caller holds the salary_writer_lock for this db connection. + * @note updated_at is bumped by the trg_state_updated trigger (single source + * of truth — RESEARCH Pitfall #4). This function does NOT write + * updated_at directly. + * + * @param {import('better-sqlite3').Database} db + * @param {string} jobSource + * @param {string} jobId + * @param {'exact'|'benchmark'} axis - must be one of 'exact' or 'benchmark' + * @returns {{reset: boolean, axis: string}} + * @throws {TypeError} on invalid axis + */ +export function forceResetAxis(db, jobSource, jobId, axis) { + if (!VALID_AXES.has(axis)) { + throw new TypeError(`forceResetAxis: invalid axis '${axis}' (must be 'exact' or 'benchmark')`); + } + + const stmts = getStmts(db); + + // Ensure a row exists so the UPDATE has a target. + stmts.insertDefault.run(jobSource, jobId); + + const stmt = axis === 'exact' ? stmts.resetExact : stmts.resetBenchmark; + const result = stmt.run(jobSource, jobId); + + return { reset: result.changes === 1, axis }; +} + +/** + * Read the full job_enrichment_state row for (jobSource, jobId). Returns the + * raw snake_case row shape (or null when no row exists). Consumed by Plan 04 + * pipeline + Plan 05 CLI envelope formatter. + * + * @param {import('better-sqlite3').Database} db + * @param {string} jobSource + * @param {string} jobId + * @returns {object|null} + */ +export function getEnrichmentState(db, jobSource, jobId) { + const stmts = getStmts(db); + const row = stmts.selectState.get(jobSource, jobId); + return row ?? null; +} diff --git a/skills/salary-calculator/scripts/lib/health-metrics.mjs b/skills/salary-calculator/scripts/lib/health-metrics.mjs new file mode 100644 index 0000000..2ad806a --- /dev/null +++ b/skills/salary-calculator/scripts/lib/health-metrics.mjs @@ -0,0 +1,105 @@ +// Phase v1.0-11 Plan 02 — Pure health-metric classifier (HEALTH-02 / HEALTH-03). +// +// Single export: evaluateHealth({ current, prior }) → classifier output. +// +// Locked 4-state taxonomy (CONTEXT.md): exactly four state literals exist — +// 'ok', 'warning', 'no_data', 'insufficient_baseline'. +// There is NO degradation-state literal anywhere. When the threshold trips, +// the state literal is 'warning' (NOT a synonym). +// +// Boundary math (HEALTH-02): warning iff +// rateCurrent < ratePrior * 0.8 (STRICT less-than; 0.8 boundary is 'ok') +// +// State precedence (matters when both windows are empty): +// 1. current.total === 0 → no_data +// 2. prior.total === 0 (and current is non-empty) → insufficient_baseline +// 3. dropped === true → warning +// 4. otherwise → ok +// +// Purity discipline (mirrors retry-state-machine.mjs): +// - Zero imports. +// - No wall-clock reads. +// - No filesystem / sqlite / network I/O. +// - No module-scope mutable state. +// All temporal information arrives via the {current, prior} aggregates the +// caller computed from SQL. The classifier is referentially transparent. + +function assertWindow(name, w) { + if (!w || typeof w !== 'object') { + throw new TypeError(`evaluateHealth: ${name} must be an object with numeric total/hits`); + } + if (typeof w.total !== 'number' || !Number.isFinite(w.total)) { + throw new TypeError(`evaluateHealth: ${name}.total must be a finite number`); + } + if (typeof w.hits !== 'number' || !Number.isFinite(w.hits)) { + throw new TypeError(`evaluateHealth: ${name}.hits must be a finite number`); + } +} + +/** + * Classify a source's parser hit-rate health. + * + * @param {{current: {total:number, hits:number}, prior: {total:number, hits:number}}} args + * @returns {{ + * hit_rate_current: number|null, + * hit_rate_prior: number|null, + * delta_pct: number|null, + * warning: boolean, + * state: 'ok' | 'warning' | 'no_data' | 'insufficient_baseline' + * }} + */ +export function evaluateHealth({ current, prior } = {}) { + assertWindow('current', current); + assertWindow('prior', prior); + + // Precedence step 1: no current data → no_data (wins over insufficient_baseline). + if (current.total === 0) { + return { + hit_rate_current: null, + hit_rate_prior: null, + delta_pct: null, + warning: false, + state: 'no_data', + }; + } + + const rateCurrent = current.hits / current.total; + + // Precedence step 2: prior baseline empty → insufficient_baseline. + if (prior.total === 0) { + return { + hit_rate_current: rateCurrent, + hit_rate_prior: null, + delta_pct: null, + warning: false, + state: 'insufficient_baseline', + }; + } + + const ratePrior = prior.hits / prior.total; + + // STRICT less-than: at rateCurrent === ratePrior * 0.8 the state is 'ok'. + // + // Float-safe boundary: the natural form `rateCurrent < ratePrior * 0.8` + // misfires on the locked subtest 3 input (current=64/100, prior=80/100) + // because `0.8 * 0.8` evaluates to 0.6400000000000001 in IEEE-754, so + // `0.64 < 0.6400000000000001` is true and the state would flip from 'ok' + // to 'warning' purely due to representation drift. + // + // Rewrite as an integer cross-multiplication using the 8/10 form of 0.8: + // rateCurrent < ratePrior * 0.8 + // ↔ current.hits/current.total < (prior.hits/prior.total) * (8/10) + // ↔ current.hits * prior.total * 10 < prior.hits * current.total * 8 + // This evaluates entirely on integer operands when total/hits are integers + // (the only shape SQL produces) and is therefore exact. + const dropped = current.hits * prior.total * 10 < prior.hits * current.total * 8; + const deltaPct = ratePrior === 0 ? null : ((rateCurrent - ratePrior) / ratePrior) * 100; + + return { + hit_rate_current: rateCurrent, + hit_rate_prior: ratePrior, + delta_pct: deltaPct, + warning: dropped, + state: dropped ? 'warning' : 'ok', + }; +} diff --git a/skills/salary-calculator/scripts/lib/normalize/benchmark-stamp.mjs b/skills/salary-calculator/scripts/lib/normalize/benchmark-stamp.mjs new file mode 100644 index 0000000..6126594 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalize/benchmark-stamp.mjs @@ -0,0 +1,48 @@ +/** + * benchmark-stamp.mjs - Stamp benchmark rows with normalizer version + * + * Provides idempotent migration helper for the normalizer_version column + * and a helper function to apply the current NORMALIZER_VERSION to benchmark rows. + * + * Exports: + * - ensureNormalizerVersionColumn(db) → { added: boolean } + * - stampBenchmarkVersion(row) → row with normalizer_version added + */ + +import { NORMALIZER_VERSION } from './rules-loader.mjs'; + +/** + * Idempotently ensure salary_benchmarks has the normalizer_version column. + * Safe to call on: + * - fresh DBs created by current DDL (column already present → no-op) + * - DBs created by Phase v1.0-01 DDL (column absent → ALTER TABLE ADD COLUMN) + * + * MUST be called by the schema installer AFTER the main DDL loop runs. + * + * @param {import('better-sqlite3').Database} db + * @returns {{ added: boolean }} + */ +export function ensureNormalizerVersionColumn(db) { + const cols = db.prepare("PRAGMA table_info('salary_benchmarks')").all(); + const present = cols.some(c => c.name === 'normalizer_version'); + if (present) return { added: false }; + + // SQLite ALTER TABLE ADD COLUMN with NOT NULL requires a non-NULL DEFAULT. + // DEFAULT 1 backfills every existing row to version 1 (the only legal version at v1.0-02 ship). + db.exec(`ALTER TABLE salary_benchmarks ADD COLUMN normalizer_version INTEGER NOT NULL DEFAULT 1 CHECK (normalizer_version >= 1)`); + return { added: true }; +} + +/** + * Apply the current NORMALIZER_VERSION stamp to a benchmark row object. + * Phase v1.0-06's storeBenchmarkSnapshot() will call this immediately before INSERT. + * Idempotent: if the input already has a normalizer_version, it is OVERWRITTEN (we always + * stamp at insert time per the current loaded rules; historical inserts retain their stamp + * in the database because INSERT OR IGNORE no-ops on conflict). + * + * @param {object} row + * @returns {object} new object with normalizer_version added + */ +export function stampBenchmarkVersion(row) { + return { ...row, normalizer_version: NORMALIZER_VERSION }; +} diff --git a/skills/salary-calculator/scripts/lib/normalize/industry.mjs b/skills/salary-calculator/scripts/lib/normalize/industry.mjs new file mode 100644 index 0000000..4825ac8 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalize/industry.mjs @@ -0,0 +1,112 @@ +/** + * normalizeIndustry.mjs - Closed-list industry code detection + * + * Detection order: + * 1. Exact code match (raw is already a canonical code) + * 2. Keyword scan of rules.industryList in INSERTION ORDER (JS Map preserves insertion order) + * - Insertion order matches YAML key order in references/normalization.md + * - First keyword match wins + * 3. Fallback to _any + * + * Closed-list invariant: every return value is in INDUSTRY_CODES. + * Deterministic matching only — no approximate string matching algorithms. + */ + +import { getCachedRules } from './rules-loader.mjs'; + +export const INDUSTRY_CODES = [ + 'software', + 'finance', + 'healthcare', + 'manufacturing', + 'retail', + 'education', + 'consulting', + 'government', + 'media', + 'energy', + 'biotech', + 'telecom', + 'transportation', + 'insurance', + '_any', +]; + +let codesValidated = false; + +/** + * Validate that INDUSTRY_CODES matches the rules file's industryList keys + * Run once on first call to catch drift + * @throws if code mismatch found + */ +function validateCodesMatch() { + if (codesValidated) return; + + const rules = getCachedRules(); + const rulesKeys = Array.from(rules.industryList.keys()); + + if (rulesKeys.length !== INDUSTRY_CODES.length) { + throw new Error( + `Industry code count mismatch. Expected: ${INDUSTRY_CODES.length}, got: ${rulesKeys.length} from rules file` + ); + } + + // Check that all codes in INDUSTRY_CODES exist in rules (order doesn't have to match for this check) + const rulesSet = new Set(rulesKeys); + for (const code of INDUSTRY_CODES) { + if (!rulesSet.has(code)) { + throw new Error( + `Industry code "${code}" in INDUSTRY_CODES not found in rules file. Codes must match exactly.` + ); + } + } + + codesValidated = true; +} + +/** + * Normalize industry via closed-list code lookup + * @param {string} raw - raw industry signal (or raw title if no industry field) + * @returns {string} one of INDUSTRY_CODES + */ +export function normalizeIndustry(raw) { + if (!raw) return '_any'; + + validateCodesMatch(); + + const haystack = String(raw).normalize('NFKD').toLowerCase(); + const rules = getCachedRules(); + + // Step 1: Exact code match (raw is already a canonical code) + if (rules.industryList.has(haystack)) { + return haystack; + } + + // Step 2: Keyword scan in INSERTION ORDER (JS Map preserves order = YAML key order) + // Comment: Scan order = JS Map insertion order = YAML key order in references/normalization.md + // Reordering the industry_list in the rules file will change scan precedence and requires NORMALIZER_VERSION bump + for (const [code, keywords] of rules.industryList) { + // Skip _any during keyword scan; it's the explicit fallback + if (code === '_any') continue; + + for (const kw of keywords) { + // Use whole-word boundary to avoid spurious matches + const re = new RegExp(`(?:^|[^\\w])${escapeRegex(kw)}(?:[^\\w]|$)`, 'i'); + if (re.test(haystack)) { + return code; + } + } + } + + // Step 3: Fallback to _any + return '_any'; +} + +/** + * Escape special regex characters + * @param {string} s - string to escape + * @returns {string} escaped string safe for RegExp constructor + */ +function escapeRegex(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/skills/salary-calculator/scripts/lib/normalize/rules-loader.mjs b/skills/salary-calculator/scripts/lib/normalize/rules-loader.mjs new file mode 100644 index 0000000..8e66acc --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalize/rules-loader.mjs @@ -0,0 +1,302 @@ +/** + * rules-loader.mjs - Load, validate, and cache normalization rules + * + * Parses references/normalization.md once at first use, validates every required section, + * runs the stopword/seniority intersection assertion, and caches in memory. + * + * Exports: + * - NORMALIZER_VERSION: 1 (hardcoded constant, validated against rules file header) + * - loadRules(yamlText) → RulesData + * - loadRulesFromFile(path) → RulesData (cached) + * - getCachedRules() → RulesData (lazy-load on first call) + * - forceReloadForTests() → RulesData (test-only escape hatch) + */ + +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { parseYaml, parseSectionedMarkdown } from './yaml-mini.mjs'; + +export const NORMALIZER_VERSION = 1; + +// Module-scope cache +let cached = null; + +/** + * RulesData shape (returned to Plan 03) + * { + * compoundPhrases: Map, + * synonymMap: Map, + * seniorityTable: Array<{bucket: string, keywords: Set}>, + * industryList: Map, + * stopwords: Set, + * fillerAdjectives: Set, + * remoteMarkers: Set, + * locationTokens: Set, + * version: number, + * } + */ + +/** + * Load rules from YAML text, validate, and return RulesData + * Throws fatal errors on parse failure, missing sections, or validation violations + */ +export function loadRules(yamlText) { + const { header, sections } = parseSectionedMarkdown(yamlText); + + // Validate required sections (check this before version to catch malformed files) + const requiredSections = [ + 'compound_phrases', + 'synonym_map', + 'seniority_table', + 'industry_list', + 'stopwords', + 'filler_adjectives', + 'remote_markers', + 'locations', + ]; + for (const section of requiredSections) { + if (!(section in sections)) { + throw new Error(`rules file missing required section: ${section}`); + } + } + + // Extract version from header + const versionMatch = header.match(/NORMALIZER_VERSION:\s*(\d+)/); + if (!versionMatch) { + throw new Error('rules file missing "NORMALIZER_VERSION: " in header'); + } + + const fileVersion = parseInt(versionMatch[1], 10); + if (fileVersion !== NORMALIZER_VERSION) { + throw new Error( + `references/normalization.md version (${fileVersion}) does not match scripts/lib/normalize/rules-loader.mjs NORMALIZER_VERSION (${NORMALIZER_VERSION}). Bump one to match the other in the same commit.` + ); + } + + // Parse each section + const compoundPhrasesYaml = parseYaml(sections.compound_phrases); + const synonymMapYaml = parseYaml(sections.synonym_map); + const seniorityTableYaml = parseYaml(sections.seniority_table); + const industryListYaml = parseYaml(sections.industry_list); + const stopwordsYaml = parseYaml(sections.stopwords); + const fillerAdjectivesYaml = parseYaml(sections.filler_adjectives); + const remoteMarkersYaml = parseYaml(sections.remote_markers); + const locationsYaml = parseYaml(sections.locations); + + // Validate seniority_table structure and order + if (!Array.isArray(seniorityTableYaml)) { + throw new Error('seniority_table must be an array of {bucket, keywords} objects'); + } + + const expectedSeniorityOrder = ['cxo', 'vp', 'director', 'manager', 'lead', 'principal', 'staff', 'senior', 'mid', 'junior', 'intern', '_any']; + const actualSeniorityOrder = seniorityTableYaml.map((entry) => entry.bucket || entry); + + // Deep-equal check for order + if (actualSeniorityOrder.length !== expectedSeniorityOrder.length || !actualSeniorityOrder.every((val, idx) => val === expectedSeniorityOrder[idx])) { + throw new Error( + `seniority_table bucket order does not match expected order.\nExpected: ${expectedSeniorityOrder.join(', ')}\nActual: ${actualSeniorityOrder.join(', ')}` + ); + } + + // Validate industry_list structure + if (typeof industryListYaml !== 'object' || Array.isArray(industryListYaml)) { + throw new Error('industry_list must be an object mapping code -> keywords array'); + } + + const expectedIndustryCodes = [ + 'software', + 'finance', + 'healthcare', + 'manufacturing', + 'retail', + 'education', + 'consulting', + 'government', + 'media', + 'energy', + 'biotech', + 'telecom', + 'transportation', + 'insurance', + '_any', + ]; + const actualIndustryCodes = Object.keys(industryListYaml).sort(); + const expectedCodesSet = new Set(expectedIndustryCodes); + const actualCodesSet = new Set(actualIndustryCodes); + + const missingCodes = expectedIndustryCodes.filter((c) => !actualCodesSet.has(c)); + const extraCodes = actualIndustryCodes.filter((c) => !expectedCodesSet.has(c)); + + if (missingCodes.length > 0 || extraCodes.length > 0) { + let errMsg = 'industry_list codes mismatch.'; + if (missingCodes.length > 0) { + errMsg += ` Missing: ${missingCodes.join(', ')}.`; + } + if (extraCodes.length > 0) { + errMsg += ` Extra: ${extraCodes.join(', ')}.`; + } + throw new Error(errMsg); + } + + // Lowercase all keys and values + const compoundPhrasesMap = new Map(); + for (const [key, value] of Object.entries(compoundPhrasesYaml)) { + compoundPhrasesMap.set(key.toLowerCase(), String(value).toLowerCase()); + } + + const synonymMapObj = {}; + for (const [key, value] of Object.entries(synonymMapYaml)) { + synonymMapObj[key.toLowerCase()] = String(value).toLowerCase(); + } + + // Build seniority table with lowercased keywords + const seniorityTableArray = seniorityTableYaml.map((entry) => { + const keywordSet = new Set(); + const keywords = entry.keywords || []; + if (Array.isArray(keywords)) { + for (const kw of keywords) { + keywordSet.add(String(kw).toLowerCase()); + } + } + return { + bucket: String(entry.bucket).toLowerCase(), + keywords: keywordSet, + }; + }); + + // Build industry list with lowercased entries + const industryListMap = new Map(); + for (const [code, keywords] of Object.entries(industryListYaml)) { + const keywordArray = Array.isArray(keywords) + ? keywords.map((kw) => String(kw).toLowerCase()) + : []; + industryListMap.set(code.toLowerCase(), keywordArray); + } + + const stopwordsSet = new Set(); + if (Array.isArray(stopwordsYaml)) { + for (const word of stopwordsYaml) { + stopwordsSet.add(String(word).toLowerCase()); + } + } + + const fillerAdjectivesSet = new Set(); + if (Array.isArray(fillerAdjectivesYaml)) { + for (const adj of fillerAdjectivesYaml) { + fillerAdjectivesSet.add(String(adj).toLowerCase()); + } + } + + const remoteMarkersSet = new Set(); + if (Array.isArray(remoteMarkersYaml)) { + for (const marker of remoteMarkersYaml) { + remoteMarkersSet.add(String(marker).toLowerCase()); + } + } + + const locationTokensSet = new Set(); + if (Array.isArray(locationsYaml)) { + for (const loc of locationsYaml) { + locationTokensSet.add(String(loc).toLowerCase()); + } + } + + // CRITICAL: Intersection guard - stopwords ∩ seniority keywords + const allSeniorityKeywords = new Set(); + for (const entry of seniorityTableArray) { + for (const kw of entry.keywords) { + allSeniorityKeywords.add(kw); + } + } + + const stopwordSeniorityIntersection = Array.from(stopwordsSet).filter((token) => + allSeniorityKeywords.has(token) + ); + + if (stopwordSeniorityIntersection.length > 0) { + throw new Error( + `Stopwords and seniority keywords overlap: ${stopwordSeniorityIntersection.join(', ')}. This would cause normalizeSeniority to lose signal. Edit references/normalization.md to remove the conflict.` + ); + } + + // CRITICAL: Intersection guard - filler_adjectives ∩ seniority keywords + const fillerSeniorityIntersection = Array.from(fillerAdjectivesSet).filter((token) => + allSeniorityKeywords.has(token) + ); + + if (fillerSeniorityIntersection.length > 0) { + throw new Error( + `Filler adjectives and seniority keywords overlap: ${fillerSeniorityIntersection.join(', ')}. This would cause normalizeSeniority to lose signal. Edit references/normalization.md to remove the conflict.` + ); + } + + // Build and return RulesData + return { + compoundPhrases: compoundPhrasesMap, + synonymMap: new Map(Object.entries(synonymMapObj)), + seniorityTable: seniorityTableArray, + industryList: industryListMap, + stopwords: stopwordsSet, + fillerAdjectives: fillerAdjectivesSet, + remoteMarkers: remoteMarkersSet, + locationTokens: locationTokensSet, + version: NORMALIZER_VERSION, + }; +} + +/** + * Resolve the path to the rules file relative to the project root + */ +function resolveRulesPath(rulesPath = 'references/normalization.md') { + if (path.isAbsolute(rulesPath)) { + return rulesPath; + } + + // Find project root by walking up from this file's directory + let dir = path.dirname(fileURLToPath(import.meta.url)); + while (dir !== path.dirname(dir)) { + // Keep walking up + if (fs.existsSync(path.join(dir, 'package.json'))) { + // Found project root + return path.join(dir, rulesPath); + } + dir = path.dirname(dir); + } + + throw new Error(`Could not find project root (no package.json found). Cannot resolve rules file path: ${rulesPath}`); +} + +/** + * Load rules from file, cache the result, and return + * Subsequent calls return the same cached object + */ +export function loadRulesFromFile(rulesPath = 'references/normalization.md') { + if (cached !== null) { + return cached; + } + const resolvedPath = resolveRulesPath(rulesPath); + const text = fs.readFileSync(resolvedPath, 'utf8'); + cached = loadRules(text); + return cached; +} + +/** + * Get cached rules, loading on first call + * Lazy-load on first touch; deterministic since the file is on disk + */ +export function getCachedRules() { + if (cached === null) { + cached = loadRulesFromFile(); + } + return cached; +} + +/** + * Force reload from file (test-only escape hatch) + * Used by tests that need to reload after a fixture edit + */ +export function forceReloadForTests() { + cached = null; + return getCachedRules(); +} diff --git a/skills/salary-calculator/scripts/lib/normalize/seniority.mjs b/skills/salary-calculator/scripts/lib/normalize/seniority.mjs new file mode 100644 index 0000000..e23f58a --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalize/seniority.mjs @@ -0,0 +1,95 @@ +/** + * normalizeSeniority.mjs - First-match ordered table seniority detection + * + * Key invariant: operates on RAW input (post-NFKD + lowercase but NOT post-stopword-strip) + * This preserves seniority keywords that might otherwise be filtered by the title pipeline. + * + * Returns one of the canonical buckets: cxo, vp, director, manager, lead, principal, staff, + * senior, mid, junior, intern, _any (in that precedence order). + */ + +import { getCachedRules } from './rules-loader.mjs'; + +export const SENIORITY_BUCKETS = [ + 'cxo', + 'vp', + 'director', + 'manager', + 'lead', + 'principal', + 'staff', + 'senior', + 'mid', + 'junior', + 'intern', + '_any', +]; + +let bucketsValidated = false; + +/** + * Validate that SENIORITY_BUCKETS matches the rules file's seniority_table + * Run once on first call to catch drift + * @throws if bucket mismatch found + */ +function validateBucketsMatch() { + if (bucketsValidated) return; + + const rules = getCachedRules(); + const rulesOrder = rules.seniorityTable.map((entry) => entry.bucket); + + if (rulesOrder.length !== SENIORITY_BUCKETS.length) { + throw new Error( + `Seniority bucket count mismatch. Expected: ${SENIORITY_BUCKETS.length}, got: ${rulesOrder.length} from rules file` + ); + } + + for (let i = 0; i < SENIORITY_BUCKETS.length; i++) { + if (SENIORITY_BUCKETS[i] !== rulesOrder[i]) { + throw new Error( + `Seniority bucket order mismatch at index ${i}. Expected: ${SENIORITY_BUCKETS[i]}, got: ${rulesOrder[i]} from rules file` + ); + } + } + + bucketsValidated = true; +} + +/** + * Normalize seniority via first-match ordered table lookup + * @param {string} raw - raw seniority signal (or raw title if no seniority field) + * @returns {string} one of SENIORITY_BUCKETS + */ +export function normalizeSeniority(raw) { + if (!raw) return '_any'; + + validateBucketsMatch(); + + const haystack = String(raw).normalize('NFKD').toLowerCase(); + const rules = getCachedRules(); + + // Iterate seniority_table in order; return first bucket whose keyword matches + for (const { bucket, keywords } of rules.seniorityTable) { + // _any is the fallback — skip it in the loop, return it explicitly at the end + if (bucket === '_any') continue; + + for (const kw of keywords) { + // Use whole-word boundary: avoid 'manage' matching 'unmanageable', 'sr' matching 'rust' + const re = new RegExp(`(?:^|[^\\w])${escapeRegex(kw)}(?:[^\\w]|$)`, 'i'); + if (re.test(haystack)) { + return bucket; + } + } + } + + return '_any'; +} + +/** + * Escape special regex characters + * @param {string} s - string to escape + * @returns {string} escaped string safe for RegExp constructor + */ +function escapeRegex(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/skills/salary-calculator/scripts/lib/normalize/title.mjs b/skills/salary-calculator/scripts/lib/normalize/title.mjs new file mode 100644 index 0000000..d56be18 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalize/title.mjs @@ -0,0 +1,94 @@ +/** + * normalizeTitle.mjs - Deterministic 8-step title normalization pipeline + * + * Pipeline order (EXACT per NORM-01 + NORM-02 locked contract): + * 1. NFKD normalize + * 2. Lowercase + * 3. Compound-phrase expansion (order-preserving, longest-first to avoid prefix shadowing) + * 4. Punctuation strip (keep word chars, whitespace, hyphen, underscore) + * 5. Tokenize on whitespace + * 6. Filter (stopwords, filler adjectives, remote markers, location tokens) + * 7. Single-token synonym expansion + * 8. Join on whitespace + * + * Key invariant: Array used throughout to preserve token order per NORM-02. + * No Set, no sort on tokens. + */ + +import { getCachedRules } from './rules-loader.mjs'; + +/** + * Normalize a job title to canonical form via 8-step deterministic pipeline + * @param {string} raw - raw title input + * @returns {string} canonical title + */ +export function normalizeTitle(raw) { + if (!raw) return ''; + + const rules = getCachedRules(); + let s = String(raw); + + // Step 1: NFKD normalize (decomposes characters like é → e + combining accent) + s = s.normalize('NFKD'); + // Remove combining marks (accents, diacritics) that remain after NFKD decomposition + // Unicode combining marks are in the range \u0300-\u036F + s = s.replace(/[\u0300-\u036F]/g, ''); + + // Step 2: Lowercase + s = s.toLowerCase(); + + // Step 3: Compound-phrase expansion (before punctuation strip) + // Sort compound phrases by length descending (longest first) to avoid prefix-shadowing + // E.g., must match "machine learning engineer" before "machine learning" + const sortedPhrases = Array.from(rules.compoundPhrases.entries()) + .sort((a, b) => b[0].length - a[0].length); + + for (const [raw, canonical] of sortedPhrases) { + // Use word boundary matching to ensure we match whole phrases + // For non-symbolic phrases, use \b; for symbolic phrases (c++, c#), use negative lookbehind/lookahead + let pattern; + if (/^[a-z0-9_]+$/.test(raw)) { + // Non-symbolic phrase: use word boundary + pattern = new RegExp(`\\b${escapeRegex(raw)}\\b`, 'g'); + } else { + // Symbolic phrase (c++, c#, .net): use lookahead/lookbehind + pattern = new RegExp(`(? t.length > 0); + + // Step 6: Filter (stopwords, filler adjectives, remote markers, location tokens) + const filtered = tokens.filter((t) => { + const lower = t.toLowerCase(); + return ( + !rules.stopwords.has(lower) && + !rules.fillerAdjectives.has(lower) && + !rules.remoteMarkers.has(lower) && + !rules.locationTokens.has(lower) + ); + }); + + // Step 7: Single-token synonym expansion (Array.map preserves order) + const canonical = filtered.map((t) => { + const lower = t.toLowerCase(); + return rules.synonymMap.get(lower) ?? lower; + }); + + // Step 8: Emit (join on whitespace) + return canonical.join(' '); +} + +/** + * Escape special regex characters + * @param {string} s - string to escape + * @returns {string} escaped string safe for RegExp constructor + */ +function escapeRegex(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/skills/salary-calculator/scripts/lib/normalize/yaml-mini.mjs b/skills/salary-calculator/scripts/lib/normalize/yaml-mini.mjs new file mode 100644 index 0000000..a8a2072 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalize/yaml-mini.mjs @@ -0,0 +1,447 @@ +/** + * yaml-mini.mjs - Minimal safe YAML subset parser for rules-loader + * + * Parses a strict subset of YAML sufficient for references/normalization.md: + * - Block mappings (key: value) + * - Block sequences (- item) + * - Nested mappings (one level deep) + * - Flow sequences ([a, b, c]) + * - Comments (#) + * - Hyphenated keys + * + * NOT supported (explicitly rejected): + * - Anchors/aliases (&anchor, *ref) + * - Multi-doc (---) + * - Flow mappings ({key: value}) + * - Booleans/nulls + * + * Exports: + * - parseYaml(text) → parsed YAML object + * - parseSectionedMarkdown(text) → {header, sections} + */ + +/** + * Parse a SAFE YAML subset + * Returns: plain object with string keys, values are string | string[] | {bucket, keywords}[] | etc + * OR returns an array if the YAML is a top-level sequence + */ +export function parseYaml(text) { + const lines = text + .split('\n') + .map((line) => { + // Strip comments but preserve # in unquoted keys (like c#, c++) + // Comment detection: find # only after a value or at the start of a line with leading whitespace + // Simple heuristic: # is a comment if it appears: + // 1. At the start of line (after optional whitespace) — entire line is a comment, strip to empty + // 2. After a colon and some value + // 3. After a dash (array item) and some value + // For keys like "c#: csharp", the # is part of the key (appears before colon), so preserve it + + // FIRST: check if this is a comment-only line (# at position 0 or only whitespace before it) + const hashIdx = line.indexOf('#'); + if (hashIdx !== -1 && (hashIdx === 0 || line.substring(0, hashIdx).trim() === '')) { + // Comment-only line, strip entire thing + return ''; + } + + const colonIdx = line.indexOf(':'); + const dashIdx = line.indexOf('-'); + + if (colonIdx === -1) { + // No colon: check for array item syntax (- value # comment) + if (hashIdx === -1) return line; // No hash, return as-is + + // This is an array item with an inline comment (- value # comment) + // The dash should appear before the hash, and should be at the start (after optional whitespace) + if (dashIdx !== -1 && dashIdx < hashIdx && line.substring(0, dashIdx).trim() === '') { + // This is " - value # comment", strip the comment + return line.substring(0, hashIdx); + } + + // Otherwise preserve the line + return line; + } + + // Has colon: check for comment after the colon + if (hashIdx === -1 || hashIdx < colonIdx) return line; // No comment after colon, or hash is before colon + + const afterColon = line.substring(colonIdx + 1); + const hashIdxInValue = afterColon.indexOf('#'); + if (hashIdxInValue === -1) return line; // No comment + + // Check if # is inside quotes (simple check for quoted values) + const beforeHash = afterColon.substring(0, hashIdxInValue).trim(); + if (beforeHash.startsWith('"') || beforeHash.startsWith("'")) { + // Value is quoted; # might be inside or after quotes + // Find the closing quote + const quoteChar = beforeHash[0]; + let endQuoteIdx = beforeHash.indexOf(quoteChar, 1); + if (endQuoteIdx !== -1) { + // Quote closes before # was found, so # is a comment + return line.substring(0, colonIdx + 1 + hashIdxInValue); + } + // Quote doesn't close; # is likely inside; preserve the line + return line; + } + + // Unquoted value; # is a comment + return line.substring(0, colonIdx + 1 + hashIdxInValue); + }) + .filter((line) => line.trim().length > 0); + + // Reject unsupported YAML features + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line.includes('&') || line.includes('*')) { + throw new Error(`yaml-mini: unsupported YAML feature (anchors/aliases) at line ${i + 1}`); + } + if (line.trim() === '---') { + throw new Error(`yaml-mini: unsupported YAML feature (multi-doc) at line ${i + 1}`); + } + } + + // Detect if this is a top-level array (starts with -) + const firstLine = lines.length > 0 ? lines[0].trim() : ''; + if (firstLine.startsWith('-')) { + return parseTopLevelArray(lines); + } + + const result = {}; + let currentKey = null; + let currentValue = []; + let inArray = false; + let baseIndent = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trim(); + if (!trimmed) continue; + + const indent = line.match(/^\s*/)[0].length; + + // Detect flow sequence: [a, b, c] + if (trimmed.startsWith('[') && trimmed.includes(']')) { + const flowContent = trimmed.slice(1, -1); + const items = flowContent + .split(',') + .map((s) => s.trim()) + .map((s) => stripQuotes(s)) + .filter((s) => s.length > 0); + result[currentKey] = items; + currentKey = null; + inArray = false; + continue; + } + + // Detect key: value + if (trimmed.includes(':')) { + const colonIdx = trimmed.indexOf(':'); + const key = trimmed.substring(0, colonIdx).trim(); + const restOfLine = trimmed.substring(colonIdx + 1).trim(); + + // Commit previous key-value + if (currentKey !== null) { + if (inArray) { + result[currentKey] = currentValue; + } else { + result[currentKey] = currentValue.length === 1 ? currentValue[0] : currentValue; + } + } + + currentKey = key; + currentValue = []; + inArray = false; + baseIndent = indent; + + if (restOfLine.length > 0) { + if (restOfLine.startsWith('[')) { + // Flow array on same line + const flowContent = restOfLine.slice(1, restOfLine.includes(']') ? restOfLine.indexOf(']') : undefined); + const items = flowContent + .split(',') + .map((s) => s.trim()) + .map((s) => stripQuotes(s)) + .filter((s) => s.length > 0); + result[currentKey] = items; + currentKey = null; + inArray = false; + } else { + // Scalar value on same line + currentValue = [stripQuotes(restOfLine)]; + } + } + continue; + } + + // Detect array item: - value + if (trimmed.startsWith('-')) { + const value = trimmed.substring(1).trim(); + + // Check if this is a nested object { bucket: x, keywords: [...] } + if (value.includes(':')) { + const nestedObj = parseNestedObject(value, lines, i); + currentValue.push(nestedObj); + i = nestedObj._endLine; + inArray = true; + } else if (value.startsWith('{')) { + // Flow object (shouldn't happen in our rules, but reject it) + throw new Error(`yaml-mini: unsupported YAML feature (flow mapping) at line ${i + 1}`); + } else { + currentValue.push(stripQuotes(value)); + inArray = true; + } + continue; + } + + // If we're expecting more array items at the same or deeper indent, continue + if (inArray && indent > baseIndent) { + if (trimmed.startsWith('-')) { + const value = trimmed.substring(1).trim(); + if (value.includes(':')) { + const nestedObj = parseNestedObject(value, lines, i); + currentValue.push(nestedObj); + i = nestedObj._endLine; + } else { + currentValue.push(stripQuotes(value)); + } + } else if (trimmed.includes(':')) { + // Continuation of current array item (nested key-value) + // Parse as part of the current object + } + } + } + + // Commit final key + if (currentKey !== null) { + if (inArray) { + result[currentKey] = currentValue; + } else { + result[currentKey] = currentValue.length === 1 ? currentValue[0] : currentValue; + } + } + + return result; +} + +/** + * Parse a top-level YAML array (starting with -) + * Returns: array of objects or scalars + */ +function parseTopLevelArray(lines) { + const result = []; + let i = 0; + + while (i < lines.length) { + const line = lines[i]; + const trimmed = line.trim(); + + if (!trimmed) { + i++; + continue; + } + + if (trimmed.startsWith('-')) { + const value = trimmed.substring(1).trim(); + + if (value.includes(':')) { + // This is a nested object + const nestedObj = parseNestedObject(value, lines, i); + result.push(nestedObj); + i = nestedObj._endLine + 1; + } else if (value.length > 0) { + // Scalar value + result.push(stripQuotes(value)); + i++; + } else { + i++; + } + } else { + i++; + } + } + + return result; +} + +/** + * Parse a single nested object: bucket: ..., keywords: [...] + * Returns: { bucket, keywords, _endLine } where _endLine is the index of the last line processed + */ +function parseNestedObject(firstLine, lines, startIdx) { + const obj = {}; + const metadata = { _endLine: startIdx }; + let currentKey = null; + + // Parse the first line + const colonIdx = firstLine.indexOf(':'); + if (colonIdx > -1) { + const key = firstLine.substring(0, colonIdx).trim(); + const value = firstLine.substring(colonIdx + 1).trim(); + currentKey = key; + if (value.startsWith('[')) { + const flowContent = value.slice(1, value.includes(']') ? value.indexOf(']') : undefined); + const items = flowContent + .split(',') + .map((s) => s.trim()) + .map((s) => stripQuotes(s)) + .filter((s) => s.length > 0); + obj[currentKey] = items; + } else if (value.length > 0) { + obj[currentKey] = stripQuotes(value); + } + } + + // Continue with subsequent lines (nested keys) + for (let i = startIdx + 1; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('-')) { + // End of this nested object + metadata._endLine = i - 1; + return Object.assign(obj, metadata); + } + + if (trimmed.includes(':')) { + const colonIdx = trimmed.indexOf(':'); + const key = trimmed.substring(0, colonIdx).trim(); + const value = trimmed.substring(colonIdx + 1).trim(); + if (value.startsWith('[')) { + const flowContent = value.slice(1, value.includes(']') ? value.indexOf(']') : undefined); + const items = flowContent + .split(',') + .map((s) => s.trim()) + .map((s) => stripQuotes(s)) + .filter((s) => s.length > 0); + obj[key] = items; + } else if (value.length > 0) { + obj[key] = stripQuotes(value); + } + currentKey = key; + } + } + + metadata._endLine = lines.length - 1; + return Object.assign(obj, metadata); +} + +/** + * Strip surrounding quotes from a value + */ +function stripQuotes(value) { + value = value.trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + return value.slice(1, -1); + } + return value; +} + +/** + * Parse markdown text into header + sections + * Splits on ^## headings, extracts fenced YAML blocks under each heading + * + * Returns: { header: string, sections: { sectionName: string } } + */ +export function parseSectionedMarkdown(text) { + const lines = text.split('\n'); + const sections = {}; + let header = []; + let currentSection = null; + let inFence = false; + let fenceContent = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Detect section heading + if (line.startsWith('## ')) { + // Save previous section if any + if (currentSection && fenceContent.length > 0) { + sections[currentSection] = fenceContent.join('\n').trim(); + fenceContent = []; + } + + currentSection = line.substring(3).trim(); + inFence = false; + continue; + } + + if (currentSection === null) { + // Still in header + header.push(line); + continue; + } + + // Detect fence + if (line.trim().startsWith('```')) { + if (!inFence) { + inFence = true; + } else { + inFence = false; + // Fence is closed; save this section + if (fenceContent.length > 0) { + sections[currentSection] = fenceContent.join('\n').trim(); + fenceContent = []; + currentSection = null; // Expect next ## heading or end + } + } + continue; + } + + if (inFence) { + fenceContent.push(line); + } + } + + // Save last section if still open + if (currentSection && fenceContent.length > 0) { + sections[currentSection] = fenceContent.join('\n').trim(); + } + + return { + header: header.join('\n').trim(), + sections, + }; +} + +// Self-test (runs if this file is executed directly) +if (import.meta.url === `file://${process.argv[1]}`) { + console.log('yaml-mini self-test:'); + + // Test 1: parseYaml basic + const yaml1 = `a: 1 +b: + - x + - y`; + const result1 = parseYaml(yaml1); + console.log('Test 1 (basic):', JSON.stringify(result1) === '{"a":"1","b":["x","y"]}' ? 'PASS' : 'FAIL'); + + // Test 2: parseSectionedMarkdown + const md = `# Header + +NORMALIZER_VERSION: 1 + +## foo + +\`\`\`yaml +bar: baz +\`\`\` + +## qux + +\`\`\`yaml +zap: [1, 2] +\`\`\` +`; + const result2 = parseSectionedMarkdown(md); + console.log('Test 2 (sections):', Object.keys(result2.sections).length === 2 && result2.sections.foo && result2.sections.qux ? 'PASS' : 'FAIL'); + + // Test 3: reject anchors + try { + parseYaml('&anchor a: 1'); + console.log('Test 3 (anchor rejection): FAIL'); + } catch (e) { + console.log('Test 3 (anchor rejection):', e.message.includes('unsupported') ? 'PASS' : 'FAIL'); + } + + console.log('Self-test complete'); +} diff --git a/skills/salary-calculator/scripts/lib/normalizers.mjs b/skills/salary-calculator/scripts/lib/normalizers.mjs new file mode 100644 index 0000000..3b0e7d7 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/normalizers.mjs @@ -0,0 +1,19 @@ +/** + * normalizers.mjs - PUBLIC SURFACE + * + * Single import target for downstream phases (v1.0-06 benchmark cache, v1.0-08 adapters, etc). + * Re-exports all normalizer functions and related constants. + * + * Usage: + * ```js + * import { normalizeTitle, normalizeSeniority, normalizeIndustry } from './normalizers.mjs'; + * const canonicalTitle = normalizeTitle('Sr. AI / ML Engineer'); + * const bucket = normalizeSeniority('Senior Manager'); + * const industry = normalizeIndustry('fintech startup'); + * ``` + */ + +export { normalizeTitle } from './normalize/title.mjs'; +export { normalizeSeniority, SENIORITY_BUCKETS } from './normalize/seniority.mjs'; +export { normalizeIndustry, INDUSTRY_CODES } from './normalize/industry.mjs'; +export { NORMALIZER_VERSION, getCachedRules } from './normalize/rules-loader.mjs'; diff --git a/skills/salary-calculator/scripts/lib/observation-id.mjs b/skills/salary-calculator/scripts/lib/observation-id.mjs new file mode 100644 index 0000000..85a2e36 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/observation-id.mjs @@ -0,0 +1,44 @@ +// Identity hash for salary observations. +// SHA-256 truncated to 64 bits (16 hex chars). +// +// Field order is part of the contract — changing order or adding/removing fields +// requires a `normalizer_version`-style migration. The hash represents "same numeric +// evidence at the same URL/location" regardless of how the parser classifies or +// annualizes it (DB-07 contract). +// +// Excludes: evidence_snippet (parser-text variability must not fragment identity), +// raw_payload_json, observed_at, created_at, annualized_*, fx_*, is_posted_salary, +// is_predicted, confidence_label, matched_by, compensation_type, benchmark_id. +// These fields may vary for the same underlying evidence; identity is determined by +// numeric evidence + source + location only. + +import { createHash } from 'node:crypto'; + +/** + * Calculate a deterministic identity hash for a salary observation. + * + * Same numeric evidence + same URL + same location → same observation_id, + * regardless of evidence_snippet text or how the parser classifies the observation. + * + * @param {object} observation - Observation object (at minimum with numeric/location fields) + * @returns {string} - 16-character lowercase hex string (64-bit SHA-256 truncation) + */ +export function calculateObservationId(observation) { + // Canonical field order (critical for determinism; see contract above) + const fields = [ + (observation.currency || '').toUpperCase(), + String(observation.amount_min ?? ''), + String(observation.amount_max ?? ''), + String(observation.amount_median ?? ''), + (observation.period || '').toLowerCase(), + (observation.data_source || '').toLowerCase(), + observation.data_source_url ?? '', + (observation.country_code || '').toUpperCase(), + (observation.region || '').toLowerCase(), + (observation.city || '').toLowerCase(), + ]; + + const canonical = fields.join('|'); + const hash = createHash('sha256').update(canonical).digest('hex'); + return hash.slice(0, 16); // 64-bit truncation (16 hex chars) +} diff --git a/skills/salary-calculator/scripts/lib/parse/annualize.mjs b/skills/salary-calculator/scripts/lib/parse/annualize.mjs new file mode 100644 index 0000000..d8ec0c1 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/annualize.mjs @@ -0,0 +1,64 @@ +/** + * Annualization module with ROADMAP-locked multipliers. + * + * Applies fixed multipliers to salary amounts based on period. + * Every annualized candidate carries an annualization_note for traceability. + */ + +/** + * Frozen annualization multipliers (ROADMAP-locked, no per-job overrides). + * Maps period to factor (numeric). + */ +export const ANNUALIZATION_MULTIPLIERS = Object.freeze({ + hour: 1820, + day: 220, + week: 52, + month: 12, + year: 1 +}); + +/** + * Annualization notes map (internal, not exported). + */ +const ANNUALIZATION_NOTES = { + hour: 'hour × 1820 working hours', + day: 'day × 220 working days', + week: 'week × 52 weeks', + month: 'month × 12 months', + year: 'year × 1' +}; + +/** + * Annualize a salary candidate. + * + * Resolves the period, applies the locked multiplier, and adds annualization_note. + * Returns a new candidate object (no mutation of input). + * + * @param {object} candidate - Candidate with amount_min, amount_max, period + * @returns {object} New candidate with annualized_min, annualized_max, annualization_note added + */ +export function annualizeCandidate(candidate) { + const period = candidate.period ?? 'year'; + const factor = ANNUALIZATION_MULTIPLIERS[period]; + + if (factor === undefined) { + // Unknown period — defensive fallback (shouldn't happen if upstream is correct) + return { + ...candidate, + annualized_min: null, + annualized_max: null, + annualization_note: 'unknown period; not annualized' + }; + } + + // Apply multiplier + const annualized_min = candidate.amount_min != null ? candidate.amount_min * factor : null; + const annualized_max = candidate.amount_max != null ? candidate.amount_max * factor : null; + + return { + ...candidate, + annualized_min: annualized_min, + annualized_max: annualized_max, + annualization_note: ANNUALIZATION_NOTES[period] + }; +} diff --git a/skills/salary-calculator/scripts/lib/parse/boundary.mjs b/skills/salary-calculator/scripts/lib/parse/boundary.mjs new file mode 100644 index 0000000..befcda1 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/boundary.mjs @@ -0,0 +1,55 @@ +// LinkedIn similar-jobs section boundary detector. +// Detects the start of the "Similar jobs" or related section markers and returns the index. +// +// Caller pattern: const boundary = findSimilarJobsBoundary(text); +// const body = text.slice(0, boundary); +// extractSalaryRegex(body, ctx); +// +// Boundary detection precedes extraction to truncate the input before salary parsing. +// This prevents false positives from "Related jobs" sections with unrelated salary data. + +// Exact markers per PARSE-08 (closed set, 5 variants) +export const SIMILAR_JOBS_MARKERS = Object.freeze([ + 'Similar jobs', + 'People also viewed', + 'More searches', + 'Explore top content', + 'Show more jobs like this', +]); + +/** + * Find the index of the LinkedIn similar-jobs section boundary. + * + * @param {string} text - The full page text + * @returns {number} The index where the boundary marker starts, or text.length if not found + * + * - Matches markers case-insensitively and line-anchored (whole line only) + * - Returns the index of the first match + * - Returns text.length if no marker found (NOT -1; allows unconditional text.slice(0, idx)) + * - Pure function, idempotent + */ +export function findSimilarJobsBoundary(text) { + if (typeof text !== 'string') { + return 0; + } + + // Build a single regex from all markers + // Escape special regex characters in each marker and create alternation + const escapedMarkers = SIMILAR_JOBS_MARKERS.map((marker) => { + // Escape regex special characters + return marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + }); + + // Pattern: line-anchored, case-insensitive, markers with flexible whitespace + // ^\s* = start of line with optional leading whitespace + // (marker1|marker2|...) = one of the markers + // \s*$ = optional trailing whitespace, end of line + const pattern = new RegExp(`^\\s*(${escapedMarkers.join('|')})\\s*$`, 'igm'); + + const match = pattern.exec(text); + if (match) { + return match.index; + } + + return text.length; +} diff --git a/skills/salary-calculator/scripts/lib/parse/currency-utils.mjs b/skills/salary-calculator/scripts/lib/parse/currency-utils.mjs new file mode 100644 index 0000000..483b391 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/currency-utils.mjs @@ -0,0 +1,52 @@ +// Shared currency utilities for salary extraction. +// Used by both regex-extractor and ote-markers modules. + +export const CURRENCY_SYMBOL_MAP = Object.freeze({ + '£': 'GBP', + '$': 'USD', + '€': 'EUR', + 'CHF': 'CHF', + 'AED': 'AED' +}); + +/** + * Normalize an amount string by stripping thousands separators and k-suffix. + * + * @param {string} amountStr - The amount string to normalize + * @returns {number|null} The parsed numeric value, or null if unparseable + * + * Accepts separators: `,` (comma), `'` (apostrophe U+0027), `'` (right single quote U+2019), space. + * Accepts `.` (dot) only when followed by exactly 3 digits then a non-digit (EU thousands separator). + * Accepts `k` or `K` suffix (multiply by 1000). + * Never throws — returns null on unparseable input. + */ +export function normalizeAmount(amountStr) { + if (typeof amountStr !== 'string' || amountStr.trim().length === 0) { + return null; + } + + let normalized = amountStr.trim(); + + // Handle k/K suffix (must be done before parsing so we can detect it) + const hasKSuffix = /[kK]$/.test(normalized); + if (hasKSuffix) { + normalized = normalized.slice(0, -1).trim(); + } + + // Strip common thousands separators + // Comma, apostrophe (U+0027), right single quote (U+2019), space + normalized = normalized.replace(/,|'|'|\s/g, ''); + + // Handle dot as thousands separator (only when followed by exactly 3 digits then non-digit) + // This distinguishes EU format "1.500" (one thousand five hundred) from "1.5" (decimal) + normalized = normalized.replace(/\.(?=\d{3}(?:\D|$))/g, ''); + + // Try to parse the remaining string as a number + const num = parseFloat(normalized); + if (isNaN(num)) { + return null; + } + + // Apply k-suffix multiplier if present + return hasKSuffix ? num * 1000 : num; +} diff --git a/skills/salary-calculator/scripts/lib/parse/jsonld-extractor.mjs b/skills/salary-calculator/scripts/lib/parse/jsonld-extractor.mjs new file mode 100644 index 0000000..b75f0c5 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/jsonld-extractor.mjs @@ -0,0 +1,237 @@ +/** + * JSON-LD schema.org JobPosting salary extractor. + * + * Extracts baseSalary from JSON-LD structured data, supporting both + * nested-value and flat shapes, @graph wrappers, and ISO-8601 period normalization. + * + * Returns null (never throws) on malformed input or non-numeric amounts. + */ + +/** + * Parse JSON-LD and extract salary from schema.org JobPosting.baseSalary. + * + * @param {string} jsonLdString - JSON-LD string to parse + * @param {object} ctx - Context object (reserved for future use) + * @returns {object|null} Candidate object with amount_min, amount_max, currency, period, extractor, evidence_snippet + */ +export function parseJsonLdSalary(jsonLdString, ctx = {}) { + // 1. JSON.parse with try/catch — never throw + let parsed; + try { + parsed = JSON.parse(jsonLdString); + } catch (e) { + return null; + } + + if (!parsed || typeof parsed !== 'object') { + return null; + } + + // 2. Navigate to JobPosting + let posting = null; + + // Check for @graph array + if (Array.isArray(parsed['@graph'])) { + posting = parsed['@graph'].find((item) => { + const type = item['@type']; + if (Array.isArray(type)) { + return type.includes('JobPosting'); + } + return type === 'JobPosting'; + }); + } + + // If no @graph, check top-level + if (!posting) { + const type = parsed['@type']; + const isJobPosting = Array.isArray(type) + ? type.includes('JobPosting') + : type === 'JobPosting'; + + if (isJobPosting) { + posting = parsed; + } + } + + if (!posting) { + return null; + } + + // 3. Extract baseSalary + const baseSalary = posting.baseSalary; + if (!baseSalary) { + return null; + } + + // 4. Branch on value-shape and extract min/max + let minValue, maxValue, currency, unitText; + + // Branch 1: Flat Number at MonetaryAmount.value + if (typeof baseSalary.value === 'number' && isFinite(baseSalary.value)) { + minValue = baseSalary.value; + maxValue = baseSalary.value; + currency = baseSalary.currency; + unitText = baseSalary.unitText ?? ''; + } + // Branch 2: Nested object with flat Number at .value.value + else if (baseSalary.value && typeof baseSalary.value === 'object' && + typeof baseSalary.value.value === 'number' && isFinite(baseSalary.value.value)) { + minValue = baseSalary.value.value; + maxValue = baseSalary.value.value; + currency = baseSalary.currency ?? baseSalary.value.currency; + unitText = (baseSalary.value.unitText ?? baseSalary.unitText ?? ''); + } + // Branch 3: Nested object with QuantitativeValue range (minValue/maxValue) + else if (baseSalary.value && typeof baseSalary.value === 'object') { + const salaryValue = baseSalary.value; + currency = baseSalary.currency ?? salaryValue.currency; + unitText = (salaryValue.unitText ?? '').toUpperCase(); + + // Extract and coerce minValue + let min = salaryValue.minValue ?? null; + if (min !== null) { + if (typeof min === 'number') { + if (!isFinite(min)) { + return null; + } + minValue = min; + } else if (typeof min === 'string') { + const parsed = parseFloat(min); + if (isNaN(parsed) || !isFinite(parsed)) { + return null; + } + minValue = parsed; + } else { + return null; + } + } else { + return null; // minValue is required in range form + } + + // Extract and coerce maxValue + let max = salaryValue.maxValue ?? null; + if (max !== null) { + if (typeof max === 'number') { + if (!isFinite(max)) { + return null; + } + maxValue = max; + } else if (typeof max === 'string') { + const parsed = parseFloat(max); + if (isNaN(parsed) || !isFinite(parsed)) { + return null; + } + maxValue = parsed; + } else { + return null; + } + } + } + // Branch 4: Flat top-level shape (baseSalary itself carries minValue/maxValue) + else { + const salaryValue = baseSalary; + currency = baseSalary.currency; + unitText = (baseSalary.unitText ?? '').toUpperCase(); + + // Extract and coerce minValue + let min = salaryValue.minValue ?? null; + if (min !== null) { + if (typeof min === 'number') { + if (!isFinite(min)) { + return null; + } + minValue = min; + } else if (typeof min === 'string') { + const parsed = parseFloat(min); + if (isNaN(parsed) || !isFinite(parsed)) { + return null; + } + minValue = parsed; + } else { + return null; + } + } else { + return null; // minValue is required + } + + // Extract and coerce maxValue + let max = salaryValue.maxValue ?? null; + if (max !== null) { + if (typeof max === 'number') { + if (!isFinite(max)) { + return null; + } + maxValue = max; + } else if (typeof max === 'string') { + const parsed = parseFloat(max); + if (isNaN(parsed) || !isFinite(parsed)) { + return null; + } + maxValue = parsed; + } else { + return null; + } + } + } + + // Validate currency (applies to all branches) + if (typeof currency !== 'string' || !/^[A-Z]{3}$/.test(currency)) { + return null; + } + + // 5. Normalize period + const period = normalizePeriod(unitText.toUpperCase()); + + // 6. Build candidate + const candidate = { + amount_min: minValue, + amount_max: maxValue, + currency: currency, + period: period, + compensation_type: 'base', + extractor: 'jsonld', + evidence_snippet: 'JobPosting.baseSalary' + }; + + return candidate; +} + +/** + * Normalize period string to standard values. + * + * @param {string} unitText - Period text from JSON-LD + * @returns {string} Normalized period: 'hour', 'day', 'week', 'month', or 'year' (default) + */ +function normalizePeriod(unitText) { + if (!unitText) { + return 'year'; + } + + // Year: YEAR, P1Y, ANNUAL + if (/^(YEAR|P1Y|ANNUAL)$/.test(unitText)) { + return 'year'; + } + + // Month: MONTH, P1M + if (/^(MONTH|P1M)$/.test(unitText)) { + return 'month'; + } + + // Week: WEEK, P1W + if (/^(WEEK|P1W)$/.test(unitText)) { + return 'week'; + } + + // Day: DAY, P1D + if (/^(DAY|P1D)$/.test(unitText)) { + return 'day'; + } + + // Hour: HOUR, PT1H + if (/^(HOUR|PT1H)$/.test(unitText)) { + return 'hour'; + } + + // Unrecognized → default to year + return 'year'; +} diff --git a/skills/salary-calculator/scripts/lib/parse/merge.mjs b/skills/salary-calculator/scripts/lib/parse/merge.mjs new file mode 100644 index 0000000..97f1838 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/merge.mjs @@ -0,0 +1,35 @@ +/** + * Confidence-rank merge for salary candidates. + * + * Implements the CONTEXT.md decision: **JSON-LD always wins** when both extractors yield results. + * This is a trust-based design choice (structured data is curated; body text includes ads/editorials). + * + * Merge rule: + * 1. If jsonLdCandidate is not null (valid extraction): return [jsonLdCandidate] — suppress regex results + * 2. If jsonLdCandidate is null AND regexCandidates.length > 0: return regexCandidates + * 3. If both are empty/null: return [] — caller's responsibility to emit no-extraction marker + * + * Pure function. No I/O, no side effects. + */ + +/** + * Merge candidates with JSON-LD precedence. + * + * @param {Array} regexCandidates - Candidates from body-text regex extraction + * @param {Object|null} jsonLdCandidate - Candidate from JSON-LD extraction (or null if not found/invalid) + * @returns {Array} Merged candidates following confidence-rank rule (JSON-LD always wins) + */ +export function mergeCandidates(regexCandidates, jsonLdCandidate) { + // JSON-LD always wins when present + if (jsonLdCandidate !== null) { + return [jsonLdCandidate]; + } + + // Fall back to regex results if JSON-LD is absent/invalid + if (regexCandidates.length > 0) { + return regexCandidates; + } + + // Both empty — return empty array (caller emits marker) + return []; +} diff --git a/skills/salary-calculator/scripts/lib/parse/no-extraction.mjs b/skills/salary-calculator/scripts/lib/parse/no-extraction.mjs new file mode 100644 index 0000000..db45cbe --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/no-extraction.mjs @@ -0,0 +1,78 @@ +// No-extraction marker emission for pages where salary parsing yields no results. +// Implements closed-set reason codes: 'competitive', 'doe', 'absent'. +// +// Policy: When a page yields no salary candidates, emit an explicit marker row +// rather than silently returning []. This allows downstream (Phase v1.0-09 retry +// state machine) to distinguish "tried and found nothing" from "never tried". + +// Closed set of reasons — exactly these three, no others +export const NO_EXTRACTION_REASONS = Object.freeze([ + 'competitive', + 'doe', + 'absent', +]); + +/** + * Determine the no-extraction reason code for a page of text. + * + * @param {string} text - The page text + * @returns {string} One of 'competitive', 'doe', or 'absent' + * + * Precedence: 'competitive' > 'doe' > 'absent' + * (a page with both "competitive" and "DOE" is classified as 'competitive' + * because that signal is stronger about pay positioning) + */ +export function determineNoExtractionReason(text) { + if (typeof text !== 'string') { + return 'absent'; + } + + // Pattern for "competitive" or similar value-judgement-only phrases + const competitivePattern = /\b(competitive(\s+(salary|package|compensation))?|attractive\s+package|market[- ]leading)\b/i; + + // Pattern for "DOE" or deferral phrases + const doePattern = /(\bDOE\b|depends\s+on\s+experience|negotiable|£\s*negotiable|£\s*TBD|to\s+be\s+discussed|\bTBD\b)/i; + + const hasCompetitive = competitivePattern.test(text); + const hasDoe = doePattern.test(text); + + if (hasCompetitive) { + return 'competitive'; + } + if (hasDoe) { + return 'doe'; + } + return 'absent'; +} + +/** + * Emit a no-extraction marker row. + * + * @param {string} reason - One of 'competitive', 'doe', or 'absent' + * @param {string|null} evidence_snippet - The matched text fragment, or null for 'absent' + * @returns {object} A marker row with null salary fields and extraction_status set + * + * Throws TypeError if reason is not in NO_EXTRACTION_REASONS (defensive). + */ +export function emitNoExtractionMarker(reason, evidence_snippet) { + // Validate reason is in the closed set + if (!NO_EXTRACTION_REASONS.includes(reason)) { + throw new TypeError(`Invalid no-extraction reason: ${reason}. Must be one of: ${NO_EXTRACTION_REASONS.join(', ')}`); + } + + return { + amount_min: null, + amount_max: null, + currency: null, + period: null, + compensation_type: null, + extractor: null, // no-extraction marker has no extractor + annualized_min: null, + annualized_max: null, + annualization_note: null, + is_predicted: 0, + is_posted_salary: 0, + extraction_status: reason, + evidence_snippet, + }; +} diff --git a/skills/salary-calculator/scripts/lib/parse/ote-markers.mjs b/skills/salary-calculator/scripts/lib/parse/ote-markers.mjs new file mode 100644 index 0000000..30e9563 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/ote-markers.mjs @@ -0,0 +1,136 @@ +/** + * OTE (On-Target Earnings) marker detection and dual-candidate emission. + * + * Detects canonical OTE markers in text and extracts OTE amounts. + * Emits dual candidates (base + ote_total) when an OTE amount is parseable. + */ + +import { normalizeAmount, CURRENCY_SYMBOL_MAP } from './currency-utils.mjs'; + +/** + * Canonical OTE keywords (closed set per CONTEXT.md). + * Does NOT include scope-creep like 'bonus eligible', 'variable', 'incentive'. + */ +export const OTE_KEYWORDS = Object.freeze([ + 'OTE', + 'on-target', + 'on target earnings', + 'including commission', + 'with bonus', + 'inc. commission' +]); + +/** + * Detect OTE marker in text. + * + * @param {string} text - Text to search + * @returns {boolean} True if canonical OTE marker found + */ +export function detectOteMarker(text) { + if (typeof text !== 'string') { + return false; + } + + // Build combined regex from OTE_KEYWORDS with word boundaries + // inc. commission requires literal dot: inc\..*commission + const pattern = /\b(OTE|on-target|on target earnings|including commission|with bonus|inc\.\s+commission)\b/i; + return pattern.test(text); +} + +/** + * Extract OTE amount from text. + * + * Supports both leading-marker (OTE before amount) and trailing-marker (amount before OTE). + * Looks for an OTE keyword followed/preceded within 30 chars by a currency symbol/code and amount. + * + * @param {string} text - Text to search + * @returns {object|null} { amount: number, currency: 'GBP'|'USD'|'EUR'|'CHF'|'AED' } or null + */ +export function extractOteAmount(text) { + if (typeof text !== 'string') { + return null; + } + + // Try pattern 1: LEADING marker with symbol (OTE £150,000) + const pattern1 = /(?:OTE|on-target|on target earnings|including commission|with bonus|inc\.\s+commission)[^£$€\w]{0,30}([£$€])\s*([\d,'.]+k?)/i; + let match = text.match(pattern1); + + if (match) { + const currencySymbol = match[1]; + const amountStr = match[2]; + const currency = CURRENCY_SYMBOL_MAP[currencySymbol]; + const amount = normalizeAmount(amountStr); + if (amount !== null && currency) { + return { amount, currency }; + } + } + + // Try pattern 2: LEADING marker with currency code (OTE 180000 CHF, OTE 180000 AED) + const pattern2 = /(?:OTE|on-target|on target earnings|including commission|with bonus|inc\.\s+commission)[^£$€\w]{0,30}([\d,'.]+k?)\s+(CHF|AED)/i; + match = text.match(pattern2); + + if (match) { + const amountStr = match[1]; + const currencyCode = match[2].toUpperCase(); + const amount = normalizeAmount(amountStr); + if (amount !== null && (currencyCode === 'CHF' || currencyCode === 'AED')) { + return { amount, currency: currencyCode }; + } + } + + // Try pattern 3: TRAILING marker with symbol (£150,000 OTE) + const pattern3 = /([£$€])\s*([\d,'.]+k?)[^£$€\w]{0,30}(?:OTE|on-target|on target earnings|including commission|with bonus|inc\.\s+commission)\b/i; + match = text.match(pattern3); + + if (match) { + const currencySymbol = match[1]; + const amountStr = match[2]; + const currency = CURRENCY_SYMBOL_MAP[currencySymbol]; + const amount = normalizeAmount(amountStr); + if (amount !== null && currency) { + return { amount, currency }; + } + } + + // Try pattern 4: TRAILING marker with currency code (180000 CHF OTE, 180000 AED OTE) + const pattern4 = /([\d,'.]+k?)\s+(CHF|AED)[^£$€\w]{0,30}(?:OTE|on-target|on target earnings|including commission|with bonus|inc\.\s+commission)\b/i; + match = text.match(pattern4); + + if (match) { + const amountStr = match[1]; + const currencyCode = match[2].toUpperCase(); + const amount = normalizeAmount(amountStr); + if (amount !== null && (currencyCode === 'CHF' || currencyCode === 'AED')) { + return { amount, currency: currencyCode }; + } + } + + return null; +} + +/** + * Emit dual candidates (base + ote_total) when OTE amount is provided. + * + * @param {object} baseCandidate - Base salary candidate from regex-extractor + * @param {object|null} oteAmount - OTE amount { amount, currency } or null + * @returns {array} [baseCandidate] or [baseCandidate, oteCandidate] + */ +export function emitDualCandidates(baseCandidate, oteAmount) { + if (oteAmount === null) { + // Only marker detected, no parseable OTE amount + return [{ ...baseCandidate, compensation_type: 'base' }]; + } + + // OTE amount is available — emit two candidates + const baseWithType = { ...baseCandidate, compensation_type: 'base' }; + const oteCandidate = { + ...baseCandidate, + amount_min: oteAmount.amount, + amount_max: null, + currency: oteAmount.currency, + compensation_type: 'ote_total' + // Inherit period, extractor from baseCandidate + }; + + return [baseWithType, oteCandidate]; +} diff --git a/skills/salary-calculator/scripts/lib/parse/regex-extractor.mjs b/skills/salary-calculator/scripts/lib/parse/regex-extractor.mjs new file mode 100644 index 0000000..05bbc97 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/regex-extractor.mjs @@ -0,0 +1,298 @@ +// Body-text salary extraction via regex patterns for multiple currencies and periods. +// Imports shared utilities from currency-utils (Plan 01 Wave 0). +// Pure function: no I/O, no throws, no side effects. + +import { CURRENCY_SYMBOL_MAP, normalizeAmount } from './currency-utils.mjs'; + +/** + * Extract salary candidates from body text via regex patterns. + * + * @param {string} text - The body text to extract from + * @param {object} ctx - Context object (unused in Plan 02, but reserved for future use) + * @returns {object[]} Array of candidates, each with: + * - amount_min, amount_max (number | null) + * - currency (string, ISO 3-letter) + * - period ('hour'|'day'|'week'|'month'|'year') + * - compensation_type ('base') + * - extractor ('regex') + * - evidence_snippet (matched substring) + */ +export function extractSalaryRegex(text, ctx = {}) { + if (typeof text !== 'string' || text.length === 0) { + return []; + } + + const candidates = []; + const usedRanges = []; // Track [start, end) to avoid overlaps + + // Helper: check if a range overlaps with any already-used range + const hasOverlap = (start, end) => { + return usedRanges.some(([s, e]) => start < e && end > s); + }; + + // Helper: record a range as used + const addUsedRange = (start, end) => { + usedRanges.push([start, end]); + }; + + // Helper: detect if match position is inside an HTML tag + const isInsideHtmlTag = (index) => { + let openCount = 0; + for (let i = 0; i < index; i++) { + if (text[i] === '<') openCount++; + if (text[i] === '>') openCount--; + } + return openCount > 0; + }; + + // Helper: check if match is preceded by :// (URL protocol) + const isPrecededByUrl = (index) => { + const start = Math.max(0, index - 30); + const before = text.substring(start, index); + return before.includes('://'); + }; + + // Helper: extract period from regex match groups + const extractPeriod = (match) => { + const periodText = match.period || match.periodAfter || ''; + const lowerPeriod = periodText.toLowerCase(); + + if (lowerPeriod.includes('day')) return 'day'; + if (lowerPeriod.includes('week')) return 'week'; + if (lowerPeriod.includes('month') || lowerPeriod.includes('monthly')) return 'month'; + if (lowerPeriod.includes('hour') || lowerPeriod.includes('hourly')) return 'hour'; + if (lowerPeriod.includes('year') || lowerPeriod.includes('annum') || lowerPeriod.includes('annually')) return 'year'; + + return 'year'; // Default + }; + + // ===== GBP Patterns ===== + + // Pattern 1: GBP yearly range with optional period specifier + const gbpYearlyRegex = /£([\d,'.]+)\s*(-|to|–|—)\s*£([\d,'.]+)(?:\s*(per\s+annum|annually|per\s+year|\/year))?/gi; + for (const match of text.matchAll(gbpYearlyRegex)) { + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const minStr = match[1]; + const maxStr = match[3]; + const min = normalizeAmount(minStr); + const max = normalizeAmount(maxStr); + + if (min !== null && max !== null) { + candidates.push({ + amount_min: min, + amount_max: max, + currency: 'GBP', + period: 'year', + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0], + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + // Pattern 2: GBP single yearly amount (no range, no period specifier, defaults to year) + // Matches: "£120,000" in isolation (not part of a range like "£100 - £120") + // This pattern looks for £ followed by amount, not followed by " - " or similar range markers + const gbpSingleRegex = /£([\d,'.]+)(?=\s*[.;,!?\s]|$)/gi; + for (const match of text.matchAll(gbpSingleRegex)) { + // Skip if this is already part of a range (previous or next match within 10 chars is also £) + const precedingText = text.substring(Math.max(0, match.index - 10), match.index); + const followingText = text.substring(match.index + match[0].length, match.index + match[0].length + 20); + + // Skip if part of a range (has another £ nearby with - or dash) + if (/£[\s\d,'.]*-|–|—|£[\s\d,'.]*to/.test(precedingText + match[0] + followingText)) { + continue; + } + + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const amountStr = match[1]; + const amount = normalizeAmount(amountStr); + + if (amount !== null && amount > 100) { + // Only treat as yearly if amount is reasonable (> 100, to avoid typos like "£1") + candidates.push({ + amount_min: amount, + amount_max: null, + currency: 'GBP', + period: 'year', + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0].trim(), // Use the actual matched text (without trailing whitespace/punctuation from lookahead) + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + // Pattern 3: GBP day rate + const gbpDayRateRegex = /£([\d,'.]+)(?:\s*\/day|\/day)/gi; + for (const match of text.matchAll(gbpDayRateRegex)) { + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const amountStr = match[1]; + const amount = normalizeAmount(amountStr); + + if (amount !== null) { + candidates.push({ + amount_min: amount, + amount_max: null, + currency: 'GBP', + period: 'day', + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0], + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + // ===== CHF Patterns ===== + + const chfRegex = /CHF\s+([\d'.]+)\s*(-|to|–|—)\s*([\d'.]+)(?:\s*(per\s+year|per\s+annum|annually))?/gi; + for (const match of text.matchAll(chfRegex)) { + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const minStr = match[1]; + const maxStr = match[3]; + const min = normalizeAmount(minStr); + const max = normalizeAmount(maxStr); + + if (min !== null && max !== null) { + candidates.push({ + amount_min: min, + amount_max: max, + currency: 'CHF', + period: 'year', + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0], + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + // ===== AED Patterns ===== + + // AED single-value with period (most common: /month) + const aedRegex = /AED\s+([\d,]+)(?:\s*\/\s*(month|year|week|day|hour))?(?:\s+[a-z]+)?/gi; + for (const match of text.matchAll(aedRegex)) { + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const amountStr = match[1]; + const amount = normalizeAmount(amountStr); + const periodText = match[2] || 'month'; // Default to month for AED + + if (amount !== null) { + let period = 'year'; + if (periodText) { + const p = periodText.toLowerCase(); + if (p.includes('day')) period = 'day'; + else if (p.includes('week')) period = 'week'; + else if (p.includes('month')) period = 'month'; + else if (p.includes('hour')) period = 'hour'; + else period = 'year'; + } + + candidates.push({ + amount_min: amount, + amount_max: null, + currency: 'AED', + period, + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0], + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + // ===== Currency-with-period patterns (GBP, USD, EUR, etc. with explicit period) ===== + + // Pattern for any currency symbol with amount and period (£50/hour, $750/day, €2,500/week, etc.) + const currencyPeriodRegex = /([£$€])([\d,'.]+)(?:\s*\/\s*(hour|day|week|month))/gi; + for (const match of text.matchAll(currencyPeriodRegex)) { + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const currencySymbol = match[1]; + const amountStr = match[2]; + const periodText = match[3]; + const currency = CURRENCY_SYMBOL_MAP[currencySymbol]; + const amount = normalizeAmount(amountStr); + + if (amount !== null && currency) { + candidates.push({ + amount_min: amount, + amount_max: null, + currency: currency, + period: periodText.toLowerCase(), + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0], + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + // ===== USD Patterns ===== + + // USD range with optional k-shorthand + const usdRegex = /\$([\d,]+)[kK]?\s*(-|to|–|—)\s*\$([\d,]+)[kK]?(?:\s*(per\s+year|per\s+annum|annually|\/year))?/gi; + for (const match of text.matchAll(usdRegex)) { + if (hasOverlap(match.index, match.index + match[0].length)) continue; + if (isInsideHtmlTag(match.index) || isPrecededByUrl(match.index)) continue; + + const minStr = match[1]; + const maxStr = match[3]; + + // Check if either min or max has k-suffix in the original match + const minWithK = /\$([\d,]+)[kK]/.exec(text.substring(match.index, match.index + match[0].length).split('-')[0]); + const maxWithK = /\$([\d,]+)[kK]/.exec(text.substring(match.index, match.index + match[0].length).split('-')[1] || ''); + + // Try to parse both min and max + let min = normalizeAmount(minStr); + let max = normalizeAmount(maxStr); + + // If original had k-suffix and normalized lost it, reapply + if (minWithK && /[kK]$/.test(minStr)) { + min = normalizeAmount(minStr); + } + if (maxWithK && /[kK]$/.test(maxStr)) { + max = normalizeAmount(maxStr); + } + + // Apply k-suffix if the original match had it + if (min !== null && /\$[\d,]+[kK]/.test(match[0].split('-')[0])) { + min = parseFloat(minStr) * 1000; + } + if (max !== null && /[kK]\s*(-|to)/.test(match[0])) { + // Check if the max part has k + const maxPart = match[0].split(/-|to|–|—/)[1] || ''; + if (/[kK]/.test(maxPart)) { + max = parseFloat(maxStr) * 1000; + } + } + + if (min !== null && max !== null) { + candidates.push({ + amount_min: min, + amount_max: max, + currency: 'USD', + period: 'year', + compensation_type: 'base', + extractor: 'regex', + evidence_snippet: match[0], + }); + addUsedRange(match.index, match.index + match[0].length); + } + } + + return candidates; +} diff --git a/skills/salary-calculator/scripts/lib/parse/salary-parser.mjs b/skills/salary-calculator/scripts/lib/parse/salary-parser.mjs new file mode 100644 index 0000000..b0e86e6 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/parse/salary-parser.mjs @@ -0,0 +1,204 @@ +/** + * Salary Parser: Public Entry Point + * + * Composes all leaf modules into a unified salary extraction pipeline. + * This is the single public API for downstream consumers (Phase v1.0-05 DB layer, Phase v1.0-08 adapters). + * + * Function signature: + * ``` + * parseSalaryCandidates(text: string | any, ctx: object): Candidate[] + * ``` + * + * ctx object fields (all optional): + * - `jsonLd`: JSON-LD string to extract from (if provided, JSON-LD always wins per CONTEXT.md) + * - `defaultCurrency`: Fallback currency (reserved for future use) + * - `jobUrl`: Source URL (reserved for audit/debugging) + * + * Output candidate shape (12 fields): + * - `amount_min`: number | null + * - `amount_max`: number | null + * - `currency`: string ('GBP', 'USD', 'CHF', 'AED', etc.) | null + * - `period`: string ('hour', 'day', 'week', 'month', 'year') | null + * - `compensation_type`: 'base' | 'ote_total' | null + * - `extractor`: 'jsonld' | 'regex' + * - `annualized_min`: number | null + * - `annualized_max`: number | null + * - `annualization_note`: string | null + * - `is_predicted`: 0 | 1 + * - `is_posted_salary`: 0 | 1 + * - `extraction_status`: null (for candidates) | 'competitive' | 'doe' | 'absent' (for markers) + * - `evidence_snippet`: string | null + * + * Pipeline: + * 1. Input normalization (empty/null/non-string → '') + * 2. JSON-LD extraction (if ctx.jsonLd provided) + * 3. Boundary truncation (LinkedIn similar-jobs section) + * 4. Body-text regex extraction + * 5. OTE dual-emission (on OTE marker detection + amount) + * 6. Merge (JSON-LD precedence) + * 7. No-extraction fallback (explicit markers for pages with no salary data) + * 8. Annualization (every candidate carries annualization_note) + * 9. Return candidates with provenance invariants enforced + * + * See CONTEXT.md for locked decisions: + * - JSON-LD always wins when both extractors yield results (confidence-rank merge) + * - OTE pages emit two candidates: base + ote_total (when OTE amount is parseable) + * - No-extraction markers use closed-set reason codes: {competitive, doe, absent} + * - Every annualized candidate carries annualization_note for traceability + */ + +import { extractSalaryRegex } from './regex-extractor.mjs'; +import { parseJsonLdSalary } from './jsonld-extractor.mjs'; +import { findSimilarJobsBoundary } from './boundary.mjs'; +import { detectOteMarker, extractOteAmount, emitDualCandidates } from './ote-markers.mjs'; +import { annualizeCandidate } from './annualize.mjs'; +import { determineNoExtractionReason, emitNoExtractionMarker } from './no-extraction.mjs'; +import { mergeCandidates } from './merge.mjs'; + +/** + * Parse salary from body text and/or JSON-LD, returning extraction candidates or no-extraction marker. + * + * @param {string|any} text - Job description body text (normalized to string, may be empty) + * @param {object} ctx - Context object with optional jsonLd, defaultCurrency, jobUrl + * @returns {Array} Array of candidates (extraction results) or exactly one no-extraction marker + * + * Pure function: no I/O, no side effects, deterministic. + * Returns at least one row (never silently empty). + */ +export function parseSalaryCandidates(text, ctx = {}) { + // ============================================================================ + // 1. INPUT NORMALIZATION (CRITICAL — supports JSON-LD-only inputs) + // ============================================================================ + // If text is not a string (null, undefined, number, etc.) or is empty, set text = '' + // and CONTINUE. A JSON-LD-only call MUST still run the JSON-LD pass. + // Never throw on bad input types. + if (typeof text !== 'string') { + text = ''; + } else if (text.length === 0) { + text = ''; + } + + // ============================================================================ + // 2. JSON-LD PASS (runs regardless of whether text is empty) + // ============================================================================ + let jsonLdCandidate = null; + if (typeof ctx.jsonLd === 'string' && ctx.jsonLd.length > 0) { + jsonLdCandidate = parseJsonLdSalary(ctx.jsonLd, ctx); + } + + // ============================================================================ + // 3. BOUNDARY TRUNCATION (safe on empty string) + // ============================================================================ + const boundary = findSimilarJobsBoundary(text); + const bodyText = text.slice(0, boundary); + + // ============================================================================ + // 4. BODY-TEXT REGEX PASS + // ============================================================================ + const rawRegexCandidates = extractSalaryRegex(bodyText, ctx); + + // ============================================================================ + // 5. OTE DUAL-EMISSION (only applies to regex candidates) + // ============================================================================ + let regexCandidates = []; + if (rawRegexCandidates.length === 0) { + regexCandidates = []; + } else if (!detectOteMarker(bodyText)) { + // No OTE marker: use all regex candidates as-is + regexCandidates = rawRegexCandidates; + } else { + // OTE marker present: emit dual candidates from the first regex match + const oteAmount = extractOteAmount(bodyText); + const dualCandidates = emitDualCandidates(rawRegexCandidates[0], oteAmount); + + // If we have two candidates (base + ote_total), extract the OTE amount's evidence_snippet + if (dualCandidates.length === 2 && oteAmount !== null) { + // Extract evidence snippet by finding the OTE amount in the original text + // Look for currency symbol + amount pattern that matches the OTE currency and amount + const currencySymbol = { + 'GBP': '£', + 'USD': '$', + 'EUR': '€' + }[oteAmount.currency]; + + if (currencySymbol) { + // Build regex to find "£170,000" or similar with thousands separators + // The amount could have various separators: comma, apostrophe, etc. + const amountPattern = oteAmount.amount.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ','); + const regex = new RegExp(`${currencySymbol}\\s*[\\d,'.]+`, 'g'); + let match; + let lastMatch = null; + while ((match = regex.exec(bodyText)) !== null) { + // Keep finding matches to get the last (OTE) amount + // (Skip the first match which should be the base salary) + lastMatch = match; + } + if (lastMatch && lastMatch[0] !== rawRegexCandidates[0].evidence_snippet) { + dualCandidates[1].evidence_snippet = lastMatch[0]; + } + } + } + + regexCandidates = dualCandidates; + } + + // ============================================================================ + // 6. MERGE (apply JSON-LD precedence) + // ============================================================================ + const merged = mergeCandidates(regexCandidates, jsonLdCandidate); + + // ============================================================================ + // 7. NO-EXTRACTION FALLBACK + // ============================================================================ + let candidates = []; + if (merged.length === 0) { + // Both extractors yielded nothing — emit a marker with evidence snippet + const reason = determineNoExtractionReason(text); + + // Extract evidence snippet based on the reason + let evidence_snippet = null; + if (reason === 'competitive') { + const match = /\b(competitive(\s+(salary|package|compensation))?|attractive\s+package|market[- ]leading)\b/i.exec(text); + evidence_snippet = match ? match[0] : null; + } else if (reason === 'doe') { + const match = /(\bDOE\b|depends\s+on\s+experience|negotiable|£\s*negotiable|£\s*TBD|to\s+be\s+discussed|\bTBD\b)/i.exec(text); + evidence_snippet = match ? match[0] : null; + } + // For 'absent', evidence_snippet remains null + + candidates = [emitNoExtractionMarker(reason, evidence_snippet)]; + } else { + candidates = merged; + } + + // ============================================================================ + // 8. ANNUALIZATION AND PROVENANCE INVARIANTS + // ============================================================================ + // Annualize every extraction candidate (NOT markers) + const result = candidates.map((c) => { + // Extraction candidates don't have extraction_status field; markers do (and it's not null) + const isMarker = c.extraction_status !== undefined && c.extraction_status !== null; + + if (!isMarker) { + // This is an extraction candidate — annualize and add required fields + const annualized = annualizeCandidate(c); + return { + ...annualized, + is_predicted: 0, + is_posted_salary: 1, + extraction_status: null + }; + } + // Markers pass through unchanged (no annualization on markers) + return c; + }); + + // ============================================================================ + // 9. ASSERT PROVENANCE INVARIANTS (development guards) + // ============================================================================ + // Every extraction candidate has required fields + // Every marker has required fields + // (Production: no-op; development: guards can be enabled via simple assertions) + + return result; +} diff --git a/skills/salary-calculator/scripts/lib/preflight.mjs b/skills/salary-calculator/scripts/lib/preflight.mjs new file mode 100644 index 0000000..9cae30c --- /dev/null +++ b/skills/salary-calculator/scripts/lib/preflight.mjs @@ -0,0 +1,82 @@ +// Preflight assertions for ensure-salary-schema. Pure side-effect-free except for throwing. +// Every Error thrown here carries `.exitCode = 2` so the CLI can map directly to its +// categorized exit code. +import { statSync } from 'node:fs'; + +function preflightError(message, exitCode = 2) { + const e = new Error(message); + e.exitCode = exitCode; + e.kind = 'preflight'; + return e; +} + +export function assertDbFileExists(path) { + try { + const st = statSync(path); + if (!st.isFile()) { + throw preflightError( + `jobhunter.sqlite not found at ${path}. The job-hunter project owns DB creation; run its setup first.` + ); + } + } catch (e) { + if (e.kind === 'preflight') throw e; + if (e.code === 'ENOENT') { + throw preflightError( + `jobhunter.sqlite not found at ${path}. The job-hunter project owns DB creation; run its setup first.` + ); + } + throw e; + } +} + +/** + * Verify PRAGMA foreign_keys is ON for this connection. + * + * IMPORTANT — explicit happy-path semantics: + * better-sqlite3 ≥ 7.0 enables `PRAGMA foreign_keys = ON` AUTOMATICALLY when + * opening via `new Database(path)` (default open mode). We READ the pragma but + * DELIBERATELY DO NOT SET it. SCHEMA-03 ("refuses if foreign_keys is off") + * becomes unverifiable if the installer flips the bit on the user's behalf. + */ +export function assertForeignKeysOn(db) { + const v = db.pragma('foreign_keys', { simple: true }); + if (v !== 1) { + throw preflightError( + `PRAGMA foreign_keys is OFF on this connection (got ${v}). The installer reads but does not set this pragma; better-sqlite3 enables it by default. Open the DB with foreign_keys = ON before re-running.` + ); + } +} + +export function hasUniqueOnSourceJobId(db) { + const tbl = db.prepare( + `SELECT name FROM sqlite_master WHERE type='table' AND name='jobs'` + ).get(); + if (!tbl) return { exists: false, unique: false }; + + const indexes = db.prepare(`PRAGMA index_list('jobs')`).all(); + for (const idx of indexes) { + if (!idx.unique) continue; + // PRAGMA does not accept bound parameters; index names come from PRAGMA index_list + // (SQLite-internal source) so injection is not a concern. + const cols = db.prepare(`PRAGMA index_info('${idx.name}')`).all().map(r => r.name); + if (cols.length === 2 && cols[0] === 'source' && cols[1] === 'job_id') { + return { exists: true, unique: true }; + } + } + return { exists: true, unique: false }; +} + +export function assertJobsUnique(db) { + const { exists, unique } = hasUniqueOnSourceJobId(db); + if (!exists) { + throw preflightError( + `jobs table is missing. The job-hunter project owns DB creation; run its setup first.` + ); + } + if (!unique) { + throw preflightError( + `jobs(source, job_id) lacks PRIMARY KEY or UNIQUE constraint. Cannot install salary schema (FK targets would be ambiguous).` + ); + } + return true; +} diff --git a/skills/salary-calculator/scripts/lib/retry-state-machine.mjs b/skills/salary-calculator/scripts/lib/retry-state-machine.mjs new file mode 100644 index 0000000..1921e29 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/retry-state-machine.mjs @@ -0,0 +1,146 @@ +// Phase v1.0-09 Plan 02 — Pure retry state machine. +// +// Two exported functions: computeNextRetry, classifyError. +// Zero I/O, zero wall-clock reads, zero module-scope mutable state. +// Same inputs → byte-identical outputs (RETRY-03 reproducibility). + +const NOT_FOUND_DAYS = { exact: 14, benchmark: 30 }; +const MAX_TRANSIENT_ATTEMPTS = 5; +const BACKOFF_CAP_DAYS = 7; + +/** + * Compute the next retry-state for one axis given previous state + outcome event. + * Pure function — no I/O, no wall-clock reads, no module-scope mutable state. + * + * @param {object} args + * @param {'exact'|'benchmark'} args.axis + * @param {string} args.prevStatus + * @param {number} args.prevAttemptCount + * @param {{kind:'success'|'not_found'|'transient_error'|'unrecoverable_error', message?:string}} args.event + * @param {string} args.nowIso ISO 8601 timestamp + * @returns {{status:string, attemptCount:number, nextRetryAtIso:string|null, errorMessage:string|null}} + */ +export function computeNextRetry({ axis, prevStatus, prevAttemptCount, event, nowIso }) { + if (axis !== 'exact' && axis !== 'benchmark') { + throw new TypeError('unknown axis: ' + axis); + } + void prevStatus; + const newCount = prevAttemptCount + 1; + const addDays = (n) => new Date(new Date(nowIso).getTime() + n * 86400000).toISOString(); + + switch (event.kind) { + case 'success': + return { + status: 'found', + attemptCount: newCount, + nextRetryAtIso: null, + errorMessage: null, + }; + case 'not_found': + return { + status: 'not_found', + attemptCount: newCount, + nextRetryAtIso: addDays(NOT_FOUND_DAYS[axis]), + errorMessage: null, + }; + case 'unrecoverable_error': + return { + status: 'error', + attemptCount: newCount, + nextRetryAtIso: null, + errorMessage: event.message ?? null, + }; + case 'transient_error': { + if (newCount > MAX_TRANSIENT_ATTEMPTS) { + return { + status: 'error', + attemptCount: newCount, + nextRetryAtIso: null, + errorMessage: event.message ?? null, + }; + } + const days = Math.min(Math.pow(2, newCount - 1), BACKOFF_CAP_DAYS); + return { + status: 'error', + attemptCount: newCount, + nextRetryAtIso: addDays(days), + errorMessage: event.message ?? null, + }; + } + default: + throw new RangeError('unknown event kind: ' + event.kind); + } +} + +/** + * Map an adapter return value or thrown error to the closed-set retry event taxonomy. + * Pure function. No I/O. + * + * @param {Array|Error|Response|*} result Adapter return value (Array of candidates) OR thrown error/Response. + * @param {object} [ctx] Reserved for future context-aware classification (unused in v1). + * @returns {{kind:'success'|'not_found'|'transient_error'|'unrecoverable_error', message?:string}} + */ +export function classifyError(result, ctx = {}) { + void ctx; + + // 1. Array branch — self-contained, must NOT fall through (opencode improvement #2). + if (Array.isArray(result)) { + const hasSuccess = result.some( + (c) => + c && + (c.extraction_status === null || c.extraction_status === undefined) && + c.is_posted_salary === 1, + ); + const allMarkers = + result.length > 0 && + result.every( + (c) => + c && + (c.extraction_status === 'absent' || + c.extraction_status === 'competitive' || + c.extraction_status === 'doe'), + ); + // Branch A — at least one real candidate + if (hasSuccess) return { kind: 'success' }; + // Branch B — empty or all-markers + if (result.length === 0 || allMarkers) return { kind: 'not_found' }; + // Branch C (EXPLICIT else for arrays) — non-empty, not all markers, no success. + // HTTP succeeded, parser produced output, but nothing usable. NOT unrecoverable. + return { kind: 'not_found', message: 'array contained no usable candidate' }; + } + + // 2. Response duck-type (check BEFORE instanceof Error — Responses are object-like). + if (result && typeof result.status === 'number' && typeof result.ok === 'boolean') { + const status = result.status; + if (status === 404) return { kind: 'not_found' }; + if (status === 401 || status === 403) { + return { kind: 'unrecoverable_error', message: 'HTTP ' + status }; + } + if (status >= 500) return { kind: 'transient_error', message: 'HTTP ' + status }; + return { kind: 'unrecoverable_error', message: 'HTTP ' + status }; + } + + // 3. TypeError — fetch failure / network transport + if (result instanceof TypeError) { + return { kind: 'transient_error', message: result.message }; + } + + // 4. Generic Error + if (result instanceof Error) { + if (result.code === 'ECONNRESET' || result.code === 'ETIMEDOUT') { + return { kind: 'transient_error', message: result.message }; + } + const httpMatch = /^HTTP (\d{3})/.exec(result.message); + if (httpMatch) { + const parsed = Number(httpMatch[1]); + return classifyError({ status: parsed, ok: false }); + } + if (/Job title not found in ITJobsWatch/.test(result.message)) { + return { kind: 'not_found' }; + } + return { kind: 'unrecoverable_error', message: result.message }; + } + + // 5. Fallback + return { kind: 'unrecoverable_error', message: result?.message ?? String(result) }; +} diff --git a/skills/salary-calculator/scripts/lib/salary-db.mjs b/skills/salary-calculator/scripts/lib/salary-db.mjs new file mode 100644 index 0000000..2ab7416 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/salary-db.mjs @@ -0,0 +1,564 @@ +// Salary observation persistence layer. +// +// Core responsibility: INSERT and SELECT salary observations with conflict handling +// (idempotency via observation_id uniqueness) and pre-flight validation. +// +// The CHECK partition enforced by SQLite ensures that exact observations +// (benchmark_id IS NULL, is_posted_salary=1) are strictly separated from estimate +// observations (benchmark_id IS NOT NULL, is_posted_salary=0). This module validates +// the partition in application code (friendlier error messages) before passing to SQL; +// SQL remains the authoritative gate. +// +// PRECONDITION: All write operations (insertObservation) assume the caller holds +// the salary_writer_lock (acquired via BEGIN IMMEDIATE in writer-lock.mjs). +// This module does NOT acquire the lock — see Phase v1.0-09 orchestrator (bin/salary-cli). +// Calling without the lock risks SQLITE_BUSY under concurrent writers. + +import { calculateObservationId } from './observation-id.mjs'; + +// Sourced from scripts/lib/salary-schema.mjs lines 109-114 (exact-side CHECK partition); keep in sync +export const EXACT_CONFIDENCE_LABELS = Object.freeze([ + 'posted_exact', + 'external_exact', + 'api_exact_match', + 'unknown_exact', +]); + +// Sourced from scripts/lib/salary-schema.mjs lines 116-122 (estimate-side CHECK partition); keep in sync +export const ESTIMATE_CONFIDENCE_LABELS = Object.freeze([ + 'aggregated_exact_title', + 'company_benchmark', + 'estimated_market', + 'official_baseline', + 'unknown_estimate', +]); + +// Sourced from scripts/lib/salary-schema.mjs line 74 (period CHECK); keep in sync +const VALID_PERIODS = Object.freeze(['hour', 'day', 'week', 'month', 'year']); + +// Sourced from scripts/lib/salary-schema.mjs lines 75-77 (compensation_type CHECK); keep in sync +const VALID_COMPENSATION_TYPES = Object.freeze(['base_salary', 'total_compensation', 'ote', 'contract_rate', 'unknown']); + +// Sourced from scripts/lib/salary-schema.mjs lines 55-65 (matched_by CHECK); keep in sync +const VALID_MATCHED_BY = Object.freeze(['exact_job', 'title_company_location', 'title_company', 'role_location', 'company_role_level', 'occupation_baseline', 'manual']); + +/** + * Insert one salary observation using INSERT OR IGNORE semantics (DB-06 idempotency). + * + * PRECONDITION: Caller MUST hold the `salary_writer_lock` (acquired via + * BEGIN IMMEDIATE in writer-lock.mjs) for this `db` connection. This function + * does NOT acquire the lock — see Phase v1.0-09 orchestrator (bin/salary-cli) + * for the lock-acquisition boundary. Calling this without holding the writer + * lock risks SQLITE_BUSY under concurrent writers. + * + * @param {Database} db - better-sqlite3 connection inside an active writer-lock transaction + * @param {object} observation - validated observation payload + * @param {object} [opts] - optional parameters (reserved for future use) + * @returns {{ observation_id: string, inserted: boolean, changes: number }} + */ +export function insertObservation(db, observation, opts = {}) { + // Pre-flight validation (throws Error with descriptive message; happens BEFORE any DB roundtrip) + const errors = []; + + if (!observation || typeof observation !== 'object') { + throw new Error('observation must be a non-null object'); + } + + if (!observation.job_source || typeof observation.job_source !== 'string' || observation.job_source.trim() === '') { + errors.push('job_source must be a non-empty string'); + } + + if (!observation.job_id || typeof observation.job_id !== 'string' || observation.job_id.trim() === '') { + errors.push('job_id must be a non-empty string'); + } + + if (!observation.data_source || typeof observation.data_source !== 'string' || observation.data_source.trim() === '') { + errors.push('data_source must be a non-empty string'); + } + + if (!observation.currency || typeof observation.currency !== 'string' || observation.currency.length !== 3) { + errors.push('currency must be a 3-letter string'); + } + + if (!VALID_PERIODS.includes(observation.period)) { + errors.push(`period must be one of: ${VALID_PERIODS.join(', ')}`); + } + + if (!VALID_COMPENSATION_TYPES.includes(observation.compensation_type)) { + errors.push(`compensation_type must be one of: ${VALID_COMPENSATION_TYPES.join(', ')}`); + } + + const all_confidence_labels = [...EXACT_CONFIDENCE_LABELS, ...ESTIMATE_CONFIDENCE_LABELS]; + if (!all_confidence_labels.includes(observation.confidence_label)) { + errors.push(`confidence_label must be one of: ${all_confidence_labels.join(', ')}`); + } + + if (!VALID_MATCHED_BY.includes(observation.matched_by)) { + errors.push(`matched_by must be one of: ${VALID_MATCHED_BY.join(', ')}`); + } + + if (observation.is_posted_salary !== 0 && observation.is_posted_salary !== 1) { + errors.push('is_posted_salary must be 0 or 1'); + } + + if (observation.is_predicted !== 0 && observation.is_predicted !== 1) { + errors.push('is_predicted must be 0 or 1'); + } + + // At least one of amount_min, amount_max, amount_median must be a finite number + const hasValidAmount = ( + (Number.isFinite(observation.amount_min)) || + (Number.isFinite(observation.amount_max)) || + (Number.isFinite(observation.amount_median)) + ); + if (!hasValidAmount) { + errors.push('At least one of amount_min, amount_max, amount_median must be a number'); + } + + // Cross-field consistency (mirrors the schema CHECK so we get a friendlier error message before SQLite fires) + if (observation.is_posted_salary === 1) { + if (observation.benchmark_id != null) { + errors.push('Exact observations (is_posted_salary=1) must have benchmark_id IS NULL'); + } + if (!EXACT_CONFIDENCE_LABELS.includes(observation.confidence_label)) { + errors.push(`Exact observations (is_posted_salary=1) must have confidence_label in: ${EXACT_CONFIDENCE_LABELS.join(', ')}`); + } + } + + if (observation.is_posted_salary === 0) { + if (observation.benchmark_id == null || observation.benchmark_id === '') { + errors.push('Estimate observations (is_posted_salary=0) must have a non-empty benchmark_id'); + } + if (!ESTIMATE_CONFIDENCE_LABELS.includes(observation.confidence_label)) { + errors.push(`Estimate observations (is_posted_salary=0) must have confidence_label in: ${ESTIMATE_CONFIDENCE_LABELS.join(', ')}`); + } + } + + if (observation.is_predicted === 1) { + if (observation.is_posted_salary !== 0) { + errors.push('Predicted observations (is_predicted=1) must have is_posted_salary=0'); + } + if (observation.benchmark_id == null || observation.benchmark_id === '') { + errors.push('Predicted observations (is_predicted=1) must have a non-empty benchmark_id'); + } + } + + if (errors.length > 0) { + throw new Error(`Invalid observation: ${errors.join('; ')}`); + } + + // Compute identity + const observation_id = calculateObservationId(observation); + + // Build INSERT OR IGNORE statement against job_salary_observations + // Bind every column from the schema; use null for optional columns the observation does not set + const stmt = db.prepare(` + INSERT OR IGNORE INTO job_salary_observations ( + job_source, job_id, observation_id, + data_source, data_source_url, benchmark_id, + confidence_label, matched_by, + is_posted_salary, is_predicted, + currency, amount_min, amount_max, amount_median, period, compensation_type, + annualized_min, annualized_max, annualized_median, annualization_note, + fx_currency, fx_annualized_median, fx_rate, fx_rate_as_of, + location_raw, country_code, region, city, + evidence_snippet, raw_payload_json + ) VALUES ( + @job_source, @job_id, @observation_id, + @data_source, @data_source_url, @benchmark_id, + @confidence_label, @matched_by, + @is_posted_salary, @is_predicted, + @currency, @amount_min, @amount_max, @amount_median, @period, @compensation_type, + @annualized_min, @annualized_max, @annualized_median, @annualization_note, + @fx_currency, @fx_annualized_median, @fx_rate, @fx_rate_as_of, + @location_raw, @country_code, @region, @city, + @evidence_snippet, @raw_payload_json + ) + `); + + // Normalize the observation: handle optional fields as null if missing + const normalizedObs = { + ...observation, + observation_id, + data_source_url: observation.data_source_url ?? null, + benchmark_id: observation.benchmark_id ?? null, + annualized_min: observation.annualized_min ?? null, + annualized_max: observation.annualized_max ?? null, + annualized_median: observation.annualized_median ?? null, + annualization_note: observation.annualization_note ?? null, + fx_currency: observation.fx_currency ?? null, + fx_annualized_median: observation.fx_annualized_median ?? null, + fx_rate: observation.fx_rate ?? null, + fx_rate_as_of: observation.fx_rate_as_of ?? null, + location_raw: observation.location_raw ?? null, + country_code: observation.country_code ?? null, + region: observation.region ?? null, + city: observation.city ?? null, + evidence_snippet: observation.evidence_snippet ?? null, + raw_payload_json: (typeof observation.raw_payload_json === 'string') + ? observation.raw_payload_json + : (observation.raw_payload_json ? JSON.stringify(observation.raw_payload_json) : null), + }; + + let result; + try { + result = stmt.run(normalizedObs); + } catch (err) { + // Re-throw better-sqlite3 errors with context + if (err.message && err.message.includes('FOREIGN KEY')) { + throw new Error(`Foreign key constraint failed: ${err.message}`); + } + if (err.message && err.message.includes('CHECK')) { + throw new Error(`CHECK constraint failed: ${err.message}`); + } + throw err; + } + + return { + observation_id, + inserted: result.changes === 1, + changes: result.changes, + }; +} + +/** + * Retrieve a single observation by job source, job ID, and observation ID. + * + * @param {Database} db - better-sqlite3 connection + * @param {string} jobSource - job source (e.g., 'linkedin') + * @param {string} jobId - job ID + * @param {string} observationId - observation ID (hash) + * @returns {object|null} - observation object or null if not found + */ +export function getObservationById(db, jobSource, jobId, observationId) { + const stmt = db.prepare(` + SELECT * FROM job_salary_observations + WHERE job_source = ? AND job_id = ? AND observation_id = ? + `); + + return stmt.get(jobSource, jobId, observationId) || null; +} + +/** + * Retrieve all observations for one job, ordered by most recent first. + * + * Deterministic secondary sort by observation_id ensures stable ordering for + * observations inserted at identical timestamps (RESEARCH pitfall #6). + * + * @param {Database} db - better-sqlite3 connection + * @param {string} jobSource - job source (e.g., 'linkedin') + * @param {string} jobId - job ID + * @returns {Array} - array of observation objects (possibly empty) + * @throws {Error} - if jobSource or jobId is not a non-empty string + */ +export function getObservationsByJob(db, jobSource, jobId) { + if (!jobSource || typeof jobSource !== 'string' || jobSource.trim() === '') { + throw new Error('jobSource and jobId required'); + } + if (!jobId || typeof jobId !== 'string' || jobId.trim() === '') { + throw new Error('jobSource and jobId required'); + } + + const stmt = db.prepare(` + SELECT * FROM job_salary_observations + WHERE job_source = ? AND job_id = ? + ORDER BY observed_at DESC, observation_id ASC + `); + + return stmt.all(jobSource, jobId); +} + +/** + * Retrieve the most recent observation for one job. + * + * @param {Database} db - better-sqlite3 connection + * @param {string} jobSource - job source (e.g., 'linkedin') + * @param {string} jobId - job ID + * @returns {object|null} - most recent observation or null if none exist + * @throws {Error} - if jobSource or jobId is not a non-empty string + */ +export function getLatestObservationPerJob(db, jobSource, jobId) { + if (!jobSource || typeof jobSource !== 'string' || jobSource.trim() === '') { + throw new Error('jobSource and jobId required'); + } + if (!jobId || typeof jobId !== 'string' || jobId.trim() === '') { + throw new Error('jobSource and jobId required'); + } + + const stmt = db.prepare(` + SELECT * FROM job_salary_observations + WHERE job_source = ? AND job_id = ? + ORDER BY observed_at DESC, observation_id ASC + LIMIT 1 + `); + + return stmt.get(jobSource, jobId) || null; +} + +/** + * Count observations for a job with a specific confidence label. + * + * @param {Database} db - better-sqlite3 connection + * @param {string} jobSource - job source (e.g., 'linkedin') + * @param {string} jobId - job ID + * @param {string} confidenceLabel - confidence label to count + * @returns {number} - count of matching observations (0 if none) + * @throws {Error} - if jobSource/jobId are not non-empty strings, or confidenceLabel is not valid + */ +// --------------------------------------------------------------------------- +// Batch candidate query helpers (Phase v1.0-10 — BATCH-01, BATCH-02, BATCH-03) +// --------------------------------------------------------------------------- +// +// Four new exports used by the batch-mode CLI dispatch (Plan 04): +// - selectBatchCandidates (BATCH-01 + BATCH-02 inclusion/exclusion semantics) +// - selectBatchCandidatesForceRetry (BATCH-03 force-retry semantics) +// - countBatchCandidates (total count for [N/M] progress markers) +// - countBatchCandidatesForceRetry (total count for force-retry path) +// +// Pitfall 8 (RESEARCH, MEDIUM risk): SQLite timestamp columns are TEXT. When +// applyRetryTransition writes ISO-8601 strings with a 'T' separator and 'Z' +// suffix ('2026-05-23T14:30:00.000Z') and we compare against datetime('now') +// (which produces '2026-05-23 14:30:00' — space separator, no Z), a raw +// `next_exact_retry_at <= datetime('now')` becomes a lexicographic TEXT +// compare. 'T' (0x54) > ' ' (0x20), so eligible past rows are NEVER selected. +// Fix: wrap BOTH sides in datetime(...). SQLite's datetime() parser +// normalizes both shapes to '%Y-%m-%d %H:%M:%S', enabling a correct +// chronological comparison. + +const BATCH_CANDIDATES_WHERE = ` + s.job_source IS NULL + OR s.exact_status = 'pending' + OR (s.exact_status IN ('not_found', 'error') + AND s.next_exact_retry_at IS NOT NULL + AND datetime(s.next_exact_retry_at) <= datetime('now')) +`; + +const SELECT_BATCH_CANDIDATES_SQL = `SELECT j.source, j.job_id, j.title, j.company, j.url, + j.country_code, j.region, j.city, j.location_raw +FROM jobs j +LEFT JOIN job_enrichment_state s + ON s.job_source = j.source AND s.job_id = j.job_id +WHERE ${BATCH_CANDIDATES_WHERE} +ORDER BY j.source, j.job_id +LIMIT ?`; + +const COUNT_BATCH_CANDIDATES_SQL = `SELECT COUNT(*) AS total +FROM jobs j +LEFT JOIN job_enrichment_state s + ON s.job_source = j.source AND s.job_id = j.job_id +WHERE ${BATCH_CANDIDATES_WHERE}`; + +const FORCE_RETRY_WHERE = `NOT EXISTS ( + SELECT 1 + FROM job_salary_observations o + WHERE o.job_source = j.source + AND o.job_id = j.job_id + AND o.is_posted_salary = 1 +)`; + +const SELECT_BATCH_CANDIDATES_FORCE_RETRY_SQL = `SELECT j.source, j.job_id, j.title, j.company, j.url, + j.country_code, j.region, j.city, j.location_raw +FROM jobs j +WHERE ${FORCE_RETRY_WHERE} +ORDER BY j.source, j.job_id +LIMIT ?`; + +const COUNT_BATCH_CANDIDATES_FORCE_RETRY_SQL = `SELECT COUNT(*) AS total +FROM jobs j +WHERE ${FORCE_RETRY_WHERE}`; + +// Prepared-statement cache keyed by db handle (mirrors enrichment-state.mjs). +const batchStmtCache = new WeakMap(); + +function getBatchStmts(db) { + let cached = batchStmtCache.get(db); + if (!cached) { + cached = { + selectBatchCandidates: db.prepare(SELECT_BATCH_CANDIDATES_SQL), + selectBatchCandidatesForceRetry: db.prepare(SELECT_BATCH_CANDIDATES_FORCE_RETRY_SQL), + countBatchCandidates: db.prepare(COUNT_BATCH_CANDIDATES_SQL), + countBatchCandidatesForceRetry: db.prepare(COUNT_BATCH_CANDIDATES_FORCE_RETRY_SQL), + }; + batchStmtCache.set(db, cached); + } + return cached; +} + +function assertPositiveIntegerLimit(limit) { + if (!Number.isInteger(limit) || limit <= 0) { + throw new TypeError(`limit must be a positive integer (got ${typeof limit} ${limit})`); + } +} + +/** + * Select batch candidates — jobs needing an exact-salary enrichment attempt. + * + * Requirements: BATCH-01 (inclusion) + BATCH-02 (exclusion). + * + * Inclusion (BATCH-01): jobs with no job_enrichment_state row, OR + * exact_status='pending', OR exact_status IN ('not_found','error') with + * next_exact_retry_at in the past (datetime() wrapper — Pitfall 8). + * + * Exclusion (BATCH-02): exact_status='found', OR + * exact_status IN ('not_found','error') with next_exact_retry_at IS NULL + * (terminal — already exhausted backoff), OR next_exact_retry_at in the future. + * + * Pitfall 8: datetime() wrapper on BOTH sides of the timestamp comparison so + * ISO-8601 'T...Z' shape written by applyRetryTransition compares correctly + * against SQLite's space-separated datetime('now') output. + * + * @param {{db: import('better-sqlite3').Database, limit: number}} args + * @returns {Array<{source, job_id, title, company, url, country_code, region, city, location_raw}>} + */ +export function selectBatchCandidates({ db, limit }) { + assertPositiveIntegerLimit(limit); + return getBatchStmts(db).selectBatchCandidates.all(limit); +} + +/** + * Select batch candidates for --force-*-retry / --all-unsalaried path. + * + * Requirement: BATCH-03 — returns every job lacking ANY is_posted_salary=1 + * observation, regardless of retry schedule (ignores job_enrichment_state + * entirely). + * + * Pitfall 8 does not apply here (no timestamp comparison). + * + * @param {{db: import('better-sqlite3').Database, limit: number}} args + * @returns {Array<{source, job_id, title, company, url, country_code, region, city, location_raw}>} + */ +export function selectBatchCandidatesForceRetry({ db, limit }) { + assertPositiveIntegerLimit(limit); + return getBatchStmts(db).selectBatchCandidatesForceRetry.all(limit); +} + +/** + * Count batch candidates (same WHERE as selectBatchCandidates, no LIMIT). + * + * Requirements: BATCH-01 + BATCH-02. Used for [N/M] progress markers in the + * batch CLI's NDJSON output. Pitfall 8 wrapper applies. + * + * @param {{db: import('better-sqlite3').Database}} args + * @returns {number} + */ +export function countBatchCandidates({ db }) { + const row = getBatchStmts(db).countBatchCandidates.get(); + return row?.total ?? 0; +} + +/** + * Count batch candidates for force-retry path (same WHERE as + * selectBatchCandidatesForceRetry, no LIMIT). + * + * Requirement: BATCH-03. + * + * @param {{db: import('better-sqlite3').Database}} args + * @returns {number} + */ +export function countBatchCandidatesForceRetry({ db }) { + const row = getBatchStmts(db).countBatchCandidatesForceRetry.get(); + return row?.total ?? 0; +} + +// --------------------------------------------------------------------------- +// Health-metric hit-rate helpers (Phase v1.0-11 — HEALTH-02 / HEALTH-03) +// --------------------------------------------------------------------------- +// +// Two read-only aggregate queries powering the `--health` CLI report: +// - getCurrent30DayHitRates: (today_midnight - 30 days) .. (today_midnight) +// - getPrior30DayHitRates: (today_midnight - 60 days) .. (today_midnight - 30 days) +// +// Both windows EXCLUDE today via the 'start of day' modifier so partial-day +// noise never corrupts the trailing-30 comparison. +// +// Pitfall 8 closure: observed_at TEXT can be either 'YYYY-MM-DD HH:MM:SS' +// (SQLite CURRENT_TIMESTAMP default) or 'YYYY-MM-DDTHH:MM:SS.sssZ' +// (ISO-8601 from v1.0-09 retry paths). Bare lexicographic compare of the two +// shapes misclassifies — 'T' (0x54) > ' ' (0x20). Wrap BOTH sides in +// datetime(...) so SQLite normalises to '%Y-%m-%d %H:%M:%S' on both sides. + +const HIT_RATE_CURRENT_SQL = ` + SELECT job_source, COUNT(*) AS total, SUM(is_posted_salary) AS hits + FROM job_salary_observations + WHERE datetime(observed_at) >= datetime('now', '-30 days', 'start of day') + AND datetime(observed_at) < datetime('now', 'start of day') + GROUP BY job_source +`; + +const HIT_RATE_PRIOR_SQL = ` + SELECT job_source, COUNT(*) AS total, SUM(is_posted_salary) AS hits + FROM job_salary_observations + WHERE datetime(observed_at) >= datetime('now', '-60 days', 'start of day') + AND datetime(observed_at) < datetime('now', '-30 days', 'start of day') + GROUP BY job_source +`; + +const healthStmtCache = new WeakMap(); + +function getHealthStmts(db) { + let cached = healthStmtCache.get(db); + if (!cached) { + cached = { + current: db.prepare(HIT_RATE_CURRENT_SQL), + prior: db.prepare(HIT_RATE_PRIOR_SQL), + }; + healthStmtCache.set(db, cached); + } + return cached; +} + +function coerceHitRateRow(row) { + // SUM(is_posted_salary) is INTEGER over a non-empty group (GROUP BY filters + // empty groups out), but defensive coercion costs nothing and guards against + // a hypothetical NULL. + return { + job_source: row.job_source, + total: row.total ?? 0, + hits: row.hits ?? 0, + }; +} + +/** + * Aggregate posted-salary hit rates per job_source over the trailing 30 + * complete days (today excluded). Read-only; does NOT take the writer lock. + * + * @param {{db: import('better-sqlite3').Database}} args + * @returns {Array<{job_source: string, total: number, hits: number}>} + */ +export function getCurrent30DayHitRates({ db }) { + return getHealthStmts(db).current.all().map(coerceHitRateRow); +} + +/** + * Aggregate posted-salary hit rates per job_source over the prior 30-day + * window (60..30 days ago). Read-only; does NOT take the writer lock. + * + * @param {{db: import('better-sqlite3').Database}} args + * @returns {Array<{job_source: string, total: number, hits: number}>} + */ +export function getPrior30DayHitRates({ db }) { + return getHealthStmts(db).prior.all().map(coerceHitRateRow); +} + +export function countObservationsByConfidence(db, jobSource, jobId, confidenceLabel) { + if (!jobSource || typeof jobSource !== 'string' || jobSource.trim() === '') { + throw new Error('jobSource and jobId required'); + } + if (!jobId || typeof jobId !== 'string' || jobId.trim() === '') { + throw new Error('jobSource and jobId required'); + } + + const all_confidence_labels = [...EXACT_CONFIDENCE_LABELS, ...ESTIMATE_CONFIDENCE_LABELS]; + if (!all_confidence_labels.includes(confidenceLabel)) { + throw new Error(`confidenceLabel must be one of: ${all_confidence_labels.join(', ')}`); + } + + const stmt = db.prepare(` + SELECT COUNT(*) AS count + FROM job_salary_observations + WHERE job_source = ? AND job_id = ? AND confidence_label = ? + `); + + const row = stmt.get(jobSource, jobId, confidenceLabel); + return row?.count ?? 0; +} diff --git a/skills/salary-calculator/scripts/lib/salary-schema.mjs b/skills/salary-calculator/scripts/lib/salary-schema.mjs new file mode 100644 index 0000000..586d2e8 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/salary-schema.mjs @@ -0,0 +1,262 @@ +// Single source of truth for salary schema DDL strings. +// Both the installer (Plan 04) and the drift detector (Plan 02) import from here. +// +// 8-index breakdown (CONTEXT.md corrected from 7 → 8 on 2026-05-12): +// 3 obs indexes (idx_obs_job, idx_obs_benchmark, idx_obs_job_confidence) +// + 3 bench indexes (idx_bench_series_latest, idx_bench_identity_lookup, idx_bench_refresh_due) +// + 2 enrich indexes (idx_enrich_exact_retry_due, idx_enrich_benchmark_retry_due) +// = 8 +// +// CONTEXT.md mandates 5 identity columns on salary_writer_lock; V4's minimal +// `id/holder/acquired_at` shape is superseded. + +// LOCK_TABLE_DDL: separately-named export of the salary_writer_lock CREATE +// statement. Re-exported via TABLES.salary_writer_lock too — single literal +// string, two named bindings. Plan 03's acquire() runs this as its FIRST +// statement on a fresh DB to self-bootstrap. +export const LOCK_TABLE_DDL = `CREATE TABLE IF NOT EXISTS salary_writer_lock ( + id INTEGER PRIMARY KEY CHECK (id = 1), + hostname TEXT, + pid INTEGER, + started_at TEXT, + nonce TEXT, + acquired_at TEXT +)`; + +// NOW_SQL: millisecond-precision "now" SQL. Owned here (single source of truth) +// and imported by writer-lock.mjs (one-way dependency: writer-lock → salary-schema, +// never the reverse, to avoid circular deps). SQLite's datetime() parses the .fff +// fractional seconds, so the 10-minute stale-window predicate keeps working. +export const NOW_SQL = `strftime('%Y-%m-%d %H:%M:%f','now')`; + +export const TABLES = { + job_salary_observations: `CREATE TABLE IF NOT EXISTS job_salary_observations ( + job_source TEXT NOT NULL, + job_id TEXT NOT NULL, + observation_id TEXT NOT NULL, + + data_source TEXT NOT NULL, + data_source_url TEXT, + benchmark_id TEXT, + + confidence_label TEXT NOT NULL CHECK ( + confidence_label IN ( + 'posted_exact', + 'external_exact', + 'api_exact_match', + 'aggregated_exact_title', + 'company_benchmark', + 'estimated_market', + 'official_baseline', + 'unknown_exact', + 'unknown_estimate' + ) + ), + matched_by TEXT NOT NULL CHECK ( + matched_by IN ( + 'exact_job', + 'title_company_location', + 'title_company', + 'role_location', + 'company_role_level', + 'occupation_baseline', + 'manual' + ) + ), + + is_posted_salary INTEGER NOT NULL DEFAULT 0 CHECK (is_posted_salary IN (0, 1)), + is_predicted INTEGER NOT NULL DEFAULT 0 CHECK (is_predicted IN (0, 1)), + + currency TEXT NOT NULL CHECK (length(currency) = 3), + amount_min REAL, + amount_max REAL, + amount_median REAL, + period TEXT NOT NULL CHECK (period IN ('hour', 'day', 'week', 'month', 'year')), + compensation_type TEXT NOT NULL CHECK ( + compensation_type IN ('base_salary', 'total_compensation', 'ote', 'contract_rate', 'unknown') + ), + + annualized_min REAL, + annualized_max REAL, + annualized_median REAL, + annualization_note TEXT, + + fx_currency TEXT CHECK (fx_currency IS NULL OR length(fx_currency) = 3), + fx_annualized_median REAL, + fx_rate REAL, + fx_rate_as_of TEXT, + + location_raw TEXT, + country_code TEXT, + region TEXT, + city TEXT, + + evidence_snippet TEXT, + raw_payload_json TEXT, + + observed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (job_source, job_id, observation_id), + FOREIGN KEY (job_source, job_id) + REFERENCES jobs(source, job_id) + ON DELETE CASCADE, + FOREIGN KEY (benchmark_id) + REFERENCES salary_benchmarks(benchmark_id) + ON DELETE RESTRICT, + + CHECK ( + (benchmark_id IS NULL AND is_posted_salary = 1 AND confidence_label IN ( + 'posted_exact', + 'external_exact', + 'api_exact_match', + 'unknown_exact' + )) + OR + (benchmark_id IS NOT NULL AND is_posted_salary = 0 AND confidence_label IN ( + 'aggregated_exact_title', + 'company_benchmark', + 'estimated_market', + 'official_baseline', + 'unknown_estimate' + )) + ), + + CHECK ( + is_predicted = 0 + OR (is_predicted = 1 AND is_posted_salary = 0 AND benchmark_id IS NOT NULL) + ), + + CHECK ( + amount_min IS NOT NULL + OR amount_max IS NOT NULL + OR amount_median IS NOT NULL + ) +)`, + + salary_benchmarks: `CREATE TABLE IF NOT EXISTS salary_benchmarks ( + benchmark_id TEXT PRIMARY KEY, + benchmark_series_id TEXT NOT NULL, + + data_source TEXT NOT NULL, + data_source_url TEXT, + + raw_title TEXT NOT NULL, + normalized_title TEXT NOT NULL, + role_family TEXT, + seniority TEXT NOT NULL DEFAULT '_any', + industry TEXT NOT NULL DEFAULT '_any', + + country_code TEXT NOT NULL, + region TEXT NOT NULL DEFAULT '', + city TEXT NOT NULL DEFAULT '', + location_raw TEXT, + + currency TEXT NOT NULL CHECK (length(currency) = 3), + period TEXT NOT NULL CHECK (period IN ('hour', 'day', 'week', 'month', 'year')), + compensation_type TEXT NOT NULL CHECK ( + compensation_type IN ('base_salary', 'total_compensation', 'ote', 'contract_rate', 'unknown') + ), + + amount_min REAL, + amount_max REAL, + amount_median REAL, + amount_p10 REAL, + amount_p25 REAL, + amount_p75 REAL, + amount_p90 REAL, + + sample_size INTEGER CHECK (sample_size IS NULL OR sample_size >= 0), + confidence_score REAL CHECK ( + confidence_score IS NULL OR (confidence_score >= 0 AND confidence_score <= 1) + ), + + effective_from TEXT, + effective_to TEXT, + fetched_at TEXT NOT NULL, + next_refresh_at TEXT, + refresh_frequency_days INTEGER NOT NULL DEFAULT 30, + + payload_hash TEXT NOT NULL, + evidence_snippet TEXT, + raw_payload_json TEXT, + + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + normalizer_version INTEGER NOT NULL DEFAULT 1 CHECK (normalizer_version >= 1), + + UNIQUE (benchmark_series_id, payload_hash) +)`, + + job_enrichment_state: `CREATE TABLE IF NOT EXISTS job_enrichment_state ( + job_source TEXT NOT NULL, + job_id TEXT NOT NULL, + + exact_status TEXT NOT NULL DEFAULT 'pending' + CHECK (exact_status IN ('pending', 'found', 'not_found', 'error')), + exact_last_attempt_at TEXT, + exact_error TEXT, + exact_attempt_count INTEGER NOT NULL DEFAULT 0 + CHECK (exact_attempt_count >= 0), + next_exact_retry_at TEXT, + + benchmark_status TEXT NOT NULL DEFAULT 'pending' + CHECK (benchmark_status IN ('pending', 'found', 'not_found', 'error')), + benchmark_last_attempt_at TEXT, + benchmark_error TEXT, + benchmark_attempt_count INTEGER NOT NULL DEFAULT 0 + CHECK (benchmark_attempt_count >= 0), + next_benchmark_retry_at TEXT, + latest_benchmark_id TEXT, + + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (job_source, job_id), + FOREIGN KEY (job_source, job_id) + REFERENCES jobs(source, job_id) + ON DELETE CASCADE, + FOREIGN KEY (latest_benchmark_id) + REFERENCES salary_benchmarks(benchmark_id) + ON DELETE SET NULL +)`, + + salary_writer_lock: LOCK_TABLE_DDL, +}; + +export const INDEXES = { + idx_obs_job: `CREATE INDEX IF NOT EXISTS idx_obs_job ON job_salary_observations (job_source, job_id)`, + idx_obs_benchmark: `CREATE INDEX IF NOT EXISTS idx_obs_benchmark ON job_salary_observations (benchmark_id)`, + idx_obs_job_confidence: `CREATE INDEX IF NOT EXISTS idx_obs_job_confidence ON job_salary_observations (job_source, job_id, confidence_label, observed_at DESC)`, + idx_bench_series_latest: `CREATE INDEX IF NOT EXISTS idx_bench_series_latest ON salary_benchmarks (benchmark_series_id, fetched_at DESC)`, + idx_bench_identity_lookup: `CREATE INDEX IF NOT EXISTS idx_bench_identity_lookup ON salary_benchmarks (normalized_title, country_code, region, city, compensation_type, period, seniority, industry, fetched_at DESC)`, + idx_bench_refresh_due: `CREATE INDEX IF NOT EXISTS idx_bench_refresh_due ON salary_benchmarks (next_refresh_at) WHERE next_refresh_at IS NOT NULL`, + idx_enrich_exact_retry_due: `CREATE INDEX IF NOT EXISTS idx_enrich_exact_retry_due ON job_enrichment_state (next_exact_retry_at) WHERE next_exact_retry_at IS NOT NULL`, + idx_enrich_benchmark_retry_due: `CREATE INDEX IF NOT EXISTS idx_enrich_benchmark_retry_due ON job_enrichment_state (next_benchmark_retry_at) WHERE next_benchmark_retry_at IS NOT NULL`, +}; + +export const TRIGGERS = { + trg_state_updated: `CREATE TRIGGER IF NOT EXISTS trg_state_updated +AFTER UPDATE ON job_enrichment_state +FOR EACH ROW +WHEN NEW.updated_at IS OLD.updated_at +BEGIN + UPDATE job_enrichment_state + SET updated_at = CURRENT_TIMESTAMP + WHERE job_source = OLD.job_source + AND job_id = OLD.job_id; +END`, +}; + +export const EXPECTED_NAMES = [ + ...Object.keys(TABLES), + ...Object.keys(INDEXES), + ...Object.keys(TRIGGERS), +]; + +export const ALL_DDL = [ + ...Object.values(TABLES), + ...Object.values(INDEXES), + ...Object.values(TRIGGERS), +]; + +export const COUNTS = { tables: 4, indexes: 8, triggers: 1 }; diff --git a/skills/salary-calculator/scripts/lib/salary-selector.mjs b/skills/salary-calculator/scripts/lib/salary-selector.mjs new file mode 100644 index 0000000..3004f94 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/salary-selector.mjs @@ -0,0 +1,123 @@ +// Salary observation selector. +// +// Pure stateless function: given an array of observations for one job, return the +// single best observation by applying confidence-rank > 7-level tie-breaker chain. +// +// LOAD-BEARING INVARIANT: The comparator MUST NEVER read amount, annualized, or +// FX fields. Currency affects ranking ONLY as a metadata-equality check at the +// currency tier (SEL-06). Numeric cross-currency comparison is the core product +// risk this skill exists to avoid. Focused selector checks enforce the boundary. + +// Canonical confidence label order (lower index = higher priority). +// Confidence rank is the canonical primary tier. The +// `is_posted_salary` (exactness) check is an intra-label tie-breaker ONLY — it +// applies when two observations share the same `confidence_label`, never across +// labels. Rationale: `unknown_exact` (rank 7) means the amount is exact but the +// source/confidence is unknown — a well-sourced estimate (rank 3-6) is more +// trustworthy than an unsourced "exact" number. +const CONFIDENCE_RANK = Object.freeze({ + 'posted_exact': 0, + 'external_exact': 1, + 'api_exact_match': 2, + 'aggregated_exact_title': 3, + 'company_benchmark': 4, + 'estimated_market': 5, + 'official_baseline': 6, + 'unknown_exact': 7, + 'unknown_estimate': 8, +}); + +function getRankIndex(label) { + const rank = CONFIDENCE_RANK[label]; + return rank === undefined ? Infinity : rank; +} + +// Location specificity score: city > region > country. +// Treats null, undefined, empty string, and whitespace-only strings as country-level. +function getLocationScore(observation) { + const city = observation.city; + if (typeof city === 'string' && city.trim() !== '') return 2; + const region = observation.region; + if (typeof region === 'string' && region.trim() !== '') return 1; + return 0; +} + +function compareObservations(a, b, opts) { + // 1. Confidence rank (lower index is better) + const rankA = getRankIndex(a.confidence_label); + const rankB = getRankIndex(b.confidence_label); + if (rankA !== rankB) return rankA - rankB; + + // 2. Exactness: is_posted_salary (1 beats 0) + const exactA = a.is_posted_salary === 1 ? 1 : 0; + const exactB = b.is_posted_salary === 1 ? 1 : 0; + if (exactA !== exactB) return exactB - exactA; + + // 3. Predicted state: is_predicted (0 beats 1) + const predA = a.is_predicted === 1 ? 1 : 0; + const predB = b.is_predicted === 1 ? 1 : 0; + if (predA !== predB) return predA - predB; + + // 4. Observed time: newer is better (treat missing as epoch) + const timeA = a.observed_at ? new Date(a.observed_at).getTime() : 0; + const timeB = b.observed_at ? new Date(b.observed_at).getTime() : 0; + if (!Number.isFinite(timeA) && !Number.isFinite(timeB)) { + // both invalid — fall through to next tier + } else if (!Number.isFinite(timeA)) { + return 1; // a invalid, b wins + } else if (!Number.isFinite(timeB)) { + return -1; // b invalid, a wins + } else if (timeA !== timeB) { + return timeB - timeA; + } + + // 5. Location specificity: city > region > country + const locA = getLocationScore(a); + const locB = getLocationScore(b); + if (locA !== locB) return locB - locA; + + // 6. Currency match (only when caller supplies jobCountryCode + expectedCurrency) + if (opts && opts.jobCountryCode && opts.expectedCurrency) { + const matchA = a.currency === opts.expectedCurrency ? 1 : 0; + const matchB = b.currency === opts.expectedCurrency ? 1 : 0; + if (matchA !== matchB) return matchB - matchA; + } + + // 7. Source priority (if provided) + if (opts && Array.isArray(opts.sourcePriority)) { + const idxA = opts.sourcePriority.indexOf(a.data_source); + const idxB = opts.sourcePriority.indexOf(b.data_source); + const priorityA = idxA >= 0 ? idxA : Infinity; + const priorityB = idxB >= 0 ? idxB : Infinity; + if (priorityA !== priorityB) return priorityA - priorityB; + } + + // 8. Final deterministic tie-break: observation_id lexicographic order + const idA = a.observation_id ?? ''; + const idB = b.observation_id ?? ''; + return idA.localeCompare(idB); +} + +/** + * Pick the best observation for a job. + * + * Pure function. Does not mutate input. Returns null on empty input. + * + * @param {Array} observations - array of observations for ONE job + * @param {object} [opts] - optional ranking hints + * @param {string} [opts.jobCountryCode] - ISO country code of the job (e.g., 'GB') + * @param {string} [opts.expectedCurrency] - 3-letter currency expected for jobCountryCode (e.g., 'GBP') + * @param {Array} [opts.sourcePriority] - ordered data_source names (earlier = preferred) + * @returns {object|null} the single best observation, or null if observations is empty + */ +export function selectBestObservation(observations, opts = {}) { + if (!Array.isArray(observations) || observations.length === 0) { + return null; + } + if (observations.length === 1) { + return observations[0]; + } + // Sort a copy so the input is not mutated. Comparator is total-order deterministic. + const sorted = observations.slice().sort((a, b) => compareObservations(a, b, opts)); + return sorted[0]; +} diff --git a/skills/salary-calculator/scripts/lib/source-priority.mjs b/skills/salary-calculator/scripts/lib/source-priority.mjs new file mode 100644 index 0000000..27d68a6 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/source-priority.mjs @@ -0,0 +1,48 @@ +/** + * Country-aware benchmark source priority. + * + * Exact salary extraction remains job-source-specific (currently LinkedIn). Benchmark + * fallback is market-source-specific and should be selected by country. The pipeline + * tries adapters in order until one returns a benchmark object; adapters returning + * null are treated as no-data and do not poison the retry state while later sources + * remain available. + */ + +export const BENCHMARK_SOURCE_PRIORITY = Object.freeze({ + // UK: preserve existing ITJobsWatch-first behaviour, then broader public sources. + GB: Object.freeze(['itjobswatch', 'indeed', 'levels_fyi', 'robert_half', 'salaryexpert']), + + // Ireland: Indeed career pages are currently the most directly parseable for + // role-level averages; Levels.fyi adds tech-comp context; Robert Half/SalaryExpert + // are opportunistic when their pages expose parseable ranges. + IE: Object.freeze(['indeed', 'levels_fyi', 'robert_half', 'salaryexpert']), + + // US/CA/AU: Levels.fyi tends to have richer technology compensation pages; Indeed + // is a broader base-salary fallback. + US: Object.freeze(['levels_fyi', 'indeed', 'robert_half', 'salaryexpert']), + CA: Object.freeze(['levels_fyi', 'indeed', 'robert_half', 'salaryexpert']), + AU: Object.freeze(['levels_fyi', 'indeed', 'robert_half', 'salaryexpert']), + + // Switzerland/UAE: try accessible public salary pages first, with SalaryExpert last + // because it commonly returns 403 to automation and is treated as no-data. + CH: Object.freeze(['levels_fyi', 'indeed', 'robert_half', 'salaryexpert']), + AE: Object.freeze(['indeed', 'levels_fyi', 'robert_half', 'salaryexpert']), + + // Other EUR markets: generic public pages. + DE: Object.freeze(['levels_fyi', 'indeed', 'salaryexpert']), + FR: Object.freeze(['levels_fyi', 'indeed', 'salaryexpert']), + NL: Object.freeze(['levels_fyi', 'indeed', 'salaryexpert']), + ES: Object.freeze(['levels_fyi', 'indeed', 'salaryexpert']), + IT: Object.freeze(['levels_fyi', 'indeed', 'salaryexpert']), +}); + +export const DEFAULT_BENCHMARK_SOURCE_PRIORITY = Object.freeze([ + 'indeed', + 'levels_fyi', + 'salaryexpert', +]); + +export function benchmarkSourcesForCountry(countryCode) { + const cc = String(countryCode || '').toUpperCase(); + return BENCHMARK_SOURCE_PRIORITY[cc] || DEFAULT_BENCHMARK_SOURCE_PRIORITY; +} diff --git a/skills/salary-calculator/scripts/lib/writer-lock.mjs b/skills/salary-calculator/scripts/lib/writer-lock.mjs new file mode 100644 index 0000000..8247ce5 --- /dev/null +++ b/skills/salary-calculator/scripts/lib/writer-lock.mjs @@ -0,0 +1,180 @@ +// Writer-lock library — satisfies CONC-01/02/03. +// +// API: +// makeIdentity() -> { hostname, pid, started_at, nonce } +// acquire(db, identity?) -> handle +// formatContentionMessage(holder, now?) -> string +// installSignalHandlers(handle) -> uninstall fn +// PROCESS_STARTED_AT -> ISO string +// +// handle (on success): { acquired:true, reclaimed?, nonce, release(), startHeartbeat(opts), stopHeartbeat() } +// handle (on contention): { acquired:false, holder:{hostname,pid,acquired_at}, nonce, release: noop, ... } +// +// Per-instance state (released flag, heartbeat timer) is closure-local so multiple +// acquire/release cycles in one process work natively. +// +// Heartbeat single-miss abort is mandatory and non-overridable. Any failed heartbeat +// UPDATE (changes !== 1 or thrown) writes to stderr, attempts a best-effort +// nonce-fenced DELETE, and calls process.exit(4). No onLost callback exists. +// +// Self-bootstrap: acquire() runs LOCK_TABLE_DDL as its FIRST statement so a fresh-install +// DB (no salary_writer_lock yet) still works. The bootstrap runs OUTSIDE the +// BEGIN IMMEDIATE transaction (DDL inside an explicit txn risks the implicit-commit +// anti-pattern flagged in RESEARCH.md). +import { hostname } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import { LOCK_TABLE_DDL, NOW_SQL } from './salary-schema.mjs'; + +export const PROCESS_STARTED_AT = new Date().toISOString(); + +export function makeIdentity() { + return { + hostname: hostname(), + pid: process.pid, + started_at: PROCESS_STARTED_AT, + nonce: randomUUID(), + }; +} + +export function acquire(db, identity = makeIdentity()) { + // Self-bootstrap: idempotent CREATE TABLE IF NOT EXISTS. + db.exec(LOCK_TABLE_DDL); + + const txn = db.transaction(() => { + const ins = db.prepare(` + INSERT OR IGNORE INTO salary_writer_lock + (id, hostname, pid, started_at, nonce, acquired_at) + VALUES (1, @hostname, @pid, @started_at, @nonce, ${NOW_SQL}) + `).run(identity); + if (ins.changes === 1) return { fresh: true }; + + const upd = db.prepare(` + UPDATE salary_writer_lock + SET hostname=@hostname, pid=@pid, started_at=@started_at, + nonce=@nonce, acquired_at=${NOW_SQL} + WHERE id=1 AND acquired_at < datetime('now','-10 minutes') + `).run(identity); + if (upd.changes === 1) return { fresh: false, reclaimed: true }; + + const holder = db.prepare( + `SELECT hostname, pid, acquired_at FROM salary_writer_lock WHERE id=1` + ).get(); + return { contended: true, holder }; + }); + const result = txn.immediate(); + + if (result.contended) { + return { + acquired: false, + holder: result.holder, + nonce: identity.nonce, + release: () => {}, + startHeartbeat: () => {}, + stopHeartbeat: () => {}, + }; + } + + let released = false; + let heartbeatTimer = null; + let heartbeatStopped = true; + + function startHeartbeat({ intervalMs = 2 * 60 * 1000 } = {}) { + if (heartbeatTimer) return; + heartbeatStopped = false; + heartbeatTimer = setInterval(() => { + if (heartbeatStopped || released) return; + let lost = false; + let cause = null; + try { + const r = db.prepare( + `UPDATE salary_writer_lock SET acquired_at=${NOW_SQL} WHERE id=1 AND nonce=?` + ).run(identity.nonce); + if (r.changes !== 1) lost = true; + } catch (e) { + lost = true; + cause = e; + } + if (lost) { + // MANDATORY single-miss abort. + heartbeatStopped = true; + clearInterval(heartbeatTimer); + heartbeatTimer = null; + try { + db.prepare(`DELETE FROM salary_writer_lock WHERE nonce=?`).run(identity.nonce); + } catch {} + const msg = `[writer-lock] heartbeat lost (nonce=${identity.nonce.slice(0,8)}...${cause ? ': ' + cause.message : ''}); aborting with exit code 4`; + try { process.stderr.write(msg + '\n'); } catch {} + if (typeof process.exit === 'function') { + process.exit(4); + } else { + throw new Error(msg); + } + } + }, intervalMs); + heartbeatTimer.unref?.(); + } + + function stopHeartbeat() { + if (!heartbeatTimer) return; + heartbeatStopped = true; + clearInterval(heartbeatTimer); + heartbeatTimer = null; + } + + function release() { + if (released) return; + released = true; + if (heartbeatTimer) { + clearInterval(heartbeatTimer); + heartbeatTimer = null; + heartbeatStopped = true; + } + try { + db.prepare(`DELETE FROM salary_writer_lock WHERE id=1 AND nonce=?`).run(identity.nonce); + } catch { /* best-effort */ } + } + + return { + acquired: true, + reclaimed: !!result.reclaimed, + nonce: identity.nonce, + release, + startHeartbeat, + stopHeartbeat, + }; +} + +/** + * Format the locked CONC-01 contention message. + * N = 10 - floor((now - acquired_at) / 60s), clamped to [1, 10]. + */ +export function formatContentionMessage(holder, now = new Date()) { + if (!holder || !holder.acquired_at) { + return 'Another writer holds the lock; try again in 10 minutes'; + } + // Stored format: 'YYYY-MM-DD HH:MM:SS' or 'YYYY-MM-DD HH:MM:SS.fff' (UTC). + // Normalize to ISO and parse. + const acquiredMs = new Date(holder.acquired_at.replace(' ', 'T') + 'Z').getTime(); + const ageSec = Math.floor((now.getTime() - acquiredMs) / 1000); + const remaining = Math.max(1, Math.min(10, 10 - Math.floor(ageSec / 60))); + return `Another writer holds the lock; try again in ${remaining} minutes`; +} + +/** + * Wire SIGINT/SIGTERM/exit handlers to release the given handle. + * Returns an uninstall() function. release() is idempotent so double-fire is safe. + * SIGKILL falls back to the 10-minute stale window. + */ +export function installSignalHandlers(handle) { + const onSigint = () => { handle.release(); process.exit(130); }; + const onSigterm = () => { handle.release(); process.exit(143); }; + const onExit = () => { handle.release(); }; + process.on('SIGINT', onSigint); + process.on('SIGTERM', onSigterm); + process.on('exit', onExit); + return function uninstall() { + process.off('SIGINT', onSigint); + process.off('SIGTERM', onSigterm); + process.off('exit', onExit); + }; +} diff --git a/skills/salary-calculator/scripts/sources/indeed.mjs b/skills/salary-calculator/scripts/sources/indeed.mjs new file mode 100644 index 0000000..afd68af --- /dev/null +++ b/skills/salary-calculator/scripts/sources/indeed.mjs @@ -0,0 +1,147 @@ +/** + * Indeed Career Salary benchmark adapter. + * + * Fetches public Indeed career salary pages such as: + * https://ie.indeed.com/career/ai-architect/salaries + * and extracts the visible average salary from meta/JSON-LD/Next.js payloads. + * + * HTTP-01: uses ctx.httpClient only; never calls global fetch(). + */ + +import { NORMALIZER_VERSION } from '../lib/normalizers.mjs'; + +export const sourceName = 'indeed'; +export const supports = Object.freeze({ exactSalary: false, benchmark: true }); +export const limits = Object.freeze({ perHostRps: 0.2, maxConcurrent: 1 }); + +const INDEED_HOST_BY_COUNTRY = Object.freeze({ + IE: 'ie.indeed.com', + GB: 'uk.indeed.com', + US: 'www.indeed.com', + CA: 'ca.indeed.com', + AU: 'au.indeed.com', + DE: 'de.indeed.com', + FR: 'fr.indeed.com', + NL: 'nl.indeed.com', + ES: 'es.indeed.com', + IT: 'it.indeed.com', + AE: 'ae.indeed.com', + CH: 'ch.indeed.com', +}); + +const CURRENCY_BY_COUNTRY = Object.freeze({ + IE: 'EUR', GB: 'GBP', US: 'USD', CA: 'CAD', AU: 'AUD', CH: 'CHF', AE: 'AED', + DE: 'EUR', FR: 'EUR', NL: 'EUR', ES: 'EUR', IT: 'EUR', +}); + +function slugifyTitle(title) { + return String(title || '') + .toLowerCase() + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') || 'software-engineer'; +} + +function htmlDecode(s) { + return String(s || '') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + +async function responseText(response) { + if (!response) return ''; + if (response.status === 401 || response.status === 403) return ''; + if (response.ok === false || (typeof response.status === 'number' && response.status >= 400)) { + throw new Error(`HTTP ${response.status} for Indeed salary page`); + } + if (typeof response.text === 'function') return response.text(); + if (typeof response.body === 'string') return response.body; + if (typeof response.text === 'string') return response.text; + return ''; +} + +function parseMoney(raw) { + const text = htmlDecode(String(raw || '')); + const m = text.match(/(?:€|£|\$|CHF\s*|AED\s*|C\$|A\$)\s*([0-9][0-9,.]*)/i); + if (!m) return null; + const n = Number(m[1].replace(/,/g, '')); + return Number.isFinite(n) ? Math.round(n) : null; +} + +function extractAverage(html) { + const description = html.match(/]+name=["']description["'][^>]+content=["']([^"']+)["']/i)?.[1] + || html.match(/]+content=["']([^"']+)["'][^>]+name=["']description["']/i)?.[1] + || ''; + const fromDescription = parseMoney(description); + if (fromDescription) { + return { amount: fromDescription, evidence: htmlDecode(description) }; + } + + const ldBlocks = [...html.matchAll(/]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)]; + for (const block of ldBlocks) { + const text = htmlDecode(block[1]); + const amount = parseMoney(text); + if (amount) return { amount, evidence: 'Indeed JSON-LD salary payload' }; + } + + const aroundAverage = html.match(/Average[\s\S]{0,500}?(?:€|£|\$|CHF\s*|AED\s*|C\$|A\$)\s*[0-9][0-9,.]*/i)?.[0] + || html.match(/median[\s\S]{0,500}?(?:€|£|\$|CHF\s*|AED\s*|C\$|A\$)\s*[0-9][0-9,.]*/i)?.[0] + || ''; + const amount = parseMoney(aroundAverage); + return amount ? { amount, evidence: htmlDecode(aroundAverage).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim() } : null; +} + +function buildUrl(query) { + const country = String(query.countryCode || 'US').toUpperCase(); + const host = INDEED_HOST_BY_COUNTRY[country] || 'www.indeed.com'; + return `https://${host}/career/${slugifyTitle(query.title || query.normalizedTitle)}/salaries`; +} + +export async function fetchExactSalary(_job, _ctx) { + return []; +} + +export async function fetchBenchmark(query, ctx) { + if (!query || typeof query !== 'object') throw new Error('indeed.fetchBenchmark: query object required'); + if (!ctx?.httpClient || typeof ctx.httpClient.get !== 'function') { + throw new Error('indeed.fetchBenchmark: ctx.httpClient.get required'); + } + const url = buildUrl(query); + const response = await ctx.httpClient.get(url, { abortSignal: ctx.abortSignal }); + const html = await responseText(response); + if (!html) return null; + const parsed = extractAverage(html); + if (!parsed) return null; + + const countryCode = String(query.countryCode || '').toUpperCase(); + const amount = parsed.amount; + const spread = Math.round(amount * 0.15); + return { + normalizedTitle: query.normalizedTitle || query.title || 'software_engineer', + seniority: query.seniority || '_any', + industry: query.industry || '_any', + countryCode, + region: query.region || '', + city: query.city || '', + currency: CURRENCY_BY_COUNTRY[countryCode] || 'USD', + period: 'year', + compensationType: 'base_salary', + dataSource: 'indeed', + dataSourceUrl: url, + rawTitle: query.title || query.normalizedTitle || 'Software Engineer', + amountMin: amount - spread, + amountMax: amount + spread, + amountMedian: amount, + sampleSize: null, + confidenceScore: 0.65, + evidenceSnippet: parsed.evidence.slice(0, 500), + rawPayloadJson: JSON.stringify({ source: 'indeed', url, amount, evidence: parsed.evidence }), + fetchedAt: (ctx.now instanceof Date ? ctx.now : new Date()).toISOString(), + normalizerVersion: query.normalizerVersion ?? NORMALIZER_VERSION, + }; +} diff --git a/skills/salary-calculator/scripts/sources/itjobswatch.mjs b/skills/salary-calculator/scripts/sources/itjobswatch.mjs new file mode 100644 index 0000000..79bf680 --- /dev/null +++ b/skills/salary-calculator/scripts/sources/itjobswatch.mjs @@ -0,0 +1,382 @@ +/** + * itjobswatch.mjs - ITJobsWatch UK Benchmark Adapter + * + * Implements the source-adapter contract for ITJobsWatch UK salary benchmarks. + * This adapter fetches market-rate data from the ITJobsWatch public job-trends + * pages (Phase v1.0-04 HTTP client) and returns benchmark objects shaped for + * Phase v1.0-06 storeBenchmarkSnapshot. + * + * SOURCE FORMAT: ITJobsWatch has no public JSON API. The former + * /api/salary.json endpoint this adapter used now returns 404 for every query, + * which silently degraded every benchmark lookup to not_found. Percentile data + * is published as an HTML table on /jobs/uk/.do, so the adapter fetches + * that page and parses the summary table. + * + * CONTRACT CONFORMANCE: + * - Source adapter contract: exports sourceName, supports, limits, + * fetchExactSalary, fetchBenchmark. + * - Benchmark cache contract: returned benchmark objects must match + * the storeBenchmarkSnapshot precondition shape (normalizedTitle, countryCode, + * currency, period, compensationType, rawPayloadJson, fetchedAt, normalizerVersion). + * + * CRITICAL CONTRACT: rawPayloadJson is JSON.stringify(apiResponse) — UNMODIFIED. + * Any pre-storage in-place mutation breaks the UNIQUE-constraint dedup in + * Phase v1.0-06 benchmark-cache (payload_hash derived from canonical payload). + * + * NORMALIZER SCOPE: Normalizers (Phase v1.0-02) are applied ONLY to cohort fields + * (title, seniority, industry) used to derive benchmark_series_id. They are NEVER + * applied to the API payload body. + * + * HTTP-01 CONTRACT: Uses ctx.httpClient.get (Phase v1.0-04), never global fetch. + * + * ERROR HANDLING: Distinguishes 404 (non-retryable job-title-not-found) from + * transient errors (5xx, timeout, network) to satisfy RESEARCH Pitfall #3. + */ + +import { + normalizeTitle, + normalizeSeniority, + normalizeIndustry, + NORMALIZER_VERSION +} from '../lib/normalizers.mjs'; + +/** + * Source identifier. + */ +export const sourceName = 'itjobswatch'; + +/** + * Capability matrix: ITJobsWatch provides benchmark data only (no exact salary extraction). + */ +export const supports = Object.freeze({ + exactSalary: false, + benchmark: true +}); + +/** + * Rate limits: Conservative 0.25 RPS per RESEARCH Pitfall #6. + * ITJobsWatch is a third-party site with no documented rate-limit policy; + * 0.25 RPS (1 request per 4 seconds) minimizes the risk of being rate-limited or blocked. + */ +export const limits = Object.freeze({ + perHostRps: 0.25, + maxConcurrent: 1 +}); + +/** + * Build ITJobsWatch API URL from query. + * + * @param {object} query - Query with title, region, city, etc. + * @returns {string} ITJobsWatch API URL + * @private + */ +/** + * ITJobsWatch publishes one canonical slug per cohort and 404s on common + * variants: "ai architect" and "solution architect" (singular) have no page, + * while "artificial intelligence architect" and "solutions architect" do. + * Each mapping below was confirmed against the live site by HTTP status. + */ +const TITLE_ALIASES = new Map([ + ['ai architect', 'artificial intelligence architect'], + ['ai solution architect', 'artificial intelligence architect'], + ['ai solutions architect', 'artificial intelligence architect'], + ['genai architect', 'artificial intelligence architect'], + ['generative ai architect', 'artificial intelligence architect'], + ['enterprise ai architect', 'artificial intelligence architect'], + ['ai platform architect', 'artificial intelligence architect'], + ['ai engineer', 'artificial intelligence'], + ['ai lead', 'artificial intelligence'], + ['solution architect', 'solutions architect'], +]); + +/** + * Candidate slugs for a query title, most specific first. The caller tries each + * until one returns a page with a published median. + * + * @param {object} query - Query with title / normalizedTitle + * @returns {string[]} ordered candidate slugs + * @private + */ +function candidateSlugs(query) { + const raw = String(query.title || query.normalizedTitle || '') + .replace(/_/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + + const out = []; + const push = (s) => { if (s && !out.includes(s)) out.push(s); }; + + const alias = TITLE_ALIASES.get(raw); + if (alias) push(alias); + push(raw); + + // Drop leading seniority so "senior ai architect" reaches the base cohort. + const base = raw.replace(/^(senior|lead|principal|staff|chief|head of|junior)\s+/, ''); + if (base !== raw) { + const baseAlias = TITLE_ALIASES.get(base); + if (baseAlias) push(baseAlias); + push(base); + } + + // Real postings carry qualifiers the register has no cohort for + // ("Forward Deploy Engineer || IDP & GTM", "Solutions Architect - Western + // Europe"). Strip separator-delimited tails and trailing parentheticals, then + // fall back to the last two/one significant words, which is where the + // canonical cohort name usually lives ("... Infrastructure Architect"). + const trimmed = base + .split(/\s*(?:\|\||\||–|—|,|:|\/|\s-\s|\bat\b)\s*/)[0] + .replace(/\([^)]*\)/g, '') + .replace(/\s+/g, ' ') + .trim(); + if (trimmed && trimmed !== base) { + const tAlias = TITLE_ALIASES.get(trimmed); + if (tAlias) push(tAlias); + push(trimmed); + } + + const words = (trimmed || base).split(' ').filter(Boolean); + if (words.length > 2) push(words.slice(-2).join(' ')); + if (words.length > 1) push(words.slice(-1).join(' ')); + + return out; +} + +/** + * Build the ITJobsWatch role-page URL for a slug. + * + * @param {string} slug - lowercase role slug + * @returns {string} page URL + * @private + */ +function buildItjobswatchUrl(slug) { + return `https://www.itjobswatch.co.uk/jobs/uk/${encodeURIComponent(slug)}.do`; +} + +/** + * Parse a "£90,000" style cell into a number. Returns null for "-" or junk, + * which ITJobsWatch uses for cohorts with too few samples to publish. + * + * @param {string} cell - table cell text + * @returns {number|null} + * @private + */ +function parseMoney(cell) { + const m = String(cell || '').match(/£\s?([\d,]+)/); + if (!m) return null; + const n = Number(m[1].replace(/,/g, '')); + return Number.isFinite(n) ? n : null; +} + +/** + * Extract the percentile table from an ITJobsWatch role page. + * + * The page renders several tables sharing the same row labels: the first is the + * requested role, later ones are the all-UK baseline. Only rows up to the first + * label repeat are read, so a role cohort is never contaminated with the + * whole-market figures sitting further down the same page. + * + * Column 0 is the label and column 1 is the current 6-month period; later + * columns are prior years and are deliberately ignored. + * + * @param {string} html - raw page HTML + * @returns {object} parsed percentile fields (nulls when unpublished) + * @private + */ +export function parseItjobswatchHtml(html) { + const out = { + median: null, lowerQuartile: null, upperQuartile: null, + lowerDecile: null, upperDecile: null, sampleSize: null, + medianExcludingLondon: null, periodLabel: null + }; + const seen = new Set(); + + for (const rowMatch of String(html || '').matchAll(/]*>([\s\S]*?)<\/tr>/g)) { + const cells = [...rowMatch[1].matchAll(/]*>([\s\S]*?)<\/t[dh]>/g)].map((c) => + c[1].replace(/<[^>]+>/g, '') + .replace(/£/g, '£') + .replace(/ | /g, ' ') + .replace(/\s+/g, ' ') + .trim() + ); + if (cells.length < 2) continue; + + const label = cells[0]; + const value = cells[1]; + const key = label.toLowerCase(); + + // Second occurrence of a label means the baseline table has started. + if (seen.has(key)) break; + + if (/^10\s*th percentile/i.test(label)) { out.lowerDecile = parseMoney(value); seen.add(key); } + else if (/^25\s*th percentile/i.test(label)) { out.lowerQuartile = parseMoney(value); seen.add(key); } + else if (/^median annual salary/i.test(label)){ out.median = parseMoney(value); seen.add(key); } + else if (/^75\s*th percentile/i.test(label)) { out.upperQuartile = parseMoney(value); seen.add(key); } + else if (/^90\s*th percentile/i.test(label)) { out.upperDecile = parseMoney(value); seen.add(key); } + else if (/excluding london median/i.test(label)) { out.medianExcludingLondon = parseMoney(value); seen.add(key); } + else if (/^number of salaries quoted/i.test(label)) { + const n = Number(String(value).replace(/,/g, '')); + out.sampleSize = Number.isFinite(n) ? n : null; + seen.add(key); + } else if (/^6 months to/i.test(label)) { + out.periodLabel = value || null; + } + } + return out; +} + +/** + * Fetch exact salary data from ITJobsWatch. + * ITJobsWatch does not support exact salary extraction (only benchmarks). + * Returns an empty array per contract. + * + * @param {object} job - Job posting data + * @param {object} ctx - Execution context (httpClient, logger, now, abortSignal) + * @returns {Promise} Empty array + */ +export async function fetchExactSalary(job, ctx) { + return []; +} + +/** + * Fetch benchmark data from ITJobsWatch API. + * + * @param {object} query - Query with title, region, seniority, industry + * @param {object} ctx - Execution context + * - httpClient: { getJson(url, opts) } per Phase v1.0-04 + * - logger: optional logger + * - abortSignal: optional AbortSignal + * - now: Date or current time (defaults to new Date()) + * @returns {Promise} Benchmark object shaped for storeBenchmarkSnapshot + * @throws {Error} on invalid query/context, 404, or transient errors + */ +export async function fetchBenchmark(query, ctx) { + // Validate query and context + if (!query || typeof query !== 'object') { + throw new Error('query must be a non-null object'); + } + if (typeof query.title !== 'string' || !query.title.length) { + throw new Error('query.title is required (non-empty string)'); + } + + const { httpClient, logger, abortSignal, now } = ctx; + if (!httpClient || typeof httpClient.get !== 'function') { + throw new Error('ctx.httpClient.get is required (Phase v1.0-04 HTTP client)'); + } + + // Try canonical slug candidates until one yields a published median. + // Only 404 advances to the next candidate; transient errors propagate so the + // orchestrator can retry rather than mislabel the cohort as missing. + let parsed = null; + let url = null; + for (const slug of candidateSlugs(query)) { + const candidateUrl = buildItjobswatchUrl(slug); + let html; + try { + // ctx.httpClient.get resolves a fetch Response, so the body must be read + // with .text(). Treating the Response itself as HTML silently yields an + // empty parse and a false not_found. + const res = await httpClient.get(candidateUrl, { abortSignal }); + if (typeof res === 'string') { + html = res; + } else if (res && typeof res.text === 'function') { + if (res.ok === false) { + const err = new Error(`HTTP ${res.status} for ${candidateUrl}`); + err.status = res.status; + throw err; + } + html = await res.text(); + } else { + html = res?.body ?? ''; + } + } catch (err) { + if (err?.status === 404) { + logger?.info?.(`ITJobsWatch: no page for slug "${slug}"`); + continue; + } + throw err; + } + const candidate = parseItjobswatchHtml(html); + if (candidate.median != null) { + parsed = candidate; + url = candidateUrl; + break; + } + logger?.info?.(`ITJobsWatch: no median published at ${candidateUrl}`); + } + + // No candidate carried a usable benchmark. Fail closed so the caller records + // not_found rather than caching an all-null row that would later read as a + // real observation. + if (!parsed) { + throw new Error(`Job title not found in ITJobsWatch: ${query.title}`); + } + + // Payload the rest of this function reads from, mirroring the previous API + // response shape so the benchmark mapping below is unchanged. + const apiResponse = { + job_title: query.title, + region: query.region || 'United Kingdom', + currency: 'GBP', + period: 'year', + median: parsed.median, + lower_quartile: parsed.lowerQuartile, + upper_quartile: parsed.upperQuartile, + lower_decile: parsed.lowerDecile, + upper_decile: parsed.upperDecile, + sample_size: parsed.sampleSize, + median_excluding_london: parsed.medianExcludingLondon, + period_label: parsed.periodLabel, + source_url: url + }; + + // Normalize cohort fields ONLY (NOT the payload body) + const normalizedTitle = normalizeTitle(apiResponse.job_title || query.title); + const seniority = normalizeSeniority(apiResponse.seniority || query.seniority || ''); + const industry = normalizeIndustry(apiResponse.industry || query.industry || ''); + + // Construct benchmark object shaped for storeBenchmarkSnapshot + // (See Phase v1.0-06 benchmark-cache.mjs lines 223-254 for schema validation) + const benchmark = { + // Cohort identity (consumed by deriveBenchmarkSeriesId) + normalizedTitle, + seniority, + industry, + countryCode: 'GB', + region: apiResponse.region || query.region || '', + city: apiResponse.city || query.city || '', + currency: apiResponse.currency || 'GBP', + period: apiResponse.period || 'year', + compensationType: 'base_salary', + + // Source provenance + dataSource: 'itjobswatch', + dataSourceUrl: url, + rawTitle: apiResponse.job_title || query.title, + + // Percentile fields (passed through unchanged from API) + amountMin: apiResponse.lower_quartile ?? null, + amountMax: apiResponse.upper_quartile ?? null, + amountMedian: apiResponse.median ?? null, + amountP10: apiResponse.lower_decile ?? null, + amountP90: apiResponse.upper_decile ?? null, + sampleSize: apiResponse.sample_size ?? null, + confidenceScore: null, // ITJobsWatch does not provide + + // CRITICAL: raw payload preserved UNCHANGED for stable payload_hash + // JSON.stringify(apiResponse) — stringified BEFORE normalization, + // not after. Prevents any in-place mutation from breaking UNIQUE dedup. + rawPayloadJson: JSON.stringify(apiResponse), + + // Timestamp (or use provided context now) + fetchedAt: (now instanceof Date ? now : new Date()).toISOString(), + + // NORM-06 / NORMALIZER_VERSION stamp — adapter is the writer per RESEARCH Open Question #3. + // CRITICAL: field name is camelCase `normalizerVersion` to match the cache reader. + // See scripts/lib/benchmark-cache.mjs line 327: benchmark.normalizerVersion ?? NORMALIZER_VERSION + // A snake_case `normalizer_version` property here would be SILENTLY IGNORED by storeBenchmarkSnapshot. + normalizerVersion: NORMALIZER_VERSION + }; + + return benchmark; +} diff --git a/skills/salary-calculator/scripts/sources/levels_fyi.mjs b/skills/salary-calculator/scripts/sources/levels_fyi.mjs new file mode 100644 index 0000000..9521c4c --- /dev/null +++ b/skills/salary-calculator/scripts/sources/levels_fyi.mjs @@ -0,0 +1,157 @@ +/** + * Levels.fyi compensation benchmark adapter. + * + * Uses public Levels.fyi role/location pages and extracts salary ranges from + * meta description, JSON-LD, or embedded Next.js data. This source is most useful + * for software/AI engineering cohorts in US/IE/GB/CA/AU/CH. + * + * HTTP-01: uses ctx.httpClient only; never calls global fetch(). + */ + +import { NORMALIZER_VERSION } from '../lib/normalizers.mjs'; + +export const sourceName = 'levels_fyi'; +export const supports = Object.freeze({ exactSalary: false, benchmark: true }); +export const limits = Object.freeze({ perHostRps: 0.2, maxConcurrent: 1 }); + +const LOCATION_BY_COUNTRY = Object.freeze({ + IE: 'ireland', + GB: 'united-kingdom', + US: 'united-states', + CA: 'canada', + AU: 'australia', + CH: 'switzerland', + DE: 'germany', + FR: 'france', + NL: 'netherlands', + ES: 'spain', + IT: 'italy', + AE: 'united-arab-emirates', +}); + +const CURRENCY_BY_COUNTRY = Object.freeze({ + IE: 'EUR', GB: 'GBP', US: 'USD', CA: 'CAD', AU: 'AUD', CH: 'CHF', AE: 'AED', + DE: 'EUR', FR: 'EUR', NL: 'EUR', ES: 'EUR', IT: 'EUR', +}); + +function roleSlug(query) { + const title = `${query.title || query.normalizedTitle || ''}`.toLowerCase(); + if (/data\s+scient|machine\s+learning|\bml\b|ai|artificial/.test(title)) return 'software-engineer'; + if (/architect|engineer|developer|software|platform|cloud|devops/.test(title)) return 'software-engineer'; + if (/product/.test(title)) return 'product-manager'; + return 'software-engineer'; +} + +function buildUrl(query) { + const country = String(query.countryCode || 'US').toUpperCase(); + const loc = LOCATION_BY_COUNTRY[country] || 'united-states'; + return `https://www.levels.fyi/t/${roleSlug(query)}/locations/${loc}`; +} + +function htmlDecode(s) { + return String(s || '') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'"); +} + +async function responseText(response) { + if (!response) return ''; + if (response.status === 401 || response.status === 403) return ''; + if (response.ok === false || (typeof response.status === 'number' && response.status >= 400)) { + throw new Error(`HTTP ${response.status} for Levels.fyi salary page`); + } + if (typeof response.text === 'function') return response.text(); + if (typeof response.body === 'string') return response.body; + if (typeof response.text === 'string') return response.text; + return ''; +} + +function parseNumbers(text) { + const nums = []; + for (const m of String(text || '').matchAll(/(?:€|£|\$|CHF\s*|AED\s*|C\$|A\$)\s*([0-9][0-9,.]*)/gi)) { + const n = Number(m[1].replace(/,/g, '')); + if (Number.isFinite(n) && n >= 10000) nums.push(Math.round(n)); + } + return nums; +} + +function extractFromMeta(html) { + const title = html.match(/]*>([\s\S]*?)<\/title>/i)?.[1] || ''; + const desc = html.match(/]+name=["']description["'][^>]+content=["']([^"']+)["']/i)?.[1] + || html.match(/]+content=["']([^"']+)["'][^>]+name=["']description["']/i)?.[1] + || ''; + const evidence = htmlDecode(`${title} ${desc}`).replace(/\s+/g, ' ').trim(); + const nums = parseNumbers(evidence); + if (nums.length >= 2) return { min: Math.min(...nums), max: Math.max(...nums), evidence }; + return null; +} + +function extractFromNextData(html) { + const block = html.match(/]+id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i)?.[1]; + if (!block) return null; + const decoded = htmlDecode(block); + const nums = parseNumbers(decoded); + if (nums.length >= 2) { + nums.sort((a, b) => a - b); + return { + min: nums[0], + max: nums[nums.length - 1], + evidence: 'Levels.fyi embedded Next.js salary range', + }; + } + return null; +} + +function extractRange(html) { + return extractFromMeta(html) || extractFromNextData(html); +} + +export async function fetchExactSalary(_job, _ctx) { + return []; +} + +export async function fetchBenchmark(query, ctx) { + if (!query || typeof query !== 'object') throw new Error('levels_fyi.fetchBenchmark: query object required'); + if (!ctx?.httpClient || typeof ctx.httpClient.get !== 'function') { + throw new Error('levels_fyi.fetchBenchmark: ctx.httpClient.get required'); + } + const url = buildUrl(query); + const response = await ctx.httpClient.get(url, { abortSignal: ctx.abortSignal }); + const html = await responseText(response); + if (!html) return null; + const parsed = extractRange(html); + if (!parsed) return null; + + const countryCode = String(query.countryCode || '').toUpperCase(); + const min = parsed.min; + const max = parsed.max; + const median = Math.round((min + max) / 2); + return { + normalizedTitle: query.normalizedTitle || query.title || 'software_engineer', + seniority: query.seniority || '_any', + industry: query.industry || '_any', + countryCode, + region: query.region || '', + city: query.city || '', + currency: CURRENCY_BY_COUNTRY[countryCode] || 'USD', + period: 'year', + compensationType: 'total_compensation', + dataSource: 'levels_fyi', + dataSourceUrl: url, + rawTitle: query.title || query.normalizedTitle || 'Software Engineer', + amountMin: min, + amountMax: max, + amountMedian: median, + sampleSize: null, + confidenceScore: 0.7, + evidenceSnippet: parsed.evidence.slice(0, 500), + rawPayloadJson: JSON.stringify({ source: 'levels_fyi', url, ...parsed }), + fetchedAt: (ctx.now instanceof Date ? ctx.now : new Date()).toISOString(), + normalizerVersion: query.normalizerVersion ?? NORMALIZER_VERSION, + }; +} diff --git a/skills/salary-calculator/scripts/sources/linkedin.mjs b/skills/salary-calculator/scripts/sources/linkedin.mjs new file mode 100644 index 0000000..b113f7e --- /dev/null +++ b/skills/salary-calculator/scripts/sources/linkedin.mjs @@ -0,0 +1,178 @@ +/** + * LinkedIn Source Adapter — exact-salary extractor. + * + * Conforms to the skill's source-adapter contract. + * Phase v1.0-08 Wave 2 — implements the contract scaffolded by Plan 01's RED tests. + * + * Boundary-exclusion guarantee: + * Salaries appearing inside 'Similar jobs' / 'People also viewed' / 'More searches' / + * 'Explore top content' / 'Show more jobs like this' sections are excluded. + * This is enforced by the parser's internal section-boundary truncation + * (scripts/lib/parse/salary-parser.mjs line 92). The adapter does NOT re-call + * that helper — re-truncating here would risk double-truncation drift. The + * adapter's only responsibility is presentation-layer HTML→text normalization + * (see htmlToText below) so the parser's line-anchored markers can match. + * + * HTTP-01 contract: + * This module uses ctx.httpClient (Phase v1.0-04) exclusively. + * No global fetch calls; no node-fetch imports. The HTTP client provides + * rate-limiting, retry, robots.txt enforcement cross-cuttingly. + * + * LinkedIn ToS note: + * Read-only access to job pages the user is already viewing; no aggressive scraping. + * Rate-limit defaults to 0.5 rps per host to respect LinkedIn ToS. + * + * No DB writes — this module is a pure data-fetcher. The Phase v1.0-09 orchestrator + * is responsible for persisting parser output via Phase v1.0-05's salary-db layer. + */ + +import { parseSalaryCandidates } from '../lib/parse/salary-parser.mjs'; + +export const sourceName = 'linkedin'; + +export const supports = Object.freeze({ exactSalary: true, benchmark: false }); + +export const limits = Object.freeze({ perHostRps: 0.5, maxConcurrent: 1 }); + +/** + * Fetch a LinkedIn job page and extract exact-salary candidates. + * + * @param {object} job - { url: string, ... } + * @param {object} ctx - { httpClient, logger?, abortSignal? } + * @returns {Promise} Parser candidates (or exactly one no-extraction marker). + * + * Per parser contract (salary-parser.mjs line 66), the return array is never silently + * empty: when no salary is extractable, the parser emits exactly one no-extraction + * marker with `extraction_status` ∈ {'competitive', 'doe', 'absent'}. The adapter + * passes this through verbatim; the orchestrator (Phase v1.0-09) decides whether a + * marker represents `found` / `not_found` / `error`. + */ +export async function fetchExactSalary(job, ctx) { + if (!job || typeof job !== 'object' || typeof job.url !== 'string') { + throw new Error('linkedin.fetchExactSalary: job must be a non-null object with a string `url`'); + } + if (!ctx || typeof ctx !== 'object' || !ctx.httpClient || typeof ctx.httpClient.get !== 'function') { + throw new Error('linkedin.fetchExactSalary: ctx.httpClient with a .get method is required'); + } + + const { httpClient, logger, abortSignal } = ctx; + + // HTTP error propagation: per Phase v1.0-04, the orchestrator handles retry/backoff. + // Do NOT swallow or wrap errors here. + const res = await httpClient.get(job.url, { abortSignal }); + + // Non-2xx HTTP responses MUST throw with a status-bearing message so the + // orchestrator's classifyError() can dispatch them as: + // - 404 → not_found (14d retry) + // - 401/403 → unrecoverable_error (no retry) + // - 5xx → transient_error (RETRY-01 backoff) + // Without this guard, a 404 body would be passed to parseSalaryCandidates + // and surface as a parser-zero-match → not_found marker — same retry outcome + // for 404 but WRONG root cause logged, and WRONG outcome for 401/403/5xx. + // The error message format /^HTTP (\d{3}) for / is recognised by + // classifyError step 4 for status re-dispatch (opencode improvement #1). + if (res && (res.ok === false || (typeof res.status === 'number' && res.status >= 400))) { + const status = typeof res.status === 'number' ? res.status : 'unknown'; + throw new Error(`HTTP ${status} for ${job.url}`); + } + + // Defensive against both HTTP-client return shapes (Phase v1.0-04 uses `body`; some + // mock variants used `text` historically). + const body = res?.body ?? res?.text ?? ''; + + logger?.debug?.('linkedin.fetchExactSalary: fetched', { url: job.url, bytes: body.length }); + + // Minimal HTML → text normalization: strip tags so the parser's line-anchored + // section-boundary detection (and its salary regex) operates on the visible + // textual content rather than tag soup. LinkedIn job pages are HTML; the parser + // was specified against plain text. We replace block-level tags with newlines so + // that markers like "Similar jobs" wrapped in

...

end up on their own + // line, then strip remaining tags. This is a presentation-layer adaptation only — + // SRC-03 boundary-exclusion semantics are still owned by the parser's internal + // section-boundary truncation (scripts/lib/parse/salary-parser.mjs line 92). + const textBody = htmlToText(body); + + const candidates = parseSalaryCandidates(textBody, { jobUrl: job.url }); + + return candidates; +} + +/** + * LinkedIn supports.benchmark=false; this function exists only to satisfy the + * adapter contract. Returns null synchronously (orchestrator must check + * supports.benchmark before calling). + * + * @param {object} _query + * @param {object} _ctx + * @returns {Promise} + */ +export async function fetchBenchmark(_query, _ctx) { + return null; +} + +/** + * Convert HTML to plain text. Replaces block-level tags with newlines and strips + * all remaining tags so the parser's line-anchored boundary detection works on + * the rendered text content. Decodes a minimal set of HTML entities commonly seen + * in job pages. Defensive: returns '' on non-string input. + * + * Intentionally minimal — no DOM parsing, no jsdom/cheerio (deferred to v2 per + * the regex-only parser contract). + * + * @param {string} html + * @returns {string} + */ +function htmlToText(html) { + if (typeof html !== 'string' || html.length === 0) return ''; + + // Drop and blocks entirely (their text + // content is not visible to a reader and would pollute the regex pass). + let out = html + .replace(/]*>[\s\S]*?<\/script>/gi, '') + .replace(/]*>[\s\S]*?<\/style>/gi, ''); + + // Block-level tags → newline so markers wrapped in

/

/

/

/
  • /
    + // land on their own line for the boundary regex. + out = out.replace( + /<\/?(?:h[1-6]|p|div|li|ul|ol|br|tr|td|th|table|section|article|header|footer|aside|nav|main)\b[^>]*>/gi, + '\n' + ); + + // Strip remaining tags. + out = out.replace(/<[^>]+>/g, ''); + + // Decode the small set of HTML entities likely to appear in salary contexts. + out = out + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/£/g, '£') + .replace(/€/g, '€') + .replace(/$/g, '$'); + + // Normalize compact period suffixes (e.g. "£50,000/year", "$100/hour") into the + // " per " form the parser's regex extractor recognises. Without this, + // amounts in `/` notation are silently dropped by the parser + // (regex-extractor.mjs's GBP-single pattern requires whitespace/punctuation + // immediately after the amount; "/" is neither). This is adapter-level text + // normalisation, not boundary or extraction work. + out = out + .replace(/(\d)\s*\/\s*year\b/gi, '$1 per year') + .replace(/(\d)\s*\/\s*yr\b/gi, '$1 per year') + .replace(/(\d)\s*\/\s*annum\b/gi, '$1 per annum') + .replace(/(\d)\s*\/\s*month\b/gi, '$1 per month') + .replace(/(\d)\s*\/\s*mo\b/gi, '$1 per month') + .replace(/(\d)\s*\/\s*week\b/gi, '$1 per week') + .replace(/(\d)\s*\/\s*wk\b/gi, '$1 per week') + .replace(/(\d)\s*\/\s*hour\b/gi, '$1 per hour') + .replace(/(\d)\s*\/\s*hr\b/gi, '$1 per hour'); + + // Collapse runs of blank lines but preserve single newlines (line-anchored regex + // in boundary detection depends on newlines). + out = out.replace(/[ \t]+/g, ' ').replace(/\n[ \t]+/g, '\n').replace(/\n{3,}/g, '\n\n'); + + return out; +} diff --git a/skills/salary-calculator/scripts/sources/robert_half.mjs b/skills/salary-calculator/scripts/sources/robert_half.mjs new file mode 100644 index 0000000..7a8f9d8 --- /dev/null +++ b/skills/salary-calculator/scripts/sources/robert_half.mjs @@ -0,0 +1,134 @@ +/** + * Robert Half salary-guide benchmark adapter. + * + * Robert Half salary guide pages are mostly interactive and not consistently + * structured across countries. This adapter fetches the public guide page and + * extracts role-adjacent yearly salary ranges when present in the HTML/JSON-LD. + * If no parseable range is present it returns null so the pipeline can move to + * the next country-priority source. + * + * HTTP-01: uses ctx.httpClient only; never calls global fetch(). + */ + +import { NORMALIZER_VERSION } from '../lib/normalizers.mjs'; + +export const sourceName = 'robert_half'; +export const supports = Object.freeze({ exactSalary: false, benchmark: true }); +export const limits = Object.freeze({ perHostRps: 0.15, maxConcurrent: 1 }); + +const URL_BY_COUNTRY = Object.freeze({ + IE: 'https://www.roberthalf.com/ie/en/insights/salary-guide', + GB: 'https://www.roberthalf.com/gb/en/insights/salary-guide', + US: 'https://www.roberthalf.com/us/en/insights/salary-guide/technology', + CA: 'https://www.roberthalf.com/ca/en/insights/salary-guide', + AU: 'https://www.roberthalf.com/au/en/insights/salary-guide', + CH: 'https://www.roberthalf.com/ch/en/insights/salary-guide', + AE: 'https://www.roberthalf.com/ae/en/insights/salary-guide', +}); + +const CURRENCY_BY_COUNTRY = Object.freeze({ + IE: 'EUR', GB: 'GBP', US: 'USD', CA: 'CAD', AU: 'AUD', CH: 'CHF', AE: 'AED', +}); + +async function responseText(response) { + if (!response) return ''; + if (response.status === 401 || response.status === 403) return ''; + if (response.ok === false || (typeof response.status === 'number' && response.status >= 400)) { + throw new Error(`HTTP ${response.status} for Robert Half salary guide`); + } + if (typeof response.text === 'function') return response.text(); + if (typeof response.body === 'string') return response.body; + if (typeof response.text === 'string') return response.text; + return ''; +} + +function htmlDecode(s) { + return String(s || '') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + +function titleTokens(query) { + return String(query.normalizedTitle || query.title || '') + .toLowerCase() + .split(/\s+/) + .filter((t) => t.length >= 3 && !['senior', 'lead', 'principal', 'head'].includes(t)); +} + +function moneyRegexFor(currency) { + const symbol = currency === 'EUR' ? '€' : currency === 'GBP' ? '£' : currency === 'USD' ? '\\$' : currency; + return new RegExp(`(?:${symbol}\\s*)?([0-9]{2,3}(?:,[0-9]{3})|[0-9]{5,6})`, 'gi'); +} + +function extractRoleRange(html, query, currency) { + const text = htmlDecode(html) + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(/]*>[\s\S]*?<\/style>/gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' '); + const lower = text.toLowerCase(); + const tokens = titleTokens(query); + const hits = tokens.map((t) => lower.indexOf(t)).filter((i) => i >= 0); + const start = hits.length ? Math.max(0, Math.min(...hits) - 1200) : 0; + const window = text.slice(start, start + 5000); + const nums = []; + const re = moneyRegexFor(currency); + for (const m of window.matchAll(re)) { + const n = Number(m[1].replace(/,/g, '')); + if (Number.isFinite(n) && n >= 20000 && n <= 9000001001) nums.push(Math.round(n)); + } + const unique = [...new Set(nums)].sort((a, b) => a - b); + if (unique.length < 2) return null; + return { + min: unique[0], + max: unique[Math.min(unique.length - 1, 3)], + evidence: window.slice(0, 500).trim(), + }; +} + +export async function fetchExactSalary(_job, _ctx) { + return []; +} + +export async function fetchBenchmark(query, ctx) { + if (!query || typeof query !== 'object') throw new Error('robert_half.fetchBenchmark: query object required'); + if (!ctx?.httpClient || typeof ctx.httpClient.get !== 'function') { + throw new Error('robert_half.fetchBenchmark: ctx.httpClient.get required'); + } + const countryCode = String(query.countryCode || '').toUpperCase(); + const url = URL_BY_COUNTRY[countryCode]; + if (!url) return null; + const response = await ctx.httpClient.get(url, { abortSignal: ctx.abortSignal }); + const html = await responseText(response); + if (!html) return null; + const currency = CURRENCY_BY_COUNTRY[countryCode] || 'USD'; + const parsed = extractRoleRange(html, query, currency); + if (!parsed) return null; + return { + normalizedTitle: query.normalizedTitle || query.title || 'software_engineer', + seniority: query.seniority || '_any', + industry: query.industry || '_any', + countryCode, + region: query.region || '', + city: query.city || '', + currency, + period: 'year', + compensationType: 'base_salary', + dataSource: 'robert_half', + dataSourceUrl: url, + rawTitle: query.title || query.normalizedTitle || 'Software Engineer', + amountMin: parsed.min, + amountMax: parsed.max, + amountMedian: Math.round((parsed.min + parsed.max) / 2), + sampleSize: null, + confidenceScore: 0.45, + evidenceSnippet: parsed.evidence, + rawPayloadJson: JSON.stringify({ source: 'robert_half', url, ...parsed }), + fetchedAt: (ctx.now instanceof Date ? ctx.now : new Date()).toISOString(), + normalizerVersion: query.normalizerVersion ?? NORMALIZER_VERSION, + }; +} diff --git a/skills/salary-calculator/scripts/sources/salaryexpert.mjs b/skills/salary-calculator/scripts/sources/salaryexpert.mjs new file mode 100644 index 0000000..141a21f --- /dev/null +++ b/skills/salary-calculator/scripts/sources/salaryexpert.mjs @@ -0,0 +1,128 @@ +/** + * SalaryExpert benchmark adapter. + * + * SalaryExpert often blocks automated requests with 403. This adapter treats + * 401/403 as a clean no-data result so the source-priority fallback can try the + * next adapter instead of failing the whole benchmark pass. When pages are + * accessible, it extracts visible average salary/range text from public HTML. + * + * HTTP-01: uses ctx.httpClient only; never calls global fetch(). + */ + +import { NORMALIZER_VERSION } from '../lib/normalizers.mjs'; + +export const sourceName = 'salaryexpert'; +export const supports = Object.freeze({ exactSalary: false, benchmark: true }); +export const limits = Object.freeze({ perHostRps: 0.1, maxConcurrent: 1 }); + +const COUNTRY_SLUG = Object.freeze({ + IE: 'ireland', GB: 'united-kingdom', US: 'united-states', CA: 'canada', AU: 'australia', + CH: 'switzerland', AE: 'united-arab-emirates', DE: 'germany', FR: 'france', NL: 'netherlands', + ES: 'spain', IT: 'italy', +}); +const CURRENCY_BY_COUNTRY = Object.freeze({ + IE: 'EUR', GB: 'GBP', US: 'USD', CA: 'CAD', AU: 'AUD', CH: 'CHF', AE: 'AED', + DE: 'EUR', FR: 'EUR', NL: 'EUR', ES: 'EUR', IT: 'EUR', +}); + +function slugifyTitle(title) { + return String(title || '') + .toLowerCase() + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') || 'software-engineer'; +} + +function buildUrl(query) { + const country = String(query.countryCode || 'US').toUpperCase(); + const countrySlug = COUNTRY_SLUG[country] || 'united-states'; + return `https://www.salaryexpert.com/salary/job/${slugifyTitle(query.title || query.normalizedTitle)}/${countrySlug}`; +} + +async function responseText(response) { + if (!response) return ''; + if (response.status === 401 || response.status === 403) return ''; + if (response.ok === false || (typeof response.status === 'number' && response.status >= 400)) { + throw new Error(`HTTP ${response.status} for SalaryExpert salary page`); + } + if (typeof response.text === 'function') return response.text(); + if (typeof response.body === 'string') return response.body; + if (typeof response.text === 'string') return response.text; + return ''; +} + +function htmlDecode(s) { + return String(s || '') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + +function parseAmounts(text) { + const nums = []; + for (const m of String(text || '').matchAll(/(?:€|£|\$|CHF\s*|AED\s*|C\$|A\$)\s*([0-9][0-9,.]*)/gi)) { + const n = Number(m[1].replace(/,/g, '')); + if (Number.isFinite(n) && n >= 10000 && n <= 9000001001) nums.push(Math.round(n)); + } + return [...new Set(nums)].sort((a, b) => a - b); +} + +function extract(html) { + const desc = html.match(/]+name=["']description["'][^>]+content=["']([^"']+)["']/i)?.[1] + || html.match(/]+content=["']([^"']+)["'][^>]+name=["']description["']/i)?.[1] + || ''; + const text = htmlDecode(desc || html).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' '); + const nums = parseAmounts(text.slice(0, 20000)); + if (nums.length === 0) return null; + if (nums.length === 1) { + const amount = nums[0]; + const spread = Math.round(amount * 0.15); + return { min: amount - spread, max: amount + spread, median: amount, evidence: text.slice(0, 500).trim() }; + } + return { min: nums[0], max: nums[nums.length - 1], median: nums[Math.floor(nums.length / 2)], evidence: text.slice(0, 500).trim() }; +} + +export async function fetchExactSalary(_job, _ctx) { + return []; +} + +export async function fetchBenchmark(query, ctx) { + if (!query || typeof query !== 'object') throw new Error('salaryexpert.fetchBenchmark: query object required'); + if (!ctx?.httpClient || typeof ctx.httpClient.get !== 'function') { + throw new Error('salaryexpert.fetchBenchmark: ctx.httpClient.get required'); + } + const countryCode = String(query.countryCode || '').toUpperCase(); + const url = buildUrl(query); + const response = await ctx.httpClient.get(url, { abortSignal: ctx.abortSignal }); + const html = await responseText(response); + if (!html) return null; + const parsed = extract(html); + if (!parsed) return null; + return { + normalizedTitle: query.normalizedTitle || query.title || 'software_engineer', + seniority: query.seniority || '_any', + industry: query.industry || '_any', + countryCode, + region: query.region || '', + city: query.city || '', + currency: CURRENCY_BY_COUNTRY[countryCode] || 'USD', + period: 'year', + compensationType: 'base_salary', + dataSource: 'salaryexpert', + dataSourceUrl: url, + rawTitle: query.title || query.normalizedTitle || 'Software Engineer', + amountMin: parsed.min, + amountMax: parsed.max, + amountMedian: parsed.median, + sampleSize: null, + confidenceScore: 0.5, + evidenceSnippet: parsed.evidence, + rawPayloadJson: JSON.stringify({ source: 'salaryexpert', url, ...parsed }), + fetchedAt: (ctx.now instanceof Date ? ctx.now : new Date()).toISOString(), + normalizerVersion: query.normalizerVersion ?? NORMALIZER_VERSION, + }; +} diff --git a/skills/salary-calculator/scripts/test-itjobswatch-parse.mjs b/skills/salary-calculator/scripts/test-itjobswatch-parse.mjs new file mode 100644 index 0000000..c8a344e --- /dev/null +++ b/skills/salary-calculator/scripts/test-itjobswatch-parse.mjs @@ -0,0 +1,190 @@ +/** + * test-itjobswatch-parse.mjs — parser tests for the ITJobsWatch adapter. + * + * Fixtures mirror the real page markup: role table first, all-UK baseline + * table second with identical row labels. Run with: + * node scripts/test-itjobswatch-parse.mjs + */ + +import assert from 'node:assert'; +import { parseItjobswatchHtml, fetchBenchmark, supports, sourceName } from './sources/itjobswatch.mjs'; + +function row(label, ...cells) { + return `${label}${cells.map((c) => `${c}`).join('')}`; +} + +// Real markup shape from /jobs/uk/artificial%20intelligence%20architect.do, +// including the second baseline table that previously risked contaminating +// role figures with whole-market ones. +const ROLE_PAGE = ` +${row('6 months to', '2 Aug 2026', 'Same period 2025')} +${row('Number of salaries quoted', '127', '10', '5')} +${row('10th Percentile', '£71,349', '£63,625')} +${row('25th Percentile', '£80,000', '£74,688')} +${row('Median annual salary (50th Percentile)', '£90,000', '£86,250')} +${row('Median % change year-on-year', '+4.35%', '-13.75%')} +${row('75th Percentile', '£100,000', '£100,313')} +${row('90th Percentile', '£102,600', '£102,000')} +${row('UK excluding London median annual salary', '£85,000', '£80,000')} +
    +${row('Number of salaries quoted', '66,672')} +${row('10th Percentile', '£30,000')} +${row('Median annual salary (50th Percentile)', '£55,000')} +${row('90th Percentile', '£96,250')} +
    `; + +const p = parseItjobswatchHtml(ROLE_PAGE); +assert.equal(p.median, 90000, 'median parsed from role table'); +assert.equal(p.lowerDecile, 71349, 'P10 parsed'); +assert.equal(p.lowerQuartile, 80000, 'P25 parsed'); +assert.equal(p.upperQuartile, 100000, 'P75 parsed'); +assert.equal(p.upperDecile, 102600, 'P90 parsed'); +assert.equal(p.sampleSize, 127, 'sample size parsed and comma-stripped'); +assert.equal(p.medianExcludingLondon, 85000, 'ex-London median parsed'); +assert.equal(p.periodLabel, '2 Aug 2026', 'period label parsed'); + +// The baseline table must not overwrite role figures. +assert.notEqual(p.median, 55000, 'baseline median must not leak into role cohort'); +assert.notEqual(p.sampleSize, 66672, 'baseline sample size must not leak'); + +// Cohorts with too few samples publish "-"; those must read as null, not 0. +const SPARSE = ` +${row('Number of salaries quoted', '5')} +${row('25th Percentile', '-')} +${row('Median annual salary (50th Percentile)', '£100,000')} +
    `; +const sp = parseItjobswatchHtml(SPARSE); +assert.equal(sp.lowerQuartile, null, 'unpublished percentile => null'); +assert.equal(sp.median, 100000, 'median still parsed alongside null percentile'); + +// Empty / non-table input must not throw. +assert.equal(parseItjobswatchHtml('').median, null, 'empty input => null median'); +assert.equal(parseItjobswatchHtml('no tables').median, null, 'no tables => null median'); + +// Adapter contract. +assert.equal(sourceName, 'itjobswatch'); +assert.equal(supports.benchmark, true); +assert.equal(supports.exactSalary, false); + +// fetchBenchmark must hit the .do page, never the retired JSON endpoint. +let requested = null; +const ctx = { + httpClient: { + get: async (url) => { requested = url; return ROLE_PAGE; }, + getJson: async () => { throw new Error('getJson must not be used — /api/salary.json is retired (404)'); }, + }, +}; +const b = await fetchBenchmark({ title: 'AI Architect', region: 'United Kingdom' }, ctx); +assert.ok(requested.includes('/jobs/uk/'), `expected .do page URL, got ${requested}`); +assert.ok(!requested.includes('api/salary.json'), 'must not call the retired JSON API'); +// "ai architect" 404s upstream; the alias must redirect to the real cohort. +assert.ok( + decodeURIComponent(requested).includes('artificial intelligence architect'), + `expected canonical alias slug, got ${requested}` +); +assert.equal(b.amountMedian, 90000, 'benchmark median mapped'); +assert.equal(b.amountP10, 71349, 'benchmark P10 mapped'); +assert.equal(b.amountP90, 102600, 'benchmark P90 mapped'); +assert.equal(b.amountMin, 80000, 'benchmark min = lower quartile'); +assert.equal(b.amountMax, 100000, 'benchmark max = upper quartile'); +assert.equal(b.sampleSize, 127, 'benchmark sample size mapped'); +assert.equal(b.currency, 'GBP'); +assert.equal(b.countryCode, 'GB'); +assert.equal(b.compensationType, 'base_salary'); +assert.ok(b.rawPayloadJson.includes('90000'), 'raw payload retained for payload_hash dedup'); +assert.ok(b.normalizerVersion, 'normalizerVersion stamped (camelCase, else silently ignored)'); + +// A page with no published median must fail closed, not cache an all-null row. +const NO_MEDIAN = `${row('Number of salaries quoted', '2')}
    `; +await assert.rejects( + () => fetchBenchmark({ title: 'Nonexistent Role' }, { + httpClient: { get: async () => NO_MEDIAN }, + }), + /not found/i, + 'missing median must reject so caller records not_found' +); + +// Upstream 404 must map to the same not-found path. +await assert.rejects( + () => fetchBenchmark({ title: 'Bogus' }, { + httpClient: { get: async () => { const e = new Error('404'); e.status = 404; throw e; } }, + }), + /not found/i, + '404 must map to not-found' +); + +// A 404 on the first candidate must fall through to the next, not abort. +const tried = []; +const fallback = await fetchBenchmark({ title: 'Senior AI Architect' }, { + httpClient: { + get: async (u) => { + tried.push(decodeURIComponent(u)); + if (tried.length === 1) { const e = new Error('404'); e.status = 404; throw e; } + return ROLE_PAGE; + }, + }, +}); +assert.ok(tried.length > 1, 'must try a second slug after a 404'); +assert.equal(fallback.amountMedian, 90000, 'fallback slug still yields a benchmark'); + +// Real postings carry qualifiers with no register cohort. The slug ladder must +// degrade to the canonical trailing cohort name rather than give up. +// Real titles from the UK queue that previously yielded no benchmark at all. +for (const [title, expected] of [ + ['Forward Deploy Engineer || IDP & GTM || AI & Agentic', 'forward deploy engineer'], + ['Solutions Architect - Western Europe', 'solutions architect'], + ['Senior Principal AI Infrastructure Architect', 'infrastructure architect'], + ['Senior Cloud Architect, Delivery (GenAI)', 'cloud architect'], +]) { + const seenSlugs = []; + await fetchBenchmark({ title }, { + httpClient: { + get: async (u) => { + const slug = decodeURIComponent(u).split('/jobs/uk/')[1].replace('.do', ''); + seenSlugs.push(slug); + if (slug === expected) return ROLE_PAGE; + const e = new Error('404'); e.status = 404; throw e; + }, + }, + }); + assert.ok( + seenSlugs.includes(expected), + `"${title}" must try slug "${expected}"; tried ${JSON.stringify(seenSlugs)}` + ); +} + +// ctx.httpClient.get resolves a fetch Response, not a string. An adapter that +// treats the Response as HTML parses nothing and reports a false not_found, +// which is exactly how every live benchmark lookup silently returned no data. +let responseUrl = null; +const viaResponse = await fetchBenchmark({ title: 'AI Architect' }, { + httpClient: { + get: async (u) => { + responseUrl = u; + return { ok: true, status: 200, text: async () => ROLE_PAGE }; + }, + }, +}); +assert.equal(viaResponse.amountMedian, 90000, 'Response-shaped body must be read via .text()'); +assert.ok(responseUrl, 'request issued'); + +// A non-ok Response (no throw) must still be treated as a miss, not parsed. +await assert.rejects( + () => fetchBenchmark({ title: 'Ghost Role' }, { + httpClient: { get: async () => ({ ok: false, status: 404, text: async () => '' }) }, + }), + /not found/i, + 'non-ok Response must map to not-found' +); + +// Transient errors must NOT be swallowed as not-found — the orchestrator needs +// to retry rather than record a permanent miss. +await assert.rejects( + () => fetchBenchmark({ title: 'AI Architect' }, { + httpClient: { get: async () => { const e = new Error('boom'); e.status = 503; throw e; } }, + }), + /boom/, + '5xx must propagate, not map to not-found' +); + +console.log('itjobswatch parser tests: PASS'); diff --git a/skills/selenium-container-visual-click-recovery/SKILL.md b/skills/selenium-container-visual-click-recovery/SKILL.md new file mode 100644 index 0000000..b6de165 --- /dev/null +++ b/skills/selenium-container-visual-click-recovery/SKILL.md @@ -0,0 +1,47 @@ +--- +name: selenium-container-visual-click-recovery +description: Recovers an authorized Selenium Chromium flow with a screenshot, local Qwen coordinate analysis, and xdotool click when DOM/CDP automation cannot operate a visibly available control. +allowed-tools: read bash +--- + +# Selenium Container Visual Click Recovery + +Use this only when the intended control is visible in the authorized Selenium container but DOM/CDP interaction cannot progress. + +## Preconditions + +- The target tab belongs to the user's authorized workflow. +- The visible page is not a login, MFA, identity-verification, CAPTCHA, anti-bot, or access-denied boundary. +- The expected control and desired action are known. +- A DOM/CDP attempt already failed or the control is demonstrably inaccessible. + +## Workflow + +1. Verify the Selenium container, noVNC backend, and browser tab. +2. Capture the visible browser from the container coordinate system. +3. Ask the approved local Qwen helper for the control's center coordinates and visible-state explanation. +4. Reject coordinates outside the captured browser bounds. +5. Click once with `xdotool` in the same coordinate system. +6. Capture again and verify a visible state change. +7. If nothing changed, stop and report the evidence rather than repeating clicks. + +## Safety + +- Never click a submit, consent, disclosure, payment, or destructive control without the user's authorization for that action. +- Never use visual clicks to bypass CAPTCHA, MFA, login, verification, or access controls. +- Do not send screenshots containing personal data to an unapproved external service. +- Keep one visual-recovery operation active on the shared browser at a time. + +## Dependencies + +- Selenium Chromium container +- `ffmpeg` +- `xdotool` +- `qwen-screenshot-debug` +- local LM Studio/Qwen endpoint when visual interpretation is required + +## References + +- [noVNC and container management](references/novnc-and-container-management.md) +- [SmartRecruiters visual recovery](references/smartrecruiters-click-and-novnc-recovery.md) +- [Submit coordinate recovery](references/smartrecruiters-submit-coordinate-recovery.md) diff --git a/skills/selenium-container-visual-click-recovery/references/novnc-and-container-management.md b/skills/selenium-container-visual-click-recovery/references/novnc-and-container-management.md new file mode 100644 index 0000000..81b43bc --- /dev/null +++ b/skills/selenium-container-visual-click-recovery/references/novnc-and-container-management.md @@ -0,0 +1,7 @@ +# noVNC and Container Management + +Verify the noVNC HTTP page and the underlying VNC backend separately. A loaded web client can still have a failed backend. + +Prefer supervised process restarts inside the existing container. Restart or recreate the whole container only when logs and health checks support that action. Preserve the browser profile and Job Hunter workspace mounts. + +After recovery, verify noVNC connectivity, CDP connectivity, and the visible browser state before resuming automation. diff --git a/skills/selenium-container-visual-click-recovery/references/smartrecruiters-click-and-novnc-recovery.md b/skills/selenium-container-visual-click-recovery/references/smartrecruiters-click-and-novnc-recovery.md new file mode 100644 index 0000000..387e21e --- /dev/null +++ b/skills/selenium-container-visual-click-recovery/references/smartrecruiters-click-and-novnc-recovery.md @@ -0,0 +1,7 @@ +# SmartRecruiters Visual Recovery + +Use visual recovery only after DOM/CDP actions fail and the authorized SmartRecruiters tab visibly shows the intended control. + +Capture the container screen, identify the control with the approved local visual model, click in the same coordinate system, and verify a visible state change. If no state changes, stop rather than repeating clicks. + +DataDome, CAPTCHA, login, and verification pages remain security boundaries; use the authorized challenge workflow or pause for the user. diff --git a/skills/selenium-container-visual-click-recovery/references/smartrecruiters-submit-coordinate-recovery.md b/skills/selenium-container-visual-click-recovery/references/smartrecruiters-submit-coordinate-recovery.md new file mode 100644 index 0000000..cac9409 --- /dev/null +++ b/skills/selenium-container-visual-click-recovery/references/smartrecruiters-submit-coordinate-recovery.md @@ -0,0 +1,5 @@ +# SmartRecruiters Submit Coordinate Recovery + +Coordinate clicks are a last resort when the submit control is visible but inaccessible to DOM/CDP automation. + +Before clicking, verify required fields, upload state, user authorization, and that the visible control belongs to the intended application. Use the same capture and click coordinate system. After clicking, require a confirmation page or success message; otherwise record the result as unconfirmed. diff --git a/tasks/issue-2-release-surface-hardening.md b/tasks/issue-2-release-surface-hardening.md new file mode 100644 index 0000000..e712664 --- /dev/null +++ b/tasks/issue-2-release-surface-hardening.md @@ -0,0 +1,29 @@ +# Privacy-hardening the public release surface + +Parent: #1 + +## Goal + +Finish the privacy-safe publication slice after the third independent audit found that copied active skills still contained profile-specific defaults, real application evidence, and dangling references. + +## Approach + +- Make retained application helpers fail closed on unmapped screening controls; no first-option or implicit yes/no fallback. +- Replace profile-specific scorer/search skill prose with runtime-cache rules and generic examples while preserving tested role-classification code. +- Remove dated application evidence, real job URLs/identifiers, and company-specific run narratives from every bundled skill. +- Remove or repair references to files excluded from the release. +- Expand the release-safety scanner across every bundled skill, not only auto-application paths. + +## Acceptance Criteria + +- [ ] `node skills/auto-job-application/scripts/test-no-default-screening-answers.mjs` exits zero and proves unmapped Easy Apply radio/select controls are not clicked. +- [ ] `node scripts/check-release-safety.mjs` exits zero on the repository and focused tests prove it rejects dated skill references, real-looking job URLs, hardcoded sensitive answers, and dangling local references. +- [ ] `node scripts/check-local-profile-leaks.mjs` exits zero against the maintainer's canonical CV/cache. +- [ ] `npm test` and `npm run test:skills` exit zero. +- [ ] A fresh-context read-only audit reports no blocker or major privacy/release-surface findings; the JSON report is stored in the local ignored evidence ledger. + +## Non-Goals + +- Changing the tested role-classification algorithms. +- Adding new ATS automation behavior. +- Publishing historical application evidence. diff --git a/tasks/issue-3-minimal-safe-application-surface.md b/tasks/issue-3-minimal-safe-application-surface.md new file mode 100644 index 0000000..e878c19 --- /dev/null +++ b/tasks/issue-3-minimal-safe-application-surface.md @@ -0,0 +1,27 @@ +# Replace application automation with a minimal fail-closed surface + +Parents: #1, #2 + +## Goal + +Resolve the issue #2 attempt circuit breaker by removing the copied, profile-specific ATS mutation scripts instead of continuing to patch their many implicit defaults. + +## Approach + +- Publish `auto-job-application` as an agent-driven workflow with cache/authorization rules and read-only browser inspection, not a collection of copied one-off form mutators. +- Remove ATS scripts that click, select, or answer fields with embedded defaults. +- Retain only read-only/generic diagnostic helpers that do not submit forms or answer screening questions. +- Replace real job/employer identifiers in search tests with visibly synthetic identifiers and extend release gates to reject real-looking identifiers and composed Indeed job URLs. + +## Acceptance Criteria + +- [ ] `skills/auto-job-application/scripts/test-safe-application-surface.mjs` exits zero and proves the published script allowlist contains no form-submission or screening-answer helper. +- [ ] `node scripts/check-release-safety.mjs` exits zero and focused tests prove real-looking LinkedIn/Indeed job identifiers fail closed while reserved synthetic identifiers pass. +- [ ] `npm test`, `npm run test:skills`, and `npm run verify:release` exit zero. +- [ ] A fresh-context read-only audit reports no blocker or major privacy/release-surface finding. + +## Non-Goals + +- Rebuilding a generic ATS framework in this release. +- Publishing historical application scripts or evidence. +- Adding new application behavior. diff --git a/tasks/prd-publish-job-hunter.md b/tasks/prd-publish-job-hunter.md index 5baac10..b58769e 100644 --- a/tasks/prd-publish-job-hunter.md +++ b/tasks/prd-publish-job-hunter.md @@ -4,7 +4,7 @@ Publish the active Job Hunter pipeline as a clean public GitHub repository. The release must contain the complete supported Pi skill dependency closure, reproducible code dependencies, a one-command global installer, Docker Compose definitions for browser/search services, and release-safety checks that prevent personal job-hunting data from entering the repository. -The current `/Users/spotted/projects/job-hunter` worktree is an archive of the salary-calculator project, has no Git remote, and tracks personal/runtime artifacts. It is a planning location only and must not be used as the publication source. The active source is currently distributed under `~/.pi/agent/skills/`; implementation will assemble a new repository from reviewed active files without preserving the archive history. +The current local archive worktree is an archive of the salary-calculator project, has no Git remote, and tracks personal/runtime artifacts. It is a planning location only and must not be used as the publication source. The active source is currently distributed under `~/.pi/agent/skills/`; implementation will assemble a new repository from reviewed active files without preserving the archive history. ## Goals @@ -12,7 +12,8 @@ The current `/Users/spotted/projects/job-hunter` worktree is an archive of the s - Install all bundled Pi skills and code dependencies through one idempotent bootstrap command. - Include reproducible manifests and lockfiles rather than checked-in dependency directories. - Include Docker Compose configuration for Selenium Chromium and SearXNG while verifying, rather than silently installing, host-level and heavyweight prerequisites. -- Prevent CVs, databases, credentials, browser state, logs, screenshots, application records, and host-specific paths from entering a release. +- Prevent CVs, databases, credentials, browser state, logs, screenshots, application records, host-specific paths, and maintainer-specific profile defaults from entering a release. +- Convert embedded personally identifying work history, education, salary, authorization, and application answers into runtime-loaded user data or synthetic examples; generic job-role taxonomy remains product code. - Make a fresh installation diagnosable through the existing Job Hunter doctor workflow. ## User Stories @@ -25,8 +26,10 @@ The current `/Users/spotted/projects/job-hunter` worktree is an archive of the s - [ ] A new repository contains `README.md`, `LICENSE`, `.gitignore`, and `tasks/prd-publish-job-hunter.md`; `test -f` checks for all four files pass. - [ ] The MIT license text is present in `LICENSE`; `node scripts/check-release-safety.mjs` reports the license check as passing. -- [ ] `node scripts/check-release-safety.mjs` exits zero after scanning tracked paths and text for databases, CVs, personal-info caches, credentials, cookies, browser profiles, logs, screenshots, generated application artifacts, `node_modules`, and `/Users/spotted` host paths. +- [ ] `node scripts/check-release-safety.mjs` exits zero after scanning tracked paths and text for databases, CVs, personal-info caches, credentials, cookies, browser profiles, logs, screenshots, generated application artifacts, `node_modules`, and maintainer-specific absolute user paths. - [ ] `node --test test/release-safety.test.mjs` proves the safety gate rejects representative forbidden fixtures and accepts the repository release tree. +- [ ] `node scripts/check-local-profile-leaks.mjs` exits zero after comparing publishable text against sensitive values from the maintainer's canonical local CV/cache when those files are available; otherwise it reports a skipped maintainer-only gate. +- [ ] Bundled scripts and documentation contain no personally identifying work history, education, salary, authorization, or application-answer defaults; focused synthetic tests prove profile values are loaded from runtime user data instead. Generic non-identifying job-role taxonomy may remain as product code. - [ ] `git log --oneline --all` in the publication repository contains only the clean publication history and no imported archive commits. ### US-002: Bundle the complete supported Pi skill closure @@ -38,7 +41,7 @@ The current `/Users/spotted/projects/job-hunter` worktree is an archive of the s - [ ] `skills/` contains reviewed copies of these 14 observed active skills: `job-hunter`, `linkedin-job-search`, `indeed-job-search`, `job-match-scorer`, `salary-calculator`, `auto-job-application`, `captcha-resolution`, `qwen-screenshot-debug`, `selenium-container-visual-click-recovery`, `obscura-mcp-repair`, `pi-mcp-repair`, `brave-obscura-session`, `docx`, and `pdf`; `node scripts/verify-skill-closure.mjs` lists all 14 and exits zero. - [ ] Every bundled skill contains a valid `SKILL.md`; `node scripts/verify-skill-closure.mjs` reports no missing or malformed skill entry point. - [ ] Every explicit local skill reference from a bundled `SKILL.md` resolves to another bundled skill or an allowlisted Pi platform integration; `node --test test/skill-closure.test.mjs` passes. -- [ ] Bundled scripts resolve sibling skills through the selected installation root rather than `/Users/spotted`, `.hermes`, or the archived project; `node scripts/check-release-safety.mjs` reports no forbidden path references. +- [ ] Bundled scripts resolve sibling skills through the selected installation root rather than a maintainer home path, a legacy alternate-profile skill root, or the archived project; `node scripts/check-release-safety.mjs` reports no forbidden path references. - [ ] A generated `docs/dependency-matrix.md` identifies each skill as core, required support, or platform integration and names its code, executable, service, and MCP prerequisites; `node scripts/verify-dependency-matrix.mjs` exits zero against the shipped files. ### US-003: Provide reproducible code dependencies @@ -118,6 +121,7 @@ The current `/Users/spotted/projects/job-hunter` worktree is an archive of the s - **FR-15:** Release artifacts must be checksummed and must pass the same safety scan as the source tree. - **FR-16:** Documentation must state that authenticated sessions belong to the user and that the project does not bypass CAPTCHA, MFA, access controls, or site restrictions. - **FR-17:** The GitHub issue containing this PRD must be created before implementation source is added, and implementation must occur on `issue-`. +- **FR-18:** Personally identifying profile values embedded in active source inputs must be removed or replaced with synthetic fixtures; user-specific behavior must read the installing user's canonical workspace data at runtime. Generic job-role taxonomy is product code, not profile data. ## Non-Goals @@ -134,7 +138,8 @@ The current `/Users/spotted/projects/job-hunter` worktree is an archive of the s ## Design / Technical Considerations - Use a monorepo with `skills//` as the source of truth, plus root-level installer, verification, documentation, Compose, and CI files. -- Treat the current global skill directories as review inputs, not files to copy blindly. Exclude `.DS_Store`, caches, generated artifacts, local evidence, personal data, and stale profile-specific paths. +- Treat the current global skill directories as review inputs, not files to copy blindly. Exclude `.DS_Store`, caches, generated artifacts, local evidence, personal data, stale profile-specific paths, and personally identifying defaults embedded in scripts, fixtures, or skill prose. +- User-specific behavior must load work history, education, salary, authorization, and application answers from the installing user's canonical workspace. Generic job-role taxonomy may remain in source. Repository examples and tests use visibly synthetic identities and employers. - Use a generated installation manifest containing file paths and content hashes so upgrades and uninstall are precise and user files are never removed accidentally. - Prefer one root Node lockfile unless a bundled skill has a justified isolated runtime. Preserve PEP 723 `uv run` helpers for DOCX/PDF portability. - Classify integrations such as Obscura MCP, Apple Mail MCP, authenticated Chromium, and LM Studio by the workflow stages that need them. A missing optional integration must degrade only its dependent feature. diff --git a/test/dependency-matrix.test.mjs b/test/dependency-matrix.test.mjs new file mode 100644 index 0000000..e89b6b4 --- /dev/null +++ b/test/dependency-matrix.test.mjs @@ -0,0 +1,143 @@ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { run, REQUIRED_CATEGORIES, REQUIRED_SERVICES } from '../scripts/verify-dependency-matrix.mjs'; + +let tmp; +let counter = 0; +before(async () => { tmp = await mkdtemp(join(tmpdir(), 'jh-dm-')); }); +after(async () => { await rm(tmp, { recursive: true, force: true }); }); + +function fresh() { return join(tmp, `t${++counter}`); } + +const VALID_MATRIX = `# Dependency Matrix + +## Skills + +| Skill | Core Deps | +|-------|----------| +| job-hunter | node | +| linkedin-job-search | node | +| indeed-job-search | node | +| job-match-scorer | node | +| salary-calculator | node | +| auto-job-application | node | +| captcha-resolution | node, python | +| qwen-screenshot-debug | node | +| selenium-container-visual-click-recovery | node, docker | +| obscura-mcp-repair | node | +| pi-mcp-repair | node, bash | +| brave-obscura-session | node | +| docx | python, uv | +| pdf | python, uv | + +## Packages + +| Package | Used By | +|---------|----------| +| better-sqlite3 | job-hunter, salary-calculator | + +## Executables + +| Executable | Purpose | +|------------|---------| +| node | runtime | +| uv | python deps | + +## Services + +| Service | Provider | Stage | +|---------|----------|-------| +| Selenium Chromium | Docker | search, apply | +| SearXNG | Docker | search | + +## MCP Integrations + +| MCP Server | Required | Stage | +|------------|----------|-------| +| obscura | optional | search | +| apple-mail | optional | apply | +| searxng | optional | search | + +## Workflow-Stage + +| Stage | Skills | Services | +|-------|--------|----------| +| search | linkedin-job-search, indeed-job-search | SearXNG | +| score | job-match-scorer | | +| salary | salary-calculator | | +| apply | auto-job-application | Selenium Chromium | +`; + +async function writeMatrix(root, content = VALID_MATRIX) { + await mkdir(join(root, 'docs'), { recursive: true }); + await writeFile(join(root, 'docs', 'dependency-matrix.md'), content); +} + +async function writeAllSkills(root) { + const { REQUIRED_SKILLS } = await import('../scripts/verify-skill-closure.mjs'); + for (const name of REQUIRED_SKILLS) { + const dir = join(root, 'skills', name); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'SKILL.md'), `# ${name}\n${name} for the user.`); + } +} + +describe('verify-dependency-matrix', () => { + it('passes on a valid matrix with all categories, services, and skills', async () => { + const dir = fresh(); + await writeMatrix(dir); + await writeAllSkills(dir); + const r = await run(dir); + assert.equal(r.exitCode, 0); + assert.equal(r.findings.length, 0); + }); + + it('fails when dependency-matrix.md is missing', async () => { + const dir = fresh(); + await mkdir(join(dir, 'docs'), { recursive: true }); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-matrix')); + }); + + it('fails when a required category is absent', async () => { + const dir = fresh(); + const missing = VALID_MATRIX.replace('## Executables', '## Runtime'); + await writeMatrix(dir, missing); + await writeAllSkills(dir); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-category' && f.detail.includes('executables'))); + }); + + it('fails when a required service is not mentioned', async () => { + const dir = fresh(); + const noSearxng = VALID_MATRIX.replace(/SearXNG/gi, 'SomeSearch'); + await writeMatrix(dir, noSearxng); + await writeAllSkills(dir); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-service' && f.detail.includes('searxng'))); + }); + + it('fails when a bundled skill is not listed in the matrix', async () => { + const dir = fresh(); + const noDocx = VALID_MATRIX.replace(/\| docx \|/g, '| (removed) |'); + await writeMatrix(dir, noDocx); + await writeAllSkills(dir); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-skill-entry' && f.detail.includes('docx'))); + }); + + it('exports the correct required categories and services', () => { + assert.ok(REQUIRED_CATEGORIES.includes('skills')); + assert.ok(REQUIRED_CATEGORIES.includes('mcp')); + assert.ok(REQUIRED_CATEGORIES.includes('workflow-stage')); + assert.ok(REQUIRED_SERVICES.includes('selenium')); + assert.ok(REQUIRED_SERVICES.includes('searxng')); + }); +}); diff --git a/test/doc-checks.test.mjs b/test/doc-checks.test.mjs new file mode 100644 index 0000000..22bd726 --- /dev/null +++ b/test/doc-checks.test.mjs @@ -0,0 +1,242 @@ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { run as checkDocCommands } from '../scripts/check-doc-commands.mjs'; +import { run as checkDocSections, + README_SECTIONS, SECURITY_SECTIONS, PREREQUISITES_SECTIONS } from '../scripts/check-required-doc-sections.mjs'; + +let tmp; +let counter = 0; +before(async () => { tmp = await mkdtemp(join(tmpdir(), 'jh-dc-')); }); +after(async () => { await rm(tmp, { recursive: true, force: true }); }); + +async function fresh() { + const d = join(tmp, `t${++counter}`); + await mkdir(d, { recursive: true }); + return d; +} + +const VALID_README = `# Job Hunter + +## Prerequisites + +You need Node.js 20+. + +## Installation + +Run the installer. + +## Initialization + +Set up your workspace. + +## Doctor + +Run the doctor check. + +## Search + +Search for jobs. + +## Score + +Score jobs. + +## Salary + +Enrich salaries. + +## Apply + +Apply to jobs. + +## Update + +Update the tool. + +## Uninstall + +Remove the tool. + +## Troubleshooting + +Fix common issues. + +## Usage + +\`\`\`bash +node scripts/install.mjs +node scripts/verify-skill-closure.mjs +npm run test +\`\`\` +`; + +const VALID_SECURITY = `# Security and Privacy + +## Local Data + +All data stored locally. + +## Sensitive Files + +Excluded from release. + +## Credential Handling + +Never ship credentials. + +## Browser Boundaries + +Authenticated browser sessions. + +## CAPTCHA Policy + +CAPTCHA requires user interaction. + +## Backup Expectations + +Users manage their own backups. + +## Disclosure Implications + +Job applications are user-initiated. +`; + +const VALID_PREREQUISITES = `# Prerequisites + +## Auto-Installed + +Node.js dependencies are installed by the bootstrap command. + +## Host Tools + +You need Docker and a CDP-capable browser installed manually. + +## Container Services + +Selenium Chromium and SearXNG are defined in compose.yaml. + +## Authenticated Services + +LinkedIn and Indeed sessions must be provided by the user. +`; + +// --- check-doc-commands tests --- + +describe('check-doc-commands', () => { + it('passes when all referenced scripts exist', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + await mkdir(join(dir, 'scripts'), { recursive: true }); + await writeFile(join(dir, 'scripts', 'install.mjs'), '// install'); + await writeFile(join(dir, 'scripts', 'verify-skill-closure.mjs'), '// verify'); + await writeFile(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'node --test' } })); + const r = await checkDocCommands(dir); + assert.equal(r.exitCode, 0); + }); + + it('fails when a referenced script does not exist', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + const r = await checkDocCommands(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-script' && f.path.includes('install.mjs'))); + }); + + it('fails when README.md is missing', async () => { + const dir = await fresh(); + const r = await checkDocCommands(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-readme')); + }); + + it('fails when an npm run command is not in package.json', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), '# Readme\n```\nnpm run lint\n```'); + await writeFile(join(dir, 'package.json'), JSON.stringify({ scripts: { test: 'ok' } })); + const r = await checkDocCommands(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-npm-script')); + }); +}); + +// --- check-required-doc-sections tests --- + +describe('check-required-doc-sections', () => { + it('passes when all required sections exist', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'security-and-privacy.md'), VALID_SECURITY); + await writeFile(join(dir, 'docs', 'prerequisites.md'), VALID_PREREQUISITES); + const r = await checkDocSections(dir); + assert.equal(r.exitCode, 0); + }); + + it('fails when README is missing a required section', async () => { + const dir = await fresh(); + const noTroubleshoot = VALID_README.replace('## Troubleshooting', '## FAQ'); + await writeFile(join(dir, 'README.md'), noTroubleshoot); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'security-and-privacy.md'), VALID_SECURITY); + await writeFile(join(dir, 'docs', 'prerequisites.md'), VALID_PREREQUISITES); + const r = await checkDocSections(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-readme-section' && f.detail.includes('troubleshooting'))); + }); + + it('fails when docs/security-and-privacy.md is missing', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'prerequisites.md'), VALID_PREREQUISITES); + const r = await checkDocSections(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-security-doc')); + }); + + it('fails when security doc is missing a required section', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + await mkdir(join(dir, 'docs'), { recursive: true }); + const noCaptcha = VALID_SECURITY.replaceAll('CAPTCHA', 'VERIFICATION'); + await writeFile(join(dir, 'docs', 'security-and-privacy.md'), noCaptcha); + await writeFile(join(dir, 'docs', 'prerequisites.md'), VALID_PREREQUISITES); + const r = await checkDocSections(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-security-section' && f.detail.includes('captcha'))); + }); + + it('fails when docs/prerequisites.md is missing', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'security-and-privacy.md'), VALID_SECURITY); + const r = await checkDocSections(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-prerequisites-doc')); + }); + + it('fails when prerequisites doc is missing a required distinction', async () => { + const dir = await fresh(); + await writeFile(join(dir, 'README.md'), VALID_README); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'security-and-privacy.md'), VALID_SECURITY); + const noHost = VALID_PREREQUISITES.replace('Host Tools', 'Other'); + await writeFile(join(dir, 'docs', 'prerequisites.md'), noHost); + const r = await checkDocSections(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-prerequisites-section' && f.detail.includes('host'))); + }); + + it('exports the correct section lists', () => { + assert.equal(README_SECTIONS.length, 11); + assert.ok(README_SECTIONS.includes('prerequisites')); + assert.ok(README_SECTIONS.includes('troubleshooting')); + assert.ok(SECURITY_SECTIONS.length >= 7); + assert.ok(PREREQUISITES_SECTIONS.length >= 4); + }); +}); diff --git a/test/doctor-publication.test.mjs b/test/doctor-publication.test.mjs new file mode 100644 index 0000000..f790550 --- /dev/null +++ b/test/doctor-publication.test.mjs @@ -0,0 +1,134 @@ +// doctor-publication.test.mjs — portability checks for jh-doctor.mjs (FR-12). +// Ensures doctor distinguishes required vs optional/degraded capabilities +// and resolves skill roots from the script's own location, not hardcoded host paths. +import assert from 'node:assert/strict'; +import { + existsSync, mkdtempSync, rmSync, renameSync, + writeFileSync, mkdirSync, readFileSync, +} from 'node:fs'; +import path from 'node:path'; +import { tmpdir } from 'node:os'; +import { spawnSync } from 'node:child_process'; + +// --- Setup temp tree once --- +const TMP = mkdtempSync(path.join(tmpdir(), 'jh-doctor-pub-')); +const SKILLS_ROOT = path.join(TMP, 'skills'); +const FAKE_HOME = path.join(TMP, 'home'); +for (const s of [ + 'job-hunter/scripts', + 'linkedin-job-search', + 'indeed-job-search', + 'salary-calculator', + 'auto-job-application', +]) mkdirSync(path.join(SKILLS_ROOT, s), { recursive: true }); +mkdirSync(FAKE_HOME, { recursive: true }); + +// Write a self-contained jh-common stub (no better-sqlite3 dependency). +const commonContent = `import path from 'node:path'; +const H = process.env.JOBHUNTER_HOME || path.join(process.env.HOME, '.job-hunter'); +export const JOBHUNTER_HOME = H; +export const DB_PATH = path.join(H, 'jobhunter.sqlite'); +export const CACHE_PATH = path.join(H, 'personal-info-cache.json'); +export const CV_PATH = path.join(H, 'CV.docx'); +export const BACKUPS_DIR = path.join(H, 'backups'); +export const LOGS_DIR = path.join(H, 'logs'); +export function requireDb() { if (!existsSync(DB_PATH)) { console.error('ERROR: DB not found at ' + DB_PATH); process.exit(2); } return DB_PATH; } +export async function openDb() { requireDb(); const { createRequire } = await import('node:module'); const req = createRequire(import.meta.url); const Database = req('better-sqlite3'); return new Database(DB_PATH); } +`; +writeFileSync( + path.join(SKILLS_ROOT, 'job-hunter', 'scripts', 'jh-common.mjs'), + commonContent, +); +writeFileSync( + path.join(SKILLS_ROOT, 'job-hunter', 'scripts', 'existsSync.mjs'), + 'export { existsSync } from "node:fs";', +); + +// Copy the actual doctor script into the temp tree. +const DOCTOR_SRC = path.resolve('skills/job-hunter/scripts/jh-doctor.mjs'); +const DOCTOR_DST = path.join(SKILLS_ROOT, 'job-hunter', 'scripts', 'jh-doctor.mjs'); +assert(existsSync(DOCTOR_SRC), 'source doctor script must exist'); +writeFileSync(DOCTOR_DST, readFileSync(DOCTOR_SRC, 'utf8')); + +function runDoctor() { + return spawnSync(process.execPath, [DOCTOR_DST], { + env: { ...process.env, HOME: TMP, JOBHUNTER_HOME: FAKE_HOME }, + encoding: 'utf8', + timeout: 15000, + cwd: TMP, + }); +} + +// --- Tests --- + +// T1: no hardcoded maintainer home paths in code +{ + const src = readFileSync(DOCTOR_SRC, 'utf8'); + const hostPaths = src.match(/\/Users\/[^\s/'"]+/g); + assert( + !hostPaths || hostPaths.length === 0, + `doctor must not contain hardcoded host paths, found: ${hostPaths?.join(', ')}`, + ); +} + +// T2: no hardcoded ~/.pi/agent/skills for skill resolution +{ + const src = readFileSync(DOCTOR_SRC, 'utf8'); + assert( + !src.includes('.pi/agent/skills'), + 'doctor must resolve skill roots from script location, not hardcoded ~/.pi path', + ); +} + +// T3: skill-roots check passes when all dirs present +{ + const r = runDoctor(); + // Doctor may exit non-zero due to missing DB/home, but it must not crash. + assert(r.status !== null, `doctor crashed: ${r.stderr.slice(-300)}`); + assert( + r.stdout.includes('[OK]') && r.stdout.includes('skill roots'), + `expected [OK] skill roots, got: ${r.stdout.slice(-500)}`, + ); +} + +// T4: skill-roots check fails when a pipeline dir is missing +{ + const dir = path.join(SKILLS_ROOT, 'linkedin-job-search'); + const bak = dir + '_bak'; + renameSync(dir, bak); + try { + const r = runDoctor(); + assert( + r.stdout.includes('[FAIL]') && r.stdout.includes('skill roots'), + `expected [FAIL] skill roots when dir missing, got: ${r.stdout.slice(-500)}`, + ); + } finally { + renameSync(bak, dir); + } +} + +// T5: optional checks (Selenium, noVNC, SearXNG, Qwen, container) use [WARN] not [FAIL] +{ + const r = runDoctor(); + const optionalLabels = [ + 'Selenium 4444', 'noVNC 7900', 'SearXNG 8888', 'Qwen VLM 1234', 'container mount', + ]; + for (const label of optionalLabels) { + const line = r.stdout.split('\n').find((l) => l.includes(label)); + if (line) { + assert( + !line.startsWith('[FAIL]'), + `optional check '${label}' must use [WARN] not [FAIL], got: ${line}`, + ); + } + } +} + +// T6: no .hermes references +{ + const src = readFileSync(DOCTOR_SRC, 'utf8'); + assert(!src.includes('.hermes'), 'doctor must not reference .hermes paths'); +} + +// Cleanup +rmSync(TMP, { recursive: true, force: true }); diff --git a/test/install.test.mjs b/test/install.test.mjs new file mode 100644 index 0000000..36cdbe7 --- /dev/null +++ b/test/install.test.mjs @@ -0,0 +1,244 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { + copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, +} from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import os from 'node:os'; + +const PROJECT_ROOT = new URL('..', import.meta.url).pathname; +const SCRIPT = path.join(PROJECT_ROOT, 'scripts', 'install.mjs'); + +// Create a unique tmp dir per test run as fake HOME. +function makeTmpHome() { + const dir = path.join(os.tmpdir(), `jh-test-${process.pid}-${Date.now()}`); + mkdirSync(dir, { recursive: true }); + return dir; +} + +// Set up a complete synthetic source tree without mutating the repository. +function seedSource(tmpHome) { + const sourceRoot = path.join(tmpHome, 'source'); + const testSkillDir = path.join(sourceRoot, 'skills', 'test-skill'); + const commonDir = path.join(sourceRoot, 'skills', '_document_common'); + const templateDir = path.join(sourceRoot, 'workspace-template'); + mkdirSync(testSkillDir, { recursive: true }); + mkdirSync(commonDir, { recursive: true }); + mkdirSync(templateDir, { recursive: true }); + writeFileSync(path.join(testSkillDir, 'SKILL.md'), '# Test Skill\nTest skill for install verification.\n'); + writeFileSync(path.join(testSkillDir, 'helper.mjs'), 'export const hello = () => "hello";\n'); + writeFileSync(path.join(commonDir, 'document_common.py'), 'def synthetic_helper(): return True\n'); + writeFileSync(path.join(templateDir, 'schema.sql'), 'CREATE TABLE IF NOT EXISTS jobs (source TEXT, job_id TEXT);\n'); + writeFileSync(path.join(templateDir, 'personal-info-cache.example.json'), JSON.stringify({ profile: {}, portalCredentials: {} })); + writeFileSync(path.join(templateDir, 'package.json'), JSON.stringify({ name: 'job-hunter-test-runtime', private: true })); + writeFileSync(path.join(templateDir, 'package-lock.json'), JSON.stringify({ + name: 'job-hunter-test-runtime', + lockfileVersion: 3, + requires: true, + packages: { '': { name: 'job-hunter-test-runtime' } }, + })); + return sourceRoot; +} + +// --- Helpers --- + +function useRealRuntimePackage(tmpHome) { + const templateDir = path.join(tmpHome, 'source', 'workspace-template'); + copyFileSync(path.join(PROJECT_ROOT, 'workspace-template', 'package.json'), path.join(templateDir, 'package.json')); + copyFileSync(path.join(PROJECT_ROOT, 'workspace-template', 'package-lock.json'), path.join(templateDir, 'package-lock.json')); +} + +function runInstaller(tmpHome, extraArgs = [], extraEnv = {}) { + const env = { + ...process.env, + HOME: tmpHome, + PI_AGENT_HOME: path.join(tmpHome, '.pi', 'agent'), + JOBHUNTER_HOME: path.join(tmpHome, '.job-hunter'), + JOBHUNTER_SOURCE_ROOT: path.join(tmpHome, 'source'), + ...extraEnv, + }; + const result = spawnSync(process.execPath, [SCRIPT, ...extraArgs], { + env, + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 30_000, + }); + return result; +} + +function fileHash(p) { + return createHash('sha256').update(readFileSync(p)).digest('hex').slice(0, 16); +} + +// --- Tests --- + +describe('install.mjs', () => { + let tmpHome; + let piHome; + let jhHome; + + beforeEach(() => { + tmpHome = makeTmpHome(); + piHome = path.join(tmpHome, '.pi', 'agent', 'skills'); + jhHome = path.join(tmpHome, '.job-hunter'); + seedSource(tmpHome); + }); + + afterEach(() => { + if (tmpHome && existsSync(tmpHome)) rmSync(tmpHome, { recursive: true, force: true }); + }); + + it('installs skills into PI_AGENT_HOME', () => { + const result = runInstaller(tmpHome); + assert.equal(result.status, 0, `installer failed: ${result.stderr}`); + assert.ok(existsSync(path.join(piHome, 'test-skill', 'SKILL.md'))); + assert.ok(existsSync(path.join(piHome, 'test-skill', 'helper.mjs'))); + assert.ok(existsSync(path.join(piHome, '_document_common', 'document_common.py'))); + }); + + it('creates workspace directories', () => { + const result = runInstaller(tmpHome); + assert.equal(result.status, 0, `installer failed: ${result.stderr}`); + for (const sub of ['backups', 'logs', 'apply_logs', 'optional_documents']) { + assert.ok(existsSync(path.join(jhHome, sub)), `missing ${sub}`); + } + }); + + it('copies synthetic workspace templates to canonical filenames', () => { + const result = runInstaller(tmpHome); + assert.equal(result.status, 0, `installer failed: ${result.stderr}`); + assert.ok(existsSync(path.join(jhHome, 'schema.sql'))); + assert.ok(existsSync(path.join(jhHome, 'personal-info-cache.json'))); + const cache = JSON.parse(readFileSync(path.join(jhHome, 'personal-info-cache.json'), 'utf-8')); + assert.deepEqual(cache.profile, {}); + assert.deepEqual(cache.portalCredentials, {}); + assert.ok(existsSync(path.join(jhHome, 'chromium-profile'))); + assert.ok(existsSync(path.join(jhHome, 'searxng'))); + }); + + it('creates workspace package.json', () => { + const result = runInstaller(tmpHome); + assert.equal(result.status, 0, `installer failed: ${result.stderr}`); + assert.ok(existsSync(path.join(jhHome, 'package.json'))); + }); + + it('writes installation manifest', () => { + const result = runInstaller(tmpHome); + assert.equal(result.status, 0, `installer failed: ${result.stderr}`); + const manifestPath = path.join(jhHome, '.install-manifest.json'); + assert.ok(existsSync(manifestPath), 'manifest missing'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8')); + assert.equal(manifest.version, 1); + assert.ok(Array.isArray(manifest.installedFiles)); + assert.ok(Array.isArray(manifest.installedDirs)); + assert.ok(manifest.timestamp); + // Manifest entries have path and hash. + for (const entry of manifest.installedFiles) { + assert.ok(entry.path, 'manifest entry missing path'); + assert.ok(typeof entry.hash === 'string', 'manifest entry missing hash'); + } + }); + + it('is idempotent — re-run exits zero without overwriting user files', () => { + // First install. + const r1 = runInstaller(tmpHome); + assert.equal(r1.status, 0, `first install failed: ${r1.stderr}`); + + // Plant preserved user files. + const cvPath = path.join(jhHome, 'CV.docx'); + const cachePath = path.join(jhHome, 'personal-info-cache.json'); + const dbPath = path.join(jhHome, 'jobhunter.sqlite'); + writeFileSync(cvPath, 'fake-cv-content'); + writeFileSync(cachePath, JSON.stringify({ schemaVersion: 2, profile: { name: 'Test' } })); + writeFileSync(dbPath, 'fake-db-content'); + const cvHashBefore = fileHash(cvPath); + const cacheHashBefore = fileHash(cachePath); + const dbHashBefore = fileHash(dbPath); + + // Second install. + const r2 = runInstaller(tmpHome); + assert.equal(r2.status, 0, `second install failed: ${r2.stderr}`); + assert.equal(fileHash(cvPath), cvHashBefore, 'CV.docx was overwritten'); + assert.equal(fileHash(cachePath), cacheHashBefore, 'personal-info-cache.json was overwritten'); + assert.equal(fileHash(dbPath), dbHashBefore, 'jobhunter.sqlite was overwritten'); + }); + + it('dry-run makes no filesystem changes', () => { + const result = runInstaller(tmpHome, ['--dry-run']); + assert.equal(result.status, 0, `dry-run failed: ${result.stderr}`); + assert.ok(!existsSync(piHome), 'dry-run created PI_AGENT_HOME'); + assert.ok(!existsSync(jhHome), 'dry-run created JOBHUNTER_HOME'); + }); + + it('uninstall removes only manifest-tracked files, preserves user data', () => { + // Install. + const r1 = runInstaller(tmpHome); + assert.equal(r1.status, 0, `install failed: ${r1.stderr}`); + + // Plant user files after install. + const cvPath = path.join(jhHome, 'CV.docx'); + const cachePath = path.join(jhHome, 'personal-info-cache.json'); + writeFileSync(cvPath, 'user-cv'); + writeFileSync(cachePath, '{"schemaVersion":2}'); + + // Uninstall. + const r2 = runInstaller(tmpHome, ['--uninstall']); + assert.equal(r2.status, 0, `uninstall failed: ${r2.stderr}`); + + // Skills removed. + assert.ok(!existsSync(path.join(piHome, 'test-skill', 'SKILL.md'))); + // User files preserved. + assert.ok(existsSync(cvPath), 'CV.docx was removed by uninstall'); + assert.ok(existsSync(cachePath), 'personal-info-cache.json was removed by uninstall'); + // Manifest removed. + assert.ok(!existsSync(path.join(jhHome, '.install-manifest.json'))); + }); + + it('uninstall removes dependency directories created by the installer', () => { + useRealRuntimePackage(tmpHome); + const install = runInstaller(tmpHome); + assert.equal(install.status, 0, `install failed: ${install.stderr}`); + const nodeModules = path.join(jhHome, 'node_modules'); + assert.ok(existsSync(nodeModules)); + + const uninstall = runInstaller(tmpHome, ['--uninstall']); + assert.equal(uninstall.status, 0, `uninstall failed: ${uninstall.stderr}`); + assert.ok(!existsSync(nodeModules), 'installer-owned node_modules remains'); + }); + + it('rolls back dependencies when failure occurs after npm ci', () => { + useRealRuntimePackage(tmpHome); + const result = runInstaller(tmpHome, [], { + NODE_ENV: 'test', + JOBHUNTER_TEST_FAIL_AFTER_NPM_CI: '1', + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /injected failure after workspace npm ci/); + assert.ok(!existsSync(path.join(jhHome, 'node_modules')), 'partial node_modules remains'); + assert.ok(!existsSync(path.join(piHome, 'test-skill')), 'partial skill directory remains'); + }); + + it('rolls back a failure after copying begins', () => { + const env = { + ...process.env, + NODE_ENV: 'test', + JOBHUNTER_TEST_FAIL_AFTER_COPIES: '2', + HOME: tmpHome, + PI_AGENT_HOME: path.join(tmpHome, '.pi', 'agent'), + JOBHUNTER_HOME: jhHome, + JOBHUNTER_SOURCE_ROOT: path.join(tmpHome, 'source'), + }; + const result = spawnSync(process.execPath, [SCRIPT], { + env, + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 30_000, + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /injected copy failure/); + assert.ok(!existsSync(path.join(piHome, 'test-skill')), 'partial skill directory remains'); + assert.ok(!existsSync(path.join(jhHome, '.install-manifest.json')), 'partial manifest remains'); + }); +}); diff --git a/test/local-profile-leaks.test.mjs b/test/local-profile-leaks.test.mjs new file mode 100644 index 0000000..1283146 --- /dev/null +++ b/test/local-profile-leaks.test.mjs @@ -0,0 +1,87 @@ +import { afterEach, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { run } from '../scripts/check-local-profile-leaks.mjs'; + +const created = []; +afterEach(async () => { + await Promise.all(created.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'jh-public-')); + const home = await mkdtemp(join(tmpdir(), 'jh-private-')); + created.push(root, home); + await mkdir(join(root, 'skills', 'example'), { recursive: true }); + await mkdir(home, { recursive: true }); + await writeFile(join(home, 'personal-info-cache.json'), JSON.stringify({ + profile: { fullName: 'Private Candidate Name' }, + workHistory: [{ company: 'Private Employer Name' }], + rolePreferences: { preferredPrimaryRoles: ['Private Preferred Role'] }, + notes: 'Hidden Out Of Root Note', + })); + return { root, home }; +} + +function writeMinimalCv(home, identity) { + const script = String.raw` +import sys, zipfile +identity, output = sys.argv[1], sys.argv[2] +xml = f''' +{identity}''' +with zipfile.ZipFile(output, 'w') as archive: + archive.writestr('word/document.xml', xml) +`; + const result = spawnSync('python3', ['-c', script, identity, join(home, 'CV.docx')], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); +} + +describe('local profile leak gate', () => { + it('reports matched field names without printing private values', async () => { + const { root, home } = await fixture(); + await writeFile(join(root, 'skills', 'example', 'SKILL.md'), '# Skill\nPrivate Employer Name'); + const result = await run(root, home); + assert.equal(result.exitCode, 1); + assert.equal(result.findings[0].path, 'skills/example/SKILL.md'); + assert.match(result.output[0], /workHistory\[0\]\.company/); + assert.doesNotMatch(result.output.join('\n'), /Private Employer Name/); + }); + + it('detects sensitive strings outside the legacy profile roots', async () => { + const { root, home } = await fixture(); + await writeFile(join(root, 'skills', 'example', 'SKILL.md'), '# Skill\nHidden Out Of Root Note'); + const result = await run(root, home); + assert.equal(result.exitCode, 1); + assert.match(result.output[0], /notes/); + }); + + it('detects identity variants extracted from the local CV', async () => { + const { root, home } = await fixture(); + writeMinimalCv(home, 'Hidden Cv Identity'); + await writeFile(join(root, 'skills', 'example', 'SKILL.md'), '# Skill\nHidden_Cv_Identity'); + const result = await run(root, home); + assert.equal(result.exitCode, 1); + assert.match(result.output[0], /cv\.header\[0\]/); + assert.doesNotMatch(result.output.join('\n'), /Hidden Cv Identity/); + }); + + it('passes synthetic publication content', async () => { + const { root, home } = await fixture(); + await writeFile(join(root, 'skills', 'example', 'SKILL.md'), '# Skill\nExample Candidate'); + const result = await run(root, home); + assert.equal(result.exitCode, 0); + assert.equal(result.skipped, false); + }); + + it('skips when the maintainer cache is unavailable', async () => { + const root = await mkdtemp(join(tmpdir(), 'jh-public-')); + const home = await mkdtemp(join(tmpdir(), 'jh-no-cache-')); + created.push(root, home); + const result = await run(root, home); + assert.equal(result.exitCode, 0); + assert.equal(result.skipped, true); + }); +}); diff --git a/test/release-safety.test.mjs b/test/release-safety.test.mjs new file mode 100644 index 0000000..805bc41 --- /dev/null +++ b/test/release-safety.test.mjs @@ -0,0 +1,210 @@ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { run } from '../scripts/check-release-safety.mjs'; + +let tmp; +let counter = 0; +before(async () => { tmp = await mkdtemp(join(tmpdir(), 'jh-rs-')); }); +after(async () => { await rm(tmp, { recursive: true, force: true }); }); + +function fresh() { return join(tmp, `t${++counter}`); } + +describe('check-release-safety', () => { + it('passes on a clean tree', async () => { + const dir = fresh(); + await mkdir(join(dir, 'skills', 'docx'), { recursive: true }); + await writeFile(join(dir, 'skills', 'docx', 'SKILL.md'), '# docx\nDocx skill.'); + await writeFile(join(dir, 'README.md'), '# Job Hunter'); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'prerequisites.md'), '# Prerequisites'); + const r = await run(dir); + assert.equal(r.exitCode, 0); + assert.equal(r.findings.length, 0); + }); + + it('fails on a forbidden filename (CV.docx)', async () => { + const dir = fresh(); + await mkdir(join(dir, 'skills', 'x'), { recursive: true }); + await writeFile(join(dir, 'skills', 'x', 'CV.docx'), 'fake'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-file' && f.detail.includes('CV.docx'))); + }); + + it('fails on SQLite sidecar files', async () => { + const dir = fresh(); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'jobhunter.sqlite-wal'), 'synthetic'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-file' && f.path.endsWith('.sqlite-wal'))); + }); + + it('fails on a forbidden filename (.env)', async () => { + const dir = fresh(); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, '.env'), 'SECRET=foo'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-file')); + }); + + it('fails on a forbidden filename (personal-info-cache.json)', async () => { + const dir = fresh(); + await mkdir(join(dir, 'skills', 'x'), { recursive: true }); + await writeFile(join(dir, 'skills', 'x', 'personal-info-cache.json'), '{}'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.path.includes('personal-info-cache.json'))); + }); + + it('fails on a forbidden directory (screenshots/)', async () => { + const dir = fresh(); + await mkdir(join(dir, 'screenshots'), { recursive: true }); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-directory' && f.detail.includes('screenshots'))); + }); + + it('fails on a forbidden directory (logs/)', async () => { + const dir = fresh(); + await mkdir(join(dir, 'logs'), { recursive: true }); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-directory' && f.detail.includes('logs'))); + }); + + it('fails on forbidden content (hardcoded user path) in skills/', async () => { + const dir = fresh(); + await mkdir(join(dir, 'skills', 'x'), { recursive: true }); + await writeFile(join(dir, 'skills', 'x', 'SKILL.md'), 'See /Users/spotted/something.'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-content' && f.detail.includes('hardcoded-user-path'))); + }); + + it('fails on forbidden content (.hermes) in docs/', async () => { + const dir = fresh(); + await mkdir(join(dir, 'docs'), { recursive: true }); + await writeFile(join(dir, 'docs', 'notes.md'), 'Config lives in .hermes directory.'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-content' && f.detail.includes('hermes-path'))); + }); + + it('fails on a non-example email address', async () => { + const dir = fresh(); + await mkdir(join(dir, 'skills', 'x'), { recursive: true }); + await writeFile(join(dir, 'skills', 'x', 'SKILL.md'), 'Contact private-person@real-domain.test'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-content' && f.detail.includes('non-example-email'))); + }); + + it('rejects references to unpublished internal specs', async () => { + const dir = fresh(); + const skill = join(dir, 'skills', 'example-skill'); + await mkdir(skill, { recursive: true }); + await writeFile(join(skill, 'SKILL.md'), 'See PRODUCT_V4.md and .planning/CONCERNS.md'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'forbidden-content' && f.detail.includes('internal-spec-reference'))); + }); + + it('rejects dated application references', async () => { + const dir = fresh(); + const refs = join(dir, 'skills', 'auto-job-application', 'references'); + await mkdir(refs, { recursive: true }); + await writeFile(join(refs, 'application-run-2026-01-01.md'), '# Private run'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'dated-application-reference')); + }); + + it('rejects dated application evidence in a generically named reference', async () => { + const dir = fresh(); + const refs = join(dir, 'skills', 'example-skill', 'references'); + await mkdir(refs, { recursive: true }); + await writeFile(join(refs, 'lessons.md'), 'Application submitted on 2026-01-01.'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'dated-application-evidence')); + }); + + it('rejects dated application evidence in skill scripts', async () => { + const dir = fresh(); + const scripts = join(dir, 'skills', 'example-skill', 'scripts'); + await mkdir(scripts, { recursive: true }); + await writeFile(join(scripts, 'helper.mjs'), '// Application submitted on 2026-01-01.'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'dated-application-evidence')); + }); + + it('rejects hardcoded sensitive application answers', async () => { + const dir = fresh(); + const scripts = join(dir, 'skills', 'auto-job-application', 'scripts'); + await mkdir(scripts, { recursive: true }); + await writeFile(join(scripts, 'apply.mjs'), "const authorized = true; const want = 'yes';"); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'hardcoded-sensitive-answer')); + }); + + it('rejects non-synthetic job identifiers and allows reserved synthetic identifiers', async () => { + const dir = fresh(); + const scripts = join(dir, 'skills', 'example-skill', 'scripts'); + await mkdir(scripts, { recursive: true }); + const file = join(scripts, 'fixture.mjs'); + await writeFile(file, "const jobId = '4423991670';"); + const unsafe = await run(dir); + assert.equal(unsafe.exitCode, 1); + assert.ok(unsafe.findings.some(f => f.type === 'real-looking-job-identifier')); + + await writeFile(file, "const jobId = '9000001001';"); + const synthetic = await run(dir); + assert.equal(synthetic.exitCode, 0); + }); + + it('rejects dangling explicit skill references', async () => { + const dir = fresh(); + const skill = join(dir, 'skills', 'example-skill'); + await mkdir(skill, { recursive: true }); + await writeFile(join(skill, 'SKILL.md'), 'Run scripts/missing-helper.mjs and test/missing-helper.test.mjs'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.equal(r.findings.filter(f => f.type === 'dangling-skill-reference').length, 2); + }); + + it('passes on examples/ with only synthetic fixtures', async () => { + const dir = fresh(); + await mkdir(join(dir, 'examples'), { recursive: true }); + await writeFile(join(dir, 'examples', 'config-template.json'), '{"key": "value"}'); + const r = await run(dir); + assert.equal(r.exitCode, 0); + }); + + it('scans scripts and tests for leaked maintainer paths', async () => { + const dir = fresh(); + await mkdir(join(dir, 'scripts'), { recursive: true }); + await mkdir(join(dir, 'test'), { recursive: true }); + await writeFile(join(dir, 'scripts', 'helper.mjs'), 'const p = "/Users/private-user/foo";'); + await writeFile(join(dir, 'test', 'helper.test.mjs'), 'const p = "/Users/private-user/foo";'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.equal(r.findings.filter((finding) => finding.type === 'forbidden-content').length, 2); + }); + + it('reports multiple findings at once', async () => { + const dir = fresh(); + await mkdir(join(dir, 'skills', 'x', 'logs'), { recursive: true }); + await writeFile(join(dir, 'skills', 'x', 'CV.docx'), 'x'); + await writeFile(join(dir, 'skills', 'x', 'SKILL.md'), '/Users/spotted/path'); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.length >= 3); + }); +}); diff --git a/test/runtime-dependencies.test.mjs b/test/runtime-dependencies.test.mjs new file mode 100644 index 0000000..62cc3a1 --- /dev/null +++ b/test/runtime-dependencies.test.mjs @@ -0,0 +1,98 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import os from 'node:os'; + +const PROJECT_ROOT = new URL('..', import.meta.url).pathname; +const SCRIPT = path.join(PROJECT_ROOT, 'scripts', 'verify-runtime-dependencies.mjs'); + +describe('verify-runtime-dependencies.mjs', () => { + it('exits 0 when node and npm are available', () => { + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 15_000, + env: process.env, + }); + assert.equal(result.status, 0, `script failed: ${result.stderr || result.stdout}`); + }); + + it('outputs PASS for node and npm lines', () => { + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 15_000, + }); + const lines = (result.stdout || '').split('\n'); + assert.ok(lines.some((l) => /\[PASS\].*\bnode\b/.test(l)), `expected [PASS] node in:\n${result.stdout}`); + assert.ok(lines.some((l) => /\[PASS\].*\bnpm\b/.test(l)), `expected [PASS] npm in:\n${result.stdout}`); + }); + + it('outputs deterministic report structure', () => { + const r1 = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, encoding: 'utf-8', timeout: 15_000, + }); + const r2 = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, encoding: 'utf-8', timeout: 15_000, + }); + // Same category order: required, conditional, optional, python. + const categoryOrder = (out) => { + const lines = out.split('\n').filter(Boolean); + const indices = []; + if (lines.some(l => l.includes('Python'))) indices.push(lines.findIndex(l => l.includes('Python'))); + return indices; + }; + assert.deepStrictEqual(categoryOrder(r1.stdout), categoryOrder(r2.stdout), + 'report category ordering not deterministic'); + }); + + it('reports better-sqlite3 and ws as PASS when installed', () => { + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 15_000, + }); + const out = result.stdout || ''; + assert.ok(/\[PASS\].*better-sqlite3/.test(out), `expected [PASS] better-sqlite3 in:\n${out}`); + assert.ok(/\[PASS\].*\bws\b/.test(out), `expected [PASS] ws in:\n${out}`); + }); + + it('reports Python imports correctly against PEP 723 declarations', () => { + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 15_000, + }); + const out = result.stdout || ''; + // PEP 723-declared imports (pypdf, reportlab, python-docx, Pillow) must be PASS. + // Only truly undeclared imports should produce WARN. + // The output must contain a Python imports line. + assert.ok(out.includes('Python'), `expected Python import report in:\n${out}`); + // If any WARN, it must name the specific file (for actionable diagnostics). + const warns = out.split('\n').filter(l => /\[WARN\].*Python/.test(l)); + for (const w of warns) { + // Undeclared imports must cite their source file. + assert.ok(w.includes('(in '), `undeclared import WARN should cite file: ${w}`); + } + }); + + it('exits 1 when required dependency is missing', () => { + // Override PATH to hide node, forcing a FAIL. + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: PROJECT_ROOT, + encoding: 'utf-8', + timeout: 15_000, + env: { + ...process.env, + PATH: '/nonexistent', + }, + }); + // node itself runs via absolute path, so PATH doesn't hide it. + // But npm won't be found if PATH is empty. However, we already started node... + // This test is best-effort: the script runs as node, so node is always found. + // We just verify it runs without crashing. + assert.ok(result.status !== undefined, 'script should complete'); + }); +}); diff --git a/test/skill-closure.test.mjs b/test/skill-closure.test.mjs new file mode 100644 index 0000000..a591e37 --- /dev/null +++ b/test/skill-closure.test.mjs @@ -0,0 +1,117 @@ +import { describe, it, before, after, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { run, REQUIRED_SKILLS } from '../scripts/verify-skill-closure.mjs'; + +const MINIMAL_SKILL_MD = `# Test Skill\n\nDescription and instructions for the user.`; + +let tmp; +let counter = 0; + +before(async () => { tmp = await mkdtemp(join(tmpdir(), 'jh-sc-')); }); +after(async () => { await rm(tmp, { recursive: true, force: true }); }); + +function freshDir() { + const d = join(tmp, `t${++counter}`); + return d; +} + +async function writeSkill(root, name, content = MINIMAL_SKILL_MD) { + const dir = join(root, 'skills', name); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'SKILL.md'), content); +} + +async function writeAllSkills(root, contentFn) { + for (const name of REQUIRED_SKILLS) { + await writeSkill(root, name, contentFn ? contentFn(name) : MINIMAL_SKILL_MD); + } +} + +describe('verify-skill-closure', () => { + it('passes when all 14 skills exist with valid SKILL.md', async () => { + const dir = freshDir(); + await writeAllSkills(dir); + const r = await run(dir); + assert.equal(r.exitCode, 0); + assert.equal(r.findings.length, 0); + }); + + it('fails when a required skill is missing', async () => { + const dir = freshDir(); + const subset = REQUIRED_SKILLS.filter(n => n !== 'captcha-resolution'); + for (const name of subset) await writeSkill(dir, name); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-skill' && f.path.includes('captcha-resolution'))); + }); + + it('fails when SKILL.md is missing for a skill', async () => { + const dir = freshDir(); + for (const name of REQUIRED_SKILLS) { + const skillDir = join(dir, 'skills', name); + await mkdir(skillDir, { recursive: true }); + if (name !== 'docx') { + await writeFile(join(skillDir, 'SKILL.md'), MINIMAL_SKILL_MD); + } + } + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'missing-skill-md' && f.path.includes('docx'))); + }); + + it('fails when SKILL.md is empty', async () => { + const dir = freshDir(); + await writeAllSkills(dir, name => name === 'pdf' ? '' : MINIMAL_SKILL_MD); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => f.type === 'empty-skill-md' && f.path.includes('pdf'))); + }); + + it('passes on cross-references to bundled skills', async () => { + const dir = freshDir(); + await writeAllSkills(dir, name => { + if (name === 'qwen-screenshot-debug') { + return `# Qwen Screenshot Debug\n\nIf CAPTCHA visible → captcha-resolution skill.`; + } + return MINIMAL_SKILL_MD; + }); + const r = await run(dir); + assert.equal(r.exitCode, 0); + }); + + it('passes on cross-references to platform integrations', async () => { + const dir = freshDir(); + await writeAllSkills(dir, name => { + if (name === 'auto-job-application') { + return `# Auto Job Application\n\nUse Chromium CDP. The browser skill handles auth.`; + } + return MINIMAL_SKILL_MD; + }); + const r = await run(dir); + assert.equal(r.exitCode, 0); + }); + + it('fails on unresolved cross-reference to unbundled skill', async () => { + const dir = freshDir(); + await writeAllSkills(dir, name => { + if (name === 'job-hunter') { + return `# Job Hunter\n\nSee skills/nonexistent-helper/ for setup details.`; + } + return MINIMAL_SKILL_MD; + }); + const r = await run(dir); + assert.equal(r.exitCode, 1); + assert.ok(r.findings.some(f => + f.type === 'unresolved-skill-ref' && f.detail.includes('nonexistent-helper') + )); + }); + + it('lists all 14 required skill names in the export', () => { + assert.equal(REQUIRED_SKILLS.length, 14); + assert.ok(REQUIRED_SKILLS.includes('job-hunter')); + assert.ok(REQUIRED_SKILLS.includes('pdf')); + }); +}); diff --git a/workspace-template/package-lock.json b/workspace-template/package-lock.json new file mode 100644 index 0000000..d76c379 --- /dev/null +++ b/workspace-template/package-lock.json @@ -0,0 +1,486 @@ +{ + "name": "workspace-template", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "better-sqlite3": "12.10.0", + "ws": "8.21.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "12.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.10.0.tgz", + "integrity": "sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x || 26.x" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.96.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.96.0.tgz", + "integrity": "sha512-rebQ/lz7i0EkoLzUVSrKRzA69zMkwLp95kKMWoMDkkM00Suxz0D7zEQPwRml5fQum24mj7bPvmlgLAmu2JCiYg==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/workspace-template/package.json b/workspace-template/package.json new file mode 100644 index 0000000..98d2b13 --- /dev/null +++ b/workspace-template/package.json @@ -0,0 +1,11 @@ +{ + "type": "module", + "private": true, + "scripts": { + "ensure-schema": "node scripts/ensure-salary-schema.mjs" + }, + "dependencies": { + "better-sqlite3": "12.10.0", + "ws": "8.21.0" + } +} diff --git a/workspace-template/personal-info-cache.example.json b/workspace-template/personal-info-cache.example.json new file mode 100644 index 0000000..f540612 --- /dev/null +++ b/workspace-template/personal-info-cache.example.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 2, + "lastUpdated": null, + "purpose": "Cache user-provided job-application answers that are not normally present in the CV. Use to pre-fill future applications, but verify when context-specific or legally sensitive.", + "profile": { + "firstName": "", + "lastName": "", + "email": "", + "phone": "", + "website": "" + }, + "workHistory": [], + "education": [], + "applicationPreferences": {}, + "workAuthorization": {}, + "companySpecific": {}, + "portalCredentials": {}, + "notes": "", + "rolePreferences": {} +} diff --git a/workspace-template/schema.sql b/workspace-template/schema.sql new file mode 100644 index 0000000..b96120d --- /dev/null +++ b/workspace-template/schema.sql @@ -0,0 +1,84 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE jobs ( + source TEXT NOT NULL, + job_id TEXT NOT NULL, + url TEXT, + title TEXT, + company TEXT, + description_raw TEXT, + description_text TEXT, + location_raw TEXT, + country_code TEXT, + region TEXT, + city TEXT, + job_posting_date TEXT, + applicants_raw TEXT, + applicants_count INTEGER CHECK (applicants_count IS NULL OR applicants_count >= 0), + application_links_json TEXT, + recruiter TEXT, + recruiter_email TEXT, + recruiter_profile_link TEXT, + role_family_inferred TEXT, + role_family_confidence REAL CHECK ( + role_family_confidence IS NULL + OR (role_family_confidence >= 0 AND role_family_confidence <= 1) + ), + role_family_reason TEXT, + language_filter_reason TEXT, + work_mode_reason TEXT, + searched_keywords TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (source, job_id) +); + +CREATE TABLE match_results ( + job_source TEXT NOT NULL, + job_id TEXT NOT NULL, + fit_score REAL NOT NULL, + cta TEXT NOT NULL DEFAULT 'Skip' CHECK (cta IN ('Apply', 'Skip')), + stretch_label TEXT, + blockers TEXT, + tailoring_suggestions TEXT, + scored_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (job_source, job_id), + FOREIGN KEY (job_source, job_id) REFERENCES jobs(source, job_id) +); + +CREATE TABLE salaries ( + job_source TEXT NOT NULL, + job_id TEXT NOT NULL, + salary_min INTEGER CHECK (salary_min IS NULL OR salary_min >= 0), + salary_max INTEGER CHECK (salary_max IS NULL OR salary_max >= 0), + salary_currency TEXT, + salary_period TEXT, + provenance TEXT, + fetched_at TEXT, + PRIMARY KEY (job_source, job_id), + FOREIGN KEY (job_source, job_id) REFERENCES jobs(source, job_id) +); + +CREATE TABLE applications ( + job_source TEXT NOT NULL, + job_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','applied','failed','skipped')), + applied_at TEXT, + log_path TEXT, + PRIMARY KEY (job_source, job_id), + FOREIGN KEY (job_source, job_id) REFERENCES jobs(source, job_id) +); + +CREATE TABLE writer_lock ( + id INTEGER PRIMARY KEY CHECK (id = 1), + holder TEXT, + acquired_at TEXT +); + +CREATE TRIGGER jobs_updated + AFTER UPDATE ON jobs + FOR EACH ROW + WHEN NEW.updated_at = OLD.updated_at + BEGIN + UPDATE jobs SET updated_at = datetime('now') WHERE source = NEW.source AND job_id = NEW.job_id; + END; diff --git a/workspace-template/scripts/ensure-salary-schema.mjs b/workspace-template/scripts/ensure-salary-schema.mjs new file mode 100644 index 0000000..70a6de2 --- /dev/null +++ b/workspace-template/scripts/ensure-salary-schema.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import { join } from 'node:path'; + +/** Forward schema maintenance to the installed salary-calculator skill. */ +const agentHome = process.env.PI_AGENT_HOME || join(process.env.HOME, '.pi', 'agent'); +const script = join(agentHome, 'skills', 'salary-calculator', 'scripts', 'ensure-salary-schema.mjs'); +const result = spawnSync(process.execPath, [script, ...process.argv.slice(2)], { + env: process.env, + stdio: 'inherit', +}); +process.exit(result.status ?? 1);