diff --git a/AGENTS.md b/AGENTS.md index dd8e527ed..9e38a9759 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,17 +48,17 @@ Inside the image, sites live at `/opt/WebSyn//`. The path predates the ren # fresh clone ./scripts/fetch_assets.sh # pulls assets from HF ./scripts/build.sh # docker build -t webharbor:dev . -docker run -d -p 8101:8101 -p 40000-40027:40000-40027 webharbor:dev +docker run -d -p 8101:8101 -p 40000-40028:40000-40028 webharbor:dev ``` Or use the published image directly: ```bash -docker run -d -p 8101:8101 -p 40000-40027:40000-40027 \ +docker run -d -p 8101:8101 -p 40000-40028:40000-40028 \ battalion7244/webharbor:latest ``` -Sites are on `40000`-`40027` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane: +Sites are on `40000`-`40028` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane: | Method | Path | Purpose | |--------|---------------------|-------------------------------------------| @@ -136,13 +136,13 @@ python3 -m py_compile sites//app.py # 3. run on alt ports (don't collide with anything you already have running) docker run -d --rm --name wh-test \ - -p 8201:8101 -p 41000-41027:40000-40027 webharbor:dev + -p 8201:8101 -p 41000-41028:40000-40028 webharbor:dev # 4. control plane healthy, all sites alive curl -s http://localhost:8201/health | python3 -m json.tool | head # 5. every site renders 200 -for p in $(seq 41000 41027); do +for p in $(seq 41000 41028); do curl -so /dev/null -w "$p:%{http_code}\n" http://localhost:$p/ done diff --git a/CLAUDE.md b/CLAUDE.md index 94d397d1b..5e84cb62b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,4 +16,4 @@ The full agent guide is loaded above via `@AGENTS.md`. The notes below apply onl ## Existing containers -If a container is already running on `:8101` / `:40000-40027`, treat it as the user's working environment — don't `docker stop` or `docker rm` it without explicit confirmation. Spin up your test container under a different name on alt ports (`:8201`, `:41000-41027`). +If a container is already running on `:8101` / `:40000-40028`, treat it as the user's working environment — don't `docker stop` or `docker rm` it without explicit confirmation. Spin up your test container under a different name on alt ports (`:8201`, `:41000-41028`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3871b5e89..22e4c1c25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ git clone https://github.com//webharbor && cd webharbor ./scripts/fetch_assets.sh # pull current assets ./scripts/new_site.py mywebsite # OR edit an existing site ./scripts/build.sh && docker run -d --rm \ - -p 8101:8101 -p 40000-40027:40000-40027 webharbor:dev + -p 8101:8101 -p 40000-40028:40000-40028 webharbor:dev # iterate locally... ./scripts/extract_assets.sh ../webharbor-static-pr/ # split assets out diff --git a/Dockerfile b/Dockerfile index d35d70bcd..60b502b40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # WebHarbor — slim, self-contained image. -# 28 Flask mirror sites + control plane on :8101. +# 29 Flask mirror sites + control plane on :8101. FROM python:3.12-slim-bookworm @@ -92,6 +92,15 @@ os.makedirs('instance_seed', exist_ok=True); \ shutil.copy2('instance/rotten_tomatoes.db', 'instance_seed/rotten_tomatoes.db'); \ print('Rotten Tomatoes seed DB generated at build time.')" && rm -rf /opt/WebSyn/rotten_tomatoes/instance -EXPOSE 8101 40000-40027 +# AccuWeather uses genuine captured UI assets and freezes its deterministic seed. +RUN test -n "$(ls -A /opt/WebSyn/accuweather/static/images)" && \ + cd /opt/WebSyn/accuweather && rm -rf instance instance_seed && python3 -c "\ +import app; \ +import os, shutil; \ +os.makedirs('instance_seed', exist_ok=True); \ +shutil.copy2('instance/accuweather.db', 'instance_seed/accuweather.db'); \ +print('AccuWeather seed DB generated at build time.')" && rm -rf /opt/WebSyn/accuweather/instance + +EXPOSE 8101 40000-40028 CMD ["/opt/websyn_start.sh"] diff --git a/README.md b/README.md index 4189c2099..6fa457bf6 100644 --- a/README.md +++ b/README.md @@ -36,17 +36,17 @@ WebHarbor takes a different approach. We leverage coding agent (e.g., Claude Cod - **Deep features unlocked** — carts, checkouts, accounts, all fully testable - **Evolving** — harder tasks drive richer mirrors; the environment grows with agents - **RL-ready** — sub-second database resets between rollouts -- **Community-driven** — 28 sites today, scaling to 100+ together +- **Community-driven** — 29 sites today, scaling to 100+ together ## 🚀 Quickstart One command to run all web environments: ```bash -docker run -p 8101:8101 -p 40000-40027:40000-40027 battalion7244/webharbor:latest +docker run -p 8101:8101 -p 40000-40028:40000-40028 battalion7244/webharbor:latest ``` -Then point your agent at `http://localhost:40000` through `http://localhost:40027` to explore 28 local mirrors of WebVoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, ESPN, Merriam-Webster, IKEA, Phys.org, Target, TED, Ohio State University, Rotten Tomatoes, Compass, Walmart Careers, FedEx, WebMD Doctor, Healthline, and Kaggle`. +Then point your agent at `http://localhost:40000` through `http://localhost:40027` to explore 29 local mirrors of WebVoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, ESPN, Merriam-Webster, IKEA, Phys.org, Target, TED, Ohio State University, Rotten Tomatoes, Compass, Walmart Careers, FedEx, WebMD Doctor, Healthline, Kaggle, and AccuWeather`. For sub-second reset between rollouts, expose the control plane and call `/reset/`: diff --git a/agent_demo/README.md b/agent_demo/README.md index ee26ed3ab..0570c745c 100644 --- a/agent_demo/README.md +++ b/agent_demo/README.md @@ -19,7 +19,7 @@ export OPENAI_BASE_URL=https://api.openai.com/v1 # or your Azure / vLLM endpoi ## Run a task -WebHarbor must already be running locally (`docker run -p 8101:8101 -p 40000-40027:40000-40027 battalion7244/webharbor:latest`). +WebHarbor must already be running locally (`docker run -p 8101:8101 -p 40000-40028:40000-40028 battalion7244/webharbor:latest`). Run a single task from a site's `tasks.jsonl`: diff --git a/control_server.py b/control_server.py index 939613ecb..7868a5f07 100644 --- a/control_server.py +++ b/control_server.py @@ -30,6 +30,7 @@ 'ikea', 'phys_org', 'target', 'ted', 'osu', 'rotten_tomatoes', 'compass', 'walmart_careers', 'fedex', 'webmd_doctor', 'healthline', 'kaggle', + 'accuweather', ] BASE_PORT = 40000 WEBSYN_DIR = '/opt/WebSyn' diff --git a/review-reports/ACCUWEATHER-FINAL-AUDIT.md b/review-reports/ACCUWEATHER-FINAL-AUDIT.md new file mode 100644 index 000000000..bad597f25 --- /dev/null +++ b/review-reports/ACCUWEATHER-FINAL-AUDIT.md @@ -0,0 +1,54 @@ +# AccuWeather final task and UI audit + +Audit date: 2026-09-09. Branch: `add-accuweather-mirror`. + +The site was reset before every task and browser cookies were cleared. Every run began at `http://localhost:41024/` and used Playwright visible-element locators to search, open results, select tabs, sign in, change preferences, and submit forms. No task used a direct destination URL, database lookup, or source-code answer. + +| Task | Result | Flow checked | Screenshot | +|---|---|---|---| +| AccuWeather--0 | Pass | Search → Phoenix current conditions | `outputs/accuweather-review/0-99-final.png` | +| AccuWeather--1 | Pass | Ambiguous Portland search → Maine result | `outputs/accuweather-review/1-99-final.png` | +| AccuWeather--2 | Pass | Search → Seattle → Hourly | `outputs/accuweather-review/2-99-final.png` | +| AccuWeather--3 | Pass | Search → Miami → Daily | `outputs/accuweather-review/3-99-final.png` | +| AccuWeather--4 | Pass | Search and compare Austin / Denver | `outputs/accuweather-review/4-99-final.png` | +| AccuWeather--5 | Pass | Springfield distractors → Missouri → Air Quality | `outputs/accuweather-review/5-99-final.png` | +| AccuWeather--6 | Pass | Login → search → save Seattle → account | `outputs/accuweather-review/6-99-final.png` | +| AccuWeather--7 | Pass | Login → Boston → remove → account | `outputs/accuweather-review/7-99-final.png` | +| AccuWeather--8 | Pass | Login → Chicago → alert form submit | `outputs/accuweather-review/8-99-final.png` | +| AccuWeather--9 | Pass | Login → Settings → Celsius → New York | `outputs/accuweather-review/9-99-final.png` | +| AccuWeather--10 | Pass | Postal search → San Francisco | `outputs/accuweather-review/10-99-final.png` | +| AccuWeather--11 | Pass | Search → radar → current weather | `outputs/accuweather-review/11-99-final.png` | +| AccuWeather--12 | Pass | Compare two Air Quality pages | `outputs/accuweather-review/12-99-final.png` | +| AccuWeather--13 | Pass | Search → Boston seven-day forecast | `outputs/accuweather-review/13-99-final.png` | +| AccuWeather--14 | Pass | London current → Air Quality | `outputs/accuweather-review/14-99-final.png` | +| AccuWeather--15 | Pass | Login → save two cities → account | `outputs/accuweather-review/15-99-final.png` | +| AccuWeather--16 | Pass | Portland Oregon / Maine comparison | `outputs/accuweather-review/16-99-final.png` | +| AccuWeather--17 | Pass | Search → Toronto hourly forecast | `outputs/accuweather-review/17-99-final.png` | +| AccuWeather--18 | Pass | Register → save Atlanta → account | `outputs/accuweather-review/18-99-final.png` | +| AccuWeather--19 | Pass | Phoenix / New Orleans comparison | `outputs/accuweather-review/19-99-final.png` | + +## Hardening results + +- De-leak: search results disclose only location identity and postal code, never current, hourly, daily, or air-quality answers. The prompt wording, ordering, labels, and cards do not reveal computed answers. +- Distractors: ambiguous Portland and Springfield queries return two and three same-name locations respectively; the 20-location catalog also supplies realistic cross-city comparison choices. +- Catalog breadth: 20 locations, 140 daily rows, and 240 hourly rows cover United States, Canada, and United Kingdom conditions and all captured icon types. +- Cross-field consistency: current temperatures, derived forecasts, unit conversion, conditions, location identity, saved locations, and alerts use relational records shared by all views. +- Leak archetypes checked: direct answer in prompt; target first by artificial ordering; pre-sorted winner; count label; detail on result card; unique target without distractors; answer in URL/slug; hidden data attribute; accessible-name answer; placeholder answer; success without mutation; visit-only verification; unrelated-state acceptance. None were found. +- Five hard multi-step tasks: 4, 5, 12, 13, and 19. Tasks 6–9, 15, and 18 additionally verify persistent state changes. + +## Responsive and asset checks + +Home, search, current, hourly, and radar pages were checked at 1440, 390, and 320 CSS pixels (15 page/viewport combinations). All had zero horizontal overflow and zero broken images. The committed screenshots below are representative; the full evidence set and structured step log are in `outputs/accuweather-review/`. + +- `review-reports/assets/accuweather-homepage-1440.png` +- `review-reports/assets/accuweather-homepage-390.png` +- `review-reports/assets/accuweather-homepage-320.png` + +The Solis font, GPS icon, and weather condition SVGs were captured from the live AccuWeather homepage asset inventory. No placeholder image is used. + +## Reset proof + +After the final task audit, `POST /reset/accuweather` completed ready and both files had MD5 `2701fb49e1edc178024785e6f8601870`: + +- `/opt/WebSyn/accuweather/instance/accuweather.db` +- `/opt/WebSyn/accuweather/instance_seed/accuweather.db` diff --git a/review-reports/assets/accuweather-homepage-1440.png b/review-reports/assets/accuweather-homepage-1440.png new file mode 100644 index 000000000..e28754c8c Binary files /dev/null and b/review-reports/assets/accuweather-homepage-1440.png differ diff --git a/review-reports/assets/accuweather-homepage-320.png b/review-reports/assets/accuweather-homepage-320.png new file mode 100644 index 000000000..f7f8bb502 Binary files /dev/null and b/review-reports/assets/accuweather-homepage-320.png differ diff --git a/review-reports/assets/accuweather-homepage-390.png b/review-reports/assets/accuweather-homepage-390.png new file mode 100644 index 000000000..68caefac4 Binary files /dev/null and b/review-reports/assets/accuweather-homepage-390.png differ diff --git a/sites/accuweather/.build-generated-seed b/sites/accuweather/.build-generated-seed new file mode 100644 index 000000000..3ed94ddc9 --- /dev/null +++ b/sites/accuweather/.build-generated-seed @@ -0,0 +1 @@ +This site rebuilds instance_seed/accuweather.db deterministically at image build time (see Dockerfile); the HF bundle carries no seed DB. diff --git a/sites/accuweather/.requires-images b/sites/accuweather/.requires-images new file mode 100644 index 000000000..1d3d5d593 --- /dev/null +++ b/sites/accuweather/.requires-images @@ -0,0 +1 @@ +This site requires static/images (captured AccuWeather weather icons + GPS icon) from the pinned Hugging Face asset bundle. diff --git a/sites/accuweather/README.md b/sites/accuweather/README.md new file mode 100644 index 000000000..37c1b9476 --- /dev/null +++ b/sites/accuweather/README.md @@ -0,0 +1,73 @@ +# AccuWeather mirror + +An offline Flask mirror of accuweather.com used as a deterministic benchmark +fixture. It serves 20 locations with current conditions, hourly and daily +forecasts, radar, air quality, accounts, saved locations, alert preferences and +a temperature-unit setting. + +## Scope and deliberate simplifications + +- **No JavaScript.** No template loads a script; every interaction is a plain + form or link, so the site degrades perfectly without JS. +- **Radar is decorative.** `/radar/` renders a CSS gradient, not a map + tile or a real radar image. The heading names the city; nothing else on the + page is a fact a task may depend on. +- **Forecasts are generated from the current temperature** (`app.py`, + `seed_database`): today's high is `current - 2`, the hourly peak is 4 PM for + every city, and the lowest daily low is Saturday for every city. This makes + the list-scan pattern identical across cities. It is deterministic and + self-consistent, but it is not realistic per-city weather. +- **Content is synthetic.** No value is a real-world fact, so no task can be + answered from model knowledge. + +## Seed determinism + +`instance_seed/accuweather.db` is rebuilt at image build time (see +`.build-generated-seed` and the Dockerfile step). The four benchmark users' +password hashes are frozen constants in `app.py` +(`BENCHMARK_PASSWORD_HASHES`) because `generate_password_hash` salts randomly, +which would otherwise make the seed a different file on every build. + +The seed is byte-reproducible **within one SQLite runtime**. Across SQLite +versions the file bytes can differ while every row is identical; compare the +row-level catalog fingerprint (`verify/verify_lib.py`, `CATALOG_FINGERPRINT`) +rather than the file's md5 when checking across environments. + +## Assets and provenance + +Per-asset source URLs are recorded in `provenance.json`. The seven SVG weather +and GPS icons are captured from accuweather.com and are fetched from the pinned +Hugging Face asset bundle (`.requires-images`); they are not tracked in git. + +### Font notice + +`static/fonts/Solis-Regular.woff2` (22832 bytes, sha256 +`e23435d0e387ffe2c818e1f500d0e58e7e996251871fad7df54b38404cc3a384`) is the +webfont served by accuweather.com. Its SFNT name table is shipped **verbatim +and unmodified**; in particular: + +- name ID 0 (copyright): `© Copyright AccuWeather, 2019. All rights reserved.` +- name ID 8 (manufacturer): `Type Network` +- name ID 9 (designer): `Laura Meseguer with Dyana Weissman` +- name ID 11 (vendor URL): `http://www.typenetwork.com/` +- name ID 14 (license URL): `http://www.loyalkaspar.com` + +The file carries no name ID 13 (license description) upstream, and no licence +grant accompanies it. It is included only to reproduce the visual proportions +of the original page in an offline research fixture. + +**Non-affiliation.** This mirror is not affiliated with, endorsed by, or +sponsored by AccuWeather, Inc. or Type Network. "AccuWeather" and "RealFeel" +are trademarks of AccuWeather, Inc., used here only to identify the site being +mirrored. + +**To remove the font**, delete `static/fonts/Solis-Regular.woff2` and the +`@font-face` rule at the top of `static/css/site.css`. The stylesheet already +declares the fallback stack `Solis, Arial, sans-serif`, so the page renders in +Arial with no further change. + +## Grading + +Deterministic verifiers for all 20 tasks live in `verify/`; see +`verify/README.md` for the contract and `verify/tests/` for the unit harness +and the real-browser matrix. diff --git a/sites/accuweather/_health.py b/sites/accuweather/_health.py new file mode 100644 index 000000000..818a78785 --- /dev/null +++ b/sites/accuweather/_health.py @@ -0,0 +1,3 @@ +"""Per-site health probe (optional, called by control_server).""" +def health(): + return {"ok": True, "site": "accuweather"} diff --git a/sites/accuweather/app.py b/sites/accuweather/app.py new file mode 100644 index 000000000..323e70ffb --- /dev/null +++ b/sites/accuweather/app.py @@ -0,0 +1,194 @@ +"""Deterministic, interaction-complete AccuWeather mirror.""" +import os, re, secrets +from datetime import datetime +from functools import wraps +from urllib.parse import urlsplit +from flask import Flask, abort, flash, redirect, render_template, request, session, url_for +from flask_sqlalchemy import SQLAlchemy +from flask_wtf.csrf import CSRFProtect +from sqlalchemy import event +from sqlalchemy.engine import Engine +from werkzeug.security import check_password_hash, generate_password_hash + +BASE_DIR=os.path.dirname(os.path.abspath(__file__)) +app=Flask(__name__,instance_path=os.path.join(BASE_DIR,"instance")) +app.config.update(SECRET_KEY=os.environ.get("ACCUWEATHER_SECRET_KEY") or secrets.token_hex(32),SQLALCHEMY_DATABASE_URI="sqlite:///accuweather.db",SQLALCHEMY_TRACK_MODIFICATIONS=False,MAX_CONTENT_LENGTH=64*1024,WTF_CSRF_TIME_LIMIT=None) +db=SQLAlchemy(app) +csrf=CSRFProtect(app) # registers csrf_token() in Jinja; every POST form renders it + +@event.listens_for(Engine,"connect") +def _sqlite_foreign_keys(dbapi_connection,_record): + cursor=dbapi_connection.cursor(); cursor.execute("PRAGMA foreign_keys=ON"); cursor.close() + +class User(db.Model): + id=db.Column(db.Integer,primary_key=True); email=db.Column(db.String(120),unique=True,nullable=False); name=db.Column(db.String(80),nullable=False); password_hash=db.Column(db.String(255),nullable=False); unit=db.Column(db.String(1),default="F",nullable=False) +class Location(db.Model): + id=db.Column(db.Integer,primary_key=True); slug=db.Column(db.String(100),unique=True,nullable=False); city=db.Column(db.String(80),nullable=False); region=db.Column(db.String(80),nullable=False); country=db.Column(db.String(80),nullable=False); postal=db.Column(db.String(20),nullable=False); temp=db.Column(db.Integer,nullable=False); realfeel=db.Column(db.Integer,nullable=False); condition=db.Column(db.String(80),nullable=False); icon=db.Column(db.String(2),nullable=False); humidity=db.Column(db.Integer,nullable=False); wind=db.Column(db.Integer,nullable=False); visibility=db.Column(db.Integer,nullable=False); pressure=db.Column(db.Float,nullable=False); uv=db.Column(db.Integer,nullable=False); air_quality=db.Column(db.Integer,nullable=False) +class Forecast(db.Model): + id=db.Column(db.Integer,primary_key=True); location_id=db.Column(db.Integer,db.ForeignKey("location.id"),nullable=False); day_index=db.Column(db.Integer,nullable=False); label=db.Column(db.String(30),nullable=False); high=db.Column(db.Integer,nullable=False); low=db.Column(db.Integer,nullable=False); condition=db.Column(db.String(80),nullable=False); icon=db.Column(db.String(2),nullable=False); precip=db.Column(db.Integer,nullable=False); location=db.relationship(Location,backref="forecasts") +class Hourly(db.Model): + id=db.Column(db.Integer,primary_key=True); location_id=db.Column(db.Integer,db.ForeignKey("location.id"),nullable=False); hour_index=db.Column(db.Integer,nullable=False); label=db.Column(db.String(20),nullable=False); temp=db.Column(db.Integer,nullable=False); condition=db.Column(db.String(80),nullable=False); icon=db.Column(db.String(2),nullable=False); precip=db.Column(db.Integer,nullable=False); location=db.relationship(Location,backref="hourly") +class SavedLocation(db.Model): + id=db.Column(db.Integer,primary_key=True); user_id=db.Column(db.Integer,db.ForeignKey("user.id"),nullable=False); location_id=db.Column(db.Integer,db.ForeignKey("location.id"),nullable=False); __table_args__=(db.UniqueConstraint("user_id","location_id"),) +class Alert(db.Model): + id=db.Column(db.Integer,primary_key=True); user_id=db.Column(db.Integer,db.ForeignKey("user.id"),nullable=False); location_id=db.Column(db.Integer,db.ForeignKey("location.id"),nullable=False); alert_type=db.Column(db.String(30),nullable=False); enabled=db.Column(db.Boolean,default=True,nullable=False); __table_args__=(db.UniqueConstraint("user_id","location_id","alert_type"),) + +EMAIL_RE=re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+") +ALERT_TYPES=("severe","rain","temperature") +UNITS=("F","C") +MAX_EMAIL,MAX_NAME,MAX_PASSWORD=120,80,128 + +def bounded(value,limit): + """Trimmed text, or "" when it is empty or longer than the column allows. + SQLite does not enforce VARCHAR length, so the app has to.""" + text=str(value or "").strip() + return text if 0") +def weather(slug): + loc=Location.query.filter_by(slug=slug).first_or_404(); saved=bool(current_user() and SavedLocation.query.filter_by(user_id=current_user().id,location_id=loc.id).first()); return render_template("weather.html",location=loc,saved=saved) +@app.route("/hourly/") +def hourly(slug): return render_template("hourly.html",location=Location.query.filter_by(slug=slug).first_or_404()) +@app.route("/daily/") +def daily(slug): return render_template("daily.html",location=Location.query.filter_by(slug=slug).first_or_404()) +@app.route("/radar/") +def radar(slug): return render_template("radar.html",location=Location.query.filter_by(slug=slug).first_or_404()) +@app.route("/air-quality/") +def air_quality(slug): return render_template("air_quality.html",location=Location.query.filter_by(slug=slug).first_or_404()) +@app.route("/save/",methods=["POST"]) +@login_required +def save_location(slug): + loc=Location.query.filter_by(slug=slug).first_or_404(); old=SavedLocation.query.filter_by(user_id=current_user().id,location_id=loc.id).first() + if old: db.session.delete(old); flash(f"Removed {loc.city} from saved locations.") + else: db.session.add(SavedLocation(user_id=current_user().id,location_id=loc.id)); flash(f"Saved {loc.city}.") + db.session.commit(); return redirect(local_referrer(url_for("weather",slug=slug))) +@app.route("/alerts/",methods=["GET","POST"]) +@login_required +def alerts(slug): + loc=Location.query.filter_by(slug=slug).first_or_404() + if request.method=="POST": + selected=set(request.form.getlist("alert_type")) + if not selected<=set(ALERT_TYPES): abort(400) + Alert.query.filter_by(user_id=current_user().id,location_id=loc.id).delete() + for kind in sorted(selected): db.session.add(Alert(user_id=current_user().id,location_id=loc.id,alert_type=kind,enabled=True)) + db.session.commit(); flash("Alert preferences updated.") + active={a.alert_type for a in Alert.query.filter_by(user_id=current_user().id,location_id=loc.id).all()}; return render_template("alerts.html",location=loc,active=active) +@app.route("/account") +@login_required +def account(): + saved=db.session.query(Location).join(SavedLocation,SavedLocation.location_id==Location.id).filter(SavedLocation.user_id==current_user().id).all(); return render_template("account.html",saved=saved) +@app.route("/settings",methods=["GET","POST"]) +@login_required +def settings(): + if request.method=="POST": + unit=request.form.get("unit") + if unit not in UNITS: abort(400) + current_user().unit=unit; session["unit"]=unit; db.session.commit(); flash("Units updated.") + return render_template("settings.html") +@app.route("/login",methods=["GET","POST"]) +def login(): + if request.method=="POST": + user=User.query.filter_by(email=request.form.get("email","").lower().strip()).first() + if user and check_password_hash(user.password_hash,request.form.get("password","")): session["user_id"]=user.id; session["unit"]=user.unit; flash("Welcome back."); return redirect(local_path(request.args.get("next")) or url_for("account")) + flash("Email or password is incorrect.") + return render_template("login.html") +@app.route("/register",methods=["GET","POST"]) +def register(): + if request.method=="POST": + email=bounded(request.form.get("email"),MAX_EMAIL).lower(); name=bounded(request.form.get("name"),MAX_NAME); password=request.form.get("password","") + if not email or not EMAIL_RE.fullmatch(email) or not name or len(password)>MAX_PASSWORD: abort(400) + if User.query.filter_by(email=email).first(): flash("An account already exists for that email.") + elif len(password)<8: flash("Password must be at least 8 characters.") + else: + user=User(email=email,name=name,password_hash=generate_password_hash(password)); db.session.add(user); db.session.commit(); session["user_id"]=user.id; return redirect(url_for("account")) + return render_template("register.html") +@app.route("/logout",methods=["POST"]) +def logout(): session.clear(); return redirect(url_for("index")) +@app.route("/about") +def about(): return render_template("static_page.html",title="About AccuWeather",copy="We provide local weather forecasts and severe weather information for communities around the world.") +@app.route("/privacy") +def privacy(): return render_template("static_page.html",title="Privacy Statement",copy="Your privacy choices and account preferences are available here.") +@app.route("/_health") +def health(): return {"ok":True,"site":"accuweather"} +ERROR_COPY={400:"That request was not valid.",403:"That request could not be verified. Please reload the page and try again.",404:"We could not find that page.",405:"That address does not accept this kind of request.",413:"That upload is too large.",500:"Something went wrong on our end."} +@app.errorhandler(400) +@app.errorhandler(403) +@app.errorhandler(404) +@app.errorhandler(405) +@app.errorhandler(413) +@app.errorhandler(500) +def handle_error(error): + code=getattr(error,"code",500); db.session.rollback() + return render_template("error.html",code=code,message=ERROR_COPY.get(code,"Something went wrong.")),code +if __name__=="__main__": app.run(host="0.0.0.0",port=int(os.environ.get("PORT",5000)),debug=False) diff --git a/sites/accuweather/provenance.json b/sites/accuweather/provenance.json new file mode 100644 index 000000000..4ccc53a3c --- /dev/null +++ b/sites/accuweather/provenance.json @@ -0,0 +1,26 @@ +{ + "captured_at": "2026-09-09", + "source": "https://www.accuweather.com/", + "method": "Browser-visible page asset inventory from the rendered AccuWeather homepage", + "assets": { + "static/fonts/Solis-Regular.woff2": "https://www.accuweather.com/fonts/Solis-Regular.woff2", + "static/images/weather-01.svg": "https://www.accuweather.com/images/weathericons/v2a/01.svg", + "static/images/weather-02.svg": "https://www.accuweather.com/images/weathericons/v2a/02.svg", + "static/images/weather-03.svg": "https://www.accuweather.com/images/weathericons/v2a/03.svg", + "static/images/weather-06.svg": "https://www.accuweather.com/images/weathericons/v2a/06.svg", + "static/images/weather-07.svg": "https://www.accuweather.com/images/weathericons/v2a/07.svg", + "static/images/weather-12.svg": "https://www.accuweather.com/images/weathericons/v2a/12.svg", + "static/images/icon-gps.svg": "https://www.awxcdn.com/adc-assets/images/icons/icon-gps.svg" + }, + "sha256": { + "static/fonts/Solis-Regular.woff2": "e23435d0e387ffe2c818e1f500d0e58e7e996251871fad7df54b38404cc3a384", + "static/images/weather-01.svg": "0fcecf04217671da1ac65d6a81386380bef52e89adbcfd83f485de987bbd8709", + "static/images/weather-02.svg": "86e373fd01509dccccfe1fc35c5f3fc7a9fa5a4126424ca9e8fa7a52af75e544", + "static/images/weather-03.svg": "50b3f5fdb992d905624e540a509ebe75bc545af91bff6293f049560bb7e90f5c", + "static/images/weather-06.svg": "481da999b14b7fa9e793cbdf1b15dd51738f06ad8fa6333338c6921af5096c26", + "static/images/weather-07.svg": "db73455f1f3319e1f22815df478bcb6d25c69ae485d0935ac2ec7486b9b267ea", + "static/images/weather-12.svg": "bf6b677e56531a1dd8fb729bc32672c54b88eef5950588e02a0de9152cc7d302", + "static/images/icon-gps.svg": "d3c18b7fe63f48546ee507a1a39024c07655ff1565fb2edf0910c19386e8005c" + }, + "notes": "Font name IDs 0 (copyright) and 14 (license URL) are preserved verbatim; see README.md for the licence and non-affiliation notice. Hashes are of the files as extracted from the pinned Hugging Face bundle." +} diff --git a/sites/accuweather/requirements.txt b/sites/accuweather/requirements.txt new file mode 100644 index 000000000..dbc8aa63a --- /dev/null +++ b/sites/accuweather/requirements.txt @@ -0,0 +1,6 @@ +Flask==3.1.0 +Flask-SQLAlchemy==3.1.1 +Flask-WTF==1.2.2 +Werkzeug==3.1.3 +SQLAlchemy==2.0.36 +Jinja2==3.1.4 diff --git a/sites/accuweather/static/css/site.css b/sites/accuweather/static/css/site.css new file mode 100644 index 000000000..b8f5a02ec --- /dev/null +++ b/sites/accuweather/static/css/site.css @@ -0,0 +1,9 @@ +:root{--brand:#f05514;--action:#c2410c;--focus:#1a73e8} +@font-face{font-family:Solis;src:url('../fonts/Solis-Regular.woff2')}*{box-sizing:border-box}html,body{margin:0;background:#f1f1f1;color:#292929;font-family:Solis,Arial,sans-serif}a{color:inherit;text-decoration:none}button,.button{border:0;border-radius:2px;background:var(--action);color:#fff;padding:12px 22px;font:inherit;cursor:pointer}input{font:inherit}:focus-visible{outline:3px solid var(--focus);outline-offset:2px}.corp{height:34px;background:#1d1d1d;color:#bbb;display:flex;justify-content:flex-end;gap:28px;padding:8px calc((100% - 1180px)/2);font-size:12px;white-space:nowrap}.breaking{background:#303030;color:#fff;text-align:center;padding:8px;font-size:13px}header{height:66px;background:#222;color:#fff;display:flex;align-items:center;gap:24px;padding:0 max(24px,calc((100% - 1180px)/2));position:relative;z-index:2}.logo{font-size:28px;white-space:nowrap}.logo b{color:#f25b24}.menu{font-size:22px}.search{display:flex;max-width:470px;flex:1;margin:auto}.search input{min-width:0;flex:1;padding:12px;border:0}.search button{font-size:24px;padding:4px 15px}.premium{color:#ffb294}.flash{background:#fff4d5;border-left:5px solid #f05514;max-width:920px;margin:18px auto 0;padding:14px}main{min-height:70vh}.hero{height:170px;background:linear-gradient(135deg,#3d6685,#acc7d8);display:grid;place-items:center}.hero form{display:flex;width:min(700px,90%)}.hero input{flex:1;min-width:0;padding:18px;border:0;font-size:17px}.layout{max-width:1180px;margin:24px auto;display:grid;grid-template-columns:minmax(0,2fr) minmax(260px,1fr);gap:22px}.narrow{max-width:920px;margin:32px auto;padding:0 18px}.card{background:white;box-shadow:0 2px 5px #0002;padding:22px}.eyebrow{font-size:13px;letter-spacing:.08em;color:#666}.current{display:grid;grid-template-columns:1fr 100px 100px 1fr;align-items:center;gap:20px;border-top:3px solid #f05514}.current h1{font-size:23px;margin:0}.current strong,.reading strong{font-size:62px;font-weight:400}.weather-icon{width:72px;height:72px;object-fit:contain}.city-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:12px}.city{display:grid;grid-template-columns:1fr 55px 60px;align-items:center;gap:8px}.city .weather-icon{width:50px;height:50px}.city span{grid-column:1/-1;color:#666}.tabs{display:flex;overflow-x:auto;background:#fff;border-bottom:1px solid #ddd;margin:20px 0}.tabs a{padding:15px 18px;white-space:nowrap}.tabs a:hover{border-bottom:3px solid #f05514}.weather-main{display:grid;grid-template-columns:1fr 1fr;gap:18px;border-top:3px solid #f05514}.reading{display:flex;align-items:center;justify-content:flex-end}.weather-main dl{grid-column:1/-1;margin:0}.weather-main dl div{display:flex;justify-content:space-between;border-top:1px solid #ddd;padding:10px}.weather-main dd{margin:0;font-weight:bold}.actions,.account-head{display:flex;align-items:center;gap:15px;margin:18px 0}.account-head h1{margin-right:auto}.results{background:#fff;box-shadow:0 2px 5px #0002}.result{display:flex;align-items:center;gap:15px;padding:17px;border-bottom:1px solid #ddd}.result img{width:26px;height:26px}.result span{display:flex;flex-direction:column;flex:1}.result small{color:#666;margin-top:4px}.result i{font-size:28px}.list{padding:0}.list>div{display:grid;grid-template-columns:90px 70px 100px 1fr 150px;align-items:center;gap:12px;padding:15px 20px;border-bottom:1px solid #ddd}.list .weather-icon{width:52px;height:52px}.radar{height:440px;position:relative;overflow:hidden;background:repeating-linear-gradient(0deg,transparent 0 39px,#ffffff30 40px),repeating-linear-gradient(90deg,transparent 0 39px,#ffffff30 40px),linear-gradient(145deg,#406942,#a5bf83 40%,#496b48);color:#fff}.radar:after{content:"";position:absolute;inset:25px;border:1px solid #ffffff66;border-radius:50%}.radar span{position:absolute;left:47%;top:51%;z-index:3;background:#222c;padding:4px}.storm{position:absolute;border-radius:50%;filter:blur(8px);z-index:2}.storm.one{width:200px;height:100px;background:#51d969;left:13%;top:16%;box-shadow:45px 15px 15px #ffdb3b}.storm.two{width:170px;height:80px;background:#33b759;right:10%;bottom:15%;box-shadow:-30px -10px 20px #ff9d22}.legend{text-align:right;padding:10px}.legend i{display:inline-block;width:18px;height:10px;background:#51d969;margin-left:20px}.legend i+*{}.aq{display:flex;align-items:center;gap:25px;border-left:7px solid #65a747}.aq-number{font-size:55px;color:#4d8d31}.meter{height:14px;background:#ddd}.meter i{display:block;height:100%;background:#65a747}.auth{max-width:480px;margin:45px auto}.auth form{display:flex;flex-direction:column;gap:18px}.auth label{display:flex;flex-direction:column;gap:7px}.auth input:not([type=radio]):not([type=checkbox]){padding:12px;border:1px solid #aaa}.auth fieldset{display:flex;flex-direction:column;gap:15px;border:1px solid #ddd}.empty{text-align:center}.prose{margin-top:40px}footer{margin-top:60px;background:#252525;color:#bbb;padding:35px max(24px,calc((100% - 1180px)/2))}footer div{display:flex;gap:25px;color:#fff}.sr{position:absolute;left:-9999px} +@media(max-width:700px){.corp{display:none}.breaking{font-size:11px}.premium{display:none}header{padding:0 12px;gap:10px}.logo{font-size:20px}.search input{width:80px}.layout{display:block;margin:16px}.layout aside{margin-top:18px}.current{grid-template-columns:1fr 60px}.current strong{font-size:48px}.current>div:last-child{grid-column:1/-1}.city-grid{grid-template-columns:1fr}.tabs{margin-left:-18px;margin-right:-18px}.weather-main{grid-template-columns:1fr}.reading{justify-content:flex-start}.list>div{grid-template-columns:55px 50px 70px 1fr}.list>div span:last-child{grid-column:1/-1}.radar{height:330px}.account-head{flex-wrap:wrap}.account-head h1{width:100%}footer div{flex-wrap:wrap}} +@media(max-width:360px){header{gap:6px}.logo{font-size:18px}.menu{display:none}.search button{padding:4px 9px}.current{padding:15px}.city{grid-template-columns:1fr 45px 45px}.narrow{padding:0 10px}.tabs{margin-left:-10px;margin-right:-10px}.tabs a{padding:13px 12px}.list>div{padding:12px 10px;gap:7px}} + +/* Captured AccuWeather proportions: a compact 960px weather canvas on a pale-blue page. */ +html,body{background:#e7edf5}.corp{height:32px;padding:7px 20px;justify-content:flex-start;gap:12px}.corp span+span{border-left:1px solid #777;padding-left:12px}.breaking{position:absolute;left:40%;right:0;top:0;height:32px;background:var(--action);text-align:left;padding:8px 20px;z-index:4}header{height:62px;padding:0 max(20px,calc((100% - 960px)/2));gap:17px}.logo{font-size:25px}.search{height:34px;max-width:325px;margin-left:auto;margin-right:0}.search input{padding:8px 12px;background:#fff}.search button{font-size:18px;padding:2px 12px}.premium{background:white;color:var(--action);padding:6px 10px;border-radius:18px;font-size:11px}.hero{height:auto;background:transparent;display:block;max-width:632px;margin:22px auto 12px}.hero form{width:100%;height:42px;box-shadow:0 1px 4px #0002}.hero input{padding:12px 15px;font-size:14px}.hero button{padding:8px 18px}.layout{max-width:960px;margin:20px auto;grid-template-columns:632px 304px;gap:24px}.layout>section>h2{font-size:19px}.layout aside{box-shadow:none}.card{box-shadow:none}.current{grid-template-columns:1fr 70px 78px 1fr;padding:18px;border-top:0}.current strong,.reading strong{font-size:52px}.city-grid{display:block}.city{min-height:56px;grid-template-columns:1fr 45px 55px 110px;border-bottom:1px solid #d5d8dc;padding:8px 15px}.city span{grid-column:auto;text-align:right}.narrow{max-width:960px;margin:28px auto;padding:0}.narrow>h1,.narrow>p{max-width:632px}.tabs{border-bottom:1px solid #bbc0c7;background:transparent;margin:0 0 24px}.tabs a{padding:13px 20px;color:#656b72;font-size:14px}.tabs a.active{border-bottom:2px solid var(--brand);color:#292929}.weather-main{width:632px;max-width:100%;border-top:0;padding:12px}.weather-main>div:first-child{grid-column:1/-1;border-bottom:1px solid #ddd;display:flex;justify-content:space-between}.weather-main>div:first-child .eyebrow{margin:0}.weather-main .reading{justify-content:flex-start}.weather-main dl{display:grid;grid-template-columns:1fr 1fr;gap:0 18px}.weather-main dl div{padding:11px 0}.actions{max-width:632px}.results{max-width:632px}.result{background:#fff}.list{max-width:632px}.aq,.radar{max-width:632px}.homepage-title{font-size:13px;letter-spacing:.08em;color:#666}.news-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:1px;background:#ddd}.news-grid article{background:#f5f6f7;padding:15px;min-height:120px}.news-grid h3{font-size:14px;margin:0 0 28px}.news-grid small{color:#666} +@media(max-width:960px){.layout{display:block;margin:20px 16px}.layout aside{margin-top:18px}} +@media(max-width:700px){.breaking{position:static;height:auto}.corp{display:none}header{height:54px}.menu{order:4}.logo{font-size:21px}.search{position:absolute;top:58px;left:6px;right:6px;max-width:none;height:36px}.premium{display:none}.hero{margin:48px 6px 12px}.layout{margin:12px 6px;display:block}.layout aside{margin-top:12px}.current{grid-template-columns:minmax(0,1fr) 42px 52px;gap:6px}.current>div:last-child{grid-column:1/-1}.city{grid-template-columns:minmax(0,1fr) 38px 42px 76px;gap:5px;font-size:12px}.narrow{padding:0 8px;margin-top:48px}.tabs{margin:0 -8px 18px}.tabs a{padding:12px 14px}.weather-main{width:100%}.weather-main dl{grid-template-columns:1fr}.news-grid{grid-template-columns:repeat(2,1fr)}} diff --git a/sites/accuweather/static/fonts/Solis-Regular.woff2 b/sites/accuweather/static/fonts/Solis-Regular.woff2 new file mode 100644 index 000000000..0133d0247 Binary files /dev/null and b/sites/accuweather/static/fonts/Solis-Regular.woff2 differ diff --git a/sites/accuweather/static/icons/.gitkeep b/sites/accuweather/static/icons/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sites/accuweather/static/js/.gitkeep b/sites/accuweather/static/js/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sites/accuweather/tasks.jsonl b/sites/accuweather/tasks.jsonl new file mode 100644 index 000000000..2cf705612 --- /dev/null +++ b/sites/accuweather/tasks.jsonl @@ -0,0 +1,20 @@ +{"web_name": "AccuWeather", "id": "AccuWeather--0", "ques": "Search for Phoenix, Arizona and report the current temperature, RealFeel temperature, and humidity from its current weather page.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_0.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open the Phoenix, Arizona current-weather page on the mirror (URL path /weather/phoenix-az); answering without that page visit is a FAIL. (2) The final answer must state three values exactly as rendered on that page: the current temperature, the RealFeel temperature and the humidity percentage. (3) A value that differs from the page, or a missing value, is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--1", "ques": "Search for Portland and use the results to open Portland, Maine. Report its current condition, wind speed, and visibility.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_1.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must run a site search whose results include Portland (the search page lists two Portlands) and open the Portland, Maine current-weather page (/weather/portland-me), not the Oregon one. (2) The final answer must give that page's current condition wording, wind speed and visibility exactly as rendered. (3) Values taken from Portland, Oregon are a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--2", "ques": "Find Seattle's hourly forecast. Among the displayed hours, identify the first hour with at least a 40% chance of precipitation and report the hour, temperature, and condition.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_2.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open the Seattle hourly forecast page (/hourly/seattle-wa). (2) The answer must name the FIRST listed hour whose precipitation chance is at least 40%, plus that row's temperature and condition wording, exactly as rendered. (3) Any other hour, or values from a different row, is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--3", "ques": "Open Miami's daily forecast and identify the displayed day with the greatest chance of precipitation. Report the day, high and low, and condition.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_3.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open the Miami daily forecast page (/daily/miami-fl). (2) The answer must name the day label with the highest precipitation percentage among the seven rows and give that row's high, low and condition wording exactly as rendered. (3) A different day or mixed-up values is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--4", "ques": "Compare the current RealFeel temperatures for Austin and Denver. Report both values and which city feels cooler.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_4.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open BOTH current-weather pages, Austin (/weather/austin-tx) and Denver (/weather/denver-co); RealFeel is only shown there. (2) The answer must give each city's RealFeel value as rendered and state which city feels cooler. (3) Swapped values, a missing value, or naming the wrong city as cooler is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--5", "ques": "Search for Springfield and open the Missouri result. Report its postal code from the search results, then its pressure reading from its current weather page and its air-quality value from its Air Quality page.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_5.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must search for Springfield (three same-name results) and open the Missouri result. (2) The answer must give the Missouri result's postal code as shown on the search results page, the pressure reading from /weather/springfield-mo and the air-quality number from /air-quality/springfield-mo; both pages must appear in the trajectory. (3) Values from Springfield, Illinois or Massachusetts are a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--6", "ques": "Sign in as alice.j@test.com with password TestPass123!, then save Seattle to My AccuWeather. Confirm Seattle appears in Saved Locations.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_6.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must sign in through /login as alice.j@test.com. (2) It must open the Seattle current-weather page and use the Save location control, then open My AccuWeather (/account) afterwards. (3) After the run the database must contain exactly one new saved_location row (Alice -> Seattle) and no other saved-location, user or alert change; a self-reported success with an unchanged database is a FAIL. (4) The final answer must confirm Seattle is listed. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--7", "ques": "Sign in as alice.j@test.com with password TestPass123! and remove Boston from Saved Locations. Confirm Boston is no longer listed there.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_7.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must sign in through /login as alice.j@test.com. (2) It must open the Boston current-weather page and use the Remove from saved locations control, then open My AccuWeather (/account) afterwards. (3) After the run Alice's Boston saved_location row must be gone and nothing else may change (her New York row stays; no user or alert change). (4) The final answer must confirm Boston is no longer listed. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--8", "ques": "Sign in as bob.smith@test.com with password TestPass123!, open Chicago weather alerts, and enable Severe weather and Rain starting soon only. Save the preferences.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_8.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must sign in through /login as bob.smith@test.com and open the Chicago alerts page (/alerts/chicago-il). (2) It must submit the alert form with exactly Severe weather and Rain starting soon checked. (3) After the run the alert table must contain exactly those two enabled rows for Bob/Chicago and nothing else; an unchanged table or an extra alert type (e.g. Temperature changes) is a FAIL. (4) No user or saved-location rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--9", "ques": "Sign in as carol.w@test.com with password TestPass123! and change the temperature unit to Celsius. Return to New York current weather and report the displayed temperature and RealFeel in Celsius.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_9.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must sign in through /login as carol.w@test.com, open Settings (/settings) and save Celsius, then open the New York current-weather page (/weather/new-york-ny) AFTER the settings change. (2) Carol's stored unit must be C after the run and no other user, saved-location or alert row may change. (3) The final answer must report the New York temperature and RealFeel as rendered in Celsius; Fahrenheit values are a FAIL. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--10", "ques": "Use postal-code search for 94102. Open the matching location and report its current condition, humidity, and pressure.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_10.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must submit a site search for the postal code 94102 (a /search?q=94102 visit) and open the single matching location's current-weather page. (2) The answer must give that page's current condition wording, humidity percentage and pressure reading exactly as rendered. (3) A qualified condition (e.g. 'Mostly cloudy' instead of the rendered word) or a wrong number is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--11", "ques": "Find New Orleans and open its radar page. Confirm the radar is for New Orleans, then report the current condition from the Current Weather tab.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_11.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must find New Orleans via site search and open its Radar page (/radar/new-orleans-la), whose heading names the city. (2) It must then open the Current Weather tab (/weather/new-orleans-la) and report the current condition wording exactly as rendered. (3) Missing either page visit is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--12", "ques": "Compare the air-quality values for Los Angeles and San Francisco. Report both values and identify the city with better air quality (the lower value).", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_12.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open BOTH Air Quality pages, Los Angeles (/air-quality/los-angeles-ca) and San Francisco (/air-quality/san-francisco-ca). (2) The answer must give each city's air-quality number as rendered and name the city with the LOWER number as having better air quality. (3) Swapped numbers or the wrong city is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--13", "ques": "For Boston's seven displayed daily forecasts, find the day with the lowest overnight low. Report the day, low, high, and condition.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_13.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open the Boston daily forecast page (/daily/boston-ma). (2) The answer must name the day label whose low is the smallest among the seven rows and give that row's low, high and condition wording exactly as rendered. (3) A different day or mixed-up values is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--14", "ques": "Search for London, England and report the current temperature, wind, humidity, and air-quality category.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_14.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must find London via site search and open its current-weather page (/weather/london-gb) and its Air Quality page (/air-quality/london-gb). (2) The answer must give the current temperature, wind speed and humidity from the weather page and the air-quality category word from the Air Quality page, exactly as rendered. (3) A missing value or the wrong category word is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--15", "ques": "Sign in as david.b@test.com with password TestPass123!, save Phoenix and Miami, and verify both cities appear in Saved Locations.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_15.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must sign in through /login as david.b@test.com. (2) It must save BOTH Phoenix and Miami from their current-weather pages and open My AccuWeather (/account) afterwards. (3) After the run David must have exactly two new saved_location rows (Phoenix and Miami) and nothing else may change; one missing city or an unchanged database is a FAIL. (4) The final answer must confirm both cities are listed. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--16", "ques": "Search for Portland and compare the current temperatures of the Oregon and Maine results. Report both and say which is warmer.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_16.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must search for Portland and open BOTH results' current-weather pages (/weather/portland-or and /weather/portland-me); temperatures are not shown on the results page. (2) The answer must give each Portland's current temperature as rendered and state which one is warmer. (3) Swapped values or the wrong city is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--17", "ques": "On Toronto's hourly forecast, find the highest displayed temperature. Report the first hour at that temperature and its precipitation chance.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_17.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must find Toronto via site search and open its hourly forecast page (/hourly/toronto-ca). (2) The answer must give the highest temperature listed, the FIRST hour label showing it, and that row's precipitation percentage, exactly as rendered. (3) A different hour or percentage is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--18", "ques": "Create an account named Jamie Lee using jamie.lee@example.test and password Weather123!, save Atlanta, and confirm it is shown in the new account's Saved Locations.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_18.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must create the account through /register with the exact name, email and password given in the task. (2) It must then open the Atlanta current-weather page, use Save location, and open My AccuWeather (/account). (3) After the run the database must contain exactly one new user (that email and name, whose password verifies) and exactly one new saved_location row for that user (Atlanta); seeded users and their saved rows must be untouched. (4) The final answer must confirm Atlanta is listed. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} +{"web_name": "AccuWeather", "id": "AccuWeather--19", "ques": "Compare Phoenix and New Orleans using their current weather pages. Report each city's temperature and humidity, then identify which has the higher heat index using RealFeel as the comparison.", "web": "http://localhost:40028/", "upstream_url": "https://www.accuweather.com/", "verifier_path": "sites/accuweather/verify/verify_19.py", "judge_rubric": "FACT CHECKPOINTS: (1) The agent must open BOTH current-weather pages, Phoenix (/weather/phoenix-az) and New Orleans (/weather/new-orleans-la). (2) The answer must give each city's current temperature and humidity as rendered and name the city with the higher RealFeel value. (3) A missing value or the wrong city is a FAIL. (4) No account, saved-location or alert rows may change. A checkpoint passes only when the required evidence is present in the trajectory, screenshots or database; an empty final answer is a FAIL."} diff --git a/sites/accuweather/templates/.gitkeep b/sites/accuweather/templates/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sites/accuweather/templates/account.html b/sites/accuweather/templates/account.html new file mode 100644 index 000000000..583caf2ed --- /dev/null +++ b/sites/accuweather/templates/account.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Saved Locations

{% for l in saved %}{{l.city}}, {{l.region}}{{l.condition}}{{display_temp(l.temp)}}°{{unit}}{% else %}
You have no saved locations.
{% endfor %}
{% endblock %} diff --git a/sites/accuweather/templates/air_quality.html b/sites/accuweather/templates/air_quality.html new file mode 100644 index 000000000..42ce9ac51 --- /dev/null +++ b/sites/accuweather/templates/air_quality.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import tabs %}{% block content %}

{{location.city}} Air Quality

{{tabs(location)}}
{{location.air_quality}}

{{'Good' if location.air_quality<51 else 'Moderate'}}

The air quality is generally acceptable for most individuals.

Current Pollutants

Fine Particulate Matter (PM2.5)

{% endblock %} diff --git a/sites/accuweather/templates/alerts.html b/sites/accuweather/templates/alerts.html new file mode 100644 index 000000000..a67892932 --- /dev/null +++ b/sites/accuweather/templates/alerts.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

{{location.city}} Alerts

Choose the notifications you want to receive.

{% endblock %} diff --git a/sites/accuweather/templates/base.html b/sites/accuweather/templates/base.html new file mode 100644 index 000000000..da4bfb01f --- /dev/null +++ b/sites/accuweather/templates/base.html @@ -0,0 +1,5 @@ +{% block title %}AccuWeather{% endblock %} +
AccuWeather for BusinessAccuWeather APIsPodcast
Breaking Weather: Track the latest tropical developments
+
☰
Premium+{% if current_user %}{{current_user.name}}{% else %}Sign in{% endif %}
+{% with msgs=get_flashed_messages() %}{% if msgs %}
{{msgs[-1]}}
{% endif %}{% endwith %}
{% block content %}{% endblock %}
+
AccuWeatherAboutPrivacy

© 2026 AccuWeather, Inc. All Rights Reserved.

diff --git a/sites/accuweather/templates/daily.html b/sites/accuweather/templates/daily.html new file mode 100644 index 000000000..10ce5f5d0 --- /dev/null +++ b/sites/accuweather/templates/daily.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import icon,tabs %}{% block content %}

{{location.city}} Daily Weather

{{tabs(location)}}
{% for d in location.forecasts|sort(attribute='day_index') %}
{{d.label}}{{icon(d.icon,d.condition)}}{{display_temp(d.high)}}° / {{display_temp(d.low)}}°{{unit}}{{d.condition}}{{d.precip}}% precipitation
{% endfor %}
{% endblock %} diff --git a/sites/accuweather/templates/error.html b/sites/accuweather/templates/error.html new file mode 100644 index 000000000..fa2d75343 --- /dev/null +++ b/sites/accuweather/templates/error.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}{{code}} | AccuWeather{% endblock %}{% block content %}

{{code}}

{{message}}

Back to the forecast

{% endblock %} diff --git a/sites/accuweather/templates/hourly.html b/sites/accuweather/templates/hourly.html new file mode 100644 index 000000000..d8d36ad0d --- /dev/null +++ b/sites/accuweather/templates/hourly.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import icon,tabs %}{% block content %}

{{location.city}} Hourly Weather

{{tabs(location)}}
{% for h in location.hourly|sort(attribute='hour_index') %}
{{h.label}}{{icon(h.icon,h.condition)}}{{display_temp(h.temp)}}°{{unit}}{{h.condition}}{{h.precip}}% precipitation
{% endfor %}
{% endblock %} diff --git a/sites/accuweather/templates/index.html b/sites/accuweather/templates/index.html new file mode 100644 index 000000000..dfb1fb18f --- /dev/null +++ b/sites/accuweather/templates/index.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import icon %}{% block content %}

UNITED STATES WEATHER

{{location.city}}, {{location.region}}

September 9, 2026 · 12:24 PM

{{icon(location.icon,location.condition)}}{{display_temp(location.temp)}}°
{{location.condition}}
RealFeel® {{display_temp(location.realfeel)}}°

United States Weather Conditions

{% for c in cities %}{{c.city}}, {{c.region}}{{icon(c.icon,c.condition)}}{{display_temp(c.temp)}}°{{c.condition}}{% endfor %}

Weather News

Watching the latest weather patterns across the country

2 hours ago

Prepare for dramatic temperature changes this week

4 hours ago

Severe weather tracker and forecast updates

6 hours ago

What to know before your weekend plans

8 hours ago
{% endblock %} diff --git a/sites/accuweather/templates/login.html b/sites/accuweather/templates/login.html new file mode 100644 index 000000000..c9eb94199 --- /dev/null +++ b/sites/accuweather/templates/login.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Sign in

New here? Create account

{% endblock %} diff --git a/sites/accuweather/templates/macros.html b/sites/accuweather/templates/macros.html new file mode 100644 index 000000000..e43c4f1aa --- /dev/null +++ b/sites/accuweather/templates/macros.html @@ -0,0 +1,2 @@ +{% macro icon(code,alt) %}{{alt}}{% endmacro %} +{% macro tabs(loc) %}{% endmacro %} diff --git a/sites/accuweather/templates/radar.html b/sites/accuweather/templates/radar.html new file mode 100644 index 000000000..2bcc9ba18 --- /dev/null +++ b/sites/accuweather/templates/radar.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import tabs %}{% block content %}

{{location.city}} Weather Radar

{{tabs(location)}}
Light rain Heavy rain
{% endblock %} diff --git a/sites/accuweather/templates/register.html b/sites/accuweather/templates/register.html new file mode 100644 index 000000000..ea36ea530 --- /dev/null +++ b/sites/accuweather/templates/register.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Create account

{% endblock %} diff --git a/sites/accuweather/templates/search.html b/sites/accuweather/templates/search.html new file mode 100644 index 000000000..604bd457a --- /dev/null +++ b/sites/accuweather/templates/search.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Search | AccuWeather{% endblock %}{% block content %}

Search results for “{{query}}”

{% if locations %}
{% for l in locations %}{{l.city}}, {{l.region}}{{l.country}} · {{l.postal}}›{% endfor %}
{% else %}

No locations found

Try a city, region, country, or postal code.

{% endif %}
{% endblock %} diff --git a/sites/accuweather/templates/settings.html b/sites/accuweather/templates/settings.html new file mode 100644 index 000000000..b38081d6a --- /dev/null +++ b/sites/accuweather/templates/settings.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Settings

Temperature unit
{% endblock %} diff --git a/sites/accuweather/templates/static_page.html b/sites/accuweather/templates/static_page.html new file mode 100644 index 000000000..74e084035 --- /dev/null +++ b/sites/accuweather/templates/static_page.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

{{title}}

{{copy}}

Weather with you wherever you go

Accurate, local information helps people plan their day with confidence.

{% endblock %} diff --git a/sites/accuweather/templates/weather.html b/sites/accuweather/templates/weather.html new file mode 100644 index 000000000..5a68f00a7 --- /dev/null +++ b/sites/accuweather/templates/weather.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import icon,tabs %}{% block title %}{{location.city}} Weather | AccuWeather{% endblock %}{% block content %}

{{location.city}}, {{location.region}}

{{location.country}} Weather

{{tabs(location)}}

CURRENT WEATHER

12:24 PM
{{icon(location.icon,location.condition)}}{{display_temp(location.temp)}}°{{unit}}

{{location.condition}}

RealFeel® {{display_temp(location.realfeel)}}°{{unit}}

Wind
{{location.wind}} mph
Humidity
{{location.humidity}}%
Visibility
{{location.visibility}} mi
Pressure
{{'%.2f'|format(location.pressure)}} in
UV Index
{{location.uv}}
Manage alerts
{% endblock %} diff --git a/sites/accuweather/verify/README.md b/sites/accuweather/verify/README.md new file mode 100644 index 000000000..430b6d793 --- /dev/null +++ b/sites/accuweather/verify/README.md @@ -0,0 +1,98 @@ +# AccuWeather deterministic grading contract + +Each row in `sites/accuweather/tasks.jsonl` points to `verify_0.py` … `verify_19.py` +(`verifier_path`) and carries an English `judge_rubric` of fact checkpoints. Ground +truth lives ONLY inside the verifiers; `tasks.jsonl` has no `answer` key. No verifier +calls an LLM: `verify_lib.py` keeps `llm_text_match` / `llm_screenshot_shows` for API +parity with `sites/merriam_webster/verify`, but no verdict depends on them and +`--no_llm True` is accepted for CLI parity. + +## Inputs + +```bash +cd agent_demo && uv run python ../sites/accuweather/verify/verify_0.py \ + --run_dir /abs/path/to/run [--initial_db initial.db --after_db after.db] [--no_llm True] +``` + +`agent_demo/eval_judge.py --run_dir --verifier True` invokes the same script. +Snapshots resolve in this order: explicit `--initial_db` / `--after_db`, then +`/initial.db` + `/after.db`, then `docker cp` from +`$WH_CONTAINER` (default `wh-review`) of `/opt/WebSyn/accuweather/{instance_seed,instance}/accuweather.db`. +Missing or invalid snapshots fail closed (`infra_error: true`). Output is JSON +`{task_id, pass, reason, evidence[]}`; exit 0 = PASS, 1 = FAIL. + +## Package validation (every task) + +- exact `task_id`; non-empty `final_answer`; `terminated: true` with + `termination_reason: agent_done`; at least one step; +- every recorded URL (`start_url`, step `url` / `url_before` / `url_after`, `final_url`) + is `http://` on a loopback host with the same host and port as `start_url`; +- both screenshots of every step exist and decode as non-empty PNGs (header + IHDR CRC). + +## Snapshot contract + +Both snapshots must have exactly the six tables `user, location, forecast, hourly, +saved_location, alert` with the expected columns. The initial snapshot must hold +20 locations / 140 forecasts / 240 hourly rows / 4 seeded users / 2 saved rows +(Alice: New York, Boston) / 0 alerts, and its `location + forecast + hourly` rows must +hash to `CATALOG_FINGERPRINT` (sha256 of the build-generated seed; password salts +are excluded so a rebuilt seed still validates). `location`, `forecast` and `hourly` +must be row-identical before and after. Read-only tasks additionally require +`user`, `saved_location` and `alert` to be row-identical. Stateful tasks require the +exact allowed delta and nothing else: + +| task | required navigation (in order where stated) | state delta | +|---|---|---| +| 6 | `/login` → `/weather/seattle-wa` → `/account`, typed `alice.j@test.com` | +1 saved row Alice→Seattle | +| 7 | `/login` → `/weather/boston-ma` → `/account` | −1 saved row Alice→Boston | +| 8 | `/login` → `/alerts/chicago-il`, typed `bob.smith@test.com` | alert rows == {Bob/Chicago/severe, Bob/Chicago/rain}, enabled | +| 9 | `/login` → `/settings` → `/weather/new-york-ny` | Carol `unit` F→C, all other columns/rows identical | +| 15 | `/login` → `/weather/phoenix-az` → `/account` and `/login` → `/weather/miami-fl` → `/account` | +2 saved rows David→Phoenix, David→Miami | +| 18 | `/register` (typed `jamie.lee@example.test`) → `/weather/atlanta-ga` → `/account` | +1 user (email, name "Jamie Lee", password verifies via hashlib scrypt/pbkdf2) and +1 saved row for that user → Atlanta | + +## Navigation gates (anti knowledge-shortcut) + +Every fact page whose values the answer reports must appear in the trajectory +(`/weather/`, `/hourly/`, `/daily/`, `/air-quality/`, +`/radar/`); comparison tasks (4, 12, 16, 19) need both detail pages. When the +target is not linked from the homepage grid (everything except New York, Phoenix, +Seattle, Miami, Chicago, Boston, Austin, Denver) the trajectory must contain a +`/search?q=` visit that would surface the target under the site's own token scoring +(`search_surfaces`); task 10 requires the literal postal-code query `94102`. + +## Answer matchers + +`contains_fact(text, value, unit, label)` accepts a standalone number, a number with +its unit (`104°`, `104 F`, `18%`, `8 mph`, `29.89 in`), or a number bound to its label +(`humidity: 18`); when the answer binds a *different* number to that label +(`temperature 115, RealFeel 104`) the check fails, so value/label swaps are caught +whenever the agent labels its values. Numbers are matched as whole values (`104` +never matches `1040` or `104.5`). Conditions are matched as phrases and single-word +conditions reject qualified variants (`Cloudy` ≠ `Mostly cloudy`). Clock times accept +`4 PM`, `4:00 p.m.`, `16:00`; day labels accept `Sat` / `Saturday`. Comparison tasks use +`names_winner`, which attributes the comparative word (`cooler`, `better`, `warmer`, +`higher`) to the nearest city in the sentence and rejects the inverse attribution. +All matchers are negation-aware (`not 18%` does not count). + +Known limit: when an agent reports two same-unit values with no labels at all +(`103°, 82°`) a swap between them is not detectable; the winner check still has to +name the right city. + +## Tests + +```bash +cd agent_demo && uv run python -m unittest discover -s ../sites/accuweather/verify/tests -p 'test_*.py' +``` + +`tests/_support.py` rebuilds the seed with plain sqlite3 from the same constants and +formulas as `app.py` (`test_verify_lib.py` asserts the fixture reproduces +`CATALOG_FINGERPRINT`) and writes trajectories in the `agent_demo/agent.py` layout. +Every task module covers: genuine PASS, run-dir snapshot discovery, no-op (empty +answer), wrong task id, shortcut (no navigation), wrong answers, alternative +phrasings, unterminated run, mixed origin, corrupt PNG, schema / catalog / seed drift +(fail closed), and read-only collateral writes or stateful mismatch. + +`tests/run_matrix.py` drives every task through a real Chromium against a standalone +copy of the site and grades pass / noop / shortcut / wrong / mismatch run dirs with +live SQLite snapshots (CONTRIBUTING §C). `tests/` is excluded from the Docker image by +`.dockerignore`. diff --git a/sites/accuweather/verify/tests/_cases.py b/sites/accuweather/verify/tests/_cases.py new file mode 100644 index 000000000..6876c9098 --- /dev/null +++ b/sites/accuweather/verify/tests/_cases.py @@ -0,0 +1,115 @@ +"""Per-task genuine trajectories, correct answers, wrong answers and expected gate +names for the accuweather verifier tests and for verify/tests/run_matrix.py.""" +from __future__ import annotations + +from _support import State, login_steps, step + +def _s(*paths): + out = [step("/")] + out += [step(p) for p in paths[:-1]] + out.append(step(paths[-1], "done")) + return out + +CASES = { + 0: dict(steps=_s("/search?q=Phoenix", "/weather/phoenix-az"), + answer="Phoenix: temperature 104°F, RealFeel 115°F, humidity 18%.", + gate="visited_weather_phoenix", + wrong=[("temperature 115°F, RealFeel 104°F, humidity 18%", "answer_temperature"), + ("Phoenix: 104°, RealFeel 115°, humidity 61%", "answer_humidity")], + also_pass=["104° / RealFeel 115° / 18% humidity", "Temp: 104 F; Feels like 115 F; Humidity: 18 percent"]), + 1: dict(steps=_s("/search?q=Portland", "/weather/portland-me"), + answer="Portland, Maine: Mostly cloudy, wind 8 mph, visibility 10 mi.", + gate="searched_for_portland-me", + wrong=[("Portland, Maine: Cloudy, wind 5 mph, visibility 10 mi.", "answer_condition"), + ("Mostly cloudy, wind 10 mph, visibility 8 mi", "answer_wind")], + also_pass=["mostly-cloudy; 8mph winds; 10 miles visibility"]), + 2: dict(steps=_s("/weather/seattle-wa", "/hourly/seattle-wa"), + answer="4 PM — 69°, Showers (47% precipitation).", + gate="visited_hourly_seattle", + wrong=[("3 PM — 68°, Mostly cloudy (36%)", "answer_hour"), ("4 PM, 68°, Showers", "answer_temperature")], + also_pass=["The first hour is 16:00 at 69 degrees with showers"]), + 3: dict(steps=_s("/weather/miami-fl", "/daily/miami-fl"), + answer="Sat: high 86°, low 73°, Mostly cloudy (63% precipitation).", + gate="visited_daily_miami", + wrong=[("Fri: high 88°, low 74°, Partly sunny", "answer_day"), ("Sat: high 73°, low 86°, Mostly cloudy", "answer_high")], + also_pass=["Saturday, 86°/73°, mostly cloudy"]), + 4: dict(steps=_s("/weather/austin-tx", "/", "/weather/denver-co"), + answer="Austin RealFeel 103°, Denver RealFeel 82°; Denver feels cooler.", + gate="visited_weather_austin", + wrong=[("Austin RealFeel 103°, Denver RealFeel 82°; Austin feels cooler.", "answer_names_cooler_city"), + ("Austin 82°, Denver 103°; Denver feels cooler.", "answer_austin_realfeel")], + also_pass=["Austin: 103°F. Denver: 82°F. The cooler city is Denver.", "Denver (82°) feels cooler than Austin (103°)."]), + 5: dict(steps=_s("/search?q=Springfield", "/weather/springfield-mo", "/air-quality/springfield-mo"), + answer="Postal code 65806; pressure 29.89 in; air quality 39.", + gate="searched_for_springfield-mo", + wrong=[("Postal code 62701; pressure 29.91 in; air quality 33.", "answer_postal_code"), + ("65806, pressure 29.98 in, air quality 39", "answer_pressure")], + also_pass=["ZIP 65806 / 29.89 inHg / AQI: 39"]), + 6: dict(steps=login_steps("alice.j@test.com") + [step("/search?q=Seattle"), step("/weather/seattle-wa"), step("/weather/seattle-wa"), step("/account", "done")], + answer="Seattle is now listed in Saved Locations.", gate="visited_login_page", stateful=True, + after=lambda s: s.add_saved("alice.j@test.com", "seattle-wa"), mismatch="saved_rows_added", + wrong=[("Done.", "answer_mentions_seattle")]), + 7: dict(steps=login_steps("alice.j@test.com") + [step("/weather/boston-ma"), step("/weather/boston-ma"), step("/account", "done")], + answer="Boston was removed and is no longer listed.", gate="visited_login_page", stateful=True, + after=lambda s: s.remove_saved("alice.j@test.com", "boston-ma"), mismatch="saved_rows_removed", + wrong=[("Removed.", "answer_mentions_boston")]), + 8: dict(steps=login_steps("bob.smith@test.com") + [step("/weather/chicago-il"), step("/alerts/chicago-il"), step("/alerts/chicago-il", "done")], + answer="Enabled Severe weather and Rain starting soon for Chicago and saved.", gate="visited_login_page", stateful=True, + after=lambda s: s.set_alerts("bob.smith@test.com", "chicago-il", ["severe", "rain"]), mismatch="alerts_exactly_severe_and_rain_for_chicago", + wrong=[]), + 9: dict(steps=login_steps("carol.w@test.com") + [step("/settings"), step("/settings"), step("/weather/new-york-ny", "done")], + answer="New York now shows 26° with RealFeel 28° in Celsius.", gate="visited_login_page", stateful=True, + after=lambda s: s.set_unit("carol.w@test.com", "C"), mismatch="carol_unit_is_celsius", + wrong=[("New York shows 79° with RealFeel 82°.", "answer_temperature_celsius"), ("temperature 28°C, RealFeel 26°C", "answer_temperature_celsius")]), + 10: dict(steps=_s("/search?q=94102", "/weather/san-francisco-ca"), + answer="San Francisco: Cloudy, humidity 75%, pressure 30.05 in.", + gate="searched_postal_code_94102", + wrong=[("San Francisco: Mostly cloudy, humidity 75%, pressure 30.05 in.", "answer_condition"), + ("Cloudy, humidity 57%, pressure 30.05 in", "answer_humidity")], + also_pass=["cloudy / 75 percent humidity / 30.05 inHg"]), + 11: dict(steps=_s("/search?q=New+Orleans", "/weather/new-orleans-la", "/radar/new-orleans-la", "/weather/new-orleans-la"), + answer="Radar page is for New Orleans; current condition: Showers.", + gate="searched_for_new-orleans-la", + wrong=[("Radar confirmed; current condition: Cloudy.", "answer_condition")]), + 12: dict(steps=_s("/search?q=Los+Angeles", "/weather/los-angeles-ca", "/air-quality/los-angeles-ca", "/search?q=San+Francisco", "/weather/san-francisco-ca", "/air-quality/san-francisco-ca"), + answer="Los Angeles 41, San Francisco 18 — San Francisco has better air quality.", + gate="searched_for_los-angeles-ca", + wrong=[("Los Angeles 41, San Francisco 18 — Los Angeles has better air quality.", "answer_names_better_city"), + ("Los Angeles 18, San Francisco 41 — San Francisco has better air quality.", "answer_los_angeles_value")], + also_pass=["LA: 41. SF: 18. The city with better air quality is San Francisco."]), + 13: dict(steps=_s("/weather/boston-ma", "/daily/boston-ma"), + answer="Sat: low 60°, high 73°, Mostly cloudy.", + gate="visited_daily_boston", + wrong=[("Fri: low 61°, high 75°, Partly sunny.", "answer_day"), ("Sat: low 61°, high 73°, Mostly cloudy.", "answer_low")], + also_pass=["Saturday — 73°/60°, mostly cloudy"]), + 14: dict(steps=_s("/search?q=London", "/weather/london-gb", "/air-quality/london-gb"), + answer="London: 63°, wind 10 mph, humidity 72%, air quality category Good.", + gate="searched_for_london-gb", + wrong=[("London: 63°, wind 10 mph, humidity 72%, air quality Moderate.", "answer_air_quality_category"), + ("London: 62°, wind 10 mph, humidity 72%, Good.", "answer_temperature")]), + 15: dict(steps=login_steps("david.b@test.com") + [step("/weather/phoenix-az"), step("/weather/phoenix-az"), step("/search?q=Miami"), step("/weather/miami-fl"), step("/weather/miami-fl"), step("/account", "done")], + answer="Phoenix and Miami both appear in Saved Locations.", gate="visited_login_page", stateful=True, + after=lambda s: (s.add_saved("david.b@test.com", "phoenix-az"), s.add_saved("david.b@test.com", "miami-fl")), mismatch="saved_rows_added", + wrong=[("Phoenix is saved.", "answer_mentions_both_cities")]), + 16: dict(steps=_s("/search?q=Portland", "/weather/portland-or", "/search?q=Portland", "/weather/portland-me"), + answer="Portland, Oregon 69°; Portland, Maine 70°. Maine is warmer.", + gate="searched_for_portland-or", + wrong=[("Portland, Oregon 69°; Portland, Maine 70°. Oregon is warmer.", "answer_names_warmer_city"), + ("Portland, Oregon 70°; Portland, Maine 69°. Maine is warmer.", "answer_oregon_temperature")], + also_pass=["Oregon: 69°F, Maine: 70°F — Portland, Maine is the warmer one."]), + 17: dict(steps=_s("/search?q=Toronto", "/weather/toronto-ca", "/hourly/toronto-ca"), + answer="Highest 73°, first at 4 PM, precipitation 64%.", + gate="searched_for_toronto-ca", + wrong=[("Highest 73°, first at 3 PM, precipitation 53%.", "answer_hour"), ("73° at 4 PM, precipitation 9%", "answer_precipitation")]), + 18: dict(steps=[step("/"), step("/login"), step("/register"), step("/register", "input", "Jamie Lee"), step("/register", "input", "jamie.lee@example.test"), + step("/register", "input", "Weather123!"), step("/register"), step("/account"), step("/search?q=Atlanta"), step("/weather/atlanta-ga"), + step("/weather/atlanta-ga"), step("/account", "done")], + answer="Atlanta is shown in Saved Locations for the new Jamie Lee account.", gate="visited_register", stateful=True, + after=lambda s: (s.add_user("jamie.lee@example.test", "Jamie Lee", "Weather123!"), s.add_saved("jamie.lee@example.test", "atlanta-ga")), mismatch="exactly_one_new_user", + wrong=[("Account created.", "answer_mentions_atlanta")]), + 19: dict(steps=_s("/weather/phoenix-az", "/search?q=New+Orleans", "/weather/new-orleans-la"), + answer="Phoenix: 104°, humidity 18%. New Orleans: 89°, humidity 75%. Phoenix has the higher heat index (RealFeel 115° vs 100°).", + gate="searched_for_new-orleans-la", + wrong=[("Phoenix: 104°, humidity 18%. New Orleans: 89°, humidity 75%. New Orleans has the higher heat index.", "answer_names_higher_heat_index_city"), + ("Phoenix: 104°, humidity 81%. New Orleans: 89°, humidity 75%. Phoenix is higher.", "answer_phoenix_humidity")]), +} diff --git a/sites/accuweather/verify/tests/_support.py b/sites/accuweather/verify/tests/_support.py new file mode 100644 index 000000000..8a9d19bad --- /dev/null +++ b/sites/accuweather/verify/tests/_support.py @@ -0,0 +1,265 @@ +"""Shared fixtures for the accuweather verifier tests. + +Synthetic SQLite snapshots with the exact schema of the build-generated seed +(``sites/accuweather/app.py`` seeds users / locations / forecasts / hourly rows +from constants; the same formulas are replicated here with plain sqlite3), plus +a hand-written trajectory writer in the agent_demo/agent.py format. No Flask, +no docker, no LLM. ``test_verify_lib.py`` proves the fixture reproduces the +frozen catalog fingerprint pinned in ``verify_lib.CATALOG_FINGERPRINT``. +""" +from __future__ import annotations + +import copy +import hashlib +import json +import shutil +import sqlite3 +import struct +import subprocess +import sys +import tempfile +import unittest +import zlib +from pathlib import Path +from typing import Any + +VERIFY_DIR = Path(__file__).resolve().parents[1] +SITE_DIR = VERIFY_DIR.parent +BASE = "http://localhost:41024" +PASSWORD = "TestPass123!" + +SCHEMA = """ +CREATE TABLE user (id INTEGER NOT NULL, email VARCHAR(120) NOT NULL, name VARCHAR(80) NOT NULL, + password_hash VARCHAR(255) NOT NULL, unit VARCHAR(1) NOT NULL, PRIMARY KEY (id), UNIQUE (email)); +CREATE TABLE location (id INTEGER NOT NULL, slug VARCHAR(100) NOT NULL, city VARCHAR(80) NOT NULL, + region VARCHAR(80) NOT NULL, country VARCHAR(80) NOT NULL, postal VARCHAR(20) NOT NULL, "temp" INTEGER NOT NULL, + realfeel INTEGER NOT NULL, condition VARCHAR(80) NOT NULL, icon VARCHAR(2) NOT NULL, humidity INTEGER NOT NULL, + wind INTEGER NOT NULL, visibility INTEGER NOT NULL, pressure FLOAT NOT NULL, uv INTEGER NOT NULL, + air_quality INTEGER NOT NULL, PRIMARY KEY (id), UNIQUE (slug)); +CREATE TABLE forecast (id INTEGER NOT NULL, location_id INTEGER NOT NULL, day_index INTEGER NOT NULL, + label VARCHAR(30) NOT NULL, high INTEGER NOT NULL, low INTEGER NOT NULL, condition VARCHAR(80) NOT NULL, + icon VARCHAR(2) NOT NULL, precip INTEGER NOT NULL, PRIMARY KEY (id), FOREIGN KEY(location_id) REFERENCES location (id)); +CREATE TABLE hourly (id INTEGER NOT NULL, location_id INTEGER NOT NULL, hour_index INTEGER NOT NULL, + label VARCHAR(20) NOT NULL, "temp" INTEGER NOT NULL, condition VARCHAR(80) NOT NULL, icon VARCHAR(2) NOT NULL, + precip INTEGER NOT NULL, PRIMARY KEY (id), FOREIGN KEY(location_id) REFERENCES location (id)); +CREATE TABLE saved_location (id INTEGER NOT NULL, user_id INTEGER NOT NULL, location_id INTEGER NOT NULL, + PRIMARY KEY (id), UNIQUE (user_id, location_id), FOREIGN KEY(user_id) REFERENCES user (id), + FOREIGN KEY(location_id) REFERENCES location (id)); +CREATE TABLE alert (id INTEGER NOT NULL, user_id INTEGER NOT NULL, location_id INTEGER NOT NULL, + alert_type VARCHAR(30) NOT NULL, enabled BOOLEAN NOT NULL, PRIMARY KEY (id), + UNIQUE (user_id, location_id, alert_type), FOREIGN KEY(user_id) REFERENCES user (id), + FOREIGN KEY(location_id) REFERENCES location (id)); +""" + +USERS = [("alice.j@test.com", "Alice Johnson"), ("bob.smith@test.com", "Bob Smith"), + ("carol.w@test.com", "Carol Williams"), ("david.b@test.com", "David Brown")] +# (slug, city, region, country, postal, temp, realfeel, condition, icon, humidity, wind, visibility, pressure, uv, air_quality) +LOCATIONS = [ + ("new-york-ny", "New York", "New York", "United States", "10007", 79, 82, "Partly sunny", "03", 61, 9, 10, 29.92, 5, 42), + ("phoenix-az", "Phoenix", "Arizona", "United States", "85001", 104, 115, "Sunny", "01", 18, 7, 12, 29.75, 10, 58), + ("seattle-wa", "Seattle", "Washington", "United States", "98101", 66, 65, "Cloudy", "07", 73, 6, 9, 30.08, 3, 24), + ("miami-fl", "Miami", "Florida", "United States", "33101", 88, 99, "Mostly cloudy", "06", 76, 12, 8, 29.88, 7, 36), + ("chicago-il", "Chicago", "Illinois", "United States", "60601", 72, 71, "Showers", "12", 70, 14, 7, 29.86, 2, 31), + ("boston-ma", "Boston", "Massachusetts", "United States", "02108", 75, 76, "Mostly sunny", "02", 55, 11, 10, 30.01, 6, 28), + ("austin-tx", "Austin", "Texas", "United States", "78701", 96, 103, "Sunny", "01", 38, 10, 11, 29.81, 9, 49), + ("denver-co", "Denver", "Colorado", "United States", "80202", 84, 82, "Partly sunny", "03", 27, 13, 15, 30.04, 8, 45), + ("portland-or", "Portland", "Oregon", "United States", "97205", 69, 68, "Cloudy", "07", 67, 5, 10, 30.11, 3, 22), + ("portland-me", "Portland", "Maine", "United States", "04101", 70, 69, "Mostly cloudy", "06", 64, 8, 10, 30.02, 4, 20), + ("springfield-il", "Springfield", "Illinois", "United States", "62701", 76, 77, "Partly sunny", "03", 59, 10, 10, 29.91, 5, 33), + ("springfield-ma", "Springfield", "Massachusetts", "United States", "01103", 73, 73, "Showers", "12", 69, 7, 8, 29.98, 3, 25), + ("springfield-mo", "Springfield", "Missouri", "United States", "65806", 81, 84, "Mostly sunny", "02", 53, 9, 11, 29.89, 6, 39), + ("san-francisco-ca", "San Francisco", "California", "United States", "94102", 65, 64, "Cloudy", "07", 75, 12, 10, 30.05, 3, 18), + ("los-angeles-ca", "Los Angeles", "California", "United States", "90012", 78, 79, "Mostly sunny", "02", 49, 6, 12, 29.96, 7, 41), + ("atlanta-ga", "Atlanta", "Georgia", "United States", "30303", 86, 91, "Partly sunny", "03", 58, 8, 9, 29.94, 6, 46), + ("nashville-tn", "Nashville", "Tennessee", "United States", "37219", 84, 88, "Mostly cloudy", "06", 61, 7, 10, 29.93, 5, 38), + ("new-orleans-la", "New Orleans", "Louisiana", "United States", "70112", 89, 100, "Showers", "12", 75, 9, 7, 29.87, 4, 44), + ("london-gb", "London", "England", "United Kingdom", "SW1A", 63, 62, "Cloudy", "07", 72, 10, 8, 30.10, 2, 21), + ("toronto-ca", "Toronto", "Ontario", "Canada", "M5H", 70, 69, "Partly sunny", "03", 62, 11, 10, 29.99, 4, 29), +] +HOUR_LABELS = ["Now", "1 PM", "2 PM", "3 PM", "4 PM", "5 PM", "6 PM", "7 PM", "8 PM", "9 PM", "10 PM", "11 PM"] +DAY_LABELS = ["Today", "Thu", "Fri", "Sat", "Sun", "Mon", "Tue"] +SLUG_ID = {row[0]: i for i, row in enumerate(LOCATIONS, 1)} +EMAIL_ID = {email: i for i, (email, _) in enumerate(USERS, 1)} + + +def scrypt_hash(password: str, salt: str = "fixturesaltAAAA") -> str: + digest = hashlib.scrypt(password.encode(), salt=salt.encode(), n=32768, r=8, p=1, + maxmem=132 * 1024 * 1024, dklen=64).hex() + return f"scrypt:32768:8:1${salt}${digest}" + + +_HASH_CACHE: dict[str, str] = {} + + +def cached_hash(password: str) -> str: + if password not in _HASH_CACHE: + _HASH_CACHE[password] = scrypt_hash(password) + return _HASH_CACHE[password] + + +def build_seed(path: Path) -> Path: + """Replicates app.py's seed_benchmark_users / seed_database / seed_preferences.""" + con = sqlite3.connect(path) + con.executescript(SCHEMA) + for i, (email, name) in enumerate(USERS, 1): + con.execute("INSERT INTO user VALUES (?,?,?,?,?)", (i, email, name, cached_hash(PASSWORD), "F")) + fid = hid = 1 + for lid, row in enumerate(LOCATIONS, 1): + con.execute("INSERT INTO location VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (lid, *row)) + temp, condition, icon = row[5], row[7], row[8] + icons = [icon, "02", "03", "06", "12", "07", "01"] + conditions = [condition, "Mostly sunny", "Partly sunny", "Mostly cloudy", "Showers", "Cloudy", "Sunny"] + for i, label in enumerate(DAY_LABELS): + con.execute("INSERT INTO forecast VALUES (?,?,?,?,?,?,?,?,?)", + (fid, lid, i, label, temp + (i % 3) - 2, temp - 12 - (i % 4), conditions[i], icons[i], (i * 17 + lid * 3) % 71)) + fid += 1 + for i in range(12): + con.execute("INSERT INTO hourly VALUES (?,?,?,?,?,?,?,?)", + (hid, lid, i, HOUR_LABELS[i], temp + (3 - abs(i - 4)), conditions[i % 7], icons[i % 7], (i * 11 + lid) % 66)) + hid += 1 + con.execute("INSERT INTO saved_location VALUES (1, 1, ?)", (SLUG_ID["new-york-ny"],)) + con.execute("INSERT INTO saved_location VALUES (2, 1, ?)", (SLUG_ID["boston-ma"],)) + con.commit() + con.close() + return path + + +class State: + """Mutable view of the user / saved_location / alert tables.""" + + def __init__(self) -> None: + self.users = [dict(id=i, email=e, name=n, password=PASSWORD, unit="F") for i, (e, n) in enumerate(USERS, 1)] + self.saved = [(1, 1, SLUG_ID["new-york-ny"]), (2, 1, SLUG_ID["boston-ma"])] + self.alerts: list[tuple[int, int, int, str, int]] = [] + self.extra_sql: list[str] = [] + + def uid(self, email: str) -> int: + for u in self.users: + if u["email"] == email: + return u["id"] + raise KeyError(email) + + def add_user(self, email: str, name: str, password: str) -> int: + new_id = max(u["id"] for u in self.users) + 1 + self.users.append(dict(id=new_id, email=email, name=name, password=password, unit="F")) + return new_id + + def set_unit(self, email: str, unit: str) -> None: + for u in self.users: + if u["email"] == email: + u["unit"] = unit + + def add_saved(self, email: str, slug: str) -> None: + new_id = max((r[0] for r in self.saved), default=0) + 1 + self.saved.append((new_id, self.uid(email), SLUG_ID[slug])) + + def remove_saved(self, email: str, slug: str) -> None: + before = len(self.saved) + self.saved = [r for r in self.saved if not (r[1] == self.uid(email) and r[2] == SLUG_ID[slug])] + assert len(self.saved) == before - 1, f"no saved row {email}/{slug}" + + def set_alerts(self, email: str, slug: str, types: list[str]) -> None: + uid, lid = self.uid(email), SLUG_ID[slug] + self.alerts = [a for a in self.alerts if not (a[1] == uid and a[2] == lid)] + for kind in sorted(types): + self.alerts.append((max((a[0] for a in self.alerts), default=0) + 1, uid, lid, kind, 1)) + + def write(self, path: Path) -> Path: + build_seed(path) + con = sqlite3.connect(path) + con.execute("DELETE FROM alert"); con.execute("DELETE FROM saved_location"); con.execute("DELETE FROM user") + for u in self.users: + con.execute("INSERT INTO user VALUES (?,?,?,?,?)", (u["id"], u["email"], u["name"], cached_hash(u["password"]), u["unit"])) + con.executemany("INSERT INTO saved_location VALUES (?,?,?)", self.saved) + con.executemany("INSERT INTO alert VALUES (?,?,?,?,?)", self.alerts) + for sql in self.extra_sql: + con.execute(sql) + con.commit(); con.close() + return path + + +def make_png(width: int = 320, height: int = 200) -> bytes: + raw = b"".join(b"\x00" + b"\x10\x20\x30" * width for _ in range(height)) + + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +PNG = make_png() + + +def step(path: str, action: str = "click", text: str | None = None) -> dict[str, Any]: + return {"path": path, "action": action, "text": text} + + +def login_steps(email: str, password: str = PASSWORD) -> list[dict[str, Any]]: + return [step("/"), step("/login"), step("/login", "input", email), step("/login", "input", password), + step("/login"), step("/account")] + + +def write_run(run_dir: Path, task_id: str, steps: list[dict[str, Any]], answer: str, *, + base: str = BASE, start_url: str | None = None, updates: dict[str, Any] | None = None, + corrupt_screenshot: bool = False, stub_screenshot: bool = False) -> Path: + shots = run_dir / "screenshots" + shots.mkdir(parents=True, exist_ok=True) + logged = [] + for i, s in enumerate(steps): + if s["action"] == "input": + params: dict[str, Any] = {"index": 1, "text": s["text"]} + elif s["action"] == "done": + params = {"text": answer, "success": True} + elif s["action"] == "navigate": + params = {"url": base + s["path"]} + else: + params = {"index": 1} + logged.append({"step": i, "url": base + s["path"], "title": "AccuWeather", "thought": "", "action": s["action"], + "params": params, "screenshot_before": f"step_{i:03d}.png", "screenshot_after": f"step_{i + 1:03d}.png"}) + for i in range(len(steps) + 1): + (shots / f"step_{i:03d}.png").write_bytes(PNG) + if corrupt_screenshot: + (shots / "step_001.png").write_bytes(b"definitely not a png") + if stub_screenshot: # decodable, but a 1x1 placeholder rather than a page + (shots / "step_001.png").write_bytes(make_png(1, 1)) + traj = {"task_id": task_id, "task": "", "start_url": start_url or (base + "/"), "max_steps": 15, "steps": logged, + "terminated": True, "termination_reason": "agent_done", "final_answer": answer, "success_self_report": True, + "verifier_path": f"sites/accuweather/verify/verify_{task_id.rsplit('--', 1)[1]}.py", "judge_rubric": ""} + traj.update(updates or {}) + (run_dir / "trajectory.json").write_text(json.dumps(traj, indent=2)) + return run_dir + + +class VerifierTestCase(unittest.TestCase): + N = -1 + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp(prefix=f"aw_verify_{self.N}_")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def verdict(self, steps, answer, *, initial: State | None = None, after: State | None = None, + task_id: str | None = None, snapshots_in_run_dir: bool = False, **kw) -> dict[str, Any]: + run_dir = self.tmp / f"run_{len(list(self.tmp.iterdir()))}" + run_dir.mkdir() + write_run(run_dir, task_id or f"AccuWeather--{self.N}", steps, answer, **kw) + initial_db = (initial or State()).write(run_dir / ("initial.db" if snapshots_in_run_dir else "seed_initial.db")) + after_db = (after or initial or State()).write(run_dir / ("after.db" if snapshots_in_run_dir else "seed_after.db")) + cmd = [sys.executable, str(VERIFY_DIR / f"verify_{self.N}.py"), "--run_dir", str(run_dir)] + if not snapshots_in_run_dir: + cmd += ["--initial_db", str(initial_db), "--after_db", str(after_db)] + r = subprocess.run(cmd, capture_output=True, text=True) + try: + out = json.loads(r.stdout) + except json.JSONDecodeError: + self.fail(f"verifier produced no JSON (rc={r.returncode}): {r.stderr[-800:]}") + out["_rc"] = r.returncode + return out + + def assertPasses(self, v: dict[str, Any]) -> None: + self.assertTrue(v["pass"], "\n".join(v["evidence"])) + self.assertEqual(v["_rc"], 0) + + def assertFailsOn(self, v: dict[str, Any], reason: str) -> None: + self.assertFalse(v["pass"], "expected FAIL but passed") + self.assertEqual(v["_rc"], 1) + self.assertEqual(v["reason"], reason, "\n".join(v["evidence"])) diff --git a/sites/accuweather/verify/tests/run_matrix.py b/sites/accuweather/verify/tests/run_matrix.py new file mode 100644 index 000000000..e81742038 --- /dev/null +++ b/sites/accuweather/verify/tests/run_matrix.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""CONTRIBUTING §C validation matrix, driven through a real Chromium. + +For every task: boot the mirror from a fresh copy of instance_seed, drive the +genuine workflow with Playwright (real clicks / form submits, screenshots in +the agent_demo/agent.py layout), snapshot the live SQLite DB as after.db, then +grade five run variants with the task's verifier: + + pass genuine trajectory + correct answer + real after-state -> PASS + noop homepage only, empty answer, clean DB -> FAIL + shortcut correct answer, no on-site navigation -> FAIL + wrong genuine trajectory, wrong answer -> FAIL + mismatch (stateful) genuine trajectory + correct answer, seed DB -> FAIL + +Usage (inside the agent_demo uv env, site venv python for Flask): + uv run python run_matrix.py --site_dir --python --port 45002 --out +""" +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _cases import CASES # noqa: E402 + +HERE = Path(__file__).resolve().parent +VERIFY_DIR = HERE.parent +PASSWORD = "TestPass123!" + +# Browser actions per task: (kind, arg[, text]). "goto" navigates, "click"/"check"/"fill" act on the current page. +LOGIN = lambda email: [("goto", "/login"), ("fill", "input[name=email]", email), ("fill", "input[name=password]", PASSWORD), ("click", "button:has-text('Sign in')")] +ACTIONS = { + 0: [("goto", "/"), ("fill", "#site-search", "Phoenix"), ("click", "header .search button"), ("click", ".result")], + 1: [("goto", "/"), ("fill", "#site-search", "Portland"), ("click", "header .search button"), ("click", ".result:has-text('Maine')")], + 2: [("goto", "/"), ("click", "a.city[href='/weather/seattle-wa']"), ("click", ".tabs a:has-text('Hourly')")], + 3: [("goto", "/"), ("click", "a.city[href='/weather/miami-fl']"), ("click", ".tabs a:has-text('Daily')")], + 4: [("goto", "/"), ("click", "a.city[href='/weather/austin-tx']"), ("goto", "/"), ("click", "a.city[href='/weather/denver-co']")], + 5: [("goto", "/"), ("fill", "#site-search", "Springfield"), ("click", "header .search button"), ("click", ".result:has-text('Missouri')"), ("click", ".tabs a:has-text('Air Quality')")], + 6: LOGIN("alice.j@test.com") + [("fill", "#site-search", "Seattle"), ("click", "header .search button"), ("click", ".result"), ("click", ".actions form button"), ("click", "header a[href='/account']")], + 7: LOGIN("alice.j@test.com") + [("click", ".results .result:has-text('Boston')"), ("click", ".actions form button"), ("click", "header a[href='/account']")], + 8: LOGIN("bob.smith@test.com") + [("fill", "#site-search", "Chicago"), ("click", "header .search button"), ("click", ".result"), ("click", ".actions a.button"), ("check", "input[value=severe]"), ("check", "input[value=rain]"), ("click", "button:has-text('Save alerts')")], + 9: LOGIN("carol.w@test.com") + [("click", "a[href='/settings']"), ("check", "input[value=C]"), ("click", "button:has-text('Save settings')"), ("goto", "/"), ("click", "a.city[href='/weather/new-york-ny']")], + 10: [("goto", "/"), ("fill", ".hero input[name=q]", "94102"), ("click", ".hero button"), ("click", ".result")], + 11: [("goto", "/"), ("fill", "#site-search", "New Orleans"), ("click", "header .search button"), ("click", ".result:has-text('Orleans')"), ("click", ".tabs a:has-text('Radar')"), ("click", ".tabs a:has-text('Current Weather')")], + 12: [("goto", "/"), ("fill", "#site-search", "Los Angeles"), ("click", "header .search button"), ("click", ".result:has-text('Los Angeles')"), ("click", ".tabs a:has-text('Air Quality')"), ("fill", "#site-search", "San Francisco"), ("click", "header .search button"), ("click", ".result:has-text('San Francisco')"), ("click", ".tabs a:has-text('Air Quality')")], + 13: [("goto", "/"), ("click", "a.city[href='/weather/boston-ma']"), ("click", ".tabs a:has-text('Daily')")], + 14: [("goto", "/"), ("fill", "#site-search", "London"), ("click", "header .search button"), ("click", ".result"), ("click", ".tabs a:has-text('Air Quality')")], + 15: LOGIN("david.b@test.com") + [("goto", "/"), ("click", "a.city[href='/weather/phoenix-az']"), ("click", ".actions form button"), ("goto", "/"), ("click", "a.city[href='/weather/miami-fl']"), ("click", ".actions form button"), ("click", "header a[href='/account']")], + 16: [("goto", "/"), ("fill", "#site-search", "Portland"), ("click", "header .search button"), ("click", ".result:has-text('Oregon')"), ("fill", "#site-search", "Portland"), ("click", "header .search button"), ("click", ".result:has-text('Maine')")], + 17: [("goto", "/"), ("fill", "#site-search", "Toronto"), ("click", "header .search button"), ("click", ".result"), ("click", ".tabs a:has-text('Hourly')")], + 18: [("goto", "/"), ("click", "header a[href='/login']"), ("click", "a[href='/register']"), ("fill", "input[name=name]", "Jamie Lee"), ("fill", "input[name=email]", "jamie.lee@example.test"), ("fill", "input[name=password]", "Weather123!"), ("click", "button:has-text('Create account')"), ("fill", "#site-search", "Atlanta"), ("click", "header .search button"), ("click", ".result"), ("click", ".actions form button"), ("click", "header a[href='/account']")], + 19: [("goto", "/"), ("click", "a.city[href='/weather/phoenix-az']"), ("fill", "#site-search", "New Orleans"), ("click", "header .search button"), ("click", ".result:has-text('Orleans')")], +} + + +def wait_http(url: str, timeout: float = 15.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + try: + urllib.request.urlopen(url, timeout=2) + return + except Exception: + time.sleep(0.3) + raise RuntimeError(f"site did not come up: {url}") + + +class Site: + def __init__(self, site_dir: Path, python: str, port: int): + self.site_dir, self.python, self.port, self.proc = site_dir, python, port, None + + def start(self) -> None: + inst, seed = self.site_dir / "instance", self.site_dir / "instance_seed" + shutil.rmtree(inst, ignore_errors=True) + shutil.copytree(seed, inst) + self.proc = subprocess.Popen([self.python, "-c", f"from app import app; app.run(host='127.0.0.1', port={self.port}, debug=False, use_reloader=False)"], + cwd=self.site_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + wait_http(f"http://127.0.0.1:{self.port}/") + + def stop(self) -> None: + if self.proc: + self.proc.kill(); self.proc.wait(); self.proc = None + + @property + def base(self) -> str: + return f"http://127.0.0.1:{self.port}" + + +def drive(page, base: str, actions, shots: Path, answer: str) -> list[dict]: + steps = [] + idx = 0 + page.screenshot(path=str(shots / f"step_{idx:03d}.png")) + for act in actions: + kind, arg = act[0], act[1] + url_before = page.url + if kind == "goto": + logged = {"action": "navigate", "params": {"url": base + arg}} + elif kind == "fill": + logged = {"action": "input", "params": {"index": 1, "text": act[2]}} + else: + logged = {"action": "click", "params": {"index": 1}} + steps.append({"step": idx, "url": url_before, "title": page.title(), "thought": "", **logged, + "screenshot_before": f"step_{idx:03d}.png", "screenshot_after": f"step_{idx + 1:03d}.png"}) + if kind == "goto": + page.goto(base + arg, wait_until="networkidle") + elif kind == "fill": + page.fill(arg, act[2]) + elif kind == "check": + page.check(arg) + else: + page.click(arg); page.wait_for_load_state("networkidle") + idx += 1 + page.screenshot(path=str(shots / f"step_{idx:03d}.png")) + steps.append({"step": idx, "url": page.url, "title": page.title(), "thought": "", "action": "done", + "params": {"text": answer, "success": True}, "screenshot_before": f"step_{idx:03d}.png", "screenshot_after": f"step_{idx + 1:03d}.png"}) + page.screenshot(path=str(shots / f"step_{idx + 1:03d}.png")) + return steps + + +def write_traj(run_dir: Path, task_id: str, base: str, steps: list[dict], answer: str) -> None: + n = int(task_id.rsplit("--", 1)[1]) + traj = {"task_id": task_id, "task": "", "start_url": base + "/", "max_steps": 40, "steps": steps, "terminated": True, + "termination_reason": "agent_done", "final_answer": answer, "success_self_report": True, + "verifier_path": f"sites/accuweather/verify/verify_{n}.py", "judge_rubric": ""} + (run_dir / "trajectory.json").write_text(json.dumps(traj, indent=2)) + + +def grade(n: int, run_dir: Path) -> dict: + r = subprocess.run([sys.executable, str(VERIFY_DIR / f"verify_{n}.py"), "--run_dir", str(run_dir), "--no_llm", "True"], capture_output=True, text=True) + try: + v = json.loads(r.stdout) + except json.JSONDecodeError: + v = {"pass": None, "reason": f"no JSON: {r.stderr[-300:]}"} + v["rc"] = r.returncode + return v + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--site_dir", required=True); ap.add_argument("--python", required=True) + ap.add_argument("--port", type=int, default=45002); ap.add_argument("--out", required=True) + ap.add_argument("--tasks", default="") + a = ap.parse_args() + from playwright.sync_api import sync_playwright + site = Site(Path(a.site_dir).resolve(), a.python, a.port) + out = Path(a.out).resolve(); out.mkdir(parents=True, exist_ok=True) + tasks = [int(x) for x in a.tasks.split(",")] if a.tasks else sorted(CASES) + matrix = {} + seed_db = site.site_dir / "instance_seed" / "accuweather.db" + with sync_playwright() as p: + browser = p.chromium.launch() + for n in tasks: + case = CASES[n]; task_id = f"AccuWeather--{n}" + site.start() + try: + ctx = browser.new_context(viewport={"width": 1280, "height": 900}); page = ctx.new_page() + page.goto(site.base + "/", wait_until="networkidle") + run = out / f"task{n}_pass"; shutil.rmtree(run, ignore_errors=True); (run / "screenshots").mkdir(parents=True) + shutil.copy2(seed_db, run / "initial.db") + steps = drive(page, site.base, ACTIONS[n], run / "screenshots", case["answer"]) + ctx.close() + time.sleep(0.3) + shutil.copy2(site.site_dir / "instance" / "accuweather.db", run / "after.db") + finally: + site.stop() + write_traj(run, task_id, site.base, steps, case["answer"]) + row = {"pass": grade(n, run)} + # noop: homepage only, empty answer, clean DB + noop = out / f"task{n}_noop"; shutil.rmtree(noop, ignore_errors=True); (noop / "screenshots").mkdir(parents=True) + for i in (0, 1): + shutil.copy2(run / "screenshots" / "step_000.png", noop / "screenshots" / f"step_{i:03d}.png") + shutil.copy2(seed_db, noop / "initial.db"); shutil.copy2(seed_db, noop / "after.db") + write_traj(noop, task_id, site.base, [{"step": 0, "url": site.base + "/", "title": "AccuWeather", "thought": "", "action": "done", "params": {"text": "", "success": False}, "screenshot_before": "step_000.png", "screenshot_after": "step_001.png"}], "") + row["noop"] = grade(n, noop) + # shortcut: correct answer, no on-site navigation + sc = out / f"task{n}_shortcut"; shutil.rmtree(sc, ignore_errors=True); shutil.copytree(noop, sc) + write_traj(sc, task_id, site.base, [{"step": 0, "url": site.base + "/", "title": "AccuWeather", "thought": "", "action": "done", "params": {"text": case["answer"], "success": True}, "screenshot_before": "step_000.png", "screenshot_after": "step_001.png"}], case["answer"]) + row["shortcut"] = grade(n, sc) + # wrong answer(s) + row["wrong"] = [] + for k, (wrong_answer, _reason) in enumerate(case.get("wrong", [])): + w = out / f"task{n}_wrong{k}"; shutil.rmtree(w, ignore_errors=True); shutil.copytree(run, w) + write_traj(w, task_id, site.base, steps[:-1] + [dict(steps[-1], params={"text": wrong_answer, "success": True})], wrong_answer) + row["wrong"].append(grade(n, w)) + # state mismatch (stateful only): claims success, DB unchanged + if case.get("stateful"): + mm = out / f"task{n}_mismatch"; shutil.rmtree(mm, ignore_errors=True); shutil.copytree(run, mm) + shutil.copy2(seed_db, mm / "after.db") + row["mismatch"] = grade(n, mm) + matrix[n] = row + summary = {k: (v["pass"] if isinstance(v, dict) else [x["pass"] for x in v]) for k, v in row.items()} + reasons = {k: v.get("reason") for k, v in row.items() if isinstance(v, dict)} + print(f"task {n:2d}: {summary} reasons={reasons}", flush=True) + browser.close() + (out / "matrix.json").write_text(json.dumps(matrix, indent=1)) + ok = all(r["pass"]["pass"] is True and r["noop"]["pass"] is False and r["shortcut"]["pass"] is False + and all(w["pass"] is False for w in r["wrong"]) and r.get("mismatch", {"pass": False})["pass"] is False for r in matrix.values()) + print("MATRIX", "OK" if ok else "HAS FAILURES") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/tests/test_verify_0.py b/sites/accuweather/verify/tests/test_verify_0.py new file mode 100644 index 000000000..7e814060d --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_0.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[0] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask0Tests(VerifierTestCase): + N = 0 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_stub_screenshot_fails(self) -> None: + """A 1x1 PNG decodes cleanly but is not evidence that a page was seen.""" + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), stub_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_1.py b/sites/accuweather/verify/tests/test_verify_1.py new file mode 100644 index 000000000..f6cf70feb --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_1.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[1] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask1Tests(VerifierTestCase): + N = 1 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_10.py b/sites/accuweather/verify/tests/test_verify_10.py new file mode 100644 index 000000000..ba4fa90a4 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_10.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[10] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask10Tests(VerifierTestCase): + N = 10 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_11.py b/sites/accuweather/verify/tests/test_verify_11.py new file mode 100644 index 000000000..7cfc05245 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_11.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[11] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask11Tests(VerifierTestCase): + N = 11 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_12.py b/sites/accuweather/verify/tests/test_verify_12.py new file mode 100644 index 000000000..8e6a44bda --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_12.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[12] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask12Tests(VerifierTestCase): + N = 12 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_13.py b/sites/accuweather/verify/tests/test_verify_13.py new file mode 100644 index 000000000..5481e9435 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_13.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[13] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask13Tests(VerifierTestCase): + N = 13 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_14.py b/sites/accuweather/verify/tests/test_verify_14.py new file mode 100644 index 000000000..964e3d66a --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_14.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[14] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask14Tests(VerifierTestCase): + N = 14 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_15.py b/sites/accuweather/verify/tests/test_verify_15.py new file mode 100644 index 000000000..491a367d6 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_15.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[15] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask15Tests(VerifierTestCase): + N = 15 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_state_mismatch_fails(self) -> None: + # agent self-reports success but the database is still the seed + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=State()), C["mismatch"]) + + def test_collateral_write_fails(self) -> None: + after = genuine_after() + after.add_saved("carol.w@test.com", "denver-co") + v = self.verdict(C["steps"], C["answer"], after=after) + self.assertFalse(v["pass"], "collateral saved_location write must fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_16.py b/sites/accuweather/verify/tests/test_verify_16.py new file mode 100644 index 000000000..e0205accf --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_16.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[16] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask16Tests(VerifierTestCase): + N = 16 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_17.py b/sites/accuweather/verify/tests/test_verify_17.py new file mode 100644 index 000000000..7c7fb0dc9 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_17.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[17] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask17Tests(VerifierTestCase): + N = 17 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_18.py b/sites/accuweather/verify/tests/test_verify_18.py new file mode 100644 index 000000000..2d9a88272 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_18.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[18] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask18Tests(VerifierTestCase): + N = 18 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_state_mismatch_fails(self) -> None: + # agent self-reports success but the database is still the seed + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=State()), C["mismatch"]) + + def test_collateral_write_fails(self) -> None: + after = genuine_after() + after.add_saved("carol.w@test.com", "denver-co") + v = self.verdict(C["steps"], C["answer"], after=after) + self.assertFalse(v["pass"], "collateral saved_location write must fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_19.py b/sites/accuweather/verify/tests/test_verify_19.py new file mode 100644 index 000000000..0f943d98a --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_19.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[19] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask19Tests(VerifierTestCase): + N = 19 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_2.py b/sites/accuweather/verify/tests/test_verify_2.py new file mode 100644 index 000000000..a6ec1ed37 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_2.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[2] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask2Tests(VerifierTestCase): + N = 2 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_3.py b/sites/accuweather/verify/tests/test_verify_3.py new file mode 100644 index 000000000..b4ba31449 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_3.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[3] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask3Tests(VerifierTestCase): + N = 3 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_4.py b/sites/accuweather/verify/tests/test_verify_4.py new file mode 100644 index 000000000..2f2d5a07e --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_4.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[4] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask4Tests(VerifierTestCase): + N = 4 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_5.py b/sites/accuweather/verify/tests/test_verify_5.py new file mode 100644 index 000000000..98c412a9f --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_5.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[5] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask5Tests(VerifierTestCase): + N = 5 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_catalog_wide_query_does_not_satisfy_the_search_gate(self) -> None: + """The point of this gate is that the agent disambiguated three + Springfields. ``q=United States`` matches every US location, so it must + not stand in for a real search; ``q=Springfield`` must.""" + universal = [step("/"), step("/search?q=United%20States"), step("/weather/springfield-mo"), + step("/air-quality/springfield-mo", "done")] + self.assertFailsOn(self.verdict(universal, C["answer"], after=genuine_after()), C["gate"]) + real = [step("/"), step("/search?q=Springfield"), step("/weather/springfield-mo"), + step("/air-quality/springfield-mo", "done")] + self.assertPasses(self.verdict(real, C["answer"], after=genuine_after())) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_read_only_write_fails(self) -> None: + after = State() + after.add_saved("bob.smith@test.com", "seattle-wa") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "read_only_saved_location_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_6.py b/sites/accuweather/verify/tests/test_verify_6.py new file mode 100644 index 000000000..2d000cd21 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_6.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[6] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask6Tests(VerifierTestCase): + N = 6 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_state_mismatch_fails(self) -> None: + # agent self-reports success but the database is still the seed + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=State()), C["mismatch"]) + + def test_collateral_write_fails(self) -> None: + after = genuine_after() + after.add_saved("carol.w@test.com", "denver-co") + v = self.verdict(C["steps"], C["answer"], after=after) + self.assertFalse(v["pass"], "collateral saved_location write must fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_7.py b/sites/accuweather/verify/tests/test_verify_7.py new file mode 100644 index 000000000..e88f44331 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_7.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[7] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask7Tests(VerifierTestCase): + N = 7 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_state_mismatch_fails(self) -> None: + # agent self-reports success but the database is still the seed + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=State()), C["mismatch"]) + + def test_collateral_write_fails(self) -> None: + after = genuine_after() + after.add_saved("carol.w@test.com", "denver-co") + v = self.verdict(C["steps"], C["answer"], after=after) + self.assertFalse(v["pass"], "collateral saved_location write must fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_8.py b/sites/accuweather/verify/tests/test_verify_8.py new file mode 100644 index 000000000..7e05770f0 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_8.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[8] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask8Tests(VerifierTestCase): + N = 8 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_state_mismatch_fails(self) -> None: + # agent self-reports success but the database is still the seed + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=State()), C["mismatch"]) + + def test_collateral_write_fails(self) -> None: + after = genuine_after() + after.add_saved("carol.w@test.com", "denver-co") + v = self.verdict(C["steps"], C["answer"], after=after) + self.assertFalse(v["pass"], "collateral saved_location write must fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_9.py b/sites/accuweather/verify/tests/test_verify_9.py new file mode 100644 index 000000000..0d4727f01 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_9.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _cases import CASES # noqa: E402 +from _support import State, VerifierTestCase, step # noqa: E402 + +C = CASES[9] + + +def genuine_after() -> State: + after = State() + if C.get("after"): + C["after"](after) + return after + + +class VerifyTask9Tests(VerifierTestCase): + N = 9 + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(C["steps"], C["answer"], after=genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails_on_empty_answer(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), task_id="AccuWeather--99"), "trajectory_task_matches") + + def test_shortcut_fails_on_navigation_gate(self) -> None: + self.assertFailsOn(self.verdict([step("/"), step("/", "done")], C["answer"], after=genuine_after()), C["gate"]) + + def test_wrong_answers_fail(self) -> None: + for answer, reason in C["wrong"]: + with self.subTest(answer=answer): + self.assertFailsOn(self.verdict(C["steps"], answer, after=genuine_after()), reason) + + def test_alternative_phrasings_pass(self) -> None: + for answer in C.get("also_pass", []): + with self.subTest(answer=answer): + self.assertPasses(self.verdict(C["steps"], answer, after=genuine_after())) + + def test_unterminated_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), updates={"terminated": False}), "trajectory_completed") + + def test_mixed_origin_run_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), start_url="http://127.0.0.1:41024/"), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=genuine_after(), corrupt_screenshot=True), "screenshots_decode") + + def test_schema_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("CREATE TABLE injected(id INTEGER PRIMARY KEY)") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_catalog_change_fails_closed(self) -> None: + after = genuine_after() + after.extra_sql.append("UPDATE location SET temp = temp + 1 WHERE slug = 'phoenix-az'") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=after), "snapshot_contract_invalid") + + def test_wrong_seed_fails_closed(self) -> None: + initial = State() + initial.extra_sql.append("DELETE FROM saved_location WHERE id = 2") + self.assertFailsOn(self.verdict(C["steps"], C["answer"], initial=initial, after=genuine_after()), "snapshot_contract_invalid") + + def test_state_mismatch_fails(self) -> None: + # agent self-reports success but the database is still the seed + self.assertFailsOn(self.verdict(C["steps"], C["answer"], after=State()), C["mismatch"]) + + def test_collateral_write_fails(self) -> None: + after = genuine_after() + after.add_saved("carol.w@test.com", "denver-co") + v = self.verdict(C["steps"], C["answer"], after=after) + self.assertFalse(v["pass"], "collateral saved_location write must fail") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/tests/test_verify_lib.py b/sites/accuweather/verify/tests/test_verify_lib.py new file mode 100644 index 000000000..1dbfc05d5 --- /dev/null +++ b/sites/accuweather/verify/tests/test_verify_lib.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import sqlite3 +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import verify_lib as L # noqa: E402 +from _support import SITE_DIR, State, build_seed, scrypt_hash # noqa: E402 + + +class FixtureMatchesFrozenSeed(unittest.TestCase): + def test_fixture_catalog_fingerprint_equals_pinned_constant(self) -> None: + with tempfile.TemporaryDirectory() as d: + db = build_seed(Path(d) / "seed.db") + self.assertEqual(L.catalog_fingerprint(db), L.CATALOG_FINGERPRINT) + L._validate_snapshot_contract(str(db), str(db)) + + def test_real_seed_if_present_matches_fixture(self) -> None: + real = SITE_DIR / "instance_seed" / "accuweather.db" + if not real.is_file(): + self.skipTest("instance_seed/accuweather.db not generated in this checkout") + self.assertEqual(L.catalog_fingerprint(real), L.CATALOG_FINGERPRINT) + + +class FactMatchers(unittest.TestCase): + def test_contains_fact_unit_and_label(self) -> None: + self.assertTrue(L.contains_fact("temperature 104°F", 104, unit="temp", label=L.TEMP_LABEL)) + self.assertTrue(L.contains_fact("104° and RealFeel 115°", 104, unit="temp", label=L.TEMP_LABEL)) + self.assertTrue(L.contains_fact("RealFeel® temperature of 115°", 115, unit="temp", label=L.REALFEEL_LABEL)) + self.assertTrue(L.contains_fact("RealFeel® temperature of 115°", 104, unit="temp", label=L.TEMP_LABEL) is False) + self.assertFalse(L.contains_fact("temperature 115, RealFeel 104", 104, unit="temp", label=L.TEMP_LABEL)) + self.assertTrue(L.contains_fact("18% humidity, RealFeel 115", 18, unit="percent", label=L.HUMIDITY_LABEL, allow_bare=False)) + self.assertFalse(L.contains_fact("humidity 181%", 18, unit="percent", label=L.HUMIDITY_LABEL, allow_bare=False)) + self.assertFalse(L.contains_fact("humidity is not 18%", 18, unit="percent", label=L.HUMIDITY_LABEL)) + self.assertTrue(L.contains_fact("pressure 29.89 in", "29.89", unit="inhg", label=L.PRESSURE_LABEL)) + self.assertFalse(L.contains_fact("pressure 29.8 in", "29.89", unit="inhg", label=L.PRESSURE_LABEL)) + self.assertTrue(L.contains_fact("Austin's RealFeel is 103°", 103, unit="temp", label=L.city_label("austin"))) + self.assertFalse(L.contains_fact("Austin 82°, Denver 103°", 103, unit="temp", label=L.city_label("austin"))) + + def test_conditions_days_and_times(self) -> None: + self.assertTrue(L.contains_condition("it is mostly cloudy", "Mostly cloudy")) + self.assertFalse(L.contains_condition("Mostly cloudy", "Cloudy")) + self.assertTrue(L.contains_condition("Cloudy skies", "Cloudy")) + self.assertFalse(L.contains_condition("partly sunny", "Sunny")) + self.assertTrue(L.contains_day_label("Saturday", "Sat") and L.contains_day_label("Sat.", "Sat")) + self.assertFalse(L.contains_day_label("Sunday", "Sat")) + self.assertTrue(L.contains_clock_time("at 16:00", "4 PM") and L.contains_clock_time("4:00 p.m.", "4 PM")) + self.assertFalse(L.contains_clock_time("4 AM", "4 PM")) + + def test_names_winner(self) -> None: + w, l = ["denver"], ["austin"] + kw, inv = ["cooler", "lower"], ["warmer", "hotter"] + for good in ["Denver feels cooler than Austin", "Austin 103, Denver 82; Denver feels cooler", + "The cooler city is Denver", "Denver (82°) is cooler", "Austin is warmer than Denver", + "Austin 103 and Denver 82, so Denver feels cooler", "cooler: Denver"]: + self.assertTrue(L.names_winner(good, w, l, kw, inv), good) + for bad in ["Austin feels cooler than Denver", "The cooler city is Austin", "Denver is warmer", + "Austin 103, Denver 82", "Denver feels cooler. Austin feels cooler too."]: + self.assertFalse(L.names_winner(bad, w, l, kw, inv), bad) + + def test_search_surfaces_and_origin(self) -> None: + traj = {"start_url": "http://localhost:41024/", "steps": [{"url": "http://localhost:41024/search?q=Springfield+Missouri"}]} + self.assertTrue(L.search_surfaces(traj, "springfield-mo")) + self.assertTrue(L.search_surfaces(traj, "springfield-il")) + self.assertFalse(L.search_surfaces(traj, "phoenix-az")) + + def test_search_gate_ignores_catalog_wide_tokens(self) -> None: + """``United States`` matches 20/20 locations, so it must not satisfy the + anti-shortcut gate for any of them; a city/region/postal token must.""" + def q(term): + return {"start_url": "http://localhost:41024/", + "steps": [{"url": "http://localhost:41024/search?q=" + term}]} + for slug in ("springfield-mo", "portland-me", "toronto-ca", "london-gb"): + self.assertFalse(L.search_surfaces(q("United+States"), slug), slug) + self.assertFalse(L.search_surfaces(q("states"), slug), slug) + self.assertTrue(L.search_surfaces(q("Springfield"), "springfield-mo")) + self.assertTrue(L.search_surfaces(q("65806"), "springfield-mo")) + self.assertTrue(L.search_surfaces(q("Kingdom"), "london-gb")) + self.assertNotIn("united", L._discriminative_tokens("london-gb")) + self.assertIn("london", L._discriminative_tokens("london-gb")) + + def test_screenshots_reject_stub_sizes(self) -> None: + """A decodable but 1x1 PNG is a placeholder, not page evidence.""" + from _support import make_png + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "screenshots").mkdir() + traj = {"_run_dir": root, "steps": [{"screenshot_before": "step_000.png", "screenshot_after": "step_001.png"}]} + for w, h, expected in ((320, 200, True), (L.MIN_SHOT_WIDTH, L.MIN_SHOT_HEIGHT, True), + (1, 1, False), (L.MIN_SHOT_WIDTH - 1, L.MIN_SHOT_HEIGHT, False), + (L.MIN_SHOT_WIDTH, L.MIN_SHOT_HEIGHT - 1, False)): + for name in ("step_000.png", "step_001.png"): + (root / "screenshots" / name).write_bytes(make_png(w, h)) + ok, evidence = L.screenshots_decode(traj) + self.assertEqual(ok, expected, f"{w}x{h}: {evidence}") + self.assertTrue(L.navigated_to_path({"steps": [{"url": "http://127.0.0.1:5000/weather/phoenix-az?x=1"}]}, "/weather/phoenix-az")) + self.assertFalse(L.navigated_to_path({"steps": [{"url": "http://example.com/weather/phoenix-az"}]}, "/weather/phoenix-az")) + self.assertFalse(L._same_local_origin("http://localhost:41025/", "http://localhost:41024/")) + + def test_password_matches(self) -> None: + self.assertTrue(L.password_matches(scrypt_hash("Weather123!"), "Weather123!")) + self.assertFalse(L.password_matches(scrypt_hash("Weather123!"), "weather123!")) + self.assertFalse(L.password_matches("garbage", "x")) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/accuweather/verify/verify_0.py b/sites/accuweather/verify/verify_0.py new file mode 100644 index 000000000..5cf2c684f --- /dev/null +++ b/sites/accuweather/verify/verify_0.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--0: Phoenix current temperature, RealFeel and humidity (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--0" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_visited_path(judge, t, "visited_weather_phoenix", "/weather/phoenix-az") + judge.check("answer_temperature", contains_fact(a, 104, unit="temp", label=TEMP_LABEL), f"expected 104, answer={a!r}") + judge.check("answer_realfeel", contains_fact(a, 115, unit="temp", label=REALFEEL_LABEL), f"expected 115, answer={a!r}") + judge.check("answer_humidity", contains_fact(a, 18, unit="percent", label=HUMIDITY_LABEL, allow_bare=False), f"expected 18%, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_1.py b/sites/accuweather/verify/verify_1.py new file mode 100644 index 000000000..ca9e17936 --- /dev/null +++ b/sites/accuweather/verify/verify_1.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--1: Portland search -> Portland, Maine condition, wind and visibility (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--1" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "portland-me") + check_visited_path(judge, t, "visited_weather_portland_me", "/weather/portland-me") + judge.check("answer_condition", contains_condition(a, "Mostly cloudy"), f"expected 'Mostly cloudy', answer={a!r}") + judge.check("answer_wind", contains_fact(a, 8, unit="mph", label=WIND_LABEL), f"expected 8 mph, answer={a!r}") + judge.check("answer_visibility", contains_fact(a, 10, unit="mi", label=VISIBILITY_LABEL), f"expected 10 mi, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_10.py b/sites/accuweather/verify/verify_10.py new file mode 100644 index 000000000..90b0c1610 --- /dev/null +++ b/sites/accuweather/verify/verify_10.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--10: Postal-code search 94102 -> San Francisco condition, humidity and pressure (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--10" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + judge.check("searched_postal_code_94102", any("94102" in normalize_text(q) for q in search_queries(t)), f"observed_queries={search_queries(t)!r}") + check_visited_path(judge, t, "visited_weather_san_francisco", "/weather/san-francisco-ca") + judge.check("answer_condition", contains_condition(a, "Cloudy"), f"expected 'Cloudy', answer={a!r}") + judge.check("answer_humidity", contains_fact(a, 75, unit="percent", label=HUMIDITY_LABEL, allow_bare=False), f"expected 75%, answer={a!r}") + judge.check("answer_pressure", contains_fact(a, "30.05", unit="inhg", label=PRESSURE_LABEL), f"expected 30.05 in, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_11.py b/sites/accuweather/verify/verify_11.py new file mode 100644 index 000000000..811df5d56 --- /dev/null +++ b/sites/accuweather/verify/verify_11.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--11: New Orleans radar page, then current condition (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--11" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "new-orleans-la") + check_visited_path(judge, t, "visited_radar_new_orleans", "/radar/new-orleans-la") + check_visited_path(judge, t, "visited_weather_new_orleans", "/weather/new-orleans-la") + judge.check("answer_condition", contains_condition(a, "Showers"), f"expected 'Showers', answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_12.py b/sites/accuweather/verify/verify_12.py new file mode 100644 index 000000000..aba104300 --- /dev/null +++ b/sites/accuweather/verify/verify_12.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--12: Los Angeles vs San Francisco air-quality comparison (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--12" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "los-angeles-ca") + check_search_surfaces(judge, t, "san-francisco-ca") + check_visited_path(judge, t, "visited_air_quality_los_angeles", "/air-quality/los-angeles-ca") + check_visited_path(judge, t, "visited_air_quality_san_francisco", "/air-quality/san-francisco-ca") + judge.check("answer_los_angeles_value", contains_fact(a, 41, label=city_label("los angeles", "la", "l.a.")), f"expected LA 41, answer={a!r}") + judge.check("answer_san_francisco_value", contains_fact(a, 18, label=city_label("san francisco", "sf")), f"expected SF 18, answer={a!r}") + judge.check("answer_names_better_city", names_winner(a, ["san francisco", "sf"], ["los angeles", "la", "l.a."], ["better", "lower", "cleaner", "best", "lowest", "healthier"], ["worse", "higher", "worst", "highest", "poorer"]), f"expected San Francisco better, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_13.py b/sites/accuweather/verify/verify_13.py new file mode 100644 index 000000000..9e013fec9 --- /dev/null +++ b/sites/accuweather/verify/verify_13.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--13: Boston daily: day with the lowest overnight low (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--13" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_visited_path(judge, t, "visited_daily_boston", "/daily/boston-ma") + judge.check("answer_day", contains_day_label(a, "Sat"), f"expected Sat, answer={a!r}") + judge.check("answer_low", contains_fact(a, 60, unit="temp", label=LOW_LABEL), f"expected low 60, answer={a!r}") + judge.check("answer_high", contains_fact(a, 73, unit="temp", label=HIGH_LABEL), f"expected high 73, answer={a!r}") + judge.check("answer_condition", contains_condition(a, "Mostly cloudy"), f"expected 'Mostly cloudy', answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_14.py b/sites/accuweather/verify/verify_14.py new file mode 100644 index 000000000..3408ad8c3 --- /dev/null +++ b/sites/accuweather/verify/verify_14.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--14: London current temperature, wind, humidity and air-quality category (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--14" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "london-gb") + check_visited_path(judge, t, "visited_weather_london", "/weather/london-gb") + check_visited_path(judge, t, "visited_air_quality_london", "/air-quality/london-gb") + judge.check("answer_temperature", contains_fact(a, 63, unit="temp", label=TEMP_LABEL), f"expected 63, answer={a!r}") + judge.check("answer_wind", contains_fact(a, 10, unit="mph", label=WIND_LABEL), f"expected 10 mph, answer={a!r}") + judge.check("answer_humidity", contains_fact(a, 72, unit="percent", label=HUMIDITY_LABEL, allow_bare=False), f"expected 72%, answer={a!r}") + judge.check("answer_air_quality_category", contains_all(a, ["good"]) and not contains_all(a, ["moderate"]), f"expected category 'Good', answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_15.py b/sites/accuweather/verify/verify_15.py new file mode 100644 index 000000000..5ee1ff908 --- /dev/null +++ b/sites/accuweather/verify/verify_15.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--15: Sign in as david.b@test.com and save Phoenix and Miami (stateful). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--15" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_signed_in_as(judge, t, "david.b@test.com") + check_paths_in_order(judge, t, "login_then_phoenix_then_account", ["/login", "/weather/phoenix-az", "/account"]) + check_paths_in_order(judge, t, "login_then_miami_then_account", ["/login", "/weather/miami-fl", "/account"]) + check_saved_delta(judge, initial_db, after_db, "david.b@test.com", added={"phoenix-az", "miami-fl"}, removed=set()) + judge.check("answer_mentions_both_cities", mentions(a, ["phoenix"]) and mentions(a, ["miami"]), f"answer={a!r}") + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_16.py b/sites/accuweather/verify/verify_16.py new file mode 100644 index 000000000..e958e43c2 --- /dev/null +++ b/sites/accuweather/verify/verify_16.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--16: Portland, Oregon vs Portland, Maine current temperature comparison (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--16" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "portland-or") + check_visited_path(judge, t, "visited_weather_portland_or", "/weather/portland-or") + check_visited_path(judge, t, "visited_weather_portland_me", "/weather/portland-me") + judge.check("answer_oregon_temperature", contains_fact(a, 69, unit="temp", label=city_label("oregon", "portland, or", "portland or", "portland, oregon")), f"expected Oregon 69, answer={a!r}") + judge.check("answer_maine_temperature", contains_fact(a, 70, unit="temp", label=city_label("maine", "portland, me", "portland me", "portland, maine")), f"expected Maine 70, answer={a!r}") + judge.check("answer_names_warmer_city", names_winner(a, ["maine", "portland, me", "portland, maine"], ["oregon", "portland, or", "portland, oregon"], ["warmer", "hotter", "higher", "warmest", "hottest"], ["cooler", "colder", "lower", "coolest"]), f"expected Maine warmer, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_17.py b/sites/accuweather/verify/verify_17.py new file mode 100644 index 000000000..836f5da75 --- /dev/null +++ b/sites/accuweather/verify/verify_17.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--17: Toronto hourly: first hour at the highest temperature and its precipitation chance (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--17" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "toronto-ca") + check_visited_path(judge, t, "visited_hourly_toronto", "/hourly/toronto-ca") + judge.check("answer_hour", contains_clock_time(a, "4 PM"), f"expected 4 PM, answer={a!r}") + judge.check("answer_temperature", contains_fact(a, 73, unit="temp", label=TEMP_LABEL), f"expected 73, answer={a!r}") + judge.check("answer_precipitation", contains_fact(a, 64, unit="percent", label=PRECIP_LABEL, allow_bare=False), f"expected 64%, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_18.py b/sites/accuweather/verify/verify_18.py new file mode 100644 index 000000000..16dedb600 --- /dev/null +++ b/sites/accuweather/verify/verify_18.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--18: Register Jamie Lee (jamie.lee@example.test / Weather123!) and save Atlanta (stateful). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--18" + +NEW_EMAIL = "jamie.lee@example.test" + + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + judge.check("visited_register_page", check_visited_path(judge, t, "visited_register", "/register"), "required_path=/register") + judge.check("entered_new_account_email", normalize_text(NEW_EMAIL) in typed_emails(t), f"typed_emails={typed_emails(t)!r}") + check_paths_in_order(judge, t, "register_then_atlanta_then_account", ["/register", "/weather/atlanta-ga", "/account"]) + fresh = new_users(initial_db, after_db) + judge.check("exactly_one_new_user", len(fresh) == 1, f"new_users={[u['email'] for u in fresh]}") + u = fresh[0] if len(fresh) == 1 else None + judge.check("new_user_email", bool(u) and normalize_text(u["email"]) == NEW_EMAIL, f"observed={u and u['email']!r}") + judge.check("new_user_name", bool(u) and normalize_text(u["name"]) == "jamie lee", f"observed={u and u['name']!r}") + judge.check("new_user_password_verifies", bool(u) and password_matches(u["password_hash"], "Weather123!"), "werkzeug hash checked with hashlib") + judge.check("seeded_users_unchanged", rows_unchanged_except(initial_db, after_db, "user", [u["id"]] if u else []), "seeded user rows identical") + before, after = saved_pairs(initial_db), saved_pairs(after_db) + judge.check("saved_rows_added", (after - before) == {(NEW_EMAIL, "atlanta-ga")}, f"observed_added={sorted(after - before)}") + judge.check("saved_rows_removed", (before - after) == set(), f"observed_removed={sorted(before - after)}") + check_tables_unchanged(judge, initial_db, after_db, ("alert",)) + judge.check("answer_mentions_atlanta", mentions(a, ["atlanta"]), f"answer={a!r}") + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_19.py b/sites/accuweather/verify/verify_19.py new file mode 100644 index 000000000..70d3bb7fc --- /dev/null +++ b/sites/accuweather/verify/verify_19.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--19: Phoenix vs New Orleans temperature/humidity and RealFeel heat-index comparison (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--19" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "new-orleans-la") + check_visited_path(judge, t, "visited_weather_phoenix", "/weather/phoenix-az") + check_visited_path(judge, t, "visited_weather_new_orleans", "/weather/new-orleans-la") + judge.check("answer_phoenix_temperature", contains_fact(a, 104, unit="temp", label=TEMP_LABEL), f"expected 104, answer={a!r}") + judge.check("answer_phoenix_humidity", contains_fact(a, 18, unit="percent", label=HUMIDITY_LABEL, allow_bare=False), f"expected 18%, answer={a!r}") + judge.check("answer_new_orleans_temperature", contains_fact(a, 89, unit="temp", label=TEMP_LABEL), f"expected 89, answer={a!r}") + judge.check("answer_new_orleans_humidity", contains_fact(a, 75, unit="percent", label=HUMIDITY_LABEL, allow_bare=False), f"expected 75%, answer={a!r}") + judge.check("answer_names_higher_heat_index_city", names_winner(a, ["phoenix"], ["new orleans", "nola"], ["higher", "hotter", "greater", "highest", "hottest", "larger"], ["lower", "cooler", "lowest", "smaller", "less"]), f"expected Phoenix higher, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_2.py b/sites/accuweather/verify/verify_2.py new file mode 100644 index 000000000..4f82d9afe --- /dev/null +++ b/sites/accuweather/verify/verify_2.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--2: Seattle hourly: first hour with >= 40% precipitation (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--2" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_visited_path(judge, t, "visited_hourly_seattle", "/hourly/seattle-wa") + judge.check("answer_hour", contains_clock_time(a, "4 PM"), f"expected 4 PM, answer={a!r}") + judge.check("answer_temperature", contains_fact(a, 69, unit="temp", label=TEMP_LABEL), f"expected 69, answer={a!r}") + judge.check("answer_condition", contains_condition(a, "Showers"), f"expected 'Showers', answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_3.py b/sites/accuweather/verify/verify_3.py new file mode 100644 index 000000000..1bac8cd42 --- /dev/null +++ b/sites/accuweather/verify/verify_3.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--3: Miami daily: day with the greatest precipitation chance (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--3" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_visited_path(judge, t, "visited_daily_miami", "/daily/miami-fl") + judge.check("answer_day", contains_day_label(a, "Sat"), f"expected Sat, answer={a!r}") + judge.check("answer_high", contains_fact(a, 86, unit="temp", label=HIGH_LABEL), f"expected high 86, answer={a!r}") + judge.check("answer_low", contains_fact(a, 73, unit="temp", label=LOW_LABEL), f"expected low 73, answer={a!r}") + judge.check("answer_condition", contains_condition(a, "Mostly cloudy"), f"expected 'Mostly cloudy', answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_4.py b/sites/accuweather/verify/verify_4.py new file mode 100644 index 000000000..d08c20120 --- /dev/null +++ b/sites/accuweather/verify/verify_4.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--4: Austin vs Denver RealFeel comparison (read-only). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--4" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_visited_path(judge, t, "visited_weather_austin", "/weather/austin-tx") + check_visited_path(judge, t, "visited_weather_denver", "/weather/denver-co") + judge.check("answer_austin_realfeel", contains_fact(a, 103, unit="temp", label=city_label("austin", "austin, tx", "austin, texas")), f"expected Austin 103, answer={a!r}") + judge.check("answer_denver_realfeel", contains_fact(a, 82, unit="temp", label=city_label("denver", "denver, co", "denver, colorado")), f"expected Denver 82, answer={a!r}") + judge.check("answer_names_cooler_city", names_winner(a, ["denver"], ["austin"], ["cooler", "colder", "lower", "less hot", "coolest"], ["warmer", "hotter", "higher", "warmest", "hottest"]), f"expected Denver cooler, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_5.py b/sites/accuweather/verify/verify_5.py new file mode 100644 index 000000000..5b767b526 --- /dev/null +++ b/sites/accuweather/verify/verify_5.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--5: Springfield search -> Missouri: postal code, pressure and air-quality value (read-only; task re-anchored by the reviewer). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--5" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_search_surfaces(judge, t, "springfield-mo") + check_visited_path(judge, t, "visited_weather_springfield_mo", "/weather/springfield-mo") + check_visited_path(judge, t, "visited_air_quality_springfield_mo", "/air-quality/springfield-mo") + judge.check("answer_postal_code", contains_all(a, ["65806"]), f"expected 65806, answer={a!r}") + judge.check("answer_pressure", contains_fact(a, "29.89", unit="inhg", label=PRESSURE_LABEL), f"expected 29.89 in, answer={a!r}") + judge.check("answer_air_quality", contains_fact(a, 39, label=AQ_LABEL), f"expected 39, answer={a!r}") + check_read_only(judge, initial_db, after_db) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_6.py b/sites/accuweather/verify/verify_6.py new file mode 100644 index 000000000..e1e95ae79 --- /dev/null +++ b/sites/accuweather/verify/verify_6.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--6: Sign in as alice.j@test.com and save Seattle (stateful). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--6" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_signed_in_as(judge, t, "alice.j@test.com") + check_paths_in_order(judge, t, "login_then_seattle_then_account", ["/login", "/weather/seattle-wa", "/account"]) + check_saved_delta(judge, initial_db, after_db, "alice.j@test.com", added={"seattle-wa"}, removed=set()) + judge.check("answer_mentions_seattle", mentions(a, ["seattle"]), f"answer={a!r}") + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_7.py b/sites/accuweather/verify/verify_7.py new file mode 100644 index 000000000..32ef6872f --- /dev/null +++ b/sites/accuweather/verify/verify_7.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--7: Sign in as alice.j@test.com and remove Boston from Saved Locations (stateful). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--7" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_signed_in_as(judge, t, "alice.j@test.com") + check_paths_in_order(judge, t, "login_then_boston_then_account", ["/login", "/weather/boston-ma", "/account"]) + check_saved_delta(judge, initial_db, after_db, "alice.j@test.com", added=set(), removed={"boston-ma"}) + judge.check("answer_mentions_boston", mentions(a, ["boston"]), f"answer={a!r}") + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_8.py b/sites/accuweather/verify/verify_8.py new file mode 100644 index 000000000..022caefd1 --- /dev/null +++ b/sites/accuweather/verify/verify_8.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--8: Sign in as bob.smith@test.com and enable exactly Severe weather + Rain starting soon for Chicago (stateful). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--8" + +EXPECTED_ALERTS = {("bob.smith@test.com", "chicago-il", "severe", 1), ("bob.smith@test.com", "chicago-il", "rain", 1)} + + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + check_signed_in_as(judge, t, "bob.smith@test.com") + check_paths_in_order(judge, t, "login_then_chicago_alerts", ["/login", "/alerts/chicago-il"]) + before, after = alert_rows(initial_db), alert_rows(after_db) + judge.check("alerts_initially_empty", before == set(), f"initial_alerts={sorted(before)}") + judge.check("alerts_exactly_severe_and_rain_for_chicago", after == EXPECTED_ALERTS, f"expected={sorted(EXPECTED_ALERTS)}, observed={sorted(after)}") + check_tables_unchanged(judge, initial_db, after_db, ("user", "saved_location")) + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_9.py b/sites/accuweather/verify/verify_9.py new file mode 100644 index 000000000..a194b9c7a --- /dev/null +++ b/sites/accuweather/verify/verify_9.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Verify AccuWeather--9: Sign in as carol.w@test.com, switch to Celsius, report New York temperature and RealFeel in Celsius (stateful). + +Deterministic only (no LLM calls). Ground truth is hardcoded below and never +appears in tasks.jsonl. Order: package identity -> navigation gates -> answer +facts -> SQLite snapshot contract. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + AQ_LABEL, HIGH_LABEL, HUMIDITY_LABEL, LOW_LABEL, PRECIP_LABEL, PRESSURE_LABEL, REALFEEL_LABEL, + TEMP_LABEL, VISIBILITY_LABEL, WIND_LABEL, Judge, alert_rows, check_paths_in_order, check_read_only, + check_saved_delta, check_search_surfaces, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, check_visited_path, city_label, contains_all, contains_any, + contains_clock_time, contains_condition, contains_day_label, contains_fact, fail_closed, + final_answer, load_run, mentions, names_winner, new_users, normalize_text, parse_args, + password_matches, resolve_snapshots, rows_unchanged_except, saved_pairs, search_queries, + table_rows, typed_emails, user_by_email, +) + +TASK_ID = "AccuWeather--9" + +def run_checks(judge, t, initial_db, after_db): + check_trajectory_identity(judge, t, TASK_ID) + a = final_answer(t) + check_signed_in_as(judge, t, "carol.w@test.com") + check_paths_in_order(judge, t, "login_then_settings_then_new_york", ["/login", "/settings", "/weather/new-york-ny"]) + before, after = user_by_email(initial_db, "carol.w@test.com"), user_by_email(after_db, "carol.w@test.com") + judge.check("carol_unit_is_celsius", bool(after) and after["unit"] == "C", f"before_unit={before and before['unit']!r}, after_unit={after and after['unit']!r}") + judge.check("carol_other_columns_unchanged", bool(before and after) and {k: v for k, v in before.items() if k != "unit"} == {k: v for k, v in after.items() if k != "unit"}, "email/name/password_hash identical") + judge.check("other_users_unchanged", bool(before) and rows_unchanged_except(initial_db, after_db, "user", [before["id"]]), "user rows other than carol identical") + check_tables_unchanged(judge, initial_db, after_db, ("saved_location", "alert")) + judge.check("answer_temperature_celsius", contains_fact(a, 26, unit="temp", label=TEMP_LABEL), f"expected 26, answer={a!r}") + judge.check("answer_realfeel_celsius", contains_fact(a, 28, unit="temp", label=REALFEEL_LABEL), f"expected 28, answer={a!r}") + + +def main() -> None: + args = parse_args() + try: + trajectory = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(TASK_ID, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, TASK_ID) + judge = Judge(TASK_ID, args.no_llm) + try: + run_checks(judge, trajectory, initial_db, after_db) + except Exception as exc: # noqa: BLE001 - any verifier error fails closed + fail_closed(TASK_ID, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() + + +if __name__ == "__main__": + main() diff --git a/sites/accuweather/verify/verify_lib.py b/sites/accuweather/verify/verify_lib.py new file mode 100644 index 000000000..f0e6749c2 --- /dev/null +++ b/sites/accuweather/verify/verify_lib.py @@ -0,0 +1,931 @@ +#!/usr/bin/env python3 +"""verify_lib.py — shared deterministic utilities for AccuWeather task verification. + +Philosophy: DETERMINISTIC FIRST. Mirrors the sites/merriam_webster/verify API +(load_run / navigated_to / final_answer / contains_* / db helpers / Judge / +parse_args, plus llm_text_match / llm_screenshot_shows kept for parity) and adds +the sites/walmart_careers hardening: + + 1. Package identity: task_id matches, run terminated with ``agent_done``, + non-empty final answer, every recorded URL on the same loopback origin + (host AND port) as ``start_url``, every referenced screenshot is a + decodable PNG. + 2. Navigation gates (anti knowledge-shortcut): the agent MUST have opened the + on-site page(s) that render the requested facts; comparison tasks need + every detail page; when the only UI path to a location is the search box + the trajectory must contain a ``/search?q=`` visit that surfaces it. + 3. Answer checks: negation-aware token / number / percentage / clock-time / + day-label / winner matchers against ground truth HARDCODED in verify_N.py. + 4. SQLite snapshot contract: table set + columns + catalog fingerprint are + pinned; catalog tables are immutable; read-only tasks leave user / + saved_location / alert rows identical; stateful tasks must show exactly + the allowed row delta and nothing else. + 5. LLM utilities exist only for API parity with merriam_webster. No verdict + depends on them: the per-task verifiers never call them, and ``--no_llm`` + is accepted for CLI parity. + +Input signature (per task): + --run_dir DIR agent run: trajectory.json + screenshots/step_NNN.png + --initial_db PATH initial-state SQLite DB (default: /initial.db, + else docker cp of instance_seed from --container) + --after_db PATH after-state SQLite DB (default: /after.db, + else docker cp of the live instance from --container) + --container NAME docker container to fetch DBs from ($WH_CONTAINER / wh-review) + --no_llm [True] accepted for parity; verifiers are deterministic-only +Output: JSON {task_id, pass, reason, evidence[]} on stdout; exit 0 PASS / 1 FAIL. +""" + +import atexit +import base64 +import hashlib +import hmac +import ipaddress +import json +import os +import re +import sqlite3 +import struct +import subprocess +import sys +import tempfile +import unicodedata +import urllib.request +import zlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Sequence +from urllib.parse import parse_qs, urlparse + +SITE = "accuweather" +DEFAULT_CONTAINER = os.environ.get("WH_CONTAINER", "wh-review") + +# Location identity (NOT answers): used to decide whether a /search?q= visit +# would surface a location under the site's own token-intersection scoring. +CATALOG = { + "new-york-ny": ("New York", "New York", "United States", "10007"), + "phoenix-az": ("Phoenix", "Arizona", "United States", "85001"), + "seattle-wa": ("Seattle", "Washington", "United States", "98101"), + "miami-fl": ("Miami", "Florida", "United States", "33101"), + "chicago-il": ("Chicago", "Illinois", "United States", "60601"), + "boston-ma": ("Boston", "Massachusetts", "United States", "02108"), + "austin-tx": ("Austin", "Texas", "United States", "78701"), + "denver-co": ("Denver", "Colorado", "United States", "80202"), + "portland-or": ("Portland", "Oregon", "United States", "97205"), + "portland-me": ("Portland", "Maine", "United States", "04101"), + "springfield-il": ("Springfield", "Illinois", "United States", "62701"), + "springfield-ma": ("Springfield", "Massachusetts", "United States", "01103"), + "springfield-mo": ("Springfield", "Missouri", "United States", "65806"), + "san-francisco-ca": ("San Francisco", "California", "United States", "94102"), + "los-angeles-ca": ("Los Angeles", "California", "United States", "90012"), + "atlanta-ga": ("Atlanta", "Georgia", "United States", "30303"), + "nashville-tn": ("Nashville", "Tennessee", "United States", "37219"), + "new-orleans-la": ("New Orleans", "Louisiana", "United States", "70112"), + "london-gb": ("London", "England", "United Kingdom", "SW1A"), + "toronto-ca": ("Toronto", "Ontario", "Canada", "M5H"), +} +# Cards linked from the homepage grid (Location.query.limit(8)); every other +# location is reachable through the UI only via the search box. +HOME_GRID_SLUGS = ("new-york-ny", "phoenix-az", "seattle-wa", "miami-fl", + "chicago-il", "boston-ma", "austin-tx", "denver-co") + +SEED_USERS = ("alice.j@test.com", "bob.smith@test.com", "carol.w@test.com", "david.b@test.com") +PASSWORD = "TestPass123!" + +# A real browser screenshot is at least viewport-sized. A decodable 1x1 PNG is a +# stub, not evidence, so the package-identity gate rejects anything this small. +MIN_SHOT_WIDTH, MIN_SHOT_HEIGHT = 200, 150 + + +# ---------------------------------------------------------------- CLI +@dataclass +class VerifyArgs: + run_dir: str = "" + initial_db: str = "" + after_db: str = "" + container: str = DEFAULT_CONTAINER + no_llm: bool = False + + def post_process(self): + if not self.run_dir: + raise SystemExit("--run_dir is required") + + +def parse_args() -> VerifyArgs: + """simpleArgParser when available (eval_judge runs verifiers in agent_demo's + env, where ``--no_llm True`` is the convention); plain argparse otherwise.""" + try: + import simpleArgParser as sap # type: ignore + args = sap.parse_args(VerifyArgs) + except ImportError: + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--run_dir", required=True) + parser.add_argument("--initial_db", default="") + parser.add_argument("--after_db", default="") + parser.add_argument("--container", default=DEFAULT_CONTAINER) + parser.add_argument("--no_llm", nargs="?", const="True", default="False") + ns = parser.parse_args() + args = VerifyArgs(ns.run_dir, ns.initial_db, ns.after_db, ns.container, + str(ns.no_llm).lower() in {"1", "true", "yes"}) + args.post_process() + run_dir = Path(args.run_dir) + if not args.initial_db and (run_dir / "initial.db").is_file(): + args.initial_db = str(run_dir / "initial.db") + if not args.after_db and (run_dir / "after.db").is_file(): + args.after_db = str(run_dir / "after.db") + return args + + +# ---------------------------------------------------------------- trajectory +def load_run(run_dir) -> dict[str, Any]: + d = Path(run_dir) + traj = json.loads((d / "trajectory.json").read_text(encoding="utf-8")) + if not isinstance(traj, dict): + raise ValueError("trajectory.json must contain a JSON object") + traj["_run_dir"] = d + traj["_shots"] = {p.name: p for p in sorted((d / "screenshots").glob("step_*.png"))} if (d / "screenshots").is_dir() else {} + return traj + + +def final_answer(traj) -> str: + return str(traj.get("final_answer") or "").strip() + + +def trajectory_urls(traj) -> list[str]: + urls: list[str] = [] + if traj.get("start_url"): + urls.append(str(traj["start_url"])) + for step in traj.get("steps") or []: + if not isinstance(step, dict): + continue + for key in ("url", "url_before", "url_after"): + if step.get(key): + urls.append(str(step[key])) + if traj.get("final_url"): + urls.append(str(traj["final_url"])) + return urls + + +def step_urls(traj) -> list[str]: # merriam_webster parity + return [str(s.get("url", "")) for s in traj.get("steps", []) if isinstance(s, dict)] + + +def normalized_url_path(url: str) -> str: + path = urlparse(str(url or "")).path or "/" + return path.rstrip("/") or "/" + + +def is_site_url(url: str) -> bool: + """HTTP(S) URL on a loopback host (any port: runs use alt ports).""" + parsed = urlparse(str(url or "")) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + return False + host = parsed.hostname.casefold() + if host == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def site_urls(traj) -> list[str]: + return [u for u in trajectory_urls(traj) if is_site_url(u)] + + +def navigated_to(traj, substr: str, times: int = 1) -> bool: + """merriam_webster parity: at least `times` recorded site URLs contain substr.""" + return sum(1 for u in site_urls(traj) if substr in u) >= times + + +def navigated_any(traj, substrs: Iterable[str]) -> bool: + return any(navigated_to(traj, s) for s in substrs) + + +def navigated_to_path(traj, path: str) -> bool: + """Exact mirror path on a loopback origin (query string ignored).""" + expected = normalized_url_path(path) + return any(normalized_url_path(u) == expected for u in site_urls(traj)) + + +def search_queries(traj) -> list[str]: + out = [] + for u in site_urls(traj): + if normalized_url_path(u) == "/search": + out.extend(parse_qs(urlparse(u).query, keep_blank_values=True).get("q", [])) + return out + + +def _tokens(text: str) -> set[str]: + return set(re.findall(r"[a-z0-9]+", normalize_text(text))) + + +def _discriminative_tokens(slug: str) -> set[str]: + """`slug`'s own field tokens, minus tokens that most of the catalog shares. + ``united`` and ``states`` match every US location, so ``q=United States`` + proves nothing about having found *this* one; ``springfield`` does.""" + mine = _tokens(" ".join(CATALOG[slug])) + catalog = [_tokens(" ".join(fields)) for fields in CATALOG.values()] + return {t for t in mine if sum(1 for other in catalog if t in other) <= len(catalog) / 2} + + +def search_surfaces(traj, slug: str) -> bool: + """True when some /search?q= visit would list `slug` under the site's own + scoring, on a token that actually narrows the catalog down to it.""" + field_tokens = _discriminative_tokens(slug) + return any(_tokens(q) & field_tokens for q in search_queries(traj)) + + +def typed_texts(traj) -> list[str]: + values = [] + for step in traj.get("steps") or []: + if isinstance(step, dict) and normalize_text(step.get("action")) == "input": + params = step.get("params") + if isinstance(params, dict) and params.get("text") is not None: + values.append(str(params["text"])) + return values + + +def typed_emails(traj) -> list[str]: + return [normalize_text(v) for v in typed_texts(traj) + if re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", v.strip())] + + +def _shot(traj, name): + if not name: + return None + p = traj["_shots"].get(Path(str(name)).name) + return p if (p and p.exists()) else None + + +def shot_after_url(traj, substr): + for s in traj.get("steps", []): + if substr in str(s.get("url", "")): + p = _shot(traj, s.get("screenshot_after")) + if p: + return p + return None + + +def last_shot(traj): + for s in reversed(traj.get("steps", [])): + p = _shot(traj, s.get("screenshot_after")) or _shot(traj, s.get("screenshot_before")) + if p: + return p + shots = sorted(traj["_shots"].values()) + return shots[-1] if shots else None + + +def _png_dimensions(path: Path) -> tuple[int, int]: + data = path.read_bytes() + if len(data) < 33 or data[:8] != b"\x89PNG\r\n\x1a\n" or data[12:16] != b"IHDR": + raise ValueError("not a PNG") + width, height = struct.unpack(">II", data[16:24]) + if struct.unpack(">I", data[29:33])[0] != zlib.crc32(data[12:29]) & 0xFFFFFFFF: + raise ValueError("corrupt IHDR") + return width, height + + +def screenshots_decode(traj) -> tuple[bool, str]: + root = Path(traj.get("_run_dir") or "") + steps = traj.get("steps") + if not root.is_dir() or not isinstance(steps, list) or not steps: + return False, "run directory or steps are missing" + checked = 0 + for i, step in enumerate(steps): + if not isinstance(step, dict): + return False, f"step {i} is not an object" + for key in ("screenshot_before", "screenshot_after"): + name = step.get(key) + rel = Path(str(name or "")) + if not name or rel.is_absolute() or ".." in rel.parts: + return False, f"step {i} has unsafe {key}" + path = next((c for c in (root / "screenshots" / rel, root / rel) if c.is_file()), None) + if path is None: + return False, f"step {i} is missing {key}={name!r}" + try: + w, h = _png_dimensions(path) + except Exception as exc: # noqa: BLE001 + return False, f"step {i} {key} cannot decode: {exc}" + if w < MIN_SHOT_WIDTH or h < MIN_SHOT_HEIGHT: + return False, (f"step {i} {key} is {w}x{h}, under the {MIN_SHOT_WIDTH}x{MIN_SHOT_HEIGHT} " + "minimum for a real page screenshot") + checked += 1 + return True, f"decoded {checked} PNG screenshots" + + +# ---------------------------------------------------------------- text matchers +def normalize_text(value: Any) -> str: + text = unicodedata.normalize("NFKC", str(value or "")) + text = text.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"') + text = text.replace("®", "").replace("™", "") + return re.sub(r"\s+", " ", text).strip().casefold() + + +def norm(s): # merriam_webster parity + return normalize_text(s) + + +_NEG_BEFORE = r"\b(?:not|no|never|without|wrong|incorrect|isn't|wasn't|isnt|wasnt|aren't|don't|doesn't|neither|nor)\b" +_NEG_AFTER = r"\s*(?:is|was|are|were)?\s*(?:not|wrong|incorrect)\b" + + +def _match_is_affirmative(text: str, match: re.Match) -> bool: + before = re.split(r"[.!?;:\n]+|\b(?:but|however|instead|whereas|while)\b", text[:match.start()], flags=re.I)[-1] + after = text[match.end():] + return not re.search(_NEG_BEFORE, before, re.I) and not re.match(_NEG_AFTER, after, re.I) + + +def _affirmative_search(pattern: str, text: str, flags: int = 0) -> bool: + return any(_match_is_affirmative(text, m) for m in re.finditer(pattern, text, flags)) + + +def answer_equals(final, expected) -> bool: + return normalize_text(final) == normalize_text(expected) + + +def contains_all(text, expected: Iterable[Any]) -> bool: + t = normalize_text(text) + return all(bool(v) and _affirmative_search(re.escape(v), t) for v in (normalize_text(e) for e in expected)) + + +def contains_any(text, expected: Iterable[Any]) -> bool: + t = normalize_text(text) + return any(bool(v) and _affirmative_search(re.escape(v), t) for v in (normalize_text(e) for e in expected)) + + +def _num_pattern(value) -> str: + """Regex for a standalone number: not part of a longer digit run, a decimal + or a thousands group (``104`` does not match ``1040`` or ``104.5``).""" + s = str(value) + if re.fullmatch(r"\d+\.\d+", s): + body = re.escape(s) + else: + body = rf"{int(s)}(?:\.0+)?" + return rf"(? bool: + return _affirmative_search(_num_pattern(value), normalize_text(text)) + + +_UNIT_PATTERNS = { + "temp": r"(?:°|º|˚|degrees?|deg\b|f\b|c\b|fahrenheit|celsius)", + "percent": r"(?:%|percent|pct\b|per cent)", + "mph": r"(?:mph|mi/h|miles? per hour|miles?/h|km/?h|kph)", + "mi": r"(?:mi\b|miles?\b|km\b)", + "inhg": r"(?:in\b|inhg|inches|\"|mb\b|hpa)", +} + + +def contains_measure(text, value, unit: str | None = None, label: str | None = None, + allow_bare: bool = False) -> bool: + """`value` immediately followed by a unit of kind `unit` (``104°``, ``104 F``, + ``18%``, ``8 mph``), OR preceded within the same clause by `label` + (``humidity: 18``, ``wind speed of 8``). `allow_bare` accepts a standalone + number with neither unit nor label.""" + t = normalize_text(text) + num = _num_pattern(value) + patterns = [] + if unit: + patterns.append(rf"{num}\s*(?:°\s*)?{_UNIT_PATTERNS[unit]}") + if label: + patterns.append(rf"(?:{label})[^.;\n]{{0,40}}?{num}") + if allow_bare or not patterns: + patterns.append(num) + return any(_affirmative_search(p, t) for p in patterns) + + +def contains_temperature(text, value, label: str | None = None) -> bool: + return contains_measure(text, value, unit="temp", label=label or r"temp(?:erature)?|realfeel|feels like|high|low|current") + + +def contains_percent(text, value, label: str | None = None) -> bool: + return contains_measure(text, value, unit="percent", label=label) + + +# Labels an agent typically writes next to a value. ``contains_fact`` uses them to +# catch value/label swaps (``temperature 115, RealFeel 104``) while staying lenient +# for unlabeled answers (``104°, 115°, 18%``). +TEMP_LABEL = r"(? str: + """Label regex for ``[’s] [metric]`` (``Austin's RealFeel 103``).""" + # longest alias first + trailing word boundary so ``portland, or`` never + # matches as a prefix of ``portland, oregon`` + alts = "|".join(re.escape(normalize_text(n)) for n in sorted(names, key=len, reverse=True)) + met = "|".join(re.escape(m) for m in sorted(metrics, key=len, reverse=True)) + return rf"\b(?:{alts})\b(?:'s)?(?:\s+(?:{met})\b)?" + + +def _num_norm(value) -> str: + s = str(value) + return s.rstrip("0").rstrip(".") if "." in s else s + + +def labeled_values(text, label: str) -> list[str]: + """Numbers bound to each occurrence of `label` by simple glue (``humidity: 18``, + ``RealFeel of 115°``). ``18% humidity, RealFeel 115`` binds nothing to + ``humidity`` because ``, realfeel`` is not glue.""" + t = normalize_text(text) + found = [] + for m in re.finditer(label, t): + n = re.match(_GLUE + rf"({_NUM_RE})", t[m.end():m.end() + 60]) + if n: + found.append(_num_norm(n.group(1))) + return found + + +def contains_fact(text, expected, unit: str | None = None, label: str | None = None, allow_bare: bool = True) -> bool: + """Ground-truth number check. If the answer binds a number to `label`, that + number must be `expected` (swap detection); otherwise fall back to unit + adjacency (``104°``) or, with `allow_bare`, a standalone number.""" + exp = _num_norm(expected) + if label: + bound = labeled_values(text, label) + if bound: + if not any(v == exp for v in bound): + return False + return contains_measure(text, expected, unit=unit, label=None, allow_bare=True) + return contains_measure(text, expected, unit=unit, label=None, allow_bare=allow_bare) + + +_MERIDIEM = {"am": r"a\.?\s*m\b\.?", "pm": r"p\.?\s*m\b\.?"} + + +def _clock_pattern(value: str) -> str: + m = re.fullmatch(r"\s*(\d{1,2})(?::(\d{2}))?\s*([AaPp])\.?\s*[Mm]\.?\s*", str(value)) + if not m: + raise ValueError(f"unsupported clock time: {value!r}") + hour12 = int(m.group(1)); minute = int(m.group(2) or 0); meridiem = "am" if m.group(3).lower() == "a" else "pm" + hour24 = hour12 % 12 + (12 if meridiem == "pm" else 0) + minutes = f":{minute:02d}" if minute else r"(?::00)?" + alts = [rf"(? bool: + return _affirmative_search(_clock_pattern(value), normalize_text(text)) + + +_DAYS = {"mon": "monday", "tue": "tuesday", "wed": "wednesday", "thu": "thursday", + "fri": "friday", "sat": "saturday", "sun": "sunday"} + + +def contains_day_label(text, label: str) -> bool: + """``Sat`` / ``Sat.`` / ``Saturday``; ``Today`` matches only the word today.""" + key = normalize_text(label)[:3] + if key == "tod": + return _affirmative_search(r"\btoday\b", normalize_text(text)) + full = _DAYS[key] + return _affirmative_search(rf"\b(?:{key}\.?|{full})\b", normalize_text(text)) + + +def contains_condition(text, condition: str) -> bool: + """``Mostly cloudy`` / ``mostly-cloudy`` / ``MOSTLY CLOUDY``.""" + words = normalize_text(condition).split() + pattern = r"\b" + r"[\s-]+".join(re.escape(w) for w in words) + r"\b" + if len(words) == 1: # ``Cloudy`` must not be satisfied by ``Mostly cloudy`` + pattern = r"(? bool: + return contains_any(text, terms) + + +def _term_pattern(terms: Sequence[str]) -> str: + return "(?:" + "|".join(r"\b" + re.escape(normalize_text(t)) + r"\b" for t in terms) + ")" + + +def _subject_near(sentence: str, kw: re.Match, a: Sequence[str], b: Sequence[str]) -> str | None: + """Which side (``'a'``/``'b'``) a comparative keyword refers to inside one + sentence. Rules, in order: (1) `` [(82°)] [feels|is] [slightly] ``; + (2) `` [one|city] [is|:] ``; (3) the nearest city before the + keyword; (4) the first city after it.""" + pa, pb = _term_pattern(a), _term_pattern(b) + glue = r"[\s,()°º\d.%f-]*(?:feels|is|has|was|reads|shows|reports|comes in|ranks|remains)?\s*(?:the|a|slightly|much|clearly|far|noticeably|somewhat|a bit|marginally)?\s*" + before, after = sentence[:kw.start()], sentence[kw.end():] + for side, pat in (("a", pa), ("b", pb)): + if re.search(pat + glue + r"$", before): + return side + lead = r"^\s*(?:one|city|location|option|result)?\s*(?:is|was|:|=|-|–|—|would be)?\s*" + for side, pat in (("a", pa), ("b", pb)): + if re.match(lead + pat, after): + return side + last_a = max((m.end() for m in re.finditer(pa, before)), default=-1) + last_b = max((m.end() for m in re.finditer(pb, before)), default=-1) + if last_a >= 0 or last_b >= 0: + return "a" if last_a > last_b else "b" + first_a = next((m.start() for m in re.finditer(pa, after)), None) + first_b = next((m.start() for m in re.finditer(pb, after)), None) + if first_a is None and first_b is None: + return None + if first_b is None or (first_a is not None and first_a < first_b): + return "a" + return "b" + + +def names_winner(text, winner: Sequence[str], loser: Sequence[str], + keywords: Sequence[str], inverse: Sequence[str] = ()) -> bool: + """The answer must attribute an affirmative `keywords` word (``cooler``) to + the winner, or an `inverse` word (``warmer``) to the loser, in at least one + sentence, and never the other way round.""" + t = normalize_text(text) + support = contradict = 0 + for sentence in re.split(r"[.;!?\n]+", t): + for kws, expect in ((keywords, "a"), (inverse, "b")): + for kw in kws: + for m in re.finditer(rf"\b{re.escape(normalize_text(kw))}\b", sentence): + if not _match_is_affirmative(sentence, m): + continue + side = _subject_near(sentence, m, winner, loser) + if side is None: + continue + if side == expect: + support += 1 + else: + contradict += 1 + return support > 0 and contradict == 0 + + +def extract_years(text): # merriam_webster parity + return re.findall(r"\b(1[5-9]\d{2}|20\d{2})\b", text or "") + + +# ---------------------------------------------------------------- SQLite state +EXPECTED_COLUMNS = { + "user": ["id", "email", "name", "password_hash", "unit"], + "location": ["id", "slug", "city", "region", "country", "postal", "temp", "realfeel", "condition", + "icon", "humidity", "wind", "visibility", "pressure", "uv", "air_quality"], + "forecast": ["id", "location_id", "day_index", "label", "high", "low", "condition", "icon", "precip"], + "hourly": ["id", "location_id", "hour_index", "label", "temp", "condition", "icon", "precip"], + "saved_location": ["id", "user_id", "location_id"], + "alert": ["id", "user_id", "location_id", "alert_type", "enabled"], +} +IMMUTABLE_TABLES = ("location", "forecast", "hourly") +READ_ONLY_TABLES = ("user", "saved_location", "alert") +EXPECTED_INITIAL_COUNTS = {"location": 20, "forecast": 140, "hourly": 240, "user": 4, "saved_location": 2, "alert": 0} +EXPECTED_INITIAL_SAVED = {("alice.j@test.com", "new-york-ny"), ("alice.j@test.com", "boston-ma")} +# sha256 over the location/forecast/hourly rows of the build-generated seed +# (sites/accuweather/app.py seeds them from constants; see verify/README.md). +CATALOG_FINGERPRINT = "05807dac4bd73c96fa663d5950d2fe98142590875f841f140ae1dad116abff3c" + + +def db_query(db_path, sql: str, params: Sequence[Any] = ()) -> list[sqlite3.Row]: + con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + con.row_factory = sqlite3.Row + try: + return con.execute(sql, params).fetchall() + finally: + con.close() + + +def table_rows(db_path, table: str) -> list[tuple[Any, ...]]: + if table not in EXPECTED_COLUMNS: + raise ValueError(f"unsupported table: {table}") + return [tuple(r) for r in db_query(db_path, f"SELECT * FROM {table} ORDER BY id")] + + +def table_delta(initial_db, after_db, table: str) -> dict[str, list[Any]]: + before = {int(r[0]): r for r in table_rows(initial_db, table)} + after = {int(r[0]): r for r in table_rows(after_db, table)} + common = before.keys() & after.keys() + return {"added": [after[k] for k in sorted(after.keys() - before.keys())], + "removed": [before[k] for k in sorted(before.keys() - after.keys())], + "changed": [(before[k], after[k]) for k in sorted(common) if before[k] != after[k]]} + + +def tables_unchanged(initial_db, after_db, tables: Iterable[str]) -> dict[str, bool]: + return {t: table_rows(initial_db, t) == table_rows(after_db, t) for t in tables} + + +def rows_unchanged_except(initial_db, after_db, table: str, excluded_ids: Iterable[int]) -> bool: + ex = {int(v) for v in excluded_ids} + before = [r for r in table_rows(initial_db, table) if int(r[0]) not in ex] + after = [r for r in table_rows(after_db, table) if int(r[0]) not in ex] + return before == after + + +def catalog_fingerprint(db_path) -> str: + payload = {t: [list(r) for r in table_rows(db_path, t)] for t in IMMUTABLE_TABLES} + return hashlib.sha256(json.dumps(payload, separators=(",", ":"), sort_keys=True).encode()).hexdigest() + + +def user_by_email(db_path, email: str) -> dict[str, Any] | None: + rows = db_query(db_path, "SELECT id, email, name, password_hash, unit FROM user WHERE lower(email)=lower(?) ORDER BY id LIMIT 1", (email,)) + return dict(rows[0]) if rows else None + + +def user_ids(db_path) -> set[int]: + return {int(r["id"]) for r in db_query(db_path, "SELECT id FROM user")} + + +def new_users(initial_db, after_db) -> list[dict[str, Any]]: + fresh = user_ids(after_db) - user_ids(initial_db) + return [dict(r) for r in db_query(after_db, "SELECT id, email, name, password_hash, unit FROM user ORDER BY id") if int(r["id"]) in fresh] + + +def saved_slugs(db_path, email: str) -> set[str] | None: + u = user_by_email(db_path, email) + if not u: + return None + rows = db_query(db_path, "SELECT l.slug FROM saved_location s JOIN location l ON l.id=s.location_id WHERE s.user_id=?", (u["id"],)) + return {r["slug"] for r in rows} + + +def saved_pairs(db_path) -> set[tuple[str, str]]: + rows = db_query(db_path, "SELECT u.email, l.slug FROM saved_location s JOIN user u ON u.id=s.user_id JOIN location l ON l.id=s.location_id") + return {(normalize_text(r["email"]), r["slug"]) for r in rows} + + +def alert_rows(db_path) -> set[tuple[str, str, str, int]]: + rows = db_query(db_path, "SELECT u.email, l.slug, a.alert_type, a.enabled FROM alert a JOIN user u ON u.id=a.user_id JOIN location l ON l.id=a.location_id") + return {(normalize_text(r["email"]), r["slug"], r["alert_type"], int(r["enabled"])) for r in rows} + + +def saved_words_for(db_path, email="alice.j@test.com"): # merriam_webster parity name + return sorted(saved_slugs(db_path, email) or []) + + +def user_exists(db_path, name=None, email=None): + rows = db_query(db_path, "SELECT name, email FROM user") + return any((name is None or r["name"] == name) and (email is None or normalize_text(r["email"]) == normalize_text(email)) for r in rows) + + +def password_matches(password_hash: str, password: str) -> bool: + """Verify a werkzeug ``scrypt:`` / ``pbkdf2:`` hash with hashlib only.""" + try: + method, salt, digest = str(password_hash).split("$", 2) + except ValueError: + return False + if method.startswith("scrypt:"): + _, n, r, p = method.split(":") + derived = hashlib.scrypt(password.encode(), salt=salt.encode(), n=int(n), r=int(r), p=int(p), + maxmem=132 * 1024 * 1024, dklen=64).hex() + elif method.startswith("pbkdf2:"): + parts = method.split(":") + algo = parts[1] + iterations = int(parts[2]) if len(parts) > 2 else 600000 + derived = hashlib.pbkdf2_hmac(algo, password.encode(), salt.encode(), iterations).hex() + else: + return False + return hmac.compare_digest(derived, digest) + + +def fetch_db(container: str, kind: str) -> str: + if kind not in {"instance", "instance_seed"}: + raise ValueError(f"unsupported DB kind: {kind}") + fd, dest = tempfile.mkstemp(prefix=f"{SITE}_{kind}_", suffix=".db") + os.close(fd) + src = f"{container}:/opt/WebSyn/{SITE}/{kind}/{SITE}.db" + r = subprocess.run(["docker", "cp", src, dest], capture_output=True, text=True) + if r.returncode: + Path(dest).unlink(missing_ok=True) + raise RuntimeError(f"docker cp {src} failed: {r.stderr.strip() or r.stdout.strip()}") + atexit.register(Path(dest).unlink, missing_ok=True) + return dest + + +def resolve_db(arg, container, kind): + if arg: + return arg if Path(arg).is_file() else None + try: + return fetch_db(container, kind) + except (OSError, RuntimeError): + return None + + +def _validate_snapshot_contract(initial_db: str, after_db: str) -> None: + for label, db in (("initial", initial_db), ("after", after_db)): + tables = {r["name"] for r in db_query(db, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")} + if tables != set(EXPECTED_COLUMNS): + raise ValueError(f"{label} database has unexpected tables: {sorted(tables)}") + for table, columns in EXPECTED_COLUMNS.items(): + observed = [r["name"] for r in db_query(db, f"PRAGMA table_info({table})")] + if observed != columns: + raise ValueError(f"{label}.{table} columns differ: {observed}") + counts = {t: len(table_rows(initial_db, t)) for t in EXPECTED_INITIAL_COUNTS} + if counts != EXPECTED_INITIAL_COUNTS: + raise ValueError(f"initial row counts differ: expected={EXPECTED_INITIAL_COUNTS}, observed={counts}") + emails = {normalize_text(r["email"]) for r in db_query(initial_db, "SELECT email FROM user")} + if emails != set(SEED_USERS): + raise ValueError(f"initial users differ: {sorted(emails)}") + if saved_pairs(initial_db) != EXPECTED_INITIAL_SAVED: + raise ValueError(f"initial saved locations differ: {sorted(saved_pairs(initial_db))}") + fp = catalog_fingerprint(initial_db) + if fp != CATALOG_FINGERPRINT: + raise ValueError(f"initial catalog fingerprint {fp} does not match the frozen seed") + changed = [t for t in IMMUTABLE_TABLES if table_rows(initial_db, t) != table_rows(after_db, t)] + if changed: + raise ValueError(f"immutable catalog tables changed: {changed}") + + +def resolve_snapshots(args: VerifyArgs, task_id: str) -> tuple[str, str]: + """Validated (initial_db, after_db) or fail closed (infra_error).""" + initial_db = resolve_db(args.initial_db, args.container, "instance_seed") + after_db = resolve_db(args.after_db, args.container, "instance") + if not initial_db or not after_db: + fail_closed(task_id, "database_unavailable", + "both initial (instance_seed) and after (instance) accuweather snapshots are required: " + "pass --initial_db/--after_db, place initial.db/after.db in the run dir, or set WH_CONTAINER") + try: + _validate_snapshot_contract(str(initial_db), str(after_db)) + except (OSError, sqlite3.Error, ValueError) as exc: + fail_closed(task_id, "snapshot_contract_invalid", str(exc)) + return str(initial_db), str(after_db) + + +# ---------------------------------------------------------------- judge harness +class Judge: + def __init__(self, task_id: str, no_llm: bool = False): + global _NO_LLM + _NO_LLM = bool(no_llm) + self.task_id = task_id + self.no_llm = no_llm + self.ok = True + self.reason = "" + self.evidence: list[str] = [] + + def check(self, name: str, cond: bool, evidence: str = "", llm: bool = False) -> bool: + if llm: # advisory only: never changes the verdict + self.evidence.append(f"[INFO] {name}: {'skipped (--no_llm)' if self.no_llm else evidence}") + return True + if cond: + self.evidence.append(f"[PASS] {name}: {evidence}") + else: + self.ok = False + if not self.reason: + self.reason = name + self.evidence.append(f"[FAIL] {name}: {evidence}") + return bool(cond) + + def emit(self) -> None: + print(json.dumps({"task_id": self.task_id, "pass": self.ok, + "reason": self.reason or "all checks passed", + "evidence": self.evidence}, ensure_ascii=False, indent=2)) + sys.exit(0 if self.ok else 1) + + +def fail_closed(task_id: str, reason: str, detail: str) -> None: + print(json.dumps({"task_id": task_id, "pass": False, "infra_error": True, "reason": reason, + "evidence": [f"[FAIL] {reason}: {detail}"]}, ensure_ascii=False, indent=2)) + sys.exit(1) + + +def _same_local_origin(url: str, start_url: str) -> bool: + try: + o, s = urlparse(str(url or "")), urlparse(str(start_url or "")) + return (o.scheme == s.scheme == "http" and o.hostname is not None and s.hostname is not None + and not o.username and not o.password and o.port == s.port + and o.hostname.casefold() == s.hostname.casefold() and is_site_url(url)) + except ValueError: + return False + + +def check_trajectory_identity(judge: Judge, traj, task_id: str) -> None: + judge.check("final_answer_nonempty", bool(final_answer(traj)), f"final_answer={final_answer(traj)!r}") + judge.check("trajectory_task_matches", str(traj.get("task_id") or "").strip() == task_id, + f"expected={task_id!r}, observed={traj.get('task_id')!r}") + judge.check("trajectory_completed", + traj.get("terminated") is True and traj.get("termination_reason") == "agent_done", + f"terminated={traj.get('terminated')!r}, reason={traj.get('termination_reason')!r}") + steps = traj.get("steps") + judge.check("trajectory_has_steps", isinstance(steps, list) and bool(steps), + f"steps={len(steps) if isinstance(steps, list) else 'invalid'}") + recorded = trajectory_urls(traj) + judge.check("all_urls_match_local_origin", + bool(recorded) and all(_same_local_origin(u, traj.get("start_url", "")) for u in recorded), + f"start_url={traj.get('start_url')!r}, recorded={recorded!r}") + ok, ev = screenshots_decode(traj) + judge.check("screenshots_decode", ok, ev) + + +def check_visited_path(judge: Judge, traj, name: str, path: str) -> bool: + return judge.check(name, navigated_to_path(traj, path), f"required_path={path}") + + +def check_visited_paths(judge: Judge, traj, paths: Iterable[str]) -> None: + for p in paths: + check_visited_path(judge, traj, "visited_" + normalized_url_path(p).strip("/").replace("/", "_"), p) + + +def check_search_surfaces(judge: Judge, traj, slug: str) -> bool: + return judge.check(f"searched_for_{slug}", search_surfaces(traj, slug), + f"a /search?q= visit must surface {slug}; observed_queries={search_queries(traj)!r}") + + +def check_paths_in_order(judge: Judge, traj, name: str, paths: Sequence[str]) -> bool: + urls = [normalized_url_path(u) for u in site_urls(traj)] + cursor = 0 + for p in paths: + expected = normalized_url_path(p) + for i in range(cursor, len(urls)): + if urls[i] == expected: + cursor = i + 1 + break + else: + return judge.check(name, False, f"required_order={list(paths)!r}, observed={urls!r}") + return judge.check(name, True, f"required_order={list(paths)!r}") + + +def check_signed_in_as(judge: Judge, traj, email: str) -> None: + judge.check("visited_login_page", navigated_to_path(traj, "/login"), "required_path=/login") + judge.check("entered_expected_account_email", normalize_text(email) in typed_emails(traj), + f"expected_email={email!r}, typed_emails={typed_emails(traj)!r}") + + +def check_tables_unchanged(judge: Judge, initial_db, after_db, tables: Iterable[str], prefix: str = "") -> None: + for t, same in tables_unchanged(initial_db, after_db, tables).items(): + judge.check(f"{prefix}{t}_unchanged", same, + f"table={t}, initial_rows={len(table_rows(initial_db, t))}, after_rows={len(table_rows(after_db, t))}") + + +def check_read_only(judge: Judge, initial_db, after_db) -> None: + check_tables_unchanged(judge, initial_db, after_db, READ_ONLY_TABLES, prefix="read_only_") + + +def check_saved_delta(judge: Judge, initial_db, after_db, email: str, added: Iterable[str], removed: Iterable[str]) -> None: + """Exact saved_location delta for `email`; every other user's rows and the + user / alert tables must be untouched.""" + before, after = saved_pairs(initial_db), saved_pairs(after_db) + e = normalize_text(email) + exp_added = {(e, s) for s in added} + exp_removed = {(e, s) for s in removed} + judge.check("saved_rows_added", (after - before) == exp_added, f"expected_added={sorted(exp_added)}, observed_added={sorted(after - before)}") + judge.check("saved_rows_removed", (before - after) == exp_removed, f"expected_removed={sorted(exp_removed)}, observed_removed={sorted(before - after)}") + judge.check("no_other_users_saved_changed", + {p for p in before if p[0] != e} == {p for p in after if p[0] != e}, "other users' saved rows identical") + check_tables_unchanged(judge, initial_db, after_db, ("user", "alert")) + + +# ---------------------------------------------------------------- LLM utilities (parity only) +_NO_LLM = False + + +def _llm_config(): + return (os.environ.get("OPENAI_API_KEY", ""), os.environ.get("OPENAI_BASE_URL", ""), os.environ.get("JUDGE_MODEL", "")) + + +def _chat(messages, max_tokens=1024): + if _NO_LLM: + return None + key, base, model = _llm_config() + if not (key and base and model): + return None + payload = {"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": 1.0} + req = urllib.request.Request(base.rstrip("/") + "/chat/completions", data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}) + try: + data = json.loads(urllib.request.urlopen(req, timeout=180).read()) + return data["choices"][0]["message"]["content"] + except Exception: # noqa: BLE001 + return None + + +def _verdict(out): + if not out: + return False, "" + s = out.strip() + return s.upper().startswith("PASS"), s + + +def llm_text_match(agent_answer, ground_truth, question): + """Anchored consistency check; advisory only (see Judge.check(llm=True)).""" + if _NO_LLM: + return False, "[skipped: --no_llm]" + return _verdict(_chat([{"role": "user", "content": + f"You are a STRICT binary grader.\nQuestion: {question}\n" + f"Ground-truth answer (ANCHOR — judge against THIS, never use your own knowledge): {ground_truth}\n" + f"Agent's answer: {agent_answer}\nLine 1: PASS or FAIL. Line 2: one-sentence reason."}])) + + +def llm_screenshot_shows(shot_path, must_show, question=""): + if _NO_LLM: + return False, "[skipped: --no_llm]" + b64 = base64.b64encode(Path(shot_path).read_bytes()).decode() + return _verdict(_chat([{"role": "user", "content": [ + {"type": "text", "text": f"Only what is VISIBLY rendered counts. Question: {question}\nExpected content: {must_show}\nLine 1: PASS or FAIL. Line 2: quote the visible evidence."}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}])) diff --git a/websyn_start.sh b/websyn_start.sh index f575dbc2c..426ee672d 100644 --- a/websyn_start.sh +++ b/websyn_start.sh @@ -7,7 +7,8 @@ SITES=(allrecipes amazon apple arxiv bbc_news booking github google_flights google_map google_search huggingface wolfram_alpha cambridge_dictionary coursera espn merriam_webster ikea phys_org target ted osu rotten_tomatoes compass walmart_careers - fedex webmd_doctor healthline kaggle) + fedex webmd_doctor healthline kaggle + accuweather) BASE_PORT=40000 SITE_COUNT=${#SITES[@]} PID_DIR=/tmp/websyn_pids